Merge pull request #20788 from JoVictorNunes/virtual-bg-enhancements-0726

enhancement(webcam): custom virtual backgrounds

bigbluebutton-html5/imports/ui/components/video-preview/component.jsx

@@ -5,6 +5,7 @@ import {
 } from 'react-intl';
 import Button from '/imports/ui/components/common/button/component';
 import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component';
+import VirtualBgService from '/imports/ui/components/video-preview/virtual-background/service';
 import logger from '/imports/startup/client/logger';
 import browserInfo from '/imports/utils/browserInfo';
 import PreviewService from './service';
@@ -19,10 +20,15 @@ import {
   setSessionVirtualBackgroundInfo,
   getSessionVirtualBackgroundInfo,
   isVirtualBackgroundSupported,
+  clearSessionVirtualBackgroundInfo,
+  getSessionVirtualBackgroundInfoWithDefault,
 } from '/imports/ui/services/virtual-background/service';
 import Settings from '/imports/ui/services/settings';
 import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
 import Checkbox from '/imports/ui/components/common/checkbox/component';
+import { CustomVirtualBackgroundsContext } from '/imports/ui/components/video-preview/virtual-background/context';
+import Auth from '/imports/ui/services/auth';
+import Users from '/imports/api/users';
 
 const VIEW_STATES = {
   finding: 'finding',
@@ -34,7 +40,9 @@ const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightne
 const CAMERA_BRIGHTNESS_AVAILABLE = ENABLE_CAMERA_BRIGHTNESS && isVirtualBackgroundSupported();
 
 const propTypes = {
-  intl: PropTypes.object.isRequired,
+  intl: PropTypes.shape({
+    formatMessage: PropTypes.func.isRequired,
+  }).isRequired,
   closeModal: PropTypes.func.isRequired,
   startSharing: PropTypes.func.isRequired,
   stopSharing: PropTypes.func.isRequired,
@@ -43,6 +51,7 @@ const propTypes = {
   hasVideoStream: PropTypes.bool.isRequired,
   webcamDeviceId: PropTypes.string,
   sharedDevices: PropTypes.arrayOf(PropTypes.string),
+  cameraAsContent: PropTypes.bool,
 };
 
 const defaultProps = {
@@ -50,6 +59,7 @@ const defaultProps = {
   camCapReached: true,
   webcamDeviceId: null,
   sharedDevices: [],
+  cameraAsContent: false,
 };
 
 const intlMessages = defineMessages({
@@ -270,9 +280,44 @@ class VideoPreview extends Component {
       webcamDeviceId,
       forceOpen,
     } = this.props;
+    const { dispatch, backgrounds } = this.context;
 
     this._isMounted = true;
 
+    // Set the custom or default virtual background
+    const webcamBackground = Users.findOne({
+      meetingId: Auth.meetingID,
+      userId: Auth.userID,
+    }, {
+      fields: {
+        webcamBackground: 1,
+      },
+    });
+
+    const webcamBackgroundURL = webcamBackground?.webcamBackground;
+    if (webcamBackgroundURL !== '' && !backgrounds.webcamBackgroundURL) {
+      VirtualBgService.getFileFromUrl(webcamBackgroundURL).then((fetchedWebcamBackground) => {
+        if (fetchedWebcamBackground) {
+          const data = URL.createObjectURL(fetchedWebcamBackground);
+          const uniqueId = 'webcamBackgroundURL';
+          const filename = webcamBackgroundURL;
+          dispatch({
+            type: 'update',
+            background: {
+              filename,
+              uniqueId,
+              data,
+              lastActivityDate: Date.now(),
+              custom: true,
+              sessionOnly: true,
+            },
+          });
+        } else {
+          logger.error('Failed to fetch custom webcam background image. Using fallback image.');
+        }
+      });
+    }
+
     if (deviceInfo.hasMediaDevices) {
       navigator.mediaDevices.enumerateDevices().then((devices) => {
         VideoService.updateNumberOfDevices(devices);
@@ -620,8 +665,54 @@ class VideoPreview extends Component {
     });
   }
 
+  async startEffects(deviceId) {
+    // Brightness and backgrounds are independent of each other,
+    // handle each one separately.
+    try {
+      await this.startCameraBrightness();
+    } catch (error) {
+      logger.warn({
+        logCode: 'brightness_effect_error',
+        extraInfo: {
+          errorName: error.name,
+          errorMessage: error.message,
+        },
+      }, 'Failed to start brightness effect');
+    }
+
+    let type;
+    let name;
+    let customParams;
+
+    const { backgrounds } = this.context;
+    const { webcamBackgroundURL } = backgrounds;
+    const storedBackgroundInfo = getSessionVirtualBackgroundInfo(deviceId);
+
+    if (storedBackgroundInfo) {
+      type = storedBackgroundInfo.type;
+      name = storedBackgroundInfo.name;
+      customParams = storedBackgroundInfo.customParams;
+    } else if (webcamBackgroundURL) {
+      const { data, filename } = webcamBackgroundURL;
+      type = EFFECT_TYPES.IMAGE_TYPE;
+      name = filename;
+      customParams = { file: data };
+    }
+
+    if (!type) return Promise.resolve(true);
+
+    try {
+      return this.handleVirtualBgSelected(type, name, customParams);
+    } catch (error) {
+      this.handleVirtualBgError(error, type, name);
+      clearSessionVirtualBackgroundInfo(deviceId);
+      throw error;
+    }
+  }
+
   getCameraStream(deviceId, profile) {
     const { webcamDeviceId } = this.state;
+    const { cameraAsContent } = this.props;
 
     this.setState({
       selectedProfile: profile.id,
@@ -635,16 +726,32 @@ class VideoPreview extends Component {
     // The return of doGUM is an instance of BBBVideoStream (a thin wrapper over a MediaStream)
     return PreviewService.doGUM(deviceId, profile).then((bbbVideoStream) => {
       // Late GUM resolve, clean up tracks, stop.
-      if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId);
+      if (!this._isMounted) {
+        this.terminateCameraStream(bbbVideoStream, deviceId);
+        this.cleanupStreamAndVideo();
+        return Promise.resolve(false);
+      }
 
       this.currentVideoStream = bbbVideoStream;
-      this.startCameraBrightness().then(() => {
-        const { type, name, customParams } = getSessionVirtualBackgroundInfo(deviceId);
-        this.handleVirtualBgSelected(type, name, customParams).then(() => {
+      this.updateDeviceId(deviceId);
+
+      if (cameraAsContent) return Promise.resolve(true);
+
+      return this.startEffects(deviceId)
+        .catch((error) => {
+          if (this.shouldSkipVideoPreview()) {
+            throw error;
+          }
+        })
+        .finally(() => {
+          if (this._isMounted) {
             this.setState({
               isStartSharingDisabled: false,
             });
-        });
+          } else {
+            this.terminateCameraStream(bbbVideoStream, deviceId);
+            this.cleanupStreamAndVideo();
+          }
         });
     }).catch((error) => {
       // When video preview is set to skip, we need some way to bubble errors
@@ -917,7 +1024,7 @@ class VideoPreview extends Component {
     const initialVirtualBgState = this.currentVideoStream ? {
       type: this.currentVideoStream.virtualBgType,
       name: this.currentVideoStream.virtualBgName
-    } : getSessionVirtualBackgroundInfo(webcamDeviceId);
+    } : getSessionVirtualBackgroundInfoWithDefault(webcamDeviceId);
 
     return (
       <VirtualBgSelector
@@ -1166,5 +1273,6 @@ class VideoPreview extends Component {
 
 VideoPreview.propTypes = propTypes;
 VideoPreview.defaultProps = defaultProps;
+VideoPreview.contextType = CustomVirtualBackgroundsContext;
 
 export default injectIntl(VideoPreview);

bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx

@@ -18,8 +18,6 @@ import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
 import 'react-loading-skeleton/dist/skeleton.css';
 import Settings from '/imports/ui/services/settings';
 import { isCustomVirtualBackgroundsEnabled } from '/imports/ui/services/features';
-import Auth from '/imports/ui/services/auth';
-import Users from '/imports/api/users';
 
 const { MIME_TYPES_ALLOWED, MAX_FILE_SIZE } = VirtualBgService;
 const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@@ -96,28 +94,6 @@ const VIRTUAL_BACKGROUNDS_CONFIG = Meteor.settings.public.virtualBackgrounds;
 const ENABLE_UPLOAD = VIRTUAL_BACKGROUNDS_CONFIG.enableVirtualBackgroundUpload;
 const shouldEnableBackgroundUpload = () => ENABLE_UPLOAD && isCustomVirtualBackgroundsEnabled();
 
-// Function to convert image URL to a File object
-async function getFileFromUrl(url) {
-  try {
-    const response = await fetch(url, {
-      credentials: 'omit',
-      mode: 'cors',
-      headers: {
-        Accept: 'image/*',
-      },
-    });
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
-    const blob = await response.blob();
-    const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
-    return file;
-  } catch (error) {
-    logger.error('Fetch error:', error);
-    return null;
-  }
-}
-
 const VirtualBgSelector = ({
   intl,
   handleVirtualBgSelected,
@@ -159,51 +135,6 @@ const VirtualBgSelector = ({
       }
       if (!loaded) loadFromDB();
     }
-
-    // Set the custom or default virtual background
-    const webcamBackground = Users.findOne({
-      meetingId: Auth.meetingID,
-      userId: Auth.userID,
-    }, {
-      fields: {
-        webcamBackground: 1,
-      },
-    });
-
-    const webcamBackgroundURL = webcamBackground?.webcamBackground;
-    if (webcamBackgroundURL !== '' && !backgrounds.webcamBackgroundURL) {
-      getFileFromUrl(webcamBackgroundURL).then((fetchedWebcamBackground) => {
-        if (fetchedWebcamBackground) {
-          const data = URL.createObjectURL(fetchedWebcamBackground);
-          const uniqueId = 'webcamBackgroundURL';
-          const filename = webcamBackgroundURL;
-          dispatch({
-            type: 'update',
-            background: {
-              filename,
-              uniqueId,
-              data,
-              lastActivityDate: Date.now(),
-              custom: true,
-              sessionOnly: true,
-            },
-          });
-          handleVirtualBgSelected(
-            EFFECT_TYPES.IMAGE_TYPE,
-            webcamBackgroundURL,
-            { file: data, uniqueId },
-          ).then((switched) => {
-            if (!switched) {
-              setCurrentVirtualBg({ type: EFFECT_TYPES.NONE_TYPE });
-              return;
-            }
-            setCurrentVirtualBg({ type: EFFECT_TYPES.IMAGE_TYPE, name: filename });
-          });
-        } else {
-          logger.error('Failed to fetch custom webcam background image. Using fallback image.');
-        }
-      });
-    }
   }, []);
 
   const _virtualBgSelected = (type, name, index, customParams) =>

bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/service.js

@@ -104,6 +104,28 @@ const update = (background) => {
   });
 };
 
+// Function to convert image URL to a File object
+async function getFileFromUrl(url) {
+  try {
+    const response = await fetch(url, {
+      credentials: 'omit',
+      mode: 'cors',
+      headers: {
+        Accept: 'image/*',
+      },
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+    const blob = await response.blob();
+    const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
+    return file;
+  } catch (error) {
+    logger.error('Fetch error:', error);
+    return null;
+  }
+}
+
 export default {
   load,
   save,
@@ -111,4 +133,5 @@ export default {
   update,
   MIME_TYPES_ALLOWED,
   MAX_FILE_SIZE,
+  getFileFromUrl,
 };

bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/styles.js

@@ -89,7 +89,8 @@ const ThumbnailButton = styled(Button)`
 
   ${({ background }) => background && `
     background-image: url(${background});
-    background-size: 46px 46px;
+    background-size: cover;
+    background-position: center;
     background-origin: padding-box;
 
     &:active {

bigbluebutton-html5/imports/ui/components/video-provider/component.jsx

@@ -17,7 +17,7 @@ import MediaStreamUtils from '/imports/utils/media-stream-utils';
 import BBBVideoStream from '/imports/ui/services/webrtc-base/bbb-video-stream';
 import {
   EFFECT_TYPES,
-  getSessionVirtualBackgroundInfo,
+  getSessionVirtualBackgroundInfoWithDefault,
 } from '/imports/ui/services/virtual-background/service';
 import { notify } from '/imports/ui/services/notification';
 import { shouldForceRelay } from '/imports/ui/services/bbb-webrtc-sfu/utils';
@@ -1049,7 +1049,7 @@ class VideoProvider extends Component {
           peer.bbbVideoStream.mediaStream,
           'video',
         );
-        const { type, name } = getSessionVirtualBackgroundInfo(deviceId);
+        const { type, name } = getSessionVirtualBackgroundInfoWithDefault(deviceId);
 
         this.restoreVirtualBackground(peer.bbbVideoStream, type, name).catch((error) => {
           this.handleVirtualBgError(error, type, name);

bigbluebutton-html5/imports/ui/services/virtual-background/service.js

@@ -73,18 +73,15 @@ const setSessionVirtualBackgroundInfo = (
   deviceId,
 ) => Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name, customParams });
 
-const getSessionVirtualBackgroundInfo = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
+const getSessionVirtualBackgroundInfo = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`);
+
+const clearSessionVirtualBackgroundInfo = (deviceId) => Session.set(`VirtualBackgroundInfo_${deviceId}`, null);
+
+const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
   type: EFFECT_TYPES.NONE_TYPE,
   name: '',
 };
 
-const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => {
-  return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
-    type: EFFECT_TYPES.BLUR_TYPE,
-    name: BLUR_FILENAME,
-  };
-}
-
 const isVirtualBackgroundSupported = () => {
   return !(deviceInfo.isIos || browserInfo.isSafari);
 }
@@ -110,4 +107,5 @@ export {
   createVirtualBackgroundStream,
   getVirtualBackgroundThumbnail,
   getVirtualBgImagePath,
-}
+  clearSessionVirtualBackgroundInfo,
+};