fix(webcam): client failing to apply virtual background effect (#20777)

* fix(webcam): client failing to apply virtual background effect

* fix: check for already dispatched background

* fix: make webcam start up with last selected virtual background

bigbluebutton-html5/imports/ui/components/video-preview/component.jsx

@@ -1,12 +1,10 @@
-import Auth from '/imports/ui/services/auth';
-import Users from '/imports/api/users';
 import PropTypes from 'prop-types';
 import React, { Component } from 'react';
 import {
   defineMessages, injectIntl, FormattedMessage,
 } from 'react-intl';
 import Button from '/imports/ui/components/common/button/component';
-import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component'
+import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component';
 import logger from '/imports/startup/client/logger';
 import browserInfo from '/imports/utils/browserInfo';
 import PreviewService from './service';
@@ -24,7 +22,7 @@ import {
 } from '/imports/ui/services/virtual-background/service';
 import Settings from '/imports/ui/services/settings';
 import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
-import Checkbox from '/imports/ui/components/common/checkbox/component'
+import Checkbox from '/imports/ui/components/common/checkbox/component';
 
 const VIEW_STATES = {
   finding: 'finding',
@@ -234,6 +232,7 @@ class VideoPreview extends Component {
     this.handleVirtualBgSelected = this.handleVirtualBgSelected.bind(this);
     this.handleLocalStreamInactive = this.handleLocalStreamInactive.bind(this);
     this.handleBrightnessAreaChange = this.handleBrightnessAreaChange.bind(this);
+    this.updateVirtualBackgroundInfo = this.updateVirtualBackgroundInfo.bind(this);
 
     this._isMounted = false;
 
@@ -327,21 +326,6 @@ class VideoPreview extends Component {
                 viewState: VIEW_STATES.found,
               });
               this.displayPreview();
-
-              // Set the custom or default virtual background
-              const webcamBackground = Users.findOne({
-                meetingId: Auth.meetingID,
-                userId: Auth.userID,
-              }, {
-                fields: {
-                  webcamBackground: 1,
-                },
-              });
-              
-              const webcamBackgroundURL = webcamBackground?.webcamBackground;
-              if (webcamBackgroundURL !== '') {
-                this.handleVirtualBgSelected(EFFECT_TYPES.IMAGE_TYPE, '', { url: webcamBackgroundURL });
-              }
             });
         } else {
           // There were no webcams coming from enumerateDevices. Throw an error.
@@ -384,30 +368,6 @@ class VideoPreview extends Component {
     this._isMounted = false;
   }
 
-  startCameraBrightness() {
-    if (CAMERA_BRIGHTNESS_AVAILABLE) {
-      const setBrightnessInfo = () => {
-        const stream = this.currentVideoStream || {};
-        const service = stream.virtualBgService || {};
-        const { brightness = 100, wholeImageBrightness = false } = service;
-        this.setState({ brightness, wholeImageBrightness });
-      };
-
-      if (!this.currentVideoStream.virtualBgService) {
-        this.startVirtualBackground(
-          this.currentVideoStream,
-          EFFECT_TYPES.NONE_TYPE,
-        ).then((switched) => {
-          if (switched) {
-            setBrightnessInfo();
-          }
-        });
-      } else {
-        setBrightnessInfo();
-      }
-    }
-  }
-
   handleSelectWebcam(event) {
     const webcamValue = event.target.value;
 
@@ -432,34 +392,22 @@ class VideoPreview extends Component {
     }
   }
 
-  updateVirtualBackgroundInfo = () => {
-    const { webcamDeviceId } = this.state;
-
-    // Update this session's virtual camera effect information if it's enabled
-    setSessionVirtualBackgroundInfo(
-      this.currentVideoStream.virtualBgType,
-      this.currentVideoStream.virtualBgName,
-      webcamDeviceId,
-    );
-  };
-
   // Resolves into true if the background switch is successful, false otherwise
   handleVirtualBgSelected(type, name, customParams) {
-    const { webcamDeviceId } = this.state;
-    const shared = this.isAlreadyShared(webcamDeviceId);
-
     if (type !== EFFECT_TYPES.NONE_TYPE || CAMERA_BRIGHTNESS_AVAILABLE) {
-      return this.startVirtualBackground(this.currentVideoStream, type, name, customParams).then((switched) => {
+      return this.startVirtualBackground(
-        // If it's not shared we don't have to update here because
+        this.currentVideoStream,
-        // it will be updated in the handleStartSharing method.
+        type,
-        if (switched && shared) this.updateVirtualBackgroundInfo();
+        name,
+        customParams,
+      ).then((switched) => {
+        if (switched) this.updateVirtualBackgroundInfo();
         return switched;
       });
-    } else {
-      this.stopVirtualBackground(this.currentVideoStream);
-      if (shared) this.updateVirtualBackgroundInfo();
-      return Promise.resolve(true);
     }
+    this.stopVirtualBackground(this.currentVideoStream);
+    this.updateVirtualBackgroundInfo();
+    return Promise.resolve(true);
   }
 
   stopVirtualBackground(bbbVideoStream) {
@@ -477,7 +425,7 @@ class VideoPreview extends Component {
     return bbbVideoStream.startVirtualBackground(type, name, customParams).then(() => {
       this.displayPreview();
       return true;
-    }).catch(error => {
+    }).catch((error) => {
       this.handleVirtualBgError(error, type, name);
       return false;
     }).finally(() => {
@@ -511,7 +459,7 @@ class VideoPreview extends Component {
     // Only streams that will be shared should be stored in the service.
     // If the store call returns false, we're duplicating stuff. So clean this one
     // up because it's an impostor.
-    if(!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) {
+    if (!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) {
       this.currentVideoStream.stop();
     }
 
@@ -523,7 +471,6 @@ class VideoPreview extends Component {
       this.stopVirtualBackground(this.currentVideoStream);
     }
 
-    this.updateVirtualBackgroundInfo();
     this.cleanupStreamAndVideo();
 
     PreviewService.changeProfile(selectedProfile);
@@ -664,7 +611,9 @@ class VideoPreview extends Component {
 
   getInitialCameraStream(deviceId) {
     const { cameraAsContent } = this.props;
-    const defaultProfile = !cameraAsContent ? PreviewService.getDefaultProfile() : PreviewService.getCameraAsContentProfile();
+    const defaultProfile = !cameraAsContent
+      ? PreviewService.getDefaultProfile()
+      : PreviewService.getCameraAsContentProfile();
 
     return this.getCameraStream(deviceId, defaultProfile).then(() => {
       this.updateDeviceId(deviceId);
@@ -689,9 +638,13 @@ class VideoPreview extends Component {
       if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId);
 
       this.currentVideoStream = bbbVideoStream;
-      this.startCameraBrightness();
+      this.startCameraBrightness().then(() => {
-      this.setState({
+        const { type, name, customParams } = getSessionVirtualBackgroundInfo(deviceId);
-        isStartSharingDisabled: false,
+        this.handleVirtualBgSelected(type, name, customParams).then(() => {
+          this.setState({
+            isStartSharingDisabled: false,
+          });
+        });
       });
     }).catch((error) => {
       // When video preview is set to skip, we need some way to bubble errors
@@ -1046,6 +999,44 @@ class VideoPreview extends Component {
     return intl.formatMessage(intlMessages.webcamSettingsTitle);
   }
 
+  startCameraBrightness() {
+    if (CAMERA_BRIGHTNESS_AVAILABLE) {
+      const setBrightnessInfo = () => {
+        const stream = this.currentVideoStream || {};
+        const service = stream.virtualBgService || {};
+        const { brightness = 100, wholeImageBrightness = false } = service;
+        this.setState({ brightness, wholeImageBrightness });
+      };
+
+      if (!this.currentVideoStream.virtualBgService) {
+        return this.startVirtualBackground(
+          this.currentVideoStream,
+          EFFECT_TYPES.NONE_TYPE,
+        ).then((switched) => {
+          if (switched) {
+            setBrightnessInfo();
+          }
+        });
+      }
+
+      setBrightnessInfo();
+    }
+
+    return Promise.resolve(true);
+  }
+
+  updateVirtualBackgroundInfo() {
+    const { webcamDeviceId } = this.state;
+
+    // Update this session's virtual camera effect information if it's enabled
+    setSessionVirtualBackgroundInfo(
+      this.currentVideoStream.virtualBgType,
+      this.currentVideoStream.virtualBgName,
+      this.currentVideoStream.customParams,
+      webcamDeviceId,
+    );
+  }
+
   renderModalContent() {
     const {
       intl,

bigbluebutton-html5/imports/ui/components/video-preview/service.js

@@ -74,6 +74,8 @@ const deleteStream = (deviceId) => {
   return VIDEO_STREAM_STORAGE.delete(deviceId);
 }
 
+const clearStreams = () => VIDEO_STREAM_STORAGE.clear();
+
 const promiseTimeout = (ms, promise) => {
   const timeout = new Promise((resolve, reject) => {
     const id = setTimeout(() => {
@@ -246,4 +248,5 @@ export default {
   getCameraProfile,
   doGUM,
   terminateCameraStream,
+  clearStreams,
 };

bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx

@@ -18,6 +18,8 @@ import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
 import 'react-loading-skeleton/dist/skeleton.css';
 import Settings from '/imports/ui/services/settings';
 import { isCustomVirtualBackgroundsEnabled } from '/imports/ui/services/features';
+import Auth from '/imports/ui/services/auth';
+import Users from '/imports/api/users';
 
 const { MIME_TYPES_ALLOWED, MAX_FILE_SIZE } = VirtualBgService;
 const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@@ -94,6 +96,28 @@ const VIRTUAL_BACKGROUNDS_CONFIG = Meteor.settings.public.virtualBackgrounds;
 const ENABLE_UPLOAD = VIRTUAL_BACKGROUNDS_CONFIG.enableVirtualBackgroundUpload;
 const shouldEnableBackgroundUpload = () => ENABLE_UPLOAD && isCustomVirtualBackgroundsEnabled();
 
+// Function to convert image URL to a File object
+async function getFileFromUrl(url) {
+  try {
+    const response = await fetch(url, {
+      credentials: 'omit',
+      mode: 'cors',
+      headers: {
+        Accept: 'image/*',
+      },
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+    const blob = await response.blob();
+    const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
+    return file;
+  } catch (error) {
+    logger.error('Fetch error:', error);
+    return null;
+  }
+}
+
 const VirtualBgSelector = ({
   intl,
   handleVirtualBgSelected,
@@ -135,6 +159,51 @@ const VirtualBgSelector = ({
       }
       if (!loaded) loadFromDB();
     }
+
+    // Set the custom or default virtual background
+    const webcamBackground = Users.findOne({
+      meetingId: Auth.meetingID,
+      userId: Auth.userID,
+    }, {
+      fields: {
+        webcamBackground: 1,
+      },
+    });
+
+    const webcamBackgroundURL = webcamBackground?.webcamBackground;
+    if (webcamBackgroundURL !== '' && !backgrounds.webcamBackgroundURL) {
+      getFileFromUrl(webcamBackgroundURL).then((fetchedWebcamBackground) => {
+        if (fetchedWebcamBackground) {
+          const data = URL.createObjectURL(fetchedWebcamBackground);
+          const uniqueId = 'webcamBackgroundURL';
+          const filename = webcamBackgroundURL;
+          dispatch({
+            type: 'update',
+            background: {
+              filename,
+              uniqueId,
+              data,
+              lastActivityDate: Date.now(),
+              custom: true,
+              sessionOnly: true,
+            },
+          });
+          handleVirtualBgSelected(
+            EFFECT_TYPES.IMAGE_TYPE,
+            webcamBackgroundURL,
+            { file: data, uniqueId },
+          ).then((switched) => {
+            if (!switched) {
+              setCurrentVirtualBg({ type: EFFECT_TYPES.NONE_TYPE });
+              return;
+            }
+            setCurrentVirtualBg({ type: EFFECT_TYPES.IMAGE_TYPE, name: filename });
+          });
+        } else {
+          logger.error('Failed to fetch custom webcam background image. Using fallback image.');
+        }
+      });
+    }
   }, []);
 
   const _virtualBgSelected = (type, name, index, customParams) =>
@@ -161,6 +230,7 @@ const VirtualBgSelector = ({
                 filename: name,
                 uniqueId: customParams.uniqueId,
                 data: customParams.file,
+                sessionOnly: customParams.sessionOnly,
                 custom: true,
                 lastActivityDate: Date.now(),
               },
@@ -315,7 +385,9 @@ const VirtualBgSelector = ({
     };
 
     const renderCustomButton = (background, index) => {
-      const { filename, data, uniqueId } = background;
+      const {
+        filename, data, uniqueId, sessionOnly,
+      } = background;
       const label = intl.formatMessage(intlMessages.backgroundWithIndex, {
         0: index + 1,
       });
@@ -339,7 +411,7 @@ const VirtualBgSelector = ({
               EFFECT_TYPES.IMAGE_TYPE,
               filename,
               index,
-              { file: data, uniqueId },
+              { file: data, uniqueId, sessionOnly },
             )}
             disabled={disabled}
             isVisualEffects={isVisualEffects}
@@ -459,10 +531,11 @@ const VirtualBgSelector = ({
                     .map((background, index) => {
                       if (background.custom !== false) {
                         return renderCustomButton(background, index);
-                      } else {
-                        const isBlur = background.uniqueId.includes('Blur');
-                        return isBlur ? renderBlurButton(index) : renderDefaultButton(background.uniqueId, index);
                       }
+                      const isBlur = background.uniqueId.includes('Blur');
+                      return isBlur
+                        ? renderBlurButton(index)
+                        : renderDefaultButton(background.uniqueId, index);
                     })}
 
                   {renderInputButton()}

bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/context.jsx

@@ -39,7 +39,7 @@ const reducer = (state, action) => {
       };
     }
     case 'update': {
-      if (action.background.custom) update(action.background);
+      if (action.background.custom && !action.background.sessionOnly) update(action.background);
       return {
         ...state,
         backgrounds: {

bigbluebutton-html5/imports/ui/components/video-provider/video-button/component.jsx

@@ -11,6 +11,7 @@ import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
 import Button from '/imports/ui/components/common/button/component';
 import VideoPreviewContainer from '/imports/ui/components/video-preview/container';
 import Settings from '/imports/ui/services/settings';
+import PreviewService from '/imports/ui/components/video-preview/service';
 
 const ENABLE_WEBCAM_SELECTOR_BUTTON = Meteor.settings.public.app.enableWebcamSelectorButton;
 const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@@ -108,6 +109,7 @@ const JoinVideoButton = ({
       default:
         if (exitVideo()) {
           VideoService.exitVideo();
+          PreviewService.clearStreams();
         } else {
           setForceOpen(isMobileSharingCamera);
           setVideoPreviewModalIsOpen(true);

bigbluebutton-html5/imports/ui/services/virtual-background/index.js

@@ -388,42 +388,8 @@ export async function createVirtualBackgroundService(parameters = null) {
     } else {
         parameters.virtualSource = virtualBackgroundImagePath + parameters.backgroundFilename;
 
-        if (parameters.customParams) {
+        if (parameters?.customParams?.file) {
-            if (parameters.customParams.file) {
+            parameters.virtualSource = parameters.customParams.file;
-                parameters.virtualSource = parameters.customParams.file;
-            } else {
-                const imageUrl = parameters.customParams.url;
-
-                // Function to convert image URL to a File object
-                async function getFileFromUrl(url) {
-                    try {
-                        const response = await fetch(url, {
-                            credentials: 'omit',
-                            mode: 'cors',
-                            headers: {
-                                'Accept': 'image/*',
-                            }
-                        });                
-                        if (!response.ok) {
-                            throw new Error(`HTTP error! status: ${response.status}`);
-                        }
-                        const blob = await response.blob();
-                        const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
-                        return file;
-                    } catch (error) {
-                        logger.error('Fetch error:', error);
-                        return null;
-                    }
-                }
-
-                let fetchedWebcamBackground = await getFileFromUrl(imageUrl);
-
-                if (fetchedWebcamBackground) {
-                    parameters.virtualSource = URL.createObjectURL(fetchedWebcamBackground);
-                } else {
-                    logger.error('Failed to fetch custom webcam background image. Using fallback image.');
-                }
-            }
         }
     }
 

bigbluebutton-html5/imports/ui/services/virtual-background/service.js

@@ -66,15 +66,17 @@ const getVirtualBackgroundThumbnail = (name) => {
 //   type: <EFFECT_TYPES>,
 //   name: effect filename, if any
 // }
-const setSessionVirtualBackgroundInfo = (type, name, deviceId) => {
+const setSessionVirtualBackgroundInfo = (
-  return Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name });
+  type,
-}
+  name,
+  customParams,
+  deviceId,
+) => Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name, customParams });
 
-const getSessionVirtualBackgroundInfo = (deviceId) => {
+const getSessionVirtualBackgroundInfo = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
-  return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
+  type: EFFECT_TYPES.NONE_TYPE,
-    type: EFFECT_TYPES.NONE_TYPE,
+  name: '',
-  };
+};
-}
 
 const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => {
   return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {

bigbluebutton-html5/imports/ui/services/webrtc-base/bbb-video-stream.js

@@ -90,6 +90,7 @@ export default class BBBVideoStream extends EventEmitter2 {
       });
       this.virtualBgType = type;
       this.virtualBgName = name;
+      this.customParams = customParams;
       return Promise.resolve();
     } catch (error) {
       return Promise.reject(error);
@@ -109,6 +110,7 @@ export default class BBBVideoStream extends EventEmitter2 {
       this.virtualBgService = service;
       this.virtualBgType = type;
       this.virtualBgName = name;
+      this.customParams = customParams;
       this.originalStream = this.mediaStream;
       this.mediaStream = effect;
       this.isVirtualBackgroundEnabled = true;