From 027115aa140de3c2cb77fbb37f1d5834699ae02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Victor=20Nunes?= <62393923+JoVictorNunes@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:49:32 -0300 Subject: [PATCH] fix(webcam): client failing to apply virtual background effect (#20777) * fix(webcam): client failing to apply virtual background effect * fix: check for already dispatched background * fix: make webcam start up with last selected virtual background --- .../ui/components/video-preview/component.jsx | 135 ++++++++---------- .../ui/components/video-preview/service.js | 3 + .../virtual-background/component.jsx | 83 ++++++++++- .../virtual-background/context.jsx | 2 +- .../video-provider/video-button/component.jsx | 2 + .../ui/services/virtual-background/index.js | 38 +---- .../ui/services/virtual-background/service.js | 18 +-- .../services/webrtc-base/bbb-video-stream.js | 2 + 8 files changed, 161 insertions(+), 122 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/components/video-preview/component.jsx b/bigbluebutton-html5/imports/ui/components/video-preview/component.jsx index 12933560cf..9f680b6509 100755 --- a/bigbluebutton-html5/imports/ui/components/video-preview/component.jsx +++ b/bigbluebutton-html5/imports/ui/components/video-preview/component.jsx @@ -1,12 +1,10 @@ -import Auth from '/imports/ui/services/auth'; -import Users from '/imports/api/users'; import PropTypes from 'prop-types'; import React, { Component } from 'react'; import { defineMessages, injectIntl, FormattedMessage, } from 'react-intl'; import Button from '/imports/ui/components/common/button/component'; -import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component' +import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component'; import logger from '/imports/startup/client/logger'; import browserInfo from '/imports/utils/browserInfo'; import PreviewService from './service'; @@ -24,7 +22,7 @@ import { } from '/imports/ui/services/virtual-background/service'; import Settings from '/imports/ui/services/settings'; import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features'; -import Checkbox from '/imports/ui/components/common/checkbox/component' +import Checkbox from '/imports/ui/components/common/checkbox/component'; const VIEW_STATES = { finding: 'finding', @@ -234,6 +232,7 @@ class VideoPreview extends Component { this.handleVirtualBgSelected = this.handleVirtualBgSelected.bind(this); this.handleLocalStreamInactive = this.handleLocalStreamInactive.bind(this); this.handleBrightnessAreaChange = this.handleBrightnessAreaChange.bind(this); + this.updateVirtualBackgroundInfo = this.updateVirtualBackgroundInfo.bind(this); this._isMounted = false; @@ -327,21 +326,6 @@ class VideoPreview extends Component { viewState: VIEW_STATES.found, }); this.displayPreview(); - - // Set the custom or default virtual background - const webcamBackground = Users.findOne({ - meetingId: Auth.meetingID, - userId: Auth.userID, - }, { - fields: { - webcamBackground: 1, - }, - }); - - const webcamBackgroundURL = webcamBackground?.webcamBackground; - if (webcamBackgroundURL !== '') { - this.handleVirtualBgSelected(EFFECT_TYPES.IMAGE_TYPE, '', { url: webcamBackgroundURL }); - } }); } else { // There were no webcams coming from enumerateDevices. Throw an error. @@ -384,30 +368,6 @@ class VideoPreview extends Component { this._isMounted = false; } - startCameraBrightness() { - if (CAMERA_BRIGHTNESS_AVAILABLE) { - const setBrightnessInfo = () => { - const stream = this.currentVideoStream || {}; - const service = stream.virtualBgService || {}; - const { brightness = 100, wholeImageBrightness = false } = service; - this.setState({ brightness, wholeImageBrightness }); - }; - - if (!this.currentVideoStream.virtualBgService) { - this.startVirtualBackground( - this.currentVideoStream, - EFFECT_TYPES.NONE_TYPE, - ).then((switched) => { - if (switched) { - setBrightnessInfo(); - } - }); - } else { - setBrightnessInfo(); - } - } - } - handleSelectWebcam(event) { const webcamValue = event.target.value; @@ -432,34 +392,22 @@ class VideoPreview extends Component { } } - updateVirtualBackgroundInfo = () => { - const { webcamDeviceId } = this.state; - - // Update this session's virtual camera effect information if it's enabled - setSessionVirtualBackgroundInfo( - this.currentVideoStream.virtualBgType, - this.currentVideoStream.virtualBgName, - webcamDeviceId, - ); - }; - // Resolves into true if the background switch is successful, false otherwise handleVirtualBgSelected(type, name, customParams) { - const { webcamDeviceId } = this.state; - const shared = this.isAlreadyShared(webcamDeviceId); - if (type !== EFFECT_TYPES.NONE_TYPE || CAMERA_BRIGHTNESS_AVAILABLE) { - return this.startVirtualBackground(this.currentVideoStream, type, name, customParams).then((switched) => { - // If it's not shared we don't have to update here because - // it will be updated in the handleStartSharing method. - if (switched && shared) this.updateVirtualBackgroundInfo(); + return this.startVirtualBackground( + this.currentVideoStream, + type, + name, + customParams, + ).then((switched) => { + if (switched) this.updateVirtualBackgroundInfo(); return switched; }); - } else { - this.stopVirtualBackground(this.currentVideoStream); - if (shared) this.updateVirtualBackgroundInfo(); - return Promise.resolve(true); } + this.stopVirtualBackground(this.currentVideoStream); + this.updateVirtualBackgroundInfo(); + return Promise.resolve(true); } stopVirtualBackground(bbbVideoStream) { @@ -477,7 +425,7 @@ class VideoPreview extends Component { return bbbVideoStream.startVirtualBackground(type, name, customParams).then(() => { this.displayPreview(); return true; - }).catch(error => { + }).catch((error) => { this.handleVirtualBgError(error, type, name); return false; }).finally(() => { @@ -511,7 +459,7 @@ class VideoPreview extends Component { // Only streams that will be shared should be stored in the service. // If the store call returns false, we're duplicating stuff. So clean this one // up because it's an impostor. - if(!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) { + if (!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) { this.currentVideoStream.stop(); } @@ -523,7 +471,6 @@ class VideoPreview extends Component { this.stopVirtualBackground(this.currentVideoStream); } - this.updateVirtualBackgroundInfo(); this.cleanupStreamAndVideo(); PreviewService.changeProfile(selectedProfile); @@ -664,7 +611,9 @@ class VideoPreview extends Component { getInitialCameraStream(deviceId) { const { cameraAsContent } = this.props; - const defaultProfile = !cameraAsContent ? PreviewService.getDefaultProfile() : PreviewService.getCameraAsContentProfile(); + const defaultProfile = !cameraAsContent + ? PreviewService.getDefaultProfile() + : PreviewService.getCameraAsContentProfile(); return this.getCameraStream(deviceId, defaultProfile).then(() => { this.updateDeviceId(deviceId); @@ -689,9 +638,13 @@ class VideoPreview extends Component { if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId); this.currentVideoStream = bbbVideoStream; - this.startCameraBrightness(); - this.setState({ - isStartSharingDisabled: false, + this.startCameraBrightness().then(() => { + const { type, name, customParams } = getSessionVirtualBackgroundInfo(deviceId); + this.handleVirtualBgSelected(type, name, customParams).then(() => { + this.setState({ + isStartSharingDisabled: false, + }); + }); }); }).catch((error) => { // When video preview is set to skip, we need some way to bubble errors @@ -1046,6 +999,44 @@ class VideoPreview extends Component { return intl.formatMessage(intlMessages.webcamSettingsTitle); } + startCameraBrightness() { + if (CAMERA_BRIGHTNESS_AVAILABLE) { + const setBrightnessInfo = () => { + const stream = this.currentVideoStream || {}; + const service = stream.virtualBgService || {}; + const { brightness = 100, wholeImageBrightness = false } = service; + this.setState({ brightness, wholeImageBrightness }); + }; + + if (!this.currentVideoStream.virtualBgService) { + return this.startVirtualBackground( + this.currentVideoStream, + EFFECT_TYPES.NONE_TYPE, + ).then((switched) => { + if (switched) { + setBrightnessInfo(); + } + }); + } + + setBrightnessInfo(); + } + + return Promise.resolve(true); + } + + updateVirtualBackgroundInfo() { + const { webcamDeviceId } = this.state; + + // Update this session's virtual camera effect information if it's enabled + setSessionVirtualBackgroundInfo( + this.currentVideoStream.virtualBgType, + this.currentVideoStream.virtualBgName, + this.currentVideoStream.customParams, + webcamDeviceId, + ); + } + renderModalContent() { const { intl, diff --git a/bigbluebutton-html5/imports/ui/components/video-preview/service.js b/bigbluebutton-html5/imports/ui/components/video-preview/service.js index e5b1ea39f5..e0701b6c7a 100755 --- a/bigbluebutton-html5/imports/ui/components/video-preview/service.js +++ b/bigbluebutton-html5/imports/ui/components/video-preview/service.js @@ -74,6 +74,8 @@ const deleteStream = (deviceId) => { return VIDEO_STREAM_STORAGE.delete(deviceId); } +const clearStreams = () => VIDEO_STREAM_STORAGE.clear(); + const promiseTimeout = (ms, promise) => { const timeout = new Promise((resolve, reject) => { const id = setTimeout(() => { @@ -246,4 +248,5 @@ export default { getCameraProfile, doGUM, terminateCameraStream, + clearStreams, }; diff --git a/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx b/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx index 9c9176a11c..8c3707f35c 100644 --- a/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx +++ b/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx @@ -18,6 +18,8 @@ import Skeleton, { SkeletonTheme } from 'react-loading-skeleton'; import 'react-loading-skeleton/dist/skeleton.css'; import Settings from '/imports/ui/services/settings'; import { isCustomVirtualBackgroundsEnabled } from '/imports/ui/services/features'; +import Auth from '/imports/ui/services/auth'; +import Users from '/imports/api/users'; const { MIME_TYPES_ALLOWED, MAX_FILE_SIZE } = VirtualBgService; const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness; @@ -94,6 +96,28 @@ const VIRTUAL_BACKGROUNDS_CONFIG = Meteor.settings.public.virtualBackgrounds; const ENABLE_UPLOAD = VIRTUAL_BACKGROUNDS_CONFIG.enableVirtualBackgroundUpload; const shouldEnableBackgroundUpload = () => ENABLE_UPLOAD && isCustomVirtualBackgroundsEnabled(); +// Function to convert image URL to a File object +async function getFileFromUrl(url) { + try { + const response = await fetch(url, { + credentials: 'omit', + mode: 'cors', + headers: { + Accept: 'image/*', + }, + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + const blob = await response.blob(); + const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type }); + return file; + } catch (error) { + logger.error('Fetch error:', error); + return null; + } +} + const VirtualBgSelector = ({ intl, handleVirtualBgSelected, @@ -135,6 +159,51 @@ const VirtualBgSelector = ({ } if (!loaded) loadFromDB(); } + + // Set the custom or default virtual background + const webcamBackground = Users.findOne({ + meetingId: Auth.meetingID, + userId: Auth.userID, + }, { + fields: { + webcamBackground: 1, + }, + }); + + const webcamBackgroundURL = webcamBackground?.webcamBackground; + if (webcamBackgroundURL !== '' && !backgrounds.webcamBackgroundURL) { + getFileFromUrl(webcamBackgroundURL).then((fetchedWebcamBackground) => { + if (fetchedWebcamBackground) { + const data = URL.createObjectURL(fetchedWebcamBackground); + const uniqueId = 'webcamBackgroundURL'; + const filename = webcamBackgroundURL; + dispatch({ + type: 'update', + background: { + filename, + uniqueId, + data, + lastActivityDate: Date.now(), + custom: true, + sessionOnly: true, + }, + }); + handleVirtualBgSelected( + EFFECT_TYPES.IMAGE_TYPE, + webcamBackgroundURL, + { file: data, uniqueId }, + ).then((switched) => { + if (!switched) { + setCurrentVirtualBg({ type: EFFECT_TYPES.NONE_TYPE }); + return; + } + setCurrentVirtualBg({ type: EFFECT_TYPES.IMAGE_TYPE, name: filename }); + }); + } else { + logger.error('Failed to fetch custom webcam background image. Using fallback image.'); + } + }); + } }, []); const _virtualBgSelected = (type, name, index, customParams) => @@ -161,6 +230,7 @@ const VirtualBgSelector = ({ filename: name, uniqueId: customParams.uniqueId, data: customParams.file, + sessionOnly: customParams.sessionOnly, custom: true, lastActivityDate: Date.now(), }, @@ -315,7 +385,9 @@ const VirtualBgSelector = ({ }; const renderCustomButton = (background, index) => { - const { filename, data, uniqueId } = background; + const { + filename, data, uniqueId, sessionOnly, + } = background; const label = intl.formatMessage(intlMessages.backgroundWithIndex, { 0: index + 1, }); @@ -339,7 +411,7 @@ const VirtualBgSelector = ({ EFFECT_TYPES.IMAGE_TYPE, filename, index, - { file: data, uniqueId }, + { file: data, uniqueId, sessionOnly }, )} disabled={disabled} isVisualEffects={isVisualEffects} @@ -459,10 +531,11 @@ const VirtualBgSelector = ({ .map((background, index) => { if (background.custom !== false) { return renderCustomButton(background, index); - } else { - const isBlur = background.uniqueId.includes('Blur'); - return isBlur ? renderBlurButton(index) : renderDefaultButton(background.uniqueId, index); } + const isBlur = background.uniqueId.includes('Blur'); + return isBlur + ? renderBlurButton(index) + : renderDefaultButton(background.uniqueId, index); })} {renderInputButton()} diff --git a/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/context.jsx b/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/context.jsx index 05e14a62bf..1f051331f2 100644 --- a/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/context.jsx +++ b/bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/context.jsx @@ -39,7 +39,7 @@ const reducer = (state, action) => { }; } case 'update': { - if (action.background.custom) update(action.background); + if (action.background.custom && !action.background.sessionOnly) update(action.background); return { ...state, backgrounds: { diff --git a/bigbluebutton-html5/imports/ui/components/video-provider/video-button/component.jsx b/bigbluebutton-html5/imports/ui/components/video-provider/video-button/component.jsx index 01932e9ca5..d5490af5a1 100755 --- a/bigbluebutton-html5/imports/ui/components/video-provider/video-button/component.jsx +++ b/bigbluebutton-html5/imports/ui/components/video-provider/video-button/component.jsx @@ -11,6 +11,7 @@ import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features'; import Button from '/imports/ui/components/common/button/component'; import VideoPreviewContainer from '/imports/ui/components/video-preview/container'; import Settings from '/imports/ui/services/settings'; +import PreviewService from '/imports/ui/components/video-preview/service'; const ENABLE_WEBCAM_SELECTOR_BUTTON = Meteor.settings.public.app.enableWebcamSelectorButton; const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness; @@ -108,6 +109,7 @@ const JoinVideoButton = ({ default: if (exitVideo()) { VideoService.exitVideo(); + PreviewService.clearStreams(); } else { setForceOpen(isMobileSharingCamera); setVideoPreviewModalIsOpen(true); diff --git a/bigbluebutton-html5/imports/ui/services/virtual-background/index.js b/bigbluebutton-html5/imports/ui/services/virtual-background/index.js index 90e7c8eab6..5497dbe253 100644 --- a/bigbluebutton-html5/imports/ui/services/virtual-background/index.js +++ b/bigbluebutton-html5/imports/ui/services/virtual-background/index.js @@ -388,42 +388,8 @@ export async function createVirtualBackgroundService(parameters = null) { } else { parameters.virtualSource = virtualBackgroundImagePath + parameters.backgroundFilename; - if (parameters.customParams) { - if (parameters.customParams.file) { - parameters.virtualSource = parameters.customParams.file; - } else { - const imageUrl = parameters.customParams.url; - - // Function to convert image URL to a File object - async function getFileFromUrl(url) { - try { - const response = await fetch(url, { - credentials: 'omit', - mode: 'cors', - headers: { - 'Accept': 'image/*', - } - }); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - const blob = await response.blob(); - const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type }); - return file; - } catch (error) { - logger.error('Fetch error:', error); - return null; - } - } - - let fetchedWebcamBackground = await getFileFromUrl(imageUrl); - - if (fetchedWebcamBackground) { - parameters.virtualSource = URL.createObjectURL(fetchedWebcamBackground); - } else { - logger.error('Failed to fetch custom webcam background image. Using fallback image.'); - } - } + if (parameters?.customParams?.file) { + parameters.virtualSource = parameters.customParams.file; } } diff --git a/bigbluebutton-html5/imports/ui/services/virtual-background/service.js b/bigbluebutton-html5/imports/ui/services/virtual-background/service.js index b220a6cd43..7386171f6a 100644 --- a/bigbluebutton-html5/imports/ui/services/virtual-background/service.js +++ b/bigbluebutton-html5/imports/ui/services/virtual-background/service.js @@ -66,15 +66,17 @@ const getVirtualBackgroundThumbnail = (name) => { // type: , // name: effect filename, if any // } -const setSessionVirtualBackgroundInfo = (type, name, deviceId) => { - return Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name }); -} +const setSessionVirtualBackgroundInfo = ( + type, + name, + customParams, + deviceId, +) => Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name, customParams }); -const getSessionVirtualBackgroundInfo = (deviceId) => { - return Session.get(`VirtualBackgroundInfo_${deviceId}`) || { - type: EFFECT_TYPES.NONE_TYPE, - }; -} +const getSessionVirtualBackgroundInfo = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`) || { + type: EFFECT_TYPES.NONE_TYPE, + name: '', +}; const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => { return Session.get(`VirtualBackgroundInfo_${deviceId}`) || { diff --git a/bigbluebutton-html5/imports/ui/services/webrtc-base/bbb-video-stream.js b/bigbluebutton-html5/imports/ui/services/webrtc-base/bbb-video-stream.js index 017bdf6869..d34d3c8ea1 100644 --- a/bigbluebutton-html5/imports/ui/services/webrtc-base/bbb-video-stream.js +++ b/bigbluebutton-html5/imports/ui/services/webrtc-base/bbb-video-stream.js @@ -90,6 +90,7 @@ export default class BBBVideoStream extends EventEmitter2 { }); this.virtualBgType = type; this.virtualBgName = name; + this.customParams = customParams; return Promise.resolve(); } catch (error) { return Promise.reject(error); @@ -109,6 +110,7 @@ export default class BBBVideoStream extends EventEmitter2 { this.virtualBgService = service; this.virtualBgType = type; this.virtualBgName = name; + this.customParams = customParams; this.originalStream = this.mediaStream; this.mediaStream = effect; this.isVirtualBackgroundEnabled = true;