fix(webcam): client failing to apply virtual background effect (#20777)

* fix(webcam): client failing to apply virtual background effect

* fix: check for already dispatched background

* fix: make webcam start up with last selected virtual background
This commit is contained in:
João Victor Nunes 2024-07-25 16:49:32 -03:00 committed by GitHub
parent e1eabefc56
commit 027115aa14
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 161 additions and 122 deletions

View File

@ -1,12 +1,10 @@
import Auth from '/imports/ui/services/auth';
import Users from '/imports/api/users';
import PropTypes from 'prop-types'; import PropTypes from 'prop-types';
import React, { Component } from 'react'; import React, { Component } from 'react';
import { import {
defineMessages, injectIntl, FormattedMessage, defineMessages, injectIntl, FormattedMessage,
} from 'react-intl'; } from 'react-intl';
import Button from '/imports/ui/components/common/button/component'; import Button from '/imports/ui/components/common/button/component';
import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component' import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component';
import logger from '/imports/startup/client/logger'; import logger from '/imports/startup/client/logger';
import browserInfo from '/imports/utils/browserInfo'; import browserInfo from '/imports/utils/browserInfo';
import PreviewService from './service'; import PreviewService from './service';
@ -24,7 +22,7 @@ import {
} from '/imports/ui/services/virtual-background/service'; } from '/imports/ui/services/virtual-background/service';
import Settings from '/imports/ui/services/settings'; import Settings from '/imports/ui/services/settings';
import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features'; import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
import Checkbox from '/imports/ui/components/common/checkbox/component' import Checkbox from '/imports/ui/components/common/checkbox/component';
const VIEW_STATES = { const VIEW_STATES = {
finding: 'finding', finding: 'finding',
@ -234,6 +232,7 @@ class VideoPreview extends Component {
this.handleVirtualBgSelected = this.handleVirtualBgSelected.bind(this); this.handleVirtualBgSelected = this.handleVirtualBgSelected.bind(this);
this.handleLocalStreamInactive = this.handleLocalStreamInactive.bind(this); this.handleLocalStreamInactive = this.handleLocalStreamInactive.bind(this);
this.handleBrightnessAreaChange = this.handleBrightnessAreaChange.bind(this); this.handleBrightnessAreaChange = this.handleBrightnessAreaChange.bind(this);
this.updateVirtualBackgroundInfo = this.updateVirtualBackgroundInfo.bind(this);
this._isMounted = false; this._isMounted = false;
@ -327,21 +326,6 @@ class VideoPreview extends Component {
viewState: VIEW_STATES.found, viewState: VIEW_STATES.found,
}); });
this.displayPreview(); this.displayPreview();
// Set the custom or default virtual background
const webcamBackground = Users.findOne({
meetingId: Auth.meetingID,
userId: Auth.userID,
}, {
fields: {
webcamBackground: 1,
},
});
const webcamBackgroundURL = webcamBackground?.webcamBackground;
if (webcamBackgroundURL !== '') {
this.handleVirtualBgSelected(EFFECT_TYPES.IMAGE_TYPE, '', { url: webcamBackgroundURL });
}
}); });
} else { } else {
// There were no webcams coming from enumerateDevices. Throw an error. // There were no webcams coming from enumerateDevices. Throw an error.
@ -384,30 +368,6 @@ class VideoPreview extends Component {
this._isMounted = false; this._isMounted = false;
} }
startCameraBrightness() {
if (CAMERA_BRIGHTNESS_AVAILABLE) {
const setBrightnessInfo = () => {
const stream = this.currentVideoStream || {};
const service = stream.virtualBgService || {};
const { brightness = 100, wholeImageBrightness = false } = service;
this.setState({ brightness, wholeImageBrightness });
};
if (!this.currentVideoStream.virtualBgService) {
this.startVirtualBackground(
this.currentVideoStream,
EFFECT_TYPES.NONE_TYPE,
).then((switched) => {
if (switched) {
setBrightnessInfo();
}
});
} else {
setBrightnessInfo();
}
}
}
handleSelectWebcam(event) { handleSelectWebcam(event) {
const webcamValue = event.target.value; const webcamValue = event.target.value;
@ -432,34 +392,22 @@ class VideoPreview extends Component {
} }
} }
updateVirtualBackgroundInfo = () => {
const { webcamDeviceId } = this.state;
// Update this session's virtual camera effect information if it's enabled
setSessionVirtualBackgroundInfo(
this.currentVideoStream.virtualBgType,
this.currentVideoStream.virtualBgName,
webcamDeviceId,
);
};
// Resolves into true if the background switch is successful, false otherwise // Resolves into true if the background switch is successful, false otherwise
handleVirtualBgSelected(type, name, customParams) { handleVirtualBgSelected(type, name, customParams) {
const { webcamDeviceId } = this.state;
const shared = this.isAlreadyShared(webcamDeviceId);
if (type !== EFFECT_TYPES.NONE_TYPE || CAMERA_BRIGHTNESS_AVAILABLE) { if (type !== EFFECT_TYPES.NONE_TYPE || CAMERA_BRIGHTNESS_AVAILABLE) {
return this.startVirtualBackground(this.currentVideoStream, type, name, customParams).then((switched) => { return this.startVirtualBackground(
// If it's not shared we don't have to update here because this.currentVideoStream,
// it will be updated in the handleStartSharing method. type,
if (switched && shared) this.updateVirtualBackgroundInfo(); name,
customParams,
).then((switched) => {
if (switched) this.updateVirtualBackgroundInfo();
return switched; return switched;
}); });
} else {
this.stopVirtualBackground(this.currentVideoStream);
if (shared) this.updateVirtualBackgroundInfo();
return Promise.resolve(true);
} }
this.stopVirtualBackground(this.currentVideoStream);
this.updateVirtualBackgroundInfo();
return Promise.resolve(true);
} }
stopVirtualBackground(bbbVideoStream) { stopVirtualBackground(bbbVideoStream) {
@ -477,7 +425,7 @@ class VideoPreview extends Component {
return bbbVideoStream.startVirtualBackground(type, name, customParams).then(() => { return bbbVideoStream.startVirtualBackground(type, name, customParams).then(() => {
this.displayPreview(); this.displayPreview();
return true; return true;
}).catch(error => { }).catch((error) => {
this.handleVirtualBgError(error, type, name); this.handleVirtualBgError(error, type, name);
return false; return false;
}).finally(() => { }).finally(() => {
@ -523,7 +471,6 @@ class VideoPreview extends Component {
this.stopVirtualBackground(this.currentVideoStream); this.stopVirtualBackground(this.currentVideoStream);
} }
this.updateVirtualBackgroundInfo();
this.cleanupStreamAndVideo(); this.cleanupStreamAndVideo();
PreviewService.changeProfile(selectedProfile); PreviewService.changeProfile(selectedProfile);
@ -664,7 +611,9 @@ class VideoPreview extends Component {
getInitialCameraStream(deviceId) { getInitialCameraStream(deviceId) {
const { cameraAsContent } = this.props; const { cameraAsContent } = this.props;
const defaultProfile = !cameraAsContent ? PreviewService.getDefaultProfile() : PreviewService.getCameraAsContentProfile(); const defaultProfile = !cameraAsContent
? PreviewService.getDefaultProfile()
: PreviewService.getCameraAsContentProfile();
return this.getCameraStream(deviceId, defaultProfile).then(() => { return this.getCameraStream(deviceId, defaultProfile).then(() => {
this.updateDeviceId(deviceId); this.updateDeviceId(deviceId);
@ -689,10 +638,14 @@ class VideoPreview extends Component {
if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId); if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId);
this.currentVideoStream = bbbVideoStream; this.currentVideoStream = bbbVideoStream;
this.startCameraBrightness(); this.startCameraBrightness().then(() => {
const { type, name, customParams } = getSessionVirtualBackgroundInfo(deviceId);
this.handleVirtualBgSelected(type, name, customParams).then(() => {
this.setState({ this.setState({
isStartSharingDisabled: false, isStartSharingDisabled: false,
}); });
});
});
}).catch((error) => { }).catch((error) => {
// When video preview is set to skip, we need some way to bubble errors // When video preview is set to skip, we need some way to bubble errors
// up to users; so re-throw the error // up to users; so re-throw the error
@ -1046,6 +999,44 @@ class VideoPreview extends Component {
return intl.formatMessage(intlMessages.webcamSettingsTitle); return intl.formatMessage(intlMessages.webcamSettingsTitle);
} }
startCameraBrightness() {
if (CAMERA_BRIGHTNESS_AVAILABLE) {
const setBrightnessInfo = () => {
const stream = this.currentVideoStream || {};
const service = stream.virtualBgService || {};
const { brightness = 100, wholeImageBrightness = false } = service;
this.setState({ brightness, wholeImageBrightness });
};
if (!this.currentVideoStream.virtualBgService) {
return this.startVirtualBackground(
this.currentVideoStream,
EFFECT_TYPES.NONE_TYPE,
).then((switched) => {
if (switched) {
setBrightnessInfo();
}
});
}
setBrightnessInfo();
}
return Promise.resolve(true);
}
updateVirtualBackgroundInfo() {
const { webcamDeviceId } = this.state;
// Update this session's virtual camera effect information if it's enabled
setSessionVirtualBackgroundInfo(
this.currentVideoStream.virtualBgType,
this.currentVideoStream.virtualBgName,
this.currentVideoStream.customParams,
webcamDeviceId,
);
}
renderModalContent() { renderModalContent() {
const { const {
intl, intl,

View File

@ -74,6 +74,8 @@ const deleteStream = (deviceId) => {
return VIDEO_STREAM_STORAGE.delete(deviceId); return VIDEO_STREAM_STORAGE.delete(deviceId);
} }
const clearStreams = () => VIDEO_STREAM_STORAGE.clear();
const promiseTimeout = (ms, promise) => { const promiseTimeout = (ms, promise) => {
const timeout = new Promise((resolve, reject) => { const timeout = new Promise((resolve, reject) => {
const id = setTimeout(() => { const id = setTimeout(() => {
@ -246,4 +248,5 @@ export default {
getCameraProfile, getCameraProfile,
doGUM, doGUM,
terminateCameraStream, terminateCameraStream,
clearStreams,
}; };

View File

@ -18,6 +18,8 @@ import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
import 'react-loading-skeleton/dist/skeleton.css'; import 'react-loading-skeleton/dist/skeleton.css';
import Settings from '/imports/ui/services/settings'; import Settings from '/imports/ui/services/settings';
import { isCustomVirtualBackgroundsEnabled } from '/imports/ui/services/features'; import { isCustomVirtualBackgroundsEnabled } from '/imports/ui/services/features';
import Auth from '/imports/ui/services/auth';
import Users from '/imports/api/users';
const { MIME_TYPES_ALLOWED, MAX_FILE_SIZE } = VirtualBgService; const { MIME_TYPES_ALLOWED, MAX_FILE_SIZE } = VirtualBgService;
const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness; const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@ -94,6 +96,28 @@ const VIRTUAL_BACKGROUNDS_CONFIG = Meteor.settings.public.virtualBackgrounds;
const ENABLE_UPLOAD = VIRTUAL_BACKGROUNDS_CONFIG.enableVirtualBackgroundUpload; const ENABLE_UPLOAD = VIRTUAL_BACKGROUNDS_CONFIG.enableVirtualBackgroundUpload;
const shouldEnableBackgroundUpload = () => ENABLE_UPLOAD && isCustomVirtualBackgroundsEnabled(); const shouldEnableBackgroundUpload = () => ENABLE_UPLOAD && isCustomVirtualBackgroundsEnabled();
// Function to convert image URL to a File object
async function getFileFromUrl(url) {
try {
const response = await fetch(url, {
credentials: 'omit',
mode: 'cors',
headers: {
Accept: 'image/*',
},
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const blob = await response.blob();
const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
return file;
} catch (error) {
logger.error('Fetch error:', error);
return null;
}
}
const VirtualBgSelector = ({ const VirtualBgSelector = ({
intl, intl,
handleVirtualBgSelected, handleVirtualBgSelected,
@ -135,6 +159,51 @@ const VirtualBgSelector = ({
} }
if (!loaded) loadFromDB(); if (!loaded) loadFromDB();
} }
// Set the custom or default virtual background
const webcamBackground = Users.findOne({
meetingId: Auth.meetingID,
userId: Auth.userID,
}, {
fields: {
webcamBackground: 1,
},
});
const webcamBackgroundURL = webcamBackground?.webcamBackground;
if (webcamBackgroundURL !== '' && !backgrounds.webcamBackgroundURL) {
getFileFromUrl(webcamBackgroundURL).then((fetchedWebcamBackground) => {
if (fetchedWebcamBackground) {
const data = URL.createObjectURL(fetchedWebcamBackground);
const uniqueId = 'webcamBackgroundURL';
const filename = webcamBackgroundURL;
dispatch({
type: 'update',
background: {
filename,
uniqueId,
data,
lastActivityDate: Date.now(),
custom: true,
sessionOnly: true,
},
});
handleVirtualBgSelected(
EFFECT_TYPES.IMAGE_TYPE,
webcamBackgroundURL,
{ file: data, uniqueId },
).then((switched) => {
if (!switched) {
setCurrentVirtualBg({ type: EFFECT_TYPES.NONE_TYPE });
return;
}
setCurrentVirtualBg({ type: EFFECT_TYPES.IMAGE_TYPE, name: filename });
});
} else {
logger.error('Failed to fetch custom webcam background image. Using fallback image.');
}
});
}
}, []); }, []);
const _virtualBgSelected = (type, name, index, customParams) => const _virtualBgSelected = (type, name, index, customParams) =>
@ -161,6 +230,7 @@ const VirtualBgSelector = ({
filename: name, filename: name,
uniqueId: customParams.uniqueId, uniqueId: customParams.uniqueId,
data: customParams.file, data: customParams.file,
sessionOnly: customParams.sessionOnly,
custom: true, custom: true,
lastActivityDate: Date.now(), lastActivityDate: Date.now(),
}, },
@ -315,7 +385,9 @@ const VirtualBgSelector = ({
}; };
const renderCustomButton = (background, index) => { const renderCustomButton = (background, index) => {
const { filename, data, uniqueId } = background; const {
filename, data, uniqueId, sessionOnly,
} = background;
const label = intl.formatMessage(intlMessages.backgroundWithIndex, { const label = intl.formatMessage(intlMessages.backgroundWithIndex, {
0: index + 1, 0: index + 1,
}); });
@ -339,7 +411,7 @@ const VirtualBgSelector = ({
EFFECT_TYPES.IMAGE_TYPE, EFFECT_TYPES.IMAGE_TYPE,
filename, filename,
index, index,
{ file: data, uniqueId }, { file: data, uniqueId, sessionOnly },
)} )}
disabled={disabled} disabled={disabled}
isVisualEffects={isVisualEffects} isVisualEffects={isVisualEffects}
@ -459,10 +531,11 @@ const VirtualBgSelector = ({
.map((background, index) => { .map((background, index) => {
if (background.custom !== false) { if (background.custom !== false) {
return renderCustomButton(background, index); return renderCustomButton(background, index);
} else {
const isBlur = background.uniqueId.includes('Blur');
return isBlur ? renderBlurButton(index) : renderDefaultButton(background.uniqueId, index);
} }
const isBlur = background.uniqueId.includes('Blur');
return isBlur
? renderBlurButton(index)
: renderDefaultButton(background.uniqueId, index);
})} })}
{renderInputButton()} {renderInputButton()}

View File

@ -39,7 +39,7 @@ const reducer = (state, action) => {
}; };
} }
case 'update': { case 'update': {
if (action.background.custom) update(action.background); if (action.background.custom && !action.background.sessionOnly) update(action.background);
return { return {
...state, ...state,
backgrounds: { backgrounds: {

View File

@ -11,6 +11,7 @@ import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
import Button from '/imports/ui/components/common/button/component'; import Button from '/imports/ui/components/common/button/component';
import VideoPreviewContainer from '/imports/ui/components/video-preview/container'; import VideoPreviewContainer from '/imports/ui/components/video-preview/container';
import Settings from '/imports/ui/services/settings'; import Settings from '/imports/ui/services/settings';
import PreviewService from '/imports/ui/components/video-preview/service';
const ENABLE_WEBCAM_SELECTOR_BUTTON = Meteor.settings.public.app.enableWebcamSelectorButton; const ENABLE_WEBCAM_SELECTOR_BUTTON = Meteor.settings.public.app.enableWebcamSelectorButton;
const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness; const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@ -108,6 +109,7 @@ const JoinVideoButton = ({
default: default:
if (exitVideo()) { if (exitVideo()) {
VideoService.exitVideo(); VideoService.exitVideo();
PreviewService.clearStreams();
} else { } else {
setForceOpen(isMobileSharingCamera); setForceOpen(isMobileSharingCamera);
setVideoPreviewModalIsOpen(true); setVideoPreviewModalIsOpen(true);

View File

@ -388,42 +388,8 @@ export async function createVirtualBackgroundService(parameters = null) {
} else { } else {
parameters.virtualSource = virtualBackgroundImagePath + parameters.backgroundFilename; parameters.virtualSource = virtualBackgroundImagePath + parameters.backgroundFilename;
if (parameters.customParams) { if (parameters?.customParams?.file) {
if (parameters.customParams.file) {
parameters.virtualSource = parameters.customParams.file; parameters.virtualSource = parameters.customParams.file;
} else {
const imageUrl = parameters.customParams.url;
// Function to convert image URL to a File object
async function getFileFromUrl(url) {
try {
const response = await fetch(url, {
credentials: 'omit',
mode: 'cors',
headers: {
'Accept': 'image/*',
}
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const blob = await response.blob();
const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
return file;
} catch (error) {
logger.error('Fetch error:', error);
return null;
}
}
let fetchedWebcamBackground = await getFileFromUrl(imageUrl);
if (fetchedWebcamBackground) {
parameters.virtualSource = URL.createObjectURL(fetchedWebcamBackground);
} else {
logger.error('Failed to fetch custom webcam background image. Using fallback image.');
}
}
} }
} }

View File

@ -66,15 +66,17 @@ const getVirtualBackgroundThumbnail = (name) => {
// type: <EFFECT_TYPES>, // type: <EFFECT_TYPES>,
// name: effect filename, if any // name: effect filename, if any
// } // }
const setSessionVirtualBackgroundInfo = (type, name, deviceId) => { const setSessionVirtualBackgroundInfo = (
return Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name }); type,
} name,
customParams,
deviceId,
) => Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name, customParams });
const getSessionVirtualBackgroundInfo = (deviceId) => { const getSessionVirtualBackgroundInfo = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
type: EFFECT_TYPES.NONE_TYPE, type: EFFECT_TYPES.NONE_TYPE,
name: '',
}; };
}
const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => { const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => {
return Session.get(`VirtualBackgroundInfo_${deviceId}`) || { return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {

View File

@ -90,6 +90,7 @@ export default class BBBVideoStream extends EventEmitter2 {
}); });
this.virtualBgType = type; this.virtualBgType = type;
this.virtualBgName = name; this.virtualBgName = name;
this.customParams = customParams;
return Promise.resolve(); return Promise.resolve();
} catch (error) { } catch (error) {
return Promise.reject(error); return Promise.reject(error);
@ -109,6 +110,7 @@ export default class BBBVideoStream extends EventEmitter2 {
this.virtualBgService = service; this.virtualBgService = service;
this.virtualBgType = type; this.virtualBgType = type;
this.virtualBgName = name; this.virtualBgName = name;
this.customParams = customParams;
this.originalStream = this.mediaStream; this.originalStream = this.mediaStream;
this.mediaStream = effect; this.mediaStream = effect;
this.isVirtualBackgroundEnabled = true; this.isVirtualBackgroundEnabled = true;