BigBlueButton/bigbluebutton-Github
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(webcam): client failing to apply virtual background effect (#20777)

Browse Source 
* fix(webcam): client failing to apply virtual background effect

* fix: check for already dispatched background

* fix: make webcam start up with last selected virtual background
This commit is contained in:
João Victor Nunes 2024-07-25 16:49:32 -03:00 committed by GitHub
parent e1eabefc56
commit 027115aa14
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 161 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
bigbluebutton-html5/imports/ui/components/video-preview/component.jsx
View File

@ -1,12 +1,10 @@
import Auth from '/imports/ui/services/auth';
import Users from '/imports/api/users';
import PropTypes from 'prop-types';
import React, { Component } from 'react';
import {
defineMessages, injectIntl, FormattedMessage,
} from 'react-intl';
import Button from '/imports/ui/components/common/button/component';
import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component'
import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component';
import logger from '/imports/startup/client/logger';
import browserInfo from '/imports/utils/browserInfo';
import PreviewService from './service';
@ -24,7 +22,7 @@ import {
} from '/imports/ui/services/virtual-background/service';
import Settings from '/imports/ui/services/settings';
import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
import Checkbox from '/imports/ui/components/common/checkbox/component'
import Checkbox from '/imports/ui/components/common/checkbox/component';
const VIEW_STATES = {
finding: 'finding',
@ -234,6 +232,7 @@ class VideoPreview extends Component {
this.handleVirtualBgSelected = this.handleVirtualBgSelected.bind(this);
this.handleLocalStreamInactive = this.handleLocalStreamInactive.bind(this);
this.handleBrightnessAreaChange = this.handleBrightnessAreaChange.bind(this);
this.updateVirtualBackgroundInfo = this.updateVirtualBackgroundInfo.bind(this);
this._isMounted = false;
@ -327,21 +326,6 @@ class VideoPreview extends Component {
viewState: VIEW_STATES.found,
});
this.displayPreview();
// Set the custom or default virtual background
const webcamBackground = Users.findOne({
meetingId: Auth.meetingID,
userId: Auth.userID,
}, {
fields: {
webcamBackground: 1,
},
});
const webcamBackgroundURL = webcamBackground?.webcamBackground;
if (webcamBackgroundURL !== '') {
this.handleVirtualBgSelected(EFFECT_TYPES.IMAGE_TYPE, '', { url: webcamBackgroundURL });
}
});
} else {
// There were no webcams coming from enumerateDevices. Throw an error.
@ -384,30 +368,6 @@ class VideoPreview extends Component {
this._isMounted = false;
}
startCameraBrightness() {
if (CAMERA_BRIGHTNESS_AVAILABLE) {
const setBrightnessInfo = () => {
const stream = this.currentVideoStream || {};
const service = stream.virtualBgService || {};
const { brightness = 100, wholeImageBrightness = false } = service;
this.setState({ brightness, wholeImageBrightness });
};
if (!this.currentVideoStream.virtualBgService) {
this.startVirtualBackground(
this.currentVideoStream,
EFFECT_TYPES.NONE_TYPE,
).then((switched) => {
if (switched) {
setBrightnessInfo();
}
});
} else {
setBrightnessInfo();
}
}
}
handleSelectWebcam(event) {
const webcamValue = event.target.value;
@ -432,34 +392,22 @@ class VideoPreview extends Component {
}
}
updateVirtualBackgroundInfo = () => {
const { webcamDeviceId } = this.state;
// Update this session's virtual camera effect information if it's enabled
setSessionVirtualBackgroundInfo(
this.currentVideoStream.virtualBgType,
this.currentVideoStream.virtualBgName,
webcamDeviceId,
);
};
// Resolves into true if the background switch is successful, false otherwise
handleVirtualBgSelected(type, name, customParams) {
const { webcamDeviceId } = this.state;
const shared = this.isAlreadyShared(webcamDeviceId);
if (type !== EFFECT_TYPES.NONE_TYPE || CAMERA_BRIGHTNESS_AVAILABLE) {
return this.startVirtualBackground(this.currentVideoStream, type, name, customParams).then((switched) => {
// If it's not shared we don't have to update here because
// it will be updated in the handleStartSharing method.
if (switched && shared) this.updateVirtualBackgroundInfo();
return this.startVirtualBackground(
this.currentVideoStream,
type,
name,
customParams,
).then((switched) => {
if (switched) this.updateVirtualBackgroundInfo();
return switched;
});
} else {
this.stopVirtualBackground(this.currentVideoStream);
if (shared) this.updateVirtualBackgroundInfo();
return Promise.resolve(true);
}
this.stopVirtualBackground(this.currentVideoStream);
this.updateVirtualBackgroundInfo();
return Promise.resolve(true);
}
stopVirtualBackground(bbbVideoStream) {
@ -477,7 +425,7 @@ class VideoPreview extends Component {
return bbbVideoStream.startVirtualBackground(type, name, customParams).then(() => {
this.displayPreview();
return true;
}).catch(error => {
}).catch((error) => {
this.handleVirtualBgError(error, type, name);
return false;
}).finally(() => {
@ -511,7 +459,7 @@ class VideoPreview extends Component {
// Only streams that will be shared should be stored in the service.
// If the store call returns false, we're duplicating stuff. So clean this one
// up because it's an impostor.
if(!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) {
if (!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) {
this.currentVideoStream.stop();
}
@ -523,7 +471,6 @@ class VideoPreview extends Component {
this.stopVirtualBackground(this.currentVideoStream);
}
this.updateVirtualBackgroundInfo();
this.cleanupStreamAndVideo();
PreviewService.changeProfile(selectedProfile);
@ -664,7 +611,9 @@ class VideoPreview extends Component {
getInitialCameraStream(deviceId) {
const { cameraAsContent } = this.props;
const defaultProfile = !cameraAsContent ? PreviewService.getDefaultProfile() : PreviewService.getCameraAsContentProfile();
const defaultProfile = !cameraAsContent
? PreviewService.getDefaultProfile()
: PreviewService.getCameraAsContentProfile();
return this.getCameraStream(deviceId, defaultProfile).then(() => {
this.updateDeviceId(deviceId);
@ -689,10 +638,14 @@ class VideoPreview extends Component {
if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId);
this.currentVideoStream = bbbVideoStream;
this.startCameraBrightness();
this.startCameraBrightness().then(() => {
const { type, name, customParams } = getSessionVirtualBackgroundInfo(deviceId);
this.handleVirtualBgSelected(type, name, customParams).then(() => {
this.setState({
isStartSharingDisabled: false,
});
});
});
}).catch((error) => {
// When video preview is set to skip, we need some way to bubble errors
// up to users; so re-throw the error
@ -1046,6 +999,44 @@ class VideoPreview extends Component {
return intl.formatMessage(intlMessages.webcamSettingsTitle);
}
startCameraBrightness() {
if (CAMERA_BRIGHTNESS_AVAILABLE) {
const setBrightnessInfo = () => {
const stream = this.currentVideoStream || {};
const service = stream.virtualBgService || {};
const { brightness = 100, wholeImageBrightness = false } = service;
this.setState({ brightness, wholeImageBrightness });
};
if (!this.currentVideoStream.virtualBgService) {
return this.startVirtualBackground(
this.currentVideoStream,
EFFECT_TYPES.NONE_TYPE,
).then((switched) => {
if (switched) {
setBrightnessInfo();
}
});
}
setBrightnessInfo();
}
return Promise.resolve(true);
}
updateVirtualBackgroundInfo() {
const { webcamDeviceId } = this.state;
// Update this session's virtual camera effect information if it's enabled
setSessionVirtualBackgroundInfo(
this.currentVideoStream.virtualBgType,
this.currentVideoStream.virtualBgName,
this.currentVideoStream.customParams,
webcamDeviceId,
);
}
renderModalContent() {
const {
intl,

3
bigbluebutton-html5/imports/ui/components/video-preview/service.js
View File

@ -74,6 +74,8 @@ const deleteStream = (deviceId) => {
return VIDEO_STREAM_STORAGE.delete(deviceId);
}
const clearStreams = () => VIDEO_STREAM_STORAGE.clear();
const promiseTimeout = (ms, promise) => {
const timeout = new Promise((resolve, reject) => {
const id = setTimeout(() => {
@ -246,4 +248,5 @@ export default {
getCameraProfile,
doGUM,
terminateCameraStream,
clearStreams,
};

83
bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/component.jsx
View File

@ -18,6 +18,8 @@ import Skeleton, { SkeletonTheme } from 'react-loading-skeleton';
import 'react-loading-skeleton/dist/skeleton.css';
import Settings from '/imports/ui/services/settings';
import { isCustomVirtualBackgroundsEnabled } from '/imports/ui/services/features';
import Auth from '/imports/ui/services/auth';
import Users from '/imports/api/users';
const { MIME_TYPES_ALLOWED, MAX_FILE_SIZE } = VirtualBgService;
const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@ -94,6 +96,28 @@ const VIRTUAL_BACKGROUNDS_CONFIG = Meteor.settings.public.virtualBackgrounds;
const ENABLE_UPLOAD = VIRTUAL_BACKGROUNDS_CONFIG.enableVirtualBackgroundUpload;
const shouldEnableBackgroundUpload = () => ENABLE_UPLOAD && isCustomVirtualBackgroundsEnabled();
// Function to convert image URL to a File object
async function getFileFromUrl(url) {
try {
const response = await fetch(url, {
credentials: 'omit',
mode: 'cors',
headers: {
Accept: 'image/*',
},
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const blob = await response.blob();
const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
return file;
} catch (error) {
logger.error('Fetch error:', error);
return null;
}
}
const VirtualBgSelector = ({
intl,
handleVirtualBgSelected,
@ -135,6 +159,51 @@ const VirtualBgSelector = ({
}
if (!loaded) loadFromDB();
}
// Set the custom or default virtual background
const webcamBackground = Users.findOne({
meetingId: Auth.meetingID,
userId: Auth.userID,
}, {
fields: {
webcamBackground: 1,
},
});
const webcamBackgroundURL = webcamBackground?.webcamBackground;
if (webcamBackgroundURL !== '' && !backgrounds.webcamBackgroundURL) {
getFileFromUrl(webcamBackgroundURL).then((fetchedWebcamBackground) => {
if (fetchedWebcamBackground) {
const data = URL.createObjectURL(fetchedWebcamBackground);
const uniqueId = 'webcamBackgroundURL';
const filename = webcamBackgroundURL;
dispatch({
type: 'update',
background: {
filename,
uniqueId,
data,
lastActivityDate: Date.now(),
custom: true,
sessionOnly: true,
},
});
handleVirtualBgSelected(
EFFECT_TYPES.IMAGE_TYPE,
webcamBackgroundURL,
{ file: data, uniqueId },
).then((switched) => {
if (!switched) {
setCurrentVirtualBg({ type: EFFECT_TYPES.NONE_TYPE });
return;
}
setCurrentVirtualBg({ type: EFFECT_TYPES.IMAGE_TYPE, name: filename });
});
} else {
logger.error('Failed to fetch custom webcam background image. Using fallback image.');
}
});
}
}, []);
const _virtualBgSelected = (type, name, index, customParams) =>
@ -161,6 +230,7 @@ const VirtualBgSelector = ({
filename: name,
uniqueId: customParams.uniqueId,
data: customParams.file,
sessionOnly: customParams.sessionOnly,
custom: true,
lastActivityDate: Date.now(),
},
@ -315,7 +385,9 @@ const VirtualBgSelector = ({
};
const renderCustomButton = (background, index) => {
const { filename, data, uniqueId } = background;
const {
filename, data, uniqueId, sessionOnly,
} = background;
const label = intl.formatMessage(intlMessages.backgroundWithIndex, {
0: index + 1,
});
@ -339,7 +411,7 @@ const VirtualBgSelector = ({
EFFECT_TYPES.IMAGE_TYPE,
filename,
index,
{ file: data, uniqueId },
{ file: data, uniqueId, sessionOnly },
)}
disabled={disabled}
isVisualEffects={isVisualEffects}
@ -459,10 +531,11 @@ const VirtualBgSelector = ({
.map((background, index) => {
if (background.custom !== false) {
return renderCustomButton(background, index);
} else {
const isBlur = background.uniqueId.includes('Blur');
return isBlur ? renderBlurButton(index) : renderDefaultButton(background.uniqueId, index);
}
const isBlur = background.uniqueId.includes('Blur');
return isBlur
? renderBlurButton(index)
: renderDefaultButton(background.uniqueId, index);
})}
{renderInputButton()}

2
bigbluebutton-html5/imports/ui/components/video-preview/virtual-background/context.jsx
View File

@ -39,7 +39,7 @@ const reducer = (state, action) => {
};
}
case 'update': {
if (action.background.custom) update(action.background);
if (action.background.custom && !action.background.sessionOnly) update(action.background);
return {
...state,
backgrounds: {

2
bigbluebutton-html5/imports/ui/components/video-provider/video-button/component.jsx
View File

@ -11,6 +11,7 @@ import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features';
import Button from '/imports/ui/components/common/button/component';
import VideoPreviewContainer from '/imports/ui/components/video-preview/container';
import Settings from '/imports/ui/services/settings';
import PreviewService from '/imports/ui/components/video-preview/service';
const ENABLE_WEBCAM_SELECTOR_BUTTON = Meteor.settings.public.app.enableWebcamSelectorButton;
const ENABLE_CAMERA_BRIGHTNESS = Meteor.settings.public.app.enableCameraBrightness;
@ -108,6 +109,7 @@ const JoinVideoButton = ({
default:
if (exitVideo()) {
VideoService.exitVideo();
PreviewService.clearStreams();
} else {
setForceOpen(isMobileSharingCamera);
setVideoPreviewModalIsOpen(true);

36
bigbluebutton-html5/imports/ui/services/virtual-background/index.js
View File

@ -388,42 +388,8 @@ export async function createVirtualBackgroundService(parameters = null) {
} else {
parameters.virtualSource = virtualBackgroundImagePath + parameters.backgroundFilename;
if (parameters.customParams) {
if (parameters.customParams.file) {
if (parameters?.customParams?.file) {
parameters.virtualSource = parameters.customParams.file;
} else {
const imageUrl = parameters.customParams.url;
// Function to convert image URL to a File object
async function getFileFromUrl(url) {
try {
const response = await fetch(url, {
credentials: 'omit',
mode: 'cors',
headers: {
'Accept': 'image/*',
}
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const blob = await response.blob();
const file = new File([blob], 'fetchedWebcamBackground', { type: blob.type });
return file;
} catch (error) {
logger.error('Fetch error:', error);
return null;
}
}
let fetchedWebcamBackground = await getFileFromUrl(imageUrl);
if (fetchedWebcamBackground) {
parameters.virtualSource = URL.createObjectURL(fetchedWebcamBackground);
} else {
logger.error('Failed to fetch custom webcam background image. Using fallback image.');
}
}
}
}

16
bigbluebutton-html5/imports/ui/services/virtual-background/service.js
View File

@ -66,15 +66,17 @@ const getVirtualBackgroundThumbnail = (name) => {
// type: <EFFECT_TYPES>,
// name: effect filename, if any
// }
const setSessionVirtualBackgroundInfo = (type, name, deviceId) => {
return Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name });
}
const setSessionVirtualBackgroundInfo = (
type,
name,
customParams,
deviceId,
) => Session.set(`VirtualBackgroundInfo_${deviceId}`, { type, name, customParams });
const getSessionVirtualBackgroundInfo = (deviceId) => {
return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
const getSessionVirtualBackgroundInfo = (deviceId) => Session.get(`VirtualBackgroundInfo_${deviceId}`) || {
type: EFFECT_TYPES.NONE_TYPE,
};
}
name: '',
};
const getSessionVirtualBackgroundInfoWithDefault = (deviceId) => {
return Session.get(`VirtualBackgroundInfo_${deviceId}`) || {

2
bigbluebutton-html5/imports/ui/services/webrtc-base/bbb-video-stream.js
View File

@ -90,6 +90,7 @@ export default class BBBVideoStream extends EventEmitter2 {
});
this.virtualBgType = type;
this.virtualBgName = name;
this.customParams = customParams;
return Promise.resolve();
} catch (error) {
return Promise.reject(error);
@ -109,6 +110,7 @@ export default class BBBVideoStream extends EventEmitter2 {
this.virtualBgService = service;
this.virtualBgType = type;
this.virtualBgName = name;
this.customParams = customParams;
this.originalStream = this.mediaStream;
this.mediaStream = effect;
this.isVirtualBackgroundEnabled = true;