import PropTypes from 'prop-types'; import React, { Component } from 'react'; import { defineMessages, injectIntl, FormattedMessage, } from 'react-intl'; import Button from '/imports/ui/components/common/button/component'; import VirtualBgSelector from '/imports/ui/components/video-preview/virtual-background/component' import logger from '/imports/startup/client/logger'; import browserInfo from '/imports/utils/browserInfo'; import PreviewService from './service'; import VideoService from '../video-provider/video-provider-graphql/service'; import Styled from './styles'; import deviceInfo from '/imports/utils/deviceInfo'; import MediaStreamUtils from '/imports/utils/media-stream-utils'; import { notify } from '/imports/ui/services/notification'; import { EFFECT_TYPES, setSessionVirtualBackgroundInfo, getSessionVirtualBackgroundInfo, isVirtualBackgroundSupported, } from '/imports/ui/services/virtual-background/service'; import { getSettingsSingletonInstance } from '/imports/ui/services/settings'; import { isVirtualBackgroundsEnabled } from '/imports/ui/services/features'; import Checkbox from '/imports/ui/components/common/checkbox/component' import AppService from '/imports/ui/components/app/service'; const VIEW_STATES = { finding: 'finding', found: 'found', error: 'error', }; const propTypes = { intl: PropTypes.object.isRequired, closeModal: PropTypes.func.isRequired, startSharing: PropTypes.func.isRequired, stopSharing: PropTypes.func.isRequired, resolve: PropTypes.func, camCapReached: PropTypes.bool, hasVideoStream: PropTypes.bool.isRequired, webcamDeviceId: PropTypes.string, sharedDevices: PropTypes.arrayOf(PropTypes.string), }; const defaultProps = { resolve: null, camCapReached: true, webcamDeviceId: null, sharedDevices: [], }; const intlMessages = defineMessages({ webcamVirtualBackgroundTitle: { id: 'app.videoPreview.webcamVirtualBackgroundLabel', description: 'Title for the virtual background modal', }, webcamSettingsTitle: { id: 'app.videoPreview.webcamSettingsTitle', description: 'Title for the video preview modal', }, closeLabel: { id: 'app.videoPreview.closeLabel', description: 'Close button label', }, cancelLabel: { id: 'app.mobileAppModal.dismissLabel', description: 'Close button label', }, webcamPreviewLabel: { id: 'app.videoPreview.webcamPreviewLabel', description: 'Webcam preview label', }, cameraLabel: { id: 'app.videoPreview.cameraLabel', description: 'Camera dropdown label', }, qualityLabel: { id: 'app.videoPreview.profileLabel', description: 'Quality dropdown label', }, low: { id: 'app.videoPreview.quality.low', description: 'Low quality option label', }, medium: { id: 'app.videoPreview.quality.medium', description: 'Medium quality option label', }, high: { id: 'app.videoPreview.quality.high', description: 'High quality option label', }, hd: { id: 'app.videoPreview.quality.hd', description: 'High definition option label', }, startSharingLabel: { id: 'app.videoPreview.startSharingLabel', description: 'Start sharing button label', }, stopSharingLabel: { id: 'app.videoPreview.stopSharingLabel', description: 'Stop sharing button label', }, stopSharingAllLabel: { id: 'app.videoPreview.stopSharingAllLabel', description: 'Stop sharing all button label', }, sharedCameraLabel: { id: 'app.videoPreview.sharedCameraLabel', description: 'Already Shared camera label', }, findingWebcamsLabel: { id: 'app.videoPreview.findingWebcamsLabel', description: 'Finding webcams label', }, webcamOptionLabel: { id: 'app.videoPreview.webcamOptionLabel', description: 'Default webcam option label', }, webcamNotFoundLabel: { id: 'app.videoPreview.webcamNotFoundLabel', description: 'Webcam not found label', }, profileNotFoundLabel: { id: 'app.videoPreview.profileNotFoundLabel', description: 'Profile not found label', }, permissionError: { id: 'app.video.permissionError', description: 'Error message for webcam permission', }, AbortError: { id: 'app.video.abortError', description: 'Some problem occurred which prevented the device from being used', }, OverconstrainedError: { id: 'app.video.overconstrainedError', description: 'No candidate devices which met the criteria requested', }, SecurityError: { id: 'app.video.securityError', description: 'Media support is disabled on the Document', }, TypeError: { id: 'app.video.typeError', description: 'List of constraints specified is empty, or has all constraints set to false', }, NotFoundError: { id: 'app.video.notFoundError', description: 'error message when can not get webcam video', }, NotAllowedError: { id: 'app.video.notAllowed', description: 'error message when webcam had permission denied', }, NotSupportedError: { id: 'app.video.notSupportedError', description: 'error message when origin do not have ssl valid', }, NotReadableError: { id: 'app.video.notReadableError', description: 'error message When the webcam is being used by other software', }, TimeoutError: { id: 'app.video.timeoutError', description: 'error message when promise did not return', }, iOSError: { id: 'app.audioModal.iOSBrowser', description: 'Audio/Video Not supported warning', }, iOSErrorDescription: { id: 'app.audioModal.iOSErrorDescription', description: 'Audio/Video not supported description', }, iOSErrorRecommendation: { id: 'app.audioModal.iOSErrorRecommendation', description: 'Audio/Video recommended action', }, genericError: { id: 'app.video.genericError', description: 'error message for when the webcam sharing fails with unknown error', }, camCapReached: { id: 'app.video.camCapReached', description: 'message for when the camera cap has been reached', }, virtualBgGenericError: { id: 'app.video.virtualBackground.genericError', description: 'Failed to apply camera effect', }, inactiveError: { id: 'app.video.inactiveError', description: 'Camera stopped unexpectedly', }, brightness: { id: 'app.videoPreview.brightness', description: 'Brightness label', }, wholeImageBrightnessLabel: { id: 'app.videoPreview.wholeImageBrightnessLabel', description: 'Whole image brightness label', }, wholeImageBrightnessDesc: { id: 'app.videoPreview.wholeImageBrightnessDesc', description: 'Whole image brightness aria description', }, cameraAsContentSettingsTitle: { id: 'app.videoPreview.cameraAsContentSettingsTitle', description: 'Title for the video preview modal when sharing camera as content', }, sliderDesc: { id: 'app.videoPreview.sliderDesc', description: 'Brightness slider aria description', }, }); class VideoPreview extends Component { constructor(props) { super(props); const { webcamDeviceId, } = props; this.handleProceed = this.handleProceed.bind(this); this.handleStartSharing = this.handleStartSharing.bind(this); this.handleStopSharing = this.handleStopSharing.bind(this); this.handleStopSharingAll = this.handleStopSharingAll.bind(this); this.handleSelectWebcam = this.handleSelectWebcam.bind(this); this.handleSelectProfile = this.handleSelectProfile.bind(this); this.handleVirtualBgSelected = this.handleVirtualBgSelected.bind(this); this.handleLocalStreamInactive = this.handleLocalStreamInactive.bind(this); this.handleBrightnessAreaChange = this.handleBrightnessAreaChange.bind(this); this.handleSelectTab = this.handleSelectTab.bind(this); this._isMounted = false; this.state = { webcamDeviceId, selectedTab: 0, availableWebcams: null, selectedProfile: null, isStartSharingDisabled: true, viewState: VIEW_STATES.finding, deviceError: null, previewError: null, brightness: 100, wholeImageBrightness: false, }; } set currentVideoStream (bbbVideoStream) { // Stream is being unset - remove gUM revocation handler to avoid false negatives if (this._currentVideoStream) { this._currentVideoStream.removeListener('inactive', this.handleLocalStreamInactive); } // Set up inactivation handler for the new stream (to, eg, detect gUM revocation) if (bbbVideoStream) { bbbVideoStream.once('inactive', this.handleLocalStreamInactive); } this._currentVideoStream = bbbVideoStream; } get currentVideoStream () { return this._currentVideoStream; } componentDidMount() { const { webcamDeviceId, forceOpen, } = this.props; this._isMounted = true; if (deviceInfo.hasMediaDevices) { navigator.mediaDevices.enumerateDevices().then((devices) => { VideoService.updateNumberOfDevices(devices); // Video preview skip is activated, short circuit via a simpler procedure if (PreviewService.getSkipVideoPreview() && !forceOpen) return this.skipVideoPreview(); // Late enumerateDevices resolution, stop. if (!this._isMounted) return; let { webcams, areLabelled, areIdentified } = PreviewService.digestVideoDevices(devices, webcamDeviceId); logger.debug({ logCode: 'video_preview_enumerate_devices', extraInfo: { devices, webcams, }, }, `Enumerate devices came back. There are ${devices.length} devices and ${webcams.length} are video inputs`); if (webcams.length > 0) { this.getInitialCameraStream(webcams[0].deviceId) .then(async () => { // Late gUM resolve, stop. if (!this._isMounted) return; if (!areLabelled || !areIdentified) { // If they aren't labelled or have nullish deviceIds, run // enumeration again and get their full versions // Why: fingerprinting countermeasures obfuscate those when // no permission was granted via gUM try { const newDevices = await navigator.mediaDevices.enumerateDevices(); webcams = PreviewService.digestVideoDevices(newDevices, webcamDeviceId).webcams; } catch (error) { // Not a critical error because it should only affect UI; log it // and go ahead logger.error({ logCode: 'video_preview_enumerate_relabel_failure', extraInfo: { errorName: error.name, errorMessage: error.message, }, }, 'enumerateDevices for relabelling failed'); } } this.setState({ availableWebcams: webcams, viewState: VIEW_STATES.found, }); this.displayPreview(); }); } else { // There were no webcams coming from enumerateDevices. Throw an error. const noWebcamsError = new Error('NotFoundError'); this.handleDeviceError('enumerate', noWebcamsError, ': no webcams found'); } }).catch((error) => { // enumerateDevices failed this.handleDeviceError('enumerate', error, 'enumerating devices'); }); } else { // Top-level navigator.mediaDevices is not supported. // The session went through the version checking, but somehow ended here. // Nothing we can do. const error = new Error('NotSupportedError'); this.handleDeviceError('mount', error, ': navigator.mediaDevices unavailable'); } } componentDidUpdate() { if (this.brightnessMarker) { const markerStyle = window.getComputedStyle(this.brightnessMarker); const left = parseFloat(markerStyle.left); const right = parseFloat(markerStyle.right); if (left < 0) { this.brightnessMarker.style.left = '0px'; this.brightnessMarker.style.right = 'auto'; } else if (right < 0) { this.brightnessMarker.style.right = '0px'; this.brightnessMarker.style.left = 'auto'; } } } componentWillUnmount() { const { webcamDeviceId } = this.state; this.terminateCameraStream(this.currentVideoStream, webcamDeviceId); this.cleanupStreamAndVideo(); this._isMounted = false; } startCameraBrightness() { const ENABLE_CAMERA_BRIGHTNESS = window.meetingClientSettings.public.app.enableCameraBrightness; const CAMERA_BRIGHTNESS_AVAILABLE = ENABLE_CAMERA_BRIGHTNESS && isVirtualBackgroundSupported(); if (CAMERA_BRIGHTNESS_AVAILABLE) { const setBrightnessInfo = () => { const stream = this.currentVideoStream || {}; const service = stream.virtualBgService || {}; const { brightness = 100, wholeImageBrightness = false } = service; this.setState({ brightness, wholeImageBrightness }); }; if (!this.currentVideoStream.virtualBgService) { this.startVirtualBackground( this.currentVideoStream, EFFECT_TYPES.NONE_TYPE, ).then((switched) => { if (switched) { setBrightnessInfo(); } }); } else { setBrightnessInfo(); } } } handleSelectWebcam(event) { const webcamValue = event.target.value; this.getInitialCameraStream(webcamValue).then(() => { this.displayPreview(); }); } handleLocalStreamInactive({ id }) { // id === MediaStream.id if (this.currentVideoStream && typeof id === 'string' && this.currentVideoStream?.mediaStream?.id === id) { this.setState({ isStartSharingDisabled: true, }); this.handlePreviewError( 'stream_inactive', new Error('inactiveError'), '- preview camera stream inactive', ); } } updateVirtualBackgroundInfo = () => { const { webcamDeviceId } = this.state; // Update this session's virtual camera effect information if it's enabled setSessionVirtualBackgroundInfo( this.currentVideoStream.virtualBgType, this.currentVideoStream.virtualBgName, webcamDeviceId, ); }; // Resolves into true if the background switch is successful, false otherwise handleVirtualBgSelected(type, name, customParams) { const { sharedDevices } = this.props; const { webcamDeviceId } = this.state; const shared = this.isAlreadyShared(webcamDeviceId); const ENABLE_CAMERA_BRIGHTNESS = window.meetingClientSettings.public.app.enableCameraBrightness; const CAMERA_BRIGHTNESS_AVAILABLE = ENABLE_CAMERA_BRIGHTNESS && isVirtualBackgroundSupported(); if (type !== EFFECT_TYPES.NONE_TYPE || CAMERA_BRIGHTNESS_AVAILABLE) { return this.startVirtualBackground(this.currentVideoStream, type, name, customParams).then((switched) => { // If it's not shared we don't have to update here because // it will be updated in the handleStartSharing method. if (switched && shared) this.updateVirtualBackgroundInfo(); return switched; }); } else { this.stopVirtualBackground(this.currentVideoStream); if (shared) this.updateVirtualBackgroundInfo(); return Promise.resolve(true); } } stopVirtualBackground(bbbVideoStream) { if (bbbVideoStream) { bbbVideoStream.stopVirtualBackground(); this.displayPreview(); } } startVirtualBackground(bbbVideoStream, type, name, customParams) { this.setState({ isStartSharingDisabled: true }); if (bbbVideoStream == null) return Promise.resolve(false); return bbbVideoStream.startVirtualBackground(type, name, customParams).then(() => { this.displayPreview(); return true; }).catch(error => { this.handleVirtualBgError(error, type, name); return false; }).finally(() => { this.setState({ isStartSharingDisabled: false }); }); } handleSelectProfile(event) { const profileValue = event.target.value; const { webcamDeviceId } = this.state; const selectedProfile = PreviewService.getCameraProfile(profileValue); this.getCameraStream(webcamDeviceId, selectedProfile).then(() => { this.displayPreview(); }); } handleStartSharing() { const { resolve, startSharing, cameraAsContent, startSharingCameraAsContent, } = this.props; const { webcamDeviceId, selectedProfile, brightness, } = this.state; // Only streams that will be shared should be stored in the service. // If the store call returns false, we're duplicating stuff. So clean this one // up because it's an impostor. if(!PreviewService.storeStream(webcamDeviceId, this.currentVideoStream)) { this.currentVideoStream.stop(); } if ( this.currentVideoStream.virtualBgService && brightness === 100 && this.currentVideoStream.virtualBgType === EFFECT_TYPES.NONE_TYPE ) { this.stopVirtualBackground(this.currentVideoStream); } this.updateVirtualBackgroundInfo(); this.cleanupStreamAndVideo(); PreviewService.changeProfile(selectedProfile); PreviewService.changeWebcam(webcamDeviceId); if (cameraAsContent) { startSharingCameraAsContent(webcamDeviceId); } else { startSharing(webcamDeviceId); } if (resolve) resolve(); } handleStopSharing() { const { resolve, stopSharing, stopSharingCameraAsContent } = this.props; const { webcamDeviceId } = this.state; if (this.isCameraAsContentDevice(webcamDeviceId)) { stopSharingCameraAsContent(); } else { PreviewService.deleteStream(webcamDeviceId); stopSharing(webcamDeviceId); this.cleanupStreamAndVideo(); } if (resolve) resolve(); } handleStopSharingAll() { const { resolve, stopSharing } = this.props; stopSharing(); if (resolve) resolve(); } handleProceed() { const { resolve, closeModal, sharedDevices } = this.props; const { webcamDeviceId, brightness } = this.state; const shared = sharedDevices.includes(webcamDeviceId); if ( (shared) && this.currentVideoStream.virtualBgService && brightness === 100 && this.currentVideoStream.virtualBgType === EFFECT_TYPES.NONE_TYPE ) { this.stopVirtualBackground(this.currentVideoStream); } this.terminateCameraStream(this.currentVideoStream, webcamDeviceId); closeModal(); if (resolve) resolve(); } handlePreviewError(logCode, error, description) { logger.warn({ logCode: `video_preview_${logCode}_error`, extraInfo: { errorName: error.name, errorMessage: error.message, }, }, `Error ${description}`); this.setState({ previewError: this.handleGUMError(error), }); } handleDeviceError(logCode, error, description) { logger.warn({ logCode: `video_preview_${logCode}_error`, extraInfo: { errorName: error.name, errorMessage: error.message, }, }, `Error ${description}`); this.setState({ viewState: VIEW_STATES.error, deviceError: this.handleGUMError(error), }); } handleGUMError(error) { const { intl } = this.props; logger.error({ logCode: 'video_preview_gum_failure', extraInfo: { errorName: error.name, errorMessage: error.message, }, }, 'getUserMedia failed in video-preview'); const intlError = intlMessages[error.name] || intlMessages[error.message]; if (intlError) { return intl.formatMessage(intlError); } return intl.formatMessage(intlMessages.genericError, { 0: `${error.name}: ${error.message}` }); } terminateCameraStream(stream, deviceId) { if (stream) { // Stream is being destroyed - remove gUM revocation handler to avoid false negatives stream.removeListener('inactive', this.handleLocalStreamInactive); PreviewService.terminateCameraStream(stream, deviceId); } } cleanupStreamAndVideo() { this.currentVideoStream = null; if (this.video) this.video.srcObject = null; } handleVirtualBgError(error, type, name) { const { intl } = this.props; logger.error({ logCode: `video_preview_virtualbg_error`, extraInfo: { errorName: error.name, errorMessage: error.message, virtualBgType: type, virtualBgName: name, }, }, `Failed to toggle virtual background: ${error.message}`); notify(intl.formatMessage(intlMessages.virtualBgGenericError), 'error', 'video'); } updateDeviceId (deviceId) { let actualDeviceId = deviceId; if (!actualDeviceId && this.currentVideoStream) { actualDeviceId = MediaStreamUtils.extractDeviceIdFromStream( this.currentVideoStream.mediaStream, 'video', ); } this.setState({ webcamDeviceId: actualDeviceId, }); } getInitialCameraStream(deviceId) { const { cameraAsContent } = this.props; const defaultProfile = !cameraAsContent ? PreviewService.getDefaultProfile() : PreviewService.getCameraAsContentProfile(); return this.getCameraStream(deviceId, defaultProfile).then(() => { this.updateDeviceId(deviceId); }); } getCameraStream(deviceId, profile) { const { webcamDeviceId } = this.state; this.setState({ selectedProfile: profile.id, isStartSharingDisabled: true, previewError: undefined, }); this.terminateCameraStream(this.currentVideoStream, webcamDeviceId); this.cleanupStreamAndVideo(); // The return of doGUM is an instance of BBBVideoStream (a thin wrapper over a MediaStream) return PreviewService.doGUM(deviceId, profile).then((bbbVideoStream) => { // Late GUM resolve, clean up tracks, stop. if (!this._isMounted) return this.terminateCameraStream(bbbVideoStream, deviceId); this.currentVideoStream = bbbVideoStream; this.startCameraBrightness(); this.setState({ isStartSharingDisabled: false, }); }).catch((error) => { // When video preview is set to skip, we need some way to bubble errors // up to users; so re-throw the error if (!PreviewService.getSkipVideoPreview()) { this.handlePreviewError('do_gum_preview', error, 'displaying final selection'); } else { throw error; } }); } displayPreview() { if (this.currentVideoStream && this.video) { this.video.srcObject = this.currentVideoStream.mediaStream; } } skipVideoPreview() { this.getInitialCameraStream().then(() => { this.handleStartSharing(); }).catch(error => { this.cleanupStreamAndVideo(); notify(this.handleGUMError(error), 'error', 'video'); }); } supportWarning() { const { intl } = this.props; return (