From 770730ba6cbde04ba4f8667ae161ba9223555ec5 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 19 Nov 2024 13:17:24 -0500 Subject: [PATCH 1/3] Don't fall back as eagerly to unselected devices Somewhere around version 131 or 132, Firefox has started being more paranoid about media device fingerprinting, and will not even give you the IDs of available devices until you've requested a media stream. Instead you only get a single audio input and video input each with the empty string as their device ID, representing the system's default device. We can recognize this case and avoid resetting the device selection. --- src/livekit/MediaDevicesContext.tsx | 61 ++++++++++++----------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index 3d85b165..a26cf722 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -24,6 +24,7 @@ import { audioInput as audioInputSetting, audioOutput as audioOutputSetting, videoInput as videoInputSetting, + Setting, } from "../settings/settings"; import { isFirefox } from "../Platform"; @@ -58,7 +59,7 @@ function useObservableState( function useMediaDevice( kind: MediaDeviceKind, - fallbackDevice: string | undefined, + setting: Setting, usingNames: boolean, alwaysDefault: boolean = false, ): MediaDevice { @@ -84,15 +85,19 @@ function useMediaDevice( [kind, requestPermissions], ); const available = useObservableState(deviceObserver, []); - const [selectedId, select] = useState(fallbackDevice); + const [selectedId, select] = useSetting(setting); return useMemo(() => { - let devId; - if (available) { - devId = available.some((d) => d.deviceId === selectedId) - ? selectedId - : available.some((d) => d.deviceId === fallbackDevice) - ? fallbackDevice + let devId: string | undefined = undefined; + if (!alwaysDefault && available) { + // If the selected device is available, use it. Or if every available + // device ID is falsy, the browser is probably just being paranoid about + // fingerprinting and we should still try using the selected device. + // Otherwise, fall back to the first available device. + devId = + available.some((d) => d.deviceId === selectedId) || + available.every((d) => d.deviceId === "") + ? selectedId : available.at(0)?.deviceId; } @@ -102,10 +107,10 @@ function useMediaDevice( // device entries for the exact same device ID; deduplicate them [...new Map(available.map((d) => [d.deviceId, d])).values()] : [], - selectedId: alwaysDefault ? undefined : devId, + selectedId: devId, select, }; - }, [available, selectedId, fallbackDevice, select, alwaysDefault]); + }, [available, selectedId, select, alwaysDefault]); } const deviceStub: MediaDevice = { @@ -141,36 +146,22 @@ export const MediaDevicesProvider: FC = ({ children }) => { // for ouput devices because the selector wont be shown on FF. const useOutputNames = usingNames && !isFirefox(); - const [storedAudioInput, setStoredAudioInput] = useSetting(audioInputSetting); - const [storedAudioOutput, setStoredAudioOutput] = - useSetting(audioOutputSetting); - const [storedVideoInput, setStoredVideoInput] = useSetting(videoInputSetting); - - const audioInput = useMediaDevice("audioinput", storedAudioInput, usingNames); + const audioInput = useMediaDevice( + "audioinput", + audioInputSetting, + usingNames, + ); const audioOutput = useMediaDevice( "audiooutput", - storedAudioOutput, + audioOutputSetting, useOutputNames, alwaysUseDefaultAudio, ); - const videoInput = useMediaDevice("videoinput", storedVideoInput, usingNames); - - useEffect(() => { - if (audioInput.selectedId !== undefined) - setStoredAudioInput(audioInput.selectedId); - }, [setStoredAudioInput, audioInput.selectedId]); - - useEffect(() => { - // Skip setting state for ff output. Redundent since it is set to always return 'undefined' - // but makes it clear while debugging that this is not happening on FF. + perf ;) - if (audioOutput.selectedId !== undefined && !isFirefox()) - setStoredAudioOutput(audioOutput.selectedId); - }, [setStoredAudioOutput, audioOutput.selectedId]); - - useEffect(() => { - if (videoInput.selectedId !== undefined) - setStoredVideoInput(videoInput.selectedId); - }, [setStoredVideoInput, videoInput.selectedId]); + const videoInput = useMediaDevice( + "videoinput", + videoInputSetting, + usingNames, + ); const startUsingDeviceNames = useCallback( () => setNumCallersUsingNames((n) => n + 1), From a2b452c1d86dde9ca18bb878a9762521441984d0 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 21 Nov 2024 11:24:45 -0500 Subject: [PATCH 2/3] Don't override the camera without pressing "switch camera" --- src/room/useSwitchCamera.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/room/useSwitchCamera.ts b/src/room/useSwitchCamera.ts index e0434566..c1b4f092 100644 --- a/src/room/useSwitchCamera.ts +++ b/src/room/useSwitchCamera.ts @@ -20,7 +20,6 @@ import { TrackEvent, } from "livekit-client"; import { useObservable, useObservableEagerState } from "observable-hooks"; -import { useEffect } from "react"; import { logger } from "matrix-js-sdk/src/logger"; import { useMediaDevices } from "../livekit/MediaDevicesContext"; @@ -35,6 +34,7 @@ export function useSwitchCamera( video: Observable, ): (() => void) | null { const mediaDevices = useMediaDevices(); + const setVideoInput = useLatest(mediaDevices.videoInput.select); // Produce an observable like the input 'video' observable, except make it // emit whenever the track is muted or the device changes @@ -75,6 +75,12 @@ export function useSwitchCamera( .restartTrack({ facingMode: facingMode === "user" ? "environment" : "user", }) + .then(() => { + // Inform the MediaDeviceContext which camera was chosen + const deviceId = + track.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) setVideoInput.current(deviceId); + }) .catch((e) => logger.error("Failed to switch camera", facingMode, e), ); @@ -83,16 +89,5 @@ export function useSwitchCamera( [videoTrack], ); - const setVideoInput = useLatest(mediaDevices.videoInput.select); - useEffect(() => { - // Watch for device changes due to switching the camera and feed them back - // into the MediaDeviceContext - const subscription = videoTrack.subscribe((track) => { - const deviceId = track?.mediaStreamTrack.getSettings().deviceId; - if (deviceId !== undefined) setVideoInput.current(deviceId); - }); - return (): void => subscription.unsubscribe(); - }, [videoTrack, setVideoInput]); - return useObservableEagerState(switchCamera); } From 0718774cef344e2a39e910b4647b1a70deffc7f2 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 21 Nov 2024 11:32:15 -0500 Subject: [PATCH 3/3] Clarify how preferred devices work --- src/livekit/MediaDevicesContext.tsx | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index a26cf722..d405eec0 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -85,19 +85,21 @@ function useMediaDevice( [kind, requestPermissions], ); const available = useObservableState(deviceObserver, []); - const [selectedId, select] = useSetting(setting); + const [preferredId, select] = useSetting(setting); return useMemo(() => { - let devId: string | undefined = undefined; + let selectedId: string | undefined = undefined; if (!alwaysDefault && available) { - // If the selected device is available, use it. Or if every available + // If the preferred device is available, use it. Or if every available // device ID is falsy, the browser is probably just being paranoid about - // fingerprinting and we should still try using the selected device. - // Otherwise, fall back to the first available device. - devId = - available.some((d) => d.deviceId === selectedId) || + // fingerprinting and we should still try using the preferred device. + // Worst case it is not available and the browser will gracefully fall + // back to some other device for us when requesting the media stream. + // Otherwise, select the first available device. + selectedId = + available.some((d) => d.deviceId === preferredId) || available.every((d) => d.deviceId === "") - ? selectedId + ? preferredId : available.at(0)?.deviceId; } @@ -107,10 +109,10 @@ function useMediaDevice( // device entries for the exact same device ID; deduplicate them [...new Map(available.map((d) => [d.deviceId, d])).values()] : [], - selectedId: devId, + selectedId, select, }; - }, [available, selectedId, select, alwaysDefault]); + }, [available, preferredId, select, alwaysDefault]); } const deviceStub: MediaDevice = {