Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2024-11-15 00:04:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make Element Call work in Firefox's resist fingerprinting mode

Browse Source 
This one is gonna take some explaining:

When in resist fingerprinting mode, Firefox exhibits some funny behavior: when we ask for the the list of media devices, it gives us fake device IDs. But when the js-sdk requests a stream for any of those devices, Firefox associates the stream with the real device ID.

Now, in order to get the names of devices included in their metadata when you query the device list, you need to be holding a stream. For this reason, useMediaHandler was set up to reload the device list whenever matrix-js-sdk got a new local stream. But because of the inconsistency in device IDs, it would enter an infinite cycle telling matrix-js-sdk to request a stream for the fake device ID, but with matrix-js-sdk always responding with the real device ID.

I already wasn't happy with useMediaHandler's use of @ts-ignore comments to inspect private js-sdk fields, and in the meantime we've come up with a simpler function for requesting device names, so I decided to refactor useMediaHandler to use it instead. Importantly, it doesn't break in resist fingerprinting mode.

This created a new UX issue though: now, when on the lobby screen, useMediaHandler would request microphone access so it could get device names, followed immediately by a *second* pop-up for the lobby screen to request camera access. That's 1 pop-up too many, so I changed useMediaHandler to only request device names when a component is mounted that actually wants to show them. Currently, the settings modal is the only such component, and users normally only open it *after* granting full audio/video access, so this solution works out quite nicely.
This commit is contained in:
Robin Townsend 2023-05-15 22:03:26 -04:00
parent d3111758b7
commit f1ee3604de
4 changed files with 104 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/media-utils.ts
View File

@ -47,7 +47,7 @@ export async function findDeviceByName(
*
* @return The available media devices
*/
export async function getDevices(): Promise<MediaDeviceInfo[]> {
export async function getNamedDevices(): Promise<MediaDeviceInfo[]> {
// First get the devices without their labels, to learn what kinds of streams
// we can request
let devices: MediaDeviceInfo[];

4
src/room/GroupCallView.tsx
View File

@ -34,7 +34,7 @@ import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler";
import { useLocationNavigation } from "../useLocationNavigation";
import { PosthogAnalytics } from "../analytics/PosthogAnalytics";
import { useMediaHandler } from "../settings/useMediaHandler";
import { findDeviceByName, getDevices } from "../media-utils";
import { findDeviceByName, getNamedDevices } from "../media-utils";
declare global {
interface Window {
@ -102,7 +102,7 @@ export function GroupCallView({
// Get the available devices so we can match the selected device
// to its ID. This involves getting a media stream (see docs on
// the function) so we only do it once and re-use the result.
const devices = await getDevices();
const devices = await getNamedDevices();
const { audioInput, videoInput } = ev.detail
.data as unknown as JoinCallData;

2
src/settings/SettingsModal.tsx
View File

@ -57,7 +57,9 @@ export const SettingsModal = (props: Props) => {
audioOutput,
audioOutputs,
setAudioOutput,
useDeviceNames,
} = useMediaHandler();
useDeviceNames();
const [spatialAudio, setSpatialAudio] = useSpatialAudio();
const [showInspector, setShowInspector] = useShowInspector();

203
src/settings/useMediaHandler.tsx
View File

@ -32,7 +32,6 @@ limitations under the License.
*/
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MediaHandlerEvent } from "matrix-js-sdk/src/webrtc/mediaHandler";
import React, {
useState,
useEffect,
@ -41,18 +40,26 @@ import React, {
useContext,
createContext,
ReactNode,
useRef,
} from "react";
import { getNamedDevices } from "../media-utils";
export interface MediaHandlerContextInterface {
audioInput: string;
audioInput: string | undefined;
audioInputs: MediaDeviceInfo[];
setAudioInput: (deviceId: string) => void;
videoInput: string;
videoInput: string | undefined;
videoInputs: MediaDeviceInfo[];
setVideoInput: (deviceId: string) => void;
audioOutput: string;
audioOutput: string | undefined;
audioOutputs: MediaDeviceInfo[];
setAudioOutput: (deviceId: string) => void;
/**
* A hook which requests for devices to be named. This requires media
* permissions.
*/
useDeviceNames: () => void;
}
const MediaHandlerContext =
@ -70,10 +77,10 @@ function getMediaPreferences(): MediaPreferences {
try {
return JSON.parse(mediaPreferences);
} catch (e) {
return undefined;
return {};
}
} else {
return undefined;
return {};
}
}
@ -103,112 +110,98 @@ export function MediaHandlerProvider({ client, children }: Props): JSX.Element {
audioOutputs,
},
setState,
] = useState(() => {
const mediaPreferences = getMediaPreferences();
const mediaHandler = client.getMediaHandler();
] = useState(() => ({
audioInput: undefined as string | undefined,
videoInput: undefined as string | undefined,
audioOutput: undefined as string | undefined,
audioInputs: [] as MediaDeviceInfo[],
videoInputs: [] as MediaDeviceInfo[],
audioOutputs: [] as MediaDeviceInfo[],
}));
mediaHandler.restoreMediaSettings(
mediaPreferences?.audioInput,
mediaPreferences?.videoInput
);
// A ref counting the number of components currently mounted that want
// to know device names
const numComponentsWantingNames = useRef(0);
return {
// @ts-ignore, ignore that audioInput is a private members of mediaHandler
audioInput: mediaHandler.audioInput,
// @ts-ignore, ignore that videoInput is a private members of mediaHandler
videoInput: mediaHandler.videoInput,
audioOutput: undefined,
audioInputs: [],
videoInputs: [],
audioOutputs: [],
};
});
const updateDevices = useCallback(
async (initial: boolean) => {
// Only request device names if components actually want them, because it
// could trigger an extra permission pop-up
const devices = await (numComponentsWantingNames.current > 0
? getNamedDevices()
: navigator.mediaDevices.enumerateDevices());
const mediaPreferences = getMediaPreferences();
const audioInputs = devices.filter((d) => d.kind === "audioinput");
const videoInputs = devices.filter((d) => d.kind === "videoinput");
const audioOutputs = devices.filter((d) => d.kind === "audiooutput");
const audioInput = (
mediaPreferences.audioInput === undefined
? audioInputs.at(0)
: audioInputs.find(
(d) => d.deviceId === mediaPreferences.audioInput
) ?? audioInputs.at(0)
)?.deviceId;
const videoInput = (
mediaPreferences.videoInput === undefined
? videoInputs.at(0)
: videoInputs.find(
(d) => d.deviceId === mediaPreferences.videoInput
) ?? videoInputs.at(0)
)?.deviceId;
const audioOutput =
mediaPreferences.audioOutput === undefined
? undefined
: audioOutputs.find(
(d) => d.deviceId === mediaPreferences.audioOutput
)?.deviceId;
updateMediaPreferences({ audioInput, videoInput, audioOutput });
setState({
audioInput,
videoInput,
audioOutput,
audioInputs,
videoInputs,
audioOutputs,
});
if (
initial ||
audioInput !== mediaPreferences.audioInput ||
videoInput !== mediaPreferences.videoInput
) {
client.getMediaHandler().setMediaInputs(audioInput, videoInput);
}
},
[client, setState]
);
const useDeviceNames = useCallback(() => {
// This is a little weird from React's perspective as it looks like a
// dynamic hook, but it works
// eslint-disable-next-line react-hooks/rules-of-hooks
useEffect(() => {
numComponentsWantingNames.current++;
if (numComponentsWantingNames.current === 1) updateDevices(false);
return () => void numComponentsWantingNames.current--;
}, []);
}, [updateDevices]);
useEffect(() => {
const mediaHandler = client.getMediaHandler();
function updateDevices(): void {
navigator.mediaDevices.enumerateDevices().then((devices) => {
const mediaPreferences = getMediaPreferences();
const audioInputs = devices.filter(
(device) => device.kind === "audioinput"
);
const audioConnected = audioInputs.some(
// @ts-ignore
(device) => device.deviceId === mediaHandler.audioInput
);
// @ts-ignore
let audioInput = mediaHandler.audioInput;
if (!audioConnected && audioInputs.length > 0) {
audioInput = audioInputs[0].deviceId;
}
const videoInputs = devices.filter(
(device) => device.kind === "videoinput"
);
const videoConnected = videoInputs.some(
// @ts-ignore
(device) => device.deviceId === mediaHandler.videoInput
);
// @ts-ignore
let videoInput = mediaHandler.videoInput;
if (!videoConnected && videoInputs.length > 0) {
videoInput = videoInputs[0].deviceId;
}
const audioOutputs = devices.filter(
(device) => device.kind === "audiooutput"
);
let audioOutput = undefined;
if (
mediaPreferences &&
audioOutputs.some(
(device) => device.deviceId === mediaPreferences.audioOutput
)
) {
audioOutput = mediaPreferences.audioOutput;
}
if (
// @ts-ignore
(mediaHandler.videoInput && mediaHandler.videoInput !== videoInput) ||
// @ts-ignore
mediaHandler.audioInput !== audioInput
) {
mediaHandler.setMediaInputs(audioInput, videoInput);
}
updateMediaPreferences({ audioInput, videoInput, audioOutput });
setState({
audioInput,
videoInput,
audioOutput,
audioInputs,
videoInputs,
audioOutputs,
});
});
}
updateDevices();
mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, updateDevices);
navigator.mediaDevices.addEventListener("devicechange", updateDevices);
updateDevices(true);
const onDeviceChange = () => updateDevices(false);
navigator.mediaDevices.addEventListener("devicechange", onDeviceChange);
return () => {
mediaHandler.removeListener(
MediaHandlerEvent.LocalStreamsChanged,
updateDevices
navigator.mediaDevices.removeEventListener(
"devicechange",
onDeviceChange
);
navigator.mediaDevices.removeEventListener("devicechange", updateDevices);
mediaHandler.stopAllStreams();
client.getMediaHandler().stopAllStreams();
};
}, [client]);
}, [client, updateDevices]);
const setAudioInput: (deviceId: string) => void = useCallback(
(deviceId: string) => {
@ -245,6 +238,7 @@ export function MediaHandlerProvider({ client, children }: Props): JSX.Element {
audioOutput,
audioOutputs,
setAudioOutput,
useDeviceNames,
}),
[
audioInput,
@ -256,6 +250,7 @@ export function MediaHandlerProvider({ client, children }: Props): JSX.Element {
audioOutput,
audioOutputs,
setAudioOutput,
useDeviceNames,
]
);