From 8c0280954c420090dc236daea5c45ee8158bdaa2 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 28 Oct 2024 17:29:26 -0400 Subject: [PATCH] Add a button to switch the camera on mobile --- public/locales/en-GB/app.json | 1 + src/button/Button.tsx | 18 +++++++ src/room/InCallView.tsx | 17 ++++-- src/room/LobbyView.tsx | 73 +++++++++++++++++++++++++- src/room/VideoPreview.tsx | 59 ++------------------- src/room/useSwitchCamera.ts | 98 +++++++++++++++++++++++++++++++++++ src/state/CallViewModel.ts | 14 +++++ src/state/MediaViewModel.ts | 2 +- 8 files changed, 219 insertions(+), 63 deletions(-) create mode 100644 src/room/useSwitchCamera.ts diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 0cfa3085..fa4066ad 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -154,6 +154,7 @@ "stop_screenshare_button_label": "Sharing screen", "stop_video_button_label": "Stop video", "submitting": "Submitting…", + "switch_camera": "Switch camera", "unauthenticated_view_body": "Not registered yet? <2>Create an account", "unauthenticated_view_eula_caption": "By clicking \"Go\", you agree to our <2>End User Licensing Agreement (EULA)", "unauthenticated_view_login_button": "Login to your account", diff --git a/src/button/Button.tsx b/src/button/Button.tsx index 5d747a03..5c85ddbf 100644 --- a/src/button/Button.tsx +++ b/src/button/Button.tsx @@ -16,6 +16,7 @@ import { EndCallIcon, ShareScreenSolidIcon, SettingsSolidIcon, + SwitchCameraSolidIcon, } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./Button.module.css"; @@ -66,6 +67,23 @@ export const VideoButton: FC = ({ muted, ...props }) => { ); }; +export const SwitchCameraButton: FC> = ( + props, +) => { + const { t } = useTranslation(); + + return ( + + + + ); +}; + interface ShareScreenButtonProps extends ComponentPropsWithoutRef<"button"> { enabled: boolean; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d50be3c9..9ec2108a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -40,6 +40,7 @@ import { VideoButton, ShareScreenButton, SettingsButton, + SwitchCameraButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { useUrlParams } from "../UrlParams"; @@ -78,6 +79,7 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout"; import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout"; import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout"; +import { useSwitchCamera } from "./useSwitchCamera"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -217,6 +219,7 @@ export const InCallView: FC = ({ const gridMode = useObservableEagerState(vm.gridMode); const showHeader = useObservableEagerState(vm.showHeader); const showFooter = useObservableEagerState(vm.showFooter); + const switchCamera = useSwitchCamera(vm.localVideo); // Ideally we could detect taps by listening for click events and checking // that the pointerType of the event is "touch", but this isn't yet supported @@ -488,14 +491,14 @@ export const InCallView: FC = ({ buttons.push( , = ({ />, ); if (!reducedControls) { + if (switchCamera !== null) + buttons.push( + , + ); if (canScreenshare && !hideScreensharing) { buttons.push( , ); } - buttons.push(); + buttons.push(); } buttons.push( = ({ ); + const devices = useMediaDevices(); + + // Capture the audio options as they were when we first mounted, because + // we're not doing anything with the audio anyway so we don't need to + // re-open the devices when they change (see below). + const initialAudioOptions = useInitial( + () => + muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId }, + ); + + const localTrackOptions = useMemo( + () => ({ + // The only reason we request audio here is to get the audio permission + // request over with at the same time. But changing the audio settings + // shouldn't cause this hook to recreate the track, which is why we + // reference the initial values here. + // We also pass in a clone because livekit mutates the object passed in, + // which would cause the devices to be re-opened on the next render. + audio: Object.assign({}, initialAudioOptions), + video: muteStates.video.enabled && { + deviceId: devices.videoInput.selectedId, + }, + }), + [ + initialAudioOptions, + devices.videoInput.selectedId, + muteStates.video.enabled, + ], + ); + + const onError = useCallback( + (error: Error) => { + logger.error("Error while creating preview Tracks:", error); + muteStates.audio.setEnabled?.(false); + muteStates.video.setEnabled?.(false); + }, + [muteStates.audio, muteStates.video], + ); + + const tracks = usePreviewTracks(localTrackOptions, onError); + + const videoTrack = useMemo( + () => + (tracks?.find((t) => t.kind === Track.Kind.Video) ?? + null) as LocalVideoTrack | null, + [tracks], + ); + + const switchCamera = useSwitchCamera( + useObservable( + (inputs) => inputs.pipe(map(([video]) => video)), + [videoTrack], + ), + ); + // TODO: Unify this component with InCallView, so we can get slick joining // animations and don't have to feel bad about reusing its CSS return ( @@ -111,7 +175,11 @@ export const LobbyView: FC = ({ )}
- +
diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index f8d45971..80aa1069 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -5,18 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { useEffect, useMemo, useRef, FC, ReactNode, useCallback } from "react"; +import { useEffect, useRef, FC, ReactNode } from "react"; import useMeasure from "react-use-measure"; -import { usePreviewTracks } from "@livekit/components-react"; -import { LocalVideoTrack, Track } from "livekit-client"; +import { LocalVideoTrack } from "livekit-client"; import classNames from "classnames"; -import { logger } from "matrix-js-sdk/src/logger"; import { Avatar } from "../Avatar"; import styles from "./VideoPreview.module.css"; -import { useMediaDevices } from "../livekit/MediaDevicesContext"; import { MuteStates } from "./MuteStates"; -import { useInitial } from "../useInitial"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; export type MatrixInfo = { @@ -33,65 +29,18 @@ export type MatrixInfo = { interface Props { matrixInfo: MatrixInfo; muteStates: MuteStates; + videoTrack: LocalVideoTrack | null; children: ReactNode; } export const VideoPreview: FC = ({ matrixInfo, muteStates, + videoTrack, children, }) => { const [previewRef, previewBounds] = useMeasure(); - const devices = useMediaDevices(); - - // Capture the audio options as they were when we first mounted, because - // we're not doing anything with the audio anyway so we don't need to - // re-open the devices when they change (see below). - const initialAudioOptions = useInitial( - () => - muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId }, - ); - - const localTrackOptions = useMemo( - () => ({ - // The only reason we request audio here is to get the audio permission - // request over with at the same time. But changing the audio settings - // shouldn't cause this hook to recreate the track, which is why we - // reference the initial values here. - // We also pass in a clone because livekit mutates the object passed in, - // which would cause the devices to be re-opened on the next render. - audio: Object.assign({}, initialAudioOptions), - video: muteStates.video.enabled && { - deviceId: devices.videoInput.selectedId, - }, - }), - [ - initialAudioOptions, - devices.videoInput.selectedId, - muteStates.video.enabled, - ], - ); - - const onError = useCallback( - (error: Error) => { - logger.error("Error while creating preview Tracks:", error); - muteStates.audio.setEnabled?.(false); - muteStates.video.setEnabled?.(false); - }, - [muteStates.audio, muteStates.video], - ); - - const tracks = usePreviewTracks(localTrackOptions, onError); - - const videoTrack = useMemo( - () => - tracks?.find((t) => t.kind === Track.Kind.Video) as - | LocalVideoTrack - | undefined, - [tracks], - ); - const videoEl = useRef(null); useEffect(() => { diff --git a/src/room/useSwitchCamera.ts b/src/room/useSwitchCamera.ts new file mode 100644 index 00000000..e0434566 --- /dev/null +++ b/src/room/useSwitchCamera.ts @@ -0,0 +1,98 @@ +/* +Copyright 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { + fromEvent, + map, + merge, + Observable, + of, + startWith, + switchMap, +} from "rxjs"; +import { + facingModeFromLocalTrack, + LocalVideoTrack, + TrackEvent, +} from "livekit-client"; +import { useObservable, useObservableEagerState } from "observable-hooks"; +import { useEffect } from "react"; +import { logger } from "matrix-js-sdk/src/logger"; + +import { useMediaDevices } from "../livekit/MediaDevicesContext"; +import { platform } from "../Platform"; +import { useLatest } from "../useLatest"; + +/** + * Determines whether the user should be shown a button to switch their camera, + * producing a callback if so. + */ +export function useSwitchCamera( + video: Observable, +): (() => void) | null { + const mediaDevices = useMediaDevices(); + + // Produce an observable like the input 'video' observable, except make it + // emit whenever the track is muted or the device changes + const videoTrack: Observable = useObservable( + (inputs) => + inputs.pipe( + switchMap(([video]) => video), + switchMap((video) => { + if (video === null) return of(null); + return merge( + fromEvent(video, TrackEvent.Restarted).pipe( + startWith(null), + map(() => video), + ), + fromEvent(video, TrackEvent.Muted).pipe(map(() => null)), + ); + }), + ), + [video], + ); + + const switchCamera: Observable<(() => void) | null> = useObservable( + (inputs) => + platform === "desktop" + ? of(null) + : inputs.pipe( + switchMap(([track]) => track), + map((track) => { + if (track === null) return null; + const facingMode = facingModeFromLocalTrack(track).facingMode; + // If the camera isn't front or back-facing, don't provide a switch + // camera shortcut at all + if (facingMode !== "user" && facingMode !== "environment") + return null; + // Restart the track with a camera facing the opposite direction + return (): void => + void track + .restartTrack({ + facingMode: facingMode === "user" ? "environment" : "user", + }) + .catch((e) => + logger.error("Failed to switch camera", facingMode, e), + ); + }), + ), + [videoTrack], + ); + + const setVideoInput = useLatest(mediaDevices.videoInput.select); + useEffect(() => { + // Watch for device changes due to switching the camera and feed them back + // into the MediaDeviceContext + const subscription = videoTrack.subscribe((track) => { + const deviceId = track?.mediaStreamTrack.getSettings().deviceId; + if (deviceId !== undefined) setVideoInput.current(deviceId); + }); + return (): void => subscription.unsubscribe(); + }, [videoTrack, setVideoInput]); + + return useObservableEagerState(switchCamera); +} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 00e10dfe..6064e611 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -13,8 +13,10 @@ import { import { Room as LivekitRoom, LocalParticipant, + LocalVideoTrack, ParticipantEvent, RemoteParticipant, + Track, } from "livekit-client"; import { Room as MatrixRoom, @@ -58,6 +60,7 @@ import { import { LocalUserMediaViewModel, MediaViewModel, + observeTrackReference, RemoteUserMediaViewModel, ScreenShareViewModel, UserMediaViewModel, @@ -259,6 +262,17 @@ function findMatrixMember( // TODO: Move wayyyy more business logic from the call and lobby views into here export class CallViewModel extends ViewModel { + public readonly localVideo: Observable = + observeTrackReference( + this.livekitRoom.localParticipant, + Track.Source.Camera, + ).pipe( + map((trackRef) => { + const track = trackRef.publication?.track; + return track instanceof LocalVideoTrack ? track : null; + }), + ); + private readonly rawRemoteParticipants = connectedParticipantsObserver( this.livekitRoom, ).pipe(this.scope.state()); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 26894f9e..50d8613a 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -65,7 +65,7 @@ export function useDisplayName(vm: MediaViewModel): string { return displayName; } -function observeTrackReference( +export function observeTrackReference( participant: Participant, source: Track.Source, ): Observable {