Merge pull request #2694 from robintown/switch-camera

Add a button to switch the camera on mobile

public/locales/en-GB/app.json

@@ -154,6 +154,7 @@
   "stop_screenshare_button_label": "Sharing screen",
   "stop_video_button_label": "Stop video",
   "submitting": "Submitting…",
+  "switch_camera": "Switch camera",
   "unauthenticated_view_body": "Not registered yet? <2>Create an account</2>",
   "unauthenticated_view_eula_caption": "By clicking \"Go\", you agree to our <2>End User Licensing Agreement (EULA)</2>",
   "unauthenticated_view_login_button": "Login to your account",

src/button/Button.tsx

@@ -16,6 +16,7 @@ import {
   EndCallIcon,
   ShareScreenSolidIcon,
   SettingsSolidIcon,
+  SwitchCameraSolidIcon,
 } from "@vector-im/compound-design-tokens/assets/web/icons";
 
 import styles from "./Button.module.css";
@@ -66,6 +67,23 @@ export const VideoButton: FC<VideoButtonProps> = ({ muted, ...props }) => {
   );
 };
 
+export const SwitchCameraButton: FC<ComponentPropsWithoutRef<"button">> = (
+  props,
+) => {
+  const { t } = useTranslation();
+
+  return (
+    <Tooltip label={t("switch_camera")}>
+      <CpdButton
+        iconOnly
+        Icon={SwitchCameraSolidIcon}
+        kind="secondary"
+        {...props}
+      />
+    </Tooltip>
+  );
+};
+
 interface ShareScreenButtonProps extends ComponentPropsWithoutRef<"button"> {
   enabled: boolean;
 }

src/room/InCallView.tsx

@@ -40,6 +40,7 @@ import {
   VideoButton,
   ShareScreenButton,
   SettingsButton,
+  SwitchCameraButton,
 } from "../button";
 import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
 import { useUrlParams } from "../UrlParams";
@@ -78,6 +79,7 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout";
 import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout";
 import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout";
 import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout";
+import { useSwitchCamera } from "./useSwitchCamera";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -217,6 +219,7 @@ export const InCallView: FC<InCallViewProps> = ({
   const gridMode = useObservableEagerState(vm.gridMode);
   const showHeader = useObservableEagerState(vm.showHeader);
   const showFooter = useObservableEagerState(vm.showFooter);
+  const switchCamera = useSwitchCamera(vm.localVideo);
 
   // Ideally we could detect taps by listening for click events and checking
   // that the pointerType of the event is "touch", but this isn't yet supported
@@ -488,14 +491,14 @@ export const InCallView: FC<InCallViewProps> = ({
 
     buttons.push(
       <MicButton
-        key="1"
+        key="audio"
         muted={!muteStates.audio.enabled}
         onClick={toggleMicrophone}
         disabled={muteStates.audio.setEnabled === null}
         data-testid="incall_mute"
       />,
       <VideoButton
-        key="2"
+        key="video"
         muted={!muteStates.video.enabled}
         onClick={toggleCamera}
         disabled={muteStates.video.setEnabled === null}
@@ -503,22 +506,26 @@ export const InCallView: FC<InCallViewProps> = ({
       />,
     );
     if (!reducedControls) {
+      if (switchCamera !== null)
+        buttons.push(
+          <SwitchCameraButton key="switch_camera" onClick={switchCamera} />,
+        );
       if (canScreenshare && !hideScreensharing) {
         buttons.push(
           <ShareScreenButton
-            key="3"
+            key="share_screen"
             enabled={isScreenShareEnabled}
             onClick={toggleScreensharing}
             data-testid="incall_screenshare"
           />,
         );
       }
-      buttons.push(<SettingsButton key="4" onClick={openSettings} />);
+      buttons.push(<SettingsButton key="settings" onClick={openSettings} />);
     }
 
     buttons.push(
       <EndCallButton
-        key="6"
+        key="end_call"
         onClick={function (): void {
           onLeave();
         }}

src/room/LobbyView.tsx

@@ -5,12 +5,17 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { FC, useCallback, useState } from "react";
+import { FC, useCallback, useMemo, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { MatrixClient } from "matrix-js-sdk/src/matrix";
 import { Button } from "@vector-im/compound-web";
 import classNames from "classnames";
 import { useHistory } from "react-router-dom";
+import { logger } from "matrix-js-sdk/src/logger";
+import { usePreviewTracks } from "@livekit/components-react";
+import { LocalVideoTrack, Track } from "livekit-client";
+import { useObservable } from "observable-hooks";
+import { map } from "rxjs";
 
 import inCallStyles from "./InCallView.module.css";
 import styles from "./LobbyView.module.css";
@@ -23,12 +28,16 @@ import {
   EndCallButton,
   MicButton,
   SettingsButton,
+  SwitchCameraButton,
   VideoButton,
 } from "../button/Button";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
 import { useMediaQuery } from "../useMediaQuery";
 import { E2eeType } from "../e2ee/e2eeType";
 import { Link } from "../button/Link";
+import { useMediaDevices } from "../livekit/MediaDevicesContext";
+import { useInitial } from "../useInitial";
+import { useSwitchCamera } from "./useSwitchCamera";
 
 interface Props {
   client: MatrixClient;
@@ -89,6 +98,61 @@ export const LobbyView: FC<Props> = ({
     </Link>
   );
 
+  const devices = useMediaDevices();
+
+  // Capture the audio options as they were when we first mounted, because
+  // we're not doing anything with the audio anyway so we don't need to
+  // re-open the devices when they change (see below).
+  const initialAudioOptions = useInitial(
+    () =>
+      muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
+  );
+
+  const localTrackOptions = useMemo(
+    () => ({
+      // The only reason we request audio here is to get the audio permission
+      // request over with at the same time. But changing the audio settings
+      // shouldn't cause this hook to recreate the track, which is why we
+      // reference the initial values here.
+      // We also pass in a clone because livekit mutates the object passed in,
+      // which would cause the devices to be re-opened on the next render.
+      audio: Object.assign({}, initialAudioOptions),
+      video: muteStates.video.enabled && {
+        deviceId: devices.videoInput.selectedId,
+      },
+    }),
+    [
+      initialAudioOptions,
+      devices.videoInput.selectedId,
+      muteStates.video.enabled,
+    ],
+  );
+
+  const onError = useCallback(
+    (error: Error) => {
+      logger.error("Error while creating preview Tracks:", error);
+      muteStates.audio.setEnabled?.(false);
+      muteStates.video.setEnabled?.(false);
+    },
+    [muteStates.audio, muteStates.video],
+  );
+
+  const tracks = usePreviewTracks(localTrackOptions, onError);
+
+  const videoTrack = useMemo(
+    () =>
+      (tracks?.find((t) => t.kind === Track.Kind.Video) ??
+        null) as LocalVideoTrack | null,
+    [tracks],
+  );
+
+  const switchCamera = useSwitchCamera(
+    useObservable(
+      (inputs) => inputs.pipe(map(([video]) => video)),
+      [videoTrack],
+    ),
+  );
+
   // TODO: Unify this component with InCallView, so we can get slick joining
   // animations and don't have to feel bad about reusing its CSS
   return (
@@ -111,7 +175,11 @@ export const LobbyView: FC<Props> = ({
           </Header>
         )}
         <div className={styles.content}>
-          <VideoPreview matrixInfo={matrixInfo} muteStates={muteStates}>
+          <VideoPreview
+            matrixInfo={matrixInfo}
+            muteStates={muteStates}
+            videoTrack={videoTrack}
+          >
             <Button
               className={classNames(styles.join, {
                 [styles.wait]: waitingForInvite,
@@ -140,6 +208,7 @@ export const LobbyView: FC<Props> = ({
               onClick={onVideoPress}
               disabled={muteStates.video.setEnabled === null}
             />
+            {switchCamera && <SwitchCameraButton onClick={switchCamera} />}
             <SettingsButton onClick={openSettings} />
             {!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
           </div>

src/room/VideoPreview.tsx

@@ -5,18 +5,14 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { useEffect, useMemo, useRef, FC, ReactNode, useCallback } from "react";
+import { useEffect, useRef, FC, ReactNode } from "react";
 import useMeasure from "react-use-measure";
-import { usePreviewTracks } from "@livekit/components-react";
+import { LocalVideoTrack } from "livekit-client";
-import { LocalVideoTrack, Track } from "livekit-client";
 import classNames from "classnames";
-import { logger } from "matrix-js-sdk/src/logger";
 
 import { Avatar } from "../Avatar";
 import styles from "./VideoPreview.module.css";
-import { useMediaDevices } from "../livekit/MediaDevicesContext";
 import { MuteStates } from "./MuteStates";
-import { useInitial } from "../useInitial";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
 
 export type MatrixInfo = {
@@ -33,65 +29,18 @@ export type MatrixInfo = {
 interface Props {
   matrixInfo: MatrixInfo;
   muteStates: MuteStates;
+  videoTrack: LocalVideoTrack | null;
   children: ReactNode;
 }
 
 export const VideoPreview: FC<Props> = ({
   matrixInfo,
   muteStates,
+  videoTrack,
   children,
 }) => {
   const [previewRef, previewBounds] = useMeasure();
 
-  const devices = useMediaDevices();
-
-  // Capture the audio options as they were when we first mounted, because
-  // we're not doing anything with the audio anyway so we don't need to
-  // re-open the devices when they change (see below).
-  const initialAudioOptions = useInitial(
-    () =>
-      muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
-  );
-
-  const localTrackOptions = useMemo(
-    () => ({
-      // The only reason we request audio here is to get the audio permission
-      // request over with at the same time. But changing the audio settings
-      // shouldn't cause this hook to recreate the track, which is why we
-      // reference the initial values here.
-      // We also pass in a clone because livekit mutates the object passed in,
-      // which would cause the devices to be re-opened on the next render.
-      audio: Object.assign({}, initialAudioOptions),
-      video: muteStates.video.enabled && {
-        deviceId: devices.videoInput.selectedId,
-      },
-    }),
-    [
-      initialAudioOptions,
-      devices.videoInput.selectedId,
-      muteStates.video.enabled,
-    ],
-  );
-
-  const onError = useCallback(
-    (error: Error) => {
-      logger.error("Error while creating preview Tracks:", error);
-      muteStates.audio.setEnabled?.(false);
-      muteStates.video.setEnabled?.(false);
-    },
-    [muteStates.audio, muteStates.video],
-  );
-
-  const tracks = usePreviewTracks(localTrackOptions, onError);
-
-  const videoTrack = useMemo(
-    () =>
-      tracks?.find((t) => t.kind === Track.Kind.Video) as
-        | LocalVideoTrack
-        | undefined,
-    [tracks],
-  );
-
   const videoEl = useRef<HTMLVideoElement | null>(null);
 
   useEffect(() => {

src/room/useSwitchCamera.ts

@@ -0,0 +1,98 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  fromEvent,
+  map,
+  merge,
+  Observable,
+  of,
+  startWith,
+  switchMap,
+} from "rxjs";
+import {
+  facingModeFromLocalTrack,
+  LocalVideoTrack,
+  TrackEvent,
+} from "livekit-client";
+import { useObservable, useObservableEagerState } from "observable-hooks";
+import { useEffect } from "react";
+import { logger } from "matrix-js-sdk/src/logger";
+
+import { useMediaDevices } from "../livekit/MediaDevicesContext";
+import { platform } from "../Platform";
+import { useLatest } from "../useLatest";
+
+/**
+ * Determines whether the user should be shown a button to switch their camera,
+ * producing a callback if so.
+ */
+export function useSwitchCamera(
+  video: Observable<LocalVideoTrack | null>,
+): (() => void) | null {
+  const mediaDevices = useMediaDevices();
+
+  // Produce an observable like the input 'video' observable, except make it
+  // emit whenever the track is muted or the device changes
+  const videoTrack: Observable<LocalVideoTrack | null> = useObservable(
+    (inputs) =>
+      inputs.pipe(
+        switchMap(([video]) => video),
+        switchMap((video) => {
+          if (video === null) return of(null);
+          return merge(
+            fromEvent(video, TrackEvent.Restarted).pipe(
+              startWith(null),
+              map(() => video),
+            ),
+            fromEvent(video, TrackEvent.Muted).pipe(map(() => null)),
+          );
+        }),
+      ),
+    [video],
+  );
+
+  const switchCamera: Observable<(() => void) | null> = useObservable(
+    (inputs) =>
+      platform === "desktop"
+        ? of(null)
+        : inputs.pipe(
+            switchMap(([track]) => track),
+            map((track) => {
+              if (track === null) return null;
+              const facingMode = facingModeFromLocalTrack(track).facingMode;
+              // If the camera isn't front or back-facing, don't provide a switch
+              // camera shortcut at all
+              if (facingMode !== "user" && facingMode !== "environment")
+                return null;
+              // Restart the track with a camera facing the opposite direction
+              return (): void =>
+                void track
+                  .restartTrack({
+                    facingMode: facingMode === "user" ? "environment" : "user",
+                  })
+                  .catch((e) =>
+                    logger.error("Failed to switch camera", facingMode, e),
+                  );
+            }),
+          ),
+    [videoTrack],
+  );
+
+  const setVideoInput = useLatest(mediaDevices.videoInput.select);
+  useEffect(() => {
+    // Watch for device changes due to switching the camera and feed them back
+    // into the MediaDeviceContext
+    const subscription = videoTrack.subscribe((track) => {
+      const deviceId = track?.mediaStreamTrack.getSettings().deviceId;
+      if (deviceId !== undefined) setVideoInput.current(deviceId);
+    });
+    return (): void => subscription.unsubscribe();
+  }, [videoTrack, setVideoInput]);
+
+  return useObservableEagerState(switchCamera);
+}

src/state/CallViewModel.ts

@@ -13,8 +13,10 @@ import {
 import {
   Room as LivekitRoom,
   LocalParticipant,
+  LocalVideoTrack,
   ParticipantEvent,
   RemoteParticipant,
+  Track,
 } from "livekit-client";
 import {
   Room as MatrixRoom,
@@ -58,6 +60,7 @@ import {
 import {
   LocalUserMediaViewModel,
   MediaViewModel,
+  observeTrackReference,
   RemoteUserMediaViewModel,
   ScreenShareViewModel,
   UserMediaViewModel,
@@ -259,6 +262,17 @@ function findMatrixMember(
 
 // TODO: Move wayyyy more business logic from the call and lobby views into here
 export class CallViewModel extends ViewModel {
+  public readonly localVideo: Observable<LocalVideoTrack | null> =
+    observeTrackReference(
+      this.livekitRoom.localParticipant,
+      Track.Source.Camera,
+    ).pipe(
+      map((trackRef) => {
+        const track = trackRef.publication?.track;
+        return track instanceof LocalVideoTrack ? track : null;
+      }),
+    );
+
   private readonly rawRemoteParticipants = connectedParticipantsObserver(
     this.livekitRoom,
   ).pipe(this.scope.state());

src/state/MediaViewModel.ts

@@ -65,7 +65,7 @@ export function useDisplayName(vm: MediaViewModel): string {
   return displayName;
 }
 
-function observeTrackReference(
+export function observeTrackReference(
   participant: Participant,
   source: Track.Source,
 ): Observable<TrackReferenceOrPlaceholder> {