Add simple global controls to put the call in picture-in-picture mode (#2573)

* Stop sharing state observables when the view model is destroyed

By default, observables running with shareReplay will continue running forever even if there are no subscribers. We need to stop them when the view model is destroyed to avoid memory leaks and other unintuitive behavior.

* Hydrate the call view model in a less hacky way

This ensures that only a single view model is created per call, unlike the previous solution which would create extra view models in strict mode which it was unable to dispose of. The other way was invalid because React gives us no way to reliably dispose of a resource created in the render phase. This is essentially a memory leak fix.

* Add simple global controls to put the call in picture-in-picture mode

Our web and mobile apps (will) all support putting calls into a picture-in-picture mode. However, it'd be nice to have a way of doing this that's more explicit than a breakpoint, because PiP views could in theory get fairly large. Specifically, on mobile, we want a way to do this that can tell you whether the call is ongoing, and that works even without the widget API (because we support SPA calls in the Element X apps…)

To this end, I've created a simple global "controls" API on the window. Right now it only has methods for controlling the picture-in-picture state, but in theory we can expand it to also control mute states, which is current possible via the widget API only.

* Fix footer appearing in large PiP views

* Add a method for whether you can enter picture-in-picture mode

* Have the controls emit booleans directly

docs/README.md

@@ -2,5 +2,6 @@
 
 This folder contains documentation for Element Call setup and usage.
 
-- [Url format and parameters](./url-params.md)
 - [Embedded vs standalone mode](./embedded-standalone.md)
+- [Url format and parameters](./url-params.md)
+- [Global JS controls](./controls.md)

docs/controls.md

@@ -0,0 +1,7 @@
+# Global JS controls
+
+A few aspects of Element Call's interface can be controlled through a global API on the `window`:
+
+- `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode.
+- `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call.
+- `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call.

src/@types/global.d.ts

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import "matrix-js-sdk/src/@types/global";
+import { Controls } from "../controls";
 
 declare global {
   interface Document {
@@ -23,6 +24,10 @@ declare global {
     webkitFullscreenElement: HTMLElement | null;
   }
 
+  interface Window {
+    controls: Controls;
+  }
+
   interface HTMLElement {
     // Safari only supports this prefixed, so tell the type system about it
     webkitRequestFullscreen: () => void;

src/controls.ts

@@ -0,0 +1,39 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Subject } from "rxjs";
+
+export interface Controls {
+  canEnterPip: () => boolean;
+  enablePip: () => void;
+  disablePip: () => void;
+}
+
+export const setPipEnabled = new Subject<boolean>();
+
+window.controls = {
+  canEnterPip(): boolean {
+    return setPipEnabled.observed;
+  },
+  enablePip(): void {
+    if (!setPipEnabled.observed) throw new Error("No call is running");
+    setPipEnabled.next(true);
+  },
+  disablePip(): void {
+    if (!setPipEnabled.observed) throw new Error("No call is running");
+    setPipEnabled.next(false);
+  },
+};

src/room/InCallView.module.css

@@ -58,6 +58,10 @@ limitations under the License.
   );
 }
 
+.footer.hidden {
+  display: none;
+}
+
 .footer.overlay {
   position: absolute;
   inset-block-end: 0;
@@ -67,6 +71,7 @@ limitations under the License.
 }
 
 .footer.overlay.hidden {
+  display: grid;
   opacity: 0;
   pointer-events: none;
 }

src/room/InCallView.tsx

@@ -69,7 +69,7 @@ import { InviteButton } from "../button/InviteButton";
 import { LayoutToggle } from "./LayoutToggle";
 import { ECConnectionState } from "../livekit/useECConnectionState";
 import { useOpenIDSFU } from "../livekit/openIDSFU";
-import { GridMode, Layout, useCallViewModel } from "../state/CallViewModel";
+import { CallViewModel, GridMode, Layout } from "../state/CallViewModel";
 import { Grid, TileProps } from "../grid/Grid";
 import { useObservable } from "../state/useObservable";
 import { useInitial } from "../useInitial";
@@ -93,7 +93,7 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 const maxTapDurationMs = 400;
 
 export interface ActiveCallProps
-  extends Omit<InCallViewProps, "livekitRoom" | "connState"> {
+  extends Omit<InCallViewProps, "vm" | "livekitRoom" | "connState"> {
   e2eeSystem: EncryptionSystem;
 }
 
@@ -105,6 +105,8 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     sfuConfig,
     props.e2eeSystem,
   );
+  const connStateObservable = useObservable(connState);
+  const [vm, setVm] = useState<CallViewModel | null>(null);
 
   useEffect(() => {
     return (): void => {
@@ -113,17 +115,41 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
-  if (!livekitRoom) return null;
+  useEffect(() => {
+    if (livekitRoom !== undefined) {
+      const vm = new CallViewModel(
+        props.rtcSession.room,
+        livekitRoom,
+        props.e2eeSystem.kind !== E2eeType.NONE,
+        connStateObservable,
+      );
+      setVm(vm);
+      return (): void => vm.destroy();
+    }
+  }, [
+    props.rtcSession.room,
+    livekitRoom,
+    props.e2eeSystem.kind,
+    connStateObservable,
+  ]);
+
+  if (livekitRoom === undefined || vm === null) return null;
 
   return (
     <RoomContext.Provider value={livekitRoom}>
-      <InCallView {...props} livekitRoom={livekitRoom} connState={connState} />
+      <InCallView
+        {...props}
+        vm={vm}
+        livekitRoom={livekitRoom}
+        connState={connState}
+      />
     </RoomContext.Provider>
   );
 };
 
 export interface InCallViewProps {
   client: MatrixClient;
+  vm: CallViewModel;
   matrixInfo: MatrixInfo;
   rtcSession: MatrixRTCSession;
   livekitRoom: Room;
@@ -138,6 +164,7 @@ export interface InCallViewProps {
 
 export const InCallView: FC<InCallViewProps> = ({
   client,
+  vm,
   matrixInfo,
   rtcSession,
   livekitRoom,
@@ -193,12 +220,6 @@ export const InCallView: FC<InCallViewProps> = ({
   const reducedControls = boundsValid && bounds.width <= 340;
   const noControls = reducedControls && bounds.height <= 400;
 
-  const vm = useCallViewModel(
-    rtcSession.room,
-    livekitRoom,
-    matrixInfo.e2eeSystem.kind !== E2eeType.NONE,
-    connState,
-  );
   const windowMode = useObservableEagerState(vm.windowMode);
   const layout = useObservableEagerState(vm.layout);
   const gridMode = useObservableEagerState(vm.gridMode);

src/state/CallViewModel.ts

@@ -26,7 +26,6 @@ import {
   RemoteParticipant,
 } from "livekit-client";
 import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
-import { useEffect, useRef } from "react";
 import {
   EMPTY,
   Observable,
@@ -44,7 +43,6 @@ import {
   race,
   sample,
   scan,
-  shareReplay,
   skip,
   startWith,
   switchAll,
@@ -58,12 +56,10 @@ import {
 import { logger } from "matrix-js-sdk/src/logger";
 
 import { ViewModel } from "./ViewModel";
-import { useObservable } from "./useObservable";
 import {
   ECAddonConnectionState,
   ECConnectionState,
 } from "../livekit/useECConnectionState";
-import { usePrevious } from "../usePrevious";
 import {
   LocalUserMediaViewModel,
   MediaViewModel,
@@ -75,6 +71,7 @@ import { accumulate, finalizeValue } from "../observable-utils";
 import { ObservableScope } from "./ObservableScope";
 import { duplicateTiles } from "../settings/settings";
 import { isFirefox } from "../Platform";
+import { setPipEnabled } from "../controls";
 
 // How long we wait after a focus switch before showing the real participant
 // list again
@@ -194,11 +191,9 @@ class UserMedia {
         ),
       ),
       startWith(false),
-      distinctUntilChanged(),
-      this.scope.bind(),
       // Make this Observable hot so that the timers don't reset when you
       // resubscribe
-      shareReplay(1),
+      this.scope.state(),
     );
 
     this.presenter = observeParticipantEvents(
@@ -261,7 +256,7 @@ function findMatrixMember(
 export class CallViewModel extends ViewModel {
   private readonly rawRemoteParticipants = connectedParticipantsObserver(
     this.livekitRoom,
-  ).pipe(shareReplay(1));
+  ).pipe(this.scope.state());
 
   // Lists of participants to "hold" on display, even if LiveKit claims that
   // they've left
@@ -383,7 +378,7 @@ export class CallViewModel extends ViewModel {
     finalizeValue((ts) => {
       for (const t of ts) t.destroy();
     }),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   private readonly userMedia: Observable<UserMedia[]> = this.mediaItems.pipe(
@@ -402,7 +397,7 @@ export class CallViewModel extends ViewModel {
       map((mediaItems) =>
         mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare),
       ),
-      shareReplay(1),
+      this.scope.state(),
     );
 
   private readonly hasRemoteScreenShares: Observable<boolean> =
@@ -443,9 +438,8 @@ export class CallViewModel extends ViewModel {
         },
         null,
       ),
-      distinctUntilChanged(),
       map((speaker) => speaker.vm),
-      shareReplay(1),
+      this.scope.state(),
       throttleTime(1600, undefined, { leading: true, trailing: true }),
     );
 
@@ -513,16 +507,17 @@ export class CallViewModel extends ViewModel {
   private readonly spotlight: Observable<MediaViewModel[]> =
     this.spotlightAndPip.pipe(
       switchMap(([spotlight]) => spotlight),
-      shareReplay(1),
+      this.scope.state(),
     );
 
   private readonly pip: Observable<UserMediaViewModel | null> =
     this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
 
-  /**
-   * The general shape of the window.
-   */
-  public readonly windowMode: Observable<WindowMode> = fromEvent(
+  private readonly pipEnabled: Observable<boolean> = setPipEnabled.pipe(
+    startWith(false),
+  );
+
+  private readonly naturalWindowMode: Observable<WindowMode> = fromEvent(
     window,
     "resize",
   ).pipe(
@@ -538,15 +533,21 @@ export class CallViewModel extends ViewModel {
       if (width <= 600) return "narrow";
       return "normal";
     }),
-    distinctUntilChanged(),
-    shareReplay(1),
+    this.scope.state(),
+  );
+
+  /**
+   * The general shape of the window.
+   */
+  public readonly windowMode: Observable<WindowMode> = this.pipEnabled.pipe(
+    switchMap((pip) => (pip ? of<WindowMode>("pip") : this.naturalWindowMode)),
   );
 
   private readonly spotlightExpandedToggle = new Subject<void>();
   public readonly spotlightExpanded: Observable<boolean> =
     this.spotlightExpandedToggle.pipe(
       accumulate(false, (expanded) => !expanded),
-      shareReplay(1),
+      this.scope.state(),
     );
 
   private readonly gridModeUserSelection = new Subject<GridMode>();
@@ -572,8 +573,7 @@ export class CallViewModel extends ViewModel {
             )
         ).pipe(startWith(userSelection ?? "grid")),
       ),
-      distinctUntilChanged(),
-      shareReplay(1),
+      this.scope.state(),
     );
 
   public setGridMode(value: GridMode): void {
@@ -629,7 +629,7 @@ export class CallViewModel extends ViewModel {
   );
 
   private readonly pipLayout: Observable<Layout> = this.spotlight.pipe(
-    map((spotlight): Layout => ({ type: "pip", spotlight })),
+    map((spotlight) => ({ type: "pip", spotlight })),
   );
 
   public readonly layout: Observable<Layout> = this.windowMode.pipe(
@@ -690,13 +690,12 @@ export class CallViewModel extends ViewModel {
           return this.pipLayout;
       }
     }),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   public showSpotlightIndicators: Observable<boolean> = this.layout.pipe(
     map((l) => l.type !== "grid"),
-    distinctUntilChanged(),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   /**
@@ -720,8 +719,7 @@ export class CallViewModel extends ViewModel {
 
   public showSpeakingIndicators: Observable<boolean> = this.layout.pipe(
     map((l) => l.type !== "one-on-one" && !l.type.startsWith("spotlight-")),
-    distinctUntilChanged(),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   public readonly toggleSpotlightExpanded: Observable<(() => void) | null> =
@@ -741,7 +739,7 @@ export class CallViewModel extends ViewModel {
       map((enabled) =>
         enabled ? (): void => this.spotlightExpandedToggle.next() : null,
       ),
-      shareReplay(1),
+      this.scope.state(),
     );
 
   private readonly screenTap = new Subject<void>();
@@ -771,8 +769,7 @@ export class CallViewModel extends ViewModel {
 
   public readonly showHeader: Observable<boolean> = this.windowMode.pipe(
     map((mode) => mode !== "pip" && mode !== "flat"),
-    distinctUntilChanged(),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   public readonly showFooter = this.windowMode.pipe(
@@ -815,8 +812,7 @@ export class CallViewModel extends ViewModel {
           );
       }
     }),
-    distinctUntilChanged(),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   public constructor(
@@ -829,34 +825,3 @@ export class CallViewModel extends ViewModel {
     super();
   }
 }
-
-export function useCallViewModel(
-  matrixRoom: MatrixRoom,
-  livekitRoom: LivekitRoom,
-  encrypted: boolean,
-  connectionState: ECConnectionState,
-): CallViewModel {
-  const prevMatrixRoom = usePrevious(matrixRoom);
-  const prevLivekitRoom = usePrevious(livekitRoom);
-  const prevEncrypted = usePrevious(encrypted);
-  const connectionStateObservable = useObservable(connectionState);
-
-  const vm = useRef<CallViewModel>();
-  if (
-    matrixRoom !== prevMatrixRoom ||
-    livekitRoom !== prevLivekitRoom ||
-    encrypted !== prevEncrypted
-  ) {
-    vm.current?.destroy();
-    vm.current = new CallViewModel(
-      matrixRoom,
-      livekitRoom,
-      encrypted,
-      connectionStateObservable,
-    );
-  }
-
-  useEffect(() => vm.current?.destroy(), []);
-
-  return vm.current!;
-}

src/state/MediaViewModel.ts

@@ -36,12 +36,10 @@ import {
   BehaviorSubject,
   Observable,
   combineLatest,
-  distinctUntilChanged,
   distinctUntilKeyChanged,
   fromEvent,
   map,
   of,
-  shareReplay,
   startWith,
   switchMap,
 } from "rxjs";
@@ -84,7 +82,6 @@ function observeTrackReference(
       source,
     })),
     distinctUntilKeyChanged("publication"),
-    shareReplay(1),
   );
 }
 
@@ -119,15 +116,19 @@ abstract class BaseMediaViewModel extends ViewModel {
     videoSource: VideoSource,
   ) {
     super();
-    const audio = observeTrackReference(participant, audioSource);
-    this.video = observeTrackReference(participant, videoSource);
+    const audio = observeTrackReference(participant, audioSource).pipe(
+      this.scope.state(),
+    );
+    this.video = observeTrackReference(participant, videoSource).pipe(
+      this.scope.state(),
+    );
     this.unencryptedWarning = combineLatest(
       [audio, this.video],
       (a, v) =>
         callEncrypted &&
         (a.publication?.isEncrypted === false ||
           v.publication?.isEncrypted === false),
-    ).pipe(distinctUntilChanged(), shareReplay(1));
+    ).pipe(this.scope.state());
   }
 }
 
@@ -151,7 +152,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
     ParticipantEvent.IsSpeakingChanged,
   ).pipe(
     map((p) => p.isSpeaking),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   /**
@@ -184,7 +185,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
       Track.Source.Camera,
     );
 
-    const media = observeParticipantMedia(participant).pipe(shareReplay(1));
+    const media = observeParticipantMedia(participant).pipe(this.scope.state());
     this.audioEnabled = media.pipe(
       map((m) => m.microphoneTrack?.isMuted === false),
     );
@@ -216,7 +217,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
         map(() => facingModeFromLocalTrack(track).facingMode === "user"),
       );
     }),
-    shareReplay(1),
+    this.scope.state(),
   );
 
   /**

src/state/ObservableScope.ts

@@ -14,7 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { MonoTypeOperatorFunction, Subject, takeUntil } from "rxjs";
+import {
+  distinctUntilChanged,
+  Observable,
+  shareReplay,
+  Subject,
+  takeUntil,
+} from "rxjs";
+
+type MonoTypeOperator = <T>(o: Observable<T>) => Observable<T>;
 
 /**
  * A scope which limits the execution lifetime of its bound Observables.
@@ -22,12 +30,26 @@ import { MonoTypeOperatorFunction, Subject, takeUntil } from "rxjs";
 export class ObservableScope {
   private readonly ended = new Subject<void>();
 
+  private readonly bindImpl: MonoTypeOperator = takeUntil(this.ended);
+
   /**
    * Binds an Observable to this scope, so that it completes when the scope
    * ends.
    */
-  public bind<T>(): MonoTypeOperatorFunction<T> {
-    return takeUntil(this.ended);
+  public bind(): MonoTypeOperator {
+    return this.bindImpl;
+  }
+
+  private readonly stateImpl: MonoTypeOperator = (o) =>
+    o.pipe(this.bind(), distinctUntilChanged(), shareReplay(1));
+
+  /**
+   * Transforms an Observable into a hot state Observable which replays its
+   * latest value upon subscription, skips updates with identical values, and
+   * is bound to this scope.
+   */
+  public state(): MonoTypeOperator {
+    return this.stateImpl;
   }
 
   /**