Bring back fullscreen and picture-in-picture modes

We're now using LiveKit's magic RoomAudioRenderer component to make sure everyone's audio is rendered regardless of whether they have a tile in the DOM.

src/room/InCallView.tsx

@@ -16,6 +16,8 @@ limitations under the License.
 
 import { ResizeObserver } from "@juggle/resize-observer";
 import {
+  RoomAudioRenderer,
+  RoomContext,
   useLocalParticipant,
   useParticipants,
   useTracks,
@@ -80,6 +82,7 @@ import { RageshakeRequestModal } from "./RageshakeRequestModal";
 import { VideoTile } from "../video-grid/VideoTile";
 import { UserChoices, useLiveKit } from "../livekit/useLiveKit";
 import { useMediaDevices } from "../livekit/useMediaDevices";
+import { useFullscreen } from "./useFullscreen";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -100,7 +103,13 @@ export function ActiveCall(props: ActiveCallProps) {
     userIdentity: `${props.client.getUserId()}:${props.client.getDeviceId()}`,
   });
 
-  return livekitRoom && <InCallView {...props} livekitRoom={livekitRoom} />;
+  return (
+    livekitRoom && (
+      <RoomContext.Provider value={livekitRoom}>
+        <InCallView {...props} livekitRoom={livekitRoom} />
+      </RoomContext.Provider>
+    )
+  );
 }
 
 interface Props {
@@ -172,9 +181,6 @@ export function InCallView({
   const toggleCamera = useCallback(async () => {
     await localParticipant.setCameraEnabled(!isCameraEnabled);
   }, [localParticipant, isCameraEnabled]);
-  const toggleScreensharing = useCallback(async () => {
-    await localParticipant.setScreenShareEnabled(!isScreenShareEnabled);
-  }, [localParticipant, isScreenShareEnabled]);
 
   const joinRule = useJoinRule(groupCall.room);
 
@@ -227,15 +233,19 @@ export function InCallView({
   const noControls = reducedControls && bounds.height <= 400;
 
   const items = useParticipantTiles(livekitRoom, participants);
+  const { fullscreenItem, toggleFullscreen, exitFullscreen } =
+    useFullscreen(items);
 
-  // The maximised participant is the focused (active) participant, given the
+  // The maximised participant: either the participant that the user has
+  // manually put in fullscreen, or the focused (active) participant if the
   // window is too small to show everyone
   const maximisedParticipant = useMemo(
     () =>
-      noControls
-        ? items.find((item) => item.focused) ?? items.at(0) ?? null
-        : null,
-    [noControls, items]
+      fullscreenItem ??
+      (noControls
+        ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null
+        : null),
+    [fullscreenItem, noControls, items]
   );
 
   const Grid =
@@ -254,6 +264,9 @@ export function InCallView({
     if (maximisedParticipant) {
       return (
         <VideoTile
+          maximised={true}
+          fullscreen={maximisedParticipant === fullscreenItem}
+          onToggleFullscreen={toggleFullscreen}
           targetHeight={bounds.height}
           targetWidth={bounds.width}
           key={maximisedParticipant.id}
@@ -272,6 +285,9 @@ export function InCallView({
       >
         {(props) => (
           <VideoTile
+            maximised={false}
+            fullscreen={false}
+            onToggleFullscreen={toggleFullscreen}
             showSpeakingIndicator={items.length > 2}
             showConnectionStats={showConnectionStats}
             {...props}
@@ -321,6 +337,11 @@ export function InCallView({
     [styles.maximised]: undefined,
   });
 
+  const toggleScreensharing = useCallback(async () => {
+    exitFullscreen();
+    await localParticipant.setScreenShareEnabled(!isScreenShareEnabled);
+  }, [localParticipant, isScreenShareEnabled, exitFullscreen]);
+
   let footer: JSX.Element | null;
 
   if (noControls) {
@@ -354,10 +375,8 @@ export function InCallView({
           />
         );
       }
-      if (!maximisedParticipant) {
       buttons.push(<SettingsButton key="4" onPress={openSettings} />);
     }
-    }
 
     buttons.push(
       <HangupButton key="6" onPress={onLeave} data-testid="incall_leave" />
@@ -367,7 +386,7 @@ export function InCallView({
 
   return (
     <div className={containerClasses} ref={containerRef}>
-      {!hideHeader && (
+      {!hideHeader && maximisedParticipant === null && (
         <Header>
           <LeftNav>
             <RoomHeaderInfo
@@ -388,6 +407,7 @@ export function InCallView({
         </Header>
       )}
       <div className={styles.controlsOverlay}>
+        <RoomAudioRenderer />
         {renderContent()}
         {footer}
       </div>
@@ -469,6 +489,7 @@ function useParticipantTiles(
           local: sfuParticipant.isLocal,
           largeBaseSize: false,
           data: {
+            id,
             member,
             sfuParticipant,
             content: TileContent.UserMedia,
@@ -478,14 +499,16 @@ function useParticipantTiles(
         // If there is a screen sharing enabled for this participant, create a tile for it as well.
         let screenShareTile: TileDescriptor<ItemData> | undefined;
         if (sfuParticipant.isScreenShareEnabled) {
+          const screenShareId = `${id}:screen-share`;
           screenShareTile = {
             ...userMediaTile,
-            id: `${id}:screen-share`,
+            id: screenShareId,
             focused: true,
             largeBaseSize: true,
             placeNear: id,
             data: {
               ...userMediaTile.data,
+              id: screenShareId,
               content: TileContent.ScreenShare,
             },
           };

src/room/useFullscreen.ts

@@ -0,0 +1,114 @@
+/*
+Copyright 2023 New Vector Ltd
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from "matrix-js-sdk/src/logger";
+import { useCallback, useLayoutEffect, useRef } from "react";
+
+import { TileDescriptor } from "../video-grid/VideoGrid";
+import { useReactiveState } from "../useReactiveState";
+import { useEventTarget } from "../useEvents";
+
+const isFullscreen = () =>
+  Boolean(document.fullscreenElement) ||
+  Boolean(document.webkitFullscreenElement);
+
+function enterFullscreen() {
+  if (document.body.requestFullscreen) {
+    document.body.requestFullscreen();
+  } else if (document.body.webkitRequestFullscreen) {
+    document.body.webkitRequestFullscreen();
+  } else {
+    logger.error("No available fullscreen API!");
+  }
+}
+
+function exitFullscreen() {
+  if (document.exitFullscreen) {
+    document.exitFullscreen();
+  } else if (document.webkitExitFullscreen) {
+    document.webkitExitFullscreen();
+  } else {
+    logger.error("No available fullscreen API!");
+  }
+}
+
+function useFullscreenChange(onFullscreenChange: () => void) {
+  useEventTarget(document.body, "fullscreenchange", onFullscreenChange);
+  useEventTarget(document.body, "webkitfullscreenchange", onFullscreenChange);
+}
+
+/**
+ * Provides callbacks for controlling the full-screen view, which can hold one
+ * item at a time.
+ */
+export function useFullscreen<T>(items: TileDescriptor<T>[]): {
+  fullscreenItem: TileDescriptor<T> | null;
+  toggleFullscreen: (itemId: string) => void;
+  exitFullscreen: () => void;
+} {
+  const [fullscreenItem, setFullscreenItem] =
+    useReactiveState<TileDescriptor<T> | null>(
+      (prevItem) =>
+        prevItem == null
+          ? null
+          : items.find((i) => i.id === prevItem.id) ?? null,
+      [items]
+    );
+
+  const latestItems = useRef<TileDescriptor<T>[]>(items);
+  latestItems.current = items;
+
+  const latestFullscreenItem = useRef<TileDescriptor<T> | null>(fullscreenItem);
+  latestFullscreenItem.current = fullscreenItem;
+
+  const toggleFullscreen = useCallback(
+    (itemId: string) => {
+      setFullscreenItem(
+        latestFullscreenItem.current === null
+          ? latestItems.current.find((i) => i.id === itemId) ?? null
+          : null
+      );
+    },
+    [setFullscreenItem]
+  );
+
+  const exitFullscreenCallback = useCallback(
+    () => setFullscreenItem(null),
+    [setFullscreenItem]
+  );
+
+  useLayoutEffect(() => {
+    // Determine whether we need to change the fullscreen state
+    if (isFullscreen() !== (fullscreenItem !== null)) {
+      (fullscreenItem === null ? exitFullscreen : enterFullscreen)();
+    }
+  }, [fullscreenItem]);
+
+  // Detect when the user exits fullscreen through an external mechanism like
+  // browser chrome or the escape key
+  useFullscreenChange(
+    useCallback(() => {
+      if (!isFullscreen()) setFullscreenItem(null);
+    }, [setFullscreenItem])
+  );
+
+  return {
+    fullscreenItem,
+    toggleFullscreen,
+    exitFullscreen: exitFullscreenCallback,
+  };
+}

src/video-grid/VideoTile.tsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React from "react";
+import React, { useCallback } from "react";
 import { animated } from "@react-spring/web";
 import classNames from "classnames";
 import { useTranslation } from "react-i18next";
@@ -34,8 +34,10 @@ import styles from "./VideoTile.module.css";
 import { ReactComponent as MicIcon } from "../icons/Mic.svg";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { useReactiveState } from "../useReactiveState";
+import { FullscreenButton } from "../button/Button";
 
 export interface ItemData {
+  id: string;
   member?: RoomMember;
   sfuParticipant: LocalParticipant | RemoteParticipant;
   content: TileContent;
@@ -48,7 +50,9 @@ export enum TileContent {
 
 interface Props {
   data: ItemData;
-
+  maximised: boolean;
+  fullscreen: boolean;
+  onToggleFullscreen: (itemId: string) => void;
   // TODO: Refactor these props.
   targetWidth: number;
   targetHeight: number;
@@ -62,6 +66,9 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
   (
     {
       data,
+      maximised,
+      fullscreen,
+      onToggleFullscreen,
       className,
       style,
       targetWidth,
@@ -93,17 +100,33 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
       }
     }, [member, setDisplayName]);
 
-    const audioEl = React.useRef<HTMLAudioElement>(null);
     const { isMuted: microphoneMuted } = useMediaTrack(
       content === TileContent.UserMedia
         ? Track.Source.Microphone
         : Track.Source.ScreenShareAudio,
-      sfuParticipant,
-      {
-        element: audioEl,
-      }
+      sfuParticipant
     );
 
+    const onFullscreen = useCallback(() => {
+      onToggleFullscreen(data.id);
+    }, [data, onToggleFullscreen]);
+
+    const toolbarButtons: JSX.Element[] = [];
+    if (!sfuParticipant.isLocal) {
+      // TODO local volume option, which would also go here
+
+      if (content === TileContent.ScreenShare) {
+        toolbarButtons.push(
+          <FullscreenButton
+            key="fullscreen"
+            className={styles.button}
+            fullscreen={fullscreen}
+            onPress={onFullscreen}
+          />
+        );
+      }
+    }
+
     // Firefox doesn't respect the disablePictureInPicture attribute
     // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
 
@@ -117,11 +140,15 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
             showSpeakingIndicator,
           [styles.muted]: microphoneMuted,
           [styles.screenshare]: content === TileContent.ScreenShare,
+          [styles.maximised]: maximised,
         })}
         style={style}
         ref={tileRef}
         data-testid="videoTile"
       >
+        {toolbarButtons.length > 0 && (!maximised || fullscreen) && (
+          <div className={classNames(styles.toolbar)}>{toolbarButtons}</div>
+        )}
         {content === TileContent.UserMedia && !sfuParticipant.isCameraEnabled && (
           <>
             <div className={styles.videoMutedOverlay} />
@@ -134,7 +161,7 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
             />
           </>
         )}
-        {content == TileContent.ScreenShare ? (
+        {content === TileContent.ScreenShare ? (
           <div className={styles.presenterLabel}>
             <span>{t("{{displayName}} is presenting", { displayName })}</span>
           </div>
@@ -155,7 +182,6 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
               : Track.Source.ScreenShare
           }
         />
-        <audio ref={audioEl} />
       </animated.div>
     );
   }