Bring back fullscreen and picture-in-picture modes

We're now using LiveKit's magic RoomAudioRenderer component to make sure everyone's audio is rendered regardless of whether they have a tile in the DOM.
2024-11-15 00:04:59 +08:00 · 2023-06-26 13:48:41 -04:00 · 2023-06-26 13:48:41 -04:00 · 276684c103
commit 276684c103
parent 41456505e0
3 changed files with 186 additions and 23 deletions
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@ -16,6 +16,8 @@ limitations under the License.

 import { ResizeObserver } from "@juggle/resize-observer";
 import {
+  RoomAudioRenderer,
+  RoomContext,
  useLocalParticipant,
  useParticipants,
  useTracks,
@ -80,6 +82,7 @@ import { RageshakeRequestModal } from "./RageshakeRequestModal";
 import { VideoTile } from "../video-grid/VideoTile";
 import { UserChoices, useLiveKit } from "../livekit/useLiveKit";
 import { useMediaDevices } from "../livekit/useMediaDevices";
+import { useFullscreen } from "./useFullscreen";

 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@ -100,7 +103,13 @@ export function ActiveCall(props: ActiveCallProps) {
    userIdentity: `${props.client.getUserId()}:${props.client.getDeviceId()}`,
  });

-  return livekitRoom && <InCallView {...props} livekitRoom={livekitRoom} />;
+  return (
+    livekitRoom && (
+      <RoomContext.Provider value={livekitRoom}>
+        <InCallView {...props} livekitRoom={livekitRoom} />
+      </RoomContext.Provider>
+    )
+  );
 }

 interface Props {
@ -172,9 +181,6 @@ export function InCallView({
  const toggleCamera = useCallback(async () => {
    await localParticipant.setCameraEnabled(!isCameraEnabled);
  }, [localParticipant, isCameraEnabled]);
-  const toggleScreensharing = useCallback(async () => {
-    await localParticipant.setScreenShareEnabled(!isScreenShareEnabled);
-  }, [localParticipant, isScreenShareEnabled]);

  const joinRule = useJoinRule(groupCall.room);

@ -227,15 +233,19 @@ export function InCallView({
  const noControls = reducedControls && bounds.height <= 400;

  const items = useParticipantTiles(livekitRoom, participants);
+  const { fullscreenItem, toggleFullscreen, exitFullscreen } =
+    useFullscreen(items);

-  // The maximised participant is the focused (active) participant, given the
+  // The maximised participant: either the participant that the user has
+  // manually put in fullscreen, or the focused (active) participant if the
  // window is too small to show everyone
  const maximisedParticipant = useMemo(
    () =>
-      noControls
-        ? items.find((item) => item.focused) ?? items.at(0) ?? null
-        : null,
-    [noControls, items]
+      fullscreenItem ??
+      (noControls
+        ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null
+        : null),
+    [fullscreenItem, noControls, items]
  );

  const Grid =
@ -254,6 +264,9 @@ export function InCallView({
    if (maximisedParticipant) {
      return (
        <VideoTile
+          maximised={true}
+          fullscreen={maximisedParticipant === fullscreenItem}
+          onToggleFullscreen={toggleFullscreen}
          targetHeight={bounds.height}
          targetWidth={bounds.width}
          key={maximisedParticipant.id}
@ -272,6 +285,9 @@ export function InCallView({
      >
        {(props) => (
          <VideoTile
+            maximised={false}
+            fullscreen={false}
+            onToggleFullscreen={toggleFullscreen}
            showSpeakingIndicator={items.length > 2}
            showConnectionStats={showConnectionStats}
            {...props}
@ -321,6 +337,11 @@ export function InCallView({
    [styles.maximised]: undefined,
  });

+  const toggleScreensharing = useCallback(async () => {
+    exitFullscreen();
+    await localParticipant.setScreenShareEnabled(!isScreenShareEnabled);
+  }, [localParticipant, isScreenShareEnabled, exitFullscreen]);
+
  let footer: JSX.Element | null;

  if (noControls) {
@ -354,9 +375,7 @@ export function InCallView({
          />
        );
      }
-      if (!maximisedParticipant) {
-        buttons.push(<SettingsButton key="4" onPress={openSettings} />);
-      }
+      buttons.push(<SettingsButton key="4" onPress={openSettings} />);
    }

    buttons.push(
@ -367,7 +386,7 @@ export function InCallView({

  return (
    <div className={containerClasses} ref={containerRef}>
-      {!hideHeader && (
+      {!hideHeader && maximisedParticipant === null && (
        <Header>
          <LeftNav>
            <RoomHeaderInfo
@ -388,6 +407,7 @@ export function InCallView({
        </Header>
      )}
      <div className={styles.controlsOverlay}>
+        <RoomAudioRenderer />
        {renderContent()}
        {footer}
      </div>
@ -469,6 +489,7 @@ function useParticipantTiles(
          local: sfuParticipant.isLocal,
          largeBaseSize: false,
          data: {
+            id,
            member,
            sfuParticipant,
            content: TileContent.UserMedia,
@ -478,14 +499,16 @@ function useParticipantTiles(
        // If there is a screen sharing enabled for this participant, create a tile for it as well.
        let screenShareTile: TileDescriptor<ItemData> | undefined;
        if (sfuParticipant.isScreenShareEnabled) {
+          const screenShareId = `${id}:screen-share`;
          screenShareTile = {
            ...userMediaTile,
-            id: `${id}:screen-share`,
+            id: screenShareId,
            focused: true,
            largeBaseSize: true,
            placeNear: id,
            data: {
              ...userMediaTile.data,
+              id: screenShareId,
              content: TileContent.ScreenShare,
            },
          };
--- a/src/room/useFullscreen.ts
+++ b/src/room/useFullscreen.ts
@ -0,0 +1,114 @@
+/*
+Copyright 2023 New Vector Ltd
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from "matrix-js-sdk/src/logger";
+import { useCallback, useLayoutEffect, useRef } from "react";
+
+import { TileDescriptor } from "../video-grid/VideoGrid";
+import { useReactiveState } from "../useReactiveState";
+import { useEventTarget } from "../useEvents";
+
+const isFullscreen = () =>
+  Boolean(document.fullscreenElement) ||
+  Boolean(document.webkitFullscreenElement);
+
+function enterFullscreen() {
+  if (document.body.requestFullscreen) {
+    document.body.requestFullscreen();
+  } else if (document.body.webkitRequestFullscreen) {
+    document.body.webkitRequestFullscreen();
+  } else {
+    logger.error("No available fullscreen API!");
+  }
+}
+
+function exitFullscreen() {
+  if (document.exitFullscreen) {
+    document.exitFullscreen();
+  } else if (document.webkitExitFullscreen) {
+    document.webkitExitFullscreen();
+  } else {
+    logger.error("No available fullscreen API!");
+  }
+}
+
+function useFullscreenChange(onFullscreenChange: () => void) {
+  useEventTarget(document.body, "fullscreenchange", onFullscreenChange);
+  useEventTarget(document.body, "webkitfullscreenchange", onFullscreenChange);
+}
+
+/**
+ * Provides callbacks for controlling the full-screen view, which can hold one
+ * item at a time.
+ */
+export function useFullscreen<T>(items: TileDescriptor<T>[]): {
+  fullscreenItem: TileDescriptor<T> | null;
+  toggleFullscreen: (itemId: string) => void;
+  exitFullscreen: () => void;
+} {
+  const [fullscreenItem, setFullscreenItem] =
+    useReactiveState<TileDescriptor<T> | null>(
+      (prevItem) =>
+        prevItem == null
+          ? null
+          : items.find((i) => i.id === prevItem.id) ?? null,
+      [items]
+    );
+
+  const latestItems = useRef<TileDescriptor<T>[]>(items);
+  latestItems.current = items;
+
+  const latestFullscreenItem = useRef<TileDescriptor<T> | null>(fullscreenItem);
+  latestFullscreenItem.current = fullscreenItem;
+
+  const toggleFullscreen = useCallback(
+    (itemId: string) => {
+      setFullscreenItem(
+        latestFullscreenItem.current === null
+          ? latestItems.current.find((i) => i.id === itemId) ?? null
+          : null
+      );
+    },
+    [setFullscreenItem]
+  );
+
+  const exitFullscreenCallback = useCallback(
+    () => setFullscreenItem(null),
+    [setFullscreenItem]
+  );
+
+  useLayoutEffect(() => {
+    // Determine whether we need to change the fullscreen state
+    if (isFullscreen() !== (fullscreenItem !== null)) {
+      (fullscreenItem === null ? exitFullscreen : enterFullscreen)();
+    }
+  }, [fullscreenItem]);
+
+  // Detect when the user exits fullscreen through an external mechanism like
+  // browser chrome or the escape key
+  useFullscreenChange(
+    useCallback(() => {
+      if (!isFullscreen()) setFullscreenItem(null);
+    }, [setFullscreenItem])
+  );
+
+  return {
+    fullscreenItem,
+    toggleFullscreen,
+    exitFullscreen: exitFullscreenCallback,
+  };
+}
--- a/src/video-grid/VideoTile.tsx
+++ b/src/video-grid/VideoTile.tsx
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import React from "react";
+import React, { useCallback } from "react";
 import { animated } from "@react-spring/web";
 import classNames from "classnames";
 import { useTranslation } from "react-i18next";
@ -34,8 +34,10 @@ import styles from "./VideoTile.module.css";
 import { ReactComponent as MicIcon } from "../icons/Mic.svg";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { useReactiveState } from "../useReactiveState";
+import { FullscreenButton } from "../button/Button";

 export interface ItemData {
+  id: string;
  member?: RoomMember;
  sfuParticipant: LocalParticipant | RemoteParticipant;
  content: TileContent;
@ -48,7 +50,9 @@ export enum TileContent {

 interface Props {
  data: ItemData;
-
+  maximised: boolean;
+  fullscreen: boolean;
+  onToggleFullscreen: (itemId: string) => void;
  // TODO: Refactor these props.
  targetWidth: number;
  targetHeight: number;
@ -62,6 +66,9 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
  (
    {
      data,
+      maximised,
+      fullscreen,
+      onToggleFullscreen,
      className,
      style,
      targetWidth,
@ -93,17 +100,33 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
      }
    }, [member, setDisplayName]);

-    const audioEl = React.useRef<HTMLAudioElement>(null);
    const { isMuted: microphoneMuted } = useMediaTrack(
      content === TileContent.UserMedia
        ? Track.Source.Microphone
        : Track.Source.ScreenShareAudio,
-      sfuParticipant,
-      {
-        element: audioEl,
-      }
+      sfuParticipant
    );

+    const onFullscreen = useCallback(() => {
+      onToggleFullscreen(data.id);
+    }, [data, onToggleFullscreen]);
+
+    const toolbarButtons: JSX.Element[] = [];
+    if (!sfuParticipant.isLocal) {
+      // TODO local volume option, which would also go here
+
+      if (content === TileContent.ScreenShare) {
+        toolbarButtons.push(
+          <FullscreenButton
+            key="fullscreen"
+            className={styles.button}
+            fullscreen={fullscreen}
+            onPress={onFullscreen}
+          />
+        );
+      }
+    }
+
    // Firefox doesn't respect the disablePictureInPicture attribute
    // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831

@ -117,11 +140,15 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
            showSpeakingIndicator,
          [styles.muted]: microphoneMuted,
          [styles.screenshare]: content === TileContent.ScreenShare,
+          [styles.maximised]: maximised,
        })}
        style={style}
        ref={tileRef}
        data-testid="videoTile"
      >
+        {toolbarButtons.length > 0 && (!maximised || fullscreen) && (
+          <div className={classNames(styles.toolbar)}>{toolbarButtons}</div>
+        )}
        {content === TileContent.UserMedia && !sfuParticipant.isCameraEnabled && (
          <>
            <div className={styles.videoMutedOverlay} />
@ -134,7 +161,7 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
            />
          </>
        )}
-        {content == TileContent.ScreenShare ? (
+        {content === TileContent.ScreenShare ? (
          <div className={styles.presenterLabel}>
            <span>{t("{{displayName}} is presenting", { displayName })}</span>
          </div>
@ -155,7 +182,6 @@ export const VideoTile = React.forwardRef<HTMLDivElement, Props>(
              : Track.Source.ScreenShare
          }
        />
-        <audio ref={audioEl} />
      </animated.div>
    );
  }