Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2024-11-15 00:04:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1161 from robintown/layouts

Browse Source 
Make NewVideoGrid support arbitrary layout systems
This commit is contained in:
Robin 2023-06-28 11:02:15 -04:00 committed by GitHub
commit fc4ef88c75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 552 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/room/InCallView.tsx
View File

@ -83,6 +83,7 @@ import { VideoTile } from "../video-grid/VideoTile";
import { UserChoices, useLiveKit } from "../livekit/useLiveKit";
import { useMediaDevices } from "../livekit/useMediaDevices";
import { useFullscreen } from "./useFullscreen";
import { useLayoutStates } from "../video-grid/Layout";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
// There is currently a bug in Safari our our code with cloning and sending MediaStreams
@ -253,6 +254,10 @@ export function InCallView({
const prefersReducedMotion = usePrefersReducedMotion();
// This state is lifted out of NewVideoGrid so that layout states can be
// restored after a layout switch or upon exiting fullscreen
const layoutStates = useLayoutStates();
const renderContent = (): JSX.Element => {
if (items.length === 0) {
return (
@ -282,6 +287,7 @@ export function InCallView({
items={items}
layout={layout}
disableAnimations={prefersReducedMotion || isSafari}
layoutStates={layoutStates}
>
{(props) => (
<VideoTile

29
src/video-grid/BigGrid.module.css Normal file
View File

@ -0,0 +1,29 @@
/*
Copyright 2023 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.bigGrid {
display: grid;
grid-auto-rows: 163px;
gap: 8px;
}
@media (min-width: 800px) {
.bigGrid {
grid-auto-rows: 183px;
column-gap: 18px;
row-gap: 21px;
}
}

258
src/video-grid/model.ts → src/video-grid/BigGrid.tsx
View File

@ -15,33 +15,48 @@ limitations under the License.
*/
import TinyQueue from "tinyqueue";
import { RectReadOnly } from "react-use-measure";
import { FC, memo, ReactNode } from "react";
import React from "react";
import { TileDescriptor } from "./VideoGrid";
import { Slot } from "./NewVideoGrid";
import { Layout } from "./Layout";
import { count, findLastIndex } from "../array-utils";
import styles from "./BigGrid.module.css";
/**
* A 1×1 cell in a grid which belongs to a tile.
*/
export interface Cell {
interface Cell {
/**
* The item displayed on the tile.
*/
item: TileDescriptor<unknown>;
readonly item: TileDescriptor<unknown>;
/**
* Whether this cell is the origin (top left corner) of the tile.
*/
origin: boolean;
readonly origin: boolean;
/**
* The width, in columns, of the tile.
*/
columns: number;
readonly columns: number;
/**
* The height, in rows, of the tile.
*/
rows: number;
readonly rows: number;
}
export interface Grid {
export interface BigGridState {
readonly columns: number;
/**
* The cells of the grid, in left-to-right top-to-bottom order.
* undefined = empty.
*/
readonly cells: (Cell | undefined)[];
}
interface MutableBigGridState {
columns: number;
/**
* The cells of the grid, in left-to-right top-to-bottom order.
@ -58,7 +73,7 @@ export interface Grid {
* @returns An array in which each cell holds the index of the next cell to move
* to to reach the destination, or null if it is the destination.
*/
export function getPaths(dest: number, g: Grid): (number | null)[] {
export function getPaths(dest: number, g: BigGridState): (number | null)[] {
const destRow = row(dest, g);
const destColumn = column(dest, g);
@ -106,18 +121,23 @@ export function getPaths(dest: number, g: Grid): (number | null)[] {
return edges as (number | null)[];
}
const findLast1By1Index = (g: Grid): number | null =>
const findLast1By1Index = (g: BigGridState): number | null =>
findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1);
export function row(index: number, g: Grid): number {
export function row(index: number, g: BigGridState): number {
return Math.floor(index / g.columns);
}
export function column(index: number, g: Grid): number {
export function column(index: number, g: BigGridState): number {
return ((index % g.columns) + g.columns) % g.columns;
}
function inArea(index: number, start: number, end: number, g: Grid): boolean {
function inArea(
index: number,
start: number,
end: number,
g: BigGridState
): boolean {
const indexColumn = column(index, g);
const indexRow = row(index, g);
return (
@ -131,7 +151,7 @@ function inArea(index: number, start: number, end: number, g: Grid): boolean {
function* cellsInArea(
start: number,
end: number,
g: Grid
g: BigGridState
): Generator<number, void, unknown> {
const startColumn = column(start, g);
const endColumn = column(end, g);
@ -149,7 +169,7 @@ function* cellsInArea(
export function forEachCellInArea(
start: number,
end: number,
g: Grid,
g: BigGridState,
fn: (c: Cell | undefined, i: number) => void
): void {
for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i);
@ -158,7 +178,7 @@ export function forEachCellInArea(
function allCellsInArea(
start: number,
end: number,
g: Grid,
g: BigGridState,
fn: (c: Cell | undefined, i: number) => boolean
): boolean {
for (const i of cellsInArea(start, end, g)) {
@ -172,16 +192,19 @@ const areaEnd = (
start: number,
columns: number,
rows: number,
g: Grid
g: BigGridState
): number => start + columns - 1 + g.columns * (rows - 1);
const cloneGrid = (g: Grid): Grid => ({ ...g, cells: [...g.cells] });
const cloneGrid = (g: BigGridState): BigGridState => ({
...g,
cells: [...g.cells],
});
/**
* Gets the index of the next gap in the grid that should be backfilled by 1×1
* tiles.
*/
function getNextGap(g: Grid): number | null {
function getNextGap(g: BigGridState): number | null {
const last1By1Index = findLast1By1Index(g);
if (last1By1Index === null) return null;
@ -204,7 +227,7 @@ function getNextGap(g: Grid): number | null {
/**
* Gets the index of the origin of the tile to which the given cell belongs.
*/
function getOrigin(g: Grid, index: number): number {
function getOrigin(g: BigGridState, index: number): number {
const initialColumn = column(index, g);
for (
@ -229,7 +252,7 @@ function getOrigin(g: Grid, index: number): number {
* along the way.
* Precondition: the destination area must consist of only 1×1 tiles.
*/
function moveTile(g: Grid, from: number, to: number) {
function moveTileUnchecked(g: BigGridState, from: number, to: number) {
const tile = g.cells[from]!;
const fromEnd = areaEnd(from, tile.columns, tile.rows, g);
const toEnd = areaEnd(to, tile.columns, tile.rows, g);
@ -262,10 +285,15 @@ function moveTile(g: Grid, from: number, to: number) {
/**
* Moves the tile at index "from" over to index "to", if there is space.
*/
export function tryMoveTile(g: Grid, from: number, to: number): Grid {
export function moveTile(
g: BigGridState,
from: number,
to: number
): BigGridState {
const tile = g.cells[from]!;
if (
to !== from && // Skip the operation if nothing would move
to >= 0 &&
to < g.cells.length &&
column(to, g) <= g.columns - tile.columns
@ -283,7 +311,7 @@ export function tryMoveTile(g: Grid, from: number, to: number): Grid {
if (allCellsInArea(to, toEnd, g, displaceable)) {
// The target space is free; move
const gClone = cloneGrid(g);
moveTile(gClone, from, to);
moveTileUnchecked(gClone, from, to);
return gClone;
}
}
@ -297,7 +325,7 @@ export function tryMoveTile(g: Grid, from: number, to: number): Grid {
* enlarged tiles around when necessary.
* @returns Whether the tile was actually pushed
*/
function pushTileUp(g: Grid, from: number): boolean {
function pushTileUp(g: BigGridState, from: number): boolean {
const tile = g.cells[from]!;
// TODO: pushing large tiles sideways might be more successful in some
@ -315,7 +343,7 @@ function pushTileUp(g: Grid, from: number): boolean {
);
if (cellsAboveAreDisplacable) {
moveTile(g, from, from - g.columns);
moveTileUnchecked(g, from, from - g.columns);
return true;
} else {
return false;
@ -325,8 +353,8 @@ function pushTileUp(g: Grid, from: number): boolean {
/**
* Backfill any gaps in the grid.
*/
export function fillGaps(g: Grid): Grid {
const result = cloneGrid(g);
export function fillGaps(g: BigGridState): BigGridState {
const result = cloneGrid(g) as MutableBigGridState;
// This will hopefully be the size of the grid after we're done here, assuming
// that we can pack the large tiles tightly enough
@ -403,7 +431,11 @@ export function fillGaps(g: Grid): Grid {
return result;
}
function createRows(g: Grid, count: number, atRow: number): Grid {
function createRows(
g: BigGridState,
count: number,
atRow: number
): BigGridState {
const result = {
columns: g.columns,
cells: new Array(g.cells.length + g.columns * count),
@ -430,9 +462,12 @@ function createRows(g: Grid, count: number, atRow: number): Grid {
}
/**
* Adds a set of new items into the grid.
* Adds a set of new items into the grid. (May leave gaps.)
*/
export function addItems(items: TileDescriptor<unknown>[], g: Grid): Grid {
export function addItems(
items: TileDescriptor<unknown>[],
g: BigGridState
): BigGridState {
let result = cloneGrid(g);
for (const item of items) {
@ -444,13 +479,11 @@ export function addItems(items: TileDescriptor<unknown>[], g: Grid): Grid {
};
let placeAt: number;
let hasGaps: boolean;
if (item.placeNear === undefined) {
// This item has no special placement requests, so let's put it
// uneventfully at the end of the grid
placeAt = result.cells.length;
hasGaps = false;
} else {
// This item wants to be placed near another; let's put it on a row
// directly below the related tile
@ -460,7 +493,6 @@ export function addItems(items: TileDescriptor<unknown>[], g: Grid): Grid {
if (placeNear === -1) {
// Can't find the related tile, so let's give up and place it at the end
placeAt = result.cells.length;
hasGaps = false;
} else {
const placeNearCell = result.cells[placeNear]!;
const placeNearEnd = areaEnd(
@ -475,7 +507,6 @@ export function addItems(items: TileDescriptor<unknown>[], g: Grid): Grid {
placeNear +
Math.floor(placeNearCell.columns / 2) +
result.columns * placeNearCell.rows;
hasGaps = true;
}
}
@ -484,21 +515,19 @@ export function addItems(items: TileDescriptor<unknown>[], g: Grid): Grid {
if (item.largeBaseSize) {
// Cycle the tile size once to set up the tile with its larger base size
// This also fills any gaps in the grid, hence no extra call to fillGaps
result = cycleTileSize(item.id, result);
} else if (hasGaps) {
result = fillGaps(result);
result = cycleTileSize(result, item);
}
}
return result;
}
const largeTileDimensions = (g: Grid): [number, number] => [
const largeTileDimensions = (g: BigGridState): [number, number] => [
Math.min(3, Math.max(2, g.columns - 1)),
2,
];
const extraLargeTileDimensions = (g: Grid): [number, number] =>
const extraLargeTileDimensions = (g: BigGridState): [number, number] =>
g.columns > 3 ? [4, 3] : [g.columns, 2];
/**
@ -507,8 +536,11 @@ const extraLargeTileDimensions = (g: Grid): [number, number] =>
* @param g The grid.
* @returns The updated grid.
*/
export function cycleTileSize(tileId: string, g: Grid): Grid {
const from = g.cells.findIndex((c) => c?.item.id === tileId);
export function cycleTileSize(
g: BigGridState,
tile: TileDescriptor<unknown>
): BigGridState {
const from = g.cells.findIndex((c) => c?.item === tile);
if (from === -1) return g; // Tile removed, no change
const fromCell = g.cells[from]!;
const fromWidth = fromCell.columns;
@ -629,8 +661,8 @@ export function cycleTileSize(tileId: string, g: Grid): Grid {
/**
* Resizes the grid to a new column width.
*/
export function resize(g: Grid, columns: number): Grid {
const result: Grid = { columns, cells: [] };
export function resize(g: BigGridState, columns: number): BigGridState {
const result: BigGridState = { columns, cells: [] };
const [largeColumns, largeRows] = largeTileDimensions(result);
// Copy each tile from the old grid to the resized one in the same order
@ -640,6 +672,7 @@ export function resize(g: Grid, columns: number): Grid {
for (const cell of g.cells) {
if (cell?.origin) {
// TODO make aware of extra large tiles
const [nextColumns, nextRows] =
cell.columns > 1 || cell.rows > 1 ? [largeColumns, largeRows] : [1, 1];
@ -672,7 +705,7 @@ export function resize(g: Grid, columns: number): Grid {
/**
* Promotes speakers to the first page of the grid.
*/
export function promoteSpeakers(g: Grid) {
export function promoteSpeakers(g: BigGridState) {
// This is all a bit of a hack right now, because we don't know if the designs
// will stick with this approach in the long run
// We assume that 4 rows are probably about 1 page
@ -694,10 +727,149 @@ export function promoteSpeakers(g: Grid) {
toCell === undefined ||
(toCell.columns === 1 && toCell.rows === 1)
) {
moveTile(g, from, to);
moveTileUnchecked(g, from, to);
break;
}
}
}
}
}
/**
* The algorithm for updating a grid with a new set of tiles.
*/
function updateTiles(
g: BigGridState,
tiles: TileDescriptor<unknown>[]
): BigGridState {
// Step 1: Update tiles that still exist, and remove tiles that have left
// the grid
const itemsById = new Map(tiles.map((i) => [i.id, i]));
const grid1: BigGridState = {
...g,
cells: g.cells.map((c) => {
if (c === undefined) return undefined;
const item = itemsById.get(c.item.id);
return item === undefined ? undefined : { ...c, item };
}),
};
// Step 2: Add new tiles
const existingItemIds = new Set(
grid1.cells.filter((c) => c !== undefined).map((c) => c!.item.id)
);
const newItems = tiles.filter((i) => !existingItemIds.has(i.id));
const grid2 = addItems(newItems, grid1);
// Step 3: Promote speakers to the top
promoteSpeakers(grid2);
return fillGaps(grid2);
}
function updateBounds(g: BigGridState, bounds: RectReadOnly): BigGridState {
const columns = Math.max(2, Math.floor(bounds.width * 0.0045));
return columns === g.columns ? g : resize(g, columns);
}
const Slots: FC<{ s: BigGridState }> = memo(({ s: g }) => {
const areas = new Array<(number | null)[]>(
Math.ceil(g.cells.length / g.columns)
);
for (let i = 0; i < areas.length; i++)
areas[i] = new Array<number | null>(g.columns).fill(null);
let slotCount = 0;
for (let i = 0; i < g.cells.length; i++) {
const cell = g.cells[i];
if (cell?.origin) {
const slotEnd = i + cell.columns - 1 + g.columns * (cell.rows - 1);
forEachCellInArea(
i,
slotEnd,
g,
(_c, j) => (areas[row(j, g)][column(j, g)] = slotCount)
);
slotCount++;
}
}
const style = {
gridTemplateAreas: areas
.map(
(row) =>
`'${row
.map((slotId) => (slotId === null ? "." : `s${slotId}`))
.join(" ")}'`
)
.join(" "),
gridTemplateColumns: `repeat(${g.columns}, 1fr)`,
};
const slots = new Array<ReactNode>(slotCount);
for (let i = 0; i < slotCount; i++)
slots[i] = <Slot key={i} style={{ gridArea: `s${i}` }} />;
return (
<div className={styles.bigGrid} style={style}>
{slots}
</div>
);
});
/**
* Given a tile and numbers in the range [0, 1) describing a position within the
* tile, this returns the index of the specific cell in which that position
* lies.
*/
function positionOnTileToCell(
g: BigGridState,
tileOriginIndex: number,
xPositionOnTile: number,
yPositionOnTile: number
): number {
const tileOrigin = g.cells[tileOriginIndex]!;
const columnOnTile = Math.floor(xPositionOnTile * tileOrigin.columns);
const rowOnTile = Math.floor(yPositionOnTile * tileOrigin.rows);
return tileOriginIndex + columnOnTile + g.columns * rowOnTile;
}
function dragTile(
g: BigGridState,
from: TileDescriptor<unknown>,
to: TileDescriptor<unknown>,
xPositionOnFrom: number,
yPositionOnFrom: number,
xPositionOnTo: number,
yPositionOnTo: number
): BigGridState {
const fromOrigin = g.cells.findIndex((c) => c?.item === from);
const toOrigin = g.cells.findIndex((c) => c?.item === to);
const fromCell = positionOnTileToCell(
g,
fromOrigin,
xPositionOnFrom,
yPositionOnFrom
);
const toCell = positionOnTileToCell(
g,
toOrigin,
xPositionOnTo,
yPositionOnTo
);
return moveTile(g, fromOrigin, fromOrigin + toCell - fromCell);
}
export const BigGrid: Layout<BigGridState> = {
emptyState: { columns: 4, cells: [] },
updateTiles,
updateBounds,
getTiles: <T,>(g) =>
g.cells.filter((c) => c?.origin).map((c) => c!.item as T),
canDragTile: () => true,
dragTile,
toggleFocus: cycleTileSize,
Slots,
rememberState: false,
};

178
src/video-grid/Layout.tsx Normal file
View File

@ -0,0 +1,178 @@
/*
Copyright 2023 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { ComponentType, useCallback, useMemo, useRef } from "react";
import type { RectReadOnly } from "react-use-measure";
import { useReactiveState } from "../useReactiveState";
import type { TileDescriptor } from "./VideoGrid";
/**
* A video grid layout system with concrete states of type State.
*/
// Ideally State would be parameterized by the tile data type, but then that
// makes Layout a higher-kinded type, which isn't achievable in TypeScript
// (unless you invoke some dark type-level computation magic… 😏)
// So we're stuck with these types being a little too strong.
export interface Layout<State> {
/**
* The layout state for zero tiles.
*/
readonly emptyState: State;
/**
* Updates/adds/removes tiles in a way that looks natural in the context of
* the given initial state.
*/
readonly updateTiles: <T>(s: State, tiles: TileDescriptor<T>[]) => State;
/**
* Adapts the layout to a new container size.
*/
readonly updateBounds: (s: State, bounds: RectReadOnly) => State;
/**
* Gets tiles in the order created by the layout.
*/
readonly getTiles: <T>(s: State) => TileDescriptor<T>[];
/**
* Determines whether a tile is draggable.
*/
readonly canDragTile: <T>(s: State, tile: TileDescriptor<T>) => boolean;
/**
* Drags the tile 'from' to the location of the tile 'to' (if possible).
* The position parameters are numbers in the range [0, 1) describing the
* specific positions on 'from' and 'to' that the drag gesture is targeting.
*/
readonly dragTile: <T>(
s: State,
from: TileDescriptor<T>,
to: TileDescriptor<T>,
xPositionOnFrom: number,
yPositionOnFrom: number,
xPositionOnTo: number,
yPositionOnTo: number
) => State;
/**
* Toggles the focus of the given tile (if this layout has the concept of
* focus).
*/
readonly toggleFocus?: <T>(s: State, tile: TileDescriptor<T>) => State;
/**
* A React component generating the slot elements for a given layout state.
*/
readonly Slots: ComponentType<{ s: State }>;
/**
* Whether the state of this layout should be remembered even while a
* different layout is active.
*/
readonly rememberState: boolean;
}
/**
* A version of Map with stronger types that allow us to save layout states in a
* type-safe way.
*/
export interface LayoutStatesMap {
get<State>(layout: Layout<State>): State | undefined;
set<State>(layout: Layout<State>, state: State): LayoutStatesMap;
delete<State>(layout: Layout<State>): boolean;
}
/**
* Hook creating a Map to store layout states in.
*/
export const useLayoutStates = (): LayoutStatesMap => {
const layoutStates = useRef<Map<unknown, unknown>>();
if (layoutStates.current === undefined) layoutStates.current = new Map();
return layoutStates.current as LayoutStatesMap;
};
/**
* Hook which uses the provided layout system to arrange a set of items into a
* concrete layout state, and provides callbacks for user interaction.
*/
export const useLayout = <State, T>(
layout: Layout<State>,
items: TileDescriptor<T>[],
bounds: RectReadOnly,
layoutStates: LayoutStatesMap
) => {
const prevLayout = useRef<Layout<unknown>>();
const prevState = layoutStates.get(layout);
const [state, setState] = useReactiveState<State>(() => {
// If the bounds aren't known yet, don't add anything to the layout
if (bounds.width === 0) {
return layout.emptyState;
} else {
if (
prevLayout.current !== undefined &&
layout !== prevLayout.current &&
!prevLayout.current.rememberState
)
layoutStates.delete(prevLayout.current);
const baseState = layoutStates.get(layout) ?? layout.emptyState;
return layout.updateTiles(layout.updateBounds(baseState, bounds), items);
}
}, [layout, items, bounds]);
const generation = useRef<number>(0);
if (state !== prevState) generation.current++;
prevLayout.current = layout as Layout<unknown>;
// No point in remembering an empty state, plus it would end up clobbering the
// real saved state while restoring a layout
if (state !== layout.emptyState) layoutStates.set(layout, state);
return {
state,
orderedItems: useMemo(() => layout.getTiles<T>(state), [layout, state]),
generation: generation.current,
canDragTile: useCallback(
(tile: TileDescriptor<T>) => layout.canDragTile(state, tile),
[layout, state]
),
dragTile: useCallback(
(
from: TileDescriptor<T>,
to: TileDescriptor<T>,
xPositionOnFrom: number,
yPositionOnFrom: number,
xPositionOnTo: number,
yPositionOnTo: number
) =>
setState((s) =>
layout.dragTile(
s,
from,
to,
xPositionOnFrom,
yPositionOnFrom,
xPositionOnTo,
yPositionOnTo
)
),
[layout, setState]
),
toggleFocus: useMemo(
() =>
layout.toggleFocus &&
((tile: TileDescriptor<T>) =>
setState((s) => layout.toggleFocus!(s, tile))),
[layout, setState]
),
slots: <layout.Slots s={state} />,
};
};

11
src/video-grid/NewVideoGrid.module.css
View File

@ -23,11 +23,8 @@ limitations under the License.
overflow-x: hidden;
}
.slotGrid {
.slots {
position: relative;
display: grid;
grid-auto-rows: 163px;
gap: 8px;
}
.slot {
@ -38,10 +35,4 @@ limitations under the License.
.grid {
padding: 0 22px var(--footerHeight);
}
.slotGrid {
grid-auto-rows: 183px;
column-gap: 18px;
row-gap: 21px;
}
}

326
src/video-grid/NewVideoGrid.tsx
View File

@ -17,17 +17,16 @@ limitations under the License.
import { SpringRef, TransitionFn, useTransition } from "@react-spring/web";
import { EventTypes, Handler, useScroll } from "@use-gesture/react";
import React, {
Dispatch,
CSSProperties,
FC,
ReactNode,
SetStateAction,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import useMeasure from "react-use-measure";
import { zipWith } from "lodash";
import { zip } from "lodash";
import styles from "./NewVideoGrid.module.css";
import {
@ -38,99 +37,9 @@ import {
} from "./VideoGrid";
import { useReactiveState } from "../useReactiveState";
import { useMergedRefs } from "../useMergedRefs";
import {
Grid,
Cell,
row,
column,
fillGaps,
forEachCellInArea,
cycleTileSize,
addItems,
tryMoveTile,
resize,
promoteSpeakers,
} from "./model";
import { TileWrapper } from "./TileWrapper";
interface GridState extends Grid {
/**
* The ID of the current state of the grid.
*/
generation: number;
}
const useGridState = (
columns: number | null,
items: TileDescriptor<unknown>[]
): [GridState | null, Dispatch<SetStateAction<Grid>>] => {
const [grid, setGrid_] = useReactiveState<GridState | null>(
(prevGrid = null) => {
if (prevGrid === null) {
// We can't do anything if the column count isn't known yet
if (columns === null) {
return null;
} else {
prevGrid = { generation: 0, columns, cells: [] };
}
}
// Step 1: Update tiles that still exist, and remove tiles that have left
// the grid
const itemsById = new Map(items.map((i) => [i.id, i]));
const grid1: Grid = {
...prevGrid,
cells: prevGrid.cells.map((c) => {
if (c === undefined) return undefined;
const item = itemsById.get(c.item.id);
return item === undefined ? undefined : { ...c, item };
}),
};
// Step 2: Resize the grid if necessary and backfill gaps left behind by
// removed tiles
// Resizing already takes care of backfilling gaps
const grid2 =
columns !== grid1.columns ? resize(grid1, columns!) : fillGaps(grid1);
// Step 3: Add new tiles to the end of the grid
const existingItemIds = new Set(
grid2.cells.filter((c) => c !== undefined).map((c) => c!.item.id)
);
const newItems = items.filter((i) => !existingItemIds.has(i.id));
const grid3 = addItems(newItems, grid2);
// Step 4: Promote speakers to the top
promoteSpeakers(grid3);
return { ...grid3, generation: prevGrid.generation + 1 };
},
[columns, items]
);
const setGrid: Dispatch<SetStateAction<Grid>> = useCallback(
(action) => {
if (typeof action === "function") {
setGrid_((prevGrid) =>
prevGrid === null
? null
: {
...(action as (prev: Grid) => Grid)(prevGrid),
generation: prevGrid.generation + 1,
}
);
} else {
setGrid_((prevGrid) => ({
...action,
generation: prevGrid?.generation ?? 1,
}));
}
},
[setGrid_]
);
return [grid, setGrid];
};
import { BigGrid } from "./BigGrid";
import { useLayout } from "./Layout";
interface Rect {
x: number;
@ -139,8 +48,8 @@ interface Rect {
height: number;
}
interface Tile extends Rect {
item: TileDescriptor<unknown>;
interface Tile<T> extends Rect {
item: TileDescriptor<T>;
}
interface DragState {
@ -151,12 +60,21 @@ interface DragState {
cursorY: number;
}
interface SlotProps {
style?: CSSProperties;
}
export const Slot: FC<SlotProps> = ({ style }) => (
<div className={styles.slot} style={style} />
);
/**
* An interactive, animated grid of video tiles.
*/
export function NewVideoGrid<T>({
items,
disableAnimations,
layoutStates,
children,
}: Props<T>) {
// Overview: This component lays out tiles by rendering an invisible template
@ -169,36 +87,36 @@ export function NewVideoGrid<T>({
// most recently rendered generation of the grid, and watch it with a
// MutationObserver.
const [slotGrid, setSlotGrid] = useState<HTMLDivElement | null>(null);
const [slotGridGeneration, setSlotGridGeneration] = useState(0);
const [slotsRoot, setSlotsRoot] = useState<HTMLDivElement | null>(null);
const [renderedGeneration, setRenderedGeneration] = useState(0);
useEffect(() => {
if (slotGrid !== null) {
setSlotGridGeneration(
parseInt(slotGrid.getAttribute("data-generation")!)
if (slotsRoot !== null) {
setRenderedGeneration(
parseInt(slotsRoot.getAttribute("data-generation")!)
);
const observer = new MutationObserver((mutations) => {
if (mutations.some((m) => m.type === "attributes")) {
setSlotGridGeneration(
parseInt(slotGrid.getAttribute("data-generation")!)
setRenderedGeneration(
parseInt(slotsRoot.getAttribute("data-generation")!)
);
}
});
observer.observe(slotGrid, { attributes: true });
observer.observe(slotsRoot, { attributes: true });
return () => observer.disconnect();
}
}, [slotGrid, setSlotGridGeneration]);
}, [slotsRoot, setRenderedGeneration]);
const [gridRef1, gridBounds] = useMeasure();
const gridRef2 = useRef<HTMLDivElement | null>(null);
const gridRef = useMergedRefs(gridRef1, gridRef2);
const slotRects = useMemo(() => {
if (slotGrid === null) return [];
if (slotsRoot === null) return [];
const slots = slotGrid.getElementsByClassName(styles.slot);
const slots = slotsRoot.getElementsByClassName(styles.slot);
const rects = new Array<Rect>(slots.length);
for (let i = 0; i < slots.length; i++) {
const slot = slots[i] as HTMLElement;
@ -214,32 +132,34 @@ export function NewVideoGrid<T>({
// The rects may change due to the grid being resized or rerendered, but
// eslint can't statically verify this
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [slotGrid, slotGridGeneration, gridBounds]);
}, [slotsRoot, renderedGeneration, gridBounds]);
const columns = useMemo(
() =>
// The grid bounds might not be known yet
gridBounds.width === 0
? null
: Math.max(2, Math.floor(gridBounds.width * 0.0045)),
[gridBounds]
);
// TODO: Implement more layouts and select the right one here
const layout = BigGrid;
const {
state: grid,
orderedItems,
generation,
canDragTile,
dragTile,
toggleFocus,
slots,
} = useLayout(layout, items, gridBounds, layoutStates);
const [grid, setGrid] = useGridState(columns, items);
const [tiles] = useReactiveState<Tile[]>(
const [tiles] = useReactiveState<Tile<T>[]>(
(prevTiles) => {
// If React hasn't yet rendered the current generation of the grid, skip
// the update, because grid and slotRects will be out of sync
if (slotGridGeneration !== grid?.generation) return prevTiles ?? [];
if (renderedGeneration !== generation) return prevTiles ?? [];
const tileCells = grid.cells.filter((c) => c?.origin) as Cell[];
const tileRects = new Map<TileDescriptor<unknown>, Rect>(
zipWith(tileCells, slotRects, (cell, rect) => [cell.item, rect])
const tileRects = new Map(
zip(orderedItems, slotRects) as [TileDescriptor<T>, Rect][]
);
// In order to not break drag gestures, it's critical that we render tiles
// in a stable order (that of 'items')
return items.map((item) => ({ ...tileRects.get(item)!, item }));
},
[slotRects, grid, slotGridGeneration]
[slotRects, grid, renderedGeneration]
);
// Drag state is stored in a ref rather than component state, because we use
@ -249,8 +169,8 @@ export function NewVideoGrid<T>({
const [tileTransitions, springRef] = useTransition(
tiles,
() => ({
key: ({ item }: Tile) => item.id,
from: ({ x, y, width, height }: Tile) => ({
key: ({ item }: Tile<T>) => item.id,
from: ({ x, y, width, height }: Tile<T>) => ({
opacity: 0,
scale: 0,
shadow: 1,
@ -263,7 +183,7 @@ export function NewVideoGrid<T>({
immediate: disableAnimations,
}),
enter: { opacity: 1, scale: 1, immediate: disableAnimations },
update: ({ item, x, y, width, height }: Tile) =>
update: ({ item, x, y, width, height }: Tile<T>) =>
item.id === dragState.current?.tileId
? null
: {
@ -277,7 +197,7 @@ export function NewVideoGrid<T>({
config: { mass: 0.7, tension: 252, friction: 25 },
})
// react-spring's types are bugged and can't infer the spring type
) as unknown as [TransitionFn<Tile, TileSpring>, SpringRef<TileSpring>];
) as unknown as [TransitionFn<Tile<T>, TileSpring>, SpringRef<TileSpring>];
// Because we're using react-spring in imperative mode, we're responsible for
// firing animations manually whenever the tiles array updates
@ -288,11 +208,9 @@ export function NewVideoGrid<T>({
const animateDraggedTile = (endOfGesture: boolean) => {
const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!;
const tile = tiles.find((t) => t.item.id === tileId)!;
const originIndex = grid!.cells.findIndex((c) => c?.item.id === tileId);
const originCell = grid!.cells[originIndex]!;
springRef.current
.find((c) => (c.item as Tile).item.id === tileId)
.find((c) => (c.item as Tile<T>).item.id === tileId)
?.start(
endOfGesture
? {
@ -320,36 +238,23 @@ export function NewVideoGrid<T>({
}
);
const columns = grid!.columns;
const rows = row(grid!.cells.length - 1, grid!) + 1;
const cursorColumn = Math.floor(
(cursorX / slotGrid!.clientWidth) * columns
);
const cursorRow = Math.floor((cursorY / slotGrid!.clientHeight) * rows);
const cursorColumnOnTile = Math.floor(
((cursorX - tileX) / tile.width) * originCell.columns
);
const cursorRowOnTile = Math.floor(
((cursorY - tileY) / tile.height) * originCell.rows
const overTile = tiles.find(
(t) =>
cursorX >= t.x &&
cursorX < t.x + t.width &&
cursorY >= t.y &&
cursorY < t.y + t.height
);
const dest =
Math.max(
0,
Math.min(
columns - originCell.columns,
cursorColumn - cursorColumnOnTile
)
) +
grid!.columns *
Math.max(
0,
Math.min(rows - originCell.rows, cursorRow - cursorRowOnTile)
);
if (dest !== originIndex) setGrid((g) => tryMoveTile(g, originIndex, dest));
if (overTile !== undefined)
dragTile(
tile.item,
overTile.item,
(cursorX - tileX) / tile.width,
(cursorY - tileY) / tile.height,
(cursorX - overTile.x) / overTile.width,
(cursorY - overTile.y) / overTile.height
);
};
// Callback for useDrag. We could call useDrag here, but the default
@ -367,29 +272,33 @@ export function NewVideoGrid<T>({
}: Parameters<Handler<"drag", EventTypes["drag"]>>[0]
) => {
if (tap) {
setGrid((g) => cycleTileSize(tileId, g!));
toggleFocus?.(items.find((i) => i.id === tileId)!);
} else {
const tileSpring = springRef.current
.find((c) => (c.item as Tile).item.id === tileId)!
.get();
const tileController = springRef.current.find(
(c) => (c.item as Tile<T>).item.id === tileId
)!;
if (dragState.current === null) {
dragState.current = {
tileId,
tileX: tileSpring.x,
tileY: tileSpring.y,
cursorX: initialX - gridBounds.x,
cursorY: initialY - gridBounds.y + scrollOffset.current,
};
if (canDragTile((tileController.item as Tile<T>).item)) {
if (dragState.current === null) {
const tileSpring = tileController.get();
dragState.current = {
tileId,
tileX: tileSpring.x,
tileY: tileSpring.y,
cursorX: initialX - gridBounds.x,
cursorY: initialY - gridBounds.y + scrollOffset.current,
};
}
dragState.current.tileX += dx;
dragState.current.tileY += dy;
dragState.current.cursorX += dx;
dragState.current.cursorY += dy;
animateDraggedTile(last);
if (last) dragState.current = null;
}
dragState.current.tileX += dx;
dragState.current.tileY += dy;
dragState.current.cursorX += dx;
dragState.current.cursorY += dy;
animateDraggedTile(last);
if (last) dragState.current = null;
}
};
@ -411,52 +320,6 @@ export function NewVideoGrid<T>({
{ target: gridRef2 }
);
const slotGridStyle = useMemo(() => {
if (grid === null) return {};
const areas = new Array<(number | null)[]>(
Math.ceil(grid.cells.length / grid.columns)
);
for (let i = 0; i < areas.length; i++)
areas[i] = new Array<number | null>(grid.columns).fill(null);
let slotId = 0;
for (let i = 0; i < grid.cells.length; i++) {
const cell = grid.cells[i];
if (cell?.origin) {
const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1);
forEachCellInArea(
i,
slotEnd,
grid,
(_c, j) => (areas[row(j, grid)][column(j, grid)] = slotId)
);
slotId++;
}
}
return {
gridTemplateAreas: areas
.map(
(row) =>
`'${row
.map((slotId) => (slotId === null ? "." : `s${slotId}`))
.join(" ")}'`
)
.join(" "),
gridTemplateColumns: `repeat(${columns}, 1fr)`,
};
}, [grid, columns]);
const slots = useMemo(() => {
const slots = new Array<ReactNode>(items.length);
for (let i = 0; i < items.length; i++)
slots[i] = (
<div className={styles.slot} key={i} style={{ gridArea: `s${i}` }} />
);
return slots;
}, [items.length]);
// Render nothing if the grid has yet to be generated
if (grid === null) {
return <div ref={gridRef} className={styles.grid} />;
@ -465,10 +328,9 @@ export function NewVideoGrid<T>({
return (
<div ref={gridRef} className={styles.grid}>
<div
style={slotGridStyle}
ref={setSlotGrid}
className={styles.slotGrid}
data-generation={grid.generation}
ref={setSlotsRoot}
className={styles.slots}
data-generation={generation}
>
{slots}
</div>
@ -482,7 +344,7 @@ export function NewVideoGrid<T>({
data={tile.item.data}
{...spring}
>
{children as (props: ChildrenProperties<unknown>) => ReactNode}
{children as (props: ChildrenProperties<T>) => ReactNode}
</TileWrapper>
))}
</div>

9
src/video-grid/TileWrapper.tsx
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { FC, memo, ReactNode, RefObject, useRef } from "react";
import React, { memo, ReactNode, RefObject, useRef } from "react";
import { EventTypes, Handler, useDrag } from "@use-gesture/react";
import { SpringValue, to } from "@react-spring/web";
@ -47,7 +47,7 @@ interface Props<T> {
* A wrapper around a tile in a video grid. This component exists to decouple
* child components from the grid.
*/
export const TileWrapper: FC<Props<unknown>> = memo(
export const TileWrapper = memo(
({
id,
onDragRef,
@ -97,4 +97,7 @@ export const TileWrapper: FC<Props<unknown>> = memo(
</>
);
}
);
// We pretend this component is a simple function rather than a
// NamedExoticComponent, because that's the only way we can fit in a type
// parameter
) as <T>(props: Props<T>) => JSX.Element;

2
src/video-grid/VideoGrid.tsx
View File

@ -42,6 +42,7 @@ import { ResizeObserver as JuggleResizeObserver } from "@juggle/resize-observer"
import styles from "./VideoGrid.module.css";
import { Layout } from "../room/GridLayoutMenu";
import { TileWrapper } from "./TileWrapper";
import { LayoutStatesMap } from "./Layout";
interface TilePosition {
x: number;
@ -817,6 +818,7 @@ export interface VideoGridProps<T> {
items: TileDescriptor<T>[];
layout: Layout;
disableAnimations: boolean;
layoutStates: LayoutStatesMap;
children: (props: ChildrenProperties<T>) => React.ReactNode;
}

51
test/video-grid/model-test.ts → test/video-grid/BigGrid-test.ts
View File

@ -20,23 +20,23 @@ import {
cycleTileSize,
fillGaps,
forEachCellInArea,
Grid,
BigGridState,
resize,
row,
tryMoveTile,
} from "../../src/video-grid/model";
moveTile,
} from "../../src/video-grid/BigGrid";
import { TileDescriptor } from "../../src/video-grid/VideoGrid";
/**
* Builds a grid from a string specifying the contents of each cell as a letter.
*/
function mkGrid(spec: string): Grid {
function mkGrid(spec: string): BigGridState {
const secondNewline = spec.indexOf("\n", 1);
const columns = secondNewline === -1 ? spec.length : secondNewline - 1;
const cells = spec.match(/[a-z ]/g) ?? ([] as string[]);
const areas = new Set(cells);
areas.delete(" "); // Space represents an empty cell, not an area
const grid: Grid = { columns, cells: new Array(cells.length) };
const grid: BigGridState = { columns, cells: new Array(cells.length) };
for (const area of areas) {
const start = cells.indexOf(area);
@ -60,12 +60,12 @@ function mkGrid(spec: string): Grid {
/**
* Turns a grid into a string showing the contents of each cell as a letter.
*/
function showGrid(g: Grid): string {
function showGrid(g: BigGridState): string {
let result = "\n";
g.cells.forEach((c, i) => {
for (let i = 0; i < g.cells.length; i++) {
if (i > 0 && i % g.columns == 0) result += "\n";
result += c?.item.id ?? " ";
});
result += g.cells[i]?.item.id ?? " ";
}
return result;
}
@ -222,21 +222,12 @@ function testCycleTileSize(
output: string
): void {
test(`cycleTileSize ${title}`, () => {
expect(showGrid(cycleTileSize(tileId, mkGrid(input)))).toBe(output);
const grid = mkGrid(input);
const tile = grid.cells.find((c) => c?.item.id === tileId)!.item;
expect(showGrid(cycleTileSize(grid, tile))).toBe(output);
});
}
testCycleTileSize(
"does nothing if the tile is not present",
"z",
`
abcd
efgh`,
`
abcd
efgh`
);
testCycleTileSize(
"expands a tile to 2×2 in a 3 column layout",
"c",
@ -345,8 +336,8 @@ abc
def`,
`
abc
gfe
d`
g
def`
);
testAddItems(
@ -362,19 +353,19 @@ gge
d`
);
function testTryMoveTile(
function testMoveTile(
title: string,
from: number,
to: number,
input: string,
output: string
): void {
test(`tryMoveTile ${title}`, () => {
expect(showGrid(tryMoveTile(mkGrid(input), from, to))).toBe(output);
test(`moveTile ${title}`, () => {
expect(showGrid(moveTile(mkGrid(input), from, to))).toBe(output);
});
}
testTryMoveTile(
testMoveTile(
"refuses to move a tile too far to the left",
1,
-1,
@ -384,7 +375,7 @@ abc`,
abc`
);
testTryMoveTile(
testMoveTile(
"refuses to move a tile too far to the right",
1,
3,
@ -394,7 +385,7 @@ abc`,
abc`
);
testTryMoveTile(
testMoveTile(
"moves a large tile to an unoccupied space",
3,
1,
@ -408,7 +399,7 @@ bcc
d e`
);
testTryMoveTile(
testMoveTile(
"refuses to move a large tile to an occupied space",
3,
1,