diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 02dd7740..abb8c67b 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -64,6 +64,12 @@ "crypto_version": "Crypto version: {{version}}", "device_id": "Device ID: {{id}}", "disconnected_banner": "Connectivity to the server has been lost.", + "e2ee_encryption_status": { + "connecting": "Connecting...", + "key_invalid": "The end-to-end encrypted media key for this person is invalid", + "key_missing": "You haven't received the current end-to-end encrypted media key for this person yet", + "password_invalid": "This person is using a different password so you won't be able to communicate with them" + }, "full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.", "full_screen_view_h1": "<0>Oops, something's gone wrong.", "group_call_loader": { diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index dcb8c2e5..7c3f21d5 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -194,6 +194,15 @@ function withCallViewModel( ), ); + const roomEventSelectorSpy = vi + .spyOn(ComponentsCore, "roomEventSelector") + .mockImplementation((room, eventType) => of()); + + const liveKitRoom = mockLivekitRoom( + { localParticipant }, + { remoteParticipants }, + ); + const vm = new CallViewModel( mockMatrixRoom({ client: { @@ -201,7 +210,7 @@ function withCallViewModel( } as Partial as MatrixClient, getMember: (userId) => members.get(userId) ?? null, }), - mockLivekitRoom({ localParticipant }), + liveKitRoom, { kind: E2eeType.PER_PARTICIPANT, }, @@ -213,6 +222,7 @@ function withCallViewModel( participantsSpy!.mockRestore(); mediaSpy!.mockRestore(); eventsSpy!.mockRestore(); + roomEventSelectorSpy!.mockRestore(); }); continuation(vm); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 37531511..27dd7aa7 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -226,6 +226,7 @@ class UserMedia { member: RoomMember | undefined, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, ) { this.vm = participant.isLocal ? new LocalUserMediaViewModel( @@ -233,12 +234,14 @@ class UserMedia { member, participant as LocalParticipant, encryptionSystem, + livekitRoom, ) : new RemoteUserMediaViewModel( id, member, participant as RemoteParticipant, encryptionSystem, + livekitRoom, ); this.speaker = this.vm.speaking.pipe( @@ -282,12 +285,14 @@ class ScreenShare { member: RoomMember | undefined, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, + liveKitRoom: LivekitRoom, ) { this.vm = new ScreenShareViewModel( id, member, participant, encryptionSystem, + liveKitRoom, ); } @@ -428,6 +433,7 @@ export class CallViewModel extends ViewModel { member, p, this.encryptionSystem, + this.livekitRoom, ), ]; @@ -441,6 +447,7 @@ export class CallViewModel extends ViewModel { member, p, this.encryptionSystem, + this.livekitRoom, ), ]; } diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 51a821af..2053d596 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -11,6 +11,7 @@ import { VideoSource, observeParticipantEvents, observeParticipantMedia, + roomEventSelector, } from "@livekit/components-core"; import { LocalParticipant, @@ -21,6 +22,9 @@ import { Track, TrackEvent, facingModeFromLocalTrack, + Room as LivekitRoom, + RoomEvent as LivekitRoomEvent, + RemoteTrack, } from "livekit-client"; import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix"; import { @@ -28,13 +32,18 @@ import { Observable, Subject, combineLatest, + distinctUntilChanged, distinctUntilKeyChanged, + filter, fromEvent, + interval, map, merge, of, + shareReplay, startWith, switchMap, + throttleTime, } from "rxjs"; import { useEffect } from "react"; @@ -81,6 +90,115 @@ export function observeTrackReference( ); } +function observeRemoteTrackReceivingOkay( + participant: Participant, + source: Track.Source, +): Observable { + let lastStats: { + framesDecoded: number | undefined; + framesDropped: number | undefined; + framesReceived: number | undefined; + } = { + framesDecoded: undefined, + framesDropped: undefined, + framesReceived: undefined, + }; + + return combineLatest([ + observeTrackReference(participant, source), + interval(1000).pipe(startWith(0)), + ]).pipe( + switchMap(async ([trackReference]) => { + const track = trackReference.publication?.track; + if (!track || !(track instanceof RemoteTrack)) { + return undefined; + } + const report = await track.getRTCStatsReport(); + if (!report) { + return undefined; + } + + for (const v of report.values()) { + if (v.type === "inbound-rtp") { + const { framesDecoded, framesDropped, framesReceived } = + v as RTCInboundRtpStreamStats; + return { + framesDecoded, + framesDropped, + framesReceived, + }; + } + } + + return undefined; + }), + filter((newStats) => !!newStats), + map((newStats): boolean | undefined => { + const oldStats = lastStats; + lastStats = newStats; + if ( + typeof newStats.framesReceived === "number" && + typeof oldStats.framesReceived === "number" && + typeof newStats.framesDecoded === "number" && + typeof oldStats.framesDecoded === "number" + ) { + const framesReceivedDelta = + newStats.framesReceived - oldStats.framesReceived; + const framesDecodedDelta = + newStats.framesDecoded - oldStats.framesDecoded; + + // if we received >0 frames and managed to decode >0 frames then we treat that as success + + if (framesReceivedDelta > 0) { + return framesDecodedDelta > 0; + } + } + + // no change + return undefined; + }), + filter((x) => typeof x === "boolean"), + startWith(undefined), + ); +} + +function encryptionErrorObservable( + room: LivekitRoom, + participant: Participant, + encryptionSystem: EncryptionSystem, + criteria: string, +): Observable { + return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe( + map((e) => { + const [err] = e; + if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + return ( + // Ideally we would pull the participant identity from the field on the error. + // However, it gets lost in the serialization process between workers. + // So, instead we do a string match + (err?.message.includes(participant.identity) && + err?.message.includes(criteria)) ?? + false + ); + } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) { + return !!err?.message.includes(criteria); + } + + return false; + }), + throttleTime(1000), // Throttle to avoid spamming the UI + startWith(false), + ); +} + +export enum EncryptionStatus { + Connecting, + Okay, + KeyMissing, + KeyInvalid, + PasswordInvalid, +} + abstract class BaseMediaViewModel extends ViewModel { /** * Whether the media belongs to the local user. @@ -95,6 +213,8 @@ abstract class BaseMediaViewModel extends ViewModel { */ public readonly unencryptedWarning: Observable; + public readonly encryptionStatus: Observable; + public constructor( /** * An opaque identifier for this media. @@ -110,6 +230,7 @@ abstract class BaseMediaViewModel extends ViewModel { encryptionSystem: EncryptionSystem, audioSource: AudioSource, videoSource: VideoSource, + livekitRoom: LivekitRoom, ) { super(); const audio = observeTrackReference(participant, audioSource).pipe( @@ -124,7 +245,64 @@ abstract class BaseMediaViewModel extends ViewModel { encryptionSystem.kind !== E2eeType.NONE && (a.publication?.isEncrypted === false || v.publication?.isEncrypted === false), - ).pipe(this.scope.state()); + ).pipe(distinctUntilChanged(), shareReplay(1)); + + if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) { + this.encryptionStatus = of(EncryptionStatus.Okay).pipe( + this.scope.state(), + ); + } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) { + this.encryptionStatus = combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "MissingKey", + ), + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => { + if (keyMissing) return EncryptionStatus.KeyMissing; + if (keyInvalid) return EncryptionStatus.KeyInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + this.scope.state(), + ); + } else { + this.encryptionStatus = combineLatest([ + encryptionErrorObservable( + livekitRoom, + participant, + encryptionSystem, + "InvalidKey", + ), + observeRemoteTrackReceivingOkay(participant, audioSource), + observeRemoteTrackReceivingOkay(participant, videoSource), + ]).pipe( + map( + ([keyInvalid, audioOkay, videoOkay]): + | EncryptionStatus + | undefined => { + if (keyInvalid) return EncryptionStatus.PasswordInvalid; + if (audioOkay || videoOkay) return EncryptionStatus.Okay; + return undefined; // no change + }, + ), + filter((x) => !!x), + startWith(EncryptionStatus.Connecting), + this.scope.state(), + ); + } } } @@ -171,6 +349,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { member: RoomMember | undefined, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, ) { super( id, @@ -179,6 +358,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { encryptionSystem, Track.Source.Microphone, Track.Source.Camera, + livekitRoom, ); const media = observeParticipantMedia(participant).pipe(this.scope.state()); @@ -228,8 +408,9 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { member: RoomMember | undefined, participant: LocalParticipant, encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, ) { - super(id, member, participant, encryptionSystem); + super(id, member, participant, encryptionSystem, livekitRoom); } } @@ -288,8 +469,9 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { member: RoomMember | undefined, participant: RemoteParticipant, encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, ) { - super(id, member, participant, encryptionSystem); + super(id, member, participant, encryptionSystem, livekitRoom); // Sync the local volume with LiveKit this.localVolume @@ -321,6 +503,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { member: RoomMember | undefined, participant: LocalParticipant | RemoteParticipant, encryptionSystem: EncryptionSystem, + livekitRoom: LivekitRoom, ) { super( id, @@ -329,6 +512,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel { encryptionSystem, Track.Source.ScreenShareAudio, Track.Source.ScreenShare, + livekitRoom, ); } } diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 3ea40c5b..e2b804e2 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -81,6 +81,7 @@ const UserMediaTile = forwardRef( const { t } = useTranslation(); const video = useObservableEagerState(vm.video); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); + const encryptionStatus = useObservableEagerState(vm.encryptionStatus); const audioEnabled = useObservableEagerState(vm.audioEnabled); const videoEnabled = useObservableEagerState(vm.videoEnabled); const speaking = useObservableEagerState(vm.speaking); @@ -122,6 +123,7 @@ const UserMediaTile = forwardRef( video={video} member={vm.member} unencryptedWarning={unencryptedWarning} + encryptionStatus={encryptionStatus} videoEnabled={videoEnabled} videoFit={cropVideo ? "cover" : "contain"} className={classNames(className, styles.tile, { diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index adde1c7b..3ed6c83d 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -85,7 +85,7 @@ unconditionally select the container so we can use cqmin units */ display: grid; grid-template-columns: 1fr auto; grid-template-rows: 1fr auto; - grid-template-areas: ". ." "nameTag button"; + grid-template-areas: "status status" "nameTag button"; gap: var(--cpd-space-1x); place-items: start; } @@ -106,6 +106,25 @@ unconditionally select the container so we can use cqmin units */ max-inline-size: 100%; } +.status { + grid-area: status; + justify-self: center; + align-self: start; + padding: var(--cpd-space-1x); + padding-block: var(--cpd-space-1x); + color: var(--cpd-color-text-primary); + background-color: var(--cpd-color-bg-canvas-default); + display: flex; + align-items: center; + border-radius: var(--cpd-radius-pill-effect); + user-select: none; + overflow: hidden; + box-shadow: var(--small-drop-shadow); + box-sizing: border-box; + max-inline-size: 100%; + text-align: center; +} + .nameTag > svg, .nameTag > span { flex-shrink: 0; diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index b14f0ac3..ea148fc0 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -17,6 +17,7 @@ import { ErrorIcon } from "@vector-im/compound-design-tokens/assets/web/icons"; import styles from "./MediaView.module.css"; import { Avatar } from "../Avatar"; +import { EncryptionStatus } from "../state/MediaViewModel"; import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; import { showHandRaisedTimer, useSetting } from "../settings/settings"; @@ -31,6 +32,7 @@ interface Props extends ComponentProps { member: RoomMember | undefined; videoEnabled: boolean; unencryptedWarning: boolean; + encryptionStatus: EncryptionStatus; nameTagLeadingIcon?: ReactNode; displayName: string; primaryButton?: ReactNode; @@ -54,6 +56,7 @@ export const MediaView = forwardRef( nameTagLeadingIcon, displayName, primaryButton, + encryptionStatus, raisedHandTime, raisedHandOnClick, ...props @@ -69,7 +72,11 @@ export const MediaView = forwardRef( ( )}
+ {encryptionStatus !== EncryptionStatus.Okay && ( +
+ + {encryptionStatus === EncryptionStatus.Connecting && + t("e2ee_encryption_status.connecting")} + {encryptionStatus === EncryptionStatus.KeyMissing && + t("e2ee_encryption_status.key_missing")} + {encryptionStatus === EncryptionStatus.KeyInvalid && + t("e2ee_encryption_status.key_invalid")} + {encryptionStatus === EncryptionStatus.PasswordInvalid && + t("e2ee_encryption_status.password_invalid")} + +
+ )} ( const displayName = useDisplayName(vm); const video = useObservableEagerState(vm.video); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); + const encryptionStatus = useObservableEagerState(vm.encryptionStatus); // Hook this item up to the intersection observer useEffect(() => { @@ -158,6 +161,7 @@ const SpotlightItem = forwardRef( member: vm.member, unencryptedWarning, displayName, + encryptionStatus, "aria-hidden": ariaHidden, }; diff --git a/src/utils/test.ts b/src/utils/test.ts index 85e72d38..2af6016c 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -4,7 +4,7 @@ Copyright 2023, 2024 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only Please see LICENSE in the repository root for full details. */ -import { map } from "rxjs"; +import { map, Observable, of } from "rxjs"; import { RunHelpers, TestScheduler } from "rxjs/testing"; import { expect, vi } from "vitest"; import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix"; @@ -99,8 +99,27 @@ export function mockMatrixRoom(room: Partial): MatrixRoom { return { ...mockEmitter(), ...room } as Partial as MatrixRoom; } -export function mockLivekitRoom(room: Partial): LivekitRoom { - return { ...mockEmitter(), ...room } as Partial as LivekitRoom; +export function mockLivekitRoom( + room: Partial, + { + remoteParticipants, + }: { remoteParticipants?: Observable } = {}, +): LivekitRoom { + const livekitRoom = { + ...mockEmitter(), + ...room, + } as Partial as LivekitRoom; + if (remoteParticipants) { + livekitRoom.remoteParticipants = new Map(); + remoteParticipants.subscribe((newRemoteParticipants) => { + livekitRoom.remoteParticipants.clear(); + newRemoteParticipants.forEach((p) => { + livekitRoom.remoteParticipants.set(p.identity, p); + }); + }); + } + + return livekitRoom; } export function mockLocalParticipant( @@ -119,13 +138,15 @@ export async function withLocalMedia( member: Partial, continuation: (vm: LocalUserMediaViewModel) => void | Promise, ): Promise { + const localParticipant = mockLocalParticipant({}); const vm = new LocalUserMediaViewModel( "local", mockMember(member), - mockLocalParticipant({}), + localParticipant, { kind: E2eeType.PER_PARTICIPANT, }, + mockLivekitRoom({ localParticipant }), ); try { await continuation(vm); @@ -152,13 +173,15 @@ export async function withRemoteMedia( participant: Partial, continuation: (vm: RemoteUserMediaViewModel) => void | Promise, ): Promise { + const remoteParticipant = mockRemoteParticipant(participant); const vm = new RemoteUserMediaViewModel( "remote", mockMember(member), - mockRemoteParticipant(participant), + remoteParticipant, { kind: E2eeType.PER_PARTICIPANT, }, + mockLivekitRoom({}, { remoteParticipants: of([remoteParticipant]) }), ); try { await continuation(vm);