Show encryption key status from LiveKit (#2700)

* Refactor to make encryption system available in view models

* WIP show encryption errors from LiveKit

* Missing CSS

* Show encryption status based on LK and RTC

* Lint

* Lint

* Fix tests

* Update wording

* Refactor

* Lint

public/locales/en-GB/app.json

@@ -64,6 +64,12 @@
   "crypto_version": "Crypto version: {{version}}",
   "device_id": "Device ID: {{id}}",
   "disconnected_banner": "Connectivity to the server has been lost.",
+  "e2ee_encryption_status": {
+    "connecting": "Connecting...",
+    "key_invalid": "The end-to-end encrypted media key for this person is invalid",
+    "key_missing": "You haven't received the current end-to-end encrypted media key for this person yet",
+    "password_invalid": "This person is using a different password so you won't be able to communicate with them"
+  },
   "full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.</0>",
   "full_screen_view_h1": "<0>Oops, something's gone wrong.</0>",
   "group_call_loader": {

src/state/CallViewModel.test.ts

@@ -194,6 +194,15 @@ function withCallViewModel(
       ),
     );
 
+  const roomEventSelectorSpy = vi
+    .spyOn(ComponentsCore, "roomEventSelector")
+    .mockImplementation((room, eventType) => of());
+
+  const liveKitRoom = mockLivekitRoom(
+    { localParticipant },
+    { remoteParticipants },
+  );
+
   const vm = new CallViewModel(
     mockMatrixRoom({
       client: {
@@ -201,7 +210,7 @@ function withCallViewModel(
       } as Partial<MatrixClient> as MatrixClient,
       getMember: (userId) => members.get(userId) ?? null,
     }),
-    mockLivekitRoom({ localParticipant }),
+    liveKitRoom,
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
@@ -213,6 +222,7 @@ function withCallViewModel(
     participantsSpy!.mockRestore();
     mediaSpy!.mockRestore();
     eventsSpy!.mockRestore();
+    roomEventSelectorSpy!.mockRestore();
   });
 
   continuation(vm);

src/state/CallViewModel.ts

@@ -226,6 +226,7 @@ class UserMedia {
     member: RoomMember | undefined,
     participant: LocalParticipant | RemoteParticipant,
     encryptionSystem: EncryptionSystem,
+    livekitRoom: LivekitRoom,
   ) {
     this.vm = participant.isLocal
       ? new LocalUserMediaViewModel(
@@ -233,12 +234,14 @@ class UserMedia {
           member,
           participant as LocalParticipant,
           encryptionSystem,
+          livekitRoom,
         )
       : new RemoteUserMediaViewModel(
           id,
           member,
           participant as RemoteParticipant,
           encryptionSystem,
+          livekitRoom,
         );
 
     this.speaker = this.vm.speaking.pipe(
@@ -282,12 +285,14 @@ class ScreenShare {
     member: RoomMember | undefined,
     participant: LocalParticipant | RemoteParticipant,
     encryptionSystem: EncryptionSystem,
+    liveKitRoom: LivekitRoom,
   ) {
     this.vm = new ScreenShareViewModel(
       id,
       member,
       participant,
       encryptionSystem,
+      liveKitRoom,
     );
   }
 
@@ -428,6 +433,7 @@ export class CallViewModel extends ViewModel {
                       member,
                       p,
                       this.encryptionSystem,
+                      this.livekitRoom,
                     ),
                 ];
 
@@ -441,6 +447,7 @@ export class CallViewModel extends ViewModel {
                         member,
                         p,
                         this.encryptionSystem,
+                        this.livekitRoom,
                       ),
                   ];
                 }

src/state/MediaViewModel.ts

@@ -11,6 +11,7 @@ import {
   VideoSource,
   observeParticipantEvents,
   observeParticipantMedia,
+  roomEventSelector,
 } from "@livekit/components-core";
 import {
   LocalParticipant,
@@ -21,6 +22,9 @@ import {
   Track,
   TrackEvent,
   facingModeFromLocalTrack,
+  Room as LivekitRoom,
+  RoomEvent as LivekitRoomEvent,
+  RemoteTrack,
 } from "livekit-client";
 import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix";
 import {
@@ -28,13 +32,18 @@ import {
   Observable,
   Subject,
   combineLatest,
+  distinctUntilChanged,
   distinctUntilKeyChanged,
+  filter,
   fromEvent,
+  interval,
   map,
   merge,
   of,
+  shareReplay,
   startWith,
   switchMap,
+  throttleTime,
 } from "rxjs";
 import { useEffect } from "react";
 
@@ -81,6 +90,115 @@ export function observeTrackReference(
   );
 }
 
+function observeRemoteTrackReceivingOkay(
+  participant: Participant,
+  source: Track.Source,
+): Observable<boolean | undefined> {
+  let lastStats: {
+    framesDecoded: number | undefined;
+    framesDropped: number | undefined;
+    framesReceived: number | undefined;
+  } = {
+    framesDecoded: undefined,
+    framesDropped: undefined,
+    framesReceived: undefined,
+  };
+
+  return combineLatest([
+    observeTrackReference(participant, source),
+    interval(1000).pipe(startWith(0)),
+  ]).pipe(
+    switchMap(async ([trackReference]) => {
+      const track = trackReference.publication?.track;
+      if (!track || !(track instanceof RemoteTrack)) {
+        return undefined;
+      }
+      const report = await track.getRTCStatsReport();
+      if (!report) {
+        return undefined;
+      }
+
+      for (const v of report.values()) {
+        if (v.type === "inbound-rtp") {
+          const { framesDecoded, framesDropped, framesReceived } =
+            v as RTCInboundRtpStreamStats;
+          return {
+            framesDecoded,
+            framesDropped,
+            framesReceived,
+          };
+        }
+      }
+
+      return undefined;
+    }),
+    filter((newStats) => !!newStats),
+    map((newStats): boolean | undefined => {
+      const oldStats = lastStats;
+      lastStats = newStats;
+      if (
+        typeof newStats.framesReceived === "number" &&
+        typeof oldStats.framesReceived === "number" &&
+        typeof newStats.framesDecoded === "number" &&
+        typeof oldStats.framesDecoded === "number"
+      ) {
+        const framesReceivedDelta =
+          newStats.framesReceived - oldStats.framesReceived;
+        const framesDecodedDelta =
+          newStats.framesDecoded - oldStats.framesDecoded;
+
+        // if we received >0 frames and managed to decode >0 frames then we treat that as success
+
+        if (framesReceivedDelta > 0) {
+          return framesDecodedDelta > 0;
+        }
+      }
+
+      // no change
+      return undefined;
+    }),
+    filter((x) => typeof x === "boolean"),
+    startWith(undefined),
+  );
+}
+
+function encryptionErrorObservable(
+  room: LivekitRoom,
+  participant: Participant,
+  encryptionSystem: EncryptionSystem,
+  criteria: string,
+): Observable<boolean> {
+  return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
+    map((e) => {
+      const [err] = e;
+      if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
+        return (
+          // Ideally we would pull the participant identity from the field on the error.
+          // However, it gets lost in the serialization process between workers.
+          // So, instead we do a string match
+          (err?.message.includes(participant.identity) &&
+            err?.message.includes(criteria)) ??
+          false
+        );
+      } else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
+        return !!err?.message.includes(criteria);
+      }
+
+      return false;
+    }),
+    throttleTime(1000), // Throttle to avoid spamming the UI
+    startWith(false),
+  );
+}
+
+export enum EncryptionStatus {
+  Connecting,
+  Okay,
+  KeyMissing,
+  KeyInvalid,
+  PasswordInvalid,
+}
+
 abstract class BaseMediaViewModel extends ViewModel {
   /**
    * Whether the media belongs to the local user.
@@ -95,6 +213,8 @@ abstract class BaseMediaViewModel extends ViewModel {
    */
   public readonly unencryptedWarning: Observable<boolean>;
 
+  public readonly encryptionStatus: Observable<EncryptionStatus>;
+
   public constructor(
     /**
      * An opaque identifier for this media.
@@ -110,6 +230,7 @@ abstract class BaseMediaViewModel extends ViewModel {
     encryptionSystem: EncryptionSystem,
     audioSource: AudioSource,
     videoSource: VideoSource,
+    livekitRoom: LivekitRoom,
   ) {
     super();
     const audio = observeTrackReference(participant, audioSource).pipe(
@@ -124,7 +245,64 @@ abstract class BaseMediaViewModel extends ViewModel {
         encryptionSystem.kind !== E2eeType.NONE &&
         (a.publication?.isEncrypted === false ||
           v.publication?.isEncrypted === false),
-    ).pipe(this.scope.state());
+    ).pipe(distinctUntilChanged(), shareReplay(1));
+
+    if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
+      this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
+        this.scope.state(),
+      );
+    } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
+      this.encryptionStatus = combineLatest([
+        encryptionErrorObservable(
+          livekitRoom,
+          participant,
+          encryptionSystem,
+          "MissingKey",
+        ),
+        encryptionErrorObservable(
+          livekitRoom,
+          participant,
+          encryptionSystem,
+          "InvalidKey",
+        ),
+        observeRemoteTrackReceivingOkay(participant, audioSource),
+        observeRemoteTrackReceivingOkay(participant, videoSource),
+      ]).pipe(
+        map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
+          if (keyMissing) return EncryptionStatus.KeyMissing;
+          if (keyInvalid) return EncryptionStatus.KeyInvalid;
+          if (audioOkay || videoOkay) return EncryptionStatus.Okay;
+          return undefined; // no change
+        }),
+        filter((x) => !!x),
+        startWith(EncryptionStatus.Connecting),
+        this.scope.state(),
+      );
+    } else {
+      this.encryptionStatus = combineLatest([
+        encryptionErrorObservable(
+          livekitRoom,
+          participant,
+          encryptionSystem,
+          "InvalidKey",
+        ),
+        observeRemoteTrackReceivingOkay(participant, audioSource),
+        observeRemoteTrackReceivingOkay(participant, videoSource),
+      ]).pipe(
+        map(
+          ([keyInvalid, audioOkay, videoOkay]):
+            | EncryptionStatus
+            | undefined => {
+            if (keyInvalid) return EncryptionStatus.PasswordInvalid;
+            if (audioOkay || videoOkay) return EncryptionStatus.Okay;
+            return undefined; // no change
+          },
+        ),
+        filter((x) => !!x),
+        startWith(EncryptionStatus.Connecting),
+        this.scope.state(),
+      );
+    }
   }
 }
 
@@ -171,6 +349,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
     member: RoomMember | undefined,
     participant: LocalParticipant | RemoteParticipant,
     encryptionSystem: EncryptionSystem,
+    livekitRoom: LivekitRoom,
   ) {
     super(
       id,
@@ -179,6 +358,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
       encryptionSystem,
       Track.Source.Microphone,
       Track.Source.Camera,
+      livekitRoom,
     );
 
     const media = observeParticipantMedia(participant).pipe(this.scope.state());
@@ -228,8 +408,9 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
     member: RoomMember | undefined,
     participant: LocalParticipant,
     encryptionSystem: EncryptionSystem,
+    livekitRoom: LivekitRoom,
   ) {
-    super(id, member, participant, encryptionSystem);
+    super(id, member, participant, encryptionSystem, livekitRoom);
   }
 }
 
@@ -288,8 +469,9 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
     member: RoomMember | undefined,
     participant: RemoteParticipant,
     encryptionSystem: EncryptionSystem,
+    livekitRoom: LivekitRoom,
   ) {
-    super(id, member, participant, encryptionSystem);
+    super(id, member, participant, encryptionSystem, livekitRoom);
 
     // Sync the local volume with LiveKit
     this.localVolume
@@ -321,6 +503,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
     member: RoomMember | undefined,
     participant: LocalParticipant | RemoteParticipant,
     encryptionSystem: EncryptionSystem,
+    livekitRoom: LivekitRoom,
   ) {
     super(
       id,
@@ -329,6 +512,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
       encryptionSystem,
       Track.Source.ScreenShareAudio,
       Track.Source.ScreenShare,
+      livekitRoom,
     );
   }
 }

src/tile/GridTile.tsx

@@ -81,6 +81,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
     const { t } = useTranslation();
     const video = useObservableEagerState(vm.video);
     const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
+    const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
     const audioEnabled = useObservableEagerState(vm.audioEnabled);
     const videoEnabled = useObservableEagerState(vm.videoEnabled);
     const speaking = useObservableEagerState(vm.speaking);
@@ -122,6 +123,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         video={video}
         member={vm.member}
         unencryptedWarning={unencryptedWarning}
+        encryptionStatus={encryptionStatus}
         videoEnabled={videoEnabled}
         videoFit={cropVideo ? "cover" : "contain"}
         className={classNames(className, styles.tile, {

src/tile/MediaView.module.css

@@ -85,7 +85,7 @@ unconditionally select the container so we can use cqmin units */
   display: grid;
   grid-template-columns: 1fr auto;
   grid-template-rows: 1fr auto;
-  grid-template-areas: ". ." "nameTag button";
+  grid-template-areas: "status status" "nameTag button";
   gap: var(--cpd-space-1x);
   place-items: start;
 }
@@ -106,6 +106,25 @@ unconditionally select the container so we can use cqmin units */
   max-inline-size: 100%;
 }
 
+.status {
+  grid-area: status;
+  justify-self: center;
+  align-self: start;
+  padding: var(--cpd-space-1x);
+  padding-block: var(--cpd-space-1x);
+  color: var(--cpd-color-text-primary);
+  background-color: var(--cpd-color-bg-canvas-default);
+  display: flex;
+  align-items: center;
+  border-radius: var(--cpd-radius-pill-effect);
+  user-select: none;
+  overflow: hidden;
+  box-shadow: var(--small-drop-shadow);
+  box-sizing: border-box;
+  max-inline-size: 100%;
+  text-align: center;
+}
+
 .nameTag > svg,
 .nameTag > span {
   flex-shrink: 0;

src/tile/MediaView.tsx

@@ -17,6 +17,7 @@ import { ErrorIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
 
 import styles from "./MediaView.module.css";
 import { Avatar } from "../Avatar";
+import { EncryptionStatus } from "../state/MediaViewModel";
 import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
 import { showHandRaisedTimer, useSetting } from "../settings/settings";
 
@@ -31,6 +32,7 @@ interface Props extends ComponentProps<typeof animated.div> {
   member: RoomMember | undefined;
   videoEnabled: boolean;
   unencryptedWarning: boolean;
+  encryptionStatus: EncryptionStatus;
   nameTagLeadingIcon?: ReactNode;
   displayName: string;
   primaryButton?: ReactNode;
@@ -54,6 +56,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       nameTagLeadingIcon,
       displayName,
       primaryButton,
+      encryptionStatus,
       raisedHandTime,
       raisedHandOnClick,
       ...props
@@ -69,7 +72,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       <animated.div
         className={classNames(styles.media, className, {
           [styles.mirror]: mirror,
-          [styles.videoMuted]: !videoEnabled,
+          [styles.videoMuted]:
+            !videoEnabled ||
+            ![EncryptionStatus.Connecting, EncryptionStatus.Okay].includes(
+              encryptionStatus,
+            ),
         })}
         style={style}
         ref={ref}
@@ -95,6 +102,20 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
           )}
         </div>
         <div className={styles.fg}>
+          {encryptionStatus !== EncryptionStatus.Okay && (
+            <div className={styles.status}>
+              <Text as="span" size="sm" weight="medium" className={styles.name}>
+                {encryptionStatus === EncryptionStatus.Connecting &&
+                  t("e2ee_encryption_status.connecting")}
+                {encryptionStatus === EncryptionStatus.KeyMissing &&
+                  t("e2ee_encryption_status.key_missing")}
+                {encryptionStatus === EncryptionStatus.KeyInvalid &&
+                  t("e2ee_encryption_status.key_invalid")}
+                {encryptionStatus === EncryptionStatus.PasswordInvalid &&
+                  t("e2ee_encryption_status.password_invalid")}
+              </Text>
+            </div>
+          )}
           <RaisedHandIndicator
             raisedHandTime={raisedHandTime}
             minature={avatarSize < 96}

src/tile/SpotlightTile.tsx

@@ -31,6 +31,7 @@ import { RoomMember } from "matrix-js-sdk/src/matrix";
 import { MediaView } from "./MediaView";
 import styles from "./SpotlightTile.module.css";
 import {
+  EncryptionStatus,
   LocalUserMediaViewModel,
   MediaViewModel,
   ScreenShareViewModel,
@@ -51,6 +52,7 @@ interface SpotlightItemBaseProps {
   video: TrackReferenceOrPlaceholder;
   member: RoomMember | undefined;
   unencryptedWarning: boolean;
+  encryptionStatus: EncryptionStatus;
   displayName: string;
   "aria-hidden"?: boolean;
 }
@@ -132,6 +134,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
     const displayName = useDisplayName(vm);
     const video = useObservableEagerState(vm.video);
     const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
+    const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
 
     // Hook this item up to the intersection observer
     useEffect(() => {
@@ -158,6 +161,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
       member: vm.member,
       unencryptedWarning,
       displayName,
+      encryptionStatus,
       "aria-hidden": ariaHidden,
     };
 

src/utils/test.ts

@@ -4,7 +4,7 @@ Copyright 2023, 2024 New Vector Ltd.
 SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
-import { map } from "rxjs";
+import { map, Observable, of } from "rxjs";
 import { RunHelpers, TestScheduler } from "rxjs/testing";
 import { expect, vi } from "vitest";
 import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix";
@@ -99,8 +99,27 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
   return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
 }
 
-export function mockLivekitRoom(room: Partial<LivekitRoom>): LivekitRoom {
+export function mockLivekitRoom(
-  return { ...mockEmitter(), ...room } as Partial<LivekitRoom> as LivekitRoom;
+  room: Partial<LivekitRoom>,
+  {
+    remoteParticipants,
+  }: { remoteParticipants?: Observable<RemoteParticipant[]> } = {},
+): LivekitRoom {
+  const livekitRoom = {
+    ...mockEmitter(),
+    ...room,
+  } as Partial<LivekitRoom> as LivekitRoom;
+  if (remoteParticipants) {
+    livekitRoom.remoteParticipants = new Map();
+    remoteParticipants.subscribe((newRemoteParticipants) => {
+      livekitRoom.remoteParticipants.clear();
+      newRemoteParticipants.forEach((p) => {
+        livekitRoom.remoteParticipants.set(p.identity, p);
+      });
+    });
+  }
+
+  return livekitRoom;
 }
 
 export function mockLocalParticipant(
@@ -119,13 +138,15 @@ export async function withLocalMedia(
   member: Partial<RoomMember>,
   continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
+  const localParticipant = mockLocalParticipant({});
   const vm = new LocalUserMediaViewModel(
     "local",
     mockMember(member),
-    mockLocalParticipant({}),
+    localParticipant,
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
+    mockLivekitRoom({ localParticipant }),
   );
   try {
     await continuation(vm);
@@ -152,13 +173,15 @@ export async function withRemoteMedia(
   participant: Partial<RemoteParticipant>,
   continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
+  const remoteParticipant = mockRemoteParticipant(participant);
   const vm = new RemoteUserMediaViewModel(
     "remote",
     mockMember(member),
-    mockRemoteParticipant(participant),
+    remoteParticipant,
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
+    mockLivekitRoom({}, { remoteParticipants: of([remoteParticipant]) }),
   );
   try {
     await continuation(vm);