Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2024-11-21 00:28:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Show encryption key status from LiveKit (#2700)

Browse Source 
* Refactor to make encryption system available in view models

* WIP show encryption errors from LiveKit

* Missing CSS

* Show encryption status based on LK and RTC

* Lint

* Lint

* Fix tests

* Update wording

* Refactor

* Lint
This commit is contained in:
Hugh Nimmo-Smith 2024-11-06 11:12:46 +00:00 committed by GitHub
parent bc0ab92394
commit c45f724279
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 287 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
public/locales/en-GB/app.json
View File

@ -64,6 +64,12 @@
"crypto_version": "Crypto version: {{version}}",
"device_id": "Device ID: {{id}}",
"disconnected_banner": "Connectivity to the server has been lost.",
"e2ee_encryption_status": {
"connecting": "Connecting...",
"key_invalid": "The end-to-end encrypted media key for this person is invalid",
"key_missing": "You haven't received the current end-to-end encrypted media key for this person yet",
"password_invalid": "This person is using a different password so you won't be able to communicate with them"
},
"full_screen_view_description": "<0>Submitting debug logs will help us track down the problem.</0>",
"full_screen_view_h1": "<0>Oops, something's gone wrong.</0>",
"group_call_loader": {

12
src/state/CallViewModel.test.ts
View File

@ -194,6 +194,15 @@ function withCallViewModel(
),
);
const roomEventSelectorSpy = vi
.spyOn(ComponentsCore, "roomEventSelector")
.mockImplementation((room, eventType) => of());
const liveKitRoom = mockLivekitRoom(
{ localParticipant },
{ remoteParticipants },
);
const vm = new CallViewModel(
mockMatrixRoom({
client: {
@ -201,7 +210,7 @@ function withCallViewModel(
} as Partial<MatrixClient> as MatrixClient,
getMember: (userId) => members.get(userId) ?? null,
}),
mockLivekitRoom({ localParticipant }),
liveKitRoom,
{
kind: E2eeType.PER_PARTICIPANT,
},
@ -213,6 +222,7 @@ function withCallViewModel(
participantsSpy!.mockRestore();
mediaSpy!.mockRestore();
eventsSpy!.mockRestore();
roomEventSelectorSpy!.mockRestore();
});
continuation(vm);

7
src/state/CallViewModel.ts
View File

@ -226,6 +226,7 @@ class UserMedia {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
this.vm = participant.isLocal
? new LocalUserMediaViewModel(
@ -233,12 +234,14 @@ class UserMedia {
member,
participant as LocalParticipant,
encryptionSystem,
livekitRoom,
)
: new RemoteUserMediaViewModel(
id,
member,
participant as RemoteParticipant,
encryptionSystem,
livekitRoom,
);
this.speaker = this.vm.speaking.pipe(
@ -282,12 +285,14 @@ class ScreenShare {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
liveKitRoom: LivekitRoom,
) {
this.vm = new ScreenShareViewModel(
id,
member,
participant,
encryptionSystem,
liveKitRoom,
);
}
@ -428,6 +433,7 @@ export class CallViewModel extends ViewModel {
member,
p,
this.encryptionSystem,
this.livekitRoom,
),
];
@ -441,6 +447,7 @@ export class CallViewModel extends ViewModel {
member,
p,
this.encryptionSystem,
this.livekitRoom,
),
];
}

190
src/state/MediaViewModel.ts
View File

@ -11,6 +11,7 @@ import {
VideoSource,
observeParticipantEvents,
observeParticipantMedia,
roomEventSelector,
} from "@livekit/components-core";
import {
LocalParticipant,
@ -21,6 +22,9 @@ import {
Track,
TrackEvent,
facingModeFromLocalTrack,
Room as LivekitRoom,
RoomEvent as LivekitRoomEvent,
RemoteTrack,
} from "livekit-client";
import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix";
import {
@ -28,13 +32,18 @@ import {
Observable,
Subject,
combineLatest,
distinctUntilChanged,
distinctUntilKeyChanged,
filter,
fromEvent,
interval,
map,
merge,
of,
shareReplay,
startWith,
switchMap,
throttleTime,
} from "rxjs";
import { useEffect } from "react";
@ -81,6 +90,115 @@ export function observeTrackReference(
);
}
function observeRemoteTrackReceivingOkay(
participant: Participant,
source: Track.Source,
): Observable<boolean | undefined> {
let lastStats: {
framesDecoded: number | undefined;
framesDropped: number | undefined;
framesReceived: number | undefined;
} = {
framesDecoded: undefined,
framesDropped: undefined,
framesReceived: undefined,
};
return combineLatest([
observeTrackReference(participant, source),
interval(1000).pipe(startWith(0)),
]).pipe(
switchMap(async ([trackReference]) => {
const track = trackReference.publication?.track;
if (!track || !(track instanceof RemoteTrack)) {
return undefined;
}
const report = await track.getRTCStatsReport();
if (!report) {
return undefined;
}
for (const v of report.values()) {
if (v.type === "inbound-rtp") {
const { framesDecoded, framesDropped, framesReceived } =
v as RTCInboundRtpStreamStats;
return {
framesDecoded,
framesDropped,
framesReceived,
};
}
}
return undefined;
}),
filter((newStats) => !!newStats),
map((newStats): boolean | undefined => {
const oldStats = lastStats;
lastStats = newStats;
if (
typeof newStats.framesReceived === "number" &&
typeof oldStats.framesReceived === "number" &&
typeof newStats.framesDecoded === "number" &&
typeof oldStats.framesDecoded === "number"
) {
const framesReceivedDelta =
newStats.framesReceived - oldStats.framesReceived;
const framesDecodedDelta =
newStats.framesDecoded - oldStats.framesDecoded;
// if we received >0 frames and managed to decode >0 frames then we treat that as success
if (framesReceivedDelta > 0) {
return framesDecodedDelta > 0;
}
}
// no change
return undefined;
}),
filter((x) => typeof x === "boolean"),
startWith(undefined),
);
}
function encryptionErrorObservable(
room: LivekitRoom,
participant: Participant,
encryptionSystem: EncryptionSystem,
criteria: string,
): Observable<boolean> {
return roomEventSelector(room, LivekitRoomEvent.EncryptionError).pipe(
map((e) => {
const [err] = e;
if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
return (
// Ideally we would pull the participant identity from the field on the error.
// However, it gets lost in the serialization process between workers.
// So, instead we do a string match
(err?.message.includes(participant.identity) &&
err?.message.includes(criteria)) ??
false
);
} else if (encryptionSystem.kind === E2eeType.SHARED_KEY) {
return !!err?.message.includes(criteria);
}
return false;
}),
throttleTime(1000), // Throttle to avoid spamming the UI
startWith(false),
);
}
export enum EncryptionStatus {
Connecting,
Okay,
KeyMissing,
KeyInvalid,
PasswordInvalid,
}
abstract class BaseMediaViewModel extends ViewModel {
/**
* Whether the media belongs to the local user.
@ -95,6 +213,8 @@ abstract class BaseMediaViewModel extends ViewModel {
*/
public readonly unencryptedWarning: Observable<boolean>;
public readonly encryptionStatus: Observable<EncryptionStatus>;
public constructor(
/**
* An opaque identifier for this media.
@ -110,6 +230,7 @@ abstract class BaseMediaViewModel extends ViewModel {
encryptionSystem: EncryptionSystem,
audioSource: AudioSource,
videoSource: VideoSource,
livekitRoom: LivekitRoom,
) {
super();
const audio = observeTrackReference(participant, audioSource).pipe(
@ -124,7 +245,64 @@ abstract class BaseMediaViewModel extends ViewModel {
encryptionSystem.kind !== E2eeType.NONE &&
(a.publication?.isEncrypted === false ||
v.publication?.isEncrypted === false),
).pipe(this.scope.state());
).pipe(distinctUntilChanged(), shareReplay(1));
if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
this.scope.state(),
);
} else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"MissingKey",
),
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
if (keyMissing) return EncryptionStatus.KeyMissing;
if (keyInvalid) return EncryptionStatus.KeyInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
}),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
} else {
this.encryptionStatus = combineLatest([
encryptionErrorObservable(
livekitRoom,
participant,
encryptionSystem,
"InvalidKey",
),
observeRemoteTrackReceivingOkay(participant, audioSource),
observeRemoteTrackReceivingOkay(participant, videoSource),
]).pipe(
map(
([keyInvalid, audioOkay, videoOkay]):
| EncryptionStatus
| undefined => {
if (keyInvalid) return EncryptionStatus.PasswordInvalid;
if (audioOkay || videoOkay) return EncryptionStatus.Okay;
return undefined; // no change
},
),
filter((x) => !!x),
startWith(EncryptionStatus.Connecting),
this.scope.state(),
);
}
}
}
@ -171,6 +349,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(
id,
@ -179,6 +358,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
encryptionSystem,
Track.Source.Microphone,
Track.Source.Camera,
livekitRoom,
);
const media = observeParticipantMedia(participant).pipe(this.scope.state());
@ -228,8 +408,9 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
member: RoomMember | undefined,
participant: LocalParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem);
super(id, member, participant, encryptionSystem, livekitRoom);
}
}
@ -288,8 +469,9 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
member: RoomMember | undefined,
participant: RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(id, member, participant, encryptionSystem);
super(id, member, participant, encryptionSystem, livekitRoom);
// Sync the local volume with LiveKit
this.localVolume
@ -321,6 +503,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
encryptionSystem: EncryptionSystem,
livekitRoom: LivekitRoom,
) {
super(
id,
@ -329,6 +512,7 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
encryptionSystem,
Track.Source.ScreenShareAudio,
Track.Source.ScreenShare,
livekitRoom,
);
}
}

2
src/tile/GridTile.tsx
View File

@ -81,6 +81,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
const { t } = useTranslation();
const video = useObservableEagerState(vm.video);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
const audioEnabled = useObservableEagerState(vm.audioEnabled);
const videoEnabled = useObservableEagerState(vm.videoEnabled);
const speaking = useObservableEagerState(vm.speaking);
@ -122,6 +123,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
video={video}
member={vm.member}
unencryptedWarning={unencryptedWarning}
encryptionStatus={encryptionStatus}
videoEnabled={videoEnabled}
videoFit={cropVideo ? "cover" : "contain"}
className={classNames(className, styles.tile, {

21
src/tile/MediaView.module.css
View File

@ -85,7 +85,7 @@ unconditionally select the container so we can use cqmin units */
display: grid;
grid-template-columns: 1fr auto;
grid-template-rows: 1fr auto;
grid-template-areas: ". ." "nameTag button";
grid-template-areas: "status status" "nameTag button";
gap: var(--cpd-space-1x);
place-items: start;
}
@ -106,6 +106,25 @@ unconditionally select the container so we can use cqmin units */
max-inline-size: 100%;
}
.status {
grid-area: status;
justify-self: center;
align-self: start;
padding: var(--cpd-space-1x);
padding-block: var(--cpd-space-1x);
color: var(--cpd-color-text-primary);
background-color: var(--cpd-color-bg-canvas-default);
display: flex;
align-items: center;
border-radius: var(--cpd-radius-pill-effect);
user-select: none;
overflow: hidden;
box-shadow: var(--small-drop-shadow);
box-sizing: border-box;
max-inline-size: 100%;
text-align: center;
}
.nameTag > svg,
.nameTag > span {
flex-shrink: 0;

23
src/tile/MediaView.tsx
View File

@ -17,6 +17,7 @@ import { ErrorIcon } from "@vector-im/compound-design-tokens/assets/web/icons";
import styles from "./MediaView.module.css";
import { Avatar } from "../Avatar";
import { EncryptionStatus } from "../state/MediaViewModel";
import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
import { showHandRaisedTimer, useSetting } from "../settings/settings";
@ -31,6 +32,7 @@ interface Props extends ComponentProps<typeof animated.div> {
member: RoomMember | undefined;
videoEnabled: boolean;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
nameTagLeadingIcon?: ReactNode;
displayName: string;
primaryButton?: ReactNode;
@ -54,6 +56,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
nameTagLeadingIcon,
displayName,
primaryButton,
encryptionStatus,
raisedHandTime,
raisedHandOnClick,
...props
@ -69,7 +72,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
<animated.div
className={classNames(styles.media, className, {
[styles.mirror]: mirror,
[styles.videoMuted]: !videoEnabled,
[styles.videoMuted]:
!videoEnabled ||
![EncryptionStatus.Connecting, EncryptionStatus.Okay].includes(
encryptionStatus,
),
})}
style={style}
ref={ref}
@ -95,6 +102,20 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
)}
</div>
<div className={styles.fg}>
{encryptionStatus !== EncryptionStatus.Okay && (
<div className={styles.status}>
<Text as="span" size="sm" weight="medium" className={styles.name}>
{encryptionStatus === EncryptionStatus.Connecting &&
t("e2ee_encryption_status.connecting")}
{encryptionStatus === EncryptionStatus.KeyMissing &&
t("e2ee_encryption_status.key_missing")}
{encryptionStatus === EncryptionStatus.KeyInvalid &&
t("e2ee_encryption_status.key_invalid")}
{encryptionStatus === EncryptionStatus.PasswordInvalid &&
t("e2ee_encryption_status.password_invalid")}
</Text>
</div>
)}
<RaisedHandIndicator
raisedHandTime={raisedHandTime}
minature={avatarSize < 96}

4
src/tile/SpotlightTile.tsx
View File

@ -31,6 +31,7 @@ import { RoomMember } from "matrix-js-sdk/src/matrix";
import { MediaView } from "./MediaView";
import styles from "./SpotlightTile.module.css";
import {
EncryptionStatus,
LocalUserMediaViewModel,
MediaViewModel,
ScreenShareViewModel,
@ -51,6 +52,7 @@ interface SpotlightItemBaseProps {
video: TrackReferenceOrPlaceholder;
member: RoomMember | undefined;
unencryptedWarning: boolean;
encryptionStatus: EncryptionStatus;
displayName: string;
"aria-hidden"?: boolean;
}
@ -132,6 +134,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
const displayName = useDisplayName(vm);
const video = useObservableEagerState(vm.video);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
// Hook this item up to the intersection observer
useEffect(() => {
@ -158,6 +161,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
member: vm.member,
unencryptedWarning,
displayName,
encryptionStatus,
"aria-hidden": ariaHidden,
};

33
src/utils/test.ts
View File

@ -4,7 +4,7 @@ Copyright 2023, 2024 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { map } from "rxjs";
import { map, Observable, of } from "rxjs";
import { RunHelpers, TestScheduler } from "rxjs/testing";
import { expect, vi } from "vitest";
import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix";
@ -99,8 +99,27 @@ export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
return { ...mockEmitter(), ...room } as Partial<MatrixRoom> as MatrixRoom;
}
export function mockLivekitRoom(room: Partial<LivekitRoom>): LivekitRoom {
return { ...mockEmitter(), ...room } as Partial<LivekitRoom> as LivekitRoom;
export function mockLivekitRoom(
room: Partial<LivekitRoom>,
{
remoteParticipants,
}: { remoteParticipants?: Observable<RemoteParticipant[]> } = {},
): LivekitRoom {
const livekitRoom = {
...mockEmitter(),
...room,
} as Partial<LivekitRoom> as LivekitRoom;
if (remoteParticipants) {
livekitRoom.remoteParticipants = new Map();
remoteParticipants.subscribe((newRemoteParticipants) => {
livekitRoom.remoteParticipants.clear();
newRemoteParticipants.forEach((p) => {
livekitRoom.remoteParticipants.set(p.identity, p);
});
});
}
return livekitRoom;
}
export function mockLocalParticipant(
@ -119,13 +138,15 @@ export async function withLocalMedia(
member: Partial<RoomMember>,
continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const localParticipant = mockLocalParticipant({});
const vm = new LocalUserMediaViewModel(
"local",
mockMember(member),
mockLocalParticipant({}),
localParticipant,
{
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({ localParticipant }),
);
try {
await continuation(vm);
@ -152,13 +173,15 @@ export async function withRemoteMedia(
participant: Partial<RemoteParticipant>,
continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
): Promise<void> {
const remoteParticipant = mockRemoteParticipant(participant);
const vm = new RemoteUserMediaViewModel(
"remote",
mockMember(member),
mockRemoteParticipant(participant),
remoteParticipant,
{
kind: E2eeType.PER_PARTICIPANT,
},
mockLivekitRoom({}, { remoteParticipants: of([remoteParticipant]) }),
);
try {
await continuation(vm);