Matrix/element-call-Github
1
0
Fork 0
mirror of https://github.com/vector-im/element-call.git synced 2024-11-21 00:28:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add background Blur

Browse Source
This commit is contained in:
Timo 2024-11-20 17:22:24 +01:00
parent 826d0ee40d
commit 5e4d1692b8
7 changed files with 140 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
locales/en-GB/app.json
View File

@ -146,6 +146,8 @@
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
"effect_volume_label": "Sound effect volume"
},
"background_blur_header": "Background",
"background_blur_label": "Enable background blurring",
"developer_settings_label": "Developer Settings",
"developer_settings_label_description": "Expose developer settings in the settings window.",
"developer_tab_title": "Developer",
@ -169,7 +171,8 @@
"preferences_tab_h4": "Preferences",
"preferences_tab_show_hand_raised_timer_description": "Show a timer when a participant raises their hand",
"preferences_tab_show_hand_raised_timer_label": "Show hand raise duration",
"speaker_device_selection_label": "Speaker"
"speaker_device_selection_label": "Speaker",
"video_tab_activate_background_blur": "Turn on background blur on your webcam video"
},
"star_rating_input_label_one": "{{count}} stars",
"star_rating_input_label_other": "{{count}} stars",

3
package.json
View File

@ -121,5 +121,8 @@
},
"resolutions": {
"strip-ansi": "6.0.1"
},
"dependencies": {
"@livekit/track-processors": "^0.3.2"
}
}

55
src/livekit/useLiveKit.ts
View File

@ -9,7 +9,9 @@ import {
ConnectionState,
E2EEOptions,
ExternalE2EEKeyProvider,
LocalTrackPublication,
Room,
RoomEvent,
RoomOptions,
Track,
} from "livekit-client";
@ -17,6 +19,7 @@ import { useEffect, useMemo, useRef } from "react";
import E2EEWorker from "livekit-client/e2ee-worker?worker";
import { logger } from "matrix-js-sdk/src/logger";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import { BackgroundBlur } from "@livekit/track-processors";
import { defaultLiveKitOptions } from "./options";
import { SFUConfig } from "./openIDSFU";
@ -26,6 +29,7 @@ import {
MediaDevices,
useMediaDevices,
} from "./MediaDevicesContext";
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
import {
ECConnectionState,
useECConnectionState,
@ -33,6 +37,7 @@ import {
import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
import { E2eeType } from "../e2ee/e2eeType";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { useSetting } from "../settings/settings";
interface UseLivekitResult {
livekitRoom?: Room;
@ -78,13 +83,16 @@ export function useLiveKit(
const initialMuteStates = useRef<MuteStates>(muteStates);
const devices = useMediaDevices();
const initialDevices = useRef<MediaDevices>(devices);
// eslint-disable-next-line new-cap
const blur = useMemo(() => BackgroundBlur(15), []);
const roomOptions = useMemo(
(): RoomOptions => ({
...defaultLiveKitOptions,
videoCaptureDefaults: {
...defaultLiveKitOptions.videoCaptureDefaults,
deviceId: initialDevices.current.videoInput.selectedId,
// eslint-disable-next-line new-cap
processor: BackgroundBlur(15),
},
audioCaptureDefaults: {
...defaultLiveKitOptions.audioCaptureDefaults,
@ -129,6 +137,51 @@ export function useLiveKit(
sfuConfig,
);
const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
const videoTrackPromise = useRef<
undefined | Promise<LocalTrackPublication | undefined>
>(undefined);
useEffect(() => {
if (!room || videoTrackPromise.current) return;
const update = async (): Promise<void> => {
let publishCallback: undefined | ((track: LocalTrackPublication) => void);
videoTrackPromise.current = new Promise<
LocalTrackPublication | undefined
>((resolve) => {
const videoTrack = Array.from(
room.localParticipant.videoTrackPublications.values(),
).find((v) => v.source === Track.Source.Camera);
if (videoTrack) {
resolve(videoTrack);
}
publishCallback = (videoTrack: LocalTrackPublication): void => {
if (videoTrack.source === Track.Source.Camera) {
resolve(videoTrack);
}
};
room.on(RoomEvent.LocalTrackPublished, publishCallback);
});
const videoTrack = await videoTrackPromise.current;
if (publishCallback)
room.off(RoomEvent.LocalTrackPublished, publishCallback);
if (videoTrack !== undefined) {
if (showBackgroundBlur) {
logger.info("Blur: set blur");
void videoTrack.track?.setProcessor(blur);
} else {
void videoTrack.track?.stopProcessor();
}
}
videoTrackPromise.current = undefined;
};
void update();
}, [blur, room, showBackgroundBlur]);
useEffect(() => {
// Sync the requested mute states with LiveKit's mute states. We do it this
// way around rather than using LiveKit as the source of truth, so that the

35
src/room/LobbyView.tsx
View File

@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
Please see LICENSE in the repository root for full details.
*/
import { FC, useCallback, useMemo, useState } from "react";
import { FC, useCallback, useEffect, useMemo, useState } from "react";
import { useTranslation } from "react-i18next";
import { MatrixClient } from "matrix-js-sdk/src/matrix";
import { Button } from "@vector-im/compound-web";
@ -16,6 +16,7 @@ import { usePreviewTracks } from "@livekit/components-react";
import { LocalVideoTrack, Track } from "livekit-client";
import { useObservable } from "observable-hooks";
import { map } from "rxjs";
import { BackgroundBlur } from "@livekit/track-processors";
import inCallStyles from "./InCallView.module.css";
import styles from "./LobbyView.module.css";
@ -32,12 +33,14 @@ import {
VideoButton,
} from "../button/Button";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
import { useMediaQuery } from "../useMediaQuery";
import { E2eeType } from "../e2ee/e2eeType";
import { Link } from "../button/Link";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { useInitial } from "../useInitial";
import { useSwitchCamera } from "./useSwitchCamera";
import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
import { useSetting } from "../settings/settings";
interface Props {
client: MatrixClient;
@ -108,6 +111,9 @@ export const LobbyView: FC<Props> = ({
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);
// eslint-disable-next-line new-cap
const blur = useMemo(() => BackgroundBlur(15), []);
const localTrackOptions = useMemo(
() => ({
// The only reason we request audio here is to get the audio permission
@ -119,12 +125,15 @@ export const LobbyView: FC<Props> = ({
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
// It should be possible to set a processor here:
// processor: blur,
// This causes a crash currently hence we do the effect below...
},
}),
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
devices.videoInput.selectedId,
],
);
@ -146,7 +155,21 @@ export const LobbyView: FC<Props> = ({
[tracks],
);
const switchCamera = useSwitchCamera(
const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
useEffect(() => {
const updateBlur = async (showBlur: boolean): Promise<void> => {
if (showBlur && !videoTrack?.getProcessor()) {
// eslint-disable-next-line new-cap
await videoTrack?.setProcessor(blur);
} else {
await videoTrack?.stopProcessor();
}
};
if (videoTrack) void updateBlur(showBackgroundBlur);
}, [videoTrack, showBackgroundBlur, blur]);
const showSwitchCamera = useShowSwitchCamera(
useObservable(
(inputs) => inputs.pipe(map(([video]) => video)),
[videoTrack],
@ -208,7 +231,9 @@ export const LobbyView: FC<Props> = ({
onClick={onVideoPress}
disabled={muteStates.video.setEnabled === null}
/>
{switchCamera && <SwitchCameraButton onClick={switchCamera} />}
{showSwitchCamera && (
<SwitchCameraButton onClick={showSwitchCamera} />
)}
<SettingsButton onClick={openSettings} />
{!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
</div>

28
src/settings/SettingsModal.tsx
View File

@ -27,6 +27,7 @@ import {
useSetting,
developerSettingsTab as developerSettingsTabSetting,
duplicateTiles as duplicateTilesSetting,
backgroundBlur as backgroundBlurSetting,
useOptInAnalytics,
soundEffectVolumeSetting,
} from "./settings";
@ -103,6 +104,25 @@ export const SettingsModal: FC<Props> = ({
/>
);
};
// Generate a `SelectInput` with a list of devices for a given device kind.
const BlurCheckbox: React.FC = (): ReactNode => {
const [blur, setBlur] = useSetting(backgroundBlurSetting);
return (
<>
<h4>{t("settings.background_blur_header")}</h4>
<FieldRow>
<InputField
id="activateBackgroundBlur"
label={t("settings.background_blur_label")}
description={t("settings.video_tab_activate_background_blur")}
type="checkbox"
checked={blur}
onChange={(b): void => setBlur(b.target.checked)}
/>
</FieldRow>
</>
);
};
const optInDescription = (
<Text size="sm">
@ -151,7 +171,13 @@ export const SettingsModal: FC<Props> = ({
const videoTab: Tab<SettingsTab> = {
key: "video",
name: t("common.video"),
content: generateDeviceSelection(devices.videoInput, t("common.camera")),
content: (
<>
{generateDeviceSelection(devices.videoInput, t("common.camera"))}
<Separator />
<BlurCheckbox />
</>
),
};
const preferencesTab: Tab<SettingsTab> = {

2
src/settings/settings.ts
View File

@ -88,6 +88,8 @@ export const videoInput = new Setting<string | undefined>(
undefined,
);
export const backgroundBlur = new Setting<boolean>("background-blur", true);
export const showHandRaisedTimer = new Setting<boolean>(
"hand-raised-show-timer",
false,

22
yarn.lock
View File

@ -1821,6 +1821,14 @@
dependencies:
"@bufbuild/protobuf" "^1.10.0"
"@livekit/track-processors@^0.3.2":
version "0.3.2"
resolved "https://registry.yarnpkg.com/@livekit/track-processors/-/track-processors-0.3.2.tgz#eaff6a48b556c25e85f5dd2c4daf6dcf1bc3b143"
integrity sha512-4JUCzb7yIKoVsTo8J6FTzLZJHcI6DihfX/pGRDg0SOGaxprcDPrt8jaDBBTsnGBSXHeMxl2ugN+xQjdCWzLKEA==
dependencies:
"@mediapipe/holistic" "0.5.1675471629"
"@mediapipe/tasks-vision" "0.10.9"
"@matrix-org/matrix-sdk-crypto-wasm@^9.0.0":
version "9.0.0"
resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-9.0.0.tgz#293fe8fcb9bc4d577c5f6cf2cbffa151c6e11329"
@ -1831,6 +1839,16 @@
resolved "https://registry.yarnpkg.com/@matrix-org/olm/-/olm-3.2.15.tgz#55f3c1b70a21bbee3f9195cecd6846b1083451ec"
integrity sha512-S7lOrndAK9/8qOtaTq/WhttJC/o4GAzdfK0MUPpo8ApzsJEC0QjtwrkC3KBXdFP1cD1MXi/mlKR7aaoVMKgs6Q==
"@mediapipe/holistic@0.5.1675471629":
version "0.5.1675471629"
resolved "https://registry.yarnpkg.com/@mediapipe/holistic/-/holistic-0.5.1675471629.tgz#f1127d43161ff27e8889d5d39aaea164f9730980"
integrity sha512-qY+cxtDeSOvVtevrLgnodiwXYaAtPi7dHZtNv/bUCGEjFicAOYtMmrZSqMmbPkTB2+4jLnPF1vgshkAqQRSYAw==
"@mediapipe/tasks-vision@0.10.9":
version "0.10.9"
resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.9.tgz#fbd669f50ac2e888b2c64c9c9863927c111da02f"
integrity sha512-/gFguyJm1ng4Qr7VVH2vKO+zZcQd8wc3YafUfvBuYFX0Y5+CvrV+VNPEVkl5W/gUZF5KNKNZAiaHPULGPCIjyQ==
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@ -6222,14 +6240,14 @@ matrix-js-sdk@matrix-org/matrix-js-sdk#8e9a04cdec0f88fc876bbbf406db55b0677f005d:
jwt-decode "^4.0.0"
loglevel "^1.7.1"
matrix-events-sdk "0.0.1"
matrix-widget-api "^1.10.0"
matrix-widget-api "^1.8.2"
oidc-client-ts "^3.0.1"
p-retry "4"
sdp-transform "^2.14.1"
unhomoglyph "^1.0.6"
uuid "11"
matrix-widget-api@^1.10.0:
matrix-widget-api@^1.10.0, matrix-widget-api@^1.8.2:
version "1.10.0"
resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.10.0.tgz#d31ea073a5871a1fb1a511ef900b0c125a37bf55"
integrity sha512-rkAJ29briYV7TJnfBVLVSKtpeBrBju15JZFSDP6wj8YdbCu1bdmlplJayQ+vYaw1x4fzI49Q+Nz3E85s46sRDw==