Add background Blur

locales/en-GB/app.json

@@ -146,6 +146,8 @@
       "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play",
       "effect_volume_label": "Sound effect volume"
     },
+    "background_blur_header": "Background",
+    "background_blur_label": "Enable background blurring",
     "developer_settings_label": "Developer Settings",
     "developer_settings_label_description": "Expose developer settings in the settings window.",
     "developer_tab_title": "Developer",
@@ -169,7 +171,8 @@
     "preferences_tab_h4": "Preferences",
     "preferences_tab_show_hand_raised_timer_description": "Show a timer when a participant raises their hand",
     "preferences_tab_show_hand_raised_timer_label": "Show hand raise duration",
-    "speaker_device_selection_label": "Speaker"
+    "speaker_device_selection_label": "Speaker",
+    "video_tab_activate_background_blur": "Turn on background blur on your webcam video"
   },
   "star_rating_input_label_one": "{{count}} stars",
   "star_rating_input_label_other": "{{count}} stars",

package.json

@@ -121,5 +121,8 @@
   },
   "resolutions": {
     "strip-ansi": "6.0.1"
+  },
+  "dependencies": {
+    "@livekit/track-processors": "^0.3.2"
   }
 }

src/livekit/useLiveKit.ts

@@ -9,7 +9,9 @@ import {
   ConnectionState,
   E2EEOptions,
   ExternalE2EEKeyProvider,
+  LocalTrackPublication,
   Room,
+  RoomEvent,
   RoomOptions,
   Track,
 } from "livekit-client";
@@ -17,6 +19,7 @@ import { useEffect, useMemo, useRef } from "react";
 import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import { logger } from "matrix-js-sdk/src/logger";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
+import { BackgroundBlur } from "@livekit/track-processors";
 
 import { defaultLiveKitOptions } from "./options";
 import { SFUConfig } from "./openIDSFU";
@@ -26,6 +29,7 @@ import {
   MediaDevices,
   useMediaDevices,
 } from "./MediaDevicesContext";
+import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
 import {
   ECConnectionState,
   useECConnectionState,
@@ -33,6 +37,7 @@ import {
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import { useSetting } from "../settings/settings";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
@@ -78,13 +83,16 @@ export function useLiveKit(
   const initialMuteStates = useRef<MuteStates>(muteStates);
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
-
+  // eslint-disable-next-line new-cap
+  const blur = useMemo(() => BackgroundBlur(15), []);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
+        // eslint-disable-next-line new-cap
+        processor: BackgroundBlur(15),
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -129,6 +137,51 @@ export function useLiveKit(
     sfuConfig,
   );
 
+  const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
+  const videoTrackPromise = useRef<
+    undefined | Promise<LocalTrackPublication | undefined>
+  >(undefined);
+
+  useEffect(() => {
+    if (!room || videoTrackPromise.current) return;
+    const update = async (): Promise<void> => {
+      let publishCallback: undefined | ((track: LocalTrackPublication) => void);
+      videoTrackPromise.current = new Promise<
+        LocalTrackPublication | undefined
+      >((resolve) => {
+        const videoTrack = Array.from(
+          room.localParticipant.videoTrackPublications.values(),
+        ).find((v) => v.source === Track.Source.Camera);
+        if (videoTrack) {
+          resolve(videoTrack);
+        }
+        publishCallback = (videoTrack: LocalTrackPublication): void => {
+          if (videoTrack.source === Track.Source.Camera) {
+            resolve(videoTrack);
+          }
+        };
+        room.on(RoomEvent.LocalTrackPublished, publishCallback);
+      });
+
+      const videoTrack = await videoTrackPromise.current;
+
+      if (publishCallback)
+        room.off(RoomEvent.LocalTrackPublished, publishCallback);
+
+      if (videoTrack !== undefined) {
+        if (showBackgroundBlur) {
+          logger.info("Blur: set blur");
+
+          void videoTrack.track?.setProcessor(blur);
+        } else {
+          void videoTrack.track?.stopProcessor();
+        }
+      }
+      videoTrackPromise.current = undefined;
+    };
+    void update();
+  }, [blur, room, showBackgroundBlur]);
+
   useEffect(() => {
     // Sync the requested mute states with LiveKit's mute states. We do it this
     // way around rather than using LiveKit as the source of truth, so that the

src/room/LobbyView.tsx

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { FC, useCallback, useMemo, useState } from "react";
+import { FC, useCallback, useEffect, useMemo, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { MatrixClient } from "matrix-js-sdk/src/matrix";
 import { Button } from "@vector-im/compound-web";
@@ -16,6 +16,7 @@ import { usePreviewTracks } from "@livekit/components-react";
 import { LocalVideoTrack, Track } from "livekit-client";
 import { useObservable } from "observable-hooks";
 import { map } from "rxjs";
+import { BackgroundBlur } from "@livekit/track-processors";
 
 import inCallStyles from "./InCallView.module.css";
 import styles from "./LobbyView.module.css";
@@ -32,12 +33,14 @@ import {
   VideoButton,
 } from "../button/Button";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
+import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
 import { useMediaQuery } from "../useMediaQuery";
 import { E2eeType } from "../e2ee/e2eeType";
 import { Link } from "../button/Link";
 import { useMediaDevices } from "../livekit/MediaDevicesContext";
 import { useInitial } from "../useInitial";
-import { useSwitchCamera } from "./useSwitchCamera";
+import { useSwitchCamera as useShowSwitchCamera } from "./useSwitchCamera";
+import { useSetting } from "../settings/settings";
 
 interface Props {
   client: MatrixClient;
@@ -108,6 +111,9 @@ export const LobbyView: FC<Props> = ({
       muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
   );
 
+  // eslint-disable-next-line new-cap
+  const blur = useMemo(() => BackgroundBlur(15), []);
+
   const localTrackOptions = useMemo(
     () => ({
       // The only reason we request audio here is to get the audio permission
@@ -119,12 +125,15 @@ export const LobbyView: FC<Props> = ({
       audio: Object.assign({}, initialAudioOptions),
       video: muteStates.video.enabled && {
         deviceId: devices.videoInput.selectedId,
+        // It should be possible to set a processor here:
+        // processor: blur,
+        // This causes a crash currently hence we do the effect below...
       },
     }),
     [
       initialAudioOptions,
-      devices.videoInput.selectedId,
       muteStates.video.enabled,
+      devices.videoInput.selectedId,
     ],
   );
 
@@ -146,7 +155,21 @@ export const LobbyView: FC<Props> = ({
     [tracks],
   );
 
-  const switchCamera = useSwitchCamera(
+  const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
+
+  useEffect(() => {
+    const updateBlur = async (showBlur: boolean): Promise<void> => {
+      if (showBlur && !videoTrack?.getProcessor()) {
+        // eslint-disable-next-line new-cap
+        await videoTrack?.setProcessor(blur);
+      } else {
+        await videoTrack?.stopProcessor();
+      }
+    };
+    if (videoTrack) void updateBlur(showBackgroundBlur);
+  }, [videoTrack, showBackgroundBlur, blur]);
+
+  const showSwitchCamera = useShowSwitchCamera(
     useObservable(
       (inputs) => inputs.pipe(map(([video]) => video)),
       [videoTrack],
@@ -208,7 +231,9 @@ export const LobbyView: FC<Props> = ({
               onClick={onVideoPress}
               disabled={muteStates.video.setEnabled === null}
             />
-            {switchCamera && <SwitchCameraButton onClick={switchCamera} />}
+            {showSwitchCamera && (
+              <SwitchCameraButton onClick={showSwitchCamera} />
+            )}
             <SettingsButton onClick={openSettings} />
             {!confineToRoom && <EndCallButton onClick={onLeaveClick} />}
           </div>

src/settings/SettingsModal.tsx

@@ -27,6 +27,7 @@ import {
   useSetting,
   developerSettingsTab as developerSettingsTabSetting,
   duplicateTiles as duplicateTilesSetting,
+  backgroundBlur as backgroundBlurSetting,
   useOptInAnalytics,
   soundEffectVolumeSetting,
 } from "./settings";
@@ -103,6 +104,25 @@ export const SettingsModal: FC<Props> = ({
       />
     );
   };
+  // Generate a `SelectInput` with a list of devices for a given device kind.
+  const BlurCheckbox: React.FC = (): ReactNode => {
+    const [blur, setBlur] = useSetting(backgroundBlurSetting);
+    return (
+      <>
+        <h4>{t("settings.background_blur_header")}</h4>
+        <FieldRow>
+          <InputField
+            id="activateBackgroundBlur"
+            label={t("settings.background_blur_label")}
+            description={t("settings.video_tab_activate_background_blur")}
+            type="checkbox"
+            checked={blur}
+            onChange={(b): void => setBlur(b.target.checked)}
+          />
+        </FieldRow>
+      </>
+    );
+  };
 
   const optInDescription = (
     <Text size="sm">
@@ -151,7 +171,13 @@ export const SettingsModal: FC<Props> = ({
   const videoTab: Tab<SettingsTab> = {
     key: "video",
     name: t("common.video"),
-    content: generateDeviceSelection(devices.videoInput, t("common.camera")),
+    content: (
+      <>
+        {generateDeviceSelection(devices.videoInput, t("common.camera"))}
+        <Separator />
+        <BlurCheckbox />
+      </>
+    ),
   };
 
   const preferencesTab: Tab<SettingsTab> = {

src/settings/settings.ts

@@ -88,6 +88,8 @@ export const videoInput = new Setting<string | undefined>(
   undefined,
 );
 
+export const backgroundBlur = new Setting<boolean>("background-blur", true);
+
 export const showHandRaisedTimer = new Setting<boolean>(
   "hand-raised-show-timer",
   false,

yarn.lock

@@ -1821,6 +1821,14 @@
   dependencies:
     "@bufbuild/protobuf" "^1.10.0"
 
+"@livekit/track-processors@^0.3.2":
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/@livekit/track-processors/-/track-processors-0.3.2.tgz#eaff6a48b556c25e85f5dd2c4daf6dcf1bc3b143"
+  integrity sha512-4JUCzb7yIKoVsTo8J6FTzLZJHcI6DihfX/pGRDg0SOGaxprcDPrt8jaDBBTsnGBSXHeMxl2ugN+xQjdCWzLKEA==
+  dependencies:
+    "@mediapipe/holistic" "0.5.1675471629"
+    "@mediapipe/tasks-vision" "0.10.9"
+
 "@matrix-org/matrix-sdk-crypto-wasm@^9.0.0":
   version "9.0.0"
   resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-9.0.0.tgz#293fe8fcb9bc4d577c5f6cf2cbffa151c6e11329"
@@ -1831,6 +1839,16 @@
   resolved "https://registry.yarnpkg.com/@matrix-org/olm/-/olm-3.2.15.tgz#55f3c1b70a21bbee3f9195cecd6846b1083451ec"
   integrity sha512-S7lOrndAK9/8qOtaTq/WhttJC/o4GAzdfK0MUPpo8ApzsJEC0QjtwrkC3KBXdFP1cD1MXi/mlKR7aaoVMKgs6Q==
 
+"@mediapipe/holistic@0.5.1675471629":
+  version "0.5.1675471629"
+  resolved "https://registry.yarnpkg.com/@mediapipe/holistic/-/holistic-0.5.1675471629.tgz#f1127d43161ff27e8889d5d39aaea164f9730980"
+  integrity sha512-qY+cxtDeSOvVtevrLgnodiwXYaAtPi7dHZtNv/bUCGEjFicAOYtMmrZSqMmbPkTB2+4jLnPF1vgshkAqQRSYAw==
+
+"@mediapipe/tasks-vision@0.10.9":
+  version "0.10.9"
+  resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.9.tgz#fbd669f50ac2e888b2c64c9c9863927c111da02f"
+  integrity sha512-/gFguyJm1ng4Qr7VVH2vKO+zZcQd8wc3YafUfvBuYFX0Y5+CvrV+VNPEVkl5W/gUZF5KNKNZAiaHPULGPCIjyQ==
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -6222,14 +6240,14 @@ matrix-js-sdk@matrix-org/matrix-js-sdk#8e9a04cdec0f88fc876bbbf406db55b0677f005d:
     jwt-decode "^4.0.0"
     loglevel "^1.7.1"
     matrix-events-sdk "0.0.1"
-    matrix-widget-api "^1.10.0"
+    matrix-widget-api "^1.8.2"
     oidc-client-ts "^3.0.1"
     p-retry "4"
     sdp-transform "^2.14.1"
     unhomoglyph "^1.0.6"
     uuid "11"
 
-matrix-widget-api@^1.10.0:
+matrix-widget-api@^1.10.0, matrix-widget-api@^1.8.2:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.10.0.tgz#d31ea073a5871a1fb1a511ef900b0c125a37bf55"
   integrity sha512-rkAJ29briYV7TJnfBVLVSKtpeBrBju15JZFSDP6wj8YdbCu1bdmlplJayQ+vYaw1x4fzI49Q+Nz3E85s46sRDw==