refactor(core-html5): custom hooks for voice data

bbb-graphql-server/bbb_schema.sql

@@ -664,8 +664,12 @@ select
 	"user_voice"."talking",
     "user_voice"."startTime",
     "user_voice"."endTime",
-	"user_voice"."voiceActivityAt"
+	"user_voice"."voiceActivityAt",
+    "user"."color",
+    "user"."name",
+    "user"."speechLocale"
 FROM "user_voice"
+JOIN "user" ON "user"."userId" = "user_voice"."userId"
 WHERE "voiceActivityAt" is not null
 AND --filter recent activities to avoid receiving all history every time it starts the streming
     ("voiceActivityAt" > current_timestamp - '10 seconds'::interval

bbb-graphql-server/metadata/databases/BigBlueButton/tables/public_v_user_voice_activity.yaml

@@ -16,6 +16,9 @@ select_permissions:
         - talking
         - userId
         - voiceActivityAt
+        - color
+        - name
+        - speechLocale
       filter:
         meetingId:
           _eq: X-Hasura-MeetingId

bigbluebutton-html5/imports/ui/components/actions-bar/reactions-button/container.jsx

@@ -7,6 +7,7 @@ import Auth from '/imports/ui/services/auth';
 import useCurrentUser from '/imports/ui/core/hooks/useCurrentUser';
 import useSettings from '/imports/ui/services/settings/hooks/useSettings';
 import { SETTINGS } from '/imports/ui/services/settings/enums';
+import useWhoIsUnmuted from '/imports/ui/core/hooks/useWhoIsUnmuted';
 
 const ReactionsButtonContainer = ({ ...props }) => {
   const layoutContextDispatch = layoutDispatch();
@@ -23,13 +24,14 @@ const ReactionsButtonContainer = ({ ...props }) => {
     voice: user.voice,
     reactionEmoji: user.reactionEmoji,
   }));
+  const { data: unmutedUsers } = useWhoIsUnmuted();
 
   const currentUser = {
     userId: Auth.userID,
     emoji: currentUserData?.emoji,
     raiseHand: currentUserData?.raiseHand,
     away: currentUserData?.away,
-    muted: currentUserData?.voice?.muted || false,
+    muted: !unmutedUsers.has(Auth.userID),
   };
 
   const { autoCloseReactionsBar } = useSettings(SETTINGS.APPLICATION);

bigbluebutton-html5/imports/ui/components/audio/audio-graphql/audio-controls/input-stream-live-selector/component.tsx

@@ -18,6 +18,8 @@ import MutedAlert from '/imports/ui/components/muted-alert/component';
 import MuteToggle from './buttons/muteToggle';
 import ListenOnly from './buttons/listenOnly';
 import LiveSelection from './buttons/LiveSelection';
+import useWhoIsTalking from '/imports/ui/core/hooks/useWhoIsTalking';
+import useWhoIsUnmuted from '/imports/ui/core/hooks/useWhoIsUnmuted';
 
 const AUDIO_INPUT = 'audioinput';
 const AUDIO_OUTPUT = 'audiooutput';
@@ -230,13 +232,16 @@ const InputStreamLiveSelectorContainer: React.FC = () => {
       locked: u?.locked ?? false,
       away: u?.away,
       voice: {
-        muted: u?.voice?.muted ?? false,
         listenOnly: u?.voice?.listenOnly ?? false,
-        talking: u?.voice?.talking ?? false,
       },
     };
   });
 
+  const { data: talkingUsers } = useWhoIsTalking();
+  const { data: unmutedUsers } = useWhoIsUnmuted();
+  const talking = Boolean(currentUser?.userId && talkingUsers[currentUser.userId]);
+  const muted = Boolean(currentUser?.userId && !unmutedUsers.has(currentUser.userId));
+
   const { data: currentMeeting } = useMeeting((m: Partial<Meeting>) => {
     return {
       lockSettings: m?.lockSettings,
@@ -264,8 +269,8 @@ const InputStreamLiveSelectorContainer: React.FC = () => {
       isAudioLocked={(!currentUser?.isModerator && currentUser?.locked
         && currentMeeting?.lockSettings?.disableMic) ?? false}
       listenOnly={currentUser?.voice?.listenOnly ?? false}
-      muted={currentUser?.voice?.muted ?? false}
-      talking={currentUser?.voice?.talking ?? false}
+      muted={muted}
+      talking={talking}
       inAudio={!!currentUser?.voice ?? false}
       showMute={(!!currentUser?.voice && !currentMeeting?.lockSettings?.disableMic) ?? false}
       isConnected={isConnected}

bigbluebutton-html5/imports/ui/components/audio/audio-graphql/hooks/useMuteMicrophone.ts

@@ -3,16 +3,15 @@ import useCurrentUser from '/imports/ui/core/hooks/useCurrentUser';
 import logger from '/imports/startup/client/logger';
 import AudioManager from '/imports/ui/services/audio-manager';
 import useToggleVoice from './useToggleVoice';
+import useWhoIsUnmuted from '/imports/ui/core/hooks/useWhoIsUnmuted';
 
 const useMuteMicrophone = () => {
   const { data: currentUser } = useCurrentUser((u) => ({
     userId: u.userId,
-    voice: {
-      muted: u.voice?.muted,
-    },
   }));
   const toggleVoice = useToggleVoice();
-  const muted = !!currentUser?.voice?.muted;
+  const { data: unmutedUsers } = useWhoIsUnmuted();
+  const muted = currentUser?.userId && !unmutedUsers.has(currentUser?.userId);
   const userId = currentUser?.userId ?? '';
 
   return useCallback(() => {

bigbluebutton-html5/imports/ui/components/audio/audio-graphql/hooks/useToggleVoice.ts

@@ -1,16 +1,10 @@
 import { useCallback } from 'react';
 import { useMutation } from '@apollo/client';
 import { USER_SET_MUTED } from '../mutations';
-import useCurrentUser from '/imports/ui/core/hooks/useCurrentUser';
 import logger from '/imports/startup/client/logger';
 
 const useToggleVoice = () => {
   const [userSetMuted] = useMutation(USER_SET_MUTED);
-  const { data: currentUserData } = useCurrentUser((u) => ({
-    voice: {
-      muted: u.voice?.muted,
-    },
-  }));
 
   const toggleVoice = async (userId: string, muted: boolean) => {
     try {
@@ -20,7 +14,7 @@ const useToggleVoice = () => {
     }
   };
 
-  return useCallback(toggleVoice, [currentUserData?.voice?.muted]);
+  return useCallback(toggleVoice, [userSetMuted]);
 };
 
 export default useToggleVoice;

bigbluebutton-html5/imports/ui/components/audio/audio-graphql/queries.ts

@@ -4,7 +4,6 @@ interface VoiceUsers {
   joined: string | null;
   listenOnly: string | null;
   muted: string | null;
-  talking: string | null;
   userId: string;
 }
 
@@ -17,8 +16,6 @@ export const VOICE_USERS_SUBSCRIPTION = gql`
     user_voice {
       joined
       listenOnly
-      muted
-      talking
       userId
     }
   }

bigbluebutton-html5/imports/ui/components/audio/container.jsx

@@ -25,6 +25,7 @@ import { SETTINGS } from '../../services/settings/enums';
 import { useStorageKey } from '../../services/storage/hooks';
 import { BREAKOUT_COUNT } from './queries';
 import useMeeting from '../../core/hooks/useMeeting';
+import useWhoIsUnmuted from '../../core/hooks/useWhoIsUnmuted';
 
 const intlMessages = defineMessages({
   joinedAudio: {
@@ -173,7 +174,9 @@ const AudioContainer = (props) => {
   const { hasBreakoutRooms: hadBreakoutRooms } = prevProps || {};
   const userIsReturningFromBreakoutRoom = hadBreakoutRooms && !hasBreakoutRooms;
 
-  const { data: currentUserMuted } = useCurrentUser((u) => u?.voice?.muted ?? false);
+  const { data: currentUser } = useCurrentUser((u) => ({ userId: u.userId }));
+  const { data: unmutedUsers } = useWhoIsUnmuted();
+  const currentUserMuted = currentUser?.userId && !unmutedUsers.has(currentUser.userId);
 
   const joinAudio = () => {
     if (Service.isConnected()) return;

bigbluebutton-html5/imports/ui/components/nav-bar/nav-bar-graphql/talking-indicator/component.tsx

@@ -4,20 +4,16 @@ import {
   IsBreakoutSubscriptionData,
   MEETING_ISBREAKOUT_SUBSCRIPTION,
 } from './queries';
-import { UserVoice } from '/imports/ui/Types/userVoice';
 import { uniqueId } from '/imports/utils/string-utils';
 import Styled from './styles';
 import { User } from '/imports/ui/Types/user';
 import useCurrentUser from '/imports/ui/core/hooks/useCurrentUser';
 import { muteUser } from './service';
 import useToggleVoice from '../../../audio/audio-graphql/hooks/useToggleVoice';
-import TALKING_INDICATOR_SUBSCRIPTION from '/imports/ui/core/graphql/queries/userVoiceSubscription';
 import { setTalkingIndicatorList } from '/imports/ui/core/hooks/useTalkingIndicator';
 import useDeduplicatedSubscription from '/imports/ui/core/hooks/useDeduplicatedSubscription';
-
-interface TalkingIndicatorSubscriptionData {
-  user_voice: Array<Partial<UserVoice>>;
-}
+import useVoiceActivity from '/imports/ui/core/hooks/useVoiceActivity';
+import { VoiceActivityResponse } from '/imports/ui/core/graphql/queries/whoIsTalking';
 
 const TALKING_INDICATORS_MAX = 8;
 
@@ -49,7 +45,7 @@ const intlMessages = defineMessages({
 });
 
 interface TalkingIndicatorProps {
-  talkingUsers: Array<Partial<UserVoice>>;
+  talkingUsers: Array<VoiceActivityResponse['user_voice_activity_stream'][number]>;
   isBreakout: boolean;
   moreThanMaxIndicators: boolean;
   isModerator: boolean;
@@ -70,14 +66,15 @@ const TalkingIndicator: React.FC<TalkingIndicatorProps> = ({
       setTalkingIndicatorList([]);
     };
   }, []);
-  const talkingElements = useMemo(() => talkingUsers.map((talkingUser: Partial<UserVoice>) => {
+  const talkingElements = useMemo(() => talkingUsers.map((talkingUser) => {
     const {
       talking,
       muted,
-      user: { color, speechLocale } = {} as Partial<User>,
+      color,
+      speechLocale,
+      name,
     } = talkingUser;
 
-    const name = talkingUser.user?.name;
     const ariaLabel = intl.formatMessage(talking
       ? intlMessages.isTalking : intlMessages.wasTalking, {
       0: name,
@@ -182,19 +179,6 @@ const TalkingIndicatorContainer: React.FC = (() => {
       isModerator: u?.isModerator,
     }));
 
-    const {
-      data: talkingIndicatorData,
-      loading: talkingIndicatorLoading,
-      error: talkingIndicatorError,
-    } = useDeduplicatedSubscription<TalkingIndicatorSubscriptionData>(
-      TALKING_INDICATOR_SUBSCRIPTION,
-      {
-        variables: {
-          limit: TALKING_INDICATORS_MAX,
-        },
-      },
-    );
-
     const {
       data: isBreakoutData,
       loading: isBreakoutLoading,
@@ -202,19 +186,31 @@ const TalkingIndicatorContainer: React.FC = (() => {
     } = useDeduplicatedSubscription<IsBreakoutSubscriptionData>(MEETING_ISBREAKOUT_SUBSCRIPTION);
 
     const toggleVoice = useToggleVoice();
+    const {
+      data: voiceActivity,
+      loading: voiceActivityLoading,
+      error: voiceActivityError,
+    } = useVoiceActivity();
+    const talkingUsers = useMemo(() => Object.values(voiceActivity)
+      .filter((v) => v.showTalkingIndicator)
+      .sort((v1, v2) => {
+        if (!v1.startTime && !v2.startTime) return 0;
+        if (!v1.startTime) return 1;
+        if (!v2.startTime) return -1;
+        return v2.startTime - v1.startTime;
+      }).slice(0, TALKING_INDICATORS_MAX), [voiceActivity]);
 
-    if (talkingIndicatorLoading || isBreakoutLoading) return null;
+    if (voiceActivityLoading || isBreakoutLoading) return null;
 
-    if (talkingIndicatorError || isBreakoutError) {
+    if (voiceActivityError || isBreakoutError) {
       return (
         <div>
           error:
-          { JSON.stringify(talkingIndicatorError || isBreakoutError) }
+          { JSON.stringify(voiceActivityError || isBreakoutError) }
         </div>
       );
     }
 
-    const talkingUsers = talkingIndicatorData?.user_voice ?? [];
     const isBreakout = isBreakoutData?.meeting[0]?.isBreakout ?? false;
     setTalkingIndicatorList(talkingUsers);
     return (

bigbluebutton-html5/imports/ui/components/user-list/user-list-content/user-participants/user-list-participants/list-item/component.tsx

@@ -17,6 +17,8 @@ import normalizeEmojiName from './service';
 import { convertRemToPixels } from '/imports/utils/dom-utils';
 import { PluginsContext } from '/imports/ui/components/components-data/plugin-context/context';
 import { useIsReactionsEnabled } from '/imports/ui/services/features';
+import useWhoIsTalking from '/imports/ui/core/hooks/useWhoIsTalking';
+import useWhoIsUnmuted from '/imports/ui/core/hooks/useWhoIsUnmuted';
 
 const messages = defineMessages({
   moderator: {
@@ -100,7 +102,13 @@ const UserListItem: React.FC<UserListItemProps> = ({ user, lockSettings }) => {
   }
 
   const intl = useIntl();
-  const voiceUser = user.voice;
+  const { data: talkingUsers } = useWhoIsTalking();
+  const { data: unmutedUsers } = useWhoIsUnmuted();
+  const voiceUser = {
+    ...user.voice,
+    talking: talkingUsers[user.userId],
+    muted: !unmutedUsers.has(user.userId),
+  };
   const subs = [];
 
   const LABEL = window.meetingClientSettings.public.user.label;

bigbluebutton-html5/imports/ui/components/video-provider/video-list/video-list-item/container.tsx

@@ -9,6 +9,8 @@ import useSettings from '/imports/ui/services/settings/hooks/useSettings';
 import { SETTINGS } from '/imports/ui/services/settings/enums';
 import { useStorageKey } from '/imports/ui/services/storage/hooks';
 import useVoiceUsers from '/imports/ui/components/audio/audio-graphql/hooks/useVoiceUsers';
+import useWhoIsTalking from '/imports/ui/core/hooks/useWhoIsTalking';
+import useWhoIsUnmuted from '/imports/ui/core/hooks/useWhoIsUnmuted';
 
 interface VideoListItemContainerProps {
   numOfStreams: number;
@@ -55,13 +57,18 @@ const VideoListItemContainer: React.FC<VideoListItemContainerProps> = (props) =>
 
   const disabledCams = useStorageKey('disabledCams') || [];
   const voiceUsers = useVoiceUsers((v) => ({
-    muted: v.muted,
     listenOnly: v.listenOnly,
-    talking: v.talking,
     joined: v.joined,
     userId: v.userId,
   }));
-  const voiceUser = voiceUsers.data?.find((v) => v.userId === userId);
+  const { data: talkingUsers } = useWhoIsTalking();
+  const { data: unmutedUsers } = useWhoIsUnmuted();
+  const voiceUser = voiceUsers.data?.find((v) => v.userId === userId) ?? {};
+  const voiceData = {
+    ...voiceUser,
+    talking: talkingUsers[userId],
+    muted: !unmutedUsers.has(userId),
+  };
 
   return (
     <VideoListItem
@@ -83,7 +90,7 @@ const VideoListItemContainer: React.FC<VideoListItemContainerProps> = (props) =>
       onVirtualBgDrop={onVirtualBgDrop}
       settingsSelfViewDisable={settingsSelfViewDisable}
       stream={stream}
-      voiceUser={voiceUser}
+      voiceUser={voiceData}
     />
   );
 };

bigbluebutton-html5/imports/ui/core/graphql/queries/userVoiceSubscription.ts

@@ -9,7 +9,6 @@ const TALKING_INDICATOR_SUBSCRIPTION = gql`
     ) {
       callerName
       spoke
-      talking
       floor
       startTime
       muted

bigbluebutton-html5/imports/ui/core/graphql/queries/users.ts

@@ -39,8 +39,6 @@ subscription UserListSubscription($offset: Int!, $limit: Int!) {
     voice {
       joined
       listenOnly
-      talking
-      muted
       voiceUserId
     }
     cameras {

bigbluebutton-html5/imports/ui/core/graphql/queries/whoIsTalking.ts

@@ -0,0 +1,34 @@
+import { gql } from '@apollo/client';
+
+export interface VoiceActivityResponse {
+  user_voice_activity_stream: Array<{
+    startTime: number | undefined;
+    endTime: number | undefined;
+    muted: boolean;
+    talking: boolean;
+    userId: string;
+    color: string;
+    name: string;
+    speechLocale: string | undefined;
+  }>;
+}
+
+export const VOICE_ACTIVITY = gql`
+  subscription UserVoiceActivity {
+    user_voice_activity_stream(
+      cursor: { initial_value: { voiceActivityAt: "2020-01-01" } },
+      batch_size: 10
+    ) {
+      muted
+      startTime
+      endTime
+      talking
+      userId
+      color
+      name
+      speechLocale
+    }
+  }
+`;
+
+export default VOICE_ACTIVITY;

bigbluebutton-html5/imports/ui/core/hooks/useVoiceActivity.ts

@@ -0,0 +1,79 @@
+import { useEffect, useRef, useState } from 'react';
+import logger from '/imports/startup/client/logger';
+import VOICE_ACTIVITY, { VoiceActivityResponse } from '/imports/ui/core/graphql/queries/whoIsTalking';
+import useDeduplicatedSubscription from './useDeduplicatedSubscription';
+
+type VoiceItem = VoiceActivityResponse['user_voice_activity_stream'][number] & {
+  showTalkingIndicator: boolean | undefined;
+};
+
+const TALKING_INDICATOR_TIMEOUT = 5000;
+
+const useVoiceActivity = () => {
+  const {
+    data,
+    loading,
+    error,
+  } = useDeduplicatedSubscription<VoiceActivityResponse>(VOICE_ACTIVITY);
+  const [record, setRecord] = useState<Record<string, VoiceItem>>({});
+  const timeoutRegistry = useRef<Record<string, NodeJS.Timeout>>({});
+
+  if (error) {
+    logger.error({
+      logCode: 'voice_activity_sub_error',
+      extraInfo: {
+        errorName: error.name,
+        errorMessage: error.message,
+      },
+    }, 'useVoiceActivity hook failed.');
+  }
+
+  useEffect(() => {
+    const voiceActivity: Record<string, VoiceItem> = { ...record };
+
+    if (data) {
+      data.user_voice_activity_stream.forEach((voice) => {
+        const {
+          userId, talking, endTime, muted,
+        } = voice;
+
+        if (muted) {
+          delete voiceActivity[userId];
+          return;
+        }
+
+        voiceActivity[userId] = Object.assign(
+          voiceActivity[userId] || {},
+          voice,
+          { showTalkingIndicator: talking || voiceActivity[userId]?.showTalkingIndicator },
+        );
+
+        if (talking && timeoutRegistry.current[userId]) {
+          clearTimeout(timeoutRegistry.current[userId]);
+        }
+
+        if (endTime) {
+          timeoutRegistry.current[userId] = setTimeout(() => {
+            setRecord((prevRecord) => ({
+              ...prevRecord,
+              [userId]: Object.assign(
+                prevRecord[userId] || {},
+                { showTalkingIndicator: false },
+              ),
+            }));
+          }, TALKING_INDICATOR_TIMEOUT);
+        }
+      });
+    }
+
+    setRecord(voiceActivity);
+  }, [data]);
+
+  return {
+    error,
+    loading,
+    data: record,
+  };
+};
+
+export default useVoiceActivity;

bigbluebutton-html5/imports/ui/core/hooks/useWhoIsTalking.ts

@@ -0,0 +1,27 @@
+import { useEffect, useState } from 'react';
+import useVoiceActivity from './useVoiceActivity';
+
+const useWhoIsTalking = () => {
+  const {
+    data: voiceActivity,
+    loading,
+  } = useVoiceActivity();
+  const [record, setRecord] = useState<Record<string, boolean>>({});
+
+  useEffect(() => {
+    const talkingUsers: Record<string, boolean> = {};
+
+    Object.keys(voiceActivity).forEach((userId) => {
+      talkingUsers[userId] = voiceActivity[userId]?.talking;
+    });
+
+    setRecord(talkingUsers);
+  }, [voiceActivity]);
+
+  return {
+    data: record,
+    loading,
+  };
+};
+
+export default useWhoIsTalking;

bigbluebutton-html5/imports/ui/core/hooks/useWhoIsUnmuted.ts

@@ -0,0 +1,21 @@
+import { useMemo } from 'react';
+import useVoiceActivity from './useVoiceActivity';
+
+const useWhoIsUnmuted = () => {
+  const {
+    data: voiceActivity,
+    loading,
+  } = useVoiceActivity();
+
+  const record = useMemo(() => {
+    const mutedUsers: Set<string> = new Set(Object.keys(voiceActivity));
+    return mutedUsers;
+  }, [voiceActivity]);
+
+  return {
+    data: record,
+    loading,
+  };
+};
+
+export default useWhoIsUnmuted;