feat(captions): audio captions app

Add a server-side app for the audio captions feature and record proto-events
for this data.

As it is, only behaves as a pass-through module. The idea is to include all
the business intelligence in this app.

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/audiocaptions/AudioCaptionsApp2x.scala

@@ -0,0 +1,8 @@
+package org.bigbluebutton.core.apps.audiocaptions
+
+import akka.actor.ActorContext
+
+class AudioCaptionsApp2x(implicit val context: ActorContext)
+  extends UpdateTranscriptPubMsgHdlr {
+
+}

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/audiocaptions/UpdateTranscriptPubMsgHdlr.scala

@@ -0,0 +1,30 @@
+package org.bigbluebutton.core.apps.audiocaptions
+
+import org.bigbluebutton.common2.msgs._
+import org.bigbluebutton.core.bus.MessageBus
+import org.bigbluebutton.core.running.LiveMeeting
+
+trait UpdateTranscriptPubMsgHdlr {
+  this: AudioCaptionsApp2x =>
+
+  def handle(msg: UpdateTranscriptPubMsg, liveMeeting: LiveMeeting, bus: MessageBus): Unit = {
+    val meetingId = liveMeeting.props.meetingProp.intId
+
+    def broadcastEvent(userId: String, transcript: String, locale: String): Unit = {
+      val routing = Routing.addMsgToClientRouting(MessageTypes.DIRECT, meetingId, "nodeJSapp")
+      val envelope = BbbCoreEnvelope(TranscriptUpdatedEvtMsg.NAME, routing)
+      val header = BbbClientMsgHeader(TranscriptUpdatedEvtMsg.NAME, meetingId, userId)
+      val body = TranscriptUpdatedEvtMsgBody(transcript, locale)
+      val event = TranscriptUpdatedEvtMsg(header, body)
+      val msgEvent = BbbCommonEnvCoreMsg(envelope, event)
+
+      bus.outGW.send(msgEvent)
+    }
+
+    broadcastEvent(
+      msg.header.userId,
+      msg.body.transcript,
+      msg.body.locale
+    )
+  }
+}

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/models/AudioCaptions.scala

@@ -0,0 +1,7 @@
+package org.bigbluebutton.core.models
+
+object AudioCaptions {
+}
+
+class AudioCaptions {
+}

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/pubsub/senders/ReceivedJsonMsgHandlerActor.scala

@@ -375,6 +375,10 @@ class ReceivedJsonMsgHandlerActor(
       case GetScreenSubscribePermissionReqMsg.NAME =>
         routeGenericMsg[GetScreenSubscribePermissionReqMsg](envelope, jsonNode)
 
+      // AudioCaptions
+      case UpdateTranscriptPubMsg.NAME =>
+        routeGenericMsg[UpdateTranscriptPubMsg](envelope, jsonNode)
+
       // GroupChats
       case GetGroupChatsReqMsg.NAME =>
         routeGenericMsg[GetGroupChatsReqMsg](envelope, jsonNode)

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/record/events/AbstractAudioCaptionsRecordEvent.scala

@@ -0,0 +1,5 @@
+package org.bigbluebutton.core.record.events
+
+trait AbstractAudioCaptionsRecordEvent extends RecordEvent {
+  setModule("AUDIO-CAPTIONS")
+}

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/record/events/TranscriptUpdatedRecordEvent.scala

@@ -0,0 +1,20 @@
+package org.bigbluebutton.core.record.events
+
+class TranscriptUpdatedRecordEvent extends AbstractAudioCaptionsRecordEvent {
+  import TranscriptUpdatedRecordEvent._
+
+  setEvent("TranscriptUpdatedEvent")
+
+  def setLocale(locale: String) {
+    eventMap.put(LOCALE, locale)
+  }
+
+  def setTranscript(transcript: String) {
+    eventMap.put(TRANSCRIPT, transcript)
+  }
+}
+
+object TranscriptUpdatedRecordEvent {
+  protected final val LOCALE = "locale"
+  protected final val TRANSCRIPT = "transcript"
+}

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/LiveMeeting.scala

@@ -9,6 +9,7 @@ class LiveMeeting(
     val props:              DefaultProps,
     val status:             MeetingStatus2x,
     val screenshareModel:   ScreenshareModel,
+    val audioCaptions:      AudioCaptions,
     val chatModel:          ChatModel,
     val externalVideoModel: ExternalVideoModel,
     val layouts:            Layouts,

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/MeetingActor.scala

@@ -18,6 +18,7 @@ import org.bigbluebutton.core.apps.chat.ChatApp2x
 import org.bigbluebutton.core.apps.externalvideo.ExternalVideoApp2x
 import org.bigbluebutton.core.apps.pads.PadsApp2x
 import org.bigbluebutton.core.apps.screenshare.ScreenshareApp2x
+import org.bigbluebutton.core.apps.audiocaptions.AudioCaptionsApp2x
 import org.bigbluebutton.core.apps.presentation.PresentationApp2x
 import org.bigbluebutton.core.apps.users.UsersApp2x
 import org.bigbluebutton.core.apps.webcam.WebcamApp2x
@@ -124,6 +125,7 @@ class MeetingActor(
 
   val presentationApp2x = new PresentationApp2x
   val screenshareApp2x = new ScreenshareApp2x
+  val audioCaptionsApp2x = new AudioCaptionsApp2x
   val captionApp2x = new CaptionApp2x
   val chatApp2x = new ChatApp2x
   val externalVideoApp2x = new ExternalVideoApp2x
@@ -558,6 +560,9 @@ class MeetingActor(
       case m: GetScreenBroadcastPermissionReqMsg             => handleGetScreenBroadcastPermissionReqMsg(m)
       case m: GetScreenSubscribePermissionReqMsg             => handleGetScreenSubscribePermissionReqMsg(m)
 
+      // AudioCaptions
+      case m: UpdateTranscriptPubMsg                         => audioCaptionsApp2x.handle(m, liveMeeting, msgBus)
+
       // GroupChat
       case m: CreateGroupChatReqMsg =>
         state = groupChatApp.handle(m, state, liveMeeting, msgBus)

akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/RunningMeeting.scala

@@ -32,12 +32,13 @@ class RunningMeeting(val props: DefaultProps, outGW: OutMessageGateway,
   private val polls2x = new Polls
   private val guestsWaiting = new GuestsWaiting
   private val deskshareModel = new ScreenshareModel
+  private val audioCaptions = new AudioCaptions
 
   // meetingModel.setGuestPolicy(props.usersProp.guestPolicy)
 
   // We extract the meeting handlers into this class so it is
   // easy to test.
-  private val liveMeeting = new LiveMeeting(props, meetingStatux2x, deskshareModel, chatModel, externalVideoModel,
+  private val liveMeeting = new LiveMeeting(props, meetingStatux2x, deskshareModel, audioCaptions, chatModel, externalVideoModel,
     layouts, pads, registeredUsers, polls2x, wbModel, presModel, captionModel,
     webcams, voiceUsers, users2x, guestsWaiting)
 

akka-bbb-apps/src/main/scala/org/bigbluebutton/endpoint/redis/RedisRecorderActor.scala

@@ -111,6 +111,9 @@ class RedisRecorderActor(
       case m: ScreenshareRtmpBroadcastStoppedEvtMsg => handleScreenshareRtmpBroadcastStoppedEvtMsg(m)
       //case m: DeskShareNotifyViewersRTMP  => handleDeskShareNotifyViewersRTMP(m)
 
+      // AudioCaptions
+      case m: TranscriptUpdatedEvtMsg               => handleTranscriptUpdatedEvtMsg(m)
+
       // Meeting
       case m: RecordingStatusChangedEvtMsg          => handleRecordingStatusChangedEvtMsg(m)
       case m: RecordStatusResetSysMsg               => handleRecordStatusResetSysMsg(m)
@@ -505,6 +508,15 @@ class RedisRecorderActor(
   }
   */
 
+  private def handleTranscriptUpdatedEvtMsg(msg: TranscriptUpdatedEvtMsg) {
+    val ev = new TranscriptUpdatedRecordEvent()
+    ev.setMeetingId(msg.header.meetingId)
+    ev.setLocale(msg.body.locale)
+    ev.setTranscript(msg.body.transcript)
+
+    record(msg.header.meetingId, ev.toMap.asJava)
+  }
+
   private def handleStartExternalVideoEvtMsg(msg: StartExternalVideoEvtMsg) {
     val ev = new StartExternalVideoRecordEvent()
     ev.setMeetingId(msg.header.meetingId)

bbb-common-message/src/main/scala/org/bigbluebutton/common2/msgs/AudioCaptionsMsgs.scala

@@ -0,0 +1,11 @@
+package org.bigbluebutton.common2.msgs
+
+// In messages
+object UpdateTranscriptPubMsg { val NAME = "UpdateTranscriptPubMsg" }
+case class UpdateTranscriptPubMsg(header: BbbClientMsgHeader, body: UpdateTranscriptPubMsgBody) extends StandardMsg
+case class UpdateTranscriptPubMsgBody(transcriptId: String, transcript: String, locale: String)
+
+// Out messages
+object TranscriptUpdatedEvtMsg { val NAME = "TranscriptUpdatedEvtMsg" }
+case class TranscriptUpdatedEvtMsg(header: BbbClientMsgHeader, body: TranscriptUpdatedEvtMsgBody) extends BbbCoreMsg
+case class TranscriptUpdatedEvtMsgBody(transcript: String, locale: String)

bigbluebutton-html5/imports/api/audio-captions/server/eventHandlers.js

@@ -0,0 +1,4 @@
+import RedisPubSub from '/imports/startup/server/redis';
+import handleTranscriptUpdated from '/imports/api/audio-captions/server/handlers/transcriptUpdated';
+
+RedisPubSub.on('TranscriptUpdatedEvtMsg', handleTranscriptUpdated);

bigbluebutton-html5/imports/api/audio-captions/server/handlers/transcriptUpdated.js

@@ -0,0 +1,9 @@
+import setTranscript from '/imports/api/audio-captions/server/modifiers/setTranscript';
+
+export default function transcriptUpdated({ header, body }) {
+  const { meetingId } = header;
+
+  const { transcript } = body;
+
+  setTranscript(meetingId, transcript);
+}

bigbluebutton-html5/imports/api/audio-captions/server/index.js

@@ -1,2 +1,3 @@
+import './eventHandlers';
 import './methods';
 import './publishers';

bigbluebutton-html5/imports/api/audio-captions/server/methods.js

@@ -1,6 +1,6 @@
 import { Meteor } from 'meteor/meteor';
-import pushAudioTranscript from '/imports/api/audio-captions/server/methods/pushAudioTranscript';
+import updateTranscript from '/imports/api/audio-captions/server/methods/updateTranscript';
 
 Meteor.methods({
-  pushAudioTranscript,
+  updateTranscript,
 });

bigbluebutton-html5/imports/api/audio-captions/server/methods/pushAudioTranscript.js

@@ -1,19 +0,0 @@
-import { check } from 'meteor/check';
-import { extractCredentials } from '/imports/api/common/server/helpers';
-import Logger from '/imports/startup/server/logger';
-import setTranscript from '/imports/api/audio-captions/server/modifiers/setTranscript';
-
-export default function pushAudioTranscript(transcript, type) {
-  try {
-    const { meetingId, requesterUserId } = extractCredentials(this.userId);
-
-    check(meetingId, String);
-    check(requesterUserId, String);
-    check(transcript, String);
-    check(type, String);
-
-    setTranscript(meetingId, transcript);
-  } catch (err) {
-    Logger.error(`Exception while invoking method pushAudioTranscript ${err.stack}`);
-  }
-}

bigbluebutton-html5/imports/api/audio-captions/server/methods/updateTranscript.js

@@ -0,0 +1,30 @@
+import { check } from 'meteor/check';
+import RedisPubSub from '/imports/startup/server/redis';
+import { extractCredentials } from '/imports/api/common/server/helpers';
+import Logger from '/imports/startup/server/logger';
+
+export default function updateTranscript(transcriptId, transcript, locale) {
+  try {
+    const REDIS_CONFIG = Meteor.settings.private.redis;
+    const CHANNEL = REDIS_CONFIG.channels.toAkkaApps;
+    const EVENT_NAME = 'UpdateTranscriptPubMsg';
+
+    const { meetingId, requesterUserId } = extractCredentials(this.userId);
+
+    check(meetingId, String);
+    check(requesterUserId, String);
+    check(transcriptId, String);
+    check(transcript, String);
+    check(locale, String);
+
+    const payload = {
+      transcriptId,
+      transcript,
+      locale,
+    };
+
+    RedisPubSub.publishUserMessage(CHANNEL, EVENT_NAME, meetingId, requesterUserId, payload);
+  } catch (err) {
+    Logger.error(`Exception while invoking method upadteTranscript ${err.stack}`);
+  }
+}

bigbluebutton-html5/imports/ui/components/audio/captions/speech/component.jsx

@@ -12,6 +12,7 @@ class Speech extends PureComponent {
     this.onResult = this.onResult.bind(this);
 
     this.result = {
+      id: Service.generateId(),
       transcript: '',
       isFinal: true,
     };
@@ -88,16 +89,19 @@ class Speech extends PureComponent {
       results,
     } = event;
 
+    const { id } = this.result;
     const { transcript } = results[resultIndex][0];
     const { isFinal } = results[resultIndex];
 
     this.result.transcript = transcript;
     this.result.isFinal = isFinal;
 
+    const { locale } = this.props;
     if (isFinal) {
-      Service.pushFinalTranscript(transcript);
+      Service.updateFinalTranscript(id, transcript, locale);
+      this.result.id = Service.generateId();
     } else {
-      Service.pushInterimTranscript(transcript);
+      Service.updateInterimTranscript(id, transcript, locale);
     }
   }
 
@@ -105,6 +109,7 @@ class Speech extends PureComponent {
     if (this.speechRecognition) {
       this.speechRecognition.lang = locale;
       try {
+        this.result.id = Service.generateId();
         this.speechRecognition.start();
         this.idle = false;
       } catch (event) {
@@ -122,7 +127,9 @@ class Speech extends PureComponent {
       } = this.result;
 
       if (!isFinal) {
-        Service.pushFinalTranscript(transcript);
+        const { locale } = this.props;
+        const { id } = this.result;
+        Service.updateFinalTranscript(id, transcript, locale);
         this.speechRecognition.abort();
       } else {
         this.speechRecognition.stop();

bigbluebutton-html5/imports/ui/components/audio/captions/speech/service.js

@@ -1,11 +1,12 @@
 import _ from 'lodash';
 import { Session } from 'meteor/session';
+import Auth from '/imports/ui/services/auth';
 import { makeCall } from '/imports/ui/services/api';
 import AudioService from '/imports/ui/components/audio/service';
 
 const DEFAULT_LANGUAGE = 'pt-BR';
 const ENABLED = Meteor.settings.public.app.enableAudioCaptions;
-const THROTTLE_TIMEOUT = 2000;
+const THROTTLE_TIMEOUT = 1000;
 
 const SpeechRecognitionAPI = window.SpeechRecognition || window.webkitSpeechRecognition;
 
@@ -24,18 +25,20 @@ const initSpeechRecognition = (locale = DEFAULT_LANGUAGE) => {
   return null;
 };
 
-const pushAudioTranscript = (transcript, type) => makeCall('pushAudioTranscript', transcript, type);
+const updateTranscript = (id, transcript, locale) => makeCall('updateTranscript', id, transcript, locale);
 
-const throttledTranscriptPush = _.throttle(pushAudioTranscript, THROTTLE_TIMEOUT, {
+const throttledTranscriptUpdate = _.throttle(updateTranscript, THROTTLE_TIMEOUT, {
   leading: false,
   trailing: true,
 });
 
-const pushInterimTranscript = (transcript) => throttledTranscriptPush(transcript, 'interim');
+const updateInterimTranscript = (id, transcript, locale) => {
+  throttledTranscriptUpdate(id, transcript, locale);
+};
 
-const pushFinalTranscript = (transcript) => {
-  throttledTranscriptPush.cancel();
-  pushAudioTranscript(transcript, 'final');
+const updateFinalTranscript = (id, transcript, locale) => {
+  throttledTranscriptUpdate.cancel();
+  updateTranscript(id, transcript, locale);
 };
 
 const getSpeech = () => Session.get('speech') || false;
@@ -59,14 +62,17 @@ const getStatus = () => {
   };
 };
 
+const generateId = () => `${Auth.userID}-${Date.now()}`;
+
 export default {
   hasSpeechRecognitionSupport,
   initSpeechRecognition,
-  pushInterimTranscript,
-  pushFinalTranscript,
+  updateInterimTranscript,
+  updateFinalTranscript,
   getSpeech,
   setSpeech,
   isEnabled,
   isActive,
   getStatus,
+  generateId,
 };