BigBlueButton/bigbluebutton-Github
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(captions): audio captions app

Browse Source 
Add a server-side app for the audio captions feature and record proto-events
for this data.

As it is, only behaves as a pass-through module. The idea is to include all
the business intelligence in this app.
This commit is contained in:
Pedro Beschorner Marin 2022-03-31 16:40:07 -03:00 committed by Arthurk12
parent 0bc730b3e3
commit 75969ec93c
19 changed files with 176 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/audiocaptions/AudioCaptionsApp2x.scala Normal file
View File

@ -0,0 +1,8 @@
package org.bigbluebutton.core.apps.audiocaptions
import akka.actor.ActorContext
class AudioCaptionsApp2x(implicit val context: ActorContext)
extends UpdateTranscriptPubMsgHdlr {
}

30
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/apps/audiocaptions/UpdateTranscriptPubMsgHdlr.scala Normal file
View File

@ -0,0 +1,30 @@
package org.bigbluebutton.core.apps.audiocaptions
import org.bigbluebutton.common2.msgs._
import org.bigbluebutton.core.bus.MessageBus
import org.bigbluebutton.core.running.LiveMeeting
trait UpdateTranscriptPubMsgHdlr {
this: AudioCaptionsApp2x =>
def handle(msg: UpdateTranscriptPubMsg, liveMeeting: LiveMeeting, bus: MessageBus): Unit = {
val meetingId = liveMeeting.props.meetingProp.intId
def broadcastEvent(userId: String, transcript: String, locale: String): Unit = {
val routing = Routing.addMsgToClientRouting(MessageTypes.DIRECT, meetingId, "nodeJSapp")
val envelope = BbbCoreEnvelope(TranscriptUpdatedEvtMsg.NAME, routing)
val header = BbbClientMsgHeader(TranscriptUpdatedEvtMsg.NAME, meetingId, userId)
val body = TranscriptUpdatedEvtMsgBody(transcript, locale)
val event = TranscriptUpdatedEvtMsg(header, body)
val msgEvent = BbbCommonEnvCoreMsg(envelope, event)
bus.outGW.send(msgEvent)
}
broadcastEvent(
msg.header.userId,
msg.body.transcript,
msg.body.locale
)
}
}

7
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/models/AudioCaptions.scala Normal file
View File

@ -0,0 +1,7 @@
package org.bigbluebutton.core.models
object AudioCaptions {
}
class AudioCaptions {
}

4
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/pubsub/senders/ReceivedJsonMsgHandlerActor.scala
View File

@ -375,6 +375,10 @@ class ReceivedJsonMsgHandlerActor(
case GetScreenSubscribePermissionReqMsg.NAME =>
routeGenericMsg[GetScreenSubscribePermissionReqMsg](envelope, jsonNode)
// AudioCaptions
case UpdateTranscriptPubMsg.NAME =>
routeGenericMsg[UpdateTranscriptPubMsg](envelope, jsonNode)
// GroupChats
case GetGroupChatsReqMsg.NAME =>
routeGenericMsg[GetGroupChatsReqMsg](envelope, jsonNode)

5
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/record/events/AbstractAudioCaptionsRecordEvent.scala Normal file
View File

@ -0,0 +1,5 @@
package org.bigbluebutton.core.record.events
trait AbstractAudioCaptionsRecordEvent extends RecordEvent {
setModule("AUDIO-CAPTIONS")
}

20
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/record/events/TranscriptUpdatedRecordEvent.scala Normal file
View File

@ -0,0 +1,20 @@
package org.bigbluebutton.core.record.events
class TranscriptUpdatedRecordEvent extends AbstractAudioCaptionsRecordEvent {
import TranscriptUpdatedRecordEvent._
setEvent("TranscriptUpdatedEvent")
def setLocale(locale: String) {
eventMap.put(LOCALE, locale)
}
def setTranscript(transcript: String) {
eventMap.put(TRANSCRIPT, transcript)
}
}
object TranscriptUpdatedRecordEvent {
protected final val LOCALE = "locale"
protected final val TRANSCRIPT = "transcript"
}

1
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/LiveMeeting.scala
View File

@ -9,6 +9,7 @@ class LiveMeeting(
val props: DefaultProps,
val status: MeetingStatus2x,
val screenshareModel: ScreenshareModel,
val audioCaptions: AudioCaptions,
val chatModel: ChatModel,
val externalVideoModel: ExternalVideoModel,
val layouts: Layouts,

5
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/MeetingActor.scala
View File

@ -18,6 +18,7 @@ import org.bigbluebutton.core.apps.chat.ChatApp2x
import org.bigbluebutton.core.apps.externalvideo.ExternalVideoApp2x
import org.bigbluebutton.core.apps.pads.PadsApp2x
import org.bigbluebutton.core.apps.screenshare.ScreenshareApp2x
import org.bigbluebutton.core.apps.audiocaptions.AudioCaptionsApp2x
import org.bigbluebutton.core.apps.presentation.PresentationApp2x
import org.bigbluebutton.core.apps.users.UsersApp2x
import org.bigbluebutton.core.apps.webcam.WebcamApp2x
@ -124,6 +125,7 @@ class MeetingActor(
val presentationApp2x = new PresentationApp2x
val screenshareApp2x = new ScreenshareApp2x
val audioCaptionsApp2x = new AudioCaptionsApp2x
val captionApp2x = new CaptionApp2x
val chatApp2x = new ChatApp2x
val externalVideoApp2x = new ExternalVideoApp2x
@ -558,6 +560,9 @@ class MeetingActor(
case m: GetScreenBroadcastPermissionReqMsg => handleGetScreenBroadcastPermissionReqMsg(m)
case m: GetScreenSubscribePermissionReqMsg => handleGetScreenSubscribePermissionReqMsg(m)
// AudioCaptions
case m: UpdateTranscriptPubMsg => audioCaptionsApp2x.handle(m, liveMeeting, msgBus)
// GroupChat
case m: CreateGroupChatReqMsg =>
state = groupChatApp.handle(m, state, liveMeeting, msgBus)

3
akka-bbb-apps/src/main/scala/org/bigbluebutton/core/running/RunningMeeting.scala
View File

@ -32,12 +32,13 @@ class RunningMeeting(val props: DefaultProps, outGW: OutMessageGateway,
private val polls2x = new Polls
private val guestsWaiting = new GuestsWaiting
private val deskshareModel = new ScreenshareModel
private val audioCaptions = new AudioCaptions
// meetingModel.setGuestPolicy(props.usersProp.guestPolicy)
// We extract the meeting handlers into this class so it is
// easy to test.
private val liveMeeting = new LiveMeeting(props, meetingStatux2x, deskshareModel, chatModel, externalVideoModel,
private val liveMeeting = new LiveMeeting(props, meetingStatux2x, deskshareModel, audioCaptions, chatModel, externalVideoModel,
layouts, pads, registeredUsers, polls2x, wbModel, presModel, captionModel,
webcams, voiceUsers, users2x, guestsWaiting)

12
akka-bbb-apps/src/main/scala/org/bigbluebutton/endpoint/redis/RedisRecorderActor.scala
View File

@ -111,6 +111,9 @@ class RedisRecorderActor(
case m: ScreenshareRtmpBroadcastStoppedEvtMsg => handleScreenshareRtmpBroadcastStoppedEvtMsg(m)
//case m: DeskShareNotifyViewersRTMP => handleDeskShareNotifyViewersRTMP(m)
// AudioCaptions
case m: TranscriptUpdatedEvtMsg => handleTranscriptUpdatedEvtMsg(m)
// Meeting
case m: RecordingStatusChangedEvtMsg => handleRecordingStatusChangedEvtMsg(m)
case m: RecordStatusResetSysMsg => handleRecordStatusResetSysMsg(m)
@ -505,6 +508,15 @@ class RedisRecorderActor(
}
*/
private def handleTranscriptUpdatedEvtMsg(msg: TranscriptUpdatedEvtMsg) {
val ev = new TranscriptUpdatedRecordEvent()
ev.setMeetingId(msg.header.meetingId)
ev.setLocale(msg.body.locale)
ev.setTranscript(msg.body.transcript)
record(msg.header.meetingId, ev.toMap.asJava)
}
private def handleStartExternalVideoEvtMsg(msg: StartExternalVideoEvtMsg) {
val ev = new StartExternalVideoRecordEvent()
ev.setMeetingId(msg.header.meetingId)

11
bbb-common-message/src/main/scala/org/bigbluebutton/common2/msgs/AudioCaptionsMsgs.scala Normal file
View File

@ -0,0 +1,11 @@
package org.bigbluebutton.common2.msgs
// In messages
object UpdateTranscriptPubMsg { val NAME = "UpdateTranscriptPubMsg" }
case class UpdateTranscriptPubMsg(header: BbbClientMsgHeader, body: UpdateTranscriptPubMsgBody) extends StandardMsg
case class UpdateTranscriptPubMsgBody(transcriptId: String, transcript: String, locale: String)
// Out messages
object TranscriptUpdatedEvtMsg { val NAME = "TranscriptUpdatedEvtMsg" }
case class TranscriptUpdatedEvtMsg(header: BbbClientMsgHeader, body: TranscriptUpdatedEvtMsgBody) extends BbbCoreMsg
case class TranscriptUpdatedEvtMsgBody(transcript: String, locale: String)

4
bigbluebutton-html5/imports/api/audio-captions/server/eventHandlers.js Normal file
View File

@ -0,0 +1,4 @@
import RedisPubSub from '/imports/startup/server/redis';
import handleTranscriptUpdated from '/imports/api/audio-captions/server/handlers/transcriptUpdated';
RedisPubSub.on('TranscriptUpdatedEvtMsg', handleTranscriptUpdated);

9
bigbluebutton-html5/imports/api/audio-captions/server/handlers/transcriptUpdated.js Normal file
View File

@ -0,0 +1,9 @@
import setTranscript from '/imports/api/audio-captions/server/modifiers/setTranscript';
export default function transcriptUpdated({ header, body }) {
const { meetingId } = header;
const { transcript } = body;
setTranscript(meetingId, transcript);
}

1
bigbluebutton-html5/imports/api/audio-captions/server/index.js
View File

@ -1,2 +1,3 @@
import './eventHandlers';
import './methods';
import './publishers';

4
bigbluebutton-html5/imports/api/audio-captions/server/methods.js
View File

@ -1,6 +1,6 @@
import { Meteor } from 'meteor/meteor';
import pushAudioTranscript from '/imports/api/audio-captions/server/methods/pushAudioTranscript';
import updateTranscript from '/imports/api/audio-captions/server/methods/updateTranscript';
Meteor.methods({
pushAudioTranscript,
updateTranscript,
});

19
bigbluebutton-html5/imports/api/audio-captions/server/methods/pushAudioTranscript.js
View File

@ -1,19 +0,0 @@
import { check } from 'meteor/check';
import { extractCredentials } from '/imports/api/common/server/helpers';
import Logger from '/imports/startup/server/logger';
import setTranscript from '/imports/api/audio-captions/server/modifiers/setTranscript';
export default function pushAudioTranscript(transcript, type) {
try {
const { meetingId, requesterUserId } = extractCredentials(this.userId);
check(meetingId, String);
check(requesterUserId, String);
check(transcript, String);
check(type, String);
setTranscript(meetingId, transcript);
} catch (err) {
Logger.error(`Exception while invoking method pushAudioTranscript ${err.stack}`);
}
}

30
bigbluebutton-html5/imports/api/audio-captions/server/methods/updateTranscript.js Normal file
View File

@ -0,0 +1,30 @@
import { check } from 'meteor/check';
import RedisPubSub from '/imports/startup/server/redis';
import { extractCredentials } from '/imports/api/common/server/helpers';
import Logger from '/imports/startup/server/logger';
export default function updateTranscript(transcriptId, transcript, locale) {
try {
const REDIS_CONFIG = Meteor.settings.private.redis;
const CHANNEL = REDIS_CONFIG.channels.toAkkaApps;
const EVENT_NAME = 'UpdateTranscriptPubMsg';
const { meetingId, requesterUserId } = extractCredentials(this.userId);
check(meetingId, String);
check(requesterUserId, String);
check(transcriptId, String);
check(transcript, String);
check(locale, String);
const payload = {
transcriptId,
transcript,
locale,
};
RedisPubSub.publishUserMessage(CHANNEL, EVENT_NAME, meetingId, requesterUserId, payload);
} catch (err) {
Logger.error(`Exception while invoking method upadteTranscript ${err.stack}`);
}
}

13
bigbluebutton-html5/imports/ui/components/audio/captions/speech/component.jsx
View File

@ -12,6 +12,7 @@ class Speech extends PureComponent {
this.onResult = this.onResult.bind(this);
this.result = {
id: Service.generateId(),
transcript: '',
isFinal: true,
};
@ -88,16 +89,19 @@ class Speech extends PureComponent {
results,
} = event;
const { id } = this.result;
const { transcript } = results[resultIndex][0];
const { isFinal } = results[resultIndex];
this.result.transcript = transcript;
this.result.isFinal = isFinal;
const { locale } = this.props;
if (isFinal) {
Service.pushFinalTranscript(transcript);
Service.updateFinalTranscript(id, transcript, locale);
this.result.id = Service.generateId();
} else {
Service.pushInterimTranscript(transcript);
Service.updateInterimTranscript(id, transcript, locale);
}
}
@ -105,6 +109,7 @@ class Speech extends PureComponent {
if (this.speechRecognition) {
this.speechRecognition.lang = locale;
try {
this.result.id = Service.generateId();
this.speechRecognition.start();
this.idle = false;
} catch (event) {
@ -122,7 +127,9 @@ class Speech extends PureComponent {
} = this.result;
if (!isFinal) {
Service.pushFinalTranscript(transcript);
const { locale } = this.props;
const { id } = this.result;
Service.updateFinalTranscript(id, transcript, locale);
this.speechRecognition.abort();
} else {
this.speechRecognition.stop();

24
bigbluebutton-html5/imports/ui/components/audio/captions/speech/service.js
View File

@ -1,11 +1,12 @@
import _ from 'lodash';
import { Session } from 'meteor/session';
import Auth from '/imports/ui/services/auth';
import { makeCall } from '/imports/ui/services/api';
import AudioService from '/imports/ui/components/audio/service';
const DEFAULT_LANGUAGE = 'pt-BR';
const ENABLED = Meteor.settings.public.app.enableAudioCaptions;
const THROTTLE_TIMEOUT = 2000;
const THROTTLE_TIMEOUT = 1000;
const SpeechRecognitionAPI = window.SpeechRecognition || window.webkitSpeechRecognition;
@ -24,18 +25,20 @@ const initSpeechRecognition = (locale = DEFAULT_LANGUAGE) => {
return null;
};
const pushAudioTranscript = (transcript, type) => makeCall('pushAudioTranscript', transcript, type);
const updateTranscript = (id, transcript, locale) => makeCall('updateTranscript', id, transcript, locale);
const throttledTranscriptPush = _.throttle(pushAudioTranscript, THROTTLE_TIMEOUT, {
const throttledTranscriptUpdate = _.throttle(updateTranscript, THROTTLE_TIMEOUT, {
leading: false,
trailing: true,
});
const pushInterimTranscript = (transcript) => throttledTranscriptPush(transcript, 'interim');
const updateInterimTranscript = (id, transcript, locale) => {
throttledTranscriptUpdate(id, transcript, locale);
};
const pushFinalTranscript = (transcript) => {
throttledTranscriptPush.cancel();
pushAudioTranscript(transcript, 'final');
const updateFinalTranscript = (id, transcript, locale) => {
throttledTranscriptUpdate.cancel();
updateTranscript(id, transcript, locale);
};
const getSpeech = () => Session.get('speech') || false;
@ -59,14 +62,17 @@ const getStatus = () => {
};
};
const generateId = () => `${Auth.userID}-${Date.now()}`;
export default {
hasSpeechRecognitionSupport,
initSpeechRecognition,
pushInterimTranscript,
pushFinalTranscript,
updateInterimTranscript,
updateFinalTranscript,
getSpeech,
setSpeech,
isEnabled,
isActive,
getStatus,
generateId,
};