bigbluebutton-Github/bigbluebutton-html5/imports/api/audio/client/bridge/sfu-audio-bridge.js

import BaseAudioBridge from './base';
import Auth from '/imports/ui/services/auth';
import logger from '/imports/startup/client/logger';
import AudioBroker from '/imports/ui/services/bbb-webrtc-sfu/audio-broker';
import loadAndPlayMediaStream from '/imports/ui/services/bbb-webrtc-sfu/load-play';
import {
  fetchWebRTCMappedStunTurnServers,
  getMappedFallbackStun,
} from '/imports/utils/fetchStunTurnServers';
import getFromMeetingSettings from '/imports/ui/services/meeting-settings';
import getFromUserSettings from '/imports/ui/services/users-settings';
import browserInfo from '/imports/utils/browserInfo';
import {
  getAudioSessionNumber,
  getAudioConstraints,
  filterSupportedConstraints,
  doGUM,
} from '/imports/api/audio/client/bridge/service';
import { shouldForceRelay } from '/imports/ui/services/bbb-webrtc-sfu/utils';

const SENDRECV_ROLE = 'sendrecv';
const RECV_ROLE = 'recv';
const PASSIVE_SENDRECV_ROLE = 'passive-sendrecv';
const BRIDGE_NAME = 'fullaudio';
const IS_CHROME = browserInfo.isChrome;

// SFU's base broker has distinct error codes so that it can be reused by different
// modules. Errors that have a valid, localized counterpart in audio manager are
// mapped so that the user gets a localized error message.
// The ones that haven't (ie SFU's servers-side errors), aren't mapped.
const errorCodeMap = {
  1301: 1001,
  1302: 1002,
  1305: 1005,
  1307: 1007,
};

// Error codes that are prone to a retry according to RETRY_THROUGH_RELAY
const RETRYABLE_ERRORS = [1007, 1010];

const mapErrorCode = (error) => {
  const { errorCode } = error;
  const mappedErrorCode = errorCodeMap[errorCode];
  if (errorCode == null || mappedErrorCode == null) return error;
  // eslint-disable-next-line no-param-reassign
  error.errorCode = mappedErrorCode;
  return error;
};

const getMediaServerAdapter = (listenOnly = false) => {
  const SETTINGS = window.meetingClientSettings;
  const MEDIA = SETTINGS.public.media;
  const DEFAULT_LISTENONLY_MEDIA_SERVER = SETTINGS.public.kurento.listenOnlyMediaServer;
  const DEFAULT_FULLAUDIO_MEDIA_SERVER = MEDIA.audio.fullAudioMediaServer;

  if (listenOnly) {
    return getFromMeetingSettings(
      'media-server-listenonly',
      DEFAULT_LISTENONLY_MEDIA_SERVER,
    );
  }

  return getFromMeetingSettings(
    'media-server-fullaudio',
    DEFAULT_FULLAUDIO_MEDIA_SERVER,
  );
};

const isTransparentListenOnlyEnabled = () => {
  const SETTINGS = window.meetingClientSettings;
  const MEDIA = SETTINGS.public.media;
  const TRANSPARENT_LISTEN_ONLY = MEDIA.transparentListenOnly;
  return getFromUserSettings(
    'bbb_transparent_listen_only',
    TRANSPARENT_LISTEN_ONLY,
  );
};

export default class SFUAudioBridge extends BaseAudioBridge {
  static getOfferingRole(isListenOnly) {
    const SETTINGS = window.meetingClientSettings;
    const MEDIA = SETTINGS.public.media;
    const LISTEN_ONLY_OFFERING = MEDIA.listenOnlyOffering;
    const FULLAUDIO_OFFERING = MEDIA.fullAudioOffering;
    return isListenOnly && !isTransparentListenOnlyEnabled()
      ? LISTEN_ONLY_OFFERING
      : (!isTransparentListenOnlyEnabled() && FULLAUDIO_OFFERING);
  }

  constructor(userData) {
    super();
    this.userId = userData.userId;
    this.name = userData.username;
    this.sessionToken = userData.sessionToken;
    this.broker = null;
    this.reconnecting = false;
    this.iceServers = [];
    this.bridgeName = BRIDGE_NAME;
    this.isListenOnly = false;
    this.bypassGUM = false;
    this.supportsTransparentListenOnly = isTransparentListenOnlyEnabled;

    this.handleTermination = this.handleTermination.bind(this);
  }

  get inputStream() {
    // Only return the stream if the broker is active and the role isn't recvonly
    // Input stream == actual input-capturing stream, not the one that's being played
    if (this.broker && this.role !== RECV_ROLE) {
      return this.broker.getLocalStream();
    }

    return null;
  }

  get role() {
    return this.broker?.role;
  }

  getBrokerRole({ hasInputStream }) {
    if (this.isListenOnly) {
      return isTransparentListenOnlyEnabled()
        ? PASSIVE_SENDRECV_ROLE
        : RECV_ROLE;
    }

    if (this.bypassGUM && !hasInputStream) return PASSIVE_SENDRECV_ROLE;

    return SENDRECV_ROLE;
  }

  setInputStream(stream) {
    if (this.broker == null) return null;

    return this.broker.setLocalStream(stream);
  }

  getPeerConnection() {
    if (!this.broker) return null;

    const { webRtcPeer } = this.broker;
    if (webRtcPeer) return webRtcPeer.peerConnection;
    return null;
  }

  // eslint-disable-next-line class-methods-use-this
  mediaStreamFactory(constraints) {
    return doGUM(constraints, true);
  }

  setConnectionTimeout() {
    if (this.connectionTimeout) this.clearConnectionTimeout();

    const SETTINGS = window.meetingClientSettings;
    const MEDIA = SETTINGS.public.media;
    const CONNECTION_TIMEOUT_MS = MEDIA.listenOnlyCallTimeout || 15000;

    this.connectionTimeout = setTimeout(() => {
      const error = new Error(`ICE negotiation timeout after ${CONNECTION_TIMEOUT_MS / 1000}s`);
      error.errorCode = 1010;
      // Duplicating key-vals because I can'decide settle on an error pattern - prlanzarin again
      error.errorCause = error.message;
      error.errorMessage = error.message;
      this.handleBrokerFailure(error);
    }, CONNECTION_TIMEOUT_MS);
  }

  clearConnectionTimeout() {
    if (this.connectionTimeout) {
      clearTimeout(this.connectionTimeout);
      this.connectionTimeout = null;
    }
  }

  dispatchAutoplayHandlingEvent(mediaElement) {
    const tagFailedEvent = new CustomEvent('audioPlayFailed', {
      detail: { mediaElement },
    });
    window.dispatchEvent(tagFailedEvent);
    this.callback({ status: this.baseCallStates.autoplayBlocked, bridge: this.bridgeName });
  }

  reconnect(options = {}) {
    // If broker has already started, fire the reconnecting callback so the user
    // knows what's going on
    if (this.broker.started) {
      this.callback({ status: this.baseCallStates.reconnecting, bridge: this.bridgeName });
    } else {
      // Otherwise: override termination handler so the ended callback doesn't get
      // triggered - this is a retry attempt and the user shouldn't be notified
      // yet.
      this.broker.onended = () => {};
    }

    this.broker.stop();
    this.reconnecting = true;
    this._startBroker({ isListenOnly: this.isListenOnly, ...options })
      .catch((error) => {
        // Error handling is a no-op because it will be "handled" in handleBrokerFailure
        logger.debug({
          logCode: 'sfuaudio_reconnect_failed',
          extraInfo: {
            errorMessage: error.errorMessage,
            reconnecting: this.reconnecting,
            bridge: this.bridgeName,
            role: this.role,
          },
        }, 'SFU audio reconnect failed');
      });
  }

  handleBrokerFailure(error) {
    return new Promise((resolve, reject) => {
      this.clearConnectionTimeout();
      mapErrorCode(error);
      const { errorMessage, errorCause, errorCode } = error;
      const SETTINGS = window.meetingClientSettings;
      const MEDIA = SETTINGS.public.media;
      const RETRY_THROUGH_RELAY = MEDIA.audio.retryThroughRelay || false;

      if (!this.reconnecting) {
        if (this.broker.started) {
          logger.error({
            logCode: 'sfuaudio_error_try_to_reconnect',
            extraInfo: {
              errorMessage,
              errorCode,
              errorCause,
              bridge: this.bridgeName,
              role: this.role,
            },
          }, 'SFU audio failed, try to reconnect');
          this.reconnect();
          return resolve();
        }

        if (RETRYABLE_ERRORS.includes(errorCode) && RETRY_THROUGH_RELAY) {
          logger.error({
            logCode: 'sfuaudio_error_retry_through_relay',
            extraInfo: {
              errorMessage,
              errorCode,
              errorCause,
              bridge: this.bridgeName,
              role: this.role,
            },
          }, 'SFU audio failed to connect, retry through relay');
          this.reconnect({ forceRelay: true });
          return resolve();
        }
      }

      // Already tried reconnecting once OR the user handn't successfully
      // connected firsthand and retrying isn't an option. Finish the session
      // and reject with the error
      logger.error({
        logCode: 'sfuaudio_error',
        extraInfo: {
          errorMessage,
          errorCode,
          errorCause,
          reconnecting: this.reconnecting,
          bridge: this.bridgeName,
          role: this.role,
        },
      }, 'SFU audio failed');
      this.clearConnectionTimeout();
      this.broker.stop();
      this.callback({
        status: this.baseCallStates.failed,
        error: errorCode,
        bridgeError: errorMessage,
        bridge: this.bridgeName,
      });
      return reject(error);
    });
  }

  handleTermination() {
    this.clearConnectionTimeout();
    return this.callback({ status: this.baseCallStates.ended, bridge: this.bridgeName });
  }

  handleStart() {
    const SETTINGS = window.meetingClientSettings;
    const MEDIA = SETTINGS.public.media;
    const MEDIA_TAG = MEDIA.mediaTag.replace(/#/g, '');
    const stream = this.broker.webRtcPeer.getRemoteStream();
    const mediaElement = document.getElementById(MEDIA_TAG);

    return loadAndPlayMediaStream(stream, mediaElement, false).then(() => {
      this.callback({
        status: this.baseCallStates.started,
        bridge: this.bridgeName,
      });
      this.clearConnectionTimeout();
      this.reconnecting = false;
    }).catch((error) => {
      // NotAllowedError equals autoplay issues, fire autoplay handling event.
      // This will be handled in audio-manager.
      if (error.name === 'NotAllowedError') {
        logger.error({
          logCode: 'sfuaudio_error_autoplay',
          extraInfo: {
            errorName: error.name,
            bridge: this.bridgeName,
            role: this.role,
          },
        }, 'SFU audio media play failed due to autoplay error');
        this.dispatchAutoplayHandlingEvent(mediaElement);
        // For connection purposes, this worked - the autoplay thing is a client
        // side soft issue to be handled at the UI/UX level, not WebRTC/negotiation
        // So: clear the connection timer
        this.clearConnectionTimeout();
        this.reconnecting = false;
      } else {
        const normalizedError = {
          errorCode: 1004,
          errorMessage: error.message || 'AUDIO_PLAY_FAILED',
        };
        this.callback({
          status: this.baseCallStates.failed,
          error: normalizedError.errorCode,
          bridgeError: normalizedError.errorMessage,
          bridge: this.bridgeName,
        });
        throw normalizedError;
      }
    });
  }

  async _startBroker(options) {
    try {
      this.iceServers = await fetchWebRTCMappedStunTurnServers(this.sessionToken);
    } catch (error) {
      logger.error({ logCode: 'sfuaudio_stun-turn_fetch_failed' },
        'SFU audio bridge failed to fetch STUN/TURN info, using default servers');
      this.iceServers = getMappedFallbackStun();
    }

    return new Promise((resolve, reject) => {
      const {
        isListenOnly,
        extension,
        inputStream,
        forceRelay: _forceRelay = false,
        bypassGUM = false,
      } = options;

      const SETTINGS = window.meetingClientSettings;
      const MEDIA = SETTINGS.public.media;
      const SIGNAL_CANDIDATES = SETTINGS.public.kurento.signalCandidates;
      const SFU_URL = SETTINGS.public.kurento.wsUrl;
      const TRACE_LOGS = SETTINGS.public.kurento.traceLogs;
      const GATHERING_TIMEOUT = SETTINGS.public.kurento.gatheringTimeout;
      const RETRY_THROUGH_RELAY = MEDIA.audio.retryThroughRelay || false;
      const { audio: NETWORK_PRIORITY } = MEDIA.networkPriorities || {};
      const {
        enabled: RESTART_ICE = false,
        retries: RESTART_ICE_RETRIES = 1,
      } = SETTINGS.public.kurento?.restartIce?.audio || {};

      const handleInitError = (_error) => {
        mapErrorCode(_error);
        if (!RETRYABLE_ERRORS.includes(_error?.errorCode)
          || !RETRY_THROUGH_RELAY
          || this.reconnecting) {
          reject(_error);
        }
      };

      try {
        this.inEchoTest = !!extension;
        this.isListenOnly = isListenOnly;
        this.bypassGUM = bypassGUM;
        const role = this.getBrokerRole({
          hasInputStream: !!inputStream,
        });

        const brokerOptions = {
          clientSessionNumber: getAudioSessionNumber(),
          extension,
          iceServers: this.iceServers,
          mediaServer: getMediaServerAdapter(isListenOnly),
          constraints: getAudioConstraints({ deviceId: this.inputDeviceId }),
          forceRelay: _forceRelay || shouldForceRelay(),
          stream: (inputStream && inputStream.active) ? inputStream : undefined,
          offering: SFUAudioBridge.getOfferingRole(this.isListenOnly),
          signalCandidates: SIGNAL_CANDIDATES,
          traceLogs: TRACE_LOGS,
          networkPriority: NETWORK_PRIORITY,
          mediaStreamFactory: this.mediaStreamFactory,
          gatheringTimeout: GATHERING_TIMEOUT,
          transparentListenOnly: isTransparentListenOnlyEnabled(),
          bypassGUM,
          // ICE restart only works for publishers right now - recvonly full
          // reconnection works ok without it.
          restartIce: RESTART_ICE && !isListenOnly,
          restartIceMaxRetries: RESTART_ICE_RETRIES,
        };

        this.broker = new AudioBroker(
          Auth.authenticateURL(SFU_URL),
          role,
          brokerOptions,
        );

        this.broker.onended = this.handleTermination.bind(this);
        this.broker.onerror = (error) => {
          this.handleBrokerFailure(error).catch(reject);
        };
        this.broker.onstart = () => {
          this.handleStart().then(resolve).catch(reject);
        };

        // Set up a connectionTimeout in case the server or network are botching
        // negotiation or conn checks.
        this.setConnectionTimeout();
        this.broker.joinAudio().catch(handleInitError);
      } catch (error) {
        handleInitError(error);
      }
    });
  }

  async joinAudio(options, callback) {
    this.callback = callback;
    this.reconnecting = false;

    return this._startBroker(options);
  }

  sendDtmf(tones) {
    if (this.broker) {
      this.broker.dtmf(tones);
    }
  }

  transferCall(onTransferSuccess) {
    this.inEchoTest = false;
    return this.trackTransferState(onTransferSuccess);
  }

  async updateAudioConstraints(constraints) {
    try {
      if (typeof constraints !== 'object') return;

      const matchConstraints = filterSupportedConstraints(constraints);

      if (IS_CHROME) {
        matchConstraints.deviceId = this.inputDeviceId;
        const stream = await doGUM({ audio: matchConstraints });
        await this.setInputStream(stream);
      } else {
        this.inputStream.getAudioTracks()
          .forEach((track) => track.applyConstraints(matchConstraints));
      }
    } catch (error) {
      logger.error({
        logCode: 'sfuaudio_audio_constraint_error',
        extraInfo: {
          errorCode: error.code,
          errorMessage: error.message,
          bridgeName: this.bridgeName,
          role: this.role,
        },
      }, 'Failed to update audio constraint');
    }
  }

  trickleIce() {
    return new Promise((resolve, reject) => {
      try {
        fetchWebRTCMappedStunTurnServers(this.sessionToken)
          .then((iceServers) => {
            const SETTINGS = window.meetingClientSettings;
            const MEDIA = SETTINGS.public.media;
            const SFU_URL = SETTINGS.public.kurento.wsUrl;
            const TRACE_LOGS = SETTINGS.public.kurento.traceLogs;
            const GATHERING_TIMEOUT = SETTINGS.public.kurento.gatheringTimeout;
            const LISTEN_ONLY_OFFERING = MEDIA.listenOnlyOffering;

            const options = {
              clientSessionNumber: getAudioSessionNumber(),
              iceServers,
              offering: LISTEN_ONLY_OFFERING,
              traceLogs: TRACE_LOGS,
              gatheringTimeout: GATHERING_TIMEOUT,
            };

            this.broker = new AudioBroker(
              Auth.authenticateURL(SFU_URL),
              RECV_ROLE,
              options,
            );

            this.broker.onstart = () => {
              const { peerConnection } = this.broker.webRtcPeer;

              if (!peerConnection) return resolve(null);

              const selectedCandidatePair = peerConnection.getReceivers()[0]
                .transport.iceTransport.getSelectedCandidatePair();

              const validIceCandidate = [selectedCandidatePair.local];

              this.broker.stop();
              return resolve(validIceCandidate);
            };

            this.broker.joinAudio().catch(reject);
          });
      } catch (error) {
        // Rollback
        this.exitAudio();
        reject(error);
      }
    });
  }

  exitAudio() {
    const SETTINGS = window.meetingClientSettings;
    const MEDIA = SETTINGS.public.media;
    const MEDIA_TAG = MEDIA.mediaTag.replace(/#/g, '');
    const mediaElement = document.getElementById(MEDIA_TAG);

    this.clearConnectionTimeout();
    this.reconnecting = false;

    if (this.broker) {
      this.broker.stop();
      this.broker = null;
    }

    if (mediaElement && typeof mediaElement.pause === 'function') {
      mediaElement.pause();
      mediaElement.srcObject = null;
    }

    return Promise.resolve();
  }
}