Merge pull request #12705 from mariogasparoni/feat-trickle-ice-mic

feat(audio): use kurento's trickle-ice to improve mic negotiation
2021-07-02 17:44:05 -04:00 · 2021-07-02 17:44:05 -04:00 · 0150862eb1
commit 0150862eb1
parent 0457868e4b c4abd274e8
5 changed files with 204 additions and 11 deletions
--- a/bigbluebutton-html5/imports/api/audio/client/bridge/kurento.js
+++ b/bigbluebutton-html5/imports/api/audio/client/bridge/kurento.js
@ -193,6 +193,48 @@ export default class KurentoAudioBridge extends BaseAudioBridge {
    });
  }

+  trickleIce() {
+    return new Promise((resolve, reject) => {
+      try {
+        fetchWebRTCMappedStunTurnServers(this.sessionToken)
+          .then((iceServers) => {
+            const options = {
+              userName: this.name,
+              caleeName: `${GLOBAL_AUDIO_PREFIX}${this.voiceBridge}`,
+              iceServers,
+            };
+
+            this.broker = new ListenOnlyBroker(
+              Auth.authenticateURL(SFU_URL),
+              this.voiceBridge,
+              this.userId,
+              this.internalMeetingID,
+              RECV_ROLE,
+              options,
+            );
+
+            this.broker.onstart = () => {
+              const { peerConnection } = this.broker.webRtcPeer;
+
+              if (!peerConnection) return resolve(null);
+
+              const selectedCandidatePair = peerConnection.getReceivers()[0]
+                .transport.iceTransport.getSelectedCandidatePair();
+
+              const validIceCandidate = [selectedCandidatePair.local];
+
+              this.broker.stop();
+              return resolve(validIceCandidate);
+            };
+
+            this.broker.listen();
+          });
+      } catch (e) {
+        reject(e);
+      }
+    });
+  }
+
  joinAudio({ isListenOnly }, callback) {
    return new Promise(async (resolve, reject) => {
      if (!isListenOnly) return reject(new Error('Invalid bridge option'));
--- a/bigbluebutton-html5/imports/api/audio/client/bridge/sip.js
+++ b/bigbluebutton-html5/imports/api/audio/client/bridge/sip.js
@ -9,6 +9,7 @@ import {
  toUnifiedPlan,
  toPlanB,
  stripMDnsCandidates,
+  filterValidIceCandidates,
  analyzeSdp,
  logSelectedCandidate,
 } from '/imports/utils/sdpUtils';
@ -227,6 +228,7 @@ class SIPSession {
    extension,
    inputDeviceId,
    outputDeviceId,
+    validIceCandidates,
  }, managerCallback) {
    return new Promise((resolve, reject) => {
      const callExtension = extension ? `${extension}${this.userData.voiceBridge}` : this.userData.voiceBridge;
@ -254,6 +256,8 @@ class SIPSession {
      // If there's an extension passed it means that we're joining the echo test first
      this.inEchoTest = !!extension;

+      this.validIceCandidates = validIceCandidates;
+
      return this.doCall({
        callExtension,
        isListenOnly,
@ -712,6 +716,58 @@ class SIPSession {
    });
  }

+  isValidIceCandidate(event) {
+    return event.candidate
+      && this.validIceCandidates
+      && this.validIceCandidates.find((validCandidate) => (
+        (validCandidate.address === event.candidate.address)
+        || (validCandidate.relatedAddress === event.candidate.address))
+        && (validCandidate.protocol === event.candidate.protocol));
+  }
+
+  onIceGatheringStateChange(event) {
+    const secondsToGatherIce = (new Date() - this._sessionStartTime) / 1000;
+
+    const iceGatheringState = event.target
+      ? event.target.iceGatheringState
+      : null;
+
+    if (iceGatheringState === 'complete') {
+      logger.info({
+        logCode: 'sip_js_ice_gathering_time',
+        extraInfo: {
+          callerIdName: this.user.callerIdName,
+        },
+      }, `ICE gathering candidates took (s): ${secondsToGatherIce}`);
+    }
+  }
+
+  onIceCandidate(sessionDescriptionHandler, event) {
+    if (this.isValidIceCandidate(event)) {
+      logger.info({
+        logCode: 'sip_js_found_valid_candidate_from_trickle_ice',
+        extraInfo: {
+          callerIdName: this.user.callerIdName,
+        },
+      }, 'Found a valid candidate from trickle ICE, finishing gathering');
+
+      if (sessionDescriptionHandler.iceGatheringCompleteResolve) {
+        sessionDescriptionHandler.iceGatheringCompleteResolve();
+      }
+    }
+  }
+
+  initSessionDescriptionHandler(sessionDescriptionHandler) {
+    /* eslint-disable no-param-reassign */
+    sessionDescriptionHandler.peerConnectionDelegate = {
+      onicecandidate:
+        this.onIceCandidate.bind(this, sessionDescriptionHandler),
+      onicegatheringstatechange:
+        this.onIceGatheringStateChange.bind(this),
+    };
+    /* eslint-enable no-param-reassign */
+  }
+
  inviteUserAgent(userAgent) {
    return new Promise((resolve, reject) => {
      if (this.userRequestedHangup === true) reject();
@ -724,6 +780,7 @@ class SIPSession {
        isListenOnly,
      } = this.callOptions;

+      this._sessionStartTime = new Date();

      const target = SIP.UserAgent.makeURI(`sip:${callExtension}@${hostname}`);

@ -739,11 +796,16 @@ class SIPSession {
          },
          iceGatheringTimeout: ICE_GATHERING_TIMEOUT,
        },
-        sessionDescriptionHandlerModifiersPostICEGathering:
-          [stripMDnsCandidates],
+        sessionDescriptionHandlerModifiersPostICEGathering: [
+          stripMDnsCandidates,
+          filterValidIceCandidates.bind(this, this.validIceCandidates),
+        ],
+        delegate: {
+          onSessionDescriptionHandler:
+            this.initSessionDescriptionHandler.bind(this),
+        },
      };

-
      if (isListenOnly) {
        inviterOptions.sessionDescriptionHandlerOptions.offerOptions = {
          offerToReceiveAudio: true,
@ -919,8 +981,8 @@ class SIPSession {
          },
        }, 'Audio call session progress update');

-        this.currentSession.sessionDescriptionHandler.peerConnectionDelegate = {
-          onconnectionstatechange: (event) => {
+        this.currentSession.sessionDescriptionHandler.peerConnectionDelegate
+          .onconnectionstatechange = (event) => {
            const peer = event.target;

            logger.info({
@ -940,8 +1002,10 @@ class SIPSession {
              default:
                break;
            }
-          },
-          oniceconnectionstatechange: (event) => {
+          };
+
+        this.currentSession.sessionDescriptionHandler.peerConnectionDelegate
+          .oniceconnectionstatechange = (event) => {
            const peer = event.target;

            switch (peer.iceConnectionState) {
@ -989,8 +1053,7 @@ class SIPSession {
              default:
                break;
            }
-          },
-        };
+          };
      };

      const handleSessionTerminated = (message) => {
@ -1255,7 +1318,7 @@ export default class SIPBridge extends BaseAudioBridge {
    return this.activeSession ? this.activeSession.inputStream : null;
  }

-  joinAudio({ isListenOnly, extension }, managerCallback) {
+  joinAudio({ isListenOnly, extension, validIceCandidates }, managerCallback) {
    const hasFallbackDomain = typeof IPV4_FALLBACK_DOMAIN === 'string' && IPV4_FALLBACK_DOMAIN !== '';

    return new Promise((resolve, reject) => {
@ -1293,6 +1356,7 @@ export default class SIPBridge extends BaseAudioBridge {
              extension: fallbackExtension,
              inputDeviceId,
              outputDeviceId,
+              validIceCandidates,
            }, callback)
              .then((value) => {
                this.changeOutputDevice(outputDeviceId, true);
@ -1312,6 +1376,7 @@ export default class SIPBridge extends BaseAudioBridge {
        extension,
        inputDeviceId,
        outputDeviceId,
+        validIceCandidates,
      }, callback)
        .then((value) => {
          this.changeOutputDevice(outputDeviceId, true);
--- a/bigbluebutton-html5/imports/ui/services/audio-manager/index.js
+++ b/bigbluebutton-html5/imports/ui/services/audio-manager/index.js
@ -11,6 +11,7 @@ import iosWebviewAudioPolyfills from '/imports/utils/ios-webview-audio-polyfills
 import { monitorAudioConnection } from '/imports/utils/stats';
 import AudioErrors from './error-codes';
 import {Meteor} from "meteor/meteor";
+import browserInfo from '/imports/utils/browserInfo';

 const STATS = Meteor.settings.public.stats;
 const MEDIA = Meteor.settings.public.media;
@ -20,6 +21,8 @@ const MAX_LISTEN_ONLY_RETRIES = 1;
 const LISTEN_ONLY_CALL_TIMEOUT_MS = MEDIA.listenOnlyCallTimeout || 25000;
 const DEFAULT_INPUT_DEVICE_ID = 'default';
 const DEFAULT_OUTPUT_DEVICE_ID = 'default';
+const EXPERIMENTAL_USE_KMS_TRICKLE_ICE_FOR_MICROPHONE = Meteor.settings
+  .public.app.experimentalUseKmsTrickleIceForMicrophone;

 const CALL_STATES = {
  STARTED: 'started',
@ -109,7 +112,29 @@ class AudioManager {
    });
  }

+  async trickleIce() {
+    const { isFirefox, isIe, isSafari } = browserInfo;
+
+    if (!this.listenOnlyBridge
+      || isFirefox
+      || isIe
+      || isSafari) return [];
+
+    if (this.validIceCandidates && this.validIceCandidates.length) {
+      logger.info({ logCode: 'audiomanager_trickle_ice_reuse_candidate' },
+        'Reusing trickle-ice information before activating microphone');
+      return this.validIceCandidates;
+    }
+
+    logger.info({ logCode: 'audiomanager_trickle_ice_get_local_candidate' },
+      'Performing trickle-ice before activating microphone');
+    this.validIceCandidates = await this.listenOnlyBridge.trickleIce() || [];
+    return this.validIceCandidates;
+  }
+
  joinMicrophone() {
+    this.audioJoinStartTime = new Date();
+    this.logAudioJoinTime = false;
    this.isListenOnly = false;
    this.isEchoTest = false;

@ -125,15 +150,23 @@ class AudioManager {
  }

  joinEchoTest() {
+    this.audioJoinStartTime = new Date();
+    this.logAudioJoinTime = false;
    this.isListenOnly = false;
    this.isEchoTest = true;

    return this.onAudioJoining.bind(this)()
-      .then(() => {
+      .then(async () => {
+        let validIceCandidates = [];
+        if (EXPERIMENTAL_USE_KMS_TRICKLE_ICE_FOR_MICROPHONE) {
+          validIceCandidates = await this.trickleIce();
+        }
+
        const callOptions = {
          isListenOnly: false,
          extension: ECHO_TEST_NUMBER,
          inputStream: this.inputStream,
+          validIceCandidates,
        };
        logger.info({ logCode: 'audiomanager_join_echotest', extraInfo: { logType: 'user_action' } }, 'User requested to join audio conference with mic');
        return this.joinAudio(callOptions, this.callStateCallback.bind(this));
@ -183,6 +216,8 @@ class AudioManager {
  }

  async joinListenOnly(r = 0) {
+    this.audioJoinStartTime = new Date();
+    this.logAudioJoinTime = false;
    let retries = r;
    this.isListenOnly = true;
    this.isEchoTest = false;
@ -333,6 +368,13 @@ class AudioManager {
        changed: (id, fields) => this.onVoiceUserChanges(fields),
      });
    }
+    const secondsToActivateAudio = (new Date() - this.audioJoinStartTime) / 1000;
+
+    if (!this.logAudioJoinTime) {
+      this.logAudioJoinTime = true;
+      logger.info({ logCode: 'audio_mic_join_time' }, 'Time needed to '
+      + `connect audio (seconds): ${secondsToActivateAudio}`);
+    }

    if (!this.isEchoTest) {
      window.parent.postMessage({ response: 'joinedAudio' }, '*');
--- a/bigbluebutton-html5/imports/utils/sdpUtils.js
+++ b/bigbluebutton-html5/imports/utils/sdpUtils.js
@ -65,6 +65,40 @@ const stripMDnsCandidates = (sdp) => {
  return { sdp: transform.write(parsedSDP), type: sdp.type };
 };

+const filterValidIceCandidates = (validIceCandidates, sdp) => {
+  if (!validIceCandidates.length) return sdp;
+
+  const matchCandidatesIp = (candidate, mediaCandidate) => (
+    (candidate.address && candidate.address.includes(mediaCandidate.ip))
+    || (candidate.relatedAddress
+      && candidate.relatedAddress.includes(mediaCandidate.ip))
+  );
+
+  const parsedSDP = transform.parse(sdp.sdp);
+  let strippedCandidates = 0;
+  parsedSDP.media.forEach((media) => {
+    if (media.candidates) {
+      media.candidates = media.candidates.filter((candidate) => {
+        if (candidate.ip
+          && candidate.type
+          && candidate.transport
+          && validIceCandidates.find((c) => (c.protocol === candidate.transport)
+              && matchCandidatesIp(c, candidate))
+        ) {
+          return true;
+        }
+        strippedCandidates += 1;
+        return false;
+      });
+    }
+  });
+  if (strippedCandidates > 0) {
+    logger.info({ logCode: 'sdp_utils_mdns_candidate_strip' },
+      `Filtered ${strippedCandidates} invalid candidates from trickle SDP`);
+  }
+  return { sdp: transform.write(parsedSDP), type: sdp.type };
+};
+
 const isPublicIpv4 = (ip) => {
  const ipParts = ip.split('.');
  switch (ipParts[0]) {
@ -305,6 +339,7 @@ export {
  toPlanB,
  toUnifiedPlan,
  stripMDnsCandidates,
+  filterValidIceCandidates,
  analyzeSdp,
  logSelectedCandidate,
 };
--- a/bigbluebutton-html5/private/config/settings.yml
+++ b/bigbluebutton-html5/private/config/settings.yml
@ -79,6 +79,15 @@ public:
    showAudioFilters: true
    raiseHandActionButton:
      enabled: true
+    # If enabled, before joining microphone the client will perform a trickle
+    # ICE against Kurento and use the information about successfull
+    # candidate-pairs to filter out local candidates in SIP.js's SDP.
+    # Try enabling this setting in scenarios where the listenonly mode works,
+    # but microphone doesn't (for example, when using VPN).
+    # For compatibility check "Browser compatbility" section in:
+    # https://developer.mozilla.org/en-US/docs/Web/API/RTCDtlsTransport/iceTransport
+    # This is an EXPERIMENTAL setting and the default value is false
+    # experimentalUseKmsTrickleIceForMicrophone: false
    defaultSettings:
      application:
        animations: true