From 75c8dcd491dc229a44103478441d2dcfbbf6b13d Mon Sep 17 00:00:00 2001 From: Tiago Jacobs Date: Wed, 16 Feb 2022 21:51:39 -0300 Subject: [PATCH 1/6] Merge 2.6 --- bigbluebutton-html5/client/main.jsx | 1 + .../actions-bar/screenshare/component.jsx | 4 +- .../ui/components/screenshare/service.js | 6 + .../imports/ui/services/mobile-app/index.js | 191 ++++++++++++++++++ .../imports/utils/browserInfo.js | 8 +- .../imports/utils/deviceInfo.js | 5 +- 6 files changed, 209 insertions(+), 6 deletions(-) create mode 100644 bigbluebutton-html5/imports/ui/services/mobile-app/index.js diff --git a/bigbluebutton-html5/client/main.jsx b/bigbluebutton-html5/client/main.jsx index 5cb59f24bb..a42c539989 100755 --- a/bigbluebutton-html5/client/main.jsx +++ b/bigbluebutton-html5/client/main.jsx @@ -21,6 +21,7 @@ import React from 'react'; import { Meteor } from 'meteor/meteor'; import { render } from 'react-dom'; import logger from '/imports/startup/client/logger'; +import '/imports/ui/services/mobile-app'; import Base from '/imports/startup/client/base'; import JoinHandler from '/imports/ui/components/join-handler/component'; import AuthenticatedHandler from '/imports/ui/components/authenticated-handler/component'; diff --git a/bigbluebutton-html5/imports/ui/components/actions-bar/screenshare/component.jsx b/bigbluebutton-html5/imports/ui/components/actions-bar/screenshare/component.jsx index 382176f7dc..cd40c7179d 100755 --- a/bigbluebutton-html5/imports/ui/components/actions-bar/screenshare/component.jsx +++ b/bigbluebutton-html5/imports/ui/components/actions-bar/screenshare/component.jsx @@ -15,7 +15,7 @@ import { import { SCREENSHARING_ERRORS } from '/imports/api/screenshare/client/bridge/errors'; const { isMobile } = deviceInfo; -const { isSafari } = browserInfo; +const { isSafari, isMobileApp } = browserInfo; const propTypes = { intl: PropTypes.objectOf(Object).isRequired, @@ -167,7 +167,7 @@ const ScreenshareButton = ({ ? intlMessages.stopDesktopShareDesc : intlMessages.desktopShareDesc; const shouldAllowScreensharing = enabled - && !isMobile + && ( !isMobile || isMobileApp) && amIPresenter; const dataTest = !screenshareDataSavingSetting ? 'screenshareLocked' diff --git a/bigbluebutton-html5/imports/ui/components/screenshare/service.js b/bigbluebutton-html5/imports/ui/components/screenshare/service.js index 22b314c596..9c8c73de99 100644 --- a/bigbluebutton-html5/imports/ui/components/screenshare/service.js +++ b/bigbluebutton-html5/imports/ui/components/screenshare/service.js @@ -10,6 +10,7 @@ import AudioService from '/imports/ui/components/audio/service'; import { Meteor } from "meteor/meteor"; import MediaStreamUtils from '/imports/utils/media-stream-utils'; import ConnectionStatusService from '/imports/ui/components/connection-status/service'; +import browserInfo from '/imports/utils/browserInfo'; const VOLUME_CONTROL_ENABLED = Meteor.settings.public.kurento.screenshare.enableVolumeControl; const SCREENSHARE_MEDIA_ELEMENT_NAME = 'screenshareVideo'; @@ -122,6 +123,11 @@ const getVolume = () => KurentoBridge.getVolume(); const shouldEnableVolumeControl = () => VOLUME_CONTROL_ENABLED && screenshareHasAudio(); const attachLocalPreviewStream = (mediaElement) => { + const {isMobileApp} = browserInfo; + if (isMobileApp) { + // We don't show preview for mobile app, as the stream is only available in native code + return; + } const stream = KurentoBridge.gdmStream; if (stream && mediaElement) { // Always muted, presenter preview. diff --git a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js new file mode 100644 index 0000000000..c4f7ec723f --- /dev/null +++ b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js @@ -0,0 +1,191 @@ +import browserInfo from '/imports/utils/browserInfo'; +import logger from '/imports/startup/client/logger'; +(function (){ + // This function must be executed during the import time, that's why it's not exported to the caller component. + // It's needed because it changes some functions provided by browser, and these functions are verified during + // import time (like in ScreenshareBridgeService) + if(browserInfo.isMobileApp) { + logger.debug("Mobile APP detected"); + + // Store the method call sequential + const sequenceHolder = {sequence: 0}; + + // Store the promise for each method call + const promisesHolder = {}; + + // Call a method in the mobile application, returning a promise for its execution + function callNativeMethod(method, arguments=[]) { + try { + const sequence = ++sequenceHolder.sequence; + + return new Promise ( (resolve, reject) => { + promisesHolder[sequence] = { + resolve, reject + }; + + window.ReactNativeWebView.postMessage(JSON.stringify({ + sequence: sequenceHolder.sequence, + method: method, + arguments: arguments, + })); + } ); + } catch(e) { + logger.error(`Error on callNativeMethod ${e.message}`, e); + } + } + + // This method is called from the mobile app to notify us about a method invocation result + window.nativeMethodCallResult = (sequence, isResolve, resultOrException) => { + + const promise = promisesHolder[sequence]; + if(promise) { + if(isResolve) { + promise.resolve( resultOrException ); + delete promisesHolder[sequence]; + } else { + promise.reject( resultOrException ); + delete promisesHolder[sequence]; + } + } + return true; + } + + // WebRTC replacement functions + const buildVideoTrack = function () {} + const stream = {}; + + // Navigator + navigator.getDisplayMedia = function() { + logger.info("BBB-MOBILE - getDisplayMedia called", arguments); + + return new Promise((resolve, reject) => { + callNativeMethod('initializeScreenShare').then( + () => { + const fakeVideoTrack = {}; + fakeVideoTrack.applyConstraints = function (constraints) { + return new Promise( + (resolve, reject) => { + // alert("Constraints applied: " + JSON.stringify(constraints)); + resolve(); + } + ); + }; + fakeVideoTrack.onended = null; // callbacks added from screenshare (we can use it later) + fakeVideoTrack.oninactive = null; // callbacks added from screenshare (we can use it later) + + const videoTracks = [ + fakeVideoTrack + ]; + stream.getTracks = stream.getVideoTracks = function () { + return videoTracks; + }; + resolve(stream); + } + ).catch( + () => alert("Não deu") + ); + }); + } + + // RTCPeerConnection + const prototype = window.RTCPeerConnection.prototype; + + prototype.createOffer = function (options) { + logger.info("BBB-MOBILE - createOffer called", {options}); + + return new Promise( (resolve, reject) => { + callNativeMethod('createOffer').then ( sdp => { + logger.info("BBB-MOBILE - createOffer resolved", {sdp}); + + // + resolve({ + type: 'offer', + sdp + }); + }); + } ); + }; + + prototype.addEventListener = function (event, callback) { + logger.info("BBB-MOBILE - addEventListener called", {event, callback}); + + switch(event) { + case 'icecandidate': + window.bbbMobileScreenShareIceCandidateCallback = function () { + console.log("Received a bbbMobileScreenShareIceCandidateCallback call with arguments", arguments); + if(callback){ + callback.apply(this, arguments); + } + return true; + } + break; + case 'signalingstatechange': + window.bbbMobileScreenShareSignalingStateChangeCallback = function (newState) { + this.signalingState = newState; + callback(); + }; + break; + } + } + + prototype.setLocalDescription = function (description, successCallback, failureCallback) { + logger.info("BBB-MOBILE - setLocalDescription called", {description, successCallback, failureCallback}); + + // store the value + this._localDescription = JSON.parse(JSON.stringify(description)); + // replace getter of localDescription to return this value + Object.defineProperty(this, 'localDescription', {get: function() {return this._localDescription;},set: function(newValue) {}}); + + // return a promise that resolves immediately + return new Promise( (resolve, reject) => { + resolve(); + }) + } + + prototype.setRemoteDescription = function (description, successCallback, failureCallback) { + logger.info("BBB-MOBILE - setRemoteDescription called", {description, successCallback, failureCallback}); + this._remoteDescription = JSON.parse(JSON.stringify(description)); + Object.defineProperty(this, 'remoteDescription', {get: function() {return this._remoteDescription;},set: function(newValue) {}}); + + return new Promise( (resolve, reject) => { + callNativeMethod('setRemoteDescription', [description]).then ( () => { + logger.info("BBB-MOBILE - setRemoteDescription resolved"); + + resolve(); + }); + } ); + } + + prototype.addTrack = function (description, successCallback, failureCallback) { + logger.info("BBB-MOBILE - addTrack called", {description, successCallback, failureCallback}); + } + + prototype.getLocalStreams = function() { + logger.info("BBB-MOBILE - getLocalStreams called", arguments); + + // + return [ + stream + ]; + } + + prototype.addTransceiver = function() { + logger.info("BBB-MOBILE - addTransceiver called", arguments); + } + + prototype.addIceCandidate = function (candidate) { + logger.info("BBB-MOBILE - addIceCandidate called", {candidate}); + + return new Promise( (resolve, reject) => { + callNativeMethod('addRemoteIceCandidate', [candidate]).then ( () => { + logger.info("BBB-MOBILE - addRemoteIceCandidate resolved"); + + resolve(); + }); + } ); + } + + } +})(); + + diff --git a/bigbluebutton-html5/imports/utils/browserInfo.js b/bigbluebutton-html5/imports/utils/browserInfo.js index 2459cb8af6..98c4d69213 100755 --- a/bigbluebutton-html5/imports/utils/browserInfo.js +++ b/bigbluebutton-html5/imports/utils/browserInfo.js @@ -1,6 +1,7 @@ import Bowser from 'bowser'; -const BOWSER_RESULTS = Bowser.parse(window.navigator.userAgent); +const userAgent = window.navigator.userAgent; +const BOWSER_RESULTS = Bowser.parse(userAgent); const isChrome = BOWSER_RESULTS.browser.name === 'Chrome'; const isSafari = BOWSER_RESULTS.browser.name === 'Safari'; @@ -11,10 +12,12 @@ const isFirefox = BOWSER_RESULTS.browser.name === 'Firefox'; const browserName = BOWSER_RESULTS.browser.name; const versionNumber = BOWSER_RESULTS.browser.version; -const isValidSafariVersion = Bowser.getParser(window.navigator.userAgent).satisfies({ +const isValidSafariVersion = Bowser.getParser(userAgent).satisfies({ safari: '>12', }); +const isMobileApp = !!(userAgent.match(/BBBMobile/i)); + const browserInfo = { isChrome, isSafari, @@ -24,6 +27,7 @@ const browserInfo = { browserName, versionNumber, isValidSafariVersion, + isMobileApp }; export default browserInfo; diff --git a/bigbluebutton-html5/imports/utils/deviceInfo.js b/bigbluebutton-html5/imports/utils/deviceInfo.js index 9471862c58..3acb0e8c52 100755 --- a/bigbluebutton-html5/imports/utils/deviceInfo.js +++ b/bigbluebutton-html5/imports/utils/deviceInfo.js @@ -1,6 +1,7 @@ import Bowser from 'bowser'; -const BOWSER_RESULTS = Bowser.parse(window.navigator.userAgent); +const userAgent = window.navigator.userAgent; +const BOWSER_RESULTS = Bowser.parse(userAgent); const isPhone = BOWSER_RESULTS.platform.type === 'mobile'; // we need a 'hack' to correctly detect ipads with ios > 13 @@ -11,7 +12,7 @@ const osName = BOWSER_RESULTS.os.name; const osVersion = BOWSER_RESULTS.os.version; const isIos = osName === 'iOS'; const isMacos = osName === 'macOS'; -const isIphone = !!(window.navigator.userAgent.match(/iPhone/i)); +const isIphone = !!(userAgent.match(/iPhone/i)); const SUPPORTED_IOS_VERSION = 12.2; const isIosVersionSupported = () => parseFloat(osVersion) >= SUPPORTED_IOS_VERSION; From f459aec949e443d88b6f9ced8055bf01324d5a03 Mon Sep 17 00:00:00 2001 From: Tiago Jacobs Date: Sat, 16 Apr 2022 20:17:50 -0300 Subject: [PATCH 2/6] Fixes variable name to avoid colision with reserved arguments variable --- bigbluebutton-html5/imports/ui/services/mobile-app/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js index c4f7ec723f..2521e81daf 100644 --- a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js +++ b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js @@ -14,7 +14,7 @@ import logger from '/imports/startup/client/logger'; const promisesHolder = {}; // Call a method in the mobile application, returning a promise for its execution - function callNativeMethod(method, arguments=[]) { + function callNativeMethod(method, args=[]) { try { const sequence = ++sequenceHolder.sequence; @@ -26,7 +26,7 @@ import logger from '/imports/startup/client/logger'; window.ReactNativeWebView.postMessage(JSON.stringify({ sequence: sequenceHolder.sequence, method: method, - arguments: arguments, + arguments: args, })); } ); } catch(e) { From 092f5bcafdb6a2057ac4f72b8abb25c92b40d91e Mon Sep 17 00:00:00 2001 From: Tiago Jacobs Date: Sun, 17 Apr 2022 10:12:28 -0300 Subject: [PATCH 3/6] Add logic to skip webRTC replacement methods when it's not a screenshare --- .../imports/ui/services/mobile-app/index.js | 100 +++++++++++++++--- 1 file changed, 84 insertions(+), 16 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js index 2521e81daf..2a00d26a39 100644 --- a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js +++ b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js @@ -5,8 +5,39 @@ import logger from '/imports/startup/client/logger'; // It's needed because it changes some functions provided by browser, and these functions are verified during // import time (like in ScreenshareBridgeService) if(browserInfo.isMobileApp) { - logger.debug("Mobile APP detected"); + logger.debug(`BBB-MOBILE - Mobile APP detected`); + // This function detects if the call happened to publish a screenshare + function isScreenShareBroadcastRelated(caller, peerConnection = null, args = null) { + // Keep track of how many webRTC evaluations was done + if(!peerConnection.isSBREvaluations) + peerConnection.isSBREvaluations = 0; + + peerConnection.isSBREvaluations ++; + + // If already successfully evaluated, reuse + if(peerConnection && peerConnection.isSBR !== undefined ) { + logger.info(`BBB-MOBILE - isScreenShareBroadcastRelated (already evaluated as ${peerConnection.isSBR})`, {caller, peerConnection}); + return peerConnection.isSBR; + } + + // Evaluate context otherwise + const e = new Error('dummy'); + const stackTrace = e.stack; + logger.info(`BBB-MOBILE - isScreenShareBroadcastRelated (evaluating)`, {caller, peerConnection, stackTrace: stackTrace.split('\n'), isSBREvaluations: peerConnection.isSBREvaluations, args}); + + // addTransceiver is the first call for screensharing and it has a startScreensharing in its stackTrace + if( peerConnection.isSBREvaluations == 1) { + if(caller == 'addTransceiver' && stackTrace.indexOf('startScreensharing') !== -1) { + peerConnection.isSBR = true; + } else { + peerConnection.isSBR = false; + } + + return peerConnection.isSBR; + } + + } // Store the method call sequential const sequenceHolder = {sequence: 0}; @@ -27,7 +58,7 @@ import logger from '/imports/startup/client/logger'; sequence: sequenceHolder.sequence, method: method, arguments: args, - })); + })); } ); } catch(e) { logger.error(`Error on callNativeMethod ${e.message}`, e); @@ -56,7 +87,7 @@ import logger from '/imports/startup/client/logger'; // Navigator navigator.getDisplayMedia = function() { - logger.info("BBB-MOBILE - getDisplayMedia called", arguments); + logger.info(`BBB-MOBILE - getDisplayMedia called`, arguments); return new Promise((resolve, reject) => { callNativeMethod('initializeScreenShare').then( @@ -65,7 +96,6 @@ import logger from '/imports/startup/client/logger'; fakeVideoTrack.applyConstraints = function (constraints) { return new Promise( (resolve, reject) => { - // alert("Constraints applied: " + JSON.stringify(constraints)); resolve(); } ); @@ -82,7 +112,9 @@ import logger from '/imports/startup/client/logger'; resolve(stream); } ).catch( - () => alert("Não deu") + (e) => { + logger.error(`Failure calling native initializeScreenShare`, e.message) + } ); }); } @@ -90,12 +122,16 @@ import logger from '/imports/startup/client/logger'; // RTCPeerConnection const prototype = window.RTCPeerConnection.prototype; + prototype.originalCreateOffer = prototype.createOffer; prototype.createOffer = function (options) { - logger.info("BBB-MOBILE - createOffer called", {options}); + if(!isScreenShareBroadcastRelated('createOffer', this)){ + return prototype.originalCreateOffer.call(this, ...arguments); + } + logger.info(`BBB-MOBILE - createOffer called`, {options}); return new Promise( (resolve, reject) => { callNativeMethod('createOffer').then ( sdp => { - logger.info("BBB-MOBILE - createOffer resolved", {sdp}); + logger.info(`BBB-MOBILE - createOffer resolved`, {sdp}); // resolve({ @@ -106,13 +142,18 @@ import logger from '/imports/startup/client/logger'; } ); }; + prototype.originalAddEventListener = prototype.addEventListener; prototype.addEventListener = function (event, callback) { - logger.info("BBB-MOBILE - addEventListener called", {event, callback}); + if(!isScreenShareBroadcastRelated('addEventListener', this, arguments)){ + return prototype.originalAddEventListener.call(this, ...arguments); + } + + logger.info(`BBB-MOBILE - addEventListener called`, {event, callback}); switch(event) { case 'icecandidate': window.bbbMobileScreenShareIceCandidateCallback = function () { - console.log("Received a bbbMobileScreenShareIceCandidateCallback call with arguments", arguments); + logger.info("Received a bbbMobileScreenShareIceCandidateCallback call with arguments", arguments); if(callback){ callback.apply(this, arguments); } @@ -128,8 +169,12 @@ import logger from '/imports/startup/client/logger'; } } + prototype.originalSetLocalDescription = prototype.setLocalDescription; prototype.setLocalDescription = function (description, successCallback, failureCallback) { - logger.info("BBB-MOBILE - setLocalDescription called", {description, successCallback, failureCallback}); + if(!isScreenShareBroadcastRelated('setLocalDescription', this)){ + return prototype.originalSetLocalDescription.call(this, ...arguments); + } + logger.info(`BBB-MOBILE - setLocalDescription called`, {description, successCallback, failureCallback}); // store the value this._localDescription = JSON.parse(JSON.stringify(description)); @@ -142,26 +187,41 @@ import logger from '/imports/startup/client/logger'; }) } + prototype.originalSetRemoteDescription = prototype.setRemoteDescription; prototype.setRemoteDescription = function (description, successCallback, failureCallback) { - logger.info("BBB-MOBILE - setRemoteDescription called", {description, successCallback, failureCallback}); + if(!isScreenShareBroadcastRelated('setRemoteDescription', this)){ + return prototype.originalSetRemoteDescription.call(this, ...arguments); + } + + logger.info(`BBB-MOBILE - setRemoteDescription called`, {description, successCallback, failureCallback}); + this._remoteDescription = JSON.parse(JSON.stringify(description)); Object.defineProperty(this, 'remoteDescription', {get: function() {return this._remoteDescription;},set: function(newValue) {}}); return new Promise( (resolve, reject) => { callNativeMethod('setRemoteDescription', [description]).then ( () => { - logger.info("BBB-MOBILE - setRemoteDescription resolved"); + logger.info(`BBB-MOBILE - setRemoteDescription resolved`); resolve(); }); } ); } + prototype.originalAddTrack = prototype.addTrack; prototype.addTrack = function (description, successCallback, failureCallback) { - logger.info("BBB-MOBILE - addTrack called", {description, successCallback, failureCallback}); + if(!isScreenShareBroadcastRelated('addTrack', this)){ + return prototype.originalAddTrack.call(this, ...arguments); + } + + logger.info(`BBB-MOBILE - addTrack called`, {description, successCallback, failureCallback}); } + prototype.originalGetLocalStreams = prototype.getLocalStreams; prototype.getLocalStreams = function() { - logger.info("BBB-MOBILE - getLocalStreams called", arguments); + if(!isScreenShareBroadcastRelated('getLocalStreams', this)){ + return prototype.originalGetLocalStreams.call(this, ...arguments); + } + logger.info(`BBB-MOBILE - getLocalStreams called`, arguments); // return [ @@ -169,12 +229,20 @@ import logger from '/imports/startup/client/logger'; ]; } + prototype.originalAddTransceiver = prototype.addTransceiver; prototype.addTransceiver = function() { - logger.info("BBB-MOBILE - addTransceiver called", arguments); + if(!isScreenShareBroadcastRelated('addTransceiver', this)){ + return prototype.originalAddTransceiver.call(this, ...arguments); + } + logger.info(`BBB-MOBILE - addTransceiver called`, arguments); } + prototype.originalAddIceCandidate = prototype.addIceCandidate; prototype.addIceCandidate = function (candidate) { - logger.info("BBB-MOBILE - addIceCandidate called", {candidate}); + if(!isScreenShareBroadcastRelated('addIceCandidate', this)){ + return prototype.originalAddIceCandidate.call(this, ...arguments); + } + logger.info(`BBB-MOBILE - addIceCandidate called`, {candidate}); return new Promise( (resolve, reject) => { callNativeMethod('addRemoteIceCandidate', [candidate]).then ( () => { From 2b953a47bb236ed04f71cd933745b957d0bbd487 Mon Sep 17 00:00:00 2001 From: Tiago Jacobs Date: Wed, 20 Apr 2022 13:05:48 -0300 Subject: [PATCH 4/6] Add structure to detect and forward full audio calls to mobile application --- .../imports/ui/services/mobile-app/index.js | 76 +++++++++++++------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js index 2a00d26a39..ecf48718ac 100644 --- a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js +++ b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js @@ -7,34 +7,40 @@ import logger from '/imports/startup/client/logger'; if(browserInfo.isMobileApp) { logger.debug(`BBB-MOBILE - Mobile APP detected`); + const WEBRTC_CALL_TYPE_FULL_AUDIO = 'full_audio'; + const WEBRTC_CALL_TYPE_SCREEN_SHARE = 'screen_share'; + const WEBRTC_CALL_TYPE_STANDARD = 'standard'; + // This function detects if the call happened to publish a screenshare - function isScreenShareBroadcastRelated(caller, peerConnection = null, args = null) { + function detectWebRtcCallType(caller, peerConnection = null, args = null) { // Keep track of how many webRTC evaluations was done - if(!peerConnection.isSBREvaluations) - peerConnection.isSBREvaluations = 0; + if(!peerConnection.detectWebRtcCallTypeEvaluations) + peerConnection.detectWebRtcCallTypeEvaluations = 0; - peerConnection.isSBREvaluations ++; + peerConnection.detectWebRtcCallTypeEvaluations ++; // If already successfully evaluated, reuse - if(peerConnection && peerConnection.isSBR !== undefined ) { - logger.info(`BBB-MOBILE - isScreenShareBroadcastRelated (already evaluated as ${peerConnection.isSBR})`, {caller, peerConnection}); - return peerConnection.isSBR; + if(peerConnection && peerConnection.webRtcCallType !== undefined ) { + logger.info(`BBB-MOBILE - detectWebRtcCallType (already evaluated as ${peerConnection.webRtcCallType})`, {caller, peerConnection}); + return peerConnection.webRtcCallType; } // Evaluate context otherwise const e = new Error('dummy'); const stackTrace = e.stack; - logger.info(`BBB-MOBILE - isScreenShareBroadcastRelated (evaluating)`, {caller, peerConnection, stackTrace: stackTrace.split('\n'), isSBREvaluations: peerConnection.isSBREvaluations, args}); + logger.info(`BBB-MOBILE - detectWebRtcCallType (evaluating)`, {caller, peerConnection, stackTrace: stackTrace.split('\n'), detectWebRtcCallTypeEvaluations: peerConnection.detectWebRtcCallTypeEvaluations, args}); // addTransceiver is the first call for screensharing and it has a startScreensharing in its stackTrace - if( peerConnection.isSBREvaluations == 1) { + if( peerConnection.detectWebRtcCallTypeEvaluations == 1) { if(caller == 'addTransceiver' && stackTrace.indexOf('startScreensharing') !== -1) { - peerConnection.isSBR = true; + peerConnection.webRtcCallType = WEBRTC_CALL_TYPE_SCREEN_SHARE; // this uses mobile app broadcast upload extension + } else if(caller == 'addEventListener' && stackTrace.indexOf('invite') !== -1) { + peerConnection.webRtcCallType = WEBRTC_CALL_TYPE_FULL_AUDIO; // this uses mobile app webRTC } else { - peerConnection.isSBR = false; + peerConnection.webRtcCallType = WEBRTC_CALL_TYPE_STANDARD; // this uses the webview webRTC } - return peerConnection.isSBR; + return peerConnection.webRtcCallType; } } @@ -84,7 +90,7 @@ import logger from '/imports/startup/client/logger'; // WebRTC replacement functions const buildVideoTrack = function () {} const stream = {}; - + // Navigator navigator.getDisplayMedia = function() { logger.info(`BBB-MOBILE - getDisplayMedia called`, arguments); @@ -124,16 +130,20 @@ import logger from '/imports/startup/client/logger'; prototype.originalCreateOffer = prototype.createOffer; prototype.createOffer = function (options) { - if(!isScreenShareBroadcastRelated('createOffer', this)){ + const webRtcCallType = detectWebRtcCallType('createOffer', this); + + if(webRtcCallType === WEBRTC_CALL_TYPE_STANDARD){ return prototype.originalCreateOffer.call(this, ...arguments); } logger.info(`BBB-MOBILE - createOffer called`, {options}); + const createOfferMethod = (webRtcCallType === WEBRTC_CALL_TYPE_SCREEN_SHARE) ? 'createScreenShareOffer' : 'createFullAudioOffer'; + return new Promise( (resolve, reject) => { - callNativeMethod('createOffer').then ( sdp => { + callNativeMethod(createOfferMethod).then ( sdp => { logger.info(`BBB-MOBILE - createOffer resolved`, {sdp}); - // + // send offer to BBB code resolve({ type: 'offer', sdp @@ -144,7 +154,7 @@ import logger from '/imports/startup/client/logger'; prototype.originalAddEventListener = prototype.addEventListener; prototype.addEventListener = function (event, callback) { - if(!isScreenShareBroadcastRelated('addEventListener', this, arguments)){ + if(WEBRTC_CALL_TYPE_STANDARD === detectWebRtcCallType('addEventListener', this, arguments)){ return prototype.originalAddEventListener.call(this, ...arguments); } @@ -171,7 +181,7 @@ import logger from '/imports/startup/client/logger'; prototype.originalSetLocalDescription = prototype.setLocalDescription; prototype.setLocalDescription = function (description, successCallback, failureCallback) { - if(!isScreenShareBroadcastRelated('setLocalDescription', this)){ + if(WEBRTC_CALL_TYPE_STANDARD === detectWebRtcCallType('setLocalDescription', this)){ return prototype.originalSetLocalDescription.call(this, ...arguments); } logger.info(`BBB-MOBILE - setLocalDescription called`, {description, successCallback, failureCallback}); @@ -189,7 +199,8 @@ import logger from '/imports/startup/client/logger'; prototype.originalSetRemoteDescription = prototype.setRemoteDescription; prototype.setRemoteDescription = function (description, successCallback, failureCallback) { - if(!isScreenShareBroadcastRelated('setRemoteDescription', this)){ + const webRtcCallType = detectWebRtcCallType('setRemoteDescription', this); + if(WEBRTC_CALL_TYPE_STANDARD === webRtcCallType){ return prototype.originalSetRemoteDescription.call(this, ...arguments); } @@ -197,28 +208,38 @@ import logger from '/imports/startup/client/logger'; this._remoteDescription = JSON.parse(JSON.stringify(description)); Object.defineProperty(this, 'remoteDescription', {get: function() {return this._remoteDescription;},set: function(newValue) {}}); + + const setRemoteDescriptionMethod = (webRtcCallType === WEBRTC_CALL_TYPE_SCREEN_SHARE) ? 'setScreenShareRemoteSDP' : 'setFullAudioRemoteSDP'; return new Promise( (resolve, reject) => { - callNativeMethod('setRemoteDescription', [description]).then ( () => { + callNativeMethod(setRemoteDescriptionMethod, [description]).then ( () => { logger.info(`BBB-MOBILE - setRemoteDescription resolved`); resolve(); + + if(webRtcCallType === WEBRTC_CALL_TYPE_FULL_AUDIO) { + Object.defineProperty(this, "iceGatheringState", {get: function() { return "complete" }, set: ()=>{} }); + Object.defineProperty(this, "iceConnectionState", {get: function() { return "completed" }, set: ()=>{} }); + this.onicegatheringstatechange && this.onicegatheringstatechange({target: this}); + this.oniceconnectionstatechange && this.oniceconnectionstatechange({target: this}); + } }); } ); } prototype.originalAddTrack = prototype.addTrack; prototype.addTrack = function (description, successCallback, failureCallback) { - if(!isScreenShareBroadcastRelated('addTrack', this)){ + if(WEBRTC_CALL_TYPE_STANDARD === detectWebRtcCallType('addTrack', this)){ return prototype.originalAddTrack.call(this, ...arguments); } logger.info(`BBB-MOBILE - addTrack called`, {description, successCallback, failureCallback}); + console.log(`BBB-MOBILE - addTrack called`, {description, successCallback, failureCallback}); } prototype.originalGetLocalStreams = prototype.getLocalStreams; prototype.getLocalStreams = function() { - if(!isScreenShareBroadcastRelated('getLocalStreams', this)){ + if(WEBRTC_CALL_TYPE_STANDARD === detectWebRtcCallType('getLocalStreams', this)){ return prototype.originalGetLocalStreams.call(this, ...arguments); } logger.info(`BBB-MOBILE - getLocalStreams called`, arguments); @@ -231,7 +252,7 @@ import logger from '/imports/startup/client/logger'; prototype.originalAddTransceiver = prototype.addTransceiver; prototype.addTransceiver = function() { - if(!isScreenShareBroadcastRelated('addTransceiver', this)){ + if(WEBRTC_CALL_TYPE_STANDARD === detectWebRtcCallType('addTransceiver', this)){ return prototype.originalAddTransceiver.call(this, ...arguments); } logger.info(`BBB-MOBILE - addTransceiver called`, arguments); @@ -239,7 +260,7 @@ import logger from '/imports/startup/client/logger'; prototype.originalAddIceCandidate = prototype.addIceCandidate; prototype.addIceCandidate = function (candidate) { - if(!isScreenShareBroadcastRelated('addIceCandidate', this)){ + if(WEBRTC_CALL_TYPE_STANDARD === detectWebRtcCallType('addIceCandidate', this)){ return prototype.originalAddIceCandidate.call(this, ...arguments); } logger.info(`BBB-MOBILE - addIceCandidate called`, {candidate}); @@ -253,6 +274,13 @@ import logger from '/imports/startup/client/logger'; } ); } + + // Application events + window.bbbMobileScreenShareBroadcastFinishedCallback = function () { + document.querySelector('[data-test="stopScreenShare"]')?.click(); + } + + } })(); From b13039c31f4af4bffd71c3a4d7d12ceb9859e9af Mon Sep 17 00:00:00 2001 From: Gustavo Emanuel Farias Rosa Date: Thu, 2 Jun 2022 21:03:06 -0300 Subject: [PATCH 5/6] Detect screen share stop and propagate it to the application --- .../imports/ui/services/mobile-app/index.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js index ecf48718ac..bd6fd26795 100644 --- a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js +++ b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js @@ -234,7 +234,6 @@ import logger from '/imports/startup/client/logger'; } logger.info(`BBB-MOBILE - addTrack called`, {description, successCallback, failureCallback}); - console.log(`BBB-MOBILE - addTrack called`, {description, successCallback, failureCallback}); } prototype.originalGetLocalStreams = prototype.getLocalStreams; @@ -274,13 +273,21 @@ import logger from '/imports/startup/client/logger'; } ); } + // Handle screenshare stop + const KurentoScreenShareBridge = require('/imports/api/screenshare/client/bridge/index.js').default; + //Kurento Screen Share + var stopOriginal = KurentoScreenShareBridge.stop.bind(KurentoScreenShareBridge); + KurentoScreenShareBridge.stop = function(){ + callNativeMethod('stopScreenShare') + logger.debug(`BBB-MOBILE - Click on stop screen share`); + stopOriginal() + } - // Application events + // Handle screenshare stop requested by application (i.e. stopped the broadcast extension) window.bbbMobileScreenShareBroadcastFinishedCallback = function () { document.querySelector('[data-test="stopScreenShare"]')?.click(); } - } })(); From 9ea4171af649f2574d4cd5e66d524d2040f0eb15 Mon Sep 17 00:00:00 2001 From: Tiago Jacobs Date: Mon, 4 Jul 2022 16:19:38 -0300 Subject: [PATCH 6/6] Add support to 2.6 by initializing stream active flag --- bigbluebutton-html5/imports/ui/services/mobile-app/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js index bd6fd26795..9f72ddb07f 100644 --- a/bigbluebutton-html5/imports/ui/services/mobile-app/index.js +++ b/bigbluebutton-html5/imports/ui/services/mobile-app/index.js @@ -108,6 +108,7 @@ import logger from '/imports/startup/client/logger'; }; fakeVideoTrack.onended = null; // callbacks added from screenshare (we can use it later) fakeVideoTrack.oninactive = null; // callbacks added from screenshare (we can use it later) + fakeVideoTrack.addEventListener = function() {}; // skip listeners const videoTracks = [ fakeVideoTrack @@ -115,6 +116,7 @@ import logger from '/imports/startup/client/logger'; stream.getTracks = stream.getVideoTracks = function () { return videoTracks; }; + stream.active=true; resolve(stream); } ).catch(