HTML5 ScreenShare Viewer

HTML5 ScreenShare Viewer

bigbluebutton-client/resources/prod/lib/kurento-extension.js

@@ -20,7 +20,7 @@ Kurento = function (
   this.screenConstraints = {};
   this.mediaCallback = null;
 
-  this.voiceBridge = voiceBridge;
+  this.voiceBridge = voiceBridge + '-SCREENSHARE';
   this.internalMeetingId = internalMeetingId;
 
   this.vid_width = window.screen.width;
@@ -75,6 +75,10 @@ KurentoManager.prototype.exitScreenShare = function () {
   }
 };
 
+KurentoManager.prototype.exitVideo = function () {
+  // TODO exitVideo
+};
+
 KurentoManager.prototype.shareScreen = function (tag) {
   this.exitScreenShare();
   var obj = Object.create(Kurento.prototype);
@@ -84,7 +88,6 @@ KurentoManager.prototype.shareScreen = function (tag) {
   this.kurentoScreenShare.setScreenShare(tag);
 };
 
-// Still unused, part of the HTML5 implementation
 KurentoManager.prototype.joinWatchVideo = function (tag) {
   this.exitVideo();
   var obj = Object.create(Kurento.prototype);
@@ -137,6 +140,9 @@ Kurento.prototype.onWSMessage = function (message) {
     case 'presenterResponse':
       kurentoHandler.presenterResponse(parsedMessage);
       break;
+    case 'viewerResponse':
+      kurentoHandler.viewerResponse(parsedMessage);
+      break;
     case 'stopSharing':
       kurentoManager.exitScreenShare();
       break;
@@ -166,6 +172,18 @@ Kurento.prototype.presenterResponse = function (message) {
   }
 }
 
+Kurento.prototype.viewerResponse = function (message) {
+  if (message.response != 'accepted') {
+    var errorMsg = message.message ? message.message : 'Unknown error';
+    console.warn('Call not accepted for the following reason: ' + errorMsg);
+    //kurentoManager.exitScreenShare(); TODO stop?
+    kurentoHandler.onFail(errorMessage);
+  } else {
+    console.log("Viewer call was accepted with SDP => " + message.sdpAnswer);
+    this.webRtcPeer.processAnswer(message.sdpAnswer);
+  }
+}
+
 Kurento.prototype.serverResponse = function (message) {
   if (message.response != 'accepted') {
     var errorMsg = message.message ? message.message : 'Unknow error';
@@ -263,6 +281,20 @@ Kurento.prototype.onIceCandidate = function(candidate) {
   kurentoHandler.sendMessage(message);
 }
 
+Kurento.prototype.onViewerIceCandidate = function(candidate) {
+  console.log('Viewer local candidate' + JSON.stringify(candidate));
+
+  var message = {
+    id : 'viewerIceCandidate',
+    type: 'screenshare',
+    voiceBridge: kurentoHandler.voiceBridge,
+    candidate : candidate,
+    callerIdName: kurentoHandler.caller_id_name
+  }
+  console.log("this object " + JSON.stringify(this, null, 2));
+  kurentoHandler.sendMessage(message);
+}
+
 Kurento.prototype.setWatchVideo = function (tag) {
   this.useVideo = true;
   this.useCamera = 'none';
@@ -276,16 +308,16 @@ Kurento.prototype.viewer = function () {
   if (!this.webRtcPeer) {
 
     var options = {
-      remoteVideo: this.renderTag,
-      onicecandidate : onIceCandidate
+      remoteVideo: document.getElementById(this.renderTag),
+      onicecandidate : this.onViewerIceCandidate
     }
 
-    webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly(options, function(error) {
+    self.webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerRecvonly(options, function(error) {
       if(error) {
         return kurentoHandler.onFail(error);
       }
 
-      this.generateOffer(onOfferViewer);
+      this.generateOffer(self.onOfferViewer);
     });
   }
 };
@@ -437,3 +469,7 @@ window.kurentoWatchVideo = function () {
   window.kurentoInitialize();
   window.kurentoManager.joinWatchVideo.apply(window.kurentoManager, arguments);
 };
+
+window.kurentoExitVideo = function () {
+  // TODO kurentoExitVideo()
+}

bigbluebutton-html5/client/main.html

@@ -50,4 +50,6 @@
   <script src="/client/lib/jquery.json-2.4.min.js"></script>
   <script src="/client/lib/verto-min.js"></script>
   <script src="/client/lib/verto_extension.js"></script>
+  <script src="/client/lib/kurento-utils.min.js"></script>
+  <script src="/client/lib/kurento-extension.js"></script>
 </body>

bigbluebutton-html5/imports/api/2.0/screenshare/client/bridge/index.js

@@ -1,5 +1,7 @@
 import VertoBridge from './verto';
+import KurentoBridge from './kurento';
 
-const screenshareBridge = new VertoBridge();
+//const screenshareBridge = new VertoBridge();
+const screenshareBridge = new KurentoBridge();
 
 export default screenshareBridge;

bigbluebutton-html5/imports/api/2.0/screenshare/client/bridge/kurento.js

@@ -0,0 +1,34 @@
+import Users from '/imports/api/2.0/users';
+import Auth from '/imports/ui/services/auth';
+import BridgeService from './service';
+
+const getUserId = () => {
+  const userID = Auth.userID;
+  return userID;
+}
+
+const getMeetingId = () => {
+  const meetingID = Auth.meetingID;
+  return meetingID;
+}
+
+const getUsername = () => {
+  return Users.findOne({ userId: getUserId() }).name;
+}
+
+export default class KurentoScreenshareBridge {
+  kurentoWatchVideo() {
+    window.kurentoWatchVideo(
+      'screenshareVideo',
+      BridgeService.getConferenceBridge(),
+      getUsername(), 
+      getMeetingId(),
+      null,
+      null,
+    );
+  }
+
+  kurentoExitVideo() {
+    window.kurentoExitVideo();
+  }
+}

bigbluebutton-html5/imports/api/2.0/screenshare/server/handlers/screenshareStarted.js

@@ -1,7 +1,7 @@
 import { check } from 'meteor/check';
 import addScreenshare from '../modifiers/addScreenshare';
 
-export default function handleBroadcastStartedVoice({ body }, meetingId) {
+export default function handleScreenshareStarted({ body }, meetingId) {
   check(meetingId, String);
   check(body, Object);
 

bigbluebutton-html5/imports/api/2.0/screenshare/server/handlers/screenshareStopped.js

@@ -1,7 +1,7 @@
 import { check } from 'meteor/check';
 import clearScreenshare from '../modifiers/clearScreenshare';
 
-export default function handleBroadcastStartedVoice({ body }, meetingId) {
+export default function handleScreenshareStopped({ body }, meetingId) {
   const { screenshareConf } = body;
 
   check(meetingId, String);

bigbluebutton-html5/imports/startup/client/base.jsx

@@ -87,7 +87,7 @@ Base.propTypes = propTypes;
 Base.defaultProps = defaultProps;
 
 const SUBSCRIPTIONS_NAME = [
-  'users2x', 'chat2x', 'cursor2x', 'meetings2x', 'polls2x', 'presentations2x', 'annotations', 'slides2x', 'captions2x', 'breakouts2x', 'voiceUsers',
+  'users2x', 'chat2x', 'cursor2x', 'meetings2x', 'polls2x', 'presentations2x', 'annotations', 'slides2x', 'captions2x', 'breakouts2x', 'voiceUsers', 'screenshare',
 ];
 
 const BaseContainer = createContainer(({ params }) => {

bigbluebutton-html5/imports/ui/components/app/container.jsx

@@ -7,6 +7,7 @@ import Auth from '/imports/ui/services/auth';
 import Users from '/imports/api/2.0/users';
 import Breakouts from '/imports/api/2.0/breakouts';
 import Meetings from '/imports/api/2.0/meetings';
+import Screenshare from '/imports/api/2.0/screenshare';
 
 import ClosedCaptionsContainer from '/imports/ui/components/closed-captions/container';
 

bigbluebutton-html5/imports/ui/components/screenshare/component.jsx

@@ -7,7 +7,7 @@ export default class ScreenshareComponent extends React.Component {
 
   render() {
     return (
-      <video id="screenshareVideo" style={{ height: '100%', width: '100%' }} />
+      <video id="screenshareVideo" style={{ height: '100%', width: '100%' }} autoPlay />
     );
   }
 }

bigbluebutton-html5/imports/ui/components/screenshare/service.js

@@ -1,5 +1,6 @@
 import Screenshare from '/imports/api/2.0/screenshare';
-import VertoBridge from '/imports/api/2.0/screenshare/client/bridge';
+//import VertoBridge from '/imports/api/2.0/screenshare/client/bridge';
+import KurentoBridge from '/imports/api/2.0/screenshare/client/bridge';
 import PresentationService from '/imports/ui/components/presentation/service';
 
 // when the meeting information has been updated check to see if it was
@@ -11,21 +12,24 @@ function isVideoBroadcasting() {
   if (!ds) {
     return false;
   }
+
   return ds.screenshare.stream && !PresentationService.isPresenter();
 }
 
 // if remote screenshare has been ended disconnect and hide the video stream
 function presenterScreenshareHasEnded() {
-  // references a function in the global namespace inside verto_extension.js
+  // references a function in the global namespace inside kurento-extension.js
   // that we load dynamically
-  VertoBridge.vertoExitVideo();
+  //VertoBridge.vertoExitVideo();
+  KurentoBridge.kurentoExitVideo();
 }
 
 // if remote screenshare has been started connect and display the video stream
 function presenterScreenshareHasStarted() {
-  // references a function in the global namespace inside verto_extension.js
+  // references a function in the global namespace inside kurento-extension.js
   // that we load dynamically
-  VertoBridge.vertoWatchVideo();
+  //VertoBridge.vertoWatchVideo();
+  KurentoBridge.kurentoWatchVideo();
 }
 
 export {

bigbluebutton-html5/public/js/bower.json

@@ -0,0 +1,32 @@
+{
+  "name": "kurento-hello-world",
+  "description": "Kurento Browser JavaScript Tutorial",
+  "authors": [
+    "Kurento <info@kurento.org>"
+  ],
+  "main": "index.html",
+  "moduleType": [
+    "globals"
+  ],
+  "license": "LGPL",
+  "homepage": "http://www.kurento.org/",
+  "private": true,
+  "ignore": [
+    "**/.*",
+    "node_modules",
+    "bower_components",
+    "test",
+    "tests"
+  ],
+  "dependencies": {
+    "adapter.js": "0.2.9",
+    "bootstrap": "3.3.6",
+    "kurento-utils": "6.5.0",
+    "react": "15.1.0",
+    "reconnectingWebsocket": "1.0.0",
+    "requirejs": "2.2.0",
+    "requirejs-react-jsx": "1.0.2",
+    "requirejs-text": "2.0.15",
+    "font-awesome": "fontawesome#^4.6.3"
+  }
+}

labs/kurento-screenshare/lib/ConnectionManager.js

@@ -14,6 +14,7 @@ const wsModule = require('./websocket');
 const http = require('http');
 const fs = require('fs');
 const BigBlueButtonGW = require('./bbb/pubsub/bbb-gw');
+const MediaController = require('./media-controller');
 var Screenshare = require('./screenshare');
 var C = require('./bbb/messages/Constants');
 
@@ -25,6 +26,8 @@ module.exports = class ConnectionManager {
     this._logger = logger;
     this._clientId = 0;
     this._app = express();
+
+    this._sessions = {};
     this._screenshareSessions = {};
 
     this._setupExpressSession();
@@ -79,6 +82,7 @@ module.exports = class ConnectionManager {
     let connectionId;
     let request = webSocket.upgradeReq;
     let sessionId;
+    let callerIdName;
     let response = {
       writeHead : {}
     };
@@ -95,7 +99,9 @@ module.exports = class ConnectionManager {
 
     webSocket.on('close', function() {
       console.log('Connection ' + connectionId + ' closed');
-      self._stopSession(sessionId);
+      if (self._screenshareSessions[sessionId] && self._screenshareSessions[sessionId].id == connectionId) { // if presenter  // FIXME  (this conditional was added to prevent screenshare stop when an iOS user quits)
+        self._stopSession(sessionId); 
+      }
     });
 
     webSocket.on('message', function(_message) {
@@ -103,7 +109,6 @@ module.exports = class ConnectionManager {
       let session;
       // The sessionId is voiceBridge for screensharing sessions
       sessionId = message.voiceBridge;
-
       if(self._screenshareSessions[sessionId]) {
         session = self._screenshareSessions[sessionId];
       }
@@ -115,6 +120,12 @@ module.exports = class ConnectionManager {
           // Checking if there's already a Screenshare session started
           // because we shouldn't overwrite it
 
+	  if (!self._screenshareSessions[message.voiceBridge]) {
+	    self._screenshareSessions[message.voiceBridge] = {}
+	    self._screenshareSessions[message.voiceBridge] = session;
+	  }
+
+          //session.on('message', self._assembleSessionMessage.bind(self));
           if(session) {
             break;
           }
@@ -147,11 +158,19 @@ module.exports = class ConnectionManager {
           break;
 
         case 'viewer':
-          console.log('Viewer message => [' + message.id + '] connection [' + connectionId + '][' + message.presenterId + '][' + message.sessionId + '][' + message.callerName + ']');
-
+          console.log("[viewer] Session output \n " + session);
+          if (message.sdpOffer && message.voiceBridge) {
+            if (session) {
+              session._startViewer(webSocket, message.voiceBridge, message.sdpOffer, message.callerIdName, self._screenshareSessions[message.voiceBridge]._presenterEndpoint);
+            } else {
+              webSocket.sendMessage("voiceBridge not recognized");
+              webSocket.sendMessage(Object.keys(self._screenshareSessions));
+              webSocket.sendMessage(message.voiceBridge);
+            }
+          }
           break;
-        case 'stop':
 
+        case 'stop':
           console.log('[' + message.id + '] connection ' + connectionId);
 
           if (session) {
@@ -176,6 +195,16 @@ module.exports = class ConnectionManager {
           }));
           break;
 
+
+	case 'viewerIceCandidate':
+	  console.log("[viewerIceCandidate] Session output => " + session);
+	  if (session) {
+	    session._onViewerIceCandidate(message.candidate, callerIdName);
+	  } else {
+	    console.log("[iceCandidate] Why is there no session on ICE CANDIDATE?");
+	  }
+	  break;
+
         default:
           webSocket.sendMessage({ id : 'error', message : 'Invalid message ' + message });
           break;

labs/kurento-screenshare/lib/screenshare.js

@@ -44,6 +44,8 @@ module.exports = class Screenshare {
     this._vw = vw;
     this._vh = vh;
     this._candidatesQueue = [];
+    this._viewersEndpoint = [];
+    this._viewersCandidatesQueue = [];
   }
 
   // TODO isolate ICE
@@ -57,6 +59,85 @@ module.exports = class Screenshare {
       this._candidatesQueue.push(candidate);
     }
   };
+  
+  _onViewerIceCandidate(_candidate, callerIdName) {
+    let candidate = kurento.getComplexType('IceCandidate')(_candidate);
+    
+    if (this._viewersEndpoint[callerIdName]) {
+      this._viewersEndpoint[callerIdName].addIceCandidate(candidate);
+    }
+    else {
+      if (!this._viewersCandidatesQueue[callerIdName]) {
+        this._viewersCandidatesQueue[callerIdName] = [];
+      }
+      this._viewersCandidatesQueue[callerIdName].push(candidate);
+    }
+  }
+
+  _startViewer(ws, voiceBridge, sdp, callerIdName, presenterEndpoint, callback) {
+    let self = this;
+    let _callback = function(){};
+    self._viewersCandidatesQueue[callerIdName] = [];
+    
+    console.log("VIEWER VOICEBRIDGE:    "+self._voiceBridge);
+ 
+    MediaController.createMediaElement(voiceBridge, C.WebRTC, function(error, webRtcEndpoint) {
+      if (error) {
+        console.log("Media elements error" + error);
+        return _callback(error);
+      }
+
+      self._viewersEndpoint[callerIdName] = webRtcEndpoint;
+
+      // QUEUES UP ICE CANDIDATES IF NEGOTIATION IS NOT YET READY
+      while(self._viewersCandidatesQueue[callerIdName].length) {
+        let candidate = self._viewersCandidatesQueue[callerIdName].shift();
+        MediaController.addIceCandidate(self._viewersEndpoint[callerIdName].id, candidate);
+      }
+      // CONNECTS TWO MEDIA ELEMENTS
+      MediaController.connectMediaElements(presenterEndpoint.id, self._viewersEndpoint[callerIdName].id, C.VIDEO, function(error) {
+        if (error) {
+          console.log("Media elements CONNECT error " + error);
+          //pipeline.release();
+          return _callback(error);
+        }
+      });
+
+      // ICE NEGOTIATION WITH THE ENDPOINT
+      self._viewersEndpoint[callerIdName].on('OnIceCandidate', function(event) {
+        let candidate = kurento.getComplexType('IceCandidate')(event.candidate);
+        ws.sendMessage({ id : 'iceCandidate', candidate : candidate });
+      });
+
+      sdp = h264_sdp.transform(sdp);
+      // PROCESS A SDP OFFER
+      MediaController.processOffer(webRtcEndpoint.id, sdp, function(error, webRtcSdpAnswer) {
+        if (error) {
+          console.log("  [webrtc] processOffer error => " + error + " for SDP " + sdp);
+          //pipeline.release();
+          return _callback(error);
+        }
+        ws.sendMessage({id: "viewerResponse", sdpAnswer: webRtcSdpAnswer, response: "accepted"});
+        console.log(" Sent sdp message to client with callerIdName:" + callerIdName);
+
+        MediaController.gatherCandidates(webRtcEndpoint.id, function(error) {
+          if (error) {
+            return _callback(error);
+          }
+
+	  self._viewersEndpoint[callerIdName].on('MediaFlowInStateChange', function(event) {
+            if (event.state === 'NOT_FLOWING') {                          
+              console.log(" NOT FLOWING ");                              
+            }                                                             
+            else if (event.state === 'FLOWING') {                         
+              console.log(" FLOWING ");      
+            }                                                             
+          });
+        });
+      });
+    });
+  }
+
 
   _startPresenter(id, ws, sdpOffer, callback) {
     let self = this;
@@ -65,6 +146,7 @@ module.exports = class Screenshare {
     // Force H264 on Firefox and Chrome
     sdpOffer = h264_sdp.transform(sdpOffer);
     console.log("Starting presenter for " + sdpOffer);
+    console.log("PRESENTER VOICEBRIDGE:   " + self._voiceBridge);
     MediaController.createMediaElement(self._voiceBridge, C.WebRTC, function(error, webRtcEndpoint) {
       if (error) {
         console.log("Media elements error" + error);

labs/kurento-screenshare/package.json

@@ -10,11 +10,12 @@
     "cookie-parser": "^1.3.5",
     "express": "~4.12.4",
     "express-session": "~1.10.3",
-    "ws": "~1.0.1",
     "kurento-client": "6.6.0",
+    "moment": "*",
     "redis": "^2.6.2",
     "sdp-transform": "*",
-    "moment": "*"
+    "uuid": "^3.1.0",
+    "ws": "~1.0.1"
   },
   "devDependencies": {
     "config": "^1.26.1",