Merge pull request #8072 from prlanzarin/2.2-abort-play

Fixes to client-side screenshare/listen only code and log improvements
This commit is contained in:
Chad Pilkey 2019-09-06 18:37:22 -04:00 committed by GitHub
commit f471b1e2b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 353 additions and 75 deletions

View File

@ -1,6 +1,7 @@
import BaseAudioBridge from './base'; import BaseAudioBridge from './base';
import Auth from '/imports/ui/services/auth'; import Auth from '/imports/ui/services/auth';
import { fetchWebRTCMappedStunTurnServers } from '/imports/utils/fetchStunTurnServers'; import { fetchWebRTCMappedStunTurnServers } from '/imports/utils/fetchStunTurnServers';
import playAndRetry from '/imports/utils/mediaElementPlayRetry';
import logger from '/imports/startup/client/logger'; import logger from '/imports/startup/client/logger';
const SFU_URL = Meteor.settings.public.kurento.wsUrl; const SFU_URL = Meteor.settings.public.kurento.wsUrl;
@ -37,6 +38,14 @@ export default class KurentoAudioBridge extends BaseAudioBridge {
this.hasSuccessfullyStarted = false; this.hasSuccessfullyStarted = false;
} }
static normalizeError(error = {}) {
const errorMessage = error.message || error.name || error.reason || 'Unknown error';
const errorCode = error.code || 'Undefined code';
const errorReason = error.reason || error.id || 'Undefined reason';
return { errorMessage, errorCode, errorReason };
}
joinAudio({ isListenOnly, inputStream }, callback) { joinAudio({ isListenOnly, inputStream }, callback) {
return new Promise(async (resolve, reject) => { return new Promise(async (resolve, reject) => {
this.callback = callback; this.callback = callback;
@ -59,34 +68,65 @@ export default class KurentoAudioBridge extends BaseAudioBridge {
inputStream, inputStream,
}; };
const audioTag = document.getElementById(MEDIA_TAG);
const playElement = () => {
const mediaTagPlayed = () => {
logger.info({
logCode: 'listenonly_media_play_success',
}, 'Listen only media played successfully');
resolve(this.callback({ status: this.baseCallStates.started }));
};
if (audioTag.paused) {
// Tag isn't playing yet. Play it.
audioTag.play()
.then(mediaTagPlayed)
.catch((error) => {
// NotAllowedError equals autoplay issues, fire autoplay handling event.
// This will be handled in audio-manager.
if (error.name === 'NotAllowedError') {
logger.error({
logCode: 'listenonly_error_autoplay',
extraInfo: { errorName: error.name },
}, 'Listen only media play failed due to autoplay error');
const tagFailedEvent = new CustomEvent('audioPlayFailed', { detail: { mediaElement: audioTag } });
window.dispatchEvent(tagFailedEvent);
resolve(this.callback({
status: this.baseCallStates.autoplayBlocked,
}));
} else {
// Tag failed for reasons other than autoplay. Log the error and
// try playing again a few times until it works or fails for good
const played = playAndRetry(audioTag);
if (!played) {
logger.error({
logCode: 'listenonly_error_media_play_failed',
extraInfo: { errorName: error.name },
}, `Listen only media play failed due to ${error.name}`);
} else {
mediaTagPlayed();
}
}
});
} else {
// Media tag is already playing, so log a success. This is really a
// logging fallback for a case that shouldn't happen. But if it does
// (ie someone re-enables the autoPlay prop in the element), then it
// means the stream is playing properly and it'll be logged.
mediaTagPlayed();
}
};
const onSuccess = () => { const onSuccess = () => {
const { webRtcPeer } = window.kurentoManager.kurentoAudio; const { webRtcPeer } = window.kurentoManager.kurentoAudio;
this.hasSuccessfullyStarted = true; this.hasSuccessfullyStarted = true;
if (webRtcPeer) { if (webRtcPeer) {
const audioTag = document.getElementById(MEDIA_TAG);
const stream = webRtcPeer.getRemoteStream(); const stream = webRtcPeer.getRemoteStream();
audioTag.pause(); audioTag.pause();
audioTag.srcObject = stream; audioTag.srcObject = stream;
audioTag.muted = false; audioTag.muted = false;
audioTag.play() playElement();
.then(() => {
resolve(this.callback({ status: this.baseCallStates.started }));
})
.catch((error) => {
// NotAllowedError equals autoplay issues, fire autoplay handling event
if (error.name === 'NotAllowedError') {
const tagFailedEvent = new CustomEvent('audioPlayFailed', { detail: { mediaElement: audioTag } });
window.dispatchEvent(tagFailedEvent);
}
logger.warn({
logCode: 'sfuaudiobridge_play_maybe_error',
extraInfo: { error },
}, `Listen only media play failed due to ${error.name}`);
resolve(this.callback({
status: this.baseCallStates.autoplayBlocked,
}));
});
} else { } else {
this.callback({ this.callback({
status: this.baseCallStates.failed, status: this.baseCallStates.failed,
@ -101,12 +141,13 @@ export default class KurentoAudioBridge extends BaseAudioBridge {
}; };
const onFail = (error) => { const onFail = (error) => {
const { errorMessage, errorCode, errorReason } = KurentoAudioBridge.normalizeError(error);
// Listen only connected successfully already and dropped mid-call. // Listen only connected successfully already and dropped mid-call.
// Try to reconnect ONCE (binded to reconnectOngoing flag) // Try to reconnect ONCE (binded to reconnectOngoing flag)
if (this.hasSuccessfullyStarted && !this.reconnectOngoing) { if (this.hasSuccessfullyStarted && !this.reconnectOngoing) {
logger.error({ logger.error({
logCode: 'sfuaudiobridge_listen_only_error_reconnect', logCode: 'listenonly_error_try_to_reconnect',
extraInfo: { error }, extraInfo: { errorMessage, errorCode, errorReason },
}, 'Listen only failed for an ongoing session, try to reconnect'); }, 'Listen only failed for an ongoing session, try to reconnect');
window.kurentoExitAudio(); window.kurentoExitAudio();
this.callback({ status: this.baseCallStates.reconnecting }); this.callback({ status: this.baseCallStates.reconnecting });
@ -135,26 +176,34 @@ export default class KurentoAudioBridge extends BaseAudioBridge {
} else { } else {
// Already tried reconnecting once OR the user handn't succesfully // Already tried reconnecting once OR the user handn't succesfully
// connected firsthand. Just finish the session and reject with error // connected firsthand. Just finish the session and reject with error
if (!this.reconnectOngoing) {
logger.error({
logCode: 'listenonly_error_failed_to_connect',
extraInfo: { errorMessage, errorCode, errorReason },
}, `Listen only failed when trying to start due to ${errorMessage}`);
} else {
logger.error({
logCode: 'listenonly_error_reconnect_failed',
extraInfo: { errorMessage, errorCode, errorReason },
}, `Listen only failed when trying to reconnect due to ${errorMessage}`);
}
this.reconnectOngoing = false; this.reconnectOngoing = false;
this.hasSuccessfullyStarted = false; this.hasSuccessfullyStarted = false;
window.kurentoExitAudio(); window.kurentoExitAudio();
let reason = 'Undefined';
if (error) {
reason = error.reason || error.id || error;
}
this.callback({ this.callback({
status: this.baseCallStates.failed, status: this.baseCallStates.failed,
error: this.baseErrorCodes.CONNECTION_ERROR, error: this.baseErrorCodes.CONNECTION_ERROR,
bridgeError: reason, bridgeError: errorReason,
}); });
reject(reason); reject(errorReason);
} }
}; };
if (!isListenOnly) { if (!isListenOnly) {
return reject('Invalid bridge option'); return reject(new Error('Invalid bridge option'));
} }
window.kurentoJoinAudio( window.kurentoJoinAudio(

View File

@ -1,6 +1,7 @@
import Auth from '/imports/ui/services/auth'; import Auth from '/imports/ui/services/auth';
import BridgeService from './service'; import BridgeService from './service';
import { fetchWebRTCMappedStunTurnServers } from '/imports/utils/fetchStunTurnServers'; import { fetchWebRTCMappedStunTurnServers } from '/imports/utils/fetchStunTurnServers';
import playAndRetry from '/imports/utils/mediaElementPlayRetry';
import logger from '/imports/startup/client/logger'; import logger from '/imports/startup/client/logger';
const SFU_CONFIG = Meteor.settings.public.kurento; const SFU_CONFIG = Meteor.settings.public.kurento;
@ -20,13 +21,40 @@ const getMeetingId = () => Auth.meetingID;
const getSessionToken = () => Auth.sessionToken; const getSessionToken = () => Auth.sessionToken;
export default class KurentoScreenshareBridge { export default class KurentoScreenshareBridge {
static normalizeError(error = {}) {
const errorMessage = error.message || error.name || error.reason || 'Unknown error';
const errorCode = error.code || 'Undefined code';
const errorReason = error.reason || error.id || 'Undefined reason';
return { errorMessage, errorCode, errorReason };
}
static handlePresenterFailure(error) {
const normalizedError = KurentoScreenshareBridge.normalizeError(error);
logger.error({
logCode: 'screenshare_presenter_error_failed_to_connect',
extraInfo: { ...normalizedError },
}, `Screenshare presenter failed when trying to start due to ${normalizedError.errorMessage}`);
return normalizedError;
}
static handleViewerFailure(error) {
const normalizedError = KurentoScreenshareBridge.normalizeError(error);
logger.error({
logCode: 'screenshare_viewer_error_failed_to_connect',
extraInfo: { ...normalizedError },
}, `Screenshare viewer failed when trying to start due to ${normalizedError.errorMessage}`);
return normalizedError;
}
async kurentoWatchVideo() { async kurentoWatchVideo() {
let iceServers = []; let iceServers = [];
try { try {
iceServers = await fetchWebRTCMappedStunTurnServers(getSessionToken()); iceServers = await fetchWebRTCMappedStunTurnServers(getSessionToken());
} catch (error) { } catch (error) {
logger.error({ logCode: 'sfuviewscreen_fetchstunturninfo_error', extraInfo: { error } }, logger.error({ logCode: 'screenshare_viwer_fetchstunturninfo_error', extraInfo: { error } },
'Screenshare bridge failed to fetch STUN/TURN info, using default'); 'Screenshare bridge failed to fetch STUN/TURN info, using default');
} finally { } finally {
const options = { const options = {
@ -35,26 +63,67 @@ export default class KurentoScreenshareBridge {
logger, logger,
}; };
const screenshareTag = document.getElementById(SCREENSHARE_VIDEO_TAG);
const playElement = () => {
const mediaTagPlayed = () => {
logger.info({
logCode: 'screenshare_viewer_media_play_success',
}, 'Screenshare viewer media played successfully');
};
if (screenshareTag.paused) {
// Tag isn't playing yet. Play it.
screenshareTag.play()
.then(mediaTagPlayed)
.catch((error) => {
// NotAllowedError equals autoplay issues, fire autoplay handling event.
// This will be handled in the screenshare react component.
if (error.name === 'NotAllowedError') {
logger.error({
logCode: 'screenshare_viewer_error_autoplay',
extraInfo: { errorName: error.name },
}, 'Screenshare viewer play failed due to autoplay error');
const tagFailedEvent = new CustomEvent('screensharePlayFailed',
{ detail: { mediaElement: screenshareTag } });
window.dispatchEvent(tagFailedEvent);
} else {
// Tag failed for reasons other than autoplay. Log the error and
// try playing again a few times until it works or fails for good
const played = playAndRetry(screenshareTag);
if (!played) {
logger.error({
logCode: 'screenshare_viewer_error_media_play_failed',
extraInfo: { errorName: error.name },
}, `Screenshare viewer media play failed due to ${error.name}`);
} else {
mediaTagPlayed();
}
}
});
} else {
// Media tag is already playing, so log a success. This is really a
// logging fallback for a case that shouldn't happen. But if it does
// (ie someone re-enables the autoPlay prop in the element), then it
// means the stream is playing properly and it'll be logged.
mediaTagPlayed();
}
};
const onFail = (error) => {
KurentoScreenshareBridge.handleViewerFailure(error);
};
// Callback for the kurento-extension.js script. It's called when the whole
// negotiation with SFU is successful. This will load the stream into the
// screenshare media element and play it manually.
const onSuccess = () => { const onSuccess = () => {
const { webRtcPeer } = window.kurentoManager.kurentoVideo; const { webRtcPeer } = window.kurentoManager.kurentoVideo;
if (webRtcPeer) { if (webRtcPeer) {
const screenshareTag = document.getElementById(SCREENSHARE_VIDEO_TAG);
const stream = webRtcPeer.getRemoteStream(); const stream = webRtcPeer.getRemoteStream();
screenshareTag.muted = true; screenshareTag.muted = true;
screenshareTag.pause(); screenshareTag.pause();
screenshareTag.srcObject = stream; screenshareTag.srcObject = stream;
screenshareTag.play().catch((error) => { playElement();
// NotAllowedError equals autoplay issues, fire autoplay handling event
if (error.name === 'NotAllowedError') {
const tagFailedEvent = new CustomEvent('screensharePlayFailed',
{ detail: { mediaElement: screenshareTag } });
window.dispatchEvent(tagFailedEvent);
}
logger.warn({
logCode: 'sfuscreenshareview_play_maybe_error',
extraInfo: { error },
}, `Screenshare viewer media play failed due to ${error.name}`);
});
} }
}; };
@ -63,7 +132,7 @@ export default class KurentoScreenshareBridge {
BridgeService.getConferenceBridge(), BridgeService.getConferenceBridge(),
getUserId(), getUserId(),
getMeetingId(), getMeetingId(),
null, onFail,
onSuccess, onSuccess,
options, options,
); );
@ -79,7 +148,7 @@ export default class KurentoScreenshareBridge {
try { try {
iceServers = await fetchWebRTCMappedStunTurnServers(getSessionToken()); iceServers = await fetchWebRTCMappedStunTurnServers(getSessionToken());
} catch (error) { } catch (error) {
logger.error({ logCode: 'sfusharescreen_fetchstunturninfo_error' }, logger.error({ logCode: 'screenshare_presenter_fetchstunturninfo_error' },
'Screenshare bridge failed to fetch STUN/TURN info, using default'); 'Screenshare bridge failed to fetch STUN/TURN info, using default');
} finally { } finally {
const options = { const options = {
@ -90,13 +159,25 @@ export default class KurentoScreenshareBridge {
iceServers, iceServers,
logger, logger,
}; };
const failureCallback = (error) => {
const normalizedError = KurentoScreenshareBridge.handlePresenterFailure(error);
onFail(normalizedError);
};
const successCallback = () => {
logger.info({
logCode: 'screenshare_presenter_start_success',
}, 'Screenshare presenter started succesfully');
};
window.kurentoShareScreen( window.kurentoShareScreen(
SCREENSHARE_VIDEO_TAG, SCREENSHARE_VIDEO_TAG,
BridgeService.getConferenceBridge(), BridgeService.getConferenceBridge(),
getUserId(), getUserId(),
getMeetingId(), getMeetingId(),
onFail, failureCallback,
null, successCallback,
options, options,
); );
} }

View File

@ -6,6 +6,8 @@ import FullscreenService from '../fullscreen-button/service';
import FullscreenButtonContainer from '../fullscreen-button/container'; import FullscreenButtonContainer from '../fullscreen-button/container';
import { styles } from './styles'; import { styles } from './styles';
import AutoplayOverlay from '../media/autoplay-overlay/component'; import AutoplayOverlay from '../media/autoplay-overlay/component';
import logger from '/imports/startup/client/logger';
import playAndRetry from '/imports/utils/mediaElementPlayRetry';
const intlMessages = defineMessages({ const intlMessages = defineMessages({
screenShareLabel: { screenShareLabel: {
@ -80,13 +82,24 @@ class ScreenshareComponent extends React.Component {
handleAllowAutoplay() { handleAllowAutoplay() {
const { autoplayBlocked } = this.state; const { autoplayBlocked } = this.state;
logger.info({
logCode: 'screenshare_autoplay_allowed',
}, 'Screenshare media autoplay allowed by the user');
window.removeEventListener('screensharePlayFailed', this.handlePlayElementFailed); window.removeEventListener('screensharePlayFailed', this.handlePlayElementFailed);
while (this.failedMediaElements.length) { while (this.failedMediaElements.length) {
const mediaElement = this.failedMediaElements.shift(); const mediaElement = this.failedMediaElements.shift();
if (mediaElement) { if (mediaElement) {
mediaElement.play().catch(() => { const played = playAndRetry(mediaElement);
// Ignore the error for now. if (!played) {
}); logger.error({
logCode: 'screenshare_autoplay_handling_failed',
}, 'Screenshare autoplay handling failed to play media');
} else {
logger.info({
logCode: 'screenshare_viewer_media_play_success',
}, 'Screenshare viewer media played successfully');
}
} }
} }
if (autoplayBlocked) { this.setState({ autoplayBlocked: false }); } if (autoplayBlocked) { this.setState({ autoplayBlocked: false }); }
@ -99,6 +112,10 @@ class ScreenshareComponent extends React.Component {
e.stopPropagation(); e.stopPropagation();
this.failedMediaElements.push(mediaElement); this.failedMediaElements.push(mediaElement);
if (!autoplayBlocked) { if (!autoplayBlocked) {
logger.info({
logCode: 'screenshare_autoplay_prompt',
}, 'Prompting user for action to play screenshare media');
this.setState({ autoplayBlocked: true }); this.setState({ autoplayBlocked: true });
} }
} }
@ -154,7 +171,6 @@ class ScreenshareComponent extends React.Component {
id="screenshareVideo" id="screenshareVideo"
key="screenshareVideo" key="screenshareVideo"
style={{ maxHeight: '100%', width: '100%' }} style={{ maxHeight: '100%', width: '100%' }}
autoPlay
playsInline playsInline
onLoadedData={this.onVideoLoad} onLoadedData={this.onVideoLoad}
ref={(ref) => { this.videoTag = ref; }} ref={(ref) => { this.videoTag = ref; }}

View File

@ -7,6 +7,8 @@ import { styles } from './styles';
import VideoListItemContainer from './video-list-item/container'; import VideoListItemContainer from './video-list-item/container';
import { withDraggableConsumer } from '../../media/webcam-draggable-overlay/context'; import { withDraggableConsumer } from '../../media/webcam-draggable-overlay/context';
import AutoplayOverlay from '../../media/autoplay-overlay/component'; import AutoplayOverlay from '../../media/autoplay-overlay/component';
import logger from '/imports/startup/client/logger';
import playAndRetry from '/imports/utils/mediaElementPlayRetry';
const propTypes = { const propTypes = {
users: PropTypes.arrayOf(PropTypes.object).isRequired, users: PropTypes.arrayOf(PropTypes.object).isRequired,
@ -145,14 +147,25 @@ class VideoList extends Component {
handleAllowAutoplay() { handleAllowAutoplay() {
const { autoplayBlocked } = this.state; const { autoplayBlocked } = this.state;
logger.info({
logCode: 'video_provider_autoplay_allowed',
}, 'Video media autoplay allowed by the user');
this.autoplayWasHandled = true; this.autoplayWasHandled = true;
window.removeEventListener('videoPlayFailed', this.handlePlayElementFailed); window.removeEventListener('videoPlayFailed', this.handlePlayElementFailed);
while (this.failedMediaElements.length) { while (this.failedMediaElements.length) {
const mediaElement = this.failedMediaElements.shift(); const mediaElement = this.failedMediaElements.shift();
if (mediaElement) { if (mediaElement) {
mediaElement.play().catch(() => { const played = playAndRetry(mediaElement);
// Ignore the error for now. if (!played) {
}); logger.error({
logCode: 'video_provider_autoplay_handling_failed',
}, 'Video autoplay handling failed to play media');
} else {
logger.info({
logCode: 'video_provider_media_play_success',
}, 'Video media played successfully');
}
} }
} }
if (autoplayBlocked) { this.setState({ autoplayBlocked: false }); } if (autoplayBlocked) { this.setState({ autoplayBlocked: false }); }
@ -165,6 +178,9 @@ class VideoList extends Component {
e.stopPropagation(); e.stopPropagation();
this.failedMediaElements.push(mediaElement); this.failedMediaElements.push(mediaElement);
if (!autoplayBlocked && !this.autoplayWasHandled) { if (!autoplayBlocked && !this.autoplayWasHandled) {
logger.info({
logCode: 'video_provider_autoplay_prompt',
}, 'Prompting user for action to play video media');
this.setState({ autoplayBlocked: true }); this.setState({ autoplayBlocked: true });
} }
} }

View File

@ -8,6 +8,7 @@ import { notify } from '/imports/ui/services/notification';
import browser from 'browser-detect'; import browser from 'browser-detect';
import iosWebviewAudioPolyfills from '../../../utils/ios-webview-audio-polyfills'; import iosWebviewAudioPolyfills from '../../../utils/ios-webview-audio-polyfills';
import { tryGenerateIceCandidates } from '../../../utils/safari-webrtc'; import { tryGenerateIceCandidates } from '../../../utils/safari-webrtc';
import playAndRetry from '/imports/utils/mediaElementPlayRetry';
const MEDIA = Meteor.settings.public.media; const MEDIA = Meteor.settings.public.media;
const MEDIA_TAG = MEDIA.mediaTag; const MEDIA_TAG = MEDIA.mediaTag;
@ -492,11 +493,25 @@ class AudioManager {
handleAllowAutoplay() { handleAllowAutoplay() {
window.removeEventListener('audioPlayFailed', this.handlePlayElementFailed); window.removeEventListener('audioPlayFailed', this.handlePlayElementFailed);
logger.info({
logCode: 'audiomanager_autoplay_allowed',
}, 'Listen only autoplay allowed by the user');
while (this.failedMediaElements.length) { while (this.failedMediaElements.length) {
const mediaElement = this.failedMediaElements.shift(); const mediaElement = this.failedMediaElements.shift();
if (mediaElement) { if (mediaElement) {
mediaElement.play().catch(() => { playAndRetry(mediaElement).then((played) => {
// Ignore the error for now. if (!played) {
logger.error({
logCode: 'audiomanager_autoplay_handling_failed',
}, 'Listen only autoplay handling failed to play media');
} else {
// logCode is listenonly_* to make it consistent with the other tag play log
logger.info({
logCode: 'listenonly_media_play_success',
}, 'Listen only media played successfully');
}
}); });
} }
} }
@ -509,6 +524,9 @@ class AudioManager {
e.stopPropagation(); e.stopPropagation();
this.failedMediaElements.push(mediaElement); this.failedMediaElements.push(mediaElement);
if (!this.autoplayBlocked) { if (!this.autoplayBlocked) {
logger.info({
logCode: 'audiomanager_autoplay_prompt',
}, 'Prompting user for action to play listen only media');
this.autoplayBlocked = true; this.autoplayBlocked = true;
} }
} }

View File

@ -0,0 +1,27 @@
const DEFAULT_MAX_RETRIES = 10;
const DEFAULT_RETRY_TIMEOUT = 500;
const playAndRetry = async (mediaElement, maxRetries = DEFAULT_MAX_RETRIES) => {
let attempt = 0;
let played = false;
const playElement = () => new Promise((resolve, reject) => {
setTimeout(() => {
mediaElement.play().then(resolve).catch(reject);
}, DEFAULT_RETRY_TIMEOUT);
});
while (!played && attempt < maxRetries && mediaElement.paused) {
try {
await playElement();
played = true;
return played;
} catch (error) {
attempt += 1;
}
}
return played || mediaElement.paused;
};
export default playAndRetry;

View File

@ -243,11 +243,12 @@ Kurento.prototype.init = function () {
}; };
this.ws.onerror = (error) => { this.ws.onerror = (error) => {
kurentoManager.exitScreenShare(); kurentoManager.exitScreenShare();
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({ this.logger.error({
logCode: 'kurentoextension_websocket_error', logCode: 'kurentoextension_websocket_error',
extraInfo: { error } extraInfo: { errorMessage, errorCode, errorReason },
}, 'Error in the WebSocket connection to SFU, screenshare/listen only will drop'); }, 'Error in the WebSocket connection to SFU, screenshare/listen only will drop');
self.onFail('Websocket connection error'); self.onFail({ errorMessage, errorCode, errorReason });
}; };
this.ws.onopen = function () { this.ws.onopen = function () {
self.pingInterval = setInterval(self.ping.bind(self), self.PING_INTERVAL); self.pingInterval = setInterval(self.ping.bind(self), self.PING_INTERVAL);
@ -269,7 +270,7 @@ Kurento.prototype.onWSMessage = function (message) {
kurentoManager.exitScreenShare(); kurentoManager.exitScreenShare();
break; break;
case 'iceCandidate': case 'iceCandidate':
this.webRtcPeer.addIceCandidate(parsedMessage.candidate); this.handleIceCandidate(parsedMessage.candidate);
break; break;
case 'webRTCAudioSuccess': case 'webRTCAudioSuccess':
this.onSuccess(parsedMessage.success); this.onSuccess(parsedMessage.success);
@ -292,6 +293,57 @@ Kurento.prototype.setRenderTag = function (tag) {
this.renderTag = tag; this.renderTag = tag;
}; };
Kurento.prototype.processIceQueue = function () {
const peer = this.webRtcPeer;
while (peer.iceQueue.length) {
const candidate = peer.iceQueue.shift();
peer.addIceCandidate(candidate, (error) => {
if (error) {
// Just log the error. We can't be sure if a candidate failure on add is
// fatal or not, so that's why we have a timeout set up for negotiations and
// listeners for ICE state transitioning to failures, so we won't act on it here
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({
logCode: 'kurentoextension_addicecandidate_error',
extraInfo: { errorMessage, errorCode, errorReason },
}, `Adding ICE candidate failed due to ${errorMessage}`);
}
});
}
}
Kurento.prototype.handleIceCandidate = function (candidate) {
const peer = this.webRtcPeer;
if (peer.negotiated) {
peer.addIceCandidate(candidate, (error) => {
if (error) {
// Just log the error. We can't be sure if a candidate failure on add is
// fatal or not, so that's why we have a timeout set up for negotiations and
// listeners for ICE state transitioning to failures, so we won't act on it here
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({
logCode: 'kurentoextension_addicecandidate_error',
extraInfo: { errorMessage, errorCode, errorReason },
}, `Adding ICE candidate failed due to ${errorMessage}`);
}
});
} else {
// ICE candidates are queued until a SDP answer has been processed.
// This was done due to a long term iOS/Safari quirk where it'd
// fail if candidates were added before the offer/answer cycle was completed.
// IT STILL HAPPENS - prlanzarin sept 2019
peer.iceQueue.push(candidate);
}
}
Kurento.prototype.normalizeError = function (error = {}) {
const errorMessage = error.message || error.name || error.reason || 'Unknown error';
const errorCode = error.code || 'Undefined code';
const errorReason = error.reason || error.id || 'Undefined reason';
return { errorMessage, errorCode, errorReason };
}
Kurento.prototype.startResponse = function (message) { Kurento.prototype.startResponse = function (message) {
if (message.response !== 'accepted') { if (message.response !== 'accepted') {
this.handleSFUError(message); this.handleSFUError(message);
@ -300,12 +352,26 @@ Kurento.prototype.startResponse = function (message) {
logCode: 'kurentoextension_start_success', logCode: 'kurentoextension_start_success',
extraInfo: { sfuResponse: message } extraInfo: { sfuResponse: message }
}, `Start request accepted for ${message.type}`); }, `Start request accepted for ${message.type}`);
this.webRtcPeer.processAnswer(message.sdpAnswer);
// audio calls gets their success callback in a subsequent step (@webRTCAudioSuccess) this.webRtcPeer.processAnswer(message.sdpAnswer, (error) => {
// due to legacy messaging which I don't intend to break now - prlanzarin if (error) {
if (message.type === 'screenshare') { const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.onSuccess() this.logger.error({
} logCode: 'kurentoextension_peerconnection_processanswer_error',
extraInfo: { errorMessage, errorCode, errorReason },
}, `Processing SDP answer from SFU for failed due to ${errorMessage}`);
return this.onFail({ errorMessage, errorCode, errorReason });
}
// Mark the peer as negotiated and flush the ICE queue
this.webRtcPeer.negotiated = true;
this.processIceQueue();
// audio calls gets their success callback in a subsequent step (@webRTCAudioSuccess)
// due to legacy messaging which I don't intend to break now - prlanzarin
if (message.type === 'screenshare') {
this.onSuccess()
}
});
} }
}; };
@ -334,18 +400,19 @@ Kurento.prototype.handleSFUError = function (sfuResponse) {
break; break;
} }
this.onFail( { code, reason } ); this.onFail({ errorMessage: reason, errorCode: code, errorReason: reason });
}; };
Kurento.prototype.onOfferPresenter = function (error, offerSdp) { Kurento.prototype.onOfferPresenter = function (error, offerSdp) {
const self = this; const self = this;
if (error) { if (error) {
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({ this.logger.error({
logCode: 'kurentoextension_screenshare_presenter_offer_failure', logCode: 'kurentoextension_screenshare_presenter_offer_failure',
extraInfo: { error } extraInfo: { errorMessage, errorCode, errorReason },
}, `Failed to generate peer connection offer for screenshare presenter with error ${error.message}`); }, `Failed to generate peer connection offer for screenshare presenter with error ${error.message}`);
this.onFail(error); this.onFail({ errorMessage, errorCode, errorReason });
return; return;
} }
@ -405,14 +472,16 @@ Kurento.prototype.startScreensharing = function () {
this.webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerSendonly(options, (error) => { this.webRtcPeer = kurentoUtils.WebRtcPeer.WebRtcPeerSendonly(options, (error) => {
if (error) { if (error) {
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({ this.logger.error({
logCode: 'kurentoextension_screenshare_peerconnection_create_error', logCode: 'kurentoextension_screenshare_peerconnection_create_error',
extraInfo: { error } extraInfo: { errorMessage, errorCode, errorReason },
}, `WebRTC peer constructor for screenshare (presenter) failed due to ${error.message}`); }, `WebRTC peer constructor for screenshare (presenter) failed due to ${error.message}`);
this.onFail(error); this.onFail({ errorMessage, errorCode, errorReason });
return kurentoManager.exitScreenShare(); return kurentoManager.exitScreenShare();
} }
this.webRtcPeer.iceQueue = [];
this.webRtcPeer.generateOffer(this.onOfferPresenter.bind(this)); this.webRtcPeer.generateOffer(this.onOfferPresenter.bind(this));
const localStream = this.webRtcPeer.peerConnection.getLocalStreams()[0]; const localStream = this.webRtcPeer.peerConnection.getLocalStreams()[0];
@ -486,7 +555,7 @@ Kurento.prototype.viewer = function () {
if (error) { if (error) {
return self.onFail(error); return self.onFail(error);
} }
self.webRtcPeer.iceQueue = [];
this.generateOffer(self.onOfferViewer.bind(self)); this.generateOffer(self.onOfferViewer.bind(self));
}); });
self.webRtcPeer.peerConnection.oniceconnectionstatechange = () => { self.webRtcPeer.peerConnection.oniceconnectionstatechange = () => {
@ -508,12 +577,13 @@ Kurento.prototype.viewer = function () {
Kurento.prototype.onOfferViewer = function (error, offerSdp) { Kurento.prototype.onOfferViewer = function (error, offerSdp) {
const self = this; const self = this;
if (error) { if (error) {
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({ this.logger.error({
logCode: 'kurentoextension_screenshare_viewer_offer_failure', logCode: 'kurentoextension_screenshare_viewer_offer_failure',
extraInfo: { error } extraInfo: { errorMessage, errorCode, errorReason },
}, `Failed to generate peer connection offer for screenshare viewer with error ${error.message}`); }, `Failed to generate peer connection offer for screenshare viewer with error ${error.message}`);
return this.onFail(error); return this.onFail({ errorMessage, errorCode, errorReason });
} }
const message = { const message = {
@ -564,7 +634,7 @@ Kurento.prototype.listenOnly = function () {
if (error) { if (error) {
return self.onFail(error); return self.onFail(error);
} }
self.webRtcPeer.iceQueue = [];
this.generateOffer(self.onOfferListenOnly.bind(self)); this.generateOffer(self.onOfferListenOnly.bind(self));
}); });
} }
@ -590,12 +660,13 @@ Kurento.prototype.onListenOnlyIceCandidate = function (candidate) {
Kurento.prototype.onOfferListenOnly = function (error, offerSdp) { Kurento.prototype.onOfferListenOnly = function (error, offerSdp) {
const self = this; const self = this;
if (error) { if (error) {
const { errorMessage, errorCode, errorReason } = this.normalizeError(error);
this.logger.error({ this.logger.error({
logCode: 'kurentoextension_listenonly_offer_failure', logCode: 'kurentoextension_listenonly_offer_failure',
extraInfo: { error } extraInfo: { errorMessage, errorCode, errorReason },
}, `Failed to generate peer connection offer for listen only with error ${error.message}`); }, `Failed to generate peer connection offer for listen only with error ${error.message}`);
return this.onFail(error); return this.onFail({ errorMessage, errorCode, errorReason });
} }
const message = { const message = {