BigBlueButton/bigbluebutton-Github
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(screenshare): check packet flow to detect unhealthy streams earlier

Browse Source 
Screen streams were only deemed unhealthy when the transport's ICE state
transitioned to failed. That was as good as nothing because the stream would
stay frozen with no visual UI feedback until it reconnected. Bad UX.

This commit addresses that issue via two changes:
  - A stream is deemed *potentially* unhealthy now if the transport's
    state becomes disconnected
  - If a stream is deemed potentially unhealthy, a monitor probe is
    started to check whether there is media/packet flow (every 500ms).
    If there's no packet flow, the stream is flagged is factually unhealthy and
    UI feedback about that is rendered.

It's still not as good as it could be - relying on disconnected still
leaves a couple of seconds of silence to be dealt with. For that to be
addressed the prober would have to run nonstop, but that's for later.
This commit is contained in:
prlanzarin 2022-05-09 01:59:25 +00:00
parent dced094ad7
commit d350afd194
2 changed files with 81 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
bigbluebutton-html5/imports/ui/components/screenshare/component.jsx
View File

@ -12,15 +12,17 @@ import playAndRetry from '/imports/utils/mediaElementPlayRetry';
import { notify } from '/imports/ui/services/notification';
import {
SCREENSHARE_MEDIA_ELEMENT_NAME,
isMediaFlowing,
screenshareHasEnded,
screenshareHasStarted,
getMediaElement,
attachLocalPreviewStream,
setVolume,
getVolume,
getStats,
} from '/imports/ui/components/screenshare/service';
import {
isStreamStateUnhealthy,
isStreamStateHealthy,
subscribeToStreamStateChange,
unsubscribeFromStreamStateChange,
} from '/imports/ui/services/bbb-webrtc-sfu/stream-state-service';
@ -64,6 +66,7 @@ const intlMessages = defineMessages({
const ALLOW_FULLSCREEN = Meteor.settings.public.app.allowFullscreen;
const MOBILE_HOVER_TIMEOUT = 5000;
const MEDIA_FLOW_PROBE_INTERVAL = 500;
class ScreenshareComponent extends React.Component {
static renderScreenshareContainerInside(mainText) {
@ -79,7 +82,7 @@ class ScreenshareComponent extends React.Component {
this.state = {
loaded: false,
autoplayBlocked: false,
isStreamHealthy: false,
mediaFlowing: false,
switched: false,
// Volume control hover toolbar
showHoverToolBar: false,
@ -96,6 +99,7 @@ class ScreenshareComponent extends React.Component {
this.volume = getVolume();
this.mobileHoverSetTimeout = null;
this.mediaFlowMonitor = null;
}
componentDidMount() {
@ -130,9 +134,7 @@ class ScreenshareComponent extends React.Component {
}
componentDidUpdate(prevProps) {
const {
isPresenter,
} = this.props;
const { isPresenter } = this.props;
if (prevProps.isPresenter && !isPresenter) {
screenshareHasEnded();
}
@ -170,6 +172,14 @@ class ScreenshareComponent extends React.Component {
});
}
this.clearMediaFlowingMonitor();
}
clearMediaFlowingMonitor() {
if (this.mediaFlowMonitor) {
Meteor.clearInterval(this.mediaFlowMonitor);
this.mediaFlowMonitor = null;
}
}
handleAllowAutoplay() {
@ -213,15 +223,25 @@ class ScreenshareComponent extends React.Component {
}
}
onStreamStateChange(event) {
const { streamState } = event.detail;
const { isStreamHealthy } = this.state;
async monitorMediaFlow() {
let previousStats = await getStats();
this.mediaFlowMonitor = Meteor.setInterval(async () => {
const { mediaFlowing: prevMediaFlowing } = this.state;
let mediaFlowing;
const newHealthState = !isStreamStateUnhealthy(streamState);
event.stopPropagation();
if (newHealthState !== isStreamHealthy) {
this.setState({ isStreamHealthy: newHealthState });
}
const currentStats = await getStats();
try {
mediaFlowing = isMediaFlowing(previousStats, currentStats);
} catch (_error) {
// Stats processing failed for whatever reason - maintain previous state
mediaFlowing = prevMediaFlowing;
}
previousStats = currentStats;
if (prevMediaFlowing !== mediaFlowing) this.setState({ mediaFlowing });
}, MEDIA_FLOW_PROBE_INTERVAL);
}
onLoadedData() {
@ -245,6 +265,22 @@ class ScreenshareComponent extends React.Component {
}
}
onStreamStateChange(event) {
const { streamState } = event.detail;
const { mediaFlowing } = this.state;
const isStreamHealthy = isStreamStateHealthy(streamState);
event.stopPropagation();
if (isStreamHealthy) {
this.clearMediaFlowingMonitor();
// Current state is media not flowing - stream is now healthy so flip it
if (!mediaFlowing) this.setState({ mediaFlowing: isStreamHealthy });
} else {
if (this.mediaFlowMonitor == null) this.monitorMediaFlow();
}
}
renderFullscreenButton() {
const { intl, fullscreenElementId, fullscreenContext } = this.props;
@ -318,7 +354,6 @@ class ScreenshareComponent extends React.Component {
if (deviceInfo.isMobile && showHoverToolBar) {
toolbarStyle = 'showMobileHoverToolbar';
}
return [(
<Styled.HoverToolbar
@ -338,12 +373,13 @@ class ScreenshareComponent extends React.Component {
renderVideo(switched) {
const { isGloballyBroadcasting } = this.props;
const { mediaFlowing } = this.state;
return (
<Styled.ScreenshareVideo
id={SCREENSHARE_MEDIA_ELEMENT_NAME}
key={SCREENSHARE_MEDIA_ELEMENT_NAME}
unhealthyStream={!isGloballyBroadcasting}
unhealthyStream={!isGloballyBroadcasting || !mediaFlowing}
style={switched
? { maxHeight: '100%', width: '100%', height: '100%' }
: { maxHeight: '25%', width: '25%', height: '25%' }}
@ -418,7 +454,7 @@ class ScreenshareComponent extends React.Component {
}
render() {
const { loaded, autoplayBlocked, isStreamHealthy } = this.state;
const { loaded, autoplayBlocked, mediaFlowing} = this.state;
const {
isPresenter,
isGloballyBroadcasting,
@ -439,7 +475,7 @@ class ScreenshareComponent extends React.Component {
// state transitioned to an unhealthy stream. tl;dr: screen sharing reconnection
const shouldRenderConnectingState = !loaded
|| (isPresenter && !isGloballyBroadcasting)
|| (!isStreamHealthy && loaded && isGloballyBroadcasting);
|| (!mediaFlowing && loaded && isGloballyBroadcasting);
const display = (width > 0 && height > 0) ? 'inherit' : 'none';
const { animations } = Settings.application;

45
bigbluebutton-html5/imports/ui/components/screenshare/service.js
View File

@ -9,14 +9,12 @@ import Auth from '/imports/ui/services/auth';
import AudioService from '/imports/ui/components/audio/service';
import { Meteor } from "meteor/meteor";
import MediaStreamUtils from '/imports/utils/media-stream-utils';
import ConnectionStatusService from '/imports/ui/components/connection-status/service';
const VOLUME_CONTROL_ENABLED = Meteor.settings.public.kurento.screenshare.enableVolumeControl;
const SCREENSHARE_MEDIA_ELEMENT_NAME = 'screenshareVideo';
/**
* Screenshare status to be filtered in getStats()
*/
const FILTER_SCREENSHARE_STATS = [
const DEFAULT_SCREENSHARE_STATS_TYPES = [
'outbound-rtp',
'inbound-rtp',
];
@ -152,33 +150,33 @@ const screenShareEndAlert = () => AudioService
const dataSavingSetting = () => Settings.dataSaving.viewScreenshare;
/**
* Get stats about all active screenshare peer.
* We filter the status based on FILTER_SCREENSHARE_STATS constant.
* Get stats about all active screenshare peers.
*
* For more information see:
* https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/getStats
* and
* https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport
* @returns An Object containing the information about each active peer
* (currently one, for screenshare). The returned format
* follows the format returned by video's service getStats, which
* considers more than one peer connection to be returned.
* - https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/getStats
* - https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport
* @param {Array[String]} statsType - An array containing valid RTCStatsType
* values to include in the return object
*
* @returns {Object} The information about each active screen sharing peer.
* The returned format follows the format returned by video's service
* getStats, which considers more than one peer connection to be returned.
* The format is given by:
* {
* peerIdString: RTCStatsReport
* }
*/
const getStats = async () => {
const getStats = async (statsTypes = DEFAULT_SCREENSHARE_STATS_TYPES) => {
const screenshareStats = {};
const peer = KurentoBridge.getPeerConnection();
if (!peer) return null;
const peerStats = await peer.getStats();
const screenshareStats = {};
peerStats.forEach((stat) => {
if (FILTER_SCREENSHARE_STATS.includes(stat.type)) {
if (statsTypes.includes(stat.type)) {
screenshareStats[stat.type] = stat;
}
});
@ -186,8 +184,21 @@ const getStats = async () => {
return { screenshareStats };
};
// This method may throw errors
const isMediaFlowing = (previousStats, currentStats) => {
const bpsData = ConnectionStatusService.calculateBitsPerSecond(
currentStats.screenshareStats,
previousStats.screenshareStats,
);
const bpsDataAggr = Object.values(bpsData)
.reduce((sum, partialBpsData = 0) => sum + parseFloat(partialBpsData), 0);
return bpsDataAggr > 0;
};
export {
SCREENSHARE_MEDIA_ELEMENT_NAME,
isMediaFlowing,
isVideoBroadcasting,
screenshareHasEnded,
screenshareHasStarted,