From c424ffc17c65ddc6a33e84ccfc1975f9d801ce53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5var=20Aamb=C3=B8=20Fosstveit?= Date: Thu, 19 Mar 2020 21:23:01 +0100 Subject: [PATCH] Fix simulcast --- app/src/RoomClient.js | 139 ++++++++++++++++++++++++----- app/src/actions/consumerActions.js | 8 ++ app/src/index.js | 15 +++- app/src/reducers/consumers.js | 9 ++ 4 files changed, 146 insertions(+), 25 deletions(-) diff --git a/app/src/RoomClient.js b/app/src/RoomClient.js index ca7c41c..1e93f80 100644 --- a/app/src/RoomClient.js +++ b/app/src/RoomClient.js @@ -83,11 +83,28 @@ const VIDEO_CONSTRAINS = } }; -const VIDEO_ENCODINGS = +const PC_PROPRIETARY_CONSTRAINTS = +{ + optional : [ { googDscp: true } ] +}; + +const VIDEO_SIMULCAST_ENCODINGS = [ - { maxBitrate: 180000, scaleResolutionDownBy: 4 }, - { maxBitrate: 360000, scaleResolutionDownBy: 2 }, - { maxBitrate: 1500000, scaleResolutionDownBy: 1 } + { scaleResolutionDownBy: 4 }, + { scaleResolutionDownBy: 2 }, + { scaleResolutionDownBy: 1 } +]; + +// Used for VP9 webcam video. +const VIDEO_KSVC_ENCODINGS = +[ + { scalabilityMode: 'S3T3_KEY' } +]; + +// Used for VP9 desktop sharing. +const VIDEO_SVC_ENCODINGS = +[ + { scalabilityMode: 'S3T3', dtx: true } ]; let store; @@ -108,7 +125,7 @@ export default class RoomClient } constructor( - { peerId, accessCode, device, useSimulcast, produce, forceTcp, displayName, muted } = {}) + { peerId, accessCode, device, useSimulcast, useSharingSimulcast, produce, forceTcp, displayName, muted } = {}) { if (!peerId) throw new Error('Missing peerId'); @@ -140,6 +157,9 @@ export default class RoomClient // Whether simulcast should be used. this._useSimulcast = useSimulcast; + // Whether simulcast should be used for sharing + this._useSharingSimulcast = useSharingSimulcast; + this._muted = muted; // This device @@ -1235,7 +1255,7 @@ export default class RoomClient { if (this._webcamProducer) await this._webcamProducer.setMaxSpatialLayer(spatialLayer); - else if (this._screenSharingProducer) + if (this._screenSharingProducer) await this._screenSharingProducer.setMaxSpatialLayer(spatialLayer); } catch (error) @@ -1264,6 +1284,38 @@ export default class RoomClient } } + async setConsumerPriority(consumerId, priority) + { + logger.debug( + 'setConsumerPriority() [consumerId:%s, priority:%d]', + consumerId, priority); + + try + { + await this.sendRequest('setConsumerPriority', { consumerId, priority }); + + store.dispatch(consumerActions.setConsumerPriority(consumerId, priority)); + } + catch (error) + { + logger.error('setConsumerPriority() | failed:%o', error); + } + } + + async requestConsumerKeyFrame(consumerId) + { + logger.debug('requestConsumerKeyFrame() [consumerId:%s]', consumerId); + + try + { + await this.sendRequest('requestConsumerKeyFrame', { consumerId }); + } + catch (error) + { + logger.error('requestConsumerKeyFrame() | failed:%o', error); + } + } + async _loadDynamicImports() { ({ default: WebTorrent } = await import( @@ -1481,6 +1533,7 @@ export default class RoomClient temporalLayers : temporalLayers, preferredSpatialLayer : spatialLayers - 1, preferredTemporalLayer : temporalLayers - 1, + priority : 1, codec : consumer.rtpParameters.codecs[0].mimeType.split('/')[1], track : consumer.track }, @@ -1849,10 +1902,10 @@ export default class RoomClient case 'newPeer': { - const { id, displayName, picture, device } = notification.data; + const { id, displayName, picture } = notification.data; store.dispatch( - peerActions.addPeer({ id, displayName, picture, device, consumers: [] })); + peerActions.addPeer({ id, displayName, picture, consumers: [] })); store.dispatch(requestActions.notify( { @@ -2017,7 +2070,8 @@ export default class RoomClient iceParameters, iceCandidates, dtlsParameters, - iceServers : ROOM_OPTIONS.turnServers + iceServers : ROOM_OPTIONS.turnServers, + proprietaryConstraints : PC_PROPRIETARY_CONSTRAINTS }); this._sendTransport.on( @@ -2034,18 +2088,26 @@ export default class RoomClient }); this._sendTransport.on( - 'produce', ({ kind, rtpParameters, appData }, callback, errback) => + 'produce', async ({ kind, rtpParameters, appData }, callback, errback) => { - this.sendRequest( - 'produce', - { - transportId : this._sendTransport.id, - kind, - rtpParameters, - appData - }) - .then(callback) - .catch(errback); + try + { + // eslint-disable-next-line no-shadow + const { id } = await this.sendRequest( + 'produce', + { + transportId : this._sendTransport.id, + kind, + rtpParameters, + appData + }); + + callback({ id }); + } + catch (error) + { + errback(error); + } }); } @@ -2496,12 +2558,30 @@ export default class RoomClient track = stream.getVideoTracks()[0]; - if (this._useSimulcast) + if (this._useSharingSimulcast) { + // If VP9 is the only available video codec then use SVC. + const firstVideoCodec = this._mediasoupDevice + .rtpCapabilities + .codecs + .find((c) => c.kind === 'video'); + + let encodings; + + if (firstVideoCodec.mimeType.toLowerCase() === 'video/vp9') + { + encodings = VIDEO_SVC_ENCODINGS; + } + else + { + encodings = VIDEO_SIMULCAST_ENCODINGS + .map((encoding) => ({ ...encoding, dtx: true })); + } + this._screenSharingProducer = await this._sendTransport.produce( { track, - encodings : VIDEO_ENCODINGS, + encodings, codecOptions : { videoGoogleStartBitrate : 1000 @@ -2652,10 +2732,23 @@ export default class RoomClient if (this._useSimulcast) { + // If VP9 is the only available video codec then use SVC. + const firstVideoCodec = this._mediasoupDevice + .rtpCapabilities + .codecs + .find((c) => c.kind === 'video'); + + let encodings; + + if (firstVideoCodec.mimeType.toLowerCase() === 'video/vp9') + encodings = VIDEO_KSVC_ENCODINGS; + else + encodings = VIDEO_SIMULCAST_ENCODINGS; + this._webcamProducer = await this._sendTransport.produce( { track, - encodings : VIDEO_ENCODINGS, + encodings, codecOptions : { videoGoogleStartBitrate : 1000 diff --git a/app/src/actions/consumerActions.js b/app/src/actions/consumerActions.js index 0c7bb65..b8460a6 100644 --- a/app/src/actions/consumerActions.js +++ b/app/src/actions/consumerActions.js @@ -34,6 +34,14 @@ export const setConsumerPreferredLayers = (consumerId, spatialLayer, temporalLay payload : { consumerId, spatialLayer, temporalLayer } }); +export const setConsumerPriority = (consumerId, priority) => + { + return { + type : 'SET_CONSUMER_PRIORITY', + payload : { consumerId, priority } + }; + }; + export const setConsumerTrack = (consumerId, track) => ({ type : 'SET_CONSUMER_TRACK', diff --git a/app/src/index.js b/app/src/index.js index d601a31..0cadc46 100644 --- a/app/src/index.js +++ b/app/src/index.js @@ -102,7 +102,8 @@ function run() const accessCode = parameters.get('code'); const produce = parameters.get('produce') !== 'false'; - const useSimulcast = parameters.get('simulcast') === 'true'; + const useSimulcast = parameters.get('simulcast') !== 'false'; + const useSharingSimulcast = parameters.get('sharingSimulcast') === 'true'; const forceTcp = parameters.get('forceTcp') === 'true'; const displayName = parameters.get('displayName'); const muted = parameters.get('muted') === 'true'; @@ -118,7 +119,17 @@ function run() ); roomClient = new RoomClient( - { peerId, accessCode, device, useSimulcast, produce, forceTcp, displayName, muted }); + { + peerId, + accessCode, + device, + useSimulcast, + useSharingSimulcast, + produce, + forceTcp, + displayName, + muted + }); global.CLIENT = roomClient; diff --git a/app/src/reducers/consumers.js b/app/src/reducers/consumers.js index 2c99416..68a4a4a 100644 --- a/app/src/reducers/consumers.js +++ b/app/src/reducers/consumers.js @@ -79,6 +79,15 @@ const consumers = (state = initialState, action) => return { ...state, [consumerId]: newConsumer }; } + case 'SET_CONSUMER_PRIORITY': + { + const { consumerId, priority } = action.payload; + const consumer = state[consumerId]; + const newConsumer = { ...consumer, priority }; + + return { ...state, [consumerId]: newConsumer }; + } + case 'SET_CONSUMER_TRACK': { const { consumerId, track } = action.payload;