From 345f8d9c3dfa54d54728d9153813a8aa8b8b1f78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5var=20Aamb=C3=B8=20Fosstveit?= Date: Wed, 24 Oct 2018 09:54:51 +0200 Subject: [PATCH 01/30] New version of lastN --- app/lib/RoomClient.js | 142 +++++++++++++++++++++++++++++++++++------- server/lib/Room.js | 28 ++++++++- 2 files changed, 147 insertions(+), 23 deletions(-) diff --git a/app/lib/RoomClient.js b/app/lib/RoomClient.js index fe0998c..d32e0d2 100644 --- a/app/lib/RoomClient.js +++ b/app/lib/RoomClient.js @@ -19,7 +19,8 @@ const ROOM_OPTIONS = { requestTimeout : requestTimeout, transportOptions : transportOptions, - turnServers : turnServers + turnServers : turnServers, + lastN : 2 }; const VIDEO_CONSTRAINS = @@ -70,6 +71,12 @@ export default class RoomClient this._room = new mediasoupClient.Room(ROOM_OPTIONS); this._room.roomId = roomId; + // LastN speakers + this._lastNSpeakers = ROOM_OPTIONS.lastN; + + // Array of lastN speakers + this._lastN = []; + // Transport for sending. this._sendTransport = null; @@ -280,7 +287,8 @@ export default class RoomClient { const { chatHistory, - fileHistory + fileHistory, + lastN } = await this.sendRequest('server-history'); if (chatHistory.length > 0) @@ -296,6 +304,19 @@ export default class RoomClient this._dispatch(stateActions.addFileHistory(fileHistory)); } + + if (lastN.length > 0) + { + logger.debug('Got lastN'); + + const index = lastN.indexOf(this._peerName); + + lastN.splice(index, 1); + + this._lastN = lastN; + + this.updateSpeakers(); + } } catch (error) { @@ -319,6 +340,74 @@ export default class RoomClient this._micProducer.resume(); } + // Resumes consumers based on lastN speakers + async updateSpeakers() + { + logger.debug('updateSpeakers()'); + + try + { + const speakers = this._lastN.slice(0, this._lastNSpeakers); + + speakers.forEach((peerName) => + { + const peer = this._room.getPeerByName(peerName); + + if (peer) + { + for (const consumer of peer.consumers) + { + if (consumer.appData.source !== 'webcam' || + !consumer.supported || + !consumer.locallyPaused) + continue; + + consumer.resume(); + } + } + }); + } + catch(error) + { + logger.error('updateSpeakers() failed: %o', error); + } + } + + handleActiveSpeaker(peerName) + { + logger.debug('handleActiveSpeaker() [peerName:"%s"]', peerName); + + const index = this._lastN.indexOf(peerName); + + if (index > -1) // We have this speaker in the list, move to front + { + if (index >= this._lastNSpeakers) // We need to remove someone + { + const removePeer = this._lastN[this._lastNSpeakers - 1]; + + this.pausePeerVideo(removePeer); + } + + this._lastN.splice(index, 1); + this._lastN = [ peerName ].concat(this._lastN); + + this.updateSpeakers(); + } + else // We don't have this speaker in the list, should not happen + { + if (this._lastN.length >= this._lastNSpeakers) + { + const removePeer = this._lastN[this._lastNSpeakers - 1]; + + this.pausePeerVideo(removePeer); + } + + this._lastN = [ peerName ].concat(this._lastN); + + this.updateSpeakers(); + } + } + installExtension() { logger.debug('installExtension()'); @@ -1099,6 +1188,13 @@ export default class RoomClient logger.debug( 'room "newpeer" event [name:"%s", peer:%o]', peer.name, peer); + const index = this._lastN.indexOf(peer.name); + + if (index === -1) // We don't have this peer in the list, add + { + this._lastN.push(peer.name); + } + this._handlePeer(peer); }); @@ -1139,31 +1235,20 @@ export default class RoomClient })); // Don't produce if explicitely requested to not to do it. - if (!this._produce) - return; + if (this._produce) + { + if (this._room.canSend('audio')) + await this._setMicProducer(); - // NOTE: Don't depend on this Promise to continue (so we don't do return). - Promise.resolve() - // Add our mic. - .then(() => + // Add our webcam (unless the cookie says no). + if (this._room.canSend('video')) { - if (!this._room.canSend('audio')) - return; - - this._setMicProducer() - .catch(() => {}); - }) - // Add our webcam (unless the cookie says no). - .then(() => - { - if (!this._room.canSend('video')) - return; - const devicesCookie = cookiesManager.getDevices(); if (!devicesCookie || devicesCookie.webcamEnabled) - this.enableWebcam(); - }); + await this.enableWebcam(); + } + } this._dispatch(stateActions.setRoomState('connected')); @@ -1661,6 +1746,14 @@ export default class RoomClient this._dispatch(stateActions.removePeer(peer.name)); + const index = this._lastN.indexOf(peer.name); + + if (index > -1) // We have this peer in the list, remove + { + this._lastN.splice(index, 1); + this.updateSpeakers(); + } + if (this._room.joined) { this.notify(`${peer.appData.displayName} left the room`); @@ -1776,6 +1869,11 @@ export default class RoomClient if (consumer.kind === 'video' && this._getState().me.audioOnly) consumer.pause('audio-only-mode'); + const index = this._lastN.indexOf(consumer.peer.name); + + if (consumer.kind === 'video' && ((index >= this._lastNSpeakers) || (index === -1))) + consumer.pause('not-speaker'); + consumer.receive(this._recvTransport) .then((track) => { diff --git a/server/lib/Room.js b/server/lib/Room.js index 9f643fd..1545fa6 100644 --- a/server/lib/Room.js +++ b/server/lib/Room.js @@ -38,6 +38,9 @@ class Room extends EventEmitter this._fileHistory = []; + this._lastN = []; + + this._io = io; this._signalingPeers = new Map(); @@ -123,6 +126,13 @@ class Room extends EventEmitter const signalingPeer = { peerName : peerName, socket : socket }; + const index = this._lastN.indexOf(peerName); + + if (index === -1) // We don't have this peer, add to end + { + this._lastN.push(peerName); + } + this._handleSignalingPeer(signalingPeer); } @@ -140,6 +150,14 @@ class Room extends EventEmitter this._currentActiveSpeaker = activePeer; + const index = this._lastN.indexOf(activePeer.name); + + if (index > -1) // We have this speaker in the list, move to front + { + this._lastN.splice(index, 1); + this._lastN = [activePeer.name].concat(this._lastN); + } + const activeVideoProducer = activePeer.producers .find((producer) => producer.kind === 'video'); @@ -269,7 +287,8 @@ class Room extends EventEmitter null, { chatHistory : this._chatHistory, - fileHistory : this._fileHistory + fileHistory : this._fileHistory, + lastN : this._lastN } ); }); @@ -325,6 +344,13 @@ class Room extends EventEmitter if (mediaPeer && !mediaPeer.closed) mediaPeer.close(); + const index = this._lastN.indexOf(signalingPeer.peerName); + + if (index > -1) // We have this peer in the list, remove + { + this._lastN.splice(index, 1); + } + // If this is the latest peer in the room, close the room. // However wait a bit (for reconnections). setTimeout(() => From 59d570aa0b984e9bcda18bcd6f56900b43d01a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5var=20Aamb=C3=B8=20Fosstveit?= Date: Wed, 24 Oct 2018 10:06:10 +0200 Subject: [PATCH 02/30] Activate lastN --- app/lib/RoomClient.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/lib/RoomClient.js b/app/lib/RoomClient.js index d32e0d2..7bf0b3e 100644 --- a/app/lib/RoomClient.js +++ b/app/lib/RoomClient.js @@ -367,7 +367,7 @@ export default class RoomClient } }); } - catch(error) + catch (error) { logger.error('updateSpeakers() failed: %o', error); } @@ -1061,6 +1061,9 @@ export default class RoomClient this._dispatch( stateActions.setRoomActiveSpeaker(peerName)); + + if (peerName && peerName !== this._peerName) + this.handleActiveSpeaker(peerName); }); this._signalingSocket.on('display-name-changed', (data) => @@ -1240,7 +1243,7 @@ export default class RoomClient if (this._room.canSend('audio')) await this._setMicProducer(); - // Add our webcam (unless the cookie says no). + // Add our webcam (unless the cookie says no). if (this._room.canSend('video')) { const devicesCookie = cookiesManager.getDevices(); From 5f4be29d40c10178da02c07dd67703458dede6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5var=20Aamb=C3=B8=20Fosstveit?= Date: Wed, 24 Oct 2018 11:23:53 +0200 Subject: [PATCH 03/30] Separated the audio from peers to their own components --- app/lib/components/Peer.jsx | 1 - app/lib/components/PeerAudio/AudioPeer.jsx | 39 ++++++++++++ app/lib/components/PeerAudio/AudioPeers.jsx | 44 ++++++++++++++ app/lib/components/PeerAudio/PeerAudio.jsx | 67 +++++++++++++++++++++ app/lib/components/Room.jsx | 3 + app/stylus/components/AudioPeers.styl | 6 ++ app/stylus/index.styl | 1 + 7 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 app/lib/components/PeerAudio/AudioPeer.jsx create mode 100644 app/lib/components/PeerAudio/AudioPeers.jsx create mode 100644 app/lib/components/PeerAudio/PeerAudio.jsx create mode 100644 app/stylus/components/AudioPeers.styl diff --git a/app/lib/components/Peer.jsx b/app/lib/components/Peer.jsx index 2aec88a..a49a688 100644 --- a/app/lib/components/Peer.jsx +++ b/app/lib/components/Peer.jsx @@ -149,7 +149,6 @@ class Peer extends Component +{ + return ( + + ); +}; + +AudioPeer.propTypes = +{ + micConsumer : appPropTypes.Consumer, + name : PropTypes.string +}; + +const mapStateToProps = (state, { name }) => +{ + const peer = state.peers[name]; + const consumersArray = peer.consumers + .map((consumerId) => state.consumers[consumerId]); + const micConsumer = + consumersArray.find((consumer) => consumer.source === 'mic'); + + return { + micConsumer + }; +}; + +const AudioPeerContainer = connect( + mapStateToProps +)(AudioPeer); + +export default AudioPeerContainer; diff --git a/app/lib/components/PeerAudio/AudioPeers.jsx b/app/lib/components/PeerAudio/AudioPeers.jsx new file mode 100644 index 0000000..3dce02e --- /dev/null +++ b/app/lib/components/PeerAudio/AudioPeers.jsx @@ -0,0 +1,44 @@ +import React from 'react'; +import { connect } from 'react-redux'; +import PropTypes from 'prop-types'; +import * as appPropTypes from '../appPropTypes'; +import AudioPeer from './AudioPeer'; + +const AudioPeers = ({ peers }) => +{ + return ( +
+ { + peers.map((peer) => + { + return ( + + ); + }) + } +
+ ); +}; + +AudioPeers.propTypes = + { + peers : PropTypes.arrayOf(appPropTypes.Peer).isRequired + }; + +const mapStateToProps = (state) => +{ + const peers = Object.values(state.peers); + + return { + peers + }; +}; + +const AudioPeersContainer = connect( + mapStateToProps +)(AudioPeers); + +export default AudioPeersContainer; diff --git a/app/lib/components/PeerAudio/PeerAudio.jsx b/app/lib/components/PeerAudio/PeerAudio.jsx new file mode 100644 index 0000000..871c8c4 --- /dev/null +++ b/app/lib/components/PeerAudio/PeerAudio.jsx @@ -0,0 +1,67 @@ +import React from 'react'; +import PropTypes from 'prop-types'; + +export default class PeerAudio extends React.Component +{ + constructor(props) + { + super(props); + + // Latest received audio track. + // @type {MediaStreamTrack} + this._audioTrack = null; + } + + render() + { + return ( +