From 7a3bd327f2bc941adb3b0626e41e085f2fbc4695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?H=C3=A5var=20Aamb=C3=B8=20Fosstveit?= <havar@fosstveit.net>
Date: Thu, 30 Apr 2020 22:42:47 +0200
Subject: [PATCH] Play audio from video elements, fixes #242

---
 app/src/components/Containers/Peer.js         |  1 +
 app/src/components/PeerAudio/AudioPeers.js    |  8 +-
 app/src/components/Selectors.js               |  9 ++
 .../components/VideoContainers/VideoView.js   | 95 +++++++++++++++----
 4 files changed, 89 insertions(+), 24 deletions(-)

diff --git a/app/src/components/Containers/Peer.js b/app/src/components/Containers/Peer.js
index 30cbecf..827550b 100644
--- a/app/src/components/Containers/Peer.js
+++ b/app/src/components/Containers/Peer.js
@@ -340,6 +340,7 @@ const Peer = (props) =>
 						videoMultiLayer={webcamConsumer && webcamConsumer.type !== 'simple'}
 						videoTrack={webcamConsumer && webcamConsumer.track}
 						videoVisible={videoVisible}
+						audioTrack={micConsumer && micConsumer.track}
 						audioCodec={micConsumer && micConsumer.codec}
 						videoCodec={webcamConsumer && webcamConsumer.codec}
 						audioScore={micConsumer ? micConsumer.score : null}
diff --git a/app/src/components/PeerAudio/AudioPeers.js b/app/src/components/PeerAudio/AudioPeers.js
index d02f26d..6e9921e 100644
--- a/app/src/components/PeerAudio/AudioPeers.js
+++ b/app/src/components/PeerAudio/AudioPeers.js
@@ -1,6 +1,6 @@
 import React from 'react';
 import { connect } from 'react-redux';
-import { micConsumerSelector } from '../Selectors';
+import { passiveMicConsumerSelector } from '../Selectors';
 import PropTypes from 'prop-types';
 import PeerAudio from './PeerAudio';
 
@@ -37,7 +37,7 @@ AudioPeers.propTypes =
 
 const mapStateToProps = (state) =>
 	({
-		micConsumers      : micConsumerSelector(state),
+		micConsumers      : passiveMicConsumerSelector(state),
 		audioOutputDevice : state.settings.selectedAudioOutputDevice
 	});
 
@@ -50,7 +50,9 @@ const AudioPeersContainer = connect(
 		{
 			return (
 				prev.consumers === next.consumers &&
-				prev.settings.selectedAudioOutputDevice === next.settings.selectedAudioOutputDevice
+				prev.room.spotlights === next.room.spotlights &&
+				prev.settings.selectedAudioOutputDevice ===
+				next.settings.selectedAudioOutputDevice
 			);
 		}
 	}
diff --git a/app/src/components/Selectors.js b/app/src/components/Selectors.js
index a87ce16..e768cd5 100644
--- a/app/src/components/Selectors.js
+++ b/app/src/components/Selectors.js
@@ -67,6 +67,15 @@ export const screenConsumerSelector = createSelector(
 	(consumers) => Object.values(consumers).filter((consumer) => consumer.source === 'screen')
 );
 
+export const passiveMicConsumerSelector = createSelector(
+	spotlightsSelector,
+	consumersSelect,
+	(spotlights, consumers) =>
+		Object.values(consumers).filter(
+			(consumer) => consumer.source === 'mic' && !spotlights.includes(consumer.peerId)
+		)
+);
+
 export const spotlightsLengthSelector = createSelector(
 	spotlightsSelector,
 	(spotlights) => spotlights.length
diff --git a/app/src/components/VideoContainers/VideoView.js b/app/src/components/VideoContainers/VideoView.js
index c978433..e6e4fa3 100644
--- a/app/src/components/VideoContainers/VideoView.js
+++ b/app/src/components/VideoContainers/VideoView.js
@@ -3,6 +3,7 @@ import PropTypes from 'prop-types';
 import classnames from 'classnames';
 import { withStyles } from '@material-ui/core/styles';
 import EditableInput from '../Controls/EditableInput';
+import Logger from '../../Logger';
 import { green, yellow, orange, red } from '@material-ui/core/colors';
 import SignalCellularOffIcon from '@material-ui/icons/SignalCellularOff';
 import SignalCellular0BarIcon from '@material-ui/icons/SignalCellular0Bar';
@@ -11,6 +12,8 @@ import SignalCellular2BarIcon from '@material-ui/icons/SignalCellular2Bar';
 import SignalCellular3BarIcon from '@material-ui/icons/SignalCellular3Bar';
 import SignalCellularAltIcon from '@material-ui/icons/SignalCellularAlt';
 
+const logger = new Logger('VideoView');
+
 const styles = (theme) =>
 	({
 		root :
@@ -134,6 +137,10 @@ class VideoView extends React.PureComponent
 			videoHeight : null
 		};
 
+		// Latest received audio track
+		// @type {MediaStreamTrack}
+		this._audioTrack = null;
+
 		// Latest received video track.
 		// @type {MediaStreamTrack}
 		this._videoTrack = null;
@@ -292,7 +299,7 @@ class VideoView extends React.PureComponent
 				</div>
 
 				<video
-					ref='video'
+					ref='videoElement'
 					className={classnames(classes.video, {
 						hidden  : !videoVisible,
 						'isMe'  : isMe && !isScreen,
@@ -300,6 +307,16 @@ class VideoView extends React.PureComponent
 					})}
 					autoPlay
 					playsInline
+					muted
+					controls={false}
+				/>
+
+				<audio
+					ref='audioElement'
+					autoPlay
+					playsInline
+					muted={isMe}
+					controls={false}
 				/>
 
 				{children}
@@ -309,52 +326,84 @@ class VideoView extends React.PureComponent
 
 	componentDidMount()
 	{
-		const { videoTrack } = this.props;
+		const { videoTrack, audioTrack } = this.props;
 
-		this._setTracks(videoTrack);
+		this._setTracks(videoTrack, audioTrack);
 	}
 
 	componentWillUnmount()
 	{
 		clearInterval(this._videoResolutionTimer);
+
+		const { videoElement } = this.refs;
+
+		if (videoElement)
+		{
+			videoElement.oncanplay = null;
+			videoElement.onplay = null;
+			videoElement.onpause = null;
+		}
 	}
 
-	// eslint-disable-next-line camelcase
-	UNSAFE_componentWillReceiveProps(nextProps)
+	componentWillUpdate()
 	{
-		const { videoTrack } = nextProps;
-
-		this._setTracks(videoTrack);
+		const { videoTrack, audioTrack } = this.props;
 
+		this._setTracks(videoTrack, audioTrack);
 	}
 
-	_setTracks(videoTrack)
+	_setTracks(videoTrack, audioTrack)
 	{
-		if (this._videoTrack === videoTrack)
+		if (this._videoTrack === videoTrack && this._audioTrack === audioTrack)
 			return;
 
 		this._videoTrack = videoTrack;
+		this._audioTrack = audioTrack;
 
 		clearInterval(this._videoResolutionTimer);
 		this._hideVideoResolution();
 
-		const { video } = this.refs;
+		const { videoElement, audioElement } = this.refs;
 
 		if (videoTrack)
 		{
 			const stream = new MediaStream();
 
-			if (videoTrack)
-				stream.addTrack(videoTrack);
+			stream.addTrack(videoTrack);
 
-			video.srcObject = stream;
+			videoElement.srcObject = stream;
 
-			if (videoTrack)
-				this._showVideoResolution();
+			videoElement.oncanplay = () => this.setState({ videoCanPlay: true });
+
+			videoElement.onplay = () =>
+			{
+				audioElement.play()
+					.catch((error) => logger.warn('audioElement.play() [error:"%o]', error));
+			};
+
+			videoElement.play()
+				.catch((error) => logger.warn('videoElement.play() [error:"%o]', error));
+
+			this._showVideoResolution();
 		}
 		else
 		{
-			video.srcObject = null;
+			videoElement.srcObject = null;
+		}
+
+		if (audioTrack)
+		{
+			const stream = new MediaStream();
+
+			stream.addTrack(audioTrack);
+			audioElement.srcObject = stream;
+
+			audioElement.play()
+				.catch((error) => logger.warn('audioElement.play() [error:"%o]', error));
+		}
+		else
+		{
+			audioElement.srcObject = null;
 		}
 	}
 
@@ -363,16 +412,19 @@ class VideoView extends React.PureComponent
 		this._videoResolutionTimer = setInterval(() =>
 		{
 			const { videoWidth, videoHeight } = this.state;
-			const { video } = this.refs;
+			const { videoElement } = this.refs;
 
 			// Don't re-render if nothing changed.
-			if (video.videoWidth === videoWidth && video.videoHeight === videoHeight)
+			if (
+				videoElement.videoWidth === videoWidth &&
+				videoElement.videoHeight === videoHeight
+			)
 				return;
 
 			this.setState(
 				{
-					videoWidth  : video.videoWidth,
-					videoHeight : video.videoHeight
+					videoWidth  : videoElement.videoWidth,
+					videoHeight : videoElement.videoHeight
 				});
 		}, 1000);
 	}
@@ -392,6 +444,7 @@ VideoView.propTypes =
 	videoContain                   : PropTypes.bool,
 	advancedMode                   : PropTypes.bool,
 	videoTrack                     : PropTypes.any,
+	audioTrack                     : PropTypes.any,
 	videoVisible                   : PropTypes.bool.isRequired,
 	consumerSpatialLayers          : PropTypes.number,
 	consumerTemporalLayers         : PropTypes.number,