From 32af8b837c46930918f35e210150b2140c58bdb2 Mon Sep 17 00:00:00 2001 From: Yordan Neshkolov Date: Mon, 4 May 2020 19:20:20 +0300 Subject: [PATCH] NY-9977 No audio between certain participants in conference --- src/call/Call.js | 39 ++++++++++++++++++--------- src/call/CallParticipantData.js | 47 ++++++++++++++++++++++++++++++++- src/call/P2pCall.js | 9 +++++-- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/src/call/Call.js b/src/call/Call.js index abc7cc0..f57a1f2 100644 --- a/src/call/Call.js +++ b/src/call/Call.js @@ -52,6 +52,10 @@ class Call { }; this._mediaStream = null; + // create audio context as well + const AudioContext = window.AudioContext || window.webkitAudioContext; + this._audioCtx = new AudioContext(); + this._offerOptions = { offerToReceiveAudio: 1, offerToReceiveVideo: 1, @@ -619,9 +623,7 @@ class Call { // mute all audio/video elements Object.values(this._callParticipantsData).forEach(callParticipantData => { - if (callParticipantData.mediaElement) { - callParticipantData.mediaElement.muted = muted; - } + callParticipantData.mute = muted; }); this._speakerMuted = muted; @@ -657,6 +659,12 @@ class Call { registerSignaling = () => { console.log("[SDK] registerSignaling"); + if (!this._audioCtx) { + // create audio context as well + const AudioContext = window.AudioContext || window.webkitAudioContext; + this._audioCtx = new AudioContext(); + } + if (this._state !== CallState.CALL_STATE_CONNECTING) { console.warn("[SDK] registerSignaling IGNORED", this._state); return; @@ -960,14 +968,13 @@ class Call { this.callState = CallStates.CALL_STATE_ESTABLISHING; if (!this._callParticipantData) { - this._callParticipantData = new CallParticipantData(this._participantId, stream.id); + this._callParticipantData = new CallParticipantData(this._participantId, null, stream.id); } else { this._callParticipantData.streamId = stream.id; this._callParticipantData.participantId = this._participantId; } if (!this._callParticipantData.mediaElement) { const mediaElement = this._createMediaElement(this._participantId); - mediaElement.muted = true; this._callParticipantData.mediaElement = mediaElement; if (this._localElementContainer) { @@ -996,10 +1003,13 @@ class Call { } if (callParticipantData.mediaStream !== mediaStream) { callParticipantData.mediaStream = mediaStream; + if (this._speakerMuted) { + callParticipantData.mute = true; + } } if (setActiveSpeaker && mediaStream && mediaStream.getAudioTracks().length > 0) { - callParticipantData.hark = hark(mediaStream, {}); + callParticipantData.hark = hark(mediaStream, { audioContext: this._audioCtx }); callParticipantData.hark.on('speaking', () => { this._notifyParticipantSpeaking(callParticipantData.participantId, true); }); @@ -1080,7 +1090,8 @@ class Call { if (streamId) { if (!this._callParticipantsData[partId]) { if (!callParticipantStreamMapping[partId]) { - this._callParticipantsData[partId] = new CallParticipantData(partId, streamId, mediaType); + this._callParticipantsData[partId] = new CallParticipantData(partId, this._audioCtx, streamId, mediaType); + this._callParticipantsData[partId].mute = this._speakerMuted; const member = this.findParticipant(partId); if (member) { this._callParticipantsData[partId].mediaContainer = member.mediaContainer; @@ -1364,6 +1375,10 @@ class Call { this._remoteScreenSharer = null; } + if (this._audioCtx) { + this._audioCtx.close(); + this._audioCtx = null; + } if (this._mediaStream) { this._mediaStream.getTracks().forEach(track => this._stopMediaStreamTrack(track)); this._mediaStream = null; @@ -1446,9 +1461,6 @@ class Call { if (!callParticipantData.mediaElement) { let mediaElement = this._createMediaElement(callParticipantData.memberId); - if (this._speakerMuted) { - mediaElement.muted = true; - } callParticipantData.mediaElement = mediaElement; } if (stream && (!callParticipantData.mediaStream || !track)) { @@ -1597,7 +1609,9 @@ class Call { if (this._audioControls) { mediaElement.setAttribute('controls', ""); } - mediaElement.setAttribute('autoplay', ""); + mediaElement.setAttribute('autoplay', true); + mediaElement.setAttribute('muted', true); + mediaElement.muted = true; mediaElement.onpause = () => { mediaElement.play(); }; return mediaElement; @@ -1605,7 +1619,8 @@ class Call { _checkCallParticipantData = (participantId) => { if (!this._callParticipantsData[participantId]) { - this._callParticipantsData[participantId] = new CallParticipantData(participantId); + this._callParticipantsData[participantId] = new CallParticipantData(participantId, this._audioCtx); + this._callParticipantsData[participantId].mute = this._speakerMuted; const member = this.findParticipant(participantId); if (member) { this._callParticipantsData[participantId].mediaContainer = member.mediaContainer; diff --git a/src/call/CallParticipantData.js b/src/call/CallParticipantData.js index 5395cde..fa4c5f6 100644 --- a/src/call/CallParticipantData.js +++ b/src/call/CallParticipantData.js @@ -1,13 +1,14 @@ import { MediaContentType } from "../libs/protobuf/definitions_pb" export default class CallParticipantData { - constructor(participantId, streamId, mediaType = MediaContentType.AUDIO) { + constructor(participantId, audioContext, streamId, mediaType = MediaContentType.AUDIO) { this._participantId = participantId; this._streamMapping = {}; if (streamId) { this._streamMapping[mediaType] = streamId; } + this.setAudioContext(audioContext); this._hark = null; this._mediaStream = null; this._mediaElement = null; @@ -21,6 +22,25 @@ export default class CallParticipantData { this._midScreenShare = null; } + setAudioContext = (audioContext) => { + if (this._audioContext) { + if (this._sourceNode) { + this._sourceNode.disconnect(); + this._sourceNode = null; + } + if (this._gainNode) { + this._gainNode.disconnect(); + this._gainNode = null; + } + } + + this._audioContext = audioContext; + if (audioContext) { + this._gainNode = this._audioContext.createGain(); + this._gainNode.connect(this._audioContext.destination); + } + } + get participantId() { return this._participantId; } @@ -69,6 +89,15 @@ export default class CallParticipantData { } set mediaStream(value) { + if (this._sourceNode) { + this._sourceNode.disconnect(); + this._sourceNode = null; + } + if (this._audioContext && value) { + this._sourceNode = this._audioContext.createMediaStreamSource(value); + this._sourceNode.connect(this._gainNode); + } + this._mediaStream = value; if (this._mediaElement) { @@ -172,6 +201,20 @@ export default class CallParticipantData { this._midScreenShare = midScreenShare; } + set mute(mute) { + if (this._gainNode) { + // this._gainNode.gain.setValueAtTime(mute ? 0 : 1, this._audioContext.currentTime); + this._gainNode.gain.value = mute ? 0 : 1; + } + } + + get mute() { + if (this._gainNode) { + return this._gainNode.gain.value === 0; + } + return false; + } + cleanUp() { if (this._hark) { this._hark.stop(); @@ -205,5 +248,7 @@ export default class CallParticipantData { this._screenShareVideoElement = null; } + + this.setAudioContext(null); } } diff --git a/src/call/P2pCall.js b/src/call/P2pCall.js index 4e8e4ba..522bd9e 100644 --- a/src/call/P2pCall.js +++ b/src/call/P2pCall.js @@ -116,6 +116,9 @@ class P2pCall extends Call { } get hasRemoteVideo() { + if (!this._pc) { + return false; + } const callParticipantData = this._getCallParticipantData(); if (!callParticipantData.midVideo) { @@ -137,7 +140,8 @@ class P2pCall extends Call { let callParticipantsData = this._callParticipantsData[other]; if (!callParticipantsData) { - this._callParticipantsData[other] = new CallParticipantData(other, streamId); + this._callParticipantsData[other] = new CallParticipantData(other, this._audioCtx, streamId); + this._callParticipantsData[other].mute = this._speakerMuted; } if (streamId) { this._callParticipantsData[other].streamId = streamId; @@ -152,7 +156,8 @@ class P2pCall extends Call { let callParticipantsData = this._callParticipantsData[other]; if (!callParticipantsData) { - callParticipantsData = new CallParticipantData(participantId); + callParticipantsData = new CallParticipantData(participantId, this._audioCtx); + this._callParticipantsData[other].mute = this._speakerMuted; this._callParticipantsData[other] = callParticipantsData; } this._callParticipantsData[participantId] = callParticipantsData; -- GitLab