// Copyright Epic Games, Inc. All Rights Reserved. // universal module definition - read https://www.davidbcalhoun.com/2014/what-is-amd-commonjs-and-umd/ (function (root, factory) { if (typeof define === 'function' && define.amd) { // AMD. Register as an anonymous module. define(["./adapter"], factory); } else if (typeof exports === 'object') { // Node. Does not work with strict CommonJS, but // only CommonJS-like environments that support module.exports, // like Node. module.exports = factory(require("./adapter")); } else { // Browser globals (root is window) root.webRtcPlayer = factory(root.adapter); } }(this, function (adapter) { function webRtcPlayer(parOptions) { parOptions = typeof parOptions !== 'undefined' ? parOptions : {}; var self = this; //********************** //Config setup //********************** this.cfg = typeof parOptions.peerConnectionOptions !== 'undefined' ? parOptions.peerConnectionOptions : {}; this.cfg.sdpSemantics = 'unified-plan'; // this.cfg.rtcAudioJitterBufferMaxPackets = 10; // this.cfg.rtcAudioJitterBufferFastAccelerate = true; // this.cfg.rtcAudioJitterBufferMinDelayMs = 0; // If this is true in Chrome 89+ SDP is sent that is incompatible with UE Pixel Streaming 4.26 and below. // However 4.27 Pixel Streaming does not need this set to false as it supports `offerExtmapAllowMixed`. // tdlr; uncomment this line for older versions of Pixel Streaming that need Chrome 89+. this.cfg.offerExtmapAllowMixed = false; //********************** //Variables //********************** this.pcClient = null; this.dcClient = null; this.tnClient = null; this.sdpConstraints = { offerToReceiveAudio: 1, //Note: if you don't need audio you can get improved latency by turning this off. offerToReceiveVideo: 1, voiceActivityDetection: false }; // See https://www.w3.org/TR/webrtc/#dom-rtcdatachannelinit for values (this is needed for Firefox to be consistent with Chrome.) this.dataChannelOptions = {ordered: true}; // This is useful if the video/audio needs to autoplay (without user input) as browsers do not allow autoplay non-muted of sound sources without user interaction. this.startVideoMuted = typeof parOptions.startVideoMuted !== 'undefined' ? parOptions.startVideoMuted : false; this.autoPlayAudio = typeof parOptions.autoPlayAudio !== 'undefined' ? parOptions.autoPlayAudio : true; // To enable mic in browser use SSL/localhost and have ?useMic in the query string. const urlParams = new URLSearchParams(window.location.search); this.useMic = urlParams.has('useMic'); if(!this.useMic) { console.log("Microphone access is not enabled. Pass ?useMic in the url to enable it."); } // When ?useMic check for SSL or localhost let isLocalhostConnection = location.hostname === "localhost" || location.hostname === "127.0.0.1"; let isHttpsConnection = location.protocol === 'https:'; if(this.useMic && !isLocalhostConnection && !isHttpsConnection) { this.useMic = false; console.error("Microphone access in the browser will not work if you are not on HTTPS or localhost. Disabling mic access."); console.error("For testing you can enable HTTP microphone access Chrome by visiting chrome://flags/ and enabling 'unsafely-treat-insecure-origin-as-secure'"); } // Latency tester this.latencyTestTimings = { TestStartTimeMs: null, UEReceiptTimeMs: null, UEPreCaptureTimeMs: null, UEPostCaptureTimeMs: null, UEPreEncodeTimeMs: null, UEPostEncodeTimeMs: null, UETransmissionTimeMs: null, BrowserReceiptTimeMs: null, FrameDisplayDeltaTimeMs: null, Reset: function() { this.TestStartTimeMs = null; this.UEReceiptTimeMs = null; this.UEPreCaptureTimeMs = null; this.UEPostCaptureTimeMs = null; this.UEPreEncodeTimeMs = null; this.UEPostEncodeTimeMs = null; this.UETransmissionTimeMs = null; this.BrowserReceiptTimeMs = null; this.FrameDisplayDeltaTimeMs = null; }, SetUETimings: function(UETimings) { this.UEReceiptTimeMs = UETimings.ReceiptTimeMs; this.UEPreCaptureTimeMs = UETimings.PreCaptureTimeMs; this.UEPostCaptureTimeMs = UETimings.PostCaptureTimeMs; this.UEPreEncodeTimeMs = UETimings.PreEncodeTimeMs; this.UEPostEncodeTimeMs = UETimings.PostEncodeTimeMs; this.UETransmissionTimeMs = UETimings.TransmissionTimeMs; this.BrowserReceiptTimeMs = Date.now(); this.OnAllLatencyTimingsReady(this); }, SetFrameDisplayDeltaTime: function(DeltaTimeMs) { if(this.FrameDisplayDeltaTimeMs == null) { this.FrameDisplayDeltaTimeMs = Math.round(DeltaTimeMs); this.OnAllLatencyTimingsReady(this); } }, OnAllLatencyTimingsReady: function(Timings){} } //********************** //Functions //********************** //Create Video element and expose that as a parameter this.createWebRtcVideo = function() { var video = document.createElement('video'); video.id = "streamingVideo"; video.playsInline = true; video.disablepictureinpicture = true; // video.muted = self.startVideoMuted; video.muted = true; video.addEventListener('loadedmetadata', function(e){ if(self.onVideoInitialised){ self.onVideoInitialised(); } }, true); // Check if request video frame callback is supported if ('requestVideoFrameCallback' in HTMLVideoElement.prototype) { // The API is supported! const onVideoFrameReady = (now, metadata) => { if(metadata.receiveTime && metadata.expectedDisplayTime) { const receiveToCompositeMs = metadata.presentationTime - metadata.receiveTime; self.aggregatedStats.receiveToCompositeMs = receiveToCompositeMs; } // Re-register the callback to be notified about the next frame. video.requestVideoFrameCallback(onVideoFrameReady); }; // Initially register the callback to be notified about the first frame. video.requestVideoFrameCallback(onVideoFrameReady); } return video; } this.video = this.createWebRtcVideo(); onsignalingstatechange = function(state) { console.info('signaling state change:', state) }; oniceconnectionstatechange = function(state) { console.info('ice connection state change:', state) }; onicegatheringstatechange = function(state) { console.info('ice gathering state change:', state) }; handleOnTrack = function(e) { console.log('handleOnTrack', e.streams); if (e.track) { console.log('Got track - ' + e.track.kind + ' id=' + e.track.id + ' readyState=' + e.track.readyState); } if(e.track.kind == "audio") { handleOnAudioTrack(e.streams[0]); return; } else(e.track.kind == "video" && self.video.srcObject !== e.streams[0]) { self.video.srcObject = e.streams[0]; console.log('Set video source from video track ontrack.'); return; } }; handleOnAudioTrack = function(audioMediaStream) { // do nothing the video has the same media stream as the audio track we have here (they are linked) if(self.video.srcObject == audioMediaStream) { return; } // video element has some other media stream that is not associated with this audio track else if(self.video.srcObject && self.video.srcObject !== audioMediaStream) { // create a new audio element let audioElem = document.createElement("Audio"); audioElem.srcObject = audioMediaStream; // there is no way to autoplay audio (even muted), so we defer audio until first click if(!self.autoPlayAudio) { let clickToPlayAudio = function() { audioElem.play(); self.video.removeEventListener("click", clickToPlayAudio); }; self.video.addEventListener("click", clickToPlayAudio); } // we assume the user has clicked somewhere on the page and autoplaying audio will work else { audioElem.play(); } console.log('Created new audio element to play seperate audio stream.'); } } setupDataChannel = function(pc, label, options) { try { let datachannel = pc.createDataChannel(label, options); console.log(`Created datachannel (${label})`) // Inform browser we would like binary data as an ArrayBuffer (FF chooses Blob by default!) datachannel.binaryType = "arraybuffer"; datachannel.onopen = function (e) { console.log(`data channel (${label}) connect`) if(self.onDataChannelConnected){ self.onDataChannelConnected(); } } datachannel.onclose = function (e) { console.log(`data channel (${label}) closed`) } datachannel.onmessage = function (e) { //console.log(`Got message (${label})`, e.data) if (self.onDataChannelMessage) self.onDataChannelMessage(e.data); } return datachannel; } catch (e) { console.warn('No data channel', e); return null; } } onicecandidate = function (e) { console.log('ICE candidate', e) if (e.candidate && e.candidate.candidate) { self.onWebRtcCandidate(e.candidate); } }; handleCreateOffer = function (pc) { pc.createOffer(self.sdpConstraints).then(function (offer) { // Munging is where we modifying the sdp string to set parameters that are not exposed to the browser's WebRTC API mungeSDPOffer(offer); // Set our munged SDP on the local peer connection so it is "set" and will be send across pc.setLocalDescription(offer); if (self.onWebRtcOffer) { self.onWebRtcOffer(offer); } }, function () { console.warn("Couldn't create offer") }); } mungeSDPOffer = function (offer) { // turn off video-timing sdp sent from browser //offer.sdp = offer.sdp.replace("http://www.webrtc.org/experiments/rtp-hdrext/playout-delay", ""); // this indicate we support stereo (Chrome needs this) offer.sdp = offer.sdp.replace('useinbandfec=1', 'useinbandfec=1;stereo=1;sprop-maxcapturerate=48000'); } setupPeerConnection = function (pc) { if (pc.SetBitrate) console.log("Hurray! there's RTCPeerConnection.SetBitrate function"); //Setup peerConnection events pc.onsignalingstatechange = onsignalingstatechange; pc.oniceconnectionstatechange = oniceconnectionstatechange; pc.onicegatheringstatechange = onicegatheringstatechange; pc.ontrack = handleOnTrack; pc.onicecandidate = onicecandidate; }; generateAggregatedStatsFunction = function(){ if(!self.aggregatedStats) self.aggregatedStats = {}; return function(stats){ //console.log('Printing Stats'); let newStat = {}; stats.forEach(stat => { // console.log(JSON.stringify(stat, undefined, 4)); if (stat.type == 'inbound-rtp' && !stat.isRemote && (stat.mediaType == 'video' || stat.id.toLowerCase().includes('video'))) { newStat.timestamp = stat.timestamp; newStat.bytesReceived = stat.bytesReceived; newStat.framesDecoded = stat.framesDecoded; newStat.packetsLost = stat.packetsLost; newStat.bytesReceivedStart = self.aggregatedStats && self.aggregatedStats.bytesReceivedStart ? self.aggregatedStats.bytesReceivedStart : stat.bytesReceived; newStat.framesDecodedStart = self.aggregatedStats && self.aggregatedStats.framesDecodedStart ? self.aggregatedStats.framesDecodedStart : stat.framesDecoded; newStat.timestampStart = self.aggregatedStats && self.aggregatedStats.timestampStart ? self.aggregatedStats.timestampStart : stat.timestamp; if(self.aggregatedStats && self.aggregatedStats.timestamp){ if(self.aggregatedStats.bytesReceived){ // bitrate = bits received since last time / number of ms since last time //This is automatically in kbits (where k=1000) since time is in ms and stat we want is in seconds (so a '* 1000' then a '/ 1000' would negate each other) newStat.bitrate = 8 * (newStat.bytesReceived - self.aggregatedStats.bytesReceived) / (newStat.timestamp - self.aggregatedStats.timestamp); newStat.bitrate = Math.floor(newStat.bitrate); newStat.lowBitrate = self.aggregatedStats.lowBitrate && self.aggregatedStats.lowBitrate < newStat.bitrate ? self.aggregatedStats.lowBitrate : newStat.bitrate newStat.highBitrate = self.aggregatedStats.highBitrate && self.aggregatedStats.highBitrate > newStat.bitrate ? self.aggregatedStats.highBitrate : newStat.bitrate } if(self.aggregatedStats.bytesReceivedStart){ newStat.avgBitrate = 8 * (newStat.bytesReceived - self.aggregatedStats.bytesReceivedStart) / (newStat.timestamp - self.aggregatedStats.timestampStart); newStat.avgBitrate = Math.floor(newStat.avgBitrate); } if(self.aggregatedStats.framesDecoded){ // framerate = frames decoded since last time / number of seconds since last time newStat.framerate = (newStat.framesDecoded - self.aggregatedStats.framesDecoded) / ((newStat.timestamp - self.aggregatedStats.timestamp) / 1000); newStat.framerate = Math.floor(newStat.framerate); newStat.lowFramerate = self.aggregatedStats.lowFramerate && self.aggregatedStats.lowFramerate < newStat.framerate ? self.aggregatedStats.lowFramerate : newStat.framerate newStat.highFramerate = self.aggregatedStats.highFramerate && self.aggregatedStats.highFramerate > newStat.framerate ? self.aggregatedStats.highFramerate : newStat.framerate } if(self.aggregatedStats.framesDecodedStart){ newStat.avgframerate = (newStat.framesDecoded - self.aggregatedStats.framesDecodedStart) / ((newStat.timestamp - self.aggregatedStats.timestampStart) / 1000); newStat.avgframerate = Math.floor(newStat.avgframerate); } } } //Read video track stats if(stat.type == 'track' && (stat.trackIdentifier == 'video_label' || stat.kind == 'video')) { newStat.framesDropped = stat.framesDropped; newStat.framesReceived = stat.framesReceived; newStat.framesDroppedPercentage = stat.framesDropped / stat.framesReceived * 100; newStat.frameHeight = stat.frameHeight; newStat.frameWidth = stat.frameWidth; newStat.frameHeightStart = self.aggregatedStats && self.aggregatedStats.frameHeightStart ? self.aggregatedStats.frameHeightStart : stat.frameHeight; newStat.frameWidthStart = self.aggregatedStats && self.aggregatedStats.frameWidthStart ? self.aggregatedStats.frameWidthStart : stat.frameWidth; } if(stat.type =='candidate-pair' && stat.hasOwnProperty('currentRoundTripTime') && stat.currentRoundTripTime != 0){ newStat.currentRoundTripTime = stat.currentRoundTripTime; } }); if(self.aggregatedStats.receiveToCompositeMs) { newStat.receiveToCompositeMs = self.aggregatedStats.receiveToCompositeMs; self.latencyTestTimings.SetFrameDisplayDeltaTime(self.aggregatedStats.receiveToCompositeMs); } self.aggregatedStats = newStat; if(self.onAggregatedStats) self.onAggregatedStats(newStat) } }; setupTracksToSendAsync = async function(pc){ // Setup a transceiver for getting UE video pc.addTransceiver("video", { direction: "recvonly" }); // Setup a transceiver for sending mic audio to UE and receiving audio from UE if(!self.useMic) { pc.addTransceiver("audio", { direction: "recvonly" }); } else { let audioSendOptions = self.useMic ? { autoGainControl: false, channelCount: 1, echoCancellation: false, latency: 0, noiseSuppression: false, sampleRate: 16000, volume: 1.0 } : false; // Note using mic on android chrome requires SSL or chrome://flags/ "unsafely-treat-insecure-origin-as-secure" const stream = await navigator.mediaDevices.getUserMedia({video: false, audio: audioSendOptions}); if(stream) { for (const track of stream.getTracks()) { if(track.kind && track.kind == "audio") { pc.addTransceiver(track, { direction: "sendrecv" }); } } } else { pc.addTransceiver("audio", { direction: "recvonly" }); } } }; //********************** //Public functions //********************** this.setVideoEnabled = function(enabled) { self.video.srcObject.getTracks().forEach(track => track.enabled = enabled); } this.startLatencyTest = function(onTestStarted) { // Can't start latency test without a video element if(!self.video) { return; } self.latencyTestTimings.Reset(); self.latencyTestTimings.TestStartTimeMs = Date.now(); onTestStarted(self.latencyTestTimings.TestStartTimeMs); } //This is called when revceiving new ice candidates individually instead of part of the offer //This is currently not used but would be called externally from this class this.handleCandidateFromServer = function(iceCandidate) { console.log("ICE candidate: ", iceCandidate); let candidate = new RTCIceCandidate(iceCandidate); self.pcClient.addIceCandidate(candidate).then(_=>{ console.log('ICE candidate successfully added'); }); }; //Called externaly to create an offer for the server this.createOffer = function() { if(self.pcClient){ console.log("Closing existing PeerConnection") self.pcClient.close(); self.pcClient = null; } self.pcClient = new RTCPeerConnection(self.cfg); setupTracksToSendAsync(self.pcClient).finally(function() { setupPeerConnection(self.pcClient); self.dcClient = setupDataChannel(self.pcClient, 'cirrus', self.dataChannelOptions); handleCreateOffer(self.pcClient); }); }; //Called externaly when an answer is received from the server this.receiveAnswer = function(answer) { console.log('Received answer:'); console.log(answer); var answerDesc = new RTCSessionDescription(answer); self.pcClient.setRemoteDescription(answerDesc); let receivers = self.pcClient.getReceivers(); for(let receiver of receivers) { receiver.playoutDelayHint = 0; } }; this.close = function(){ if(self.pcClient){ console.log("Closing existing peerClient") self.pcClient.close(); self.pcClient = null; } if(self.aggregateStatsIntervalId) clearInterval(self.aggregateStatsIntervalId); } //Sends data across the datachannel this.send = function(data){ if(self.dcClient && self.dcClient.readyState == 'open'){ //console.log('Sending data on dataconnection', self.dcClient) self.dcClient.send(data); } }; this.getStats = function(onStats){ if(self.pcClient && onStats){ self.pcClient.getStats(null).then((stats) => { onStats(stats); }); } } this.aggregateStats = function(checkInterval){ let calcAggregatedStats = generateAggregatedStatsFunction(); let printAggregatedStats = () => { self.getStats(calcAggregatedStats); } self.aggregateStatsIntervalId = setInterval(printAggregatedStats, checkInterval); } }; return webRtcPlayer; }));