This article extends the Jocly WebRTC announcement article with technical details.
We cannot share the whole Jocly code, but here are the parts we use to make WebRTC work for connected users playing a live game, in the hope it can help developers setting up similar services.
This code works fine (with the exception of the latest Firefox nightly currently not being able to copy the remote video stream to a 3D mesh), but very certainly contains bugs. We appreciate you feedback in the comments or through any other way you find suitable.
The Firefox/Chrome interoperable WebRTC code is adapted from the apprtc application from http://code.google.com/p/webrtc-samples/source/browse/trunk/apprtc/
In the case of Jocly, we already have the two players in relation as they already agreed to play a game against each other, so there is no formal call establishment. Instead, if we detect that both players have WebRTC capabilities, the program requests each side to start the local media. When both sides have authorized their camera, we automatically start the WebRTC call, choosing arbitrarily which one is the calling party, which doesn’t make any difference from the user point of view since the callee will answer automatically.
The main client code is here:
JoclyHub.webrtc = {
RTCPeerConnection : null,
getUserMedia : null,
attachMediaStream : null,
reattachMediaStream : null,
webrtcDetectedBrowser : null,
detachMediaStream : null,
localStream : null,
remoteStream : null,
}
if (navigator.mozGetUserMedia) {
$.extend(JoclyHub.webrtc, {
webrtcDetectedBrowser : "firefox",
pc_config : {
"iceServers" : [ {
"url" : "stun:23.21.150.121"
} ]
},
pc_constraint : {
'mandatory' : {},
'optional' : [ {
'DtlsSrtpKeyAgreement' : 'true'
} ]
},
offer_constraints : {
'mandatory' : {
'MozDontOfferDataChannel' : true
},
'optional' : []
},
RTCPeerConnection : mozRTCPeerConnection,
RTCSessionDescription : mozRTCSessionDescription,
RTCIceCandidate : mozRTCIceCandidate,
getUserMedia : navigator.mozGetUserMedia.bind(navigator),
attachMediaStream : function(element, stream) {
if ($(element).attr("webrtc-attached") == "1")
console.log("Element already attached");
$(element).attr("webrtc-attached", "1");
element.mozSrcObject = stream;
element.play();
},
reattachMediaStream : function(to, from) {
to.mozSrcObject = from.mozSrcObject;
to.play();
},
detachMediaStream : function(element) {
$(element).attr("webrtc-attached", "0");
element.mozSrcObject = null;
element.pause();
},
});
MediaStream.prototype.getVideoTracks = function() {
return [];
};
MediaStream.prototype.getAudioTracks = function() {
return [];
};
} else if (navigator.webkitGetUserMedia) {
$.extend(JoclyHub.webrtc, {
webrtcDetectedBrowser : "chrome",
pc_config : {
"iceServers" : [ {
"url" : "stun:stun.l.google.com:19302"
} ]
},
pc_constraints : {
'mandatory' : {},
'optional' : [ {
'DtlsSrtpKeyAgreement' : 'true'
} ]
},
offer_constraints : {
'mandatory' : {},
'optional' : []
},
RTCPeerConnection : webkitRTCPeerConnection,
RTCSessionDescription : RTCSessionDescription,
RTCIceCandidate : RTCIceCandidate,
getUserMedia : navigator.webkitGetUserMedia.bind(navigator),
attachMediaStream : function(element, stream) {
element.src = webkitURL.createObjectURL(stream);
element.play();
$(element).attr("webrtc-attached", "1");
},
reattachMediaStream : function(to, from) {
to.src = from.src;
},
detachMediaStream : function(element) {
$(element).attr("webrtc-attached", "0");
element.src = '';
element.pause();
},
});
if (!webkitMediaStream.prototype.getVideoTracks) {
webkitMediaStream.prototype.getVideoTracks = function() {
return this.videoTracks;
};
webkitMediaStream.prototype.getAudioTracks = function() {
return this.audioTracks;
};
}
if (!webkitRTCPeerConnection.prototype.getLocalStreams) {
webkitRTCPeerConnection.prototype.getLocalStreams = function() {
return this.localStreams;
};
webkitRTCPeerConnection.prototype.getRemoteStreams = function() {
return this.remoteStreams;
};
}
} else {
console.log("Browser does not appear to be WebRTC-capable");
}
$.extend(JoclyHub.webrtc,
{
media_constraints : {
'optional' : [],
'mandatory' : {}
},
init : function(stream) {
console.log("WebRTC.Init");
var $this = this;
this.localStream = stream;
this.audioMute(stream,false);
var pc_config = $.extend(true,
JoclyHub.webrtc.pc_config, {
optional : [ {
'DataChannels' : true
} ],
});
try {
this.peerConn = new this.RTCPeerConnection(
pc_config, JoclyHub.webrtc.pc_constraints);
} catch (e) {
console.log("Failed to create PeerConnection, exception: " + e.message);
}
this.peerConn.onicecandidate = function(event) {
if (event.candidate) {
//console.log("Sending ICE candidate...", Date.now(), event.candidate);
JoclyHub.frameData.liveSend({
type : "event",
event : "E_WEBRTC",
webrtc : {
type : "signaling",
message : {
type : 'candidate',
label : event.candidate.sdpMLineIndex,
id : event.candidate.sdpMid,
candidate : event.candidate.candidate
},
},
});
} else {
//console.log("End of candidates.");
}
}
this.peerConn.onaddstream = function(event) {
console.log("addstream");
$this.audioMute(event.stream,false);
$this.remoteStream = event.stream;
$(document).trigger("joclyhub.webrtc", {
webrtcType : "mediaOn",
stream : event.stream,
side : $this.peerSide
});
}
this.peerConn.onremovestream = function(event) {
console.log("removestream");
$(document).trigger("joclyhub.webrtc", {
webrtcType : "mediaOff",
side : $this.peerSide
});
$this.remoteStream = null;
}
this.peerConn.addStream(stream);
console.log("inited");
},
startLocal : function() {
var $this = this;
var constraints = $.extend(true, {
mandatory : {
minWidth : 160,
minHeight : 120,
},
}, this.media_constraints);
this.getUserMedia({
'audio' : true,
'video' : constraints,
}, function(stream) {
$(document).trigger("joclyhub.webrtc", {
webrtcType : "mediaOn",
stream : stream,
side : $this.selfSide
})
$(document).trigger("joclyhub.webrtc", {
webrtcType : "localMediaOn",
stream : stream
})
}, function(error) {
console.log("UserMedia failure", error);
});
},
preferOpus : function(sdp) {
var sdpLines = sdp.split('\r\n');
for ( var i = 0; i < sdpLines.length; i++) {
if (sdpLines[i].search('m=audio') !== -1) {
var mLineIndex = i;
break;
}
}
if (mLineIndex === null)
return sdp;
for ( var i = 0; i < sdpLines.length; i++) {
if (sdpLines[i].search('opus/48000') !== -1) {
var opusPayload = this.extractSdp(sdpLines[i],
/:(\d+) opus\/48000/i);
if (opusPayload)
sdpLines[mLineIndex] = this
.setDefaultCodec(
sdpLines[mLineIndex],
opusPayload);
break;
}
}
sdpLines = this.removeCN(sdpLines, mLineIndex);
sdp = sdpLines.join('\r\n');
return sdp;
},
extractSdp : function(sdpLine, pattern) {
var result = sdpLine.match(pattern);
return (result && result.length == 2) ? result[1]
: null;
},
setDefaultCodec : function(mLine, payload) {
var elements = mLine.split(' ');
var newLine = new Array();
var index = 0;
for ( var i = 0; i < elements.length; i++) {
if (index === 3) // Format of media starts from
// the fourth.
newLine[index++] = payload; // Put target
// payload to the
// first.
if (elements[i] !== payload)
newLine[index++] = elements[i];
}
return newLine.join(' ');
},
removeCN : function(sdpLines, mLineIndex) {
var mLineElements = sdpLines[mLineIndex].split(' ');
for ( var i = sdpLines.length - 1; i >= 0; i--) {
var payload = this.extractSdp(sdpLines[i],
/a=rtpmap:(\d+) CN\/\d+/i);
if (payload) {
var cnPos = mLineElements.indexOf(payload);
if (cnPos !== -1) {
mLineElements.splice(cnPos, 1);
}
sdpLines.splice(i, 1);
}
}
sdpLines[mLineIndex] = mLineElements.join(' ');
return sdpLines;
},
handleMessage : function(event, data) {
try {
// console.log("WebRTC: received
// joclyhub.webrtc",data.webrtcType);
switch (data.webrtcType) {
case "peerDetails":
console.log("peerDetails", data);
JoclyHub.webrtc.peerPlayer = data.peer;
JoclyHub.webrtc.selfSide = data.playingAs == 'a' ? 1
: -1;
JoclyHub.webrtc.peerSide = data.playingAs == 'a' ? -1
: 1;
JoclyHub.frameData.liveSend({
type : "event",
event : "E_WEBRTC",
webrtc : {
type : "mediaConfig",
mediaConfig : "reciprocal",
},
});
break;
case "startLocalMedia":
this.startLocal();
break;
case "mediaOn":
console.log("webrtc.handleMessage: received mediaOn",data);
break;
case "localMediaOn":
console.log("webrtc.handleMessage: received localMediaOn",data);
JoclyHub.webrtc.init(data.stream);
JoclyHub.frameData.liveSend({
type : "event",
event : "E_WEBRTC",
webrtc : {
type : "mediaReady",
},
});
break;
case "initiateCall":
console.log("webrtc.handleMessage: received initiateCall",data);
JoclyHub.webrtc.peerConn.createOffer(
function(offer) {
offer.sdp = JoclyHub.webrtc
.preferOpus(offer.sdp);
JoclyHub.webrtc.peerConn
.setLocalDescription(offer);
JoclyHub.frameData
.liveSend({
type : "event",
event : "E_WEBRTC",
webrtc : {
type : "signaling",
message : offer,
},
});
},
function(error) {
console.log("Could not create SDP offer! Reason: "+ error);
},
$.extend(true,
{
'mandatory' : {
'OfferToReceiveAudio' : true,
'OfferToReceiveVideo' : true,
},
},
JoclyHub.webrtc.offer_constraints)
);
break;
case "expectCall":
console.log("webrtc.handleMessage: received expectCall",data);
break;
case "signaling":
var msg = data.message;
//console.log("webrtc.handleMessage: received signaling",msg);
if (msg.type == "offer") {
JoclyHub.webrtc.peerConn.setRemoteDescription(new JoclyHub.webrtc.RTCSessionDescription(msg));
JoclyHub.webrtc.peerConn.createAnswer(
function(answer) {
answer.sdp = JoclyHub.webrtc
.preferOpus(answer.sdp);
JoclyHub.webrtc.peerConn
.setLocalDescription(answer);
JoclyHub.frameData
.liveSend({
type : "event",
event : "E_WEBRTC",
webrtc : {
type : "signaling",
message : answer,
},
});
},
function(error) {
console
.log("Could not create SDP answer! Reason: "
+ error);
},
{
'mandatory' : {
'OfferToReceiveAudio' : true,
'OfferToReceiveVideo' : true,
}
}
);
} else if (msg.type == "answer") {
JoclyHub.webrtc.peerConn.setRemoteDescription(new JoclyHub.webrtc.RTCSessionDescription(msg));
} else if (msg.type == 'candidate') {
var candidate = new JoclyHub.webrtc.RTCIceCandidate(
{
sdpMLineIndex : msg.label,
candidate : msg.candidate,
});
//console.log("iceCandidate added to peerConnection",Date.now());
JoclyHub.webrtc.peerConn.addIceCandidate(
candidate,
function(error) {
if(error)
console.log("addIceCandidate error",error);
});
}
break;
default:
console.log("webrtc.handleMessage: ignored message",data);
}
} catch (e) {
console.log("joclyhub.webrtc handleMessage Error",e);
}
},
audioMute: function(stream,muted) {
var audioTracks=stream.getAudioTracks();
for(var i=0;i<audioTracks.length;i++)
audioTracks[i].enabled = !muted;
},
});
$(document).bind("joclyhub.webrtc", function(event, data) {
JoclyHub.webrtc.handleMessage.call(JoclyHub.webrtc, event, data);
});
$(window).bind("unload",function() {
JoclyHub.frameData
.liveSend({
type : "event",
event : "E_WEBRTC",
webrtc : {
type : "signaling",
message : {
type: "bye",
},
},
});
$(document).trigger("joclyhub.webrtc", {
webrtcType : "localMediaOff",
stream : null
});
$(document).trigger("joclyhub.webrtc", {
webrtcType : "mediaOff",
side : null,
stream : null
});
if (JoclyHub.webrtc.localStream)
JoclyHub.webrtc.localStream.stop();
JoclyHub.webrtc.localStream = null;
if (JoclyHub.webrtc.remoteStream)
JoclyHub.webrtc.remoteStream.stop();
JoclyHub.webrtc.remoteStream = null;
});
Notes for code above:
-
the whole Jocly client code uses jQuery, hence the use of $.extend as a convenience
-
JoclyHub.frameData.liveSend() is being used to send data to the node.js server through web sockets
-
incoming web sockets messages from the node.js server show up as joclyhub.webrtc events
-
this while code is only loaded and executed if we already checked the browser was WebRTC capable
The original apprtc code uses python running on Google appengine server. In Jocly, we already have a node.js server in place for the playing interactions, so we use this server to carry on the signaling between the two parties. The following code portion handles the WebRTC-specific messages.
JG.prototype.$handleWebRTC = function(args) {
console.log("handleWebRTC", args.webrtc);
function FakeCrypto(msg) {
if (msg.sdp && !/a=crypto/.test(msg.sdp)) {
msg.sdp = msg.sdp
.replace(
/c=IN/g,
"a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:BAADBAADBAADBAADBAADBAADBAADBAADBAADBAAD\r\nc=IN");
}
}
switch (args.webrtc.type) {
case "mediaConfig":
if (args.player == 1)
this.webrtc.player1.mediaConfig = args.webrtc.mediaConfig;
else if (args.player == 2)
this.webrtc.player2.mediaConfig = args.webrtc.mediaConfig;
if (this.webrtc.player1.mediaConfig !== null
&& this.webrtc.player2.mediaConfig !== null) {
if (this.webrtc.player1.mediaConfig == "yes"
|| (this.webrtc.player1.mediaConfig == "reciprocal" && (this.webrtc.player2.mediaConfig == "yes" || this.webrtc.player2.mediaConfig == "reciprocal")))
this.player1.send({
type : 'webrtc',
webrtcType : 'startLocalMedia',
});
if (this.webrtc.player2.mediaConfig == "yes"
|| (this.webrtc.player2.mediaConfig == "reciprocal" && (this.webrtc.player1.mediaConfig == "yes" || this.webrtc.player1.mediaConfig == "reciprocal")))
this.player2.send({
type : 'webrtc',
webrtcType : 'startLocalMedia',
});
}
break;
case "mediaReady":
if (args.player == 1)
this.webrtc.player1.media = true;
else if (args.player == 2)
this.webrtc.player2.media = true;
console.log("webrtc", this.webrtc);
if (this.webrtc.player1.media == true
&& this.webrtc.player2.media == true) {
console.log("ready to connect");
this.player1.send({
type : 'webrtc',
webrtcType : 'expectCall',
opponent : this.player2.identity.uid,
});
this.player2.send({
type : 'webrtc',
webrtcType : 'initiateCall',
opponent : this.player1.identity.uid,
});
console.log("initiateCall sent");
}
break;
case "signaling":
var msg = args.webrtc.message;
FakeCrypto(msg);
var destPlayer = null;
if (args.player == 1)
destPlayer = this.player2;
else if (args.player == 2)
destPlayer = this.player1;
if (destPlayer)
destPlayer.send({
type : 'webrtc',
webrtcType : 'signaling',
message : msg,
});
else
console.error("No destination player in message", args);
break;
}
}
In order to determine if a user has WebRTC capabilities in his browser, we use this function:
function CanWebRTC() {
if (navigator.mozGetUserMedia) {
var m = /Firefox\/([0-9]+(\.[0-9]+)?)/.exec(navigator.userAgent);
if (m) {
var version = parseInt(m[1]);
if (!isNaN(version) && version >= 22)
return true;
}
} else if (navigator.webkitGetUserMedia) {
var m = /Chrom(?:[^\/]*)\/([0-9]+)\.([0-9]+)\.([0-9]+)/
.exec(navigator.userAgent);
if (m) {
var v1 = parseInt(m[1]);
var v2 = parseInt(m[2]);
var v3 = parseInt(m[3]);
if (!isNaN(v1) && !isNaN(v2) && !isNaN(v3) && v1 >= 25
&& (v2 > 0 || v3 >= 1364))
return true;
}
}
return false;
}
When using a WebGL 3D theme to display the game scene, we integrate the video streams from the following code, adapted from https://github.com/stemkoski/stemkoski.github.com/blob/master/Three.js/Webcam-Texture.html
var Gadget3DVideo = GadgetMesh.extend({
init : function(gadget, options) {
options = $.extend(true, {
scale : [ 1, 1, 1 ],
playerSide : 1,
makeMesh : function(videoTexture) {
var material = new THREE.MeshBasicMaterial({
map : videoTexture,
overdraw : true,
// side:THREE.DoubleSide
});
var geometry = new THREE.PlaneGeometry(12, 9, 1, 1);
var mesh = new THREE.Mesh(geometry, material);
return mesh;
},
videoPlaying : function(on) {
},
}, options);
this._super.call(this, gadget, options);
this.videoConnected = false;
this.videoErrorCount = 0;
this.videoSkipError = false;
this.shouldBeVisible = false;
},
createObject : function() {
var streamReady=Gadget3DVideo.addAvatar(this,this.options.playerSide);
var mesh = this.options.makeMesh.call(this, Gadget3DVideo.getVideoTexture(this.options.playerSide));
if (mesh) {
mesh.visible = false;
this.objectReady(mesh);
this.streamReady(streamReady);
}
},
remove : function() {
//console.log("remove", this.gadget.id);
Gadget3DVideo.removeAvatar(this,this.options.playerSide);
this._super.apply(this, arguments);
},
show : function() {
this.shouldBeVisible = true;
if (this.object3d && this.videoConnected)
this.object3d.visible = true;
},
hide : function() {
this.shouldBeVisible = false;
if (this.object3d)
this.object3d.visible = false;
},
streamReady : function(on) {
//console.log("streamReady",this.options.playerSide,this.gadget.id,on);
this.videoConnected = on;
if(this.shouldBeVisible && this.object3d)
this.object3d.visible = on;
},
GetResource : GetResource,
});
Gadget3DVideo.streams={}
Gadget3DVideo.renderLoopHooked=false;
Gadget3DVideo.getStream=function(playerSide) {
if(!this.streams[playerSide]) {
var vStream={
stream: null,
avatars: [],
video: null,
videoImage: null,
videoContext: null,
videoTexture: null,
streamReady: false,
ownVideoElement: false,
mustAttachVideo: false,
mustDetachVideo: false,
errorCount: 0,
loopCount: 0,
}
var video = $("video[joclyhub-video='" + playerSide + "']");
if (video.length > 0) {
vStream.video = video;
} else {
//console.log("creating own video element");
vStream.ownVideoElement = true;
vStream.mustAttachVideo = true;
vStream.mustDetachVideo = true;
vStream.video = $("<video/>").attr("autoplay", "autoplay").width(
160).height(120).css({
visibility : "hidden",
float : "left",
}).appendTo("body");
}
vStream.videoImage = $("<canvas/>").attr("width", 160).attr("height",
120).width(160).height(120)
.css({
visibility : "hidden",
float : "left",
}).appendTo("body");
vStream.videoImageContext = vStream.videoImage[0].getContext('2d');
vStream.videoTexture = new THREE.Texture(vStream.videoImage[0]);
vStream.videoTexture.minFilter = THREE.LinearFilter;
vStream.videoTexture.magFilter = THREE.LinearFilter;
this.streams[playerSide]=vStream;
}
return this.streams[playerSide];
}
Gadget3DVideo.addStream=function(playerSide,stream) {
var $this=this;
var vStream=this.getStream(playerSide);
vStream.stream = stream;
if(vStream.mustAttachVideo) {
vStream.mustAttachVideo=false;
//console.warn("attaching stream");
JoclyHub.webrtc.attachMediaStream(vStream.video,stream);
}
if(!this.renderLoopHooked) {
//console.log("start rendering",playerSide);
this.renderLoopHooked=true;
threeCtx.animateCallbacks["Gadget3DVideo"] = {
_this : $this,
callback : $this.animate,
}
}
}
Gadget3DVideo.removeStream=function(playerSide) {
var vStream=this.streams[playerSide];
if(vStream) {
if(vStream.streamReady)
for(var i=0;i<vStream.avatars.length;i++)
vStream.avatars[i].streamReady(false);
if(vStream.mustDetachVideo)
JoclyHub.webrtc.detachMediaStream(vStream.video);
vStream.videoImage.remove();
if(vStream.ownVideoElement)
vStream.video.remove();
delete this.streams[playerSide];
if(this.renderLoopHooked) {
var streamCount=0;
for(var s in this.streams)
streamCount++;
if(streamCount==0) {
delete threeCtx.animateCallbacks["Gadget3DVideo"];
this.renderLoopHooked=false;
}
}
}
}
Gadget3DVideo.addAvatar=function(avatar,playerSide) {
var vStream=this.getStream(playerSide);
vStream.avatars.push(avatar);
//console.log("addAvatar",playerSide,vStream.streamReady)
return vStream.streamReady;
}
Gadget3DVideo.getVideoTexture=function(playerSide) {
return this.getStream(playerSide).videoTexture;
}
Gadget3DVideo.removeAvatar=function(avatar,playerSide) {
var vStream=this.streams[playerSide];
if(vStream)
for(var i=0;i<vStream.avatars.length;i++)
if(avatar==vStream.avatars[i]) {
vStream.avatars.splice(i,1);
break;
}
}
Gadget3DVideo.animate=function() {
for(var side in this.streams) {
var vStream=this.streams[side];
try {
if(vStream.loopCount % 100 ==0)
console.error("Gadget3DVideo.animate loop",vStream.loopCount,side,vStream.video[0].readyState === vStream.video[0].HAVE_ENOUGH_DATA,
vStream.video[0].getAttribute("joclyhub-video"));
vStream.loopCount++;
if (vStream.video[0].getAttribute("webrtc-attached")==="1" && vStream.video[0].readyState === vStream.video[0].HAVE_ENOUGH_DATA) {
vStream.videoImageContext.drawImage(vStream.video[0], 0, 0,
vStream.videoImage[0].width, vStream.videoImage[0].height);
vStream.videoTexture.needsUpdate = true;
if(!vStream.streamReady) {
//console.log("start copying",side);
vStream.streamReady = true;
for(var i=0;i<vStream.avatars.length;i++)
vStream.avatars[i].streamReady(true);
}
}
} catch(e) {
if(vStream.errorCount % 20 ==0)
console.error("Gadget3DVideo.animate error",vStream.errorCount,side,e);
vStream.errorCount++;
}
}
}
$(document).bind("joclyhub.webrtc",
function(event, data) {
try {
if (data.webrtcType == "mediaOn")
Gadget3DVideo.addStream(data.side,data.stream);
if (data.webrtcType == "mediaOff")
Gadget3DVideo.removeStream(data.side);
} catch(e) {
console.error("xd-view joclyhub.webrtc error",e);
}
});
Note for the code above:
-
this code is integrated into the Jocly 2D/3D abstraction mechanism and won't work "as is"
-
the animate member function is being called from the main rendering loop
If you are interested in Jocly code, you can have a look at
jjBlocks, the library we developed and use for dynamic layouts, and the
Jocly dev API to develop or modify your own games.
