code changs for multiple protobuf models
[face-privacy-filter.git] / web_demo / face-privacy.js
1 /**
2  face-privacy.js - send frames to an face privacy service
3
4  Videos or camera are displayed locally and frames are periodically sent to GPU image-net classifier service (developed by Zhu Liu) via http post.
5  For webRTC, See: https://gist.github.com/greenido/6238800
6
7  D. Gibbon 6/3/15
8  D. Gibbon 4/19/17 updated to new getUserMedia api, https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia
9  D. Gibbon 8/1/17 adapted for system
10  E. Zavesky 10/19/17 adapted for video+image
11  */
12
13 "use strict";
14
15 /**
16  * main entry point
17  */
18 $(document).ready(function() {
19     var urlDefault = getUrlParameter('url-image');
20     if (!urlDefault)
21         urlDefault = "http://localhost:8884/transform";
22
23         $(document.body).data('hdparams', {     // store global vars in the body element
24                 classificationServer: urlDefault,
25                 protoObj: null,   // to be back-filled after protobuf load {'root':obj, 'methods':{'xx':{'typeIn':x, 'typeOut':y}} }
26                 protoPayloadInput: null,   //payload for encoded message download (if desired)
27                 protoPayloadOutput: null,   //payload for encoded message download (if desired)
28                 protoRes: null,  //TEMPORARY
29                 frameCounter: 0,
30                 frameInterval: 500,             // Milliseconds to sleep between sending frames to reduce server load and reduce results updates
31                 frameTimer: -1,         // frame clock for processing
32                 imageIsWaiting: false,  // blocking to prevent too many queued frames
33                 // Objects from DOM elements
34                 srcImgCanvas: document.getElementById('srcImgCanvas'),  // we have a 'src' source image
35                 destImg: document.getElementById('destImg'),    // we have a 'src' source image
36                 video: document.getElementById('srcVideo'),
37         });
38     $(document.body).data('hdparams')['canvasMaxH'] = $(document.body).data('hdparams')['srcImgCanvas'].height;
39     $(document.body).data('hdparams')['canvasMaxW'] = $(document.body).data('hdparams')['srcImgCanvas'].width;
40
41     $("#protoInput").prop("disabled",true).click(downloadBlobIn);
42     $("#protoOutput").prop("disabled",true).click(downloadBlobOut);
43     $("#resultText").hide();
44
45         //add text input tweak
46         $("#serverUrl").change(function() {
47             $(document.body).data('hdparams')['classificationServer'] = $(this).val();
48         updateLink("serverLink");
49         }).val($(document.body).data('hdparams')['classificationServer'])
50         //set launch link at first
51     updateLink("serverLink");
52
53         // add buttons to change video
54         $("#sourceRibbon div").click(function() {
55             var $this = $(this);
56             $this.siblings().removeClass('selected'); //clear other selection
57             $this.addClass('selected');
58             var objImg = $this.children('img')[0];
59             var hd = $(document.body).data('hdparams');
60             if (objImg) {
61                 switchImage(objImg.src);
62             clearInterval(hd.frameTimer);       // stop the processing
63
64             var movieAttr = $(objImg).attr('movie');
65             if (movieAttr) {
66                 // Set the video source based on URL specified in the 'videos' list, or select camera input
67                 $(hd.video).show();
68                 $(srcImgCanvas).hide();
69                 if (movieAttr == "Camera") {
70                     var constraints = {audio: false, video: true};
71                     navigator.mediaDevices.getUserMedia(constraints)
72                         .then(function(mediaStream) {
73                             hd.video.srcObject = mediaStream;
74                             hd.video.play();
75                         })
76                         .catch(function(err) {
77                             console.log(err.name + ": " + err.message);
78                         });
79                 } else {
80                     var mp4 = document.getElementById("mp4");
81                     mp4.setAttribute("src", movieAttr);
82                     hd.video.load();
83                     newVideo();
84                 }
85             }
86             else {
87                 hd.video.pause();
88                 $(hd.video).hide();
89                 $(srcImgCanvas).show();
90             }
91             }
92         });
93
94         //allow user-uploaded images
95     var imageLoader = document.getElementById('imageLoader');
96     imageLoader.addEventListener('change', handleImage, false);
97
98     //if protobuf is enabled, fire load event for it as well
99     $(document.body).data('hdparams').protoObj = {};  //clear from last load
100     protobuf_load("model.pixelate.proto", true);
101     protobuf_load("model.detect.proto");
102
103     //trigger first click
104     $("#sourceRibbon div")[0].click();
105 });
106
107
108 function protobuf_load(pathProto, forceSelect) {
109     protobuf.load(pathProto, function(err, root) {
110         if (err) {
111             console.log("[protobuf]: Error!: "+err);
112             throw err;
113         }
114         var domSelect = $("#protoMethod");
115         var numMethods = domSelect.children().length;
116         $.each(root.nested, function(namePackage, objPackage) {    // walk all
117             if ('Model' in objPackage && 'methods' in objPackage.Model) {    // walk to model and functions...
118                 var typeSummary = {'root':root, 'methods':{} };
119                 $.each(objPackage.Model.methods, function(nameMethod, objMethod) {  // walk methods
120                     typeSummary['methods'][nameMethod] = {};
121                     typeSummary['methods'][nameMethod]['typeIn'] = namePackage+'.'+objMethod.requestType;
122                     typeSummary['methods'][nameMethod]['typeOut'] = namePackage+'.'+objMethod.responseType;
123                     typeSummary['methods'][nameMethod]['service'] = namePackage+'.'+nameMethod;
124
125                     //create HTML object as well
126                     var namePretty = namePackage+"."+nameMethod;
127                     var domOpt = $("<option />").attr("value", namePretty).text(
128                         nameMethod+ " (input: "+objMethod.requestType
129                         +", output: "+objMethod.responseType+")");
130                     if (numMethods==0) {    // first method discovery
131                         domSelect.append($("<option />").attr("value","").text("(disabled, not loaded)")); //add 'disabled'
132                     }
133                     if (forceSelect) {
134                         domOpt.attr("selected", 1);
135                     }
136                     domSelect.append(domOpt);
137                     numMethods++;
138                 });
139                 $(document.body).data('hdparams').protoObj[namePackage] = typeSummary;   //save new method set
140                 $("#protoContainer").show();
141             }
142         });
143         console.log("[protobuf]: Load successful, found "+numMethods+" model methods.");
144     });
145 }
146
147 /**
148  * Called after a new video has loaded (at least the video metadata has loaded)
149  */
150 function newVideo() {
151         var hd = $(document.body).data('hdparams');
152         hd.frameCounter = 0;
153         hd.imageIsWaiting = false;
154         hd.video.play();
155
156         // set processing canvas size based on source video
157         var pwidth = hd.video.videoWidth;
158         var pheight = hd.video.videoHeight;
159         if (pwidth > hd.maxSrcVideoWidth) {
160                 pwidth = hd.maxSrcVideoWidth;
161                 pheight = Math.floor((pwidth / hd.video.videoWidth) * pheight); // preserve aspect ratio
162         }
163         hd.srcImgCanvas.width = pwidth;
164         hd.srcImgCanvas.height = pheight;
165
166         hd.frameTimer = setInterval(nextFrame, hd.frameInterval); // start the processing
167 }
168
169 /**
170  * process the next video frame
171  */
172 function nextFrame() {
173         var hd = $(document.body).data('hdparams');
174         if (hd.video.ended || hd.video.paused) {
175                 return;
176         }
177     switchImage(hd.video, true);
178 }
179
180 function updateLink(domId, newServer) {
181     var sPageURL = decodeURIComponent(window.location.search.split('?')[0]);
182     if (newServer==undefined) {
183         newServer = $(document.body).data('hdparams')['classificationServer'];
184     }
185     else {
186         $("#serverUrl").val(newServer);
187     }
188     var sNewUrl = sPageURL+"?url-image="+newServer;
189     $("#"+domId).attr('href', sNewUrl);
190 }
191
192 function switchImage(imgSrc, isVideo) {
193     var canvas = $(document.body).data('hdparams')['srcImgCanvas'];
194     if (!isVideo) {
195         var img = new Image();
196         img.onload = function () {
197             var ctx = canvas.getContext('2d');
198             var canvasCopy = document.createElement("canvas");
199             var copyContext = canvasCopy.getContext("2d");
200
201             var ratio = 1;
202
203             //console.log( $(document.body).data('hdparams'));
204             //console.log( [ img.width, img.height]);
205             // https://stackoverflow.com/a/2412606
206             if(img.width > $(document.body).data('hdparams')['canvasMaxW'])
207                 ratio = $(document.body).data('hdparams')['canvasMaxW'] / img.width;
208             if(ratio*img.height > $(document.body).data('hdparams')['canvasMaxH'])
209                 ratio = $(document.body).data('hdparams')['canvasMaxH'] / img.height;
210
211             canvasCopy.width = img.width;
212             canvasCopy.height = img.height;
213             copyContext.drawImage(img, 0, 0);
214
215             canvas.width = img.width * ratio;
216             canvas.height = img.height * ratio;
217             ctx.drawImage(canvasCopy, 0, 0, canvasCopy.width, canvasCopy.height, 0, 0, canvas.width, canvas.height);
218             //document.removeChild(canvasCopy);
219             doPostImage(canvas, '#destImg', canvas.toDataURL());
220         }
221         img.src = imgSrc;  //copy source, let image load
222     }
223     else if (!$(document.body).data('hdparams').imageIsWaiting) {
224         var ctx = canvas.getContext('2d');
225         var canvasCopy = document.createElement("canvas");
226         var copyContext = canvasCopy.getContext("2d");
227         var ratio = 1;
228
229         if(imgSrc.videoWidth > $(document.body).data('hdparams')['canvasMaxW'])
230             ratio = $(document.body).data('hdparams')['canvasMaxW'] / imgSrc.videoWidth;
231         if(ratio*imgSrc.videoHeight > $(document.body).data('hdparams')['canvasMaxH'])
232             ratio = $(document.body).data('hdparams')['canvasMaxH'] / canvasCopy.height;
233
234         //console.log("Canvas Copy:"+canvasCopy.width+"/"+canvasCopy.height);
235         //console.log("Canvas Ratio:"+ratio);
236         //console.log("Video: "+imgSrc.videoWidth+"x"+imgSrc.videoHeight);
237         canvasCopy.width = imgSrc.videoWidth;     //large as possible
238         canvasCopy.height = imgSrc.videoHeight;
239         copyContext.drawImage(imgSrc, 0, 0);
240
241         canvas.width = canvasCopy.width * ratio;
242         canvas.height = canvasCopy.height * ratio;
243         ctx.drawImage(canvasCopy, 0, 0, canvasCopy.width, canvasCopy.height, 0, 0, canvas.width, canvas.height);
244         //document.removeChild(canvasCopy);
245         doPostImage(canvas, '#destImg', canvas.toDataURL());
246     }
247 }
248
249
250 //load image that has been uploaded into a vancas
251 function handleImage(e){
252     var reader = new FileReader();
253     reader.onload = function(event){
254         switchImage(event.target.result);
255     }
256     reader.readAsDataURL(e.target.files[0]);
257 }
258
259
260
261 // https://stackoverflow.com/questions/19491336/get-url-parameter-jquery-or-how-to-get-query-string-values-in-js
262 function getUrlParameter(sParam) {
263     var sPageURL = decodeURIComponent(window.location.search.substring(1)),
264         sURLVariables = sPageURL.split('&'),
265         sParameterName,
266         i;
267
268     for (i = 0; i < sURLVariables.length; i++) {
269         sParameterName = sURLVariables[i].split('=');
270
271         if (sParameterName[0] === sParam) {
272             return sParameterName[1] === undefined ? true : sParameterName[1];
273         }
274     }
275 };
276
277
278 /**
279  * post an image from the canvas to the service
280  */
281 function doPostImage(srcCanvas, dstImg, dataPlaceholder) {
282     var dataURL = srcCanvas.toDataURL('image/jpeg', 1.0);
283     var hd = $(document.body).data('hdparams');
284     var sendPayload = null;
285
286     var nameProtoMethod = $("#protoMethod option:selected").attr('value');
287     var methodKeys = null;
288     if (nameProtoMethod && nameProtoMethod.length) {     //valid protobuf type?
289         var partsURL = hd.classificationServer.split("/");
290         methodKeys = nameProtoMethod.split(".", 2);       //modified for multiple detect/pixelate models
291         partsURL[partsURL.length-1] = methodKeys[1];
292         hd.classificationServer = partsURL.join("/");   //rejoin with new endpoint
293         updateLink("serverLink", hd.classificationServer);
294     }
295
296     var serviceURL = hd.classificationServer;
297     var request = new XMLHttpRequest();     // create request to manipulate
298     request.open("POST", serviceURL, true);
299
300     //console.log("[doPostImage]: Selected method ... '"+typeInput+"'");
301     if (nameProtoMethod && nameProtoMethod.length) {     //valid protobuf type?
302         var blob = dataURItoBlob(dataURL, true);
303
304         // fields from .proto file at time of writing...
305         //    message FaceImage {
306         //      repeated string mime_type = 1;   -> becomes "mimeType" (NOTE repeated type)
307         //      repeated bytes image_binary = 2; -> becomes "imageBinary"
308         //    }
309
310         //TODO: should we always assume this is input? answer: for now, YES, always image input!
311         var inputPayload = { "mimeType": [blob.type], "imageBinary": [blob.bytes] };
312
313         // ---- method for processing from a type ----
314         var msgInput = hd.protoObj[methodKeys[0]]['root'].lookupType(hd.protoObj[methodKeys[0]]['methods'][methodKeys[1]]['typeIn']);
315         // Verify the payload if necessary (i.e. when possibly incomplete or invalid)
316         var errMsg = msgInput.verify(inputPayload);
317         if (errMsg) {
318             console.log("[doPostImage]: Error during type verify for object input into protobuf method.");
319             throw Error(errMsg);
320         }
321         // Create a new message
322         var msgTransmit = msgInput.create(inputPayload);
323         // Encode a message to an Uint8Array (browser) or Buffer (node)
324         sendPayload = msgInput.encode(msgTransmit).finish();
325
326         // ----------
327
328         /*
329         // ---- method for processing from a service ----
330         var serviceInput = hd.protoObj['root'].lookup(hd.protoObj['methods'][nameProtoMethod]['service']);
331
332         function rpcImpl(method, requestData, callback) {
333             // perform the request using an HTTP request or a WebSocket for example
334             var responseData = ...;
335             // and call the callback with the binary response afterwards:
336             callback(null, responseData);
337         }
338         var serviceCall = serviceInput.create(rpcImpl, false, false); //request dlimited? response delimited?
339
340         serviceCall.sayHello(sendPayload).then(response) {
341             console.log('Greeting:', response.message);
342         });
343         // ---------------------------
344         */
345
346         //downloadBlob(sendPayload, 'protobuf.bin', 'application/octet-stream');
347         // NOTE: TO TEST THIS BINARY BLOB, use some command-line magic like this...
348         //  protoc --decode=mMJuVapnmIbrHlZGKyuuPDXsrkzpGqcr.FaceImage model.proto < protobuf.bin
349         $("#protoInput").prop("disabled",false);
350         hd.protoPayloadInput = sendPayload;
351
352         // append our encoded chunk
353         //console.log(sendPayload);
354         //console.log(typeof(blob.type));
355         // console.log(nameProtoMethod);
356         request.setRequestHeader("Content-type", "text/plain;charset=UTF-8");
357         request.responseType = 'arraybuffer';
358     }
359     else {
360         var blob = dataURItoBlob(dataURL, false);
361         sendPayload = new FormData();
362         sendPayload.append("image_binary", blob);
363         sendPayload.append("mime_type", blob.type);
364     }
365     //$(dstImg).addClaas('workingImage').siblings('.spinner').remove().after($("<span class='spinner'>&nbsp;</span>"));
366     $(document.body).data('hdparams').imageIsWaiting = true;
367     var $dstImg = $(dstImg);
368     if ($dstImg.attr('src')=='') {
369         $dstImg.attr('src', dataPlaceholder);
370         //$(dstImg).addClass('workingImage').attr('src', dataPlaceholder);
371     }
372
373     hd.imageIsWaiting = true;
374     request.onreadystatechange=function() {
375         if (request.readyState==4 && request.status>=200 && request.status<300) {
376             if (methodKeys!=null) {     //valid protobuf type?
377                 //console.log(request);
378                 var bodyEncodedInString = new Uint8Array(request.response);
379                 //console.log(bodyEncodedInString);
380                 //console.log(bodyEncodedInString.length);
381                 $("#protoOutput").prop("disabled",false);
382                 hd.protoPayloadOutput = bodyEncodedInString;
383
384                 // ---- method for processing from a type ----
385                 var msgOutput = hd.protoObj[methodKeys[0]]['root'].lookupType(hd.protoObj[methodKeys[0]]['methods'][methodKeys[1]]['typeOut']);
386                 var objRecv = msgOutput.decode(hd.protoPayloadOutput);
387                 //console.log(objRecv);
388                 hd.protoRes = objRecv;
389
390                 // detect what mode we're in (detect alone or processed?)...
391                 if (!Array.isArray(objRecv.mimeType)) {
392                     $dstImg.attr('src', "data:"+objRecv.mimeType+";base64,"+objRecv.imageBinary).removeClass('workingImage');
393                 }
394                 else {
395                     var domResult = $("#resultText");
396                     var domTable = $("<tr />");
397                     var arrNames = [];
398                     $.each(msgOutput.fields, function(name, val) {           //collect field names
399                         var nameClean = val.name;
400                         if (nameClean != 'imageBinary') {
401                             domTable.append($("<th />").html(nameClean));
402                             arrNames.push(nameClean);
403                         }
404                     });
405                     domTable = $("<table />").append(domTable);     // create embedded table
406
407                     var idxImg = -1;
408                     if ('region' in msgOutput.fields) {             // did we get regions?
409                         for (var i=0; i<objRecv.region.length; i++) {       //find the right region
410                             if (objRecv.region[i]==-1) {                    //special indicator for original image
411                                 idxImg = i;
412                             }
413                             var domRow = $("<tr />");
414                             var strDisplay = [];
415                             $.each(arrNames, function(idx, name) {      //collect data from each column
416                                 domRow.append($("<td />").html(objRecv[name][i]));
417                             });
418                             domTable.append(domRow);
419                             //domResult.append($("div").html(objRecv.region));
420                         }
421                         domResult.empty().append($("<strong />").html("Results")).show();
422                         domResult.append(domTable);
423                     }
424                     else {                                  //got images, get that chunk directly
425                         idxImg = 0;
426                     }
427
428                     if (idxImg != -1) {                     //got any valid image? display it
429                         //console.log(objRecv.mimeType[idxImg]);
430                         //console.log(objRecv.imageBinary[idxImg]);
431                         //var strImage = Uint8ToString(objRecv.imageBinary[idxImg]);
432                         var strImage = btoa(String.fromCharCode.apply(null, objRecv.imageBinary[idxImg]));
433                         $dstImg.attr('src', "data:"+objRecv.mimeType[idxImg]+";base64,"+strImage).removeClass('workingImage');
434                     }
435                 }
436
437             }
438             else {
439                 var responseJson = $.parseJSON(request.responseText);
440                 var respImage = responseJson[0];
441                 // https://stackoverflow.com/questions/21227078/convert-base64-to-image-in-javascript-jquery
442                 $dstImg.attr('src', "data:"+respImage['mime_type']+";base64,"+respImage['image_binary']).removeClass('workingImage');
443                 //genClassTable($.parseJSON(request.responseText), dstDiv);
444             }
445             hd.imageIsWaiting = false;
446         }
447         }
448         request.send(sendPayload);
449         $(document.body).data('hdparams').imageIsWaiting = false;
450 }
451
452
453 /**
454  * convert base64/URLEncoded data component to raw binary data held in a string
455  *
456  * Stoive, http://stackoverflow.com/questions/4998908/convert-data-uri-to-file-then-append-to-formdata
457  */
458 function dataURItoBlob(dataURI, wantBytes) {
459     // convert base64/URLEncoded data component to raw binary data held in a string
460     var byteString;
461     if (dataURI.split(',')[0].indexOf('base64') >= 0)
462         byteString = atob(dataURI.split(',')[1]);
463     else
464         byteString = unescape(dataURI.split(',')[1]);
465
466     // separate out the mime component
467     var mimeString = dataURI.split(',')[0].split(':')[1].split(';')[0];
468
469     // write the bytes of the string to a typed array
470     var ia = new Uint8Array(byteString.length);
471     for (var i = 0; i < byteString.length; i++) {
472         ia[i] = byteString.charCodeAt(i);
473     }
474     //added for returning bytes directly
475     if (wantBytes) {
476         return {'bytes':ia, 'type':mimeString};
477     }
478     return new Blob([ia], {type:mimeString});
479 }
480
481 function Uint8ToString(u8a){
482   var CHUNK_SZ = 0x8000;
483   var c = [];
484   for (var i=0; i < u8a.length; i+=CHUNK_SZ) {
485     c.push(String.fromCharCode.apply(null, u8a.subarray(i, i+CHUNK_SZ)));
486   }
487   return c.join("");
488 }
489
490
491 // ----- diagnostic tool to download binary blobs ----
492 function downloadBlobOut() {
493     return downloadBlob($(document.body).data('hdparams').protoPayloadOutput, "protobuf.out.bin");
494 }
495
496 function downloadBlobIn() {
497     return downloadBlob($(document.body).data('hdparams').protoPayloadInput, "protobuf.in.bin");
498 }
499
500 //  https://stackoverflow.com/a/33622881
501 function downloadBlob(data, fileName, mimeType) {
502   //if there is no data, filename, or mime provided, make our own
503   if (!data)
504       data = $(document.body).data('hdparams').protoPayloadInput;
505   if (!fileName)
506       fileName = "protobuf.bin";
507   if (!mimeType)
508       mimeType = "application/octet-stream";
509
510   var blob, url;
511   blob = new Blob([data], {
512     type: mimeType
513   });
514   url = window.URL.createObjectURL(blob);
515   downloadURL(url, fileName, mimeType);
516   setTimeout(function() {
517     return window.URL.revokeObjectURL(url);
518   }, 1000);
519 };
520
521 function downloadURL(data, fileName) {
522   var a;
523   a = document.createElement('a');
524   a.href = data;
525   a.download = fileName;
526   document.body.appendChild(a);
527   a.style = 'display: none';
528   a.click();
529   a.remove();
530 };