Video stream to canvas to image using JavaScript for Project Oxford

Question

I've been able to use Project Oxford, specifically the Emotions API by manually asking a user to upload pictures. However, I'm now looking to do this using a video stream (the webpage shows a live video stream from their webcam and the user can take a snapshot of the stream and then choose to have the emotions API run on that image, which will then run the API and show the scores in a dialogue window - this is the part which I am currently stuck with).

Below I've written the HTML and JavaScript code which sets up the video stream on the webpage and then allows the user to take a snapshot, which then converts the canvas to an image.

<video id="video" width="640" height="480" autoplay></video>
<button id="snap">Snap Photo</button>
<canvas id="canvas" width="640" height="480"></canvas>
    <img id="file" src = "" alt="Emotions">
    <!--input type="file" id="file" name="filename"-->
    <button id="btn">Click here</button>

<script>

    // Put event listeners into place
    window.addEventListener("DOMContentLoaded", function() {
        // Grab elements, create settings, etc.
        var canvas = document.getElementById("canvas"),
            context = canvas.getContext("2d"),
            video = document.getElementById("video"),
            videoObj = { "video": true },
            errBack = function(error) {
                console.log("Video capture error: ", error.code); 
            };

        // Put video listeners into place
        if(navigator.getUserMedia) { // Standard
            navigator.getUserMedia(videoObj, function(stream) {
                video.src = stream;
                video.play();
            }, errBack);
        } else if(navigator.webkitGetUserMedia) { // WebKit-prefixed
            navigator.webkitGetUserMedia(videoObj, function(stream){
                video.src = window.webkitURL.createObjectURL(stream);
                video.play();
            }, errBack);
        } else if(navigator.mozGetUserMedia) { // WebKit-prefixed
            navigator.mozGetUserMedia(videoObj, function(stream){
                video.src = window.URL.createObjectURL(stream);
                video.play();
            }, errBack);
        }

        // Trigger photo take
        document.getElementById("snap").addEventListener("click", function() {
            context.drawImage(video, 0, 0, 640, 480);
                            var dataURL = document.getElementById("canvas");
                            if (dataURL.getContext){
                                var ctx = dataURL.getContext("2d");
                                var link = document.createElement('a');
                                link.download = "test.png";
                                var myImage = link.href = dataURL.toDataURL("image/png").replace("image/png", "image/octet-stream");
                                link.click();
                            }
                            var imageElement = document.getElementById("file");
                            imageElement.src = myImage;
                            //document.getElementById("file").src = dataURL;
                     $('#btn').click(function () {
                            //var file = document.getElementById('file').files[0];


                        $.ajax({
                                url: "https://api.projectoxford.ai/emotion/v1.0/recognize",
                                beforeSend: function(xhrObj) {
                                    // Request headers
                                    xhrObj.setRequestHeader("Content-Type", "application/octet-stream");
                                    xhrObj.setRequestHeader("Ocp-Apim-Subscription-Key", "my-key");
                                },
                                type: "POST",
                                data: imageElement,
                                processData: false
                            })
                            .done(function(data) {
                                JSON.stringify(data);
                                alert(JSON.stringify(data));
                            })
                            .fail(function(error) {
                                //alert(error.getAllResponseHeaders());
                                alert("Error");
                            });
                        });
        });
    }, false);

</script>

I'm not able to get the API running on the image after it's been converted as a canvas. After reading around, I believe this has something to do with the file extension that's given to the image after it's been converted as a canvas (there is none), as the API only allows for image file types. I believe this might be where the problem lies and any help would be very much appreciated. Many thanks!

Answer 1

This is the code I'm using to upload the picture to a backend API, which then use the .NET SDK after adding some business logic. It is not the exact same scenario but the Javascript code might help you:

var url = 'Home/upload';

var data = new FormData();
data.append('file', imgData);

var imgData = canvas.msToBlob('image/jpeg');

$.ajax({
    type: 'POST',
    url: url,
    data: data,
    processData: false,
    contentType: false
})
.done(function (response) {
    loading.style.display = "none";

    if (response.error != "") {
        result.innerText = response.error;
    }
    else {
        result.innerHTML = response.result;
    }
})
.fail(function (response) {
    alert('Error : ' + response.error);
})
.complete(function (response) {
    restartButton.style.display = 'inline-block';
});

The following thread helped me: Pass Blob through ajax to generate a file