How do I pass an image from javascript to a Python Flask app

Question

I have a working Flask/Python app that uses a Mask RCNN and some OpenCV python functions. I got this working on localhost fine but learnt that to use a user's webcam I need to use Javascript. I found some working .js code that retrieves the webcam and has a button to take a screen shot.

What is the easiest way to pass this image to Python for processing. I know tensorflow.js and opencv.js exist but as I don't know js I assume it'll be much easier to just pass it back to Python than learning another language to the same level as my Python knowledge.

I have read that saving to the web server with .js isn't possible without another intermediary language. I could separate the steps and use the .js to save the file to the user's computer and then ask them to upload it into the Python script but this adds another step for the user I would like to avoid.

JS code:

(function() {

  // For javascript we must define if it is a constant or variable before giving it a name - akin to in SWIFT.
  // The width and height of the captured photo. We will set the
  // width to the value defined here, but the height will be
  // calculated based on the aspect ratio of the input stream.


  var width = 320;    // We will scale the photo width to this
  var height = 0;     // This will be computed based on the input stream

  // |streaming| indicates whether or not we're currently streaming
  // video from the camera. Obviously, we start at false.

  var streaming = false;

  // The various HTML elements we need to configure or control. These
  // will be set by the startup() function.

  var video = null;
  var canvas = null;
  var photo = null;
  var startbutton = null;

  function startup() {
    // These elementId's are passed to the javascript file from the html file. Therefore, we must ensure they match.
    video = document.getElementById('video');
    canvas = document.getElementById('canvas');
    photo = document.getElementById('photo');
    startbutton = document.getElementById('startbutton');

    navigator.mediaDevices.getUserMedia({video: true, audio: false})
    .then(function(stream) {
      video.srcObject = stream;
      video.play();
    })
    .catch(function(err) {
      console.log("An error occurred: " + err);
    });

    video.addEventListener('canplay', function(ev){
      if (!streaming) {
        height = video.videoHeight / (video.videoWidth/width);

        // Firefox currently has a bug where the height can't be read from
        // the video, so we will make assumptions if this happens.

        if (isNaN(height)) {
          height = width / (4/3);
        }

        video.setAttribute('width', width);
        video.setAttribute('height', height);
        canvas.setAttribute('width', width);
        canvas.setAttribute('height', height);
        streaming = true;
      }
    }, false);

    // When we click the button, we run the takepicture() function
    startbutton.addEventListener('click', function(ev){
      takepicture();
      ev.preventDefault();
    }, false);

    // clearphoto() is a function as defined below that makes the photo grey until we populate it with a screenshot.
    clearphoto();
  }


  // Fill the photo with an indication that none has been
  // captured.

  function clearphoto() {
    var context = canvas.getContext('2d');
    context.fillStyle = "#AAA";
    context.fillRect(0, 0, canvas.width, canvas.height);

    var data = canvas.toDataURL('image/png');
    photo.setAttribute('src', data);
  }



  // Capture a photo by fetching the current contents of the video
  // and drawing it into a canvas, then converting that to a PNG
  // format data URL. By drawing it on an offscreen canvas and then
  // drawing that to the screen, we can change its size and/or apply
  // other changes before drawing it.

  function takepicture() {
    var context = canvas.getContext('2d');
    const download = document.getElementById('download');

    if (width && height) {
      canvas.width = width;
      canvas.height = height;
      context.drawImage(video, 0, 0, width, height); // Draw the video onto the canvas.

      var data = canvas.toDataURL('image/png');

      photo.setAttribute('src', data); // elements - in this case photo - have attributes, an attribute of this photos element is the src.
      // Here we are changing the source to be data - which encapsulates the canvas we have just drawn and edited.

    } else {
      clearphoto();
    }
  }


  // Set up our event listener to run the startup process
  // once loading is complete.
  window.addEventListener('load', startup, false);
})();

HTML code:

<!doctype html>
<html>
<head>
    <title>WebRTC: Still photo capture demo</title>
    <meta charset='utf-8'>

    <script src="Webcam.js">
    </script>
</head>
<body>
<div class="contentarea">
  <div class="camera">
    <video id="video">Video stream not available.</video>
    <button id="startbutton">Take photo</button>
  </div>
  <canvas id="canvas">
  </canvas>
  <div class="output">
    <img id="photo" alt="The screen capture will appear in this box.">
  </div>
</div>
</body>
</html>

Edit: After furas' comments I tried to implement the scripts but the effects to my screenshot are being displayed in the server panel and not on top of the screnshot itself. When I include my RCNN processing (commented out) it performs it one the next incoming frame of the webcam and not just once on the screenshot as I'd like.

Flask:

import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=UserWarning)
from flask import Flask, render_template, request, make_response
import cv2 as cv
import numpy as np
from test_routes import create_model
from mrcnn import visualize
import time

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('furas.html', width=320, height=240)


def send_file_data(data, mimetype='image/jpeg', filename='output.jpg'):
    # https://stackoverflow.com/questions/11017466/flask-to-return-image-stored-in-database/11017839

    response = make_response(data)
    response.headers.set('Content-Type', mimetype)
    response.headers.set('Content-Disposition', 'attachment', filename=filename)

    return response


@app.route('/upload', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        fs = request.files.get('snap')
        if fs:
            #model = create_model()
            img = cv.imdecode(np.frombuffer(fs.read(), np.uint8), cv.IMREAD_UNCHANGED)

            # Detect objects
            #r = model.detect([img], verbose=0)[0]

            #masked_image = visualize.display_instances(img, r['rois'], r['masks'], r['class_ids'], ["BG", "culture"], r['scores'], show_bbox=True)
            #masked_image = cv.putText(img=np.float32(masked_image), text=f'{r["masks"].shape[2]} colonies', org=(50, 50),
                                      #fontFace=cv.FONT_HERSHEY_DUPLEX, fontScale=2, lineType=cv.LINE_AA, color=(256, 0, 0), thickness=2)


            img = cv.cvtColor(img, cv.COLOR_RGB2GRAY)
            ret, buf = cv.imencode('.jpg', img)

            return send_file_data(buf.tobytes())


    return 'Hello World!'


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=False, ssl_context='adhoc')

New HTML Code:

<!DOCTYPE html>
<html lang="en">
<table>
<tr>
    <td>VIDEO</td>
    <td>CANVAS</td>
    <td>SERVER</td>
    <td>SCREENSHOT</td>
</tr>
<tr>
    <td>- assign camera (media device)<br>- play stream</td>
    <td>- draw video on context 2d,<br>- convert to jpg file (Blob)<br>- upload to server as POST</td>
    <td>- get jpg file (Blob) from server<br>- convert to BASE64 url<br>- assign to img</td>
    <td>- get canvas as BASE64 url<br>- assign to img</td>
</tr>
<tr>
    <td><video id="video" width="{{ width }}" height="{{ height }}" autoplay style="background-color: grey"></video></td>
    <td><canvas id="canvas" width="{{ width }}" height="{{ height }}" style="background-color: grey"></canvas></td>
    <td><img id="image" src="" width="{{ width }}" height="{{ height }}" style="background-color: grey" /></td>
    <td><img id="image64" src="" width="{{ width }}" height="{{ height }}" style="background-color: grey" /></td>
</tr>
<tr>
    <td></td>
    <td></td>
    <td></td>
    <td><button id="send">Take Photo</button></td>
</tr>
</table>
<script>
// Elements for taking the snapshot
var video = document.getElementById('video');
// Element to display snapshot
    // you need canvas to get image - canvas can be hidden using `createElement("canvas")`
    // var canvas = document.createElement("canvas");
var canvas = document.getElementById('canvas');
var context = canvas.getContext('2d');
var image = document.getElementById('image');
var image64 = document.getElementById('image64');
console.log("before get camera");
console.log(navigator.mediaDevices && navigator.mediaDevices.getUserMedia);


// Get access to the camera!
if(navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
    // Not adding `{ audio: true }` since we only want video now
    navigator.mediaDevices.getUserMedia({ video: true }).then(function(stream) {
        //video.src = window.URL.createObjectURL(stream);
        video.srcObject = stream;
        video.play();
        //console.log('setInterval')
        window.setInterval(function() {
            context.drawImage(video, 0, 0, {{ width }}, {{ height }}); // better use size because camera may gives data in different size then <video> is displaying
            canvas.toBlob(upload, "image/jpeg");
        }, 100);
    });
}

// Trigger photo take on button click. We know which button because inside the HTML there is a button called 'send' also.
document.getElementById("send").addEventListener("click", function() {
    // copy frame from video to canvas as context 2d
    context.drawImage(video, 0, 0, {{ width }}, {{ height }}); // better use size because camera may gives data in different size then <video> is displaying
    // convert to BASE64 url and assign to <img> to display it
    image64.src = canvas.toDataURL();
});


function upload(file) {
    // Create a form using FormData. This holds key value pairs.
    var formdata = new FormData();

    // Add a key value pair ("snap", file) to the FormData object. file is the original file passed sent to the upload function.
    formdata.append("snap", file);

    // create AJAX connection
    // The fetch() method is used to request from the server and load the information in the webpages.
    // The request can be of any APIs that returns the data of the format JSON or XML. This method returns a promise.

    // The fetch() method returns a Promise and has the syntax of if true, do these sequential functions, else in the case of an error do Y.
    fetch("{{ url_for('upload') }}", {
        method: 'POST',
        body: formdata,
    }).then(function(response) {
        return response.blob();
    }).then(function(blob) {
        image.src = URL.createObjectURL(blob);
    }).catch(function(err) {
        console.log('Fetch problem: ' + err.message);
    });
}
</script>
</html>

Thanks.