I am doing some image processing stuff on a live webRTC camera view in a <video> with OpenCV.js. The processing has a single particularly slow operation that can take between 100ms to 400ms. Even though I execute this within a delay / setTimeout loop and have also tried promises, this leads to a significant amount of blocking and lag / choppiness in the video.
As a result I want to move this operation to a WebWorker. The issue I am facing is that there is no obvious way to transfer video to a WebWorker. I read this can be done through a canvas but this seems pretty convoluted / inefficient. Is there an easier way to do this?
I'm using an OpenCV.js implementation like this but mine is a bit more computationally expensive and with webRTC.
In essence my issue with a WebWorker is two fold:
- Images in OpenCV.js are read from
<img>or<canvas>tags. How can I pass these to aWebWorker? - Videos in OpenCV.js are read directly from the
<video>tag. Which is super convenient is my current setup but how can I pass this into aWebWorker. - How can I load the WASM (opencv.js) file from within the
WebWorker
My Code
<!DOCTYPE html>
<html lang="en">
<script src="./opencv.js" type="text/javascript"></script>
<head>
<meta charset="utf-8" />
</head>
<body>
<video id="video"></video>
<script type="text/javascript">
var video; // INPUT WEBRTC as <video> tag
var drawingCanvas;
var img; // INPUT AS <img> tag
var cameras = ["user", "environment"]; // USUAL CAMERA TYPES
// WAITS FOR WASM TO LOAD
cv['onRuntimeInitialized'] = () => {
// LOADING INDICATOR STOP
var options = []; // CAMERAS AVAILABLE
navigator.mediaDevices.enumerateDevices().then((devices) => {
let index = 0;
devices.find((device) => {
if (device.kind === 'videoinput') {
if (device.deviceId == '') {
options.push({
audio: false,
video: {
facingMode: {
exact: cameras[index]
}
}
});
index++;
} else {
options.push({
audio: false,
video: {
deviceId: {
exact: device.deviceId
}
}
});
}
}
});
if (options.length == 0) {
console.log("NO DEVICES FOUND");
} else {
navigator.mediaDevices.getUserMedia(options[options.length - 1]).then(stream => {
video = document.getElementById('video');
video.setAttribute('playsinline', 'playsinline');
video.setAttribute('id', 'v');
video.setAttribute('position', 'absolute');
video.setAttribute('top', '0');
video.setAttribute('left', '0');
//document.body.appendChild(video);
try {
video.srcObject = stream;
video.style.display = 'block';
video.play();
} catch (error) {
video.src = URL.createObjectURL(stream);
video.style.display = 'block';
video.play();
}
let videoSettings = stream.getVideoTracks()[0].getSettings()
video.width = window.innerWidth; //videoSettings.width/2;
video.height = window.innerWidth * (videoSettings.height / videoSettings.width); //
img = document.createElement('img');
img.setAttribute("id", "img1");
img.setAttribute("src", "volviclogo.png");
document.body.appendChild(img);
brisk = new cv.BRISK(30, 3, 1); //new cv.BRISK(30, 1, 3);
isDetecting = true;
setTimeout(mainFunction, 1000);
})
}
}).catch(err => {
console.log(err);
});
}
// VIDEO MAX FRAME RATE
var FPS = 24;
// OPENCV VARS CACHED
var brisk;
var img1;
var kp1;
var des1;
var img1Raw;
// ADDITIONAL LOADINGS FIRST PASS
var isFirst = true;
// TRACKING OF VIDEO FEED
var cap;
var frame;
function mainFunction() {
if (isFirst) {
cap = new cv.VideoCapture(video);
frame = new cv.Mat(video.height, video.width, cv.CV_8UC4);
cap.read(frame);
}
subFunction();
function subFunction() {
try {
let begin = Date.now();
cap.read(frame);
img2 = new cv.Mat();
const mask = new cv.Mat();
kp2 = new cv.KeyPointVector();
des2 = new cv.Mat();
cv.cvtColor(frame, img2, cv.COLOR_RGBA2GRAY, 0);
brisk.detectAndCompute(img2, mask, kp2, des2, false); // SUPER SLOW
img2.delete();
if (isFirst) {
img1Raw = cv.imread("img1");
img1 = new cv.Mat();
cv.cvtColor(img1Raw, img1, cv.COLOR_RGBA2GRAY, 0);
kp1 = new cv.KeyPointVector();
des1 = new cv.Mat();
brisk.detectAndCompute(img1, mask, kp1, des1, false); // SUPER SLOW BUT ONLY ONCE
isFirst = false;
}
kp2.delete();
des2.delete();
let delay = 1000 / FPS - (Date.now() - begin);
setTimeout(subFunction, delay);
} catch (err) {
console.log("ERROR");
console.log(err);
}
}
}
</script>
</body>
</html>
