Getting an openCV JS error after running the code for longer time

Question

I am trying to implement beautification in JS using opencv JS but after running for a minute, an error is logged in the console. Can someone please explain why is this happening? Is it related to openCV or is it a memory issue? Here is my code.

let video = document.getElementById("videoInput"); // video is the id of video tag
let canvas = document.getElementById('canvasOutput');

let ctx = canvas.getContext("2d");
video.width = 640;
video.height = 480;
navigator.mediaDevices
.getUserMedia({ video: true, audio: false })
.then(function(stream) {
video.srcObject = stream;
video.play();

const FPS = 30;
function processVideo() {
  let src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
  let img = new cv.Mat();
  let cap = new cv.VideoCapture(video);
  
  let blur = new cv.Mat(video.height, video.width, cv.CV_8UC4);
  let hpf1 = new cv.Mat();
  let hpf = new cv.Mat(video.height, video.width, cv.CV_8UC4);
    
  let skinArea = new cv.Mat(video.height, video.width, cv.CV_8UC1);
  let begin = Date.now();
  cap.read(src);
  img=src
  cv.GaussianBlur(img, blur, new cv.Size(15, 15), 0,0,cv.BORDER_DEFAULT)
  cv.GaussianBlur(img, hpf1, new cv.Size(3, 3), 0,0,cv.BORDER_DEFAULT)

  cv.subtract(img,hpf1,hpf1)
  let minRange = new cv.Mat(img.rows, img.cols, img.type(), [120, 80, 60, 0]);
  let maxRange = new cv.Mat(img.rows, img.cols, img.type(), [255, 190, 120, 255]);
  cv.inRange(img, minRange, maxRange, skinArea);
  
  maxRange.delete()
  minRange.delete()

  let mat_1 = new cv.Mat();
  cv.imshow("canvasOutput1", skinArea);

  mat_1 = new cv.Mat(img.rows, img.cols, cv.CV_8UC1,new cv.Scalar(1,1,1))
  let blur_hpf= new cv.Mat();
  cv.add(blur,hpf1,blur_hpf)
  hpf1.delete()
  blur.delete()
  let blurred = new cv.Mat(),background = new cv.Mat();

  let ones = new cv.Mat(img.rows, img.cols, cv.CV_8UC1, new cv.Scalar(1));

  cv.min(skinArea, ones, skinArea)
  ones.delete()
  cv.subtract(mat_1, skinArea,blurred)
  cv.cvtColor(skinArea, skinArea, cv.COLOR_GRAY2BGRA);
  cv.cvtColor(blurred, blurred, cv.COLOR_GRAY2BGRA);

  cv.multiply(skinArea,blur_hpf,skinArea)

  let final_out = new cv.Mat()
  if(blurred == undefined){
    background = img
    final_out = img
  }
  else if(blurred == undefined || img == undefined){
    console.log("not image")
  }
  else{
  cv.multiply(blurred,img,background)
  cv.add(skinArea,background,final_out)
  }
  
  background.delete()
  blurred.delete()
  skinArea.delete()
  let kernel = cv.matFromArray(3,3,cv.CV_32FC1, [0.110, 0.160, 0.07,0.145, 0.143, 0.12,0.150, 0.150, 0.09]);
  cv.filter2D(final_out, final_out, -1, kernel);
  
  cv.imshow("canvasOutput", final_out);
  
  src.delete()
  final_out.delete()
          
  let delay = 100/FPS - (Date.now() - begin);
  setTimeout(processVideo, 0);
}
// schedule first one.
setTimeout(processVideo, 0);});

It is giving error in cv multiply after the code has run correctly for a minute.

Rotem · Answer 1 · 2022-10-08T21:41:32.223

The script crashes after long execution time due to memory leaks.

Execute cap = new cv.VideoCapture(video); only once before the starting processVideo().
When working with video, we don't suppose to create new VideoCapture object multiple times, and delete it multiple times, because cap = new cv.VideoCapture(video) supposes to open a new video stream (it may not be the case in your specific example).
Create the VideoCapture object once, and use it inside processVideo() multiple times.
Make sure do delete all the cv.Mat objects inside processVideo().
Consider creating the cv.Mat objects before processVideo() instead of executing new and delete multiple times - it's safer, and more efficient.

In case you intend to add a stop function, you may execute cap.release() and cap.delete() in the stop function.

Modified (part) code sample:

let video = document.getElementById("videoInput"); // video is the id of video tag
let cap = new cv.VideoCapture(video); // Create VideoCapture object once before starting processVideo()

const FPS = 30;
function processVideo() {
  let src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
  //let img = new cv.Mat();
  //let cap = new cv.VideoCapture(video);

  let blur = new cv.Mat(video.height, video.width, cv.CV_8UC4);
  let hpf1 = new cv.Mat();
  //let hpf = new cv.Mat(video.height, video.width, cv.CV_8UC4);

  let skinArea = new cv.Mat(video.height, video.width, cv.CV_8UC1);
  let begin = Date.now();
  cap.read(src);
  //img=src
  let img = src;

  cv.GaussianBlur(img, blur, new cv.Size(15, 15), 0,0,cv.BORDER_DEFAULT)
  cv.GaussianBlur(img, hpf1, new cv.Size(3, 3), 0,0,cv.BORDER_DEFAULT)

  cv.subtract(img,hpf1,hpf1)
  let minRange = new cv.Mat(img.rows, img.cols, img.type(), [120, 80, 60, 0]);
  let maxRange = new cv.Mat(img.rows, img.cols, img.type(), [255, 190, 120, 255]);
  cv.inRange(img, minRange, maxRange, skinArea);

  maxRange.delete()
  minRange.delete()

  //let mat_1 = new cv.Mat();
  cv.imshow("canvasOutput1", skinArea);

  let mat_1 = new cv.Mat(img.rows, img.cols, cv.CV_8UC1,new cv.Scalar(1,1,1))
  let blur_hpf= new cv.Mat();
  cv.add(blur,hpf1,blur_hpf)
  hpf1.delete()
  blur.delete()
  let blurred = new cv.Mat(),background = new cv.Mat();

  let ones = new cv.Mat(img.rows, img.cols, cv.CV_8UC1, new cv.Scalar(1));

  cv.min(skinArea, ones, skinArea)
  ones.delete()
  cv.subtract(mat_1, skinArea,blurred)
  cv.cvtColor(skinArea, skinArea, cv.COLOR_GRAY2BGRA);
  cv.cvtColor(blurred, blurred, cv.COLOR_GRAY2BGRA);

  cv.multiply(skinArea,blur_hpf,skinArea)

  let final_out = new cv.Mat()
  if(blurred == undefined){
    background = img
    final_out = img
  }
  else if(blurred == undefined || img == undefined){
    console.log("not image")
  }
  else{
    cv.multiply(blurred,img,background)
    cv.add(skinArea,background,final_out)
  }

  background.delete()
  blurred.delete()
  skinArea.delete()
  let kernel = cv.matFromArray(3,3,cv.CV_32FC1, [0.110, 0.160, 0.07,0.145, 0.143, 0.12,0.150, 0.150, 0.09]);
  cv.filter2D(final_out, final_out, -1, kernel);

  cv.imshow("canvasOutput", final_out);

  src.delete()
  final_out.delete()
  mat_1.delete()
  blur_hpf.delete()

  let delay = 100/FPS - (Date.now() - begin);
  setTimeout(processVideo, 0);
};

// schedule the first one.
setTimeout(processVideo, 0);

Update:

Monitoring memory usage:

One way to monitor the memory usage is getting the use head size before start executing onVideoStarted(), getting the size at the end of onVideoStarted(), compute the delta, and print it to the log periodically.

According to the following post, we may use window.performance.memory in Chrome (only) browser.

For getting used heap size we may execute:

const heapUsed = window.performance.memory['usedJSHeapSize'];

Execute the following code before onVideoStarted():

 const memUsedBefore = window.performance.memory['usedJSHeapSize']; //Used memory before executing processVideo (in bytes)

Execute the following code before the end of onVideoStarted():

 const additionalMemUsed = window.performance.memory['usedJSHeapSize'] - memUsedBefore;
 console.log(`Additional mem used: ${Math.round(additionalMemUsed/(1024*1024))} MB`);

The used memory supposed to go up and down.
In case the value keeps raising along time, there is a "memory leak".

Code sample (relevant part):

const FPS = 30;
const memUsedBefore = window.performance.memory['usedJSHeapSize']; //Used memory before executing processVideo (in bytes)

function processVideo() {
  let src = new cv.Mat(video.height, video.width, cv.CV_8UC4);
  //let img = new cv.Mat();
  //let cap = new cv.VideoCapture(video);

  let blur = new cv.Mat(video.height, video.width, cv.CV_8UC4);
  let hpf1 = new cv.Mat();
  //let hpf = new cv.Mat(video.height, video.width, cv.CV_8UC4);

  let skinArea = new cv.Mat(video.height, video.width, cv.CV_8UC1);
  let begin = Date.now();
  cap.read(src);
  //img=src
  let img = src;

  cv.GaussianBlur(img, blur, new cv.Size(15, 15), 0,0,cv.BORDER_DEFAULT)
  cv.GaussianBlur(img, hpf1, new cv.Size(3, 3), 0,0,cv.BORDER_DEFAULT)

  cv.subtract(img,hpf1,hpf1)
  let minRange = new cv.Mat(img.rows, img.cols, img.type(), [120, 80, 60, 0]);
  let maxRange = new cv.Mat(img.rows, img.cols, img.type(), [255, 190, 120, 255]);
  cv.inRange(img, minRange, maxRange, skinArea);

  maxRange.delete()
  minRange.delete()

  //let mat_1 = new cv.Mat();
  cv.imshow("canvasOutput1", skinArea);

  let mat_1 = new cv.Mat(img.rows, img.cols, cv.CV_8UC1,new cv.Scalar(1,1,1))
  let blur_hpf= new cv.Mat();
  cv.add(blur,hpf1,blur_hpf)
  hpf1.delete()
  blur.delete()
  let blurred = new cv.Mat(),background = new cv.Mat();

  let ones = new cv.Mat(img.rows, img.cols, cv.CV_8UC1, new cv.Scalar(1));

  cv.min(skinArea, ones, skinArea)
  ones.delete()
  cv.subtract(mat_1, skinArea,blurred)
  cv.cvtColor(skinArea, skinArea, cv.COLOR_GRAY2BGRA);
  cv.cvtColor(blurred, blurred, cv.COLOR_GRAY2BGRA);

  cv.multiply(skinArea,blur_hpf,skinArea)

  let final_out = new cv.Mat()
  if(blurred == undefined){
    background = img
    final_out = img
  }
  else if(blurred == undefined || img == undefined){
    console.log("not image")
  }
  else{
    cv.multiply(blurred,img,background)
    cv.add(skinArea,background,final_out)
  }

  background.delete()
  blurred.delete()
  skinArea.delete()
  let kernel = cv.matFromArray(3,3,cv.CV_32FC1, [0.110, 0.160, 0.07,0.145, 0.143, 0.12,0.150, 0.150, 0.09]);
  cv.filter2D(final_out, final_out, -1, kernel);

  cv.imshow("canvasOutput", final_out);

  src.delete()
  final_out.delete()
  mat_1.delete()
  blur_hpf.delete()

  let delay = 100/FPS - (Date.now() - begin);

  //Subtract memUsedBefore from the current allocated heap size
  const additionalMemUsed = window.performance.memory['usedJSHeapSize'] - memUsedBefore;
  console.log(`Additional mem used: ${Math.round(additionalMemUsed/(1024*1024))} MB`); //Print to log - should be around zero.

  setTimeout(processVideo, 0);
};

I tried this but still facing the same issue. Also if I delete img mat, it is giving binding error. — vaibhav jain, Oct 08 '22 at 09:50
Compare your code with the code sample in my answer. There is no need to delete a reference to an object. Using Windows Task Manager, it's simple to see the memory usage keeps rising (I didn't try other OS). — Rotem, Oct 08 '22 at 11:26
I updated my answer with memory usage monitoring example. In case the issue is not a "memory leak", you may post the HTML, and other dependencies. I used the example from [here](https://docs.opencv.org/4.5.0/dd/d00/tutorial_js_video_display.html), for testing. It's working in my machine (with my Webcam)... — Rotem, Oct 08 '22 at 21:47

Getting an openCV JS error after running the code for longer time

1 Answers1

Update: