bytedeco javcv OpenCVFrameGrabber.grab is too slow for 25 fps

Question

I've got the following code and somehow the logged time between 1 and 2 is 65-95 ms all the time so I can only record a video grabbed from my webcam with 10-12 fps :-/

(For instance) Looking at How to capture and record video from webcam using JavaCV I shouldn't be that far off from being able to capture a video with 25 or 30 fps from the webcam

                grabber = OpenCVFrameGrabber.createDefault(0);
                //grabber.setFrameRate(25);
                grabber.setImageWidth(1280);
                grabber.setImageHeight(720);
                grabber.start();

                ExecutorService executorService = Executors.newSingleThreadExecutor();
                executorService.execute(() -> {
                    try {
                        while (!isStopRequested) {
                            LOGGER.debug("1: {}", System.currentTimeMillis());
                            Frame frame = grabber.grab(); // this takes 65-95 ms
                            LOGGER.debug("2: {}", System.currentTimeMillis());
                            WritableImage image = frameToImage(frame);
                            recordingController.processImage(image);
                            if (isStarted) {
                                recorder.record(frame);
                            }
                        }
                    } catch (FrameGrabber.Exception fe) {
                        LOGGER.error("FrameGrabber.Exception when grabbing frame from camera: {}", fe.getMessage());
                    } catch (FrameRecorder.Exception fre) {
                        LOGGER.error("FrameRecorder.Exception when recording frame from camera: {}", fre.getMessage());
                    }
                });

The code runs in Java 17 on a laptop with an i7-11800H @ 2.30GHz and 64 GB ram, it has 8 cores and not even one of the 8 is fully utilized when running this software, so I don't think it's a hardware issue.

Answer 1

What works for me is this code, in which not only the Thread.sleep() is essential but also the amount of sleep, as 10 or 30 didn't give me good results for the duration, but 40 does (2-2-3-2-2 etc). Don't know why, though and because of this uncertainty I won't accept my own answer as THE answer. I might have to get back to this later

import javafx.beans.property.SimpleBooleanProperty;
import javafx.scene.image.*;
import ??.screens.RecordingController;
import org.bytedeco.javacv.*;
import org.bytedeco.javacv.Frame;
import org.bytedeco.opencv.global.opencv_imgproc;
import org.bytedeco.opencv.opencv_core.Mat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import java.awt.*;
import java.io.File;
import java.nio.ByteBuffer;

import static org.opencv.imgproc.Imgproc.COLOR_BGR2BGRA;

@Component
public class WebcamGrabber {
    private static final Logger LOGGER = LoggerFactory.getLogger(WebcamGrabber.class);

    private RecordingController recordingController;

    Mat javaCVMat = new Mat();

    /**
     * create buffer only once saves much time!
     */
    WritablePixelFormat<ByteBuffer> formatByte = PixelFormat.getByteBgraPreInstance();

    OpenCVFrameConverter<Mat> javaCVConv = new OpenCVFrameConverter.ToMat();

    /**
     * controls if application closes
     */
    SimpleBooleanProperty cameraActiveProperty = new SimpleBooleanProperty(false);

    OpenCVFrameGrabber frameGrabber;

    FFmpegFrameRecorder recorder;

    ByteBuffer buffer;

    Dimension dimension;

    boolean isStarted;

    public void init(RecordingController recordingController) {
        this.recordingController = recordingController;
    }


    protected void updateView(Frame frame) {
        int w = frame.imageWidth;
        int h = frame.imageHeight;

        Mat mat = javaCVConv.convert(frame);
        opencv_imgproc.cvtColor(mat, javaCVMat, COLOR_BGR2BGRA);
        if (buffer == null) {
            buffer = javaCVMat.createBuffer();
        }

        PixelBuffer<ByteBuffer> pb = new PixelBuffer<>(w, h, buffer, formatByte);
        final WritableImage wi = new WritableImage(pb);
        recordingController.processImage(wi);
    }


    public void setCameraActive(Boolean isActive) {
        cameraActiveProperty.set(isActive);
    }

    public Boolean getCameraActive() {
        return cameraActiveProperty.get();
    }

    public void shutdown() {
        setCameraActive(false);
    }

    void setVideoView(Frame mat) {
        updateView(mat);
    }

    public void startViewing(Dimension dimension, int cameraIndex) {
        this.dimension = dimension;
        frameGrabber = new OpenCVFrameGrabber(cameraIndex);
        frameGrabber.setFrameRate(25);
        frameGrabber.setImageWidth(dimension.width);
        frameGrabber.setImageHeight(dimension.height);

        try {
            frameGrabber.start();
        } catch (FrameGrabber.Exception fe) {
            LOGGER.error("Exception when trying to start grabbing from camera: {}", fe.getMessage());
        }
        new Thread(() -> {
            setCameraActive(true);
            while (getCameraActive()) {
                try {
                    new Thread(() -> {
                        try {
                            long startTime = System.currentTimeMillis();
                            Frame frame = frameGrabber.grab();
                            System.out.println("Duration: " + (System.currentTimeMillis() - startTime));
                            setVideoView(frame);
                            if (isStarted) {
                                recorder.record(frame);
                            }
                        } catch (FrameGrabber.Exception fe) {
                            LOGGER.error("Exception when grabbing frame from camera: {}", fe.getMessage());
                        } catch (FrameRecorder.Exception fe) {
                            LOGGER.error("Exception when recording frame from camera: {}", fe.getMessage());
                        }
                    }).start();

                    Thread.sleep(40);
                } catch (InterruptedException ie) {
                    LOGGER.error("InterruptedException when grabbing frame from camera: {}", ie.getMessage());
                }
            }
            try {
                frameGrabber.release();
                recorder.stop();
            } catch (FrameGrabber.Exception fe) {
                LOGGER.error("Exception when releasing frame grabber: {}", fe.getMessage());
            } catch (FrameRecorder.Exception fe) {
                LOGGER.error("Exception when releasing frame recorder: {}", fe.getMessage());
            }
        }).start();
    }

    public void startRecording() {
        long start = System.currentTimeMillis();
        recorder = new FFmpegFrameRecorder(new File("D:\\data\\" + start + ".mp4"), dimension.width, dimension.height, 2);
        recorder.setFormat("mp4");
        recorder.setFrameRate(25);
        try {
            recorder.start();
            isStarted = true;
        } catch (FrameRecorder.Exception fre) {
            LOGGER.error("FrameRecorder.Exception when starting recording: {}", fre.getMessage());
        }
        Context.getInstance().writeMetadata(start);
    }

    public void stop() {
        if (isStarted) {
            isStarted = false;
            LOGGER.info("Recording stopped");
        }
    }
}