How to convert a JavaFX image to a OpenCV matrix?

Question

I am unable to successfully convert a javafx.scene.image.Image to a org.opencv.core.Mat. The resulting matrix produces a black image. I've not used PixelReader before so I am unsure wether or not I am using it correctly.

Here is my code:

public static Mat imageToMat(Image image) {
    int width = (int) image.getWidth();
    int height = (int) image.getHeight();
    byte[] buffer = new byte[width * height * 3];

    PixelReader reader = image.getPixelReader();
    WritablePixelFormat format = WritablePixelFormat.getByteBgraInstance();
    reader.getPixels(0, 0, width, height, format, buffer, 0, 0);

    Mat mat = new Mat(height, width, CvType.CV_8UC3);
    mat.put(0, 0, buffer);
    return mat;
}

Any help/solutions would be greatly appreciated! :) Thank you.

I have a code very similar to this, only essential difference being: `reader.getPixels(0, 0, width, height, format, buffer, 0, 3*width);` - see the `3*width` instead of `0`. — Nikos Paraskevopoulos, Jan 13 '16 at 17:10

score 4 · Answer 1 · answered Jan 13 '16 at 16:49

That stuff is still circumstantial. I've found 2 working solutions. I'll just post my OpenCvUtils class, hope it helps until someone comes up with a better solution:

import java.awt.AlphaComposite;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.ByteArrayInputStream;
import java.net.URISyntaxException;
import java.nio.file.Paths;

import javafx.embed.swing.SwingFXUtils;
import javafx.scene.image.Image;

import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.imgcodecs.Imgcodecs;

public class OpenCvUtils {

    /**
     * Convert a Mat object (OpenCV) in the corresponding Image for JavaFX
     *
     * @param frame
     *            the {@link Mat} representing the current frame
     * @return the {@link Image} to show
     */
    public static Image mat2Image(Mat frame) {
        // create a temporary buffer
        MatOfByte buffer = new MatOfByte();
        // encode the frame in the buffer, according to the PNG format
        Imgcodecs.imencode(".png", frame, buffer);
        // build and return an Image created from the image encoded in the
        // buffer
        return new Image(new ByteArrayInputStream(buffer.toArray()));
    }


    public static Mat image2Mat( Image image) {


        BufferedImage bImage = SwingFXUtils.fromFXImage(image, null);

        return bufferedImage2Mat( bImage);

    }

    // http://www.codeproject.com/Tips/752511/How-to-Convert-Mat-to-BufferedImage-Vice-Versa
    public static Mat bufferedImage2Mat(BufferedImage in)
    {
          Mat out;
          byte[] data;
          int r, g, b;
          int height = in.getHeight();
          int width = in.getWidth();
          if(in.getType() == BufferedImage.TYPE_INT_RGB || in.getType() == BufferedImage.TYPE_INT_ARGB)
          {
              out = new Mat(height, width, CvType.CV_8UC3);
              data = new byte[height * width * (int)out.elemSize()];
              int[] dataBuff = in.getRGB(0, 0, width, height, null, 0, width);
              for(int i = 0; i < dataBuff.length; i++)
              {
                  data[i*3 + 2] = (byte) ((dataBuff[i] >> 16) & 0xFF);
                  data[i*3 + 1] = (byte) ((dataBuff[i] >> 8) & 0xFF);
                  data[i*3] = (byte) ((dataBuff[i] >> 0) & 0xFF);
              }
          }
          else
          {
              out = new Mat(height, width, CvType.CV_8UC1);
              data = new byte[height * width * (int)out.elemSize()];
              int[] dataBuff = in.getRGB(0, 0, width, height, null, 0, width);
              for(int i = 0; i < dataBuff.length; i++)
              {
                r = (byte) ((dataBuff[i] >> 16) & 0xFF);
                g = (byte) ((dataBuff[i] >> 8) & 0xFF);
                b = (byte) ((dataBuff[i] >> 0) & 0xFF);
                data[i] = (byte)((0.21 * r) + (0.71 * g) + (0.07 * b)); //luminosity
              }
           }
           out.put(0, 0, data);
           return out;
     } 

    public static String getOpenCvResource(Class<?> clazz, String path) {
        try {
            return Paths.get( clazz.getResource(path).toURI()).toString();
        } catch (URISyntaxException e) {
            throw new RuntimeException(e);
        }
    }

    // Convert image to Mat
    // alternate version http://stackoverflow.com/questions/21740729/converting-bufferedimage-to-mat-opencv-in-java
    public static Mat bufferedImage2Mat_v2(BufferedImage im) {

        im = toBufferedImageOfType(im, BufferedImage.TYPE_3BYTE_BGR);

        // Convert INT to BYTE
        //im = new BufferedImage(im.getWidth(), im.getHeight(),BufferedImage.TYPE_3BYTE_BGR);
        // Convert bufferedimage to byte array
        byte[] pixels = ((DataBufferByte) im.getRaster().getDataBuffer()).getData();

        // Create a Matrix the same size of image
        Mat image = new Mat(im.getHeight(), im.getWidth(), CvType.CV_8UC3);
        // Fill Matrix with image values
        image.put(0, 0, pixels);

        return image;

    }

    private static BufferedImage toBufferedImageOfType(BufferedImage original, int type) {
        if (original == null) {
            throw new IllegalArgumentException("original == null");
        }

        // Don't convert if it already has correct type
        if (original.getType() == type) {
            return original;
        }

        // Create a buffered image
        BufferedImage image = new BufferedImage(original.getWidth(), original.getHeight(), type);

        // Draw the image onto the new buffer
        Graphics2D g = image.createGraphics();
        try {
            g.setComposite(AlphaComposite.Src);
            g.drawImage(original, 0, 0, null);
        }
        finally {
            g.dispose();
        }

        return image;
    }

}

score 3 · Accepted Answer · edited May 23 '17 at 12:10

Thanks to Nikos Paraskevopoulos for suggesting setting the scanlineStride parameter of the PixelReader::getPixels() method, this has solved it. :)

Working code below:

public static Mat imageToMat(Image image) {
    int width = (int) image.getWidth();
    int height = (int) image.getHeight();
    byte[] buffer = new byte[width * height * 4];

    PixelReader reader = image.getPixelReader();
    WritablePixelFormat<ByteBuffer> format = WritablePixelFormat.getByteBgraInstance();
    reader.getPixels(0, 0, width, height, format, buffer, 0, width * 4);

    Mat mat = new Mat(height, width, CvType.CV_8UC4);
    mat.put(0, 0, buffer);
    return mat;
}

score 1 · Answer 3 · answered Jan 03 '17 at 13:18

You need convert : Mat > BufferedImage > FXImage

private Image mat2Image(Mat src)
{
    BufferedImage image = ImageConverter.toImage(src);
    return SwingFXUtils.toFXImage(image, null);
}

Class: public class ImageConverter {

    /**
     * Converts/writes a Mat into a BufferedImage.
     *
     * @param src Mat of type CV_8UC3 or CV_8UC1
     * @return BufferedImage of type TYPE_3BYTE_BGR or TYPE_BYTE_GRAY
     */
    public static BufferedImage toImage(Mat src)
    {
        if ( src != null ) {

            int cols = src.cols();
            int rows = src.rows();
            int elemSize = (int)src.elemSize();
            byte[] data = new byte[cols * rows * elemSize];
            int type;
            src.data().get(data);
            switch (src.channels()) {
                case 1:
                    type = BufferedImage.TYPE_BYTE_GRAY;
                    break;
                case 3:
                    type = BufferedImage.TYPE_3BYTE_BGR;
                    // bgr to rgb
                    byte b;
                    for(int i=0; i<data.length; i=i+3) {
                        b = data[i];
                        data[i] = data[i+2];
                        data[i+2] = b;
                    }
                    break;
                default:
                    return null;
            }

            BufferedImage bimg = new BufferedImage(cols, rows, type);
            bimg.getRaster().setDataElements(0, 0, cols, rows, data);

            return bimg;
        }
        return null;
    }

}

score 0 · Answer 4 · edited Mar 26 '17 at 18:56

0

Following the solution above it may be also necessary to convert the format from four bytes (CvType.CV_8UC4) to three bytes (CvType.CV_8UC3) depending on what you are finally seeking. For example, if I read a xx.jpa image, it is RGB format.

if (isRGB)
     Imgproc.cvtColor(mat,mat,Imgproc.COLOR_RGBA2RGB);
    //or...COLOR_BGR2RGB,COLOR_BGRA2RGB,COLOR_BGR2BGRA

edited Mar 26 '17 at 18:56

Alexei - check Codidact

22,016
16
145
164

answered Mar 26 '17 at 18:33

J.E.Tkaczyk

557
1
8
19

How to convert a JavaFX image to a OpenCV matrix?

4 Answers4