Convert OpenCV Mat to Frame type of Realsense in Librealsense SDK

Question

I have RGB data as rs2::frame, I converted it to cv::Mat and send via TCP connection, on the server (receiver) side I am storing buffer into a cv::Mat. My question is How can I convert cv::Mat to rs2::frame on the receiver side, so I can use SDK functions that support rs2::frame type?

Answer 1

You need to simulate a software device in order to have a rs2::frame.

Following this example you can write your own class that creates the synthetic streams, taking data from the cv::Mat instances.

For example, here's something I have done to solve the problem.

---

rsImageConverter.h

#include <librealsense2/rs.hpp>
#include <librealsense2/hpp/rs_internal.hpp>

class rsImageConverter
{
public:
    rsImageConverter(int w, int h, int bpp);

    bool convertFrame(uint8_t* depth_data, uint8_t*  color_data);

    rs2::frame getDepth() const;
    rs2::frame getColor() const;

private:
    int w = 640;
    int h = 480;
    int bpp = 2;

    rs2::software_device dev;
    rs2::software_sensor depth_sensor;
    rs2::software_sensor color_sensor;
    rs2::stream_profile depth_stream;
    rs2::stream_profile color_stream;
    rs2::syncer syncer;

    rs2::frame depth;
    rs2::frame color;

    int ind = 0;
};

---

rsImageConverter.cpp

#include "rsimageconverter.h"

rsImageConverter::rsImageConverter(int w, int h, int bpp) :
    w(w),
    h(h),
    bpp(bpp),
    depth_sensor(dev.add_sensor("Depth")), // initializing depth sensor
    color_sensor(dev.add_sensor("Color")) // initializing color sensor
{
    rs2_intrinsics depth_intrinsics{ w, h, (float)(w / 2), (float)(h / 2), (float) w , (float) h , RS2_DISTORTION_BROWN_CONRADY ,{ 0,0,0,0,0 } };
    depth_stream = depth_sensor.add_video_stream({  RS2_STREAM_DEPTH, 0, 0,
                                w, h, 60, bpp,
                                RS2_FORMAT_Z16, depth_intrinsics });
    depth_sensor.add_read_only_option(RS2_OPTION_DEPTH_UNITS, 0.001f); // setting depth units option to the virtual sensor

    rs2_intrinsics color_intrinsics = { w, h,
            (float)w / 2, (float)h / 2,
            (float)w / 2, (float)h / 2,
            RS2_DISTORTION_BROWN_CONRADY ,{ 0,0,0,0,0 } };
    color_stream = color_sensor.add_video_stream({ RS2_STREAM_COLOR, 0, 1, w,
                                    h, 60, bpp,
                                    RS2_FORMAT_RGB8, color_intrinsics });


    dev.create_matcher(RS2_MATCHER_DLR_C); // create the matcher with the RGB frame

    depth_sensor.open(depth_stream);
    color_sensor.open(color_stream);

    depth_sensor.start(syncer);
    color_sensor.start(syncer);

    depth_stream.register_extrinsics_to(color_stream, { { 1,0,0,0,1,0,0,0,1 },{ 0,0,0 } });
}

bool rsImageConverter::convertFrame(uint8_t*  depth_data, uint8_t*  color_data)
{
    depth_sensor.on_video_frame({ depth_data, // Frame pixels
                                      [](void*) {}, // Custom deleter (if required)
                                      w*bpp, bpp, // Stride and Bytes-per-pixel
                                      (rs2_time_t)ind * 16, RS2_TIMESTAMP_DOMAIN_HARDWARE_CLOCK, ind, // Timestamp, Frame# for potential sync services
                                      depth_stream });
    color_sensor.on_video_frame({ color_data, // Frame pixels from capture API
        [](void*) {}, // Custom deleter (if required)
        w*bpp, bpp, // Stride and Bytes-per-pixel
        (rs2_time_t)ind * 16, RS2_TIMESTAMP_DOMAIN_HARDWARE_CLOCK, ind, // Timestamp, Frame# for potential sync services
        color_stream });

    ind++;

    rs2::frameset fset = syncer.wait_for_frames();
    depth = fset.first_or_default(RS2_STREAM_DEPTH);
    color = fset.first_or_default(RS2_STREAM_COLOR);

    return (depth && color); // return true if everything went good
}

rs2::frame rsImageConverter::getDepth() const
{
    return depth;
}

rs2::frame rsImageConverter::getColor() const
{
    return color;
}

---

And then you can use it like this (assuming depth and rgb as two cv::Mat, where depth is converted in CV_16U and rgb is CV_8UC3 with a conversion from BGR to RGB):

rsImageConverter* converter = new rsImageConverter(640, 480, 2);

...

if(converter->convertFrame(depth.data, rgb.data)) 
{
    rs2::frame rs2depth = converter->getDepth();
    rs2::frame rs2rgb = converter->getColor();

    ... // Here you use these frames
}

By the way, I designed this class with the use of both depth and RGB. To convert only one of these you can simply pass an empty frame to the other argument, or change the class.