Error in function input in cv::Mat format in C# and C++ use marshaling

Question

I have a DLL in C++ whose function input is cv::Mat. It gives an error when I try to call this function with the frame input that I receive from opencvsharp as Mat in C#. How do I fix this problem?

How can I match Mat in C++ and Mat in C# to prevent errors?

Do I need to change the C++ function or do I need to do something else in C# to access the data inside the Mat as input to the C++ function?

C++ function:

extern "C" __declspec(dllexport)  vector<std::string> __cdecl ProcessFrame(cv::Mat image);



     vector<std::string> ProcessFrame(cv::Mat image)
    {
        int k = 0;
        cv::Mat croppedimage;
        cv::Mat finalcropped;
        string filename;
        Mat result_image;
        vector<string> listName;
        Module module = torch::jit::load("D:/Project/libfacedetection/example/converted.pt");


        int* pResults = NULL;


        unsigned char* pBuffer = (unsigned char*)malloc(DETECT_BUFFER_SIZE);
        if (!pBuffer)
        {
            fprintf(stderr, "Can not alloc buffer.\n");
            return listName;
        }


        TickMeter cvtm;
        cvtm.start();


        pResults = facedetect_cnn(pBuffer, (unsigned char*)(image.ptr(0)), image.cols, image.rows, (int)image.step);


        int face_num = (pResults ? *pResults : 0);

        if (*pResults != 0)
        {
            result_image = image.clone();

            for (int i = 0; i < face_num; i++)
            {
                try
                {
                    short* p = ((short*)(pResults + 1)) + 142 * i;
                    int confidence = p[0];
                    int x = p[1];
                    int y = p[2];
                    int w = p[3];
                    int h = p[4];

                    char sScore[256];


                    if (confidence >= 95)
                    {

                        //////////////////////////////////////////////////////////////////////////////
                        ////////////// Rotate and Crop
                        //////////////////////////////////////////////////////////////////////////////

                        short angle = Face_rotate(p);

                        cv::Rect rc = AlignCordinates(x, y, w, h, result_image.cols, result_image.rows);

                        cv::Rect myroi(x, y, w, h);
                        cv::Rect newroi((x - rc.x) / 2, (y - rc.y) / 2, w, h);

                        croppedimage = result_image(rc);
                        //imshow("1", croppedimage);

                        croppedimage = croppedimage.clone();
                        croppedimage = rotate(croppedimage, (angle));


                        //imshow("Rotate", croppedimage);


                        croppedimage = croppedimage(newroi).clone();

                        finalcropped = Mat(112, 112, croppedimage.type());
                        //imshow("dst", croppedimage);


                        cv::resize(croppedimage, finalcropped, finalcropped.size());
                        //imshow("resize", finalcropped);
                        Mat flipimage;
                        flip(finalcropped, flipimage, 1);



                        torch::Tensor img_tensor = torch::from_blob(finalcropped.data, { finalcropped.rows,finalcropped.cols ,3 }, torch::kByte);
                        torch::Tensor img_tensor_flip = torch::from_blob(flipimage.data, { flipimage.rows, flipimage.cols, 3 }, torch::kByte);

                        //torch::Tensor img_tensor_final = img_tensor + img_tensor_flip;

                        img_tensor = img_tensor.to(at::kFloat).div(255).unsqueeze(0);
                        img_tensor = img_tensor.sub_(0.5);
                        img_tensor = img_tensor.permute({ 0,3,1,2 });

                        img_tensor_flip = img_tensor_flip.to(at::kFloat).div(255).unsqueeze(0);
                        img_tensor_flip = img_tensor_flip.sub_(0.5);
                        img_tensor_flip = img_tensor_flip.permute({ 0,3,1,2 });



                        at::Tensor output_org = module.forward({ img_tensor }).toTensor();
                        at::Tensor output_flip = module.forward({ img_tensor_flip }).toTensor();

                        std::vector<double> out;


                        for (int i = 0; i < 512; i++)
                        {
                            out.push_back(output_org[0][i].item().to<double>() + output_flip[0][i].item().to<double>());
                        }

                        out = l2_norm(out);




                        std::ifstream file("D:/Project/libfacedetection/example/facebank.json");
                        json object = json::parse(file);




                        double min_dis = 1000;
                        std::string min_name;

                        for (auto& x : object.items()) {
                            auto dataSize = std::size(x.value());

                            std::vector<double> vec1 = x.value();



                            double res = cosine_similarity_vectors(vec1, out);
                            res = (res * -1) + 1;
                            //double res = distance(vec1, out);


                            if (res <= min_dis) {
                                min_dis = res;
                                min_name = x.key();
                            }
                        }




                        std::cout << "One Frame   " << min_name << " " << min_dis << std::endl;


                        if (min_dis < 0.8) {

                            listName.push_back(min_name);
                        }
                        else
                        {
                            listName.push_back("Unknown");
                        }
                    }

                    else
                    {
                        listName.push_back("conf_low");

                    }


                }
                catch (const std::exception& ex)
                {
                    cout << "NASHOD" << endl;

                    //std::cout << ex.what();
                }



            }
        }


        else
        {
            listName.push_back("No_Body");
        }
        cvtm.stop();



        //printf("time = %gms\n", cvtm.getTimeMilli());
        //printf("%d faces detected.\n", (pResults ? *pResults : 0));
        free(pBuffer);

        return listName;
    }

C#:

   [DllImport("detect-camera.dll", CallingConvention = CallingConvention.Cdecl)]
        public static extern List<string> ProcessFrame(Mat image);
        private void button1_Click(object sender, EventArgs e)
        {
            Mat image = Cv2.ImRead("D:/c++/ImportCallFunction/ImportCallFunction/123.jpg");

            List<string> facelist = ProcessFrame(image);


            foreach (var item in facelist)
            {
                listBox1.Items.Add(item);
            }

Error:

System.Runtime.InteropServices.MarshalDirectiveException: 'Cannot marshal 'return value': Generic types cannot be marshaled.'

Answer 1

The error that you've encountered is not related to the parameter type cv::Mat but the return type of the function which is declared as vector<std::string>.

First, a note about the parameter type: you might want to make it const cv::Mat& to avoid copying the whole matrix into the function on every frame. So it will be like:

    std::vector<std::string> ProcessFrame(const cv::Mat& image)

You will also need a wrapper function that is written in C++/CLI and serves as the interface between the C# code and the C++ code. It performs the custom marshalling required by the function both for the input argument and for the return value. Note that you should place the wrapper function in a compile unit that is compiled with /clr (to enable C++/CLI). Your original (native) function doesn't need to be compiled with the /clr option. The wrapper function declaration might look like this:

System::Collections::Generic<System::String>^ ProcessFrameWrapper(
    OpenCvSharp::Mat^ mat);

In the C# code, you will call the wrapper function now:

    [DllImport("detect-camera.dll", CallingConvention = CallingConvention.Cdecl)]
    public static extern List<string> ProcessFrameWrapper(Mat image);
    // ...
            List<string> facelist = ProcessFrameWrapper(image);

To summarize, you will need these files:

DetectCamera.h:

// ...
std::vector<std::string> ProcessFrame(const cv::Mat& image);
// ... other native declarations

DetectCamera.cpp:

// ...

std::vector<std::string> ProcessFrame(const cv::Mat& image)
{
    // actual function implementation
}

// ... other function implementations

DetectCameraWrapper.h:

// ...
System::String^ ProcessFrameWrapper(OpenCvSharp::Mat^ mat);
// ... other wrapper functions ...

DetectCameraWrapper.cpp:

// ...

System::Collections::Generic<System::String>^ ProcessFrameWrapper(
    OpenCvSharp::Mat^ mat)
{
    var names = gcnew System::Collections::Generic<System::String>();
    auto matNativePtr = 
        reinterpret_cast<cv::Mat*>(marshal_as<void*>(mat->CvPtr));
    auto namesNative = ProcessFrame(*matNativePtr);
    for (const auto& nameNative : namesNative)
    {
        names->Add(marshal_as<System::String^>(nameNative));
    }
    return names;
}

// ... other wrapper function implementations

DetectCamera.cs:

    // ... 
    [DllImport("detect-camera.dll", CallingConvention = CallingConvention.Cdecl)]
    public static extern List<string> ProcessFrameWrapper(Mat image);

    private void button1_Click(object sender, EventArgs e)
    {
        Mat image = Cv2.ImRead("D:/c++/ImportCallFunction/ImportCallFunction/123.jpg");
        List<string> facelist = ProcessFrameWrapper(image);
        foreach (var item in facelist)
        {
            listBox1.Items.Add(item);
        }
    // ...

These files can be organized in two or three separate projects:

• DetectCamera.cs is placed in a C# project - call it ProjCSharp.
• DetectCameraWrapper.cpp is placed in a C++/CLI DLL project (with /clr) - call it ProjWrapper.
• DetectCamera.cpp could be placed either within the same project ProjWrapper, or in a separate native library project (.lib) - call it ProjNative. I recommend the latter. If it is placed in a separate library (ProjNative), the DLL project ProjWrapper must be linked to the library ProjNative.

The reason I recommend placing the native code inside a separate library is modularity and code reusability.

Answer 2

I do not quite remember the Interop and was never an expert although I used to do pretty advanced stuff.

The advice from @misoboute is to use CLI which I have never used so I wouldn't know. It is surely possible to do that that and as he has explained, such C++ will understand the generic C# list and be able to marshal successfully.

But it appears to me, this is NOT your problem. You simply want to return a bunch of strings. It really does not have to be defined as List<string>. It can be defined as string[], an array of string.

So I would try

[DllImport("detect-camera.dll", CallingConvention = CallingConvention.Cdecl)]
public static extern string[] ProcessFrame(Mat image);

And then, in my C++ code, return an array of string by representing the vector of string as an array:

How to convert vector to array

Sorry but that is all I have for you. It is possible that after this, you will hit other problems.

Answer 3

You have two "standard" ways to get from C++ to C#.

The first is C++/CLI. In this case you will build a C++/CLI library that takes the std::vectorstd::string and converting that into a System::vectorSystem::string. Then you can use it freely as a System.String[] in C#.

The other is COM. There you create a COM interface that returns a SAFEARRAY containing BSTR string. This COM interface is then instantiated though the System.Runtime.InteropServices in C#. The SAFEARRAY is then a Object[] which can be cased to single string objects.

The facility to load C interfaces into C# is basically restricted to C. Any C++ will fail and Pete provides that "non standard" approach. (It works very well, just not what MS wants you to do.)

c++ :

extern "C" __declspec(dllexport) LPSAFEARRAY ListDevices();

LPSAFEARRAY ProcessFrame(int width, int height, unsigned char* data)
{
    CComSafeArray<BSTR> a(listName.size()); // cool ATL helper that requires atlsafe.h

    std::vector<std::string>::const_iterator it;
    int i = 0;

    for (it = listName.begin(); it != listName.end(); ++it, ++i)
    {
        // note: you could also use std::wstring instead and avoid A2W conversion
        a.SetAt(i, A2BSTR_EX((*it).c_str()), FALSE);
    }

return a.Detach();

}

c#:

[DllImport("detectcameratest.dll")]
        [return: MarshalAs(UnmanagedType.SafeArray)]
        private extern static string[] ProcessFrame2(int width, int height, IntPtr data);
        private void button2_Click(object sender, EventArgs e)
        {
            Mat img = Cv2.ImRead(@"C:\Users\Subtek\source\repos\WindowsFormsApp17\WindowsFormsApp17\bin\x64\Debug\1.jpg");

            var facelist = ProcessFrame2(img.Width, img.Height, img.Data);


            foreach (var s in facelist)
            {
                listBox1.Items.Add(s);
            }