0

Backgroud: similiar issue here and i use code segments from the answer. However, it does not fix the issue in my case. I have to add that i am a beginner in Unity programming !

Goal: receive a video livestream through a opencv c++ script and access it from a unity script to display in a scene.

Approach to display the livestream in the scence: Sprite renderer.

Issue: Looking at the screenshot below, the webcam output gets displayed four times next to each other and with stripes (it shall show me in front of the camera). The imshow shows the correct webcam image.

enter image description here

Current code:

C++:

...

extern "C" void __declspec(dllexport) Init()
{
    _capture.open(0);
}

extern "C" void __declspec(dllexport) NDVIStreamer(unsigned char* data, int width, int height)
{
    Mat _currentFrame(height, width, CV_8UC4, data);
    _capture >> _currentFrame;
  
    imshow("lennart", _currentFrame);

    memcpy(data, _currentFrame.data, _currentFrame.total() * _currentFrame.elemSize());

    
}

Notes: having the data pointer assinged to _currentFrame doesnt change anything eventhough - from my perspective - that should already do the job. It doesnt.

C#:

...

[RequireComponent(typeof(SpriteRenderer))]
public class SpriteMaker : MonoBehaviour
{
    SpriteRenderer rend;

    [DllImport("NDVIstreamer_dll2")]
    private static extern void NDVIStreamer(IntPtr data, int width, int height);

    [DllImport("NDVIstreamer_dll2")]
    private static extern void Init();

    private Texture2D tex;
    private Color32[] pixel32;

    private GCHandle pixelHandle;
    private IntPtr pixelPtr;
    
    void Start()
    {
        rend = GetComponent<SpriteRenderer>();

        tex = new Texture2D(640, 240, TextureFormat.RGBA32, false);
        pixel32 = tex.GetPixels32();
        //Pin pixel32 array
        pixelHandle = GCHandle.Alloc(pixel32, GCHandleType.Pinned);
        //Get the pinned address
        pixelPtr = pixelHandle.AddrOfPinnedObject();

        Init();

        MatToTexture2D();

        //create a sprite from that texture
        Sprite newSprite = Sprite.Create(tex, new Rect(0, 0, tex.width, tex.height), Vector2.one * 0.5f);
        rend.sprite = newSprite;

        pixelHandle.Free();
    }

    void MatToTexture2D()
    {
        NDVIStreamer(pixelPtr, tex.width, tex.height);

        tex.SetPixels32(pixel32);
        tex.Apply();
    }
}
Lennart
  • 33
  • 7
  • opencv's VideoCapture returns BGR (3 channel, CV_8UC3) frames, not RGBA / CV_8UC4. preallocating the Mat does not do anything useful, you probably need to cvtConvert() it after reading – berak Dec 03 '21 at 13:03
  • @berek: when doing so "cvtColor(_currentFrameBGR,_currentFrameRGB, COLOR_BGR2RGB);" only the color changes but not the issue with the wrong size and the stripes. – Lennart Dec 03 '21 at 13:41
  • COLOR_BGR2RGB**A** – berak Dec 03 '21 at 15:03

1 Answers1

0

adding this code does the job (the code is taken from Programmer here):

    Mat resizedMat(height, width, _currentFrame.type());
    resize(_currentFrame, resizedMat, resizedMat.size());

    Mat argb_img;
    cvtColor(resizedMat, argb_img, COLOR_RGB2BGRA);
    vector<Mat> bgra;
    split(argb_img, bgra);
    swap(bgra[0], bgra[3]);
    swap(bgra[1], bgra[2]);

    flip(argb_img, argb_img, -1);
Lennart
  • 33
  • 7