1

This question refers to this one : How to render Android's YUV-NV21 camera image on the background in libgdx with OpenGLES 2.0 in real-time?

It is well explained in the best answer given by the author, but I have a little different issue concerning YV12 instead of NV12. (Here are some specs : https://wiki.videolan.org/YUV and https://www.fourcc.org/yuv.php )

What about YUV-YV12 ? The Y buffer is the same, but the UV is not entrelaced so i looks like 2 buffers for V and U. But then, who to do to give them to the Shader ? Using an Pixmap.Format.Intensity texture I think, setting GL_LUMINANCE ?

I don't understand how the NV12 "UVUV" buffer is converted into RGBA with RGB = V and A = U using GL_LUMINANCE and Pixmap format with GL_LUMINANCEALPHA ?

YV12 is using "VVUU" buffer, so it is easy to split in V and U buffers, but how to bind them and get u and v in the shader ?

Thanks for any help, this sample is awesome ! but I need something a bit different and for that I need to understand deep in details the shader binding behavior.

Thanks!

Community
  • 1
  • 1
Vince
  • 70
  • 8

1 Answers1

1

Ok, I got it : YUV-YV12 is 12 bytes per pixels : 8 bit Y plane followed by 8 bit 2x2 subsampled V and U planes.

Based on this answer (detailing the whole YUV-NV12 to RGB shader display) https://stackoverflow.com/a/22456885/4311503 let's make some littles changes.

So, we can split the buffer in 3 pieces

    yBuffer = ByteBuffer.allocateDirect(640*480);
    uBuffer = ByteBuffer.allocateDirect(640*480/4); //We have (width/2*height/2) pixels, each pixel is 2 bytes
    vBuffer = ByteBuffer.allocateDirect(640*480/4); //We have (width/2*height/2) pixels, each pixel is 2 bytes

Then getting the datas

yBuffer.put(frame.getData(), 0, size);
yBuffer.position(0);
//YV12 : Y(8 bytes) then V(2 bytes) then U(2 bytes)
vBuffer.put(frame.getData(), size, size/4);
vBuffer.position(0);
uBuffer.put(frame.getData(), size  * 5 / 4, size/4);
uBuffer.position(0);

Now, prepare the texture : 

yTexture = new Texture(640, 480, Pixmap.Format.Intensity); //A 8-bit per pixel format
uTexture = new Texture(640 / 2, 480 / 2, Pixmap.Format.Intensity); //A 8-bit per pixel format
vTexture = new Texture(640 / 2, 480 / 2, Pixmap.Format.Intensity); //A 8-bit per pixel format

And change a bit the binding because we now use 3 textures instead of two :

//Set texture slot 0 as active and bind our texture object to it
Gdx.gl.glActiveTexture(GL20.GL_TEXTURE0);
yTexture.bind();

//Y texture is (width*height) in size and each pixel is one byte;
//by setting GL_LUMINANCE, OpenGL puts this byte into R,G and B
//components of the texture
Gdx.gl.glTexImage2D(GL20.GL_TEXTURE_2D, 0, GL20.GL_LUMINANCE,
        640, 480, 0, GL20.GL_LUMINANCE, GL20.GL_UNSIGNED_BYTE, yBuffer);

//Use linear interpolation when magnifying/minifying the texture to
//areas larger/smaller than the texture size
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_MIN_FILTER, GL20.GL_LINEAR);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_MAG_FILTER, GL20.GL_LINEAR);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_WRAP_S, GL20.GL_CLAMP_TO_EDGE);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_WRAP_T, GL20.GL_CLAMP_TO_EDGE);

/*
* Prepare the UV channel texture
*/

//Set texture slot 1 as active and bind our texture object to it
Gdx.gl.glActiveTexture(GL20.GL_TEXTURE1);
uTexture.bind();

Gdx.gl.glTexImage2D(GL20.GL_TEXTURE_2D, 0, GL20.GL_LUMINANCE,
        640 / 2, 480 / 2, 0, GL20.GL_LUMINANCE, GL20.GL_UNSIGNED_BYTE,
        uBuffer);

//Use linear interpolation when magnifying/minifying the texture to
//areas larger/smaller than the texture size
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_MIN_FILTER, GL20.GL_LINEAR);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_MAG_FILTER, GL20.GL_LINEAR);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_WRAP_S, GL20.GL_CLAMP_TO_EDGE);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_WRAP_T, GL20.GL_CLAMP_TO_EDGE);

//Set texture slot 1 as active and bind our texture object to it
Gdx.gl.glActiveTexture(GL20.GL_TEXTURE2);
vTexture.bind();

//UV texture is (width/2*height/2) Using GL_Luminance, each pixel will match a buffer component
Gdx.gl.glTexImage2D(GL20.GL_TEXTURE_2D, 0, GL20.GL_LUMINANCE,
        640 / 2, 480 / 2, 0, GL20.GL_LUMINANCE, GL20.GL_UNSIGNED_BYTE,
        vBuffer);

//Use linear interpolation when magnifying/minifying the texture to
//areas larger/smaller than the texture size
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_MIN_FILTER, GL20.GL_LINEAR);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_MAG_FILTER, GL20.GL_LINEAR);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_WRAP_S, GL20.GL_CLAMP_TO_EDGE);
Gdx.gl.glTexParameterf(GL20.GL_TEXTURE_2D,
        GL20.GL_TEXTURE_WRAP_T, GL20.GL_CLAMP_TO_EDGE);


shader.begin();

//Set the uniform y_texture object to the texture at slot 0
shader.setUniformi("y_texture", 0);

//Set the uniform uv_texture object to the texture at slot 1
shader.setUniformi("u_texture", 1);
shader.setUniformi("v_texture", 2);

mesh.render(shader, GL20.GL_TRIANGLES);

shader.end();

Last use the following shader (just changed a bit the fragment u and v texture part)

    //Our vertex shader code; nothing special
    String vertexShader =
            "attribute vec4 a_position;                         \n" +
                    "attribute vec2 a_texCoord;                         \n" +
                    "varying vec2 v_texCoord;                           \n" +

                    "void main(){                                       \n" +
                    "   gl_Position = a_position;                       \n" +
                    "   v_texCoord = a_texCoord;                        \n" +
                    "}                                                  \n";

    //Our fragment shader code; takes Y,U,V values for each pixel and calculates R,G,B colors,
    //Effectively making YUV to RGB conversion
    String fragmentShader =
            "#ifdef GL_ES                                       \n" +
                    "precision highp float;                             \n" +
                    "#endif                                             \n" +

                    "varying vec2 v_texCoord;                           \n" +
                    "uniform sampler2D y_texture;                       \n" +
                    "uniform sampler2D u_texture;                       \n" +
                    "uniform sampler2D v_texture;                       \n" +

                    "void main (void){                                  \n" +
                    "   float r, g, b, y, u, v;                         \n" +

                    //We had put the Y values of each pixel to the R,G,B components by GL_LUMINANCE,
                    //that's why we're pulling it from the R component, we could also use G or B
                    //see https://stackoverflow.com/questions/12130790/yuv-to-rgb-conversion-by-fragment-shader/17615696#17615696
                    //and https://stackoverflow.com/questions/22456884/how-to-render-androids-yuv-nv21-camera-image-on-the-background-in-libgdx-with-o
                    "   y = texture2D(y_texture, v_texCoord).r;         \n" +

                    //Since we use GL_LUMINANCE, each compoentn it on it own map
                    "   u = texture2D(u_texture, v_texCoord).r - 0.5;  \n" +
                    "   v = texture2D(v_texture, v_texCoord).r - 0.5;  \n" +


                    //The numbers are just YUV to RGB conversion constants
                    "   r = y + 1.13983*v;                              \n" +
                    "   g = y - 0.39465*u - 0.58060*v;                  \n" +
                    "   b = y + 2.03211*u;                              \n" +

                    //We finally set the RGB color of our pixel
                    "   gl_FragColor = vec4(r, g, b, 1.0);              \n" +
                    "}                                                  \n";

Here it is !

Community
  • 1
  • 1
Vince
  • 70
  • 8