I'm attempting to convert camera frames which are in YUV 4.2.2 (NV12) format to YUV 4.2.0 (YV12/i420) format. The U and V channels of NV12 are interleaved, while the YV12 are not.
For performance reasons, I'm trying to do this conversion in an OpenGl ES 2.0 shader.
The shaders are fairly easy. I separate out the Y and the UV channels and then pass the UV to a shader that separates it out into a U channel in gl_FragData[0] and V channel in gl_FragData[1]
Here is the Fragment Shader:
precision mediump float;
uniform sampler2D t_texture_uv;
varying highp vec2 uvVarying;
void main()
{
/*
vec2 uvu,uvv;
int x,y;
x = uvVarying.x;
y = uvVarying.y;
uvu = vec2(x*2,y);
uvv = vec2((x*2)+1,y);
gl_FragData[0] = texture2D(t_texture_uv, uvu);
gl_FragData[1] = texture2D(t_texture_uv, uvv);
*/
// this is the same as the logic above, but is less processor intensive
gl_FragData[0] = texture2D(t_texture_uv, vec2( uvVarying.x*2 ,uvVarying.y)); // u-plane
gl_FragData[1] = texture2D(t_texture_uv, vec2((uvVarying.x*2)+1,uvVarying.y)); // v-plane
}
And the vertex shader:
attribute vec2 position;
attribute vec2 uv;
uniform mat4 proj_matrix;
varying vec2 uvVarying;
void main()
{
gl_Position = proj_matrix * vec4(vec3(position.xy, 0.0), 1.0);
uvVarying = uv;
}
My problem is that I'm lost at how to retrieve the data from the Gl_FragData buffers back in my Objective C code. I tried glReadPixels, but it doesn't seem to work.
This is my code, but I think I've gone down the wrong path.
- (void) nv12_to_yv12_converter:(unsigned char*)YBuf UVBuff:(unsigned char*)UVBuf Width:(GLsizei)UVWidth Height:(GLsizei)UVHeight {
unsigned char *Ubufout;
unsigned char *Vbufout;
EAGLContext *context;
context = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
if (!context || ![EAGLContext setCurrentContext:context]) {
NSLog(@"Failed to create ES context");
}
[self loadShaders];
glGenFramebuffers(1, &_framebuffer);
glGenRenderbuffers(1, &_renderbuffer);
glBindFramebuffer(GL_FRAMEBUFFER, _framebuffer);
glBindRenderbuffer(GL_RENDERBUFFER, _renderbuffer);
;
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16, UVWidth, UVHeight);
// [context renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer*)self.view.layer];
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &_backingWidth);
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &_backingHeight);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _renderbuffer);
_vertices[0] = -1.0f; // x0
_vertices[1] = -1.0f; // y0
_vertices[2] = 1.0f; // ..
_vertices[3] = -1.0f;
_vertices[4] = -1.0f;
_vertices[5] = 1.0f;
_vertices[6] = 1.0f; // x3
_vertices[7] = 1.0f; // y3
int orientation = 0;
static const GLfloat texCoords[] = {
0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f,
};
glViewport(0, 0, _backingWidth, _backingHeight);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
glUseProgram(_program);
if (!validateProgram(_program))
{
NSLog(@"Failed to validate program");
return;
}
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
glDeleteTextures(1, &_textures[0]);
glActiveTexture(GL_TEXTURE0);
glGenTextures(1, &_textures[0]);
glBindTexture(GL_TEXTURE_2D, _textures[0]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glEnable(GL_TEXTURE_2D);
glUniform1i(glGetUniformLocation(_program, "t_texture_uv"), 0);
glTexImage2D(GL_TEXTURE_2D,
0,
GL_LUMINANCE,
16,
8,
0,
GL_LUMINANCE,
GL_UNSIGNED_BYTE,
UVBuf);
_uniformMatrix = glGetUniformLocation(_program, "proj_matrix");
GLfloat modelviewProj[16];
mat4f_LoadOrtho(-1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, modelviewProj);
glUniformMatrix4fv(_uniformMatrix, 1, GL_FALSE, modelviewProj);
//draw RECT
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, _vertices);
glEnableVertexAttribArray(ATTRIB_VERTEX);
//ATTRIB_TEXTUREPOSITON
glVertexAttribPointer(ATTRIB_UV, 2, GL_FLOAT, 0, 0, texCoords);
glEnableVertexAttribArray(ATTRIB_UV);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
glReadPixels(0, 0, UVWidth, UVHeight, GL_COLOR_ATTACHMENT0, GL_UNSIGNED_BYTE, &Ubufout);
glReadPixels(0, 0, UVWidth, UVHeight, GL_COLOR_ATTACHMENT0+1, GL_UNSIGNED_BYTE, &Vbufout);
NSLog(@"Complete");
}
Can anyone shed some light on how to retrieve the gl_FragData buffers?