How to use a stencil buffer with an alpha mask

Question

I am trying to use the stencil buffer with an alpha mask to prevent drawing of part of a texture.

Init Code

if(SDL_Init(SDL_INIT_EVERYTHING) < 0) {
    fprintf(stderr,"%s:%d\n    SDL_Init call failed.\n",__FILE__,__LINE__);
    return false;
}

// Request use of the stencil buffer
SDL_GL_SetAttribute( SDL_GL_STENCIL_SIZE, 1 );

if((Surf_Display = SDL_SetVideoMode(WWIDTH, WHEIGHT, 32,  SDL_GL_DOUBLEBUFFER | SDL_OPENGL | SDL_RESIZABLE)) == NULL) {
    fprintf(stderr,"%s:%d\n    SDL_SetVideoMode call failed.\n",__FILE__,__LINE__);
    return false;
}

// Init GLDebug system
GLDebug::Init();

// Init GL system
glClearColor(0, 0, 0, 1);

glClearDepth(1.0f);

glViewport(0, 0, WWIDTH, WHEIGHT);

glMatrixMode(GL_PROJECTION);

glLoadIdentity();

glOrtho(0, WWIDTH,  WHEIGHT, 0, 1, -1);

glMatrixMode(GL_MODELVIEW);

glHint(GL_POINT_SMOOTH_HINT, GL_NICEST);
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST);
glHint(GL_POLYGON_SMOOTH_HINT, GL_NICEST);

glEnable(GL_POINT_SMOOTH);
glEnable(GL_LINE_SMOOTH);
glEnable(GL_POLYGON_SMOOTH);

glLoadIdentity();

Source

glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

// Enable use of textures
glEnable(GL_TEXTURE_2D);

// Disable writing to any of the color fields
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glStencilOp(GL_KEEP, GL_KEEP, GL_INCR); 
glStencilFunc(GL_ALWAYS, 0,0);
glEnable(GL_ALPHA_TEST);
glEnable(GL_STENCIL_TEST);

glBindTexture(GL_TEXTURE_2D,(&StencilTex)[0]);

// Draw our stencil buffer texture
glBegin(GL_POLYGON);                                   
    glTexCoord2d( 0, 0 );    glVertex2f( 50,     50 );
    glTexCoord2d( 0, 1 );    glVertex2f( 50,     50+128 );
    glTexCoord2d( 1, 1 );    glVertex2f( 50+128, 50+128 );
    glTexCoord2d( 1, 0 );    glVertex2f( 50+128, 50 );
glEnd();

glDisable(GL_ALPHA_TEST);

// Re enable drawing of colors
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);

glStencilFunc(GL_GREATER, 0, -1);

// Bind desired texture for drawing
glBindTexture(GL_TEXTURE_2D,(&texture)[0]);

// Draw the box with colors (this should be masked off)
glBegin(GL_QUADS);                                   
    glTexCoord2d( 0, 0 );    glVertex2f( 50,     50 );
    glTexCoord2d( 0, 1 );    glVertex2f( 50,     50+128 );
    glTexCoord2d( 1, 1 );    glVertex2f( 50+128, 50+128 );
    glTexCoord2d( 1, 0 );    glVertex2f( 50+128, 50 );
glEnd();
glDisable(GL_STENCIL_TEST);

// Swap buffers and display!
SDL_GL_SwapBuffers();

I am going off the reference from this question as posted by Tim. The setup I have works for raw drawing of pixels, but I cannot either A) Make heads or tails of how this process is supposed to work or B) get it to work from an alpha texture. All I get is a black screen, or the full texture.

Anyone care to break this into someone reasonable steps? I have a vague understanding of how the glStencilOp and glStencilFunc options work, but their relationship to the use of an alpha test and stencil test baffle me. How do we decide to keep the alpha layer only from the texture?

To review:

• I have a texture that I want to draw to a scene, but I need to block of very specific areas off.
• The alpha texture of the object cannot be modified at run time to achieve this effect.
• I am using fixed pipeline, I plan to learn how to use VBOs later.
• The given code results in nothing being drawn when the second texture is called.

[Edit 0]

The accepted solution works! Here is the final code that does as I would expect.

glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

// // Enable use of textures
glEnable(GL_TEXTURE_2D);

// Disable writing to any of the color fields
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_TRUE);
glStencilOp(GL_KEEP, GL_KEEP, GL_INCR); 

glAlphaFunc(GL_GREATER, 0.05);
glStencilFunc(GL_ALWAYS, 0,0);

glEnable(GL_STENCIL_TEST);
glEnable(GL_ALPHA_TEST);

glBindTexture(GL_TEXTURE_2D,(&StencilTex)[0]);

// Draw our blocking poly
glBegin(GL_POLYGON);                                   
    glTexCoord2d( 0, 0 );    glVertex2f( 50,     50 );
    glTexCoord2d( 0, 1 );    glVertex2f( 50,     50+128 );
    glTexCoord2d( 1, 1 );    glVertex2f( 50+128, 50+128 );
    glTexCoord2d( 1, 0 );    glVertex2f( 50+128, 50 );
glEnd();
glDisable(GL_ALPHA_TEST);


// Re enable drawing of colors
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);

glStencilFunc(GL_LESS, 0, -1);

// Bind desired texture for drawing
glBindTexture(GL_TEXTURE_2D,(&texture)[0]);

// Draw the box with colors
glBegin(GL_QUADS);                                   
    glTexCoord2d( 0, 0 );    glVertex2f( 50,     50 );
    glTexCoord2d( 0, 1 );    glVertex2f( 50,     50+128 );
    glTexCoord2d( 1, 1 );    glVertex2f( 50+128, 50+128 );
    glTexCoord2d( 1, 0 );    glVertex2f( 50+128, 50 );
glEnd();
glDisable(GL_STENCIL_TEST);


// Swap buffers and display!
SDL_GL_SwapBuffers();

Answer 1

I see you have enabled alpha test, but I do not see any call to set the alpha function (glAlphaFunc)?

If you do not set the alpha function, the alpha test always passes. You must tell the state machine which pixels to discard.

Typically I would set it to something like:

glAlphaFunc(GL_GREATER, 0.05); //reject any pixels with alpha less than 0.05

I'm sorry if you didn't understand clearly from my previous answer, though if you are confused about how the stencil functions work you can always consult their reference pages glStencilFunc glStencilOp.