OpenGL stencil buffer concave polygon

Question

I'm trying to do this example.

I couldn't understand how it works still and I'm having difficulties in visualizing how the code would look like.

I stumbled upon this post which was the closest I think I could find.

From the redbook link, how can I do a "two-pass algorithm" to get the desired output? How do I use GL_INVERT and check if the pixel is covered even number of times?

I somehow managed to understand how the stencil works for the concave polygon but my problem now is the polygon is being rendered incorrectly.

http://img.photobucket.com/albums/v442/ardo/ScreenShot2013-12-30at24155PM.png

The location of where I am drawing this is in a framebuffer which is set up like this:

            //multisample
            glGenRenderbuffersEXT(1, &colorBuffer);
            glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, colorBuffer);
            glRenderbufferStorageMultisampleEXT(GL_RENDERBUFFER_EXT, multisampling, GL_RGBA,800, 600);

            //multi sample depth
            glGenRenderbuffersEXT(1, &depthBuffer);
            glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, depthBuffer);
            glRenderbufferStorageMultisampleEXT(GL_RENDERBUFFER_EXT, multisampling, GL_DEPTH24_STENCIL8, 800, 600);

            //multisamplefbo
            glGenFramebuffersEXT(1, &mFBO);
            glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mFBO);
            glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT, GL_RENDERBUFFER_EXT, colorBuffer);
            glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER_EXT, depthBuffer);

Then my drawing code is:

glEnable(GL_STENCIL_TEST);         
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE); 
glStencilFunc(GL_ALWAYS, 0x1, 0x1);
glStencilOp(GL_KEEP, GL_INVERT, GL_INVERT);

glBegin(GL_TRIANGLE_FAN);

glVertex3f(1.396900,3.130690,0);
glVertex3f(2.034830,2.466900,0);
glVertex3f(2.486338,2.441036,0);
glVertex3f(2.802204,2.437803,0);
glVertex3f(2.910181,2.447098,0);
glVertex3f(2.957240,2.466900,0);
glVertex3f(3.019335,2.552570,0);
glVertex3f(3.130733,2.673799,0);
glVertex3f(3.284830,2.828970,0);
glVertex3f(3.474490,3.087590,0);
glVertex3f(4.319320,2.596210,0);
glVertex3f(4.508980,2.147930,0);
glVertex3f(4.714798,1.866680,0);
glVertex3f(4.907215,1.620586,0);
glVertex3f(5.101651,1.397930,0);
glVertex3f(5.186114,1.317364,0);
glVertex3f(5.254869,1.269024,0);
glVertex3f(5.302510,1.261700,0);
glVertex3f(5.323633,1.304180,0);
glVertex3f(5.312832,1.405254,0);
glVertex3f(5.264703,1.573711,0);
glVertex3f(5.173841,1.818340,0);
glVertex3f(5.034840,2.147930,0);
glVertex3f(4.875695,2.492816,0);
glVertex3f(4.727900,2.780675,0);
glVertex3f(4.591307,3.016304,0);
glVertex3f(4.465765,3.204503,0);
glVertex3f(4.351127,3.350070,0);
glVertex3f(4.247241,3.457803,0);
glVertex3f(4.153958,3.532502,0);
glVertex3f(4.071129,3.578965,0);
glVertex3f(3.998605,3.601990,0);
glVertex3f(3.936235,3.606376,0);
glVertex3f(3.841363,3.578427,0);
glVertex3f(3.766900,3.510000,0);
glVertex3f(2.974490,3.458280,0);
glVertex3f(2.672770,2.958280,0);
glVertex3f(2.379670,3.070350,0);
glVertex3f(1.853810,3.303110,0);
glVertex3f(1.396910,3.303110,0);
glVertex3f(1.396910,3.130690,0);
glVertex3f(1.396900,3.130690,0);
glVertex3f(1.396900,3.130690,0);

   glEnd();


glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); 
glStencilFunc(GL_EQUAL, 0x1, 0x1);                  
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
glColor3f(1,0,1);
glBegin(GL_TRIANGLE_FAN);

glVertex3f(1.396900,3.130690,0);
glVertex3f(2.034830,2.466900,0);
glVertex3f(2.486338,2.441036,0);
glVertex3f(2.802204,2.437803,0);
glVertex3f(2.910181,2.447098,0);
glVertex3f(2.957240,2.466900,0);
glVertex3f(3.019335,2.552570,0);
glVertex3f(3.130733,2.673799,0);
glVertex3f(3.284830,2.828970,0);
glVertex3f(3.474490,3.087590,0);
glVertex3f(4.319320,2.596210,0);
glVertex3f(4.508980,2.147930,0);
glVertex3f(4.714798,1.866680,0);
glVertex3f(4.907215,1.620586,0);
glVertex3f(5.101651,1.397930,0);
glVertex3f(5.186114,1.317364,0);
glVertex3f(5.254869,1.269024,0);
glVertex3f(5.302510,1.261700,0);
glVertex3f(5.323633,1.304180,0);
glVertex3f(5.312832,1.405254,0);
glVertex3f(5.264703,1.573711,0);
glVertex3f(5.173841,1.818340,0);
glVertex3f(5.034840,2.147930,0);
glVertex3f(4.875695,2.492816,0);
glVertex3f(4.727900,2.780675,0);
glVertex3f(4.591307,3.016304,0);
glVertex3f(4.465765,3.204503,0);
glVertex3f(4.351127,3.350070,0);
glVertex3f(4.247241,3.457803,0);
glVertex3f(4.153958,3.532502,0);
glVertex3f(4.071129,3.578965,0);
glVertex3f(3.998605,3.601990,0);
glVertex3f(3.936235,3.606376,0);
glVertex3f(3.841363,3.578427,0);
glVertex3f(3.766900,3.510000,0);
glVertex3f(2.974490,3.458280,0);
glVertex3f(2.672770,2.958280,0);
glVertex3f(2.379670,3.070350,0);
glVertex3f(1.853810,3.303110,0);
glVertex3f(1.396910,3.303110,0);
glVertex3f(1.396910,3.130690,0);
glVertex3f(1.396900,3.130690,0);
glVertex3f(1.396900,3.130690,0);

    glEnd();

glDisable(GL_STENCIL_TEST);

The way I have my multisampling set up is here (This is before the drawing is written)

[self checkError];
        glEnable(GL_DEPTH_TEST);
        [self checkError];
        glDepthMask(GL_TRUE);
        [self checkError];
        glDepthFunc(GL_LESS);
        [self checkError];
        glClearDepth(10000.0);
        [self checkError];
        glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
        [self checkError];

        if(multisampling != 0){
            //Set multisampled framebuffer
            glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mFBO);
            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

            //Antialiasing functions
            glEnable (GL_POLYGON_SMOOTH);
            glEnable (GL_BLEND);
            glBlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
            glHint (GL_POLYGON_SMOOTH_HINT, GL_NICEST);
            glLineWidth (1.5);
        }

        glDisable(GL_CULL_FACE);

And my blit call:

glBindFramebufferEXT(GL_READ_FRAMEBUFFER_EXT, mFBO);
            glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER_EXT, currentFramebuffer);

            glBlitFramebuffer(0, 0, 800, 600, 0, 0, 800, 600, GL_COLOR_BUFFER_BIT, GL_NEAREST);

            glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mFBO);
            glReadBuffer(GL_COLOR_ATTACHMENT0_EXT);

            glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER_EXT, currentFramebuffer);
            glBindFramebufferEXT(GL_READ_FRAMEBUFFER_EXT, currentFramebuffer);
            glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, currentFramebuffer);

Smoothed polygons should use a different blending function than the one you are currently using. They accumulate the coverage in the destination alpha channel (thus you also need a pixel format that stores alpha), and `GL_SRC_ALPHA_SATURATE`, `GL_ONE` is optimal for smoothed polygons. Of course since hardware generally does not accelerate this, and you are already using multisampling, I would say simply stop enabling `GL_POLYGON_SMOOTH`. — Andon M. Coleman, Dec 31 '13 at 04:06
@AndonM.Coleman I tried removing the GL_POLYGON_SMOOTH but i still end up with this: http://img.photobucket.com/albums/v442/ardo/ScreenShot2013-12-31at93129AM.png — ardowz, Dec 31 '13 at 17:32
I did not mean to imply that would fix your problem... only that it was a problem. Does using a single-sampled render buffer change anything, by the way? — Andon M. Coleman, Dec 31 '13 at 17:38
@AndonM.Coleman Oh I see, I tried it without multisampling and lowered multisampling but the problem still persists. — ardowz, Dec 31 '13 at 17:51

score 0 · Answer 1 · answered Dec 30 '13 at 17:48

What don't you get? The example starts out with a stencil buffer cleared to 0x00 and each time you draw a fragment (whether it passes or fails a depth/stencil test) it performs a bitwise inversion of the stencil buffer (e.g. ~0x00 --> 0xff). If you do this an odd number of times, the stencil buffer will be non-zero, but if you have an even number of fragments it will be zero.

Some things that might be confusing are the use of 1 as the bitmask for the stencil test and the stencil op in the second pass. This effectively limits the test to a single bit. In other words, the bit flipping I mentioned earlier will only happen for bit 1. Thus, you can actually simplify your test for even/odd to a test of a single bit... if the stencil buffer stores a value of 1 then you drew an odd number of fragments. If it stores a value of 0 then you drew an even number.

The second pass in your example actually does the exact stencil test I described. It tests the stencil buffer for 1 and fails the stencil test if the first bit != 1.

// The stencil op below is a convoluted way of clearing the stencil buffer
glStencilOp   (GL_ZERO,  GL_ZERO, GL_ZERO);
glStencilFunc (GL_EQUAL, 1,       1);       // Test: (Stencil & 1) == 1

I updated my question with what my output is. I'm still confused with the whole stencil thing and here is what I came up with. But Its not rendering correctly. — ardowz, Dec 30 '13 at 22:53
@ardowz: Why did you change the stencil op. to `GL_KEEP, GL_KEEP, GL_KEEP`? And where in your code are you actually setting the stencil clear value and clearing the stencil buffer? Your updated code adds more questions than it answers. Can you include more of your stencil buffer setup code, as well as where you do the MSAA resolve blit (e.g. `glBlitFramebuffer (...)`), etc. — Andon M. Coleman, Dec 30 '13 at 23:36
I was trying with and without the stencilOp change. Somehow both yields different results, with the stencilOp it provides a similar output and without the stencilOp, it provides a somewhat cleaner output: — ardowz, Dec 31 '13 at 00:05

OpenGL stencil buffer concave polygon

1 Answers1