How to correctly make a depth cubemap for shadow mapping?

Question

I have written code to render my scene objects to a cubemap texture of format GL_DEPTH_COMPONENT and then use this texture in a shader to determine whether a fragment is being directly lit or not, for shadowing purposes. However, my cubemap appears to come out as black. I suppose I am not setting up my FBO or rendering context sufficiently, but fail to see what is missing.

Using GL 3.3 in compatibility profile.

This is my code for creating the FBO and cubemap texture:

    glGenFramebuffers(1, &fboShadow);
    glGenTextures(1, &texShadow);
    glBindTexture(GL_TEXTURE_CUBE_MAP, texShadow);
    for (int sideId = 0; sideId < 6; sideId++) {
        // Make sure GL knows what this is going to be.
        glTexImage2D(GL_TEXTURE_CUBE_MAP_POSITIVE_X + sideId, 0, GL_DEPTH_COMPONENT, 512, 512, 0, GL_DEPTH_COMPONENT, GL_FLOAT, NULL);
    }

    // Don't interpolate depth value sampling. Between occluder and occludee there will
    // be an instant jump in depth value, not a linear transition.
    glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_CUBE_MAP, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE);

    glBindTexture(GL_TEXTURE_CUBE_MAP, 0);

My full rendering function then looks like so:

void render() {

    // --- MAKE DEPTH CUBEMAP ---

    // Set shader program for depth testing
    glUseProgram(progShadow);

    // Get the light for which we want to generate a depth cubemap
    PointLight p = pointLights.at(0);

    // Bind our framebuffer for drawing; clean it up
    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fboShadow);
    glClear(GL_DEPTH_BUFFER_BIT);

    // Make 1:1-ratio, 90-degree view frustum for a 512x512 texture.
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    gluPerspective(90.0, 1, 16.0, 16384.0);
    glViewport(0, 0, 512, 512);
    glMatrixMode(GL_MODELVIEW);

    // Set modelview and projection matrix uniforms
    setShadowUniforms();

    // Need 6 renderpasses to complete each side of the cubemap
    for (int sideId = 0; sideId < 6; sideId++) {
        // Attach depth attachment of current framebuffer to level 0 of currently relevant target of texShadow cubemap texture.
        glFramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_CUBE_MAP_POSITIVE_X + sideId, texShadow, 0);

        // All is fine.
        GLenum status = glCheckFramebufferStatus(GL_DRAW_FRAMEBUFFER);
        if (status != GL_FRAMEBUFFER_COMPLETE) {
            std::cout << "Shadow FBO is broken with code " << status << std::endl;
        }

        // Push modelview matrix stack because we need to rotate and move camera every time
        glPushMatrix();

        // This does a switch-case with glRotatefs
        rotateCameraForSide(GL_TEXTURE_CUBE_MAP_POSITIVE_X + sideId);

        // Render from light's position.
        glTranslatef(-p.getX(), -p.getY(), -p.getZ());

        // Render all objects.
        for (ObjectList::iterator it = objectList.begin(); it != objectList.end(); it++) {
            (*it)->render();
        }

        glPopMatrix();
    }


    // --- RENDER SCENE ---

    // Bind default framebuffer
    glBindFramebuffer(GL_FRAMEBUFFER, 0);

    // Setup proper projection matrix with 70 degree vertical FOV and ratio according to window frame dimensions.
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();
    gluPerspective(70.0, ((float)vpWidth) / ((float)vpHeight), 16.0, 16384.0);
    glViewport(0, 0, vpWidth, vpHeight);

    glUseProgram(prog);

    glMatrixMode(GL_MODELVIEW);
    glPushMatrix();

    applyCameraPerspective();

    // My PointLight class has both a position (world space) and renderPosition (camera space) Vec3f variable;
    // The lights' renderPositions get transformed with the modelview matrix by this.
    updateLights();

    // And here, among other things, the lights' camera space coordinates go to the shader.
    setUniforms();

    // Render all objects
    for (ObjectList::iterator it = objectList.begin(); it != objectList.end(); it++) {

        // Object texture goes to texture unit 0
        GLuint usedTexture = glTextureList.find((*it)->getTextureName())->second;
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, usedTexture);
        glUniform1i(textureLoc, 0);

        // Cubemap goes to texture unit 1
        glActiveTexture(GL_TEXTURE1);
        glBindTexture(GL_TEXTURE_CUBE_MAP, texShadow);
        glUniform1i(shadowLoc, 1);

        (*it)->render();
    }

    glPopMatrix();
    frameCount++;
}

The shader program for rendering depth values ("progShadow") is simple.

Vertex shader:

#version 330

in vec3 position;

uniform mat4 modelViewMatrix, projectionMatrix;

void main() {
    gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1);
}

Fragment shader:

#version 330

void main() {
    // OpenGL sets the depth anyway. Nothing to do here.
}

The shader program for final rendering ("prog") has a fragment shader which looks something like this:

#version 330

#define MAX_LIGHTS 8

in vec3 fragPosition;
in vec3 fragNormal;
in vec2 fragTexCoordinates;
out vec4 fragColor;

uniform sampler2D colorTexture;
uniform samplerCubeShadow shadowCube;

uniform uint activeLightCount;

struct Light {
    vec3 position;
    vec3 diffuse;
    float cAtt;
    float lAtt;
    float qAtt;
};

// Index 0 to (activeLightCount - 1) need to be the active lights.
uniform Light lights[MAX_LIGHTS];

void main() {
    vec3 lightColor = vec3(0, 0, 0);
    vec3 normalFragmentToLight[MAX_LIGHTS];
    float distFragmentToLight[MAX_LIGHTS];
    float distEyeToFragment = length(fragPosition);

    // Accumulate all light in "lightColor" variable
    for (uint i = uint(0); i < activeLightCount; i++) {
        normalFragmentToLight[i] = normalize(lights[i].position - fragPosition);
        distFragmentToLight[i] = distance(fragPosition, lights[i].position);
        float attenuation = (lights[i].cAtt
            + lights[i].lAtt * distFragmentToLight[i]
            + lights[i].qAtt * pow(distFragmentToLight[i], 2.0));

        float dotProduct = dot(fragNormal, normalFragmentToLight[i]);
        lightColor += lights[i].diffuse * max(dotProduct, 0.0) / attenuation;
    }

    // Shadow mapping only for light at index 0 for now.
    float distOccluderToLight = texture(shadowCube, vec4(normalFragmentToLight[0], 1));

    // My geometries use inches as units, hence a large bias of 1
    bool isLit = (distOccluderToLight + 1) < distFragmentToLight[0];
    fragColor = texture2D(colorTexture, fragTexCoordinates) * vec4(lightColor, 1.0f) * int(isLit);
}

I have verified that all uniform location variables are set to a proper value (i.e. not -1).

It might be worth noting I do no call to glBindFragDataLocation() for "progShadow" prior to linking it, because no color value should be written by that shader.

See anything obviously wrong here?

Answer 1

For shadow maps, depth buffer internal format is pretty important (too small and things look awful, too large and you eat memory bandwidth). You should use a sized format (e.g. GL_DEPTH_COMPONENT24) to guarantee a certain size, otherwise the implementation will pick whatever it wants. As for debugging a cubemap shadow map, the easiest thing to do is actually to draw the scene into each cube face and output color instead of depth. Then, where you currently try to use the cubemap to sample depth, write the sampled color to fragColor instead. You can rule out view issues immediately this way.

There is another much more serious issue, however. You are using samplerCubeShadow, but you have not set GL_TEXTURE_COMPARE_MODE for your cube map. Attempting to sample from a depth texture with this sampler type and without GL_TEXTURE_COMPARE_MODE = GL_COMPARE_REF_TO_TEXTURE will produce undefined results. Even if you did have this mode set properly, the 4th component of the texture coordinates are used as the depth comparison reference -- a constant value of 1.0 is NOT what you want.

Likewise, the depth buffer does not store linear distance, you cannot directly compare the distance you computed here:

distFragmentToLight[i] = distance(fragPosition, lights[i].position);

Instead, something like this will be necessary:

float VectorToDepth (vec3 Vec)
{
    vec3 AbsVec = abs(Vec);
    float LocalZcomp = max(AbsVec.x, max(AbsVec.y, AbsVec.z));

    // Replace f and n with the far and near plane values you used when
    //   you drew your cube map.
    const float f = 2048.0;
    const float n = 1.0;

    float NormZComp = (f+n) / (f-n) - (2*f*n)/(f-n)/LocalZcomp;
    return (NormZComp + 1.0) * 0.5;
}

float LightDepth    = VectorToDepth (fragPosition - lights [i].position);
float depth_compare = texture(shadowCube,vec4(normalFragmentToLight[0],LightDepth));

^{* Code for float VectorToDepth (vec3 Vec)borrowed from Omnidirectional shadow mapping with depth cubemap}

Now depth_compare will be a value between 0.0 (completely in shadow) and 1.0 (completely out of shadow). If you have linear texture filtering enabled, the hardware will sample the depth at 4 points and may give you a form of 2x2 PCF filtering. If you have nearest texture filtering, then it will either be 1.0 or 0.0.