Perspective projection and view matrix: Both depth buffer and triangle face orientation are reversed in OpenGL

Question

I am having trouble with my scene in OpenGL. Objects that are supposed to be further away are drawn closer etc AND front facing triangles are being culled instead of back facing ones. They are drawn in the correct orientation as it is a package I have used before. I am convinced that it is something to do with my projection or veiwModel matrix. I can not see anything wrong with these though!

AV4X4FLOAT formProjMatrix(float FOVangle,float aspect,float nearz,float farz)
{
    AV4X4FLOAT A;

    A.m[0] = 1/(aspect*tanf(FOVangle/2));
    A.m[5] = 1/tanf(FOVangle/2);
    A.m[10] = farz/(farz-nearz);
    A.m[11] = -nearz*farz/(farz-nearz);
    A.m[14] = 1;
    return A;
}

AV4X4FLOAT formViewModelMatrix(AV4FLOAT pos,AV4FLOAT target,AV4FLOAT up)
{ 
    AV4X4FLOAT M;
    AV4X4FLOAT R;
    AV4FLOAT u;
    AV4FLOAT v;
    AV4FLOAT W;

    W.x = -pos.x + target.x;
    W.y = -pos.y + target.y;
    W.z = -pos.z + target.z;

    W.w = 0;
    W.normalize();

    u.x = up.y*W.z-W.y*up.z;
    u.y = -up.x*W.z+W.x*up.z;
    u.z = up.x*W.y-W.x*up.y;
    u.w = 0;
    u.normalize();

    v.x = W.y*u.z-u.y*W.z;
    v.y = -W.x*u.z+u.x*W.z;
    v.z = W.x*u.y-u.x*W.y;
    v.w = 0;

    M.m[0]  = u.x;  M.m[1]  = u.y;  M.m[2]  = u.z;  M.m[3]  = 0;
    M.m[4]  = v.x;  M.m[5]  = v.y;  M.m[6]  = v.z;  M.m[7]  = 0;
    M.m[8]  = -W.x; M.m[9]  = -W.y; M.m[10] = -W.z; M.m[11] = 0;
    M.m[12] = 0;    M.m[13] = 0;    M.m[14] = 0;    M.m[15] = 1;

    R.m[0] = 1;
    R.m[5] = 1;
    R.m[10] = 1;
    R.m[15] = 1;
    R.m[12] = -pos.x;
    R.m[13] = -pos.y;
    R.m[14] = -pos.z;

//the opposite of what you expect because of the way we overload mult operator!
    M.display ();
    R.display ();
    return M*R;
}

This is what I call in my drawing routine.

glMatrixMode(GL_PROJECTION);
glLoadMatrixf(projMatrix.m);

glMatrixMode(GL_MODELVIEW);
glLoadMatrixf(viewModelMatrix.m);

Some otherinfo,

Yes, I have enabled depth testing!

Rabbid76 · Accepted Answer · 2018-03-25T21:33:41.913

There are some issues with in the calculation of the projection matrix. You have to adapt your code like this:

AV4X4FLOAT formProjMatrix(float FOVangle,float aspect,float nearz,float farz)
{
    AV4X4FLOAT A;

    A.m[0]  = 1.0 / (aspect*tanf(FOVangle/2));
    A.m[5]  = 1.0 / tanf(FOVangle/2);
    A.m[10] =  (nearz+farz)/(farz-nearz);
    A.m[11] = - 2.0 * nearz*farz/(farz-nearz);
    A.m[14] = - 1.0;
    return A;
}

The Perspective Projection Matrix looks like this:

r = right, l = left, b = bottom, t = top, n = near, f = far

2*n/(r-l)      0              0               0
0              2*n/(t-b)      0               0
(r+l)/(r-l)    (t+b)/(t-b)    -(f+n)/(f-n)   -1    
0              0              -2*f*n/(f-n)    0

it follows:

aspect = w / h
tanFov = tan( fov_y * 0.5 );

p[0][0] = 2*n/(r-l) = 1.0 / (tanFov * aspect)
p[1][1] = 2*n/(t-b) = 1.0 / tanFov

The following function will calculate the same projection matrix as gluPerspective or glm::perspective does:

#include <array>

const float cPI = 3.14159265f;
float ToRad( float deg ) { return deg * cPI / 180.0f; }

using TVec4  = std::array< float, 4 >;
using TMat44 = std::array< TVec4, 4 >;

TMat44 Perspective( float fov_y, float aspect )
{
    float fn = far + near
    float f_n = far - near;
    float r = aspect;
    float t = 1.0f / tan( ToRad( fov_y ) / 2.0f );

    return TMat44{ 
        TVec4{ t / r, 0.0f,  0.0f,                 0.0f },
        TVec4{ 0.0f,  t,     0.0f,                 0.0f },
        TVec4{ 0.0f,  0.0f, -fn / f_n,            -1.0f },
        TVec4{ 0.0f,  0.0f, -2.0f*far*near / f_n,  0.0f }
    };
}

On the viewport the X-axis points to the left, the Y-axis up and the Z-axis out of the view (Note in a right hand system the Z-Axis is the cross product of the X-Axis and the Y-Axis).

The following code does the same as gluLookAt or glm::lookAt does:

using TVec3  = std::array< float, 3 >;
using TVec4  = std::array< float, 4 >;
using TMat44 = std::array< TVec4, 4 >;

TVec3 Cross( TVec3 a, TVec3 b ) { return { a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0] }; }
float Dot( TVec3 a, TVec3 b ) { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2]; }
void Normalize( TVec3 & v )
{
    float len = sqrt( v[0] * v[0] + v[1] * v[1] + v[2] * v[2] );
    v[0] /= len; v[1] /= len; v[2] /= len;
}

TMat44 Camera::LookAt( const TVec3 &pos, const TVec3 &target, const TVec3 &up )
{ 
    TVec3 mz = { pos[0] - target[0], pos[1] - target[1], pos[2] - target[2] };
    Normalize( mz );
    TVec3 my = { up[0], up[1], up[2] };
    TVec3 mx = Cross( my, mz );
    Normalize( mx );
    my = Cross( mz, mx );

    TMat44 v{
        TVec4{ mx[0], my[0], mz[0], 0.0f },
        TVec4{ mx[1], my[1], mz[1], 0.0f },
        TVec4{ mx[2], my[2], mz[2], 0.0f },
        TVec4{ Dot(mx, pos), Dot(my, pos), -Dot(mz, pos), 1.0f }
    };

    return v;
}

Adapt your code like this:

AV4X4FLOAT formViewModelMatrix(AV4FLOAT pos,AV4FLOAT target,AV4FLOAT up)
{ 
    AV4FLOAT mz;
    mz.x = pos.x - target.x; mz.y = pos.y - target.y; mz.z = pos.z - target.z; mz.w = 1.0f;
    mz.normalize();

    AV4FLOAT my;
    my.x = up.x; my.y = up.y; my.z = up.z; my.w = 1.0f;

    AV4FLOAT mx;
    mx.x = my.y*mz.z - my.z*mz.y; mx.y = my.z*mz.x - my.x*mz.z; mx.z = my.x*mz.y - my.y*mz.x; mx.w = 1.0f;
    mx.normylize();

    my.x = mz.y*mx.z - mz.z*mx.y; my.y = mz.z*mx.x - mz.x*mx.z; my.z = mz.x*mx.y - mz.y*mx.x; my.w = 1.0f;

    AV4FLOAT t;
    t.x = mx.x*pos.x + mx.y*pos.y + mx.z*pos.z; 
    t.y = my.x*pos.x + my.y*pos.y + my.z*pos.z; 
    t.z = -(mz.x*pos.x + mz.y*pos.y + mz.z*pos.z); 

    AV4X4FLOAT m;
    m[0]  = mx.x;  m[1]  = my.x;  m[2]  = mz.x;  m[3]  = 0.0f;
    m[4]  = mx.y;  m[5]  = my.y;  m[6]  = mz.y;  m[7]  = 0.0f;
    m[8]  = mx.z;  m[9]  = my.z;  m[10] = mz.z;  m[11] = 0.0f;
    m[12] = t.x;   m[13] = t.y;   m[14] = t.z;   m[15] = 1.0f;

    return m
}

See further the answers to the following question:

Your use of images add a lot of depth (ha ha) to your answers:) — Brett Hale, Sep 15 '17 at 12:54
The images are a brilliant addition! I think Brett was joking — , Sep 15 '17 at 13:06
First and foremost, thank you! Now, I have updated my projection matrix and it rectifies the depth buffer issue. The problem that I am now having is that the orientation of the triangles is still the wrong way. If I modify my ModelView to that in the 3rd link in your answer I get total jibberish on screen. — , Sep 15 '17 at 13:15
Perhaps I should add that If I instead use glLookAt, everything is fine. — , Sep 15 '17 at 13:17

Perspective projection and view matrix: Both depth buffer and triangle face orientation are reversed in OpenGL

1 Answers1

Linked