Been following questions like this one (Calculating normals for a height map) to calculate the normals across a noisy terrain. Because I'm doing a planet, I figured I could multiply by the TBN matrix (using tangent, bitangent and normal from original mesh, before displacement) to calculate this accurately. The results are... wrong. Here is my tessalation evaluation shader code:
vec2 pointCoord = outputVertex.textureCoord;
//Use the original TBN matrix to calculate this properly
float heightL = getNoiseAtPoint(pointCoord + vec2(-delta, 0));
float heightR = getNoiseAtPoint(pointCoord + vec2(delta, 0));
float heightU = getNoiseAtPoint(pointCoord + vec2( 0, delta));
float heightD = getNoiseAtPoint(pointCoord + vec2( 0, -delta));
vec3 tangentNormal = normalize(vec3(heightL - heightR, 2.0f, heightU - heightD));
vec3 T = normalize(interpolate3D(inputVertex[0].tangent, inputVertex[1].tangent, inputVertex[2].tangent));
vec3 B = normalize(interpolate3D(inputVertex[0].bitangent, inputVertex[1].bitangent, inputVertex[2].bitangent));
mat3 TBN = mat3(T, normalize(vecToOrigin), B);
vec3 worldNormal = normalize( TBN * tangentNormal);
outputVertex.normal = worldNormal;
vec3 debugColour;
debugColour.x = ((2.0 * worldNormal.x) - 1.0);
debugColour.y = ((2.0 * worldNormal.y) - 1.0);
debugColour.z = ((2.0 * worldNormal.z) - 1.0);
outputVertex.debugTexture = debugColour;
I've been struggling with this for a while now, and getting consistently wrong results. Here is what I'm seeing:
And here's the worldNormals visualized as rgb:
Same two pictures, but this time from the perspective of the light. Strikes me there's a lot of blue (z) across the whole sphere...
Anyone got any ideas? I'm fairly sure my original tangents and bitangents are correct as if I remove displacement I can do regular normal mapping just fine Thanks
EDIT: correct debug colours being in wrong range, uploaded the corresponding photos etc
EDIT 2 - vertex shader-
void calculateTBN()
{
vec4 tan = tangent / tangent.w;
vec3 T = normalize(vec3(transform * tan));
vec4 bitan = bitangent / bitangent.w;
vec3 B = normalize(vec3(transform * bitan));
vec3 N = normalize(vec3(normalMat * vec4(normal, 0.0).xyz));
TBN = mat3(T, B, N);
outputVertex.TBN = TBN;
outputVertex.tangent = T;
outputVertex.bitangent = B;
}
void main(void) {
//Calculate transform matrix
transform = T * R * S;
//Calculate pos
pos = vec4(position.x, position.y, position.z, 1.0) + posRelParent;
//Pass through the texture coords
outputVertex.textureCoord = textureCoord;
//Transform matrix dealt with
pos = transform * pos;
//Output vertex pos
outputVertex.vert = pos.xyz;
//Work out the normal for lighting
normalMat = transpose(inverse(mat3(transform)));
outputVertex.normal = normalize(normalMat * normal);
calculateTBN();
gl_Position = pos;
}