So I have this function for getting the intersection of a ray with an OBB:
std::optional<float> Ray::hitsOBB(const glm::vec3& min, const glm::vec3& max, const glm::mat4& modelMatrix) {
float tMin = 0.0f;
float tMax = 100000.0f;
glm::vec3 OBBposition_worldspace(modelMatrix[3].x, modelMatrix[3].y, modelMatrix[3].z);
glm::vec3 delta = OBBposition_worldspace - origin;
{
glm::vec3 xaxis(modelMatrix[0].x, modelMatrix[0].y, modelMatrix[0].z);
float e = glm::dot(xaxis, delta);
float f = glm::dot(direction, xaxis);
if (fabs(f) > 0.001f) {
float t1 = (e + min.x) / f;
float t2 = (e + max.x) / f;
if (t1 > t2) std::swap(t1, t2);
if (t2 < tMax) tMax = t2;
if (t1 > tMin) tMin = t1;
if (tMin > tMax) return {};
}
else {
if (-e + min.x > 0.0f || -e + max.x < 0.0f) return {};
}
}
{
glm::vec3 yaxis(modelMatrix[1].x, modelMatrix[1].y, modelMatrix[1].z);
float e = glm::dot(yaxis, delta);
float f = glm::dot(direction, yaxis);
if (fabs(f) > 0.001f) {
float t1 = (e + min.y) / f;
float t2 = (e + max.y) / f;
if (t1 > t2) std::swap(t1, t2);
if (t2 < tMax) tMax = t2;
if (t1 > tMin) tMin = t1;
if (tMin > tMax) return {};
}
else {
if (-e + min.y > 0.0f || -e + max.y < 0.0f) return {};
}
}
{
glm::vec3 zaxis(modelMatrix[2].x, modelMatrix[2].y, modelMatrix[2].z);
float e = glm::dot(zaxis, delta);
float f = glm::dot(direction, zaxis);
if (fabs(f) > 0.001f) {
float t1 = (e + min.z) / f;
float t2 = (e + max.z) / f;
if (t1 > t2) std::swap(t1, t2);
if (t2 < tMax) tMax = t2;
if (t1 > tMin) tMin = t1;
if (tMin > tMax) return {};
}
else {
if (-e + min.z > 0.0f || -e + max.z < 0.0f) return {};
}
}
return tMin;
}
I use it to click on cubes in OpenGL. The test case is a cube with min and max of (-1, -1, -1) and (1, 1, 1). Now this works fine when rotating and translating the modelMatrix but it does not seem to take scale into account. I've tried pre-multiplying the min and max with the modelMatrix scale but to no avail. Anyone know what's going on?