I'm trying to create a see-through window effect in OpenGL. By means of a background quad, I render a background texture. After that, I render the see-through window texture on top and fill the stencil buffer with ones everywhere the window is. Finally, I render a mesh only where the stencil buffer has ones.
Here is my OpenGL initialization code and my Draw
method:
// OpenGL state
glClearColor(0.0f, 0.0f, 0.4f, 0.0f);
glEnable(GL_DEPTH_TEST);
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glEnable(GL_STENCIL_TEST);
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
// Draw method
auto virtual Draw() -> void override
{
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
// Update camera UBO
...
// Render background
glDisable(GL_STENCIL_TEST);
glDisable(GL_DEPTH_TEST);
prg_texture2D->UseProgram();
prg_texture2D->SetUniform("modelMatrix", glm::mat4());
glActiveTexture(GL_TEXTURE0);
tex_image->Bind();
backgroundQuad->Render();
tex_image->Unbind();
prg_texture2D->UnUseProgram();
glEnable(GL_DEPTH_TEST);
glEnable(GL_STENCIL_TEST);
// Render window
double x, y;
glfwGetCursorPos(m_window, &x, &y);
glViewport(x - 150, m_windowDimensions.y - y - 150, 300, 300); // see-through window size = 300
glClear(GL_STENCIL_BUFFER_BIT);
glStencilFunc(GL_ALWAYS, 1, 0xFF);
glStencilMask(0xFF);
glDisable(GL_DEPTH_TEST);
prg_texture2D->UseProgram();
prg_texture2D->SetUniform("modelMatrix", glm::mat4());
glActiveTexture(GL_TEXTURE0);
tex_window->Bind();
backgroundQuad->Render();
tex_window->Unbind();
prg_texture2D->UnUseProgram();
glEnable(GL_DEPTH_TEST);
backgroundQuad->UpdatePositions(vec_positions);
glViewport(0, 0, m_windowDimensions.x, m_windowDimensions.y);
// Update camera UBO
...
// Render skull
glStencilFunc(GL_EQUAL, 1, 0xFF);
glStencilMask(0x00);
prg_lighting->UseProgram();
skull->ResetModelMatrix();
skull->ScaleModel(glm::vec3(0.005f, 0.005f, 0.005f));
skull->RotateModel(glm::radians(180.0f), glm::vec3(0.0f, 1.0f, 0.0f));
prg_lighting->SetUniform("modelMatrix", skull->GetModelMatrix());
skull->Render(GL_TRIANGLES);
prg_lighting->UnUseProgram();
Without the calls to glViewport
, the see-through texture is streched across the whole screen, which is not what want as the window should be movable. However, in this case, the stencil functionality works fine and the mesh (which I can rotate, translate, etc.) is only visible if behind the see-through window.
With the calls to glViewport
, the behavious is very strange and the stencil test passes for every portion of the screen where the see-through window has been moved since the start of the program, i.e. if I move the see-through window over the entire screen, the mesh is visible all the time.
Instead of changing the viewport, I also tried to update the vertex positions of the background quad which I use to render the textures to the screen. However, the effect is exactly the same. Here is the code using the background quad update:
// Render window
double x, y;
glfwGetCursorPos(m_window, &x, &y);
float p = 0.1; // size of see-through window
std::array<glm::vec3, 6> vec_windowPositions = // All in the range of [-1, 1]
{
// First triangle
glm::vec3(2 * (x / m_windowDimensions.x - p / 2) - 1, 2 * ((m_windowDimensions.y - y) / m_windowDimensions.y - p / 2) - 1, 0.0f), // {0}
glm::vec3(2 * (x / m_windowDimensions.x + p / 2) - 1, 2 * ((m_windowDimensions.y - y) / m_windowDimensions.y - p / 2) - 1, 0.0f), // {1}
glm::vec3(2 * (x / m_windowDimensions.x + p / 2) - 1, 2 * ((m_windowDimensions.y - y) / m_windowDimensions.y + p / 2) - 1, 0.0f), // {2}
// Second triangle
glm::vec3(2 * (x / m_windowDimensions.x + p / 2) - 1, 2 * ((m_windowDimensions.y - y) / m_windowDimensions.y + p / 2) - 1, 0.0f), // {2}
glm::vec3(2 * (x / m_windowDimensions.x - p / 2) - 1, 2 * ((m_windowDimensions.y - y) / m_windowDimensions.y + p / 2) - 1, 0.0f), // {3}
glm::vec3(2 * (x / m_windowDimensions.x - p / 2) - 1, 2 * ((m_windowDimensions.y - y) / m_windowDimensions.y - p / 2) - 1, 0.0f) // {0}
};
backgroundQuad->UpdatePositions(vec_windowPositions); // Update new vertex positions
glStencilFunc(GL_ALWAYS, 1, 0xFF);
glStencilMask(0xFF);
glDisable(GL_DEPTH_TEST);
prg_texture2D->UseProgram();
prg_texture2D->SetUniform("modelMatrix", glm::mat4());
glActiveTexture(GL_TEXTURE0);
tex_window->Bind();
backgroundQuad->Render();
tex_window->Unbind();
prg_texture2D->UnUseProgram();
glEnable(GL_DEPTH_TEST);
backgroundQuad->UpdatePositions(vec_positions); // Update to old, fulscreen vertex positions
Here's a screenshot of what it looks like. Any suggestions how to solve this?