OpenGL: The roll rotation has an unwanted impact on other rotations

Question

I wrote a simple program in the C language using the GLEW, SDL and cglm libraries to draw a triangle with OpenGL 3.3. I would like to be able to rotate the camera around the 3 axis:

• Move the mouse to the left or right to rotate around the Y axis (the yaw rotation)
• Move the mouse to the down or up to rotate around the X axis (the pitch rotation)
• Press the Q or E key to rotate around the Z axis (the roll rotation)

The problem: If I do a positive roll rotation (I press the E key) then a positive yaw rotation (move the mouse to the right) then this last rotation is rotating to the bottom of the screen instead of to the right of the screen. So, the roll rotation has an unwanted impact on other rotations. Do you know how I can fix that?

The code in "main.c":

#include <GL/glew.h> /* for glClear */
#include <SDL.h> /* for SDL_Init */
#include <cglm/cglm.h> /* for vec3 */
#include <fcntl.h> /* for open */
#include <unistd.h> /* for read */
#define BUF_SIZE 512
static int prog_create_shader(GLuint *shader, const char *spath, GLenum stype)
{
    int fd = -1;
    int nbytes = -1;
    GLchar sbuf[BUF_SIZE];
    const GLchar *ssrc = NULL;
    GLint ret = -1;
    GLchar log[BUF_SIZE];
    /* read the shader source */
    fd = open(spath, O_RDONLY);
    if(fd == -1)
    {
        SDL_Log("Error: open");
        return -1;
    }
    nbytes = read(fd, &sbuf, BUF_SIZE);
    if(nbytes == -1)
    {
        SDL_Log("Error: read");
        return -1;
    }
    sbuf[nbytes] = '\0';
    ssrc = sbuf;
    if(close(fd) == -1)
    {
        SDL_Log("Error: close");
        return -1;
    }
    /* create the shader */
    *shader = glCreateShader(stype);
    glShaderSource(*shader, 1, &ssrc, NULL);
    glCompileShader(*shader);
    glGetShaderiv(*shader, GL_COMPILE_STATUS, &ret);
    if(ret == GL_FALSE)
    {
        glGetShaderInfoLog(*shader, BUF_SIZE, NULL, log);
        SDL_Log("Error: glGetShaderiv: %s", log);
        return -1;
    }
    return 0;
}
static int prog_create(GLuint *prog, const char *vspath, const char *fspath)
{
    GLuint vshader = -1;
    GLuint fshader = -1;
    GLint ret = -1;
    GLchar log[BUF_SIZE];
    /* create the vertex shader */
    if(prog_create_shader(&vshader, vspath, GL_VERTEX_SHADER) == -1)
    {
        SDL_Log("Error: prog_create_shader (%s)", vspath);
        return -1;
    }
    /* create the fragment shader */
    if(prog_create_shader(&fshader, fspath, GL_FRAGMENT_SHADER) == -1)
    {
        SDL_Log("Error: prog_create_shader (%s)", fspath);
        return -1;
    }
    /* create the program */
    *prog = glCreateProgram();
    glAttachShader(*prog, vshader);
    glAttachShader(*prog, fshader);
    glLinkProgram(*prog);
    glGetProgramiv(*prog, GL_LINK_STATUS, &ret);
    if(ret == GL_FALSE)
    {
        glGetProgramInfoLog(*prog, BUF_SIZE, NULL, log);
        SDL_Log("Error: glGetShaderiv: %s", log);
        return -1;
    }
    glDeleteShader(fshader);
    glDeleteShader(vshader);
    glUseProgram(*prog);
    return 0;
}
static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, float *yaw, float *pitch, float *roll, vec3 camera_front, vec3 camera_up)
{
    float xoffset = xrel;
    float yoffset = yrel;
    float zoffset = zrel;
    float sensitivity = 0.1;
    xoffset *= sensitivity;
    yoffset *= sensitivity;
    zoffset *= sensitivity;
    *yaw += xoffset;
    *pitch += -yoffset;
    *roll += zoffset;
    vec3 front =
    {
        cos(glm_rad(*yaw)) * cos(glm_rad(*pitch)),
        sin(glm_rad(*pitch)),
        sin(glm_rad(*yaw)) * cos(glm_rad(*pitch))
    };
    glm_vec3_normalize_to(front, camera_front);
    vec3 up =
    {
        cos(glm_rad(*roll)),
        sin(glm_rad(*roll)),
        0
    };
    glm_vec3_normalize_to(up, camera_up);
}
static void input(int *loop, vec3 camera_position, vec3 camera_front, vec3 camera_up, float *yaw, float *pitch, float *roll)
{
    SDL_Event event = {0};
    float cameraSpeed = 0.01;
    vec3 tmp = {0};
    const Uint8* keystates = NULL;
    while(SDL_PollEvent(&event))
    {
        switch(event.type)
        {
            case SDL_MOUSEMOTION:
                input_mouse(event.motion.xrel, event.motion.yrel, 0, yaw, pitch, roll, camera_front, camera_up);
                break;
            case SDL_KEYDOWN:
                switch(event.key.keysym.sym)
                {
                    case SDLK_ESCAPE:
                        *loop = 0;
                        break;
                }
                break;
            case SDL_WINDOWEVENT:
                switch(event.window.event)
                {
                    case SDL_WINDOWEVENT_CLOSE:
                        *loop = 0;
                        break;
                    case SDL_WINDOWEVENT_RESIZED:
                        glViewport(0, 0, event.window.data1, event.window.data2);
                        break;
                }
                break;
        }
    }
    keystates = SDL_GetKeyboardState(NULL);
    if(keystates[SDL_SCANCODE_Q])
    {
        input_mouse(0, 0, -1, yaw, pitch, roll, camera_front, camera_up);
    }
    if(keystates[SDL_SCANCODE_E])
    {
        input_mouse(0, 0, 1, yaw, pitch, roll, camera_front, camera_up);
    }
    if(keystates[SDL_SCANCODE_W])
    {
        glm_vec3_scale(camera_front, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_S])
    {
        glm_vec3_scale(camera_front, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_A])
    {
        glm_vec3_cross(camera_front, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_D])
    {
        glm_vec3_cross(camera_front, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
}
static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_front, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
{
    /* declare variables */
    mat4 view = GLM_MAT4_IDENTITY_INIT;
    mat4 projection = GLM_MAT4_IDENTITY_INIT;
    mat4 model = GLM_MAT4_IDENTITY_INIT;
    GLuint uniform = 0;
    vec3 tmp = {0};
    int width = 0;
    int height = 0;
    /* clear */
    glClearColor(1, 1, 1, 0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    /* update the view matrix */
    glm_vec3_add(camera_position, camera_front, tmp);
    glm_lookat(camera_position, tmp, camera_up, view);
    uniform = glGetUniformLocation(prog, "view");
    glUniformMatrix4fv(uniform, 1, GL_FALSE, view[0]);
    /* update the projection matrix */
    SDL_GetWindowSize(window, &width, &height);
    glm_perspective(glm_rad(45), width / (float)height, 0.1, 100, projection);
    uniform = glGetUniformLocation(prog, "projection");
    glUniformMatrix4fv(uniform, 1, GL_FALSE, projection[0]);
    /* update the model matrix */
    glm_translate(model, triangle_position);
    uniform = glGetUniformLocation(prog, "model");
    glUniformMatrix4fv(uniform, 1, GL_FALSE, model[0]);
    /* draw the triangle */
    glBindVertexArray(triangle_vao);
    glDrawArrays(GL_TRIANGLES, 0, 3);
    SDL_GL_SwapWindow(window);
}
int main()
{
    /* declare variables */
    int loop = 1;
    SDL_DisplayMode dm = {0};
    SDL_Window *window = NULL;
    SDL_GLContext glcontext = NULL;
    GLuint prog = 0;
    GLuint triangle_vao = 0;
    GLuint triangle_vbo = 0;
    vec3 camera_position = {0, 0, 3};
    vec3 camera_front = {0, 0, -1};
    vec3 camera_up = {0, 1, 0};
    float yaw = -90;
    float pitch = 0;
    float roll = 90;
    float triangle_vertices[] =
    {
        -1, -1, +0,
        +0, +1, +0,
        +1, -1, +0
    };
    vec3 triangle_position = {0, 0, 0};
    /* initialize SDL */
    if(SDL_Init(SDL_INIT_VIDEO) != 0)
    {
        SDL_Log("Error: SDL_Init: %s", SDL_GetError());
        return 1;
    }
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 3);
    SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE);
    if(SDL_GetCurrentDisplayMode(0, &dm) < 0)
    {
        SDL_Log("Error: SDL_GetCurrentDisplayMode");
        return 1;
    }
    window = SDL_CreateWindow("sdl2 test", SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, dm.w, dm.h, SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE);
    if(window == NULL)
    {
        SDL_Log("Error: SDL_CreateWindow");
        return 1;
    }
    glcontext = SDL_GL_CreateContext(window);
    SDL_SetRelativeMouseMode(SDL_TRUE);
    /* initialize Glew */
    if(glewInit() != GLEW_OK)
    {
        SDL_Log("Error: glewInit");
        return 1;
    }
    /* print information */
    printf("screen width: %d\n", dm.w);
    printf("screen height: %d\n", dm.h);
    printf("GL_VENDOR: %s\n", glGetString(GL_VENDOR));
    printf("GL_RENDERER: %s\n", glGetString(GL_RENDERER));
    printf("GL_VERSION: %s\n", glGetString(GL_VERSION));
    printf("GL_SHADING_LANGUAGE_VERSION: %s\n", glGetString(GL_SHADING_LANGUAGE_VERSION));
    /* create the program */
    if(prog_create(&prog, "vs.sha", "fs.sha") == -1)
    {
        SDL_Log("Error: prog_create");
        return 1;
    }
    /* enable features */
    glEnable(GL_DEPTH_TEST);
    /* create the vertex buffer object */
    glGenBuffers(1, &triangle_vbo);
    glBindBuffer(GL_ARRAY_BUFFER, triangle_vbo);
    glBufferData(GL_ARRAY_BUFFER, sizeof(triangle_vertices), triangle_vertices, GL_STATIC_DRAW);
    /* create the vertex array object */
    glGenVertexArrays(1, &triangle_vao);
    glBindVertexArray(triangle_vao);
    glEnableVertexAttribArray(0);
    glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 3 * sizeof(float), (void*)0);
    /* loop */
    while(loop)
    {
        input(&loop, camera_position, camera_front, camera_up, &yaw, &pitch, &roll);
        display(window, prog, camera_position, camera_front, camera_up, triangle_position, triangle_vao);
    }
    /* free */
    glDeleteBuffers(1, &triangle_vbo);
    glDeleteVertexArrays(1, &triangle_vao);
    glDeleteProgram(prog);
    SDL_GL_DeleteContext(glcontext);
    SDL_Quit();
    return 0;
}

The code in "vs.sha" (the vertex shader):

#version 330 core
layout (location = 0) in vec3 pos;
uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;
void main()
{
    gl_Position = projection * view * model * vec4(pos, 1);
}

The code in "fs.sha" (the fragment shader):

#version 330 core
out vec4 color;
void main()
{
    color = vec4(1, 0.5, 0, 1);
}

The result (video of the SDL window): https://youtu.be/XIfA6pRNi24

PATCH #1

I tried to apply the suggestion of RedRuin. Here's the patch:

$ cat main.c.patch 
--- old/main.c  2022-07-25 06:36:48.165479193 +0200
+++ new/main.c  2022-07-26 01:53:55.221830761 +0200
@@ -4,6 +4,21 @@
 #include <fcntl.h> /* for open */
 #include <unistd.h> /* for read */
 #define BUF_SIZE 512
+void rotate_camera_x(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+    glm_vec3_rotate(camera_forward, angle, camera_right);
+    glm_vec3_rotate(camera_up,      angle, camera_right);
+}
+void rotate_camera_y(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+    glm_vec3_rotate(camera_forward, angle, camera_up);
+    glm_vec3_rotate(camera_right,   angle, camera_up);
+}
+void rotate_camera_z(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+    glm_vec3_rotate(camera_up,      angle, camera_forward);
+    glm_vec3_rotate(camera_right,   angle, camera_forward);
+}
 static int prog_create_shader(GLuint *shader, const char *spath, GLenum stype)
 {
    int fd = -1;
@@ -80,7 +95,7 @@
    glUseProgram(*prog);
    return 0;
 }
-static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, float *yaw, float *pitch, float *roll, vec3 camera_front, vec3 camera_up)
+static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, float *yaw, float *pitch, float *roll, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
 {
    float xoffset = xrel;
    float yoffset = yrel;
@@ -92,22 +107,14 @@
    *yaw += xoffset;
    *pitch += -yoffset;
    *roll += zoffset;
-   vec3 front =
-   {
-       cos(glm_rad(*yaw)) * cos(glm_rad(*pitch)),
-       sin(glm_rad(*pitch)),
-       sin(glm_rad(*yaw)) * cos(glm_rad(*pitch))
-   };
-   glm_vec3_normalize_to(front, camera_front);
-   vec3 up =
-   {
-       cos(glm_rad(*roll)),
-       sin(glm_rad(*roll)),
-       0
-   };
-   glm_vec3_normalize_to(up, camera_up);
+   if(xrel != 0)
+       rotate_camera_y(*yaw, camera_forward, camera_up, camera_right);
+   if(yrel != 0)
+       rotate_camera_x(*pitch, camera_forward, camera_up, camera_right);
+   if(zrel != 0)
+       rotate_camera_z(*roll, camera_forward, camera_up, camera_right);
 }
-static void input(int *loop, vec3 camera_position, vec3 camera_front, vec3 camera_up, float *yaw, float *pitch, float *roll)
+static void input(int *loop, vec3 camera_position, vec3 camera_forward, vec3 camera_up, vec3 camera_right, float *yaw, float *pitch, float *roll)
 {
    SDL_Event event = {0};
    float cameraSpeed = 0.01;
@@ -118,7 +125,7 @@
        switch(event.type)
        {
            case SDL_MOUSEMOTION:
-               input_mouse(event.motion.xrel, event.motion.yrel, 0, yaw, pitch, roll, camera_front, camera_up);
+               input_mouse(event.motion.xrel, event.motion.yrel, 0, yaw, pitch, roll, camera_forward, camera_up, camera_right);
                break;
            case SDL_KEYDOWN:
                switch(event.key.keysym.sym)
@@ -144,38 +151,38 @@
    keystates = SDL_GetKeyboardState(NULL);
    if(keystates[SDL_SCANCODE_Q])
    {
-       input_mouse(0, 0, -1, yaw, pitch, roll, camera_front, camera_up);
+       input_mouse(0, 0, -1, yaw, pitch, roll, camera_forward, camera_up, camera_right);
    }
    if(keystates[SDL_SCANCODE_E])
    {
-       input_mouse(0, 0, 1, yaw, pitch, roll, camera_front, camera_up);
+       input_mouse(0, 0, 1, yaw, pitch, roll, camera_forward, camera_up, camera_right);
    }
    if(keystates[SDL_SCANCODE_W])
    {
-       glm_vec3_scale(camera_front, cameraSpeed, tmp);
+       glm_vec3_scale(camera_forward, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_S])
    {
-       glm_vec3_scale(camera_front, cameraSpeed, tmp);
+       glm_vec3_scale(camera_forward, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_A])
    {
-       glm_vec3_cross(camera_front, camera_up, tmp);
+       glm_vec3_cross(camera_forward, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_D])
    {
-       glm_vec3_cross(camera_front, camera_up, tmp);
+       glm_vec3_cross(camera_forward, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
 }
-static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_front, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
+static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_forward, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
 {
    /* declare variables */
    mat4 view = GLM_MAT4_IDENTITY_INIT;
@@ -189,7 +196,7 @@
    glClearColor(1, 1, 1, 0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    /* update the view matrix */
-   glm_vec3_add(camera_position, camera_front, tmp);
+   glm_vec3_add(camera_position, camera_forward, tmp);
    glm_lookat(camera_position, tmp, camera_up, view);
    uniform = glGetUniformLocation(prog, "view");
    glUniformMatrix4fv(uniform, 1, GL_FALSE, view[0]);
@@ -218,8 +225,9 @@
    GLuint triangle_vao = 0;
    GLuint triangle_vbo = 0;
    vec3 camera_position = {0, 0, 3};
-   vec3 camera_front = {0, 0, -1};
+   vec3 camera_forward = {0, 0, -1};
    vec3 camera_up = {0, 1, 0};
+   vec3 camera_right = {1, 0, 0};
    float yaw = -90;
    float pitch = 0;
    float roll = 90;
@@ -285,8 +293,8 @@
    /* loop */
    while(loop)
    {
-       input(&loop, camera_position, camera_front, camera_up, &yaw, &pitch, &roll);
-       display(window, prog, camera_position, camera_front, camera_up, triangle_position, triangle_vao);
+       input(&loop, camera_position, camera_forward, camera_up, camera_right, &yaw, &pitch, &roll);
+       display(window, prog, camera_position, camera_forward, camera_up, triangle_position, triangle_vao);
    }
    /* free */
    glDeleteBuffers(1, &triangle_vbo);
$ patch -p1 < main.c.patch
$ gcc -std=c99 -pedantic -Wall -Werror -g `pkg-config sdl2 --cflags` `pkg-config glew --cflags` -o main.out main.c  `pkg-config sdl2 --libs-only-L` `pkg-config glew --libs-only-L` `pkg-config sdl2 --libs-only-l` `pkg-config glew --libs-only-l` -lm
$ ./main.out 
screen width: 1920
screen height: 1080
GL_VENDOR: AMD
GL_RENDERER: AMD Radeon RX 580 Series (polaris10, LLVM 11.0.0, DRM 3.46, 5.18.10-desktop)
GL_VERSION: 4.6 (Core Profile) Mesa 22.1.3
GL_SHADING_LANGUAGE_VERSION: 4.60

But the rotation is now doing weird things when I move the mouse or when I press the Q or E key.

PATCH #2

I tried to apply the new suggestions of RedRuin. Here's the patch:

$ cat main.c.patch 
--- 20220725/main.c 2022-07-25 06:36:48.165479193 +0200
+++ 20220727/main.c 2022-07-27 22:55:56.805646313 +0200
@@ -4,6 +4,21 @@
 #include <fcntl.h> /* for open */
 #include <unistd.h> /* for read */
 #define BUF_SIZE 512
+void rotate_camera_y(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+   printf("rotate_camera_y: [1] angle(%+.3f deg, %+.3f rad), camera_forward(%+.3f, %+.3f, %+.3f), camera_right(%+.3f, %+.3f, %+.3f)\n",
+           angle,
+           glm_rad(angle),
+           camera_forward[0],  camera_forward[1],  camera_forward[2],
+           camera_up[0],  camera_up[1],  camera_up[2]);
+   glm_vec3_rotate(camera_forward, glm_rad(angle), camera_up);
+   glm_vec3_rotate(camera_right,   glm_rad(angle), camera_up);
+   printf("rotate_camera_y: [2] angle(%+.3f deg, %+.3f rad), camera_forward(%+.3f, %+.3f, %+.3f), camera_right(%+.3f, %+.3f, %+.3f)\n",
+           angle,
+           glm_rad(angle),
+           camera_forward[0],  camera_forward[1],  camera_forward[2],
+           camera_up[0],  camera_up[1],  camera_up[2]);
+}
 static int prog_create_shader(GLuint *shader, const char *spath, GLenum stype)
 {
    int fd = -1;
@@ -80,7 +95,7 @@
    glUseProgram(*prog);
    return 0;
 }
-static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, float *yaw, float *pitch, float *roll, vec3 camera_front, vec3 camera_up)
+static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, float *yaw, float *pitch, float *roll, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
 {
    float xoffset = xrel;
    float yoffset = yrel;
@@ -92,22 +107,10 @@
    *yaw += xoffset;
    *pitch += -yoffset;
    *roll += zoffset;
-   vec3 front =
-   {
-       cos(glm_rad(*yaw)) * cos(glm_rad(*pitch)),
-       sin(glm_rad(*pitch)),
-       sin(glm_rad(*yaw)) * cos(glm_rad(*pitch))
-   };
-   glm_vec3_normalize_to(front, camera_front);
-   vec3 up =
-   {
-       cos(glm_rad(*roll)),
-       sin(glm_rad(*roll)),
-       0
-   };
-   glm_vec3_normalize_to(up, camera_up);
+   if(xrel != 0)
+       rotate_camera_y(45, camera_forward, camera_up, camera_right);
 }
-static void input(int *loop, vec3 camera_position, vec3 camera_front, vec3 camera_up, float *yaw, float *pitch, float *roll)
+static void input(int *loop, vec3 camera_position, vec3 camera_forward, vec3 camera_up, vec3 camera_right, float *yaw, float *pitch, float *roll)
 {
    SDL_Event event = {0};
    float cameraSpeed = 0.01;
@@ -118,7 +121,7 @@
        switch(event.type)
        {
            case SDL_MOUSEMOTION:
-               input_mouse(event.motion.xrel, event.motion.yrel, 0, yaw, pitch, roll, camera_front, camera_up);
+               input_mouse(event.motion.xrel, event.motion.yrel, 0, yaw, pitch, roll, camera_forward, camera_up, camera_right);
                break;
            case SDL_KEYDOWN:
                switch(event.key.keysym.sym)
@@ -144,38 +147,38 @@
    keystates = SDL_GetKeyboardState(NULL);
    if(keystates[SDL_SCANCODE_Q])
    {
-       input_mouse(0, 0, -1, yaw, pitch, roll, camera_front, camera_up);
+       input_mouse(0, 0, -1, yaw, pitch, roll, camera_forward, camera_up, camera_right);
    }
    if(keystates[SDL_SCANCODE_E])
    {
-       input_mouse(0, 0, 1, yaw, pitch, roll, camera_front, camera_up);
+       input_mouse(0, 0, 1, yaw, pitch, roll, camera_forward, camera_up, camera_right);
    }
    if(keystates[SDL_SCANCODE_W])
    {
-       glm_vec3_scale(camera_front, cameraSpeed, tmp);
+       glm_vec3_scale(camera_forward, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_S])
    {
-       glm_vec3_scale(camera_front, cameraSpeed, tmp);
+       glm_vec3_scale(camera_forward, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_A])
    {
-       glm_vec3_cross(camera_front, camera_up, tmp);
+       glm_vec3_cross(camera_forward, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_D])
    {
-       glm_vec3_cross(camera_front, camera_up, tmp);
+       glm_vec3_cross(camera_forward, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
 }
-static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_front, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
+static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_forward, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
 {
    /* declare variables */
    mat4 view = GLM_MAT4_IDENTITY_INIT;
@@ -189,7 +192,7 @@
    glClearColor(1, 1, 1, 0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    /* update the view matrix */
-   glm_vec3_add(camera_position, camera_front, tmp);
+   glm_vec3_add(camera_position, camera_forward, tmp);
    glm_lookat(camera_position, tmp, camera_up, view);
    uniform = glGetUniformLocation(prog, "view");
    glUniformMatrix4fv(uniform, 1, GL_FALSE, view[0]);
@@ -218,8 +221,9 @@
    GLuint triangle_vao = 0;
    GLuint triangle_vbo = 0;
    vec3 camera_position = {0, 0, 3};
-   vec3 camera_front = {0, 0, -1};
+   vec3 camera_forward = {0, 0, -1};
    vec3 camera_up = {0, 1, 0};
+   vec3 camera_right = {1, 0, 0};
    float yaw = -90;
    float pitch = 0;
    float roll = 90;
@@ -285,8 +289,8 @@
    /* loop */
    while(loop)
    {
-       input(&loop, camera_position, camera_front, camera_up, &yaw, &pitch, &roll);
-       display(window, prog, camera_position, camera_front, camera_up, triangle_position, triangle_vao);
+       input(&loop, camera_position, camera_forward, camera_up, camera_right, &yaw, &pitch, &roll);
+       display(window, prog, camera_position, camera_forward, camera_up, triangle_position, triangle_vao);
    }
    /* free */
    glDeleteBuffers(1, &triangle_vbo);

So it seems that the printed values are the good ones. In the patch #1, when I said that "the rotation is now doing weird things", I mean, it's hard to describe but it's like it's rotating too fast but on the good axis.

PATCH #3

I finally understood what was going wrong with the rotation that was too fast. In the original code (not patched) I was using absolute values for the rotation angle, it was a big value stored into the variables pitch, yaw and roll. Now, in the patched code, I have to use relative values for the rotation angle, it is a small value and I don't need anymore to store it into the variables pitch, yaw and roll.

I tried to apply this. Here's the patch:

$ cat main.c.patch 
--- 20220725/main.c 2022-07-25 06:36:48.165479193 +0200
+++ 20220728/main.c 2022-07-28 04:05:37.677937509 +0200
@@ -4,6 +4,21 @@
 #include <fcntl.h> /* for open */
 #include <unistd.h> /* for read */
 #define BUF_SIZE 512
+void rotate_camera_x(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+   glm_vec3_rotate(camera_forward, glm_rad(angle), camera_right);
+   glm_vec3_rotate(camera_up,      glm_rad(angle), camera_right);
+}
+void rotate_camera_y(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+   glm_vec3_rotate(camera_forward, glm_rad(angle), camera_up);
+   glm_vec3_rotate(camera_right,   glm_rad(angle), camera_up);
+}
+void rotate_camera_z(float angle, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
+{
+   glm_vec3_rotate(camera_up,      glm_rad(angle), camera_forward);
+   glm_vec3_rotate(camera_right,   glm_rad(angle), camera_forward);
+}
 static int prog_create_shader(GLuint *shader, const char *spath, GLenum stype)
 {
    int fd = -1;
@@ -80,34 +95,23 @@
    glUseProgram(*prog);
    return 0;
 }
-static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, float *yaw, float *pitch, float *roll, vec3 camera_front, vec3 camera_up)
+static void input_mouse(Sint16 xrel, Sint16 yrel, Sint16 zrel, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
 {
-   float xoffset = xrel;
-   float yoffset = yrel;
+   float xoffset = -xrel;
+   float yoffset = -yrel;
    float zoffset = zrel;
    float sensitivity = 0.1;
    xoffset *= sensitivity;
    yoffset *= sensitivity;
    zoffset *= sensitivity;
-   *yaw += xoffset;
-   *pitch += -yoffset;
-   *roll += zoffset;
-   vec3 front =
-   {
-       cos(glm_rad(*yaw)) * cos(glm_rad(*pitch)),
-       sin(glm_rad(*pitch)),
-       sin(glm_rad(*yaw)) * cos(glm_rad(*pitch))
-   };
-   glm_vec3_normalize_to(front, camera_front);
-   vec3 up =
-   {
-       cos(glm_rad(*roll)),
-       sin(glm_rad(*roll)),
-       0
-   };
-   glm_vec3_normalize_to(up, camera_up);
+   if(xrel != 0)
+       rotate_camera_y(xoffset, camera_forward, camera_up, camera_right);
+   if(yrel != 0)
+       rotate_camera_x(yoffset, camera_forward, camera_up, camera_right);
+   if(zrel != 0)
+       rotate_camera_z(zoffset, camera_forward, camera_up, camera_right);
 }
-static void input(int *loop, vec3 camera_position, vec3 camera_front, vec3 camera_up, float *yaw, float *pitch, float *roll)
+static void input(int *loop, vec3 camera_position, vec3 camera_forward, vec3 camera_up, vec3 camera_right)
 {
    SDL_Event event = {0};
    float cameraSpeed = 0.01;
@@ -118,7 +122,7 @@
        switch(event.type)
        {
            case SDL_MOUSEMOTION:
-               input_mouse(event.motion.xrel, event.motion.yrel, 0, yaw, pitch, roll, camera_front, camera_up);
+               input_mouse(event.motion.xrel, event.motion.yrel, 0, camera_forward, camera_up, camera_right);
                break;
            case SDL_KEYDOWN:
                switch(event.key.keysym.sym)
@@ -144,38 +148,38 @@
    keystates = SDL_GetKeyboardState(NULL);
    if(keystates[SDL_SCANCODE_Q])
    {
-       input_mouse(0, 0, -1, yaw, pitch, roll, camera_front, camera_up);
+       input_mouse(0, 0, -1, camera_forward, camera_up, camera_right);
    }
    if(keystates[SDL_SCANCODE_E])
    {
-       input_mouse(0, 0, 1, yaw, pitch, roll, camera_front, camera_up);
+       input_mouse(0, 0, 1, camera_forward, camera_up, camera_right);
    }
    if(keystates[SDL_SCANCODE_W])
    {
-       glm_vec3_scale(camera_front, cameraSpeed, tmp);
+       glm_vec3_scale(camera_forward, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_S])
    {
-       glm_vec3_scale(camera_front, cameraSpeed, tmp);
+       glm_vec3_scale(camera_forward, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_A])
    {
-       glm_vec3_cross(camera_front, camera_up, tmp);
+       glm_vec3_cross(camera_forward, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_sub(camera_position, tmp, camera_position);
    }
    if(keystates[SDL_SCANCODE_D])
    {
-       glm_vec3_cross(camera_front, camera_up, tmp);
+       glm_vec3_cross(camera_forward, camera_up, tmp);
        glm_vec3_normalize(tmp);
        glm_vec3_scale(tmp, cameraSpeed, tmp);
        glm_vec3_add(camera_position, tmp, camera_position);
    }
 }
-static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_front, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
+static void display(SDL_Window *window, GLuint prog, vec3 camera_position, vec3 camera_forward, vec3 camera_up, vec3 triangle_position, GLuint triangle_vao)
 {
    /* declare variables */
    mat4 view = GLM_MAT4_IDENTITY_INIT;
@@ -189,7 +193,7 @@
    glClearColor(1, 1, 1, 0);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
    /* update the view matrix */
-   glm_vec3_add(camera_position, camera_front, tmp);
+   glm_vec3_add(camera_position, camera_forward, tmp);
    glm_lookat(camera_position, tmp, camera_up, view);
    uniform = glGetUniformLocation(prog, "view");
    glUniformMatrix4fv(uniform, 1, GL_FALSE, view[0]);
@@ -218,11 +222,9 @@
    GLuint triangle_vao = 0;
    GLuint triangle_vbo = 0;
    vec3 camera_position = {0, 0, 3};
-   vec3 camera_front = {0, 0, -1};
+   vec3 camera_forward = {0, 0, -1};
    vec3 camera_up = {0, 1, 0};
-   float yaw = -90;
-   float pitch = 0;
-   float roll = 90;
+   vec3 camera_right = {1, 0, 0};
    float triangle_vertices[] =
    {
        -1, -1, +0,
@@ -285,8 +287,8 @@
    /* loop */
    while(loop)
    {
-       input(&loop, camera_position, camera_front, camera_up, &yaw, &pitch, &roll);
-       display(window, prog, camera_position, camera_front, camera_up, triangle_position, triangle_vao);
+       input(&loop, camera_position, camera_forward, camera_up, camera_right);
+       display(window, prog, camera_position, camera_forward, camera_up, triangle_position, triangle_vao);
    }
    /* free */
    glDeleteBuffers(1, &triangle_vbo);

So it just works perfectly. Problem fixed.

The result (video of the SDL window): https://youtu.be/w_4J_XwmnSg

Answer 1

Consider what this does:

vec3 front =
{
    cos(glm_rad(*yaw)) * cos(glm_rad(*pitch)),
    sin(glm_rad(*pitch)),
    sin(glm_rad(*yaw)) * cos(glm_rad(*pitch))
};

This calculation takes into account the yaw and pitch, but not the roll. Thus, while your up vector is being rolled, your front vector has no knowledge of this, and (based on OpenGL's coordinate system) assumes that "up" is always positive Y. To this bit of code, a rotation to the "right" is always the same direction, regardless of whatever way the camera is oriented. This is why moving the mouse to the right moves the triangle up; the orientation of the camera changed but the axis the rotation is around hasn't.

You could try updating your front vector to include the roll parameters (pulled from this answer):

vec3 front =
{
    -cos(glm_rad(*yaw)) * sin(glm_rad(*pitch)) * sin(glm_rad(*roll)) - sin(glm_rad(*yaw)) * cos(glm_rad(*roll)),
    -sin(glm_rad(*yaw)) * sin(glm_rad(*pitch)) * sin(glm_rad(*roll)) + cos(glm_rad(*yaw)) * cos(glm_rad(*roll)),
    cos(glm_rad(*pitch)) * sin(glm_rad(*roll))
}

Unfortunately this is pretty unweildly and chances are sooner or later you'll run into Gimbal Lock. You can use a matrix or a special 4-dimensional vector called a Quaternion to completely circumvent gimbal lock, but they are hard to intuit (compared to roll-pitch-yaw) and might be overkill for a simple application such as this.

I'll propose a middle ground, one I've had success with for simple cameras such as this. Keep track of your camera's 3 basis vectors that define it's reference orientation:

// For example, we might start with basis vectors like this
vec3 camera_forward = {0, 0, -1};
vec3 camera_up = {0, 1, 0};
vec3 camera_right = {1, 0, 0};

Then we can use the glm_vec3_rotate() function to rotate these basis vectors around each other instead of the cardinal XYZ axes:

void rotate_camera_x(float angle)
{
    // rotate around the camera's x axis, or camera_right (pitch)
    glm_vec3_rotate(&camera_forward, angle, camera_right)
    glm_vec3_rotate(&camera_up,      angle, camera_right)
    // technically we could also rotate camera_right around camera_right, but that's redundant
}

void rotate_camera_y(float angle)
{
    // rotate around the camera's y axis, or camera_up (yaw)
    glm_vec3_rotate(&camera_forward, angle, camera_up)
    glm_vec3_rotate(&camera_right,   angle, camera_up)
}

void rotate_camera_z(float angle)
{
    // rotate around the camera's z axis, or camera_forward (roll)
    glm_vec3_rotate(&camera_up,      angle, camera_forward)
    glm_vec3_rotate(&camera_right,   angle, camera_forward)
}

The trick with this is that the axes we rotate change along with the rotation of the camera; now when you roll, you're changing the direction of camera_right and camera_up, changing the axes the next rotations will be around. This ensures that the camera's movement will always be tied to it's orientation.