Bug can be seen here: https://imgur.com/a/ZnmC1

cube_broken_camera.cpp

// Taken from Mesa3D
// void gluPerspective(double fovy, double aspect, double zNear, double zFar) {
//     double xmin, xmax, ymin, ymax;
// 
//     ymax = zNear * tan(fovy * M_PI / 360.0);
//     ymin = -ymax;
//     xmin = ymin * aspect;
//     xmax = ymax * aspect;
// 
//     glFrustum(xmin, xmax, ymin, ymax, zNear, zFar);
// }
// 
// void gluPickMatrix(double x, double y, double width, double height,
//                    const int viewport[16]) {
//     float sx, sy;
//     float tx, ty;
// 
//     sx = (float) viewport[2] / (float) width;
//     sy = (float) viewport[3] / (float) height;
//     tx = ((float) viewport[2] + 2.0f * ((float) viewport[0] - (float) x)) /
//          (float) width;
//     ty = ((float) viewport[3] + 2.0f * ((float) viewport[1] - (float) y)) /
//          (float) height;
// 
//     float fb[16] = {sx,   0.0f, 0.0f, tx,   0.0f, sy,   0.0f, ty,
//                     0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f};
// 
//     glMultMatrixf(fb);
// }

void setupCamera(float a) {
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();

    glm::mat4 projection = glm::perspective(glm::radians(FOV), (float) WIDTH / (float) HEIGHT, 0.05f, 1000.0f);
    glMultMatrixf(glm::value_ptr(projection));

    // gluPerspective(FOV, (float) WIDTH / (float) HEIGHT, 0.05f, 1000.0f);

    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    moveCameraToPlayer(a);
}

void setupPickCamera(float a, int x, int y) {
    glMatrixMode(GL_PROJECTION);
    glLoadIdentity();


    // memset(viewportBuffer, 0, sizeof(viewportBuffer));
    // glGetIntegerv(GL_VIEWPORT, viewportBuffer);
    // gluPickMatrix((float) x, (float) y, 5.0F, 5.0F, viewportBuffer);

    glm::ivec4 viewport;
    glGetIntegerv(GL_VIEWPORT, glm::value_ptr(viewport));
    glm::mat4 pick = glm::pickMatrix(glm::vec2(x, y), glm::vec2(5.0f, 5.0f), viewport);
    glMultMatrixf(glm::value_ptr(pick));


    glm::mat4 projection = glm::perspective(glm::radians(FOV), (float) WIDTH / (float) HEIGHT, 0.05f, 1000.0f);
    glMultMatrixf(glm::value_ptr(projection));
    
    // gluPerspective(FOV, (float) WIDTH / (float) HEIGHT, 0.05f, 1000.0f);


    glMatrixMode(GL_MODELVIEW);
    glLoadIdentity();
    moveCameraToPlayer(a);
}