i am trying to get the mouse cursor in world space pos from window space(-1,1 window width and height) using the viewprojectionmatrix.

This is how i calculate my projection matrix:

static mat4 Perspective4x4(float FOV, float AspectRatio, float FarC, float NearC)
{
    // Positive x is right
    // Positive y is up
    // Positive z is forward into the screen

    float Cotangent = 1.0f / tanf((FOV)*0.5f);
    float Depth = NearC - FarC;
    float A = (-FarC - NearC) / Depth;
    float B = 2.0f * FarC * NearC / Depth;
    return
    {
        Cotangent/AspectRatio, 0.0f, 0.0f, 0.0f,
        0.0f, Cotangent, 0.0f, 0.0f,
        0.0f, 0.0f, -A, -B,
        0.0f, 0.0f, 1.0f, 0.0f,
    };
}

...

float WidthOverHeight = ...;
mat4 ProjectionMatrix = Perspective4x4(DegreesToRadians(90.0f), WidthOverHeight, 50.0f, 0.1f);

This is how i calculate my view matrix:

static mat4 Translate4x4(vec3 V)
{
    return
    {
        1.0f, 0.0f, 0.0f, V.X,
        0.0f, 1.0f, 0.0f, V.Y,
        0.0f, 0.0f, 1.0f, V.Z,
        0.0f, 0.0f, 0.0f, 1.0f
    };
}

...

mat4 ViewMatrix = Translate4x4(-CameraPosition);

This is how i calculate the cursor position in worldspace:

mat4 ProjectionMatrix = ...;
mat4 ViewMatrix = ...;
    
vec2 CursorP = GetBilateralCursorPos(Input); // Values between -1 and 1


v4_f32 WorldSpacePNear = Inverse(ProjectionMatrix) * V4F32(CursorP, -1.0f, 1.0f);
WorldSpacePNear /= WorldSpacePNear.W;
WorldSpacePNear = Inverse(ViewMatrix) * WorldSpacePNear;
    
v4_f32 WorldSpacePFar = Inverse(ProjectionMatrix) * V4F32(CursorP, 1.0f, 1.0f);
WorldSpacePFar /= WorldSpacePFar.W;
WorldSpacePFar = Inverse(ViewMatrix) * WorldSpacePFar;
    
WorldSpacePFar.Z *= -1.0f;
WorldSpacePNear.Z *= -1.0f;

EDIT: I also tried dividing by W at the end but it doesnt work properly either.

This is how i send the matrices to OpenGL (legacy):

// I send them transposed because my matrices are row-major
mat4 ProjectionMatrix = Transpose(...);
mat4 ViewMatrix = Transpose(...);

glMatrixMode(GL_PROJECTION);
glLoadMatrixf(ProjectionMatrix.E);
        
glMatrixMode(GL_MODELVIEW);
glLoadMatrixf(ViewMatrix.E);

The resulting positions do not seem to take in account the view projection because a change in the camera position will offset them making them not accurate.

NOTES:

1. vec4, vec2, mat4 are custom float-based math types. (they are what you would expect)
2. I know legacy OpenGL is deprecated, in fact im going to switch to modern opengl very soon, i just want to get this working.

CodePudding user response：

i fixed the issue calculating the cursor pos like this:

mat4 ProjectionMatrix = ...;
mat4 ViewMatrix = ...;

// Z Distance from camera pos
float WorldDistanceFromCameraZ = 1.0f;

vec2 CursorP = GetBilateralCursorPos(Input); // Values between -1 and 1
    
vec4 ProbeZ = V4F32(World->Camera.P - WorldDistanceFromCameraZ*World->Camera.P.Z, 1.0f);
ProbeZ = (ProjectionMatrix*ViewMatrix) * ProbeZ;
    
vec4 ClipP = V4F32(CursorP.X*ProbeZ.W, CursorP.Y*ProbeZ.W, ProbeZ.Z, ProbeZ.W);
vec4 WorldP = Inverse(ProjectionMatrix*ViewMatrix) * ClipP;