Getting close to a formula for determining tessellation LOD that takes camera position, direction, face normal and distance into account.

Still need to involve face area into this as well.

world.c

@@ -7,18 +7,18 @@
 
 static P_COLOR colors[] = 
 {
-    { (255 << 16), 0 },
-    { (255 << 8), 0 },
-    { (255 << 0), 0 },
-    { (255 << 16) | (255 << 8), 0 },
-    { (255 << 16) | (255 << 0), 0 },
-    { (255 << 8)  | (255 << 0), 0 },
-    { (128 << 8)  | (255 << 8), 0 },
-    { (255 << 16) | (128 << 8), 0 },
-    { (128 << 16) | (255 << 0), 0 },
-    { (255 << 16) | (128 << 0), 0 },
-    { (128 << 8)  | (255 << 0), 0 },
-    { (128 << 8)  | (128 << 0), 0 },
+    { (255 << 0), 0 },  // Red
+    { (255 << 8), 0 },  // Green
+    { (255 << 16), 0 }, // Blue
+    { (255 << 0) | (255 << 8), 0 },
+    { (255 << 0) | (255 << 16), 0 },
+    { (255 << 8) | (255 << 16), 0 },
+    { (128 << 8) | (255 << 8), 0 },
+    { (255 << 0) | (128 << 8), 0 },
+    { (128 << 0) | (255 << 16), 0 },
+    { (255 << 0) | (128 << 16), 0 },
+    { (128 << 8) | (255 << 16), 0 },
+    { (128 << 8) | (128 << 16), 0 },
 };
 static const int numColors = sizeof(colors) / sizeof(P_COLOR);
 
@@ -73,7 +73,7 @@ static INLINE short world_planeDot(const SVECTOR *dir, const ps1bsp_plane_t *pla
     return (short)m_dot12(dir, &plane->normal);
 }
 
-static INLINE char world_cull_backface(const world_t *world, const ps1bsp_face_t *face, short *dot)
+static INLINE short world_cull_backface(const world_t *world, const ps1bsp_face_t *face)
 {
     // Backface culling using the face's plane and center point
     // This eliminates the need for normal clipping per polygon
@@ -84,9 +84,9 @@ static INLINE char world_cull_backface(const world_t *world, const ps1bsp_face_t
 
     // Check if the face's plane points towards the camera
     const ps1bsp_plane_t *plane = &world->planes[face->planeId];
-    *dot = world_planeDot(&cam_vec, plane);
-    if ((*dot >= 0) ^ face->side)
-        return 1;
+    int planeDot = world_planeDot(&cam_vec, plane);
+    if ((planeDot >= 0) ^ face->side)
+        return 0;
  
     // Check if the face is behind the camera 
     // TODO: this may cull faces close to the camera, use VectorNormalS to normalize and check for angles < -60 degrees
@@ -104,10 +104,34 @@ static INLINE char world_cull_backface(const world_t *world, const ps1bsp_face_t
 
     // NOTE: disabling the behind-the-camera check does *not* actually solve the problem of polygons disappearing when too close to the camera!
     // This means that the GPU probably already clips polygons that stretch too far outside the drawing area. Tessellation should solve this.
-    if (m_dot12(&cam_vec, &cam_dir) >= 0)
+    int camDot = m_dot12(&cam_vec, &cam_dir);
+    if (camDot < 0)
         return 0;
 
-    return 1;
+    int vecLenSqr = m_dot12(&cam_vec, &cam_vec);
+
+    // Flip angle for faces that are on the opposite side of their plane
+    char flip = 2 * face->side - 1;
+    planeDot = planeDot * flip;
+
+    int foo = (planeDot * vecLenSqr) / (camDot  + 1);
+    // TODO: include face area
+    return (short)foo;
+}
+
+static u_short world_leafAtPoint(const world_t *world, const VECTOR *point)
+{
+    short nodeIdx = 0;
+    while (nodeIdx >= 0)
+    {
+        const ps1bsp_node_t *node = &world->nodes[nodeIdx];
+        const ps1bsp_plane_t *plane = &world->planes[node->planeId];
+
+        short dist = world_pointPlaneDist(point, plane);
+        nodeIdx = node->children[(dist > 0) ^ 1];
+    }
+
+    return ~nodeIdx;
 }
 
 static void world_drawface_flat(const world_t *world, const ps1bsp_face_t *face, P_COLOR *color, u_long *ot)
@@ -141,6 +165,13 @@ static void world_drawface_textured(const world_t *world, const ps1bsp_face_t *f
     const ps1bsp_texture_t *texture = &world->textures[face->textureId];
     const ps1bsp_polygon_t* poly = &world->polygons[face->firstPolygon];
 
+    u_char tess = face->flags & 3;
+    if (tess)
+    {
+        world_drawface_flat(world, face, &colors[tess - 1], ot);
+        return;
+    }
+
     if (face->flags & SURF_DRAWLIQUID)
     {
         for (u_char polyIdx = 0; polyIdx < face->numPolygons; ++polyIdx, ++poly)
@@ -206,10 +237,18 @@ static ps1bsp_face_t *world_sortFaces(const world_t *world, const ps1bsp_leaf_t
 
             // NOTE: this value could be REALLY useful for determining the tessellation subdivisions. It has camera distance *and* angle in it.
             // Just include the face size/area for an approximate screen size. Maybe also separate x/y/z for angle-dependent tessellation.
-            short dot;
-            if (world_cull_backface(world, face, &dot))
+            short dot = world_cull_backface(world, face);
+            if (!dot)
                 continue;
 
+            face->flags &= ~3;
+            if (dot < 64)
+                face->flags |= 1;
+            else if (dot < 128)
+                face->flags |= 2;
+            else if (dot < 256)
+                face->flags |= 3;
+
             // Sort the faces to draw in front-to-back order
             face->nextFace = firstFace;
             firstFace = face;