From 9474739c26b3d55ff80420e5fc0a1221a85e8626 Mon Sep 17 00:00:00 2001
From: Nico de Poel <ndepoel@gmail.com>
Date: Tue, 7 Feb 2023 16:18:50 +0100
Subject: [PATCH] Sort faces for brush models into the overall faces list. This
 works well to draw models correctly by themselves, but when placed in the
 world they may still have clipping issues with other models and/or
 neighboring leafs.

---
 world.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 68 insertions(+), 15 deletions(-)

diff --git a/world.c b/world.c
index 3a83b28..f329c90 100644
--- a/world.c
+++ b/world.c
@@ -194,6 +194,66 @@ static void world_drawFaces(const world_t *world, const ps1bsp_face_t *firstFace
     }
 }
 
+// Simplified BSP subtree traversal specifically for sorting faces of brush models.
+// This skips the frustum culling and PVS culling stages, as this will have already been done on the leafs containing the model.
+static ps1bsp_face_t *world_sortModelFaces(const world_t* world, const ps1bsp_model_t* model, u_long frameNum, ps1bsp_face_t **lastFace)
+{
+    short *nodeStack = (short*)(scratchpad + 256);
+    u_char stackPtr = 0;
+
+    // Push the start node onto the stack
+    nodeStack[stackPtr++] = model->nodeId0;
+
+    ps1bsp_face_t *firstFace = NULL;
+    while (stackPtr > 0)
+    {
+        // Pop a node off the stack
+        short nodeIdx = nodeStack[--stackPtr];
+
+        if (nodeIdx < 0)    // Leaf node
+        {
+            u_short leafIdx = ~nodeIdx;
+            if (leafIdx == 0)   // Leaf 0 should not be drawn
+                continue;
+
+            const ps1bsp_leaf_t *leaf = &world->leaves[leafIdx];
+            const u_short *leafFaces = &world->leafFaces[leaf->firstLeafFace];
+            for (u_short leafFaceIdx = 0; leafFaceIdx < leaf->numLeafFaces; ++leafFaceIdx)
+            {
+                ps1bsp_face_t *face = &world->faces[leafFaces[leafFaceIdx]];
+
+                // Make sure we draw each face only once per frame
+                if (face->drawFrame == frameNum)
+                    continue;
+
+                face->drawFrame = frameNum;
+
+                // Cull faces that are facing away from the camera
+                if (!world_cull_backface(world, face))
+                    continue;
+
+                if (!firstFace)
+                    *lastFace = face;
+
+                face->nextFace = firstFace;
+                firstFace = face;
+            }
+            continue;
+        }
+
+        const ps1bsp_node_t *node = &world->nodes[nodeIdx];
+        const ps1bsp_plane_t *plane = &world->planes[node->planeId];
+        short dist = world_pointPlaneDist(&cam_pos, plane); // TODO: add/subtract model origin from camera to sort faces in model space
+        
+        // Traverse the subtree in back-to-front order
+        char order = dist < 0;
+        nodeStack[stackPtr++] = node->children[order];
+        nodeStack[stackPtr++] = node->children[order ^ 1];
+    }
+
+    return firstFace;
+}
+
 static ps1bsp_face_t *world_sortFaces(const world_t *world, const ps1bsp_leaf_t *firstLeaf)
 {
     ps1bsp_face_t *firstFace = NULL;
@@ -233,24 +293,17 @@ static ps1bsp_face_t *world_sortFaces(const world_t *world, const ps1bsp_leaf_t
             firstFace = face;
         }
 
-        // Draw models in this leaf
+        // Sort faces for the models in this leaf
         for (const ps1bsp_model_t* model = leaf->models; model != NULL; model = model->nextModel)
         {
-            // Hmmm... either just draw all the faces in this model, or do the whole BSP sorting thing
-            // Guess we could call sortLeafs from here
-            // If we don't, things like tech doors which are concave might render improperly
-            ps1bsp_face_t* face = &world->faces[model->firstFace];
-            for (u_short faceIdx = 0; faceIdx < model->numFaces; ++faceIdx, ++face)
+            // This successfully sorts the faces within a single model.
+            // However, several models occupying the same leaf will still clip through each other
+            // and since we're looking at only a single leaf per model, large brush models will often have neighbouring leafs drawing over them.
+            ps1bsp_face_t* lastFace;
+            ps1bsp_face_t* face = world_sortModelFaces(world, model, frameNum, &lastFace);
+            if (face != NULL)
             {
-                if (face->drawFrame == frameNum)
-                    continue;
-
-                face->drawFrame = frameNum;
-
-                if (!world_cull_backface(world, face))
-                    continue;
-
-                face->nextFace = firstFace;
+                lastFace->nextFace = firstFace;
                 firstFace = face;
             }
         }