First attempt at calculating lighting per face-vertex with smoothing based on the angle between faces, using edge adjacency to determine neighbouring faces.

Not quite right yet but it does get a result, and it's given me a good idea of what the correct solution for this should be.

PS1BSP.vcxproj

@@ -141,10 +141,13 @@
     </Link>
   </ItemDefinitionGroup>
   <ItemGroup>
+    <ClCompile Include="lighting.cpp" />
     <ClCompile Include="main.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="bsp.h" />
+    <ClInclude Include="common.h" />
+    <ClInclude Include="lighting.h" />
     <ClInclude Include="ps1bsp.h" />
     <ClInclude Include="ps1types.h" />
     <ClInclude Include="rectpack\best_bin_finder.h" />

PS1BSP.vcxproj.filters

@@ -21,6 +21,9 @@
     <ClCompile Include="main.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="lighting.cpp">
+      <Filter>Header Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="bsp.h">
@@ -50,5 +53,11 @@
     <ClInclude Include="ps1types.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="lighting.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
+    <ClInclude Include="common.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
 </Project>

bsp.h

@@ -35,18 +35,6 @@ typedef struct                 // The BSP file header
                                  // nummodels = Size/sizeof(model_t)
 } dheader_t;
 
-typedef float scalar_t;        // Scalar value,
-
-typedef struct Vec3                 // Vector or Position
-{
-    scalar_t x;                  // horizontal
-    scalar_t y;                  // horizontal
-    scalar_t z;                  // vertical
-
-    Vec3() : x(0), y(0), z(0) { }
-    Vec3(float x, float y, float z) : x(x), y(y), z(z) { }
-} vec3_t;
-
 typedef struct
 {
     vec3_t normal;               // Vector orthogonal to plane (Nx,Ny,Nz)

common.h

@@ -0,0 +1,39 @@
+#pragma once
+#include <memory.h>
+#include <cstdlib>
+#include <cstdio>
+#include <cmath>
+#include <vector>
+#include <unordered_map>
+
+typedef float scalar_t;        // Scalar value,
+
+typedef struct Vec3                 // Vector or Position
+{
+    scalar_t x;                  // horizontal
+    scalar_t y;                  // horizontal
+    scalar_t z;                  // vertical
+
+    Vec3() : x(0), y(0), z(0) { }
+    Vec3(float x, float y, float z) : x(x), y(y), z(z) { }
+
+    Vec3 operator+(const Vec3& other) const
+    {
+        return Vec3(x + other.x, y + other.y, z + other.z);
+    }
+
+    Vec3 operator/(float div) const
+    {
+        return Vec3(x / div, y / div, z / div);
+    }
+
+    Vec3 operator-() const
+    {
+        return Vec3(-x, -y, -z);
+    }
+} vec3_t;
+
+inline float dotProduct(vec3_t a, vec3_t b)
+{
+	return a.x * b.x + a.y * b.y + a.z * b.z;
+}

lighting.cpp

@@ -0,0 +1,214 @@
+#include "common.h"
+#include "lighting.h"
+
+unsigned char sample_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, const Vec3& point)
+{
+	if (face->lightmap < 0)
+		return 0;
+
+	const unsigned char* lightmap = &world->lightmap[face->lightmap];
+	const plane_t* plane = &world->planes[face->plane_id];
+
+	int width, height;
+	float u, v;
+	switch (plane->type)
+	{
+	case 0:
+	case 3:
+		// Towards X
+		width = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16)) * 16;
+		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16)) * 16;
+		u = (point.y - bounds.min.y) / (bounds.max.y - bounds.min.y);
+		v = (point.z - bounds.min.z) / (bounds.max.z - bounds.min.z);
+		break;
+	case 1:
+	case 4:
+		// Towards Y
+		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16)) * 16;
+		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16)) * 16;
+		u = (point.x - bounds.min.x) / (bounds.max.x - bounds.min.x);
+		v = (point.z - bounds.min.z) / (bounds.max.z - bounds.min.z);
+		break;
+	case 2:
+	case 5:
+		// Towards Z
+		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16)) * 16;
+		height = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16)) * 16;
+		u = (point.x - bounds.min.x) / (bounds.max.x - bounds.min.x);
+		v = (point.y - bounds.min.y) / (bounds.max.y - bounds.min.y);
+		break;
+	default:
+		printf("Error: unknown plane type %d\n", plane->type);
+		return 0;
+	}
+
+	height >>= 4;
+	width >>= 4;
+
+	return lightmap[(int)(v * (width + 1) + u)];
+}
+
+void export_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, int faceIdx)
+{
+	if (face->lightmap < 0)
+		return;
+
+	const unsigned char* lightmap = &world->lightmap[face->lightmap];
+	const plane_t* plane = &world->planes[face->plane_id];
+
+	int width, height;
+	switch (plane->type)
+	{
+	case 0:
+	case 3:
+		// Towards X
+		width = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16));
+		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16));
+		break;
+	case 1:
+	case 4:
+		// Towards Y
+		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16));
+		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16));
+		break;
+	case 2:
+	case 5:
+		// Towards Z
+		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16));
+		height = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16));
+		break;
+	default:
+		printf("Error: unknown plane type %d\n", plane->type);
+		return;
+	}
+
+	width += 1;
+
+	char path[_MAX_PATH];
+	sprintf_s(path, _MAX_PATH, "lightmap_face%d_e%d_PT%d_%dx%d.raw", faceIdx, face->ledge_num, plane->type, width, height);
+	FILE* flm;
+	fopen_s(&flm, path, "wb");
+	if (!flm)
+		return;
+
+	for (int y = 0; y < height; ++y)
+	{
+		fwrite(&lightmap[y * width], sizeof(unsigned char), width, flm);
+	}
+
+	fclose(flm);
+}
+
+std::unordered_map<const edge_t*, EdgeData> analyze_edges(const world_t* world)
+{
+	std::unordered_map<const edge_t*, EdgeData> edgeData;
+
+	for (int faceIdx = 0; faceIdx < world->numFaces; ++faceIdx)
+	{
+		const face_t* face = &world->faces[faceIdx];
+		const int* edgeList = &world->edgeList[face->ledge_id];
+		for (int i = 0; i < face->ledge_num; ++i, ++edgeList)
+		{
+			int edgeIdx = *edgeList;
+			if (edgeIdx < 0)
+				edgeIdx = -edgeIdx;	// Reverse direction edge
+
+			const edge_t* edge = &world->edges[edgeIdx];
+
+			auto iter = edgeData.find(edge);
+			if (iter != edgeData.end())
+			{
+				iter->second.faces.push_back(face);
+			}
+			else
+			{
+				EdgeData newData = { 0 };
+				newData.edgeIndex = edgeIdx;
+				newData.faces.push_back(face);
+				edgeData[edge] = newData;
+			}
+		}
+	}
+
+	for (auto iter = edgeData.begin(); iter != edgeData.end(); ++iter)
+	{
+		size_t numFaces = iter->second.faces.size();
+		switch (numFaces)
+		{
+		case 1:
+			iter->second.isSharpEdge = true;
+			break;
+		case 2:
+		{
+			// TODO: take into account the face's side
+			auto faceA = iter->second.faces[0];
+			auto faceB = iter->second.faces[1];
+			const plane_t* planeA = &world->planes[faceA->plane_id];
+			const plane_t* planeB = &world->planes[faceB->plane_id];
+			vec3_t normalA = faceA->side ? -planeA->normal : planeA->normal;
+			vec3_t normalB = faceB->side ? -planeB->normal : planeB->normal;
+			float dot = dotProduct(planeA->normal, planeB->normal);
+			bool isSmooth = dot >= 0.5f;//&& dot <= 1;
+			iter->second.isSharpEdge = !isSmooth;
+			break;
+		}
+		default:
+			printf("Edge at index %d has %d adjacent face(s), weird\n", iter->second.edgeIndex, numFaces);
+			break;
+		}
+	}
+
+	return edgeData;
+}
+
+unsigned char compute_faceVertex_light(const world_t* world, const face_t* face, unsigned short vertexIndex, const std::unordered_map<const face_t*, BoundBox> faceBounds, const std::unordered_map<const edge_t*, EdgeData>& edgeData)
+{
+	const vertex_t* vertex = &world->vertices[vertexIndex];
+	auto point = vertex->toVec();
+
+	// Sample this face's lighting contribution
+	unsigned int light = sample_lightmap(world, face, faceBounds.find(face)->second, point) + (0xFF - face->baselight);
+	int numSamples = 1;
+
+	// Collect edges connected to this vertex, filter out the smooth ones only
+	std::vector<const edge_t*> smoothEdges;
+	for (auto iter = edgeData.begin(); iter != edgeData.end(); ++iter)
+	{
+		auto edge = iter->first;
+		if (edge->vertex0 != vertexIndex && edge->vertex1 != vertexIndex)
+			continue;
+
+		if (iter->second.isSharpEdge)
+			continue;
+
+		// If the current face doesn't appear in this edge's adjacency list, we're not interested
+		for (auto faceIter = iter->second.faces.begin(); faceIter != iter->second.faces.end(); ++faceIter)
+		{
+			// TODO: actually I don't think this is the correct solution. We're allowed to sample light contributions from edges that aren't connected to this face.
+			// However we need to ensure we sample contributions from each face only once, and we need to check the angle between faces on a case-by-case basis.
+			// In fact I don't think we're interested in edges at all? Just in the faces that connect to a certain vertex.
+			if (*faceIter == face)
+			{
+				smoothEdges.push_back(edge);
+				break;
+			}
+		}
+	}
+
+	// Gather lighting contributions from neigbouring faces
+	for (auto edgeIter = smoothEdges.begin(); edgeIter != smoothEdges.end(); ++edgeIter)
+	{
+		auto faces = edgeData.find(*edgeIter)->second.faces;
+		for (auto faceIter = faces.begin(); faceIter != faces.end(); ++faceIter)	// FIXME: "this" face doesn't always appear in faces list, when it absolutely should!
+		{
+			const face_t* otherFace = *faceIter;
+			if (otherFace == face)	// Skip the current face, we only sample it once
+				continue;
+
+			light += sample_lightmap(world, otherFace, faceBounds.find(otherFace)->second, point) + (0xFF - otherFace->baselight);
+			++numSamples;
+		}
+	}
+
+	return (unsigned char)(light / numSamples);
+}

lighting.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "bsp.h"
+
+struct EdgeData
+{
+	int edgeIndex = 0;
+	const edge_t* edge = nullptr;
+	std::vector<const face_t*> faces;
+	bool isSharpEdge = false;
+};
+
+unsigned char sample_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, const Vec3& point);
+void export_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, int faceIdx);
+
+std::unordered_map<const edge_t*, EdgeData> analyze_edges(const world_t* world);
+unsigned char compute_faceVertex_light(const world_t* world, const face_t* face, unsigned short vertexIndex, const std::unordered_map<const face_t*, BoundBox> faceBounds, const std::unordered_map<const edge_t*, EdgeData>& edgeData);

main.cpp

@@ -1,12 +1,9 @@
-#include <memory.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <vector>
-#include <unordered_map>
+#include "common.h"
 #include "bsp.h"
 #include "rectpack/finders_interface.h"
 #include "ps1types.h"
 #include "ps1bsp.h"
+#include "lighting.h"
 
 static char path[_MAX_PATH];
 
@@ -165,104 +162,6 @@ static void leaf_zone(const dleaf_t* leaf, short zone[3])
 	zone[2] = midZ & mask;
 }
 
-static unsigned char sample_lightmap(const world_t* world, const face_t *face, const BoundBox& bounds, const Vec3& point)
-{
-	if (face->lightmap < 0)
-		return 0;
-
-	const unsigned char* lightmap = &world->lightmap[face->lightmap];
-	const plane_t* plane = &world->planes[face->plane_id];
-
-	int width, height;
-	float u, v;
-	switch (plane->type)
-	{
-	case 0:
-	case 3:
-		// Towards X
-		width = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16)) * 16;
-		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16)) * 16;
-		u = (point.y - bounds.min.y) / (bounds.max.y - bounds.min.y);
-		v = (point.z - bounds.min.z) / (bounds.max.z - bounds.min.z);
-		break;
-	case 1:
-	case 4:
-		// Towards Y
-		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16)) * 16;
-		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16)) * 16;
-		u = (point.x - bounds.min.x) / (bounds.max.x - bounds.min.x);
-		v = (point.z - bounds.min.z) / (bounds.max.z - bounds.min.z);
-		break;
-	case 2:
-	case 5:
-		// Towards Z
-		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16)) * 16;
-		height = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16)) * 16;
-		u = (point.x - bounds.min.x) / (bounds.max.x - bounds.min.x);
-		v = (point.y - bounds.min.y) / (bounds.max.y - bounds.min.y);
-		break;
-	default:
-		printf("Error: unknown plane type %d\n", plane->type);
-		return 0;
-	}
-
-	height >>= 4;
-	width >>= 4;
-
-	return lightmap[(int)(v * (width + 1) + u)];
-}
-
-static void export_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, int faceIdx)
-{
-	if (face->lightmap < 0)
-		return;
-
-	const unsigned char* lightmap = &world->lightmap[face->lightmap];
-	const plane_t* plane = &world->planes[face->plane_id];
-
-	int width, height;
-	switch (plane->type)
-	{
-	case 0:
-	case 3:
-		// Towards X
-		width = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16));
-		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16));
-		break;
-	case 1:
-	case 4:
-		// Towards Y
-		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16));
-		height = (int)(ceil(bounds.max.z / 16) - floor(bounds.min.z / 16));
-		break;
-	case 2:
-	case 5:
-		// Towards Z
-		width = (int)(ceil(bounds.max.x / 16) - floor(bounds.min.x / 16));
-		height = (int)(ceil(bounds.max.y / 16) - floor(bounds.min.y / 16));
-		break;
-	default:
-		printf("Error: unknown plane type %d\n", plane->type);
-		return;
-	}
-
-	width += 1;
-
-	char path[_MAX_PATH];
-	sprintf_s(path, _MAX_PATH, "lightmap_face%d_e%d_PT%d_%dx%d.raw", faceIdx, face->ledge_num, plane->type, width, height);
-	FILE* flm;
-	fopen_s(&flm, path, "wb");
-	if (!flm)
-		return;
-
-	for (int y = 0; y < height; ++y)
-	{
-		fwrite(&lightmap[y * width], sizeof(unsigned char), width, flm);
-	}
-
-	fclose(flm);
-}
-
 template<class TData> size_t writeMapData(const std::vector<TData>& data, ps1bsp_dentry_t& dentry, FILE* f)
 {
 	dentry.offset = (unsigned int)ftell(f);
@@ -302,6 +201,8 @@ int process_faces(const world_t* world)
 	outHeader.version = 1;
 	fwrite(&outHeader, sizeof(ps1bsp_header_t), 1, fbsp);
 
+	auto edgeData = analyze_edges(world);
+
 	// Convert vertex data (no vertex splitting yet)
 	std::vector<ps1bsp_vertex_t> outVertices;
 	for (unsigned short i = 0; i < world->numVertices; ++i)
@@ -329,6 +230,7 @@ int process_faces(const world_t* world)
 	// Convert faces defined by edges into faces defined by vertex indices
 	std::vector<ps1bsp_face_t> outFaces;
 	std::vector<ps1bsp_facevertex_t> outFaceVertices;
+	std::unordered_map<const face_t*, BoundBox> faceBounds;
 	for (int faceIdx = 0; faceIdx < world->numFaces; ++faceIdx)
 	{
 		face_t* face = &world->faces[faceIdx];
@@ -367,12 +269,28 @@ int process_faces(const world_t* world)
 			vertexSum = vertexSum + vertexPoint;
 		}
 
+		faceBounds[face] = bounds;
+
+		// For visualizing and debugging lightmaps
+		//if (face->ledge_num >= 10)
+		//	export_lightmap(world, face, bounds, faceIdx);
+
+		outFace.numFaceVertices = (unsigned short)(outFaceVertices.size() - outFace.firstFaceVertex);
+		outFace.centerPoint = convertPoint(vertexSum / outFace.numFaceVertices);
+		outFaces.push_back(outFace);
+	}
+
+	// Iterate over all faces again; now that we know the bounds of each face, we can calculate lighting for all of them
+	for (int faceIdx = 0; faceIdx < world->numFaces; ++faceIdx)
+	{
+		face_t* face = &world->faces[faceIdx];
+		ps1bsp_face_t& outFace = outFaces[faceIdx];
+
 		// Sample lightmap contribution of this face on each vertex
-		for (size_t faceVertIdx = outFace.firstFaceVertex; faceVertIdx < outFaceVertices.size(); ++faceVertIdx)
+		for (size_t faceVertIdx = 0; faceVertIdx < outFace.numFaceVertices; ++faceVertIdx)
 		{
-			ps1bsp_facevertex_t& faceVertex = outFaceVertices[faceVertIdx];
-			unsigned char lightmap = sample_lightmap(world, face, bounds, world->vertices[faceVertex.index].toVec());
-			faceVertex.light = lightmap + (0xFF - face->baselight);
+			ps1bsp_facevertex_t& faceVertex = outFaceVertices[outFace.firstFaceVertex + faceVertIdx];
+			faceVertex.light = compute_faceVertex_light(world, face, faceVertex.index, faceBounds, edgeData);
 
 			if (face->lightmap >= 0)
 			{
@@ -382,13 +300,8 @@ int process_faces(const world_t* world)
 			}
 		}
 
-		// For visualizing and debugging lightmaps
-		//if (face->ledge_num >= 10)
-		//	export_lightmap(world, face, bounds, faceIdx);
-
-		outFace.numFaceVertices = (unsigned short)(outFaceVertices.size() - outFace.firstFaceVertex);
-		outFace.centerPoint = convertPoint(vertexSum / outFace.numFaceVertices);
-		outFaces.push_back(outFace);
+		if (faceIdx > 0 && faceIdx % 100 == 0)
+			printf("Calculated vertex lighting for face %d...\n", faceIdx);
 	}
 
 	// Average the lightmap values for each vertex
@@ -425,8 +338,8 @@ int process_faces(const world_t* world)
 
 		ps1bsp_node_t outNode;
 		outNode.planeId = node->plane_id;
-		outNode.front = node->front;
-		outNode.back = node->back;
+		outNode.children[0] = node->front;
+		outNode.children[1] = node->back;
 
 		outNode.firstFace = node->face_id;
 		outNode.numFaces = node->face_num;

ps1bsp.h

@@ -92,8 +92,7 @@ typedef struct
 typedef struct
 {
 	int planeId;
-	short front;
-	short back;
+	short children[2];
 
 	// TODO: add bounding box for frustum culling