Implemented an alternative method for computing lighting per face vertex, one that's simpler, faster and that produces better results.

Still not quite perfect yet but about where I want it to be for now.

common.h

@@ -5,6 +5,7 @@
 #include <cmath>
 #include <vector>
 #include <unordered_map>
+#include <unordered_set>
 
 typedef float scalar_t;        // Scalar value,
 

lighting.cpp

@@ -147,7 +147,7 @@ std::unordered_map<const edge_t*, EdgeData> analyze_edges(const world_t* world)
 			const plane_t* planeB = &world->planes[faceB->plane_id];
 			vec3_t normalA = faceA->side ? -planeA->normal : planeA->normal;
 			vec3_t normalB = faceB->side ? -planeB->normal : planeB->normal;
-			float dot = dotProduct(planeA->normal, planeB->normal);
+			float dot = dotProduct(normalA, normalB);
 			bool isSmooth = dot >= 0.5f;//&& dot <= 1;
 			iter->second.isSharpEdge = !isSmooth;
 			break;
@@ -161,7 +161,7 @@ std::unordered_map<const edge_t*, EdgeData> analyze_edges(const world_t* world)
 	return edgeData;
 }
 
-unsigned char compute_faceVertex_light(const world_t* world, const face_t* face, unsigned short vertexIndex, const std::unordered_map<const face_t*, BoundBox> faceBounds, const std::unordered_map<const edge_t*, EdgeData>& edgeData)
+unsigned char compute_faceVertex_light(const world_t* world, const face_t* face, unsigned short vertexIndex, const FaceBounds& faceBounds, const std::unordered_map<const edge_t*, EdgeData>& edgeData)
 {
 	const vertex_t* vertex = &world->vertices[vertexIndex];
 	auto point = vertex->toVec();
@@ -212,3 +212,40 @@ unsigned char compute_faceVertex_light(const world_t* world, const face_t* face,
 
 	return (unsigned char)(light / numSamples);
 }
+
+unsigned char compute_faceVertex_light2(const world_t* world, const face_t* face, unsigned short vertexIndex, const FaceBounds& faceBounds, const VertexFaces& vertexFaces)
+{
+	const vertex_t* vertex = &world->vertices[vertexIndex];
+	auto vertexFaceIter = vertexFaces.find(vertex);
+	if (vertexFaceIter == vertexFaces.end())
+		return 0;
+
+	auto point = vertex->toVec();
+
+	// Sample this face's lighting contribution
+	unsigned int light = sample_lightmap(world, face, faceBounds.find(face)->second, point) + (0xFF - face->baselight);
+	int numSamples = 1;
+
+	const plane_t* thisPlane = &world->planes[face->plane_id];
+	vec3_t thisNormal = face->side ? -thisPlane->normal : thisPlane->normal;
+
+	// Gather light samples from other faces adjacent to this vertex
+	for (auto faceIter = vertexFaceIter->second.begin(); faceIter != vertexFaceIter->second.end(); ++faceIter)
+	{
+		const face_t* otherFace = *faceIter;
+		if (otherFace == face)
+			continue;
+
+		const plane_t* otherPlane = &world->planes[otherFace->plane_id];
+		vec3_t otherNormal = otherFace->side ? -otherPlane->normal : otherPlane->normal;
+
+		float dot = dotProduct(thisNormal, otherNormal);
+		if (dot < 0.5f)
+			continue;	// Sharp edge, we don't want light contribution from this face
+
+		light += sample_lightmap(world, otherFace, faceBounds.find(otherFace)->second, point) + (0xFF - otherFace->baselight);
+		++numSamples;
+	}
+
+	return (unsigned char)(light / numSamples);
+}

lighting.h

@@ -10,8 +10,12 @@ struct EdgeData
 	bool isSharpEdge = false;
 };
 
+typedef std::unordered_map<const face_t*, BoundBox> FaceBounds;
+typedef std::unordered_map<const vertex_t*, std::unordered_set<const face_t*>> VertexFaces;
+
 unsigned char sample_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, const Vec3& point);
 void export_lightmap(const world_t* world, const face_t* face, const BoundBox& bounds, int faceIdx);
 
 std::unordered_map<const edge_t*, EdgeData> analyze_edges(const world_t* world);
-unsigned char compute_faceVertex_light(const world_t* world, const face_t* face, unsigned short vertexIndex, const std::unordered_map<const face_t*, BoundBox> faceBounds, const std::unordered_map<const edge_t*, EdgeData>& edgeData);
+unsigned char compute_faceVertex_light(const world_t* world, const face_t* face, unsigned short vertexIndex, const FaceBounds& faceBounds, const std::unordered_map<const edge_t*, EdgeData>& edgeData);
+unsigned char compute_faceVertex_light2(const world_t* world, const face_t* face, unsigned short vertexIndex, const FaceBounds& faceBounds, const VertexFaces& vertexFaces);

main.cpp

@@ -230,7 +230,8 @@ int process_faces(const world_t* world)
 	// Convert faces defined by edges into faces defined by vertex indices
 	std::vector<ps1bsp_face_t> outFaces;
 	std::vector<ps1bsp_facevertex_t> outFaceVertices;
-	std::unordered_map<const face_t*, BoundBox> faceBounds;
+	FaceBounds faceBounds;
+	VertexFaces vertexFaces;
 	for (int faceIdx = 0; faceIdx < world->numFaces; ++faceIdx)
 	{
 		face_t* face = &world->faces[faceIdx];
@@ -265,6 +266,12 @@ int process_faces(const world_t* world)
 			else
 				bounds.includePoint(vertexPoint);
 
+			auto vertexIter = vertexFaces.find(vertex);
+			if (vertexIter == vertexFaces.end())
+				vertexFaces[vertex] = std::unordered_set<const face_t*>{ face };
+			else
+				vertexIter->second.insert(face);
+
 			// Sum all vertices to calculate an average center point
 			vertexSum = vertexSum + vertexPoint;
 		}
@@ -290,7 +297,7 @@ int process_faces(const world_t* world)
 		for (size_t faceVertIdx = 0; faceVertIdx < outFace.numFaceVertices; ++faceVertIdx)
 		{
 			ps1bsp_facevertex_t& faceVertex = outFaceVertices[outFace.firstFaceVertex + faceVertIdx];
-			faceVertex.light = compute_faceVertex_light(world, face, faceVertex.index, faceBounds, edgeData);
+			faceVertex.light = compute_faceVertex_light2(world, face, faceVertex.index, faceBounds, vertexFaces);
 
 			if (face->lightmap >= 0)
 			{
@@ -299,9 +306,6 @@ int process_faces(const world_t* world)
 				vertex.r++;
 			}
 		}
-
-		if (faceIdx > 0 && faceIdx % 100 == 0)
-			printf("Calculated vertex lighting for face %d...\n", faceIdx);
 	}
 
 	// Average the lightmap values for each vertex