Reworked lightmap sampling using a lightmap space transform and correctly rounding values, using the Quake source for reference. Finally fixes a bunch of lingering lighting artifacts.

lighting.cpp

@@ -6,49 +6,23 @@ bool sample_lightmap(const world_t* world, const face_t* face, const FaceBound&
 	if (face->lightmap < 0)
 		return false;
 
-	const unsigned char* lightmap = &world->lightmap[face->lightmap];
-	const plane_t* plane = &world->planes[face->plane_id];
+	Vec3 minBounds = (bounds.lightmapBounds.min / 16).floor();
+	Vec3 maxBounds = (bounds.lightmapBounds.max / 16).ceil();
 
-	Vec3 minBounds = (bounds.worldBounds.min / 16).floor() * 16.f;
-	Vec3 maxBounds = (bounds.worldBounds.max / 16).ceil() * 16.f;
+	int width = (int)((maxBounds.x - minBounds.x) * 16.f);	// extents[0]
+	int height = (int)((maxBounds.y - minBounds.y) * 16.f);	// extents[1]
 
-	int width, height;
-	int u, v;
-	switch (plane->type)
-	{
-	case 0:
-	case 3:
-		// Towards X
-		width = (int)(maxBounds.y - minBounds.y);
-		height = (int)(maxBounds.z - minBounds.z);
-		u = (int)(point.y - minBounds.y);
-		v = (int)(point.z - minBounds.z);
-		break;
-	case 1:
-	case 4:
-		// Towards Y
-		width = (int)(maxBounds.x - minBounds.x);
-		height = (int)(maxBounds.z - minBounds.z);
-		u = (int)(point.x - minBounds.x);
-		v = (int)(point.z - minBounds.z);
-		break;
-	case 2:
-	case 5:
-		// Towards Z
-		width = (int)(maxBounds.x - minBounds.x);
-		height = (int)(maxBounds.y - minBounds.y);
-		u = (int)(point.x - minBounds.x);
-		v = (int)(point.y - minBounds.y);
-		break;
-	default:
-		printf("Error: unknown plane type %d\n", plane->type);
-		return 0;
-	}
+	minBounds = minBounds * 16.f;	// texturemins.xy
+
+	Vec3 uv = bounds.lightmapTransform.TransformPoint(point);
+	int u = (int)(uv.x - minBounds.x);
+	int v = (int)(uv.y - minBounds.y);
 
 	if (u < 0 || v < 0 || u > width || v > height)
 		return false;
 
-	*outSample = lightmap[(v >> 4) * (width >> 4) + (u >> 4)];
+	const unsigned char* lightmap = &world->lightmap[face->lightmap];
+	*outSample = lightmap[(v >> 4) * ((width >> 4) + 1) + (u >> 4)];
 	return true;
 }
 

lighting.h

@@ -21,11 +21,18 @@ struct FaceBound
 	BoundBox worldBounds;
 	Matrix4x4 textureTransform;
 	BoundBox textureBounds;
+	Matrix4x4 lightmapTransform;
+	BoundBox lightmapBounds;
 
 	void addTexturePoint(double s, double t)
 	{
 		textureBounds.includePoint(Vec3(s, t, 0.0));
 	}
+
+	void addLightmapPoint(double s, double t)
+	{
+		lightmapBounds.includePoint(Vec3(s, t, 0.0));
+	}
 };
 
 typedef std::unordered_map<const face_t*, FaceBound> FaceBounds;

main.cpp

@@ -112,7 +112,8 @@ int process_faces(const world_t* world, const TextureList& textures)
 		outFace.textureId = (unsigned char)texinfo->texture_id;
 
 		FaceBound bounds;
-		bounds.textureTransform = tesselator.buildTextureSpaceTransform(texinfo, miptex, plane);
+		bounds.textureTransform = Tesselator::buildTextureSpaceTransform(texinfo, miptex, plane);
+		bounds.lightmapTransform = Tesselator::buildLightmapTransform(texinfo, plane);
 
 		// Traverse the list of face edges to collect all of the face's vertices
 		Vec3 vertexSum;
@@ -136,6 +137,9 @@ int process_faces(const world_t* world, const TextureList& textures)
 			Vec3 texturePoint = bounds.textureTransform.TransformPoint(vertexPoint);
 			bounds.addTexturePoint(texturePoint.x, texturePoint.y);
 
+			Vec3 lightmapPoint = bounds.lightmapTransform.TransformPoint(vertexPoint);
+			bounds.addLightmapPoint(lightmapPoint.x, lightmapPoint.y);
+
 			// Sum all vertices to calculate an average center point
 			vertexSum = vertexSum + vertexPoint;
 

tesselate.cpp

@@ -118,3 +118,13 @@ Matrix4x4 Tesselator::buildTextureSpaceTransform(const texinfo_t* texinfo, const
 	matrix.SetTranslation(Vec3(texinfo->distS / miptex->width, texinfo->distT / miptex->height, -plane->dist));
 	return matrix;
 }
+
+Matrix4x4 Tesselator::buildLightmapTransform(const texinfo_t* texinfo, const plane_t* plane)
+{
+	Matrix4x4 matrix;
+	matrix.SetAxis(0, texinfo->vectorS);
+	matrix.SetAxis(1, texinfo->vectorT);
+	matrix.SetAxis(2, plane->normal);
+	matrix.SetTranslation(Vec3(texinfo->distS, texinfo->distT, -plane->dist));
+	return matrix;
+}

tesselate.h

@@ -48,6 +48,7 @@ public:
 	std::vector<Polygon> tesselateFace(const face_t* face);
 
 	static Matrix4x4 buildTextureSpaceTransform(const texinfo_t* texinfo, const miptex_t* miptex, const plane_t* plane);
+	static Matrix4x4 buildLightmapTransform(const texinfo_t* texinfo, const plane_t* plane);
 
 private:
 	const world_t* world;