Introduced some matrix math to properly construct a texture space transform, that we can also invert to create a transform back to world space. Finally our tesselated polygons are working perfectly!

PS1BSP.vcxproj

@@ -153,6 +153,7 @@
     <ClInclude Include="common.h" />
     <ClInclude Include="gpc.h" />
     <ClInclude Include="lighting.h" />
+    <ClInclude Include="matrix.h" />
     <ClInclude Include="ps1bsp.h" />
     <ClInclude Include="ps1types.h" />
     <ClInclude Include="rectpack\best_bin_finder.h" />

PS1BSP.vcxproj.filters

@@ -83,6 +83,9 @@
     <ClInclude Include="tesselate.h">
       <Filter>Header Files</Filter>
     </ClInclude>
+    <ClInclude Include="matrix.h">
+      <Filter>Header Files</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <CopyFileToFolders Include="palette.lmp">

bsp.h

@@ -71,6 +71,12 @@ typedef struct Plane
         tangent = normal.crossProduct(dir).normalized();
         bitangent = tangent.crossProduct(normal);
     }
+
+    Vec3 projectPoint(const Vec3& point) const
+    {
+        double pointDist = normal.dotProduct(point) - dist;
+        return point - normal * pointDist;
+    }
 } plane_t;
 
 typedef struct BoundBox                 // Bounding Box, Float values

matrix.h

@@ -0,0 +1,172 @@
+#pragma once
+
+class Matrix4x4
+{
+public:
+	union
+	{
+		double m[16];
+		double _m[4][4];
+	};
+
+	Matrix4x4() { LoadIdentity(); }
+
+	void LoadNull()
+	{
+		for (int i = 0; i < 16; i++)
+			m[i] = 0;
+	}
+
+	void LoadIdentity()
+	{
+		m[0] = m[5] = m[10] = m[15] = 1;
+		m[1] = m[2] = m[3] = m[4] =
+		m[6] = m[7] = m[8] = m[9] =
+		m[11] = m[12] = m[13] = m[14] = 0;
+	}
+
+	void SetAxis(int axis, const Vec3& value)
+	{
+		m[0 + axis] = value.x;
+		m[4 + axis] = value.y;
+		m[8 + axis] = value.z;
+	}
+
+	void SetTranslation(const Vec3& value)
+	{
+		m[12] = value.x;
+		m[13] = value.y;
+		m[14] = value.z;
+	}
+
+	Vec3 TransformPoint(const Vec3& point)
+	{
+		return Vec3(
+			(double)point.x * m[0] +
+			(double)point.y * m[4] +
+			(double)point.z * m[8] +
+			m[12],
+
+			(double)point.x * m[1] +
+			(double)point.y * m[5] +
+			(double)point.z * m[9] +
+			m[13],
+
+			(double)point.x * m[2] +
+			(double)point.y * m[6] +
+			(double)point.z * m[10] +
+			m[14]);
+	}
+
+	Vec3 TransformDirection(const Vec3& dir)
+	{
+		return Vec3(
+			(double)dir.x * m[0] +
+			(double)dir.y * m[4] +
+			(double)dir.z * m[8],
+		
+			(double)dir.x * m[1] +
+			(double)dir.y * m[5] +
+			(double)dir.z * m[9],
+
+			(double)dir.x * m[2] +
+			(double)dir.y * m[6] +
+			(double)dir.z * m[10]);
+	}
+
+	bool Invert()
+	{
+		double tmp[12];
+		double src[16];
+		double dst[16];
+
+		// Transpose matrix
+		for (int i = 0; i < 4; i++) {
+			src[i + 0] = m[i * 4 + 0];
+			src[i + 4] = m[i * 4 + 1];
+			src[i + 8] = m[i * 4 + 2];
+			src[i + 12] = m[i * 4 + 3];
+		}
+
+		// Calculate pairs for first 8 elements (cofactors) 
+		tmp[0] = src[10] * src[15];
+		tmp[1] = src[11] * src[14];
+		tmp[2] = src[9] * src[15];
+		tmp[3] = src[11] * src[13];
+		tmp[4] = src[9] * src[14];
+		tmp[5] = src[10] * src[13];
+		tmp[6] = src[8] * src[15];
+		tmp[7] = src[11] * src[12];
+		tmp[8] = src[8] * src[14];
+		tmp[9] = src[10] * src[12];
+		tmp[10] = src[8] * src[13];
+		tmp[11] = src[9] * src[12];
+
+		// Calculate first 8 elements (cofactors)
+		dst[0] = tmp[0] * src[5] + tmp[3] * src[6] + tmp[4] * src[7];
+		dst[0] -= tmp[1] * src[5] + tmp[2] * src[6] + tmp[5] * src[7];
+		dst[1] = tmp[1] * src[4] + tmp[6] * src[6] + tmp[9] * src[7];
+		dst[1] -= tmp[0] * src[4] + tmp[7] * src[6] + tmp[8] * src[7];
+		dst[2] = tmp[2] * src[4] + tmp[7] * src[5] + tmp[10] * src[7];
+		dst[2] -= tmp[3] * src[4] + tmp[6] * src[5] + tmp[11] * src[7];
+		dst[3] = tmp[5] * src[4] + tmp[8] * src[5] + tmp[11] * src[6];
+		dst[3] -= tmp[4] * src[4] + tmp[9] * src[5] + tmp[10] * src[6];
+		dst[4] = tmp[1] * src[1] + tmp[2] * src[2] + tmp[5] * src[3];
+		dst[4] -= tmp[0] * src[1] + tmp[3] * src[2] + tmp[4] * src[3];
+		dst[5] = tmp[0] * src[0] + tmp[7] * src[2] + tmp[8] * src[3];
+		dst[5] -= tmp[1] * src[0] + tmp[6] * src[2] + tmp[9] * src[3];
+		dst[6] = tmp[3] * src[0] + tmp[6] * src[1] + tmp[11] * src[3];
+		dst[6] -= tmp[2] * src[0] + tmp[7] * src[1] + tmp[10] * src[3];
+		dst[7] = tmp[4] * src[0] + tmp[9] * src[1] + tmp[10] * src[2];
+		dst[7] -= tmp[5] * src[0] + tmp[8] * src[1] + tmp[11] * src[2];
+
+		// Calculate pairs for second 8 elements (cofactors)
+		tmp[0] = src[2] * src[7];
+		tmp[1] = src[3] * src[6];
+		tmp[2] = src[1] * src[7];
+		tmp[3] = src[3] * src[5];
+		tmp[4] = src[1] * src[6];
+		tmp[5] = src[2] * src[5];
+		tmp[6] = src[0] * src[7];
+		tmp[7] = src[3] * src[4];
+		tmp[8] = src[0] * src[6];
+		tmp[9] = src[2] * src[4];
+		tmp[10] = src[0] * src[5];
+		tmp[11] = src[1] * src[4];
+
+		// Calculate second 8 elements (cofactors)
+		dst[8] = tmp[0] * src[13] + tmp[3] * src[14] + tmp[4] * src[15];
+		dst[8] -= tmp[1] * src[13] + tmp[2] * src[14] + tmp[5] * src[15];
+		dst[9] = tmp[1] * src[12] + tmp[6] * src[14] + tmp[9] * src[15];
+		dst[9] -= tmp[0] * src[12] + tmp[7] * src[14] + tmp[8] * src[15];
+		dst[10] = tmp[2] * src[12] + tmp[7] * src[13] + tmp[10] * src[15];
+		dst[10] -= tmp[3] * src[12] + tmp[6] * src[13] + tmp[11] * src[15];
+		dst[11] = tmp[5] * src[12] + tmp[8] * src[13] + tmp[11] * src[14];
+		dst[11] -= tmp[4] * src[12] + tmp[9] * src[13] + tmp[10] * src[14];
+		dst[12] = tmp[2] * src[10] + tmp[5] * src[11] + tmp[1] * src[9];
+		dst[12] -= tmp[4] * src[11] + tmp[0] * src[9] + tmp[3] * src[10];
+		dst[13] = tmp[8] * src[11] + tmp[0] * src[8] + tmp[7] * src[10];
+		dst[13] -= tmp[6] * src[10] + tmp[9] * src[11] + tmp[1] * src[8];
+		dst[14] = tmp[6] * src[9] + tmp[11] * src[11] + tmp[3] * src[8];
+		dst[14] -= tmp[10] * src[11] + tmp[2] * src[8] + tmp[7] * src[9];
+		dst[15] = tmp[10] * src[10] + tmp[4] * src[8] + tmp[9] * src[9];
+		dst[15] -= tmp[8] * src[9] + tmp[11] * src[10] + tmp[5] * src[8];
+
+		// Calculate determinant
+		double det = src[0] * dst[0] + src[1] * dst[1] + src[2] * dst[2] + src[3] * dst[3];
+
+		if (fabs(det) < FLT_EPSILON)
+		{
+			return false;
+		}
+		else
+		{
+			// Calculate matrix inverse
+			det = 1.0 / det;
+			for (int i = 0; i < 16; i++)
+				m[i] = dst[i] * det;
+		}
+
+		return true;
+	}
+};

tesselate.cpp

@@ -2,6 +2,7 @@
 #include "bsp.h"
 #include "tesselate.h"
 #include "gpc.h"
+#include "matrix.h"
 
 std::vector<Tesselator::Polygon> Tesselator::tesselateFace(const face_t* face)
 {
@@ -11,9 +12,6 @@ std::vector<Tesselator::Polygon> Tesselator::tesselateFace(const face_t* face)
 	const miptex_t* miptex = &world->miptexes[texinfo->texture_id];
 	const plane_t* plane = &world->planes[face->plane_id];
 
-	double minS = DBL_MAX, minT = DBL_MAX;
-	double maxS = DBL_MIN, maxT = DBL_MIN;
-
 	gpc_polygon facePolygon = { 0 };
 	gpc_vertex_list contour;
 	contour.num_vertices = face->ledge_num;
@@ -21,8 +19,17 @@ std::vector<Tesselator::Polygon> Tesselator::tesselateFace(const face_t* face)
 	if (contour.vertex == NULL)
 		return polygons;
 
-	double invSLenSqr = 1.0 / texinfo->vectorS.sqrMagnitude();
-	double invTLenSqr = 1.0 / texinfo->vectorT.sqrMagnitude();
+	double minS = DBL_MAX, minT = DBL_MAX;
+	double maxS = DBL_MIN, maxT = DBL_MIN;
+
+	// vectorS and vectorT are normally perpendicular (dot product is 0), magnitude isn't always 1 but that's fine.
+	// Means we can construct a coordinate space from them (plane normal for the third vector) and transform the vertices to texture space.
+	// And we can create an inverse transform, meaning we can transform vertices from 2D texture space back to 3D world space.
+	Matrix4x4 textureTrsf;
+	textureTrsf.SetAxis(0, texinfo->vectorS / (float)miptex->width);
+	textureTrsf.SetAxis(1, texinfo->vectorT / (float)miptex->height);
+	textureTrsf.SetAxis(2, plane->normal);
+	textureTrsf.SetTranslation(Vec3(texinfo->distS / miptex->width, texinfo->distT / miptex->height, -plane->dist));
 	
 	// Build a polygon in normalized 2D texture space from the original face data
 	std::vector<Vec3> faceVertices;
@@ -38,22 +45,22 @@ std::vector<Tesselator::Polygon> Tesselator::tesselateFace(const face_t* face)
 		Vec3 vertexPoint = vertex->toVec();
 		faceVertices.push_back(vertexPoint);
 
-		// vectorS and vectorT are normally perpendicular (dot product is 0), magnitude isn't always 1 but that's fine
-		// Means we can construct a coordinate space from them (cross product for the third vector) and transform the vertex point to ST-space
-		// And we can create an inverse transform... though just having s and t values probably isn't enough to completely transform back...
+		// Transform the vertex to texture space and calculate the texture UV bounds
+		Vec3 st = textureTrsf.TransformPoint(vertexPoint);
+		if (st.x > maxS) maxS = st.x; if (st.x < minS) minS = st.x;
+		if (st.y > maxT) maxT = st.y; if (st.y < minT) minT = st.y;
 
-		// Calculate texture UV bounds
-		double s = (vertexPoint.dotProduct(texinfo->vectorS) + texinfo->distS) / miptex->width;
-		double t = (vertexPoint.dotProduct(texinfo->vectorT) + texinfo->distT) / miptex->height;
-		if (s > maxS) maxS = s; if (s < minS) minS = s;
-		if (t > maxT) maxT = t; if (t < minT) minT = t;
-
-		contour.vertex[edgeListIdx] = gpc_vertex{ s, t };
+		contour.vertex[edgeListIdx] = gpc_vertex{ st.x, st.y };
 	}
 
 	gpc_add_contour(&facePolygon, &contour, 0);
 
-	auto faceVert = *faceVertices.begin();
+	// Invert the texture matrix so we can transform vertices from 2D texture space back to 3D world space
+	if (!textureTrsf.Invert())
+	{
+		printf("Failed to invert texture space transform!\n");
+		return polygons;
+	}
 
 	// Create a virtual grid at the texture bounds and iterate over each cell to break up the face into repeating tiles
 	for (double y = floor(minT); y <= ceil(maxT); y += 1.0)
@@ -85,10 +92,9 @@ std::vector<Tesselator::Polygon> Tesselator::tesselateFace(const face_t* face)
 			for (int v = 0; v < result.contour[0].num_vertices; ++v)
 			{
 				const auto vert = &result.contour[0].vertex[v];
-				Vec3 newVert =
-					plane->normal * plane->dist +
-					texinfo->vectorS * (vert->x * miptex->width - texinfo->distS) * invSLenSqr +
-					texinfo->vectorT * (vert->y * miptex->height - texinfo->distT) * invTLenSqr;
+
+				// Transform the vertex back to world space
+				Vec3 newVert = textureTrsf.TransformPoint(Vec3(vert->x, vert->y, 0));
 
 				size_t vertIndex = addVertex(newVert);
 				Vec3 normalizedUV(vert->x - x, vert->y - y, 0);	// Normalize the UV to fall within [0..1] range
@@ -103,12 +109,6 @@ std::vector<Tesselator::Polygon> Tesselator::tesselateFace(const face_t* face)
 		}
 	}
 
-	if (vertexIndices.find(faceVert) == vertexIndices.end())
-	{
-		gpc_free_polygon(&facePolygon);
-		return polygons;
-	}
-
 	gpc_free_polygon(&facePolygon);
 	return polygons;
 }