Fixed meshlet culling

res/shaders/Debug.slang

@@ -4,25 +4,26 @@ struct DebugVertex
 {
 	float3 position;
 	float3 color;
-}
-
-struct VertexOutput
-{
-	float4 clipPosition : SV_Position;
-	float3 color : COLOR0;
 };
 
-[shader("vertex")]
-VertexOutput vertexMain(DebugVertex input)
+struct Params
 {
-	VertexOutput output;
-	output.clipPosition = mul(pViewParams.projectionMatrix, mul(pViewParams.viewMatrix, float4(input.position, 1.0f)));
-	output.color = input.color;
-	return output;
+	float4 pos: SV_Position;
+	float3 color: COLOR0;
+}
+
+[shader("vertex")]
+Params vertexMain(
+	DebugVertex vert
+){
+	Params result;
+	result.pos = mul(pViewParams.projectionMatrix, mul(pViewParams.viewMatrix, float4(vert.position, 1)));
+	result.color = vert.color;
+	return result;
 }
 
 [shader("pixel")]
-float4 fragmentMain(VertexOutput input)
+float4 fragmentMain(in Params params) : SV_Target
 {
-	return float4(input.color, 1.0f);
-}
+	return float4(params.color, 1);
+}

res/shaders/MeshletBasePass.slang

@@ -12,7 +12,7 @@ struct MeshPayload
 
 groupshared MeshPayload p;
 groupshared uint head;
-groupshared float4x4 localToClip;
+groupshared float4x4 localToView;
 groupshared Frustum viewFrustum;
 
 [numthreads(TASK_GROUP_SIZE, 1, 1)]
@@ -26,19 +26,18 @@ void taskMain(
 	if(threadID == 0)
 	{
 		head = 0;
-		localToClip = mul(pViewParams.projectionMatrix, mul(pViewParams.viewMatrix, instance.transformMatrix));
-		// Left
-		viewFrustum.sides[0].n = float3(1, 0, 0);
-		viewFrustum.sides[0].d = -1;
-		// Right
-		viewFrustum.sides[1].n = float3(-1, 0, 0);
-		viewFrustum.sides[1].d = -1;
-		// Top
-		viewFrustum.sides[2].n = float3(0, -1, 0);
-		viewFrustum.sides[2].d = -1;
-		// Bottom
-		viewFrustum.sides[3].n = float3(0, 1, 0);
-		viewFrustum.sides[3].d = -1;
+		localToView = mul(pViewParams.viewMatrix, instance.transformMatrix);
+		float3 origin = float3(0, 0, 0);
+		float3 corners[4] = {
+			screenToView(float4(pViewParams.screenDimensions, -1.0f, 1.0f)).xyz,
+			screenToView(float4(pViewParams.screenDimensions, -1.0f, 1.0f)).xyz,
+			screenToView(float4(pViewParams.screenDimensions, -1.0f, 1.0f)).xyz,
+			screenToView(float4(pViewParams.screenDimensions, -1.0f, 1.0f)).xyz
+		};
+		viewFrustum.sides[0] = computePlane(origin, corners[2], corners[0]);
+		viewFrustum.sides[1] = computePlane(origin, corners[1], corners[3]);
+		viewFrustum.sides[2] = computePlane(origin, corners[0], corners[1]);
+		viewFrustum.sides[3] = computePlane(origin, corners[3], corners[2]);
 	}
 	GroupMemoryBarrierWithGroupSync();
 	MeshData mesh = pScene.meshData[groupID];
@@ -48,7 +47,7 @@ void taskMain(
 		{
 			uint m = mesh.meshletOffset + i;
 			MeshletDescription meshlet = pScene.meshletInfos[m];
-			//if(meshlet.boundingBox.insideFrustum(localToClip, viewFrustum))
+			if(meshlet.bounding.insideFrustum(localToView, viewFrustum))
 			{
 				uint index;
 				InterlockedAdd(head, 1, index);

res/shaders/lib/Bounding.slang

@@ -11,17 +11,20 @@ struct BoundingSphere
     {
         return centerRadius.w;
     }
-    bool pointInside(float3 p)
-    {
-        if(abs(getCenter() - p) > getRadius())
-        {
-            return false;
-        }
-        return true;
-    }
     bool insideFrustum(float4x4 transform, Frustum frustum)
     {
-        
+        float3 transformed = mul(transform, float4(getCenter(), 1)).xyz;
+        float maxScale = max(max(transform[0][0], transform[1][1]), transform[2][2]);
+        float scaledRange = getRadius() * maxScale;
+        bool result = true;
+        for(int i = 0; i < 4 && result; ++i)
+        {
+            if(dot(frustum.sides[i].n, transformed) - frustum.sides[i].d < scaledRange)
+            {
+                result = false;
+            }
+        }
+        return result;
     }
 };
 
@@ -45,7 +48,7 @@ struct AABB
         for(int i = 0; i < 8; ++i)
         {
             float3 adjusted = corners[i].xyz / corners[i].w;
-            if(frustum.pointInside(adjusted))
+            if(frustum.pointInside(corners[i].xyz))
             {
                 return true;
             }

res/shaders/lib/DebugVertexData.slang

@@ -2,7 +2,7 @@ import Common;
 import VertexData;
 import MaterialParameter;
 
-struct StaticMeshVertexData : IVertexData
+struct DebugVertexData : IVertexData
 {
 	VertexAttributes getAttributes(uint index)
 	{

res/shaders/lib/Scene.slang

@@ -2,7 +2,7 @@ import Bounding;
 
 struct MeshletDescription
 {
-	BoundingSphere bounding;
+	AABB bounding;
 	uint32_t vertexCount;
 	uint32_t primitiveCount;
 	uint32_t vertexOffset;
@@ -13,7 +13,7 @@ struct MeshletDescription
 
 struct MeshData
 {
-	BoundingSphere bounding;
+	AABB bounding;
     uint32_t numMeshlets;
     uint32_t meshletOffset;
     uint32_t firstIndex;