Clean up bilateral blur

Instead of using relative depth deltas we really need to use absolute to
have consistent object separation; for this to work on sloped surfaces
we need to reproject estimated delta based on gradient, and cut that off
if the gradient is too steep.

src/niagara.cpp

@@ -864,6 +864,7 @@ int main(int argc, const char** argv)
 	camera.position = { 0.0f, 0.0f, 0.0f };
 	camera.orientation = { 0.0f, 0.0f, 0.0f, 1.0f };
 	camera.fovY = glm::radians(70.f);
+	camera.znear = 0.1f;
 
 	vec3 sunDirection = normalize(vec3(1.0f, 1.0f, 1.0f));
 
@@ -1364,8 +1365,7 @@ int main(int argc, const char** argv)
 		view = inverse(view);
 		view = glm::scale(glm::identity<glm::mat4>(), vec3(1, 1, -1)) * view;
 
-		float znear = 0.1f;
-		mat4 projection = perspectiveProjection(camera.fovY, float(swapchain.width) / float(swapchain.height), znear);
+		mat4 projection = perspectiveProjection(camera.fovY, float(swapchain.width) / float(swapchain.height), camera.znear);
 
 		mat4 projectionT = transpose(projection);
 
@@ -1376,7 +1376,7 @@ int main(int argc, const char** argv)
 		cullData.view = view;
 		cullData.P00 = projection[0][0];
 		cullData.P11 = projection[1][1];
-		cullData.znear = znear;
+		cullData.znear = camera.znear;
 		cullData.zfar = drawDistance;
 		cullData.frustum[0] = frustumX.x;
 		cullData.frustum[1] = frustumX.z;
@@ -1831,7 +1831,7 @@ int main(int argc, const char** argv)
 				DescriptorInfo descriptors[] = { { blurTo.imageView, VK_IMAGE_LAYOUT_GENERAL }, { readSampler, blurFrom.imageView, VK_IMAGE_LAYOUT_GENERAL }, { readSampler, depthTarget.imageView, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL } };
 				vkCmdPushDescriptorSetWithTemplateKHR(commandBuffer, shadowblurProgram.updateTemplate, shadowblurProgram.layout, 0, descriptors);
 
-				vec4 blurData = vec4(float(swapchain.width), float(swapchain.height), pass == 0 ? 1 : 0, 0);
+				vec4 blurData = vec4(float(swapchain.width), float(swapchain.height), pass == 0 ? 1 : 0, camera.znear);
 
 				vkCmdPushConstants(commandBuffer, shadowblurProgram.layout, shadowblurProgram.pushConstantStages, 0, sizeof(blurData), &blurData);
 				vkCmdDispatch(commandBuffer, getGroupCount(swapchain.width, shadowblurProgram.localSizeX), getGroupCount(swapchain.height, shadowblurProgram.localSizeY), 1);

src/scene.h

@@ -88,6 +88,7 @@ struct Camera
 	vec3 position;
 	quat orientation;
 	float fovY;
+	float znear;
 };
 
 struct Keyframe

src/shaders/shadow.comp.glsl

@@ -117,11 +117,11 @@ void main()
 {
 	uvec2 pos = gl_GlobalInvocationID.xy;
 
-	if (shadowData.checkerboard == 1)
+	if (shadowData.checkerboard > 0)
 	{
 		// checkerboard even
 		pos.x *= 2;
-		pos.x += pos.y & 1;
+		pos.x += (pos.y ^ shadowData.checkerboard) & 1;
 	}
 
 	vec2 uv = (vec2(pos) + 0.5) / shadowData.imageSize;

src/shaders/shadowblur.comp.glsl

@@ -12,6 +12,7 @@ layout(push_constant) uniform block
 {
 	vec2 imageSize;
 	float direction;
+	float znear;
 };
 
 layout(binding = 0) uniform writeonly image2D outImage;
@@ -27,27 +28,30 @@ void main()
 	float shadow = texelFetch(shadowImage, ivec2(pos), 0).r;
 	float accumw = 1;
 
-	float znear = 1;
 	float depth = znear / texelFetch(depthImage, ivec2(pos), 0).r;
 
 	ivec2 offsetMask = -ivec2(direction, 1 - direction);
 
 	const int KERNEL = 10;
 
-	for (int i = -KERNEL; i <= KERNEL; ++i)
+	for (int sign = -1; sign <= 1; sign += 2)
 	{
-		if (i == 0)
-			continue;
-
-        ivec2 uvoff = ivec2(pos) + (ivec2(i) & offsetMask);
-
-		float gw = exp2(-abs(i) / 10);
-		float dv = znear / texelFetch(depthImage, uvoff, 0).r;
-		float dw = exp2(-abs(depth / dv - 1) * 200);
-		float fw = gw * dw;
-
-		shadow += texelFetch(shadowImage, uvoff, 0).r * fw;
-		accumw += fw;
+		ivec2 uvnext = ivec2(pos) + (ivec2(sign) & offsetMask);
+		float dnext = znear / texelFetch(depthImage, uvnext, 0).r;
+		float dgrad = abs(depth - dnext) < 0.1 ? dnext - depth : 0;
+
+		for (int i = 1; i <= KERNEL; ++i)
+		{
+	        ivec2 uvoff = ivec2(pos) + (ivec2(i * sign) & offsetMask);
+
+			float gw = exp2(-i * i / 50);
+			float dv = znear / texelFetch(depthImage, uvoff, 0).r;
+			float dw = exp2(-abs(dv - (depth + dgrad * i)) * 100);
+			float fw = gw * dw;
+
+			shadow += texelFetch(shadowImage, uvoff, 0).r * fw;
+			accumw += fw;
+		}
 	}
 
 	shadow /= accumw;

src/shaders/shadowfill.comp.glsl

@@ -18,7 +18,7 @@ void main()
 
 	// checkerboard odd
 	pos.x *= 2;
-	pos.x += 1 - (pos.y & 1);
+	pos.x += pos.y & 1;
 
 	float depth = texelFetch(depthImage, pos, 0).r;