From 28ca91c32f9d3832396a03b46f74509167e0b477 Mon Sep 17 00:00:00 2001 From: Nico de Poel Date: Fri, 26 Jul 2024 20:00:33 +0200 Subject: [PATCH] Implemented prepare process and started on main dispatch --- .../FrameInterpolation/FrameInterpolation.cs | 2 +- .../FrameInterpolationContext.cs | 134 ++++++++++++++++++ .../FrameInterpolationPass.cs | 33 ++--- 3 files changed, 146 insertions(+), 23 deletions(-) diff --git a/Runtime/FrameInterpolation/FrameInterpolation.cs b/Runtime/FrameInterpolation/FrameInterpolation.cs index 5e36c67..5bd8538 100644 --- a/Runtime/FrameInterpolation/FrameInterpolation.cs +++ b/Runtime/FrameInterpolation/FrameInterpolation.cs @@ -41,7 +41,7 @@ namespace FidelityFX.FrameGen public Vector2Int displaySize; public Vector2Int renderSize; public ResourceView currentBackBuffer; - public ResourceView currentBackBuffer_HUDLess; + public ResourceView currentBackBuffer_HUDLess; // Optional public ResourceView output; public RectInt interpolationRect; diff --git a/Runtime/FrameInterpolation/FrameInterpolationContext.cs b/Runtime/FrameInterpolation/FrameInterpolationContext.cs index 415e4e7..bd9e420 100644 --- a/Runtime/FrameInterpolation/FrameInterpolationContext.cs +++ b/Runtime/FrameInterpolation/FrameInterpolationContext.cs @@ -1,5 +1,6 @@ using System.Runtime.InteropServices; using UnityEngine; +using UnityEngine.Assertions; using UnityEngine.Profiling; using UnityEngine.Rendering; @@ -47,6 +48,8 @@ namespace FidelityFX.FrameGen _spdConstantsBuffer = CreateConstantBuffer(); _firstExecution = true; + _dispatchCount = 0; + _previousFrameID = 0; _asyncSupported = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableAsyncSupport) == FrameInterpolation.InitializationFlags.EnableAsyncSupport; Constants.maxRenderSize = _contextDescription.maxRenderSize; @@ -98,6 +101,30 @@ namespace FidelityFX.FrameGen public void Prepare(CommandBuffer commandBuffer, FrameInterpolation.PrepareDescription prepareDescription) { commandBuffer.BeginSample(_prepareSampler); + + int doubleBufferId = _asyncSupported ? (int)(prepareDescription.frameID & 1) : 0; + + Constants.renderSize = prepareDescription.renderSize; + Constants.jitter = prepareDescription.jitterOffset; + Vector2Int motionVectorsTargetSize = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDisplayResolutionMotionVectors) != 0 ? Constants.displaySize : Constants.renderSize; + Constants.motionVectorScale.x = prepareDescription.motionVectorScale.x / motionVectorsTargetSize.x; + Constants.motionVectorScale.y = prepareDescription.motionVectorScale.y / motionVectorsTargetSize.y; + + commandBuffer.SetBufferData(_frameInterpolationConstantsBuffer, _frameInterpolationConstantsArray); + + Assert.IsTrue(prepareDescription.depth.IsValid); + Assert.IsTrue(prepareDescription.motionVectors.IsValid); + + // clear estimated depth resources + { + bool inverted = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDepthInverted) == FrameInterpolation.InitializationFlags.EnableDepthInverted; + commandBuffer.SetRenderTarget(_resources.ReconstructedDepth[doubleBufferId]); + commandBuffer.ClearRenderTarget(false, true, inverted ? Color.clear : Color.white); + } + + int renderDispatchSizeX = (prepareDescription.renderSize.x + 7) / 8; + int renderDispatchSizeY = (prepareDescription.renderSize.y + 7) / 8; + ((FrameInterpolationReconstructAndDilatePass)_reconstructAndDilatePass).ScheduleDispatch(commandBuffer, prepareDescription, doubleBufferId, renderDispatchSizeX, renderDispatchSizeY); commandBuffer.EndSample(_prepareSampler); } @@ -105,10 +132,117 @@ namespace FidelityFX.FrameGen public void Dispatch(CommandBuffer commandBuffer, FrameInterpolation.DispatchDescription dispatchDescription) { commandBuffer.BeginSample(_sampler); + + bool reset = _dispatchCount == 0 || dispatchDescription.reset; + Assert.IsTrue(!_asyncSupported || reset || dispatchDescription.frameID > _previousFrameID, + "When async support is enabled, and the reset flag is not set, frame ID must increment in each dispatch."); + + bool frameIdDecreased = dispatchDescription.frameID < _previousFrameID; + bool frameIdSkipped = (dispatchDescription.frameID - _previousFrameID) > 1; + bool disjointFrameId = frameIdDecreased || frameIdSkipped; + _previousFrameID = dispatchDescription.frameID; + _dispatchCount++; // TODO: this is pointless, it does the same as _firstExecution, no need to do any counting + + Constants.renderSize = dispatchDescription.renderSize; + Constants.displaySize = dispatchDescription.displaySize; + Constants.displaySizeRcp.x = 1.0f / dispatchDescription.displaySize.x; + Constants.displaySizeRcp.y = 1.0f / dispatchDescription.displaySize.y; + Constants.upscalerTargetSize = dispatchDescription.interpolationRect.size; + Constants.mode = 0; + Constants.reset = (reset || disjointFrameId) ? 1 : 0; + Constants.deltaTime = dispatchDescription.frameTimeDelta; + Constants.HUDLessAttachedFactor = dispatchDescription.currentBackBuffer_HUDLess.IsValid ? 1 : 0; + + Constants.opticalFlowScale = dispatchDescription.opticalFlowScale; + Constants.opticalFlowBlockSize = dispatchDescription.opticalFlowBlockSize; + Constants.dispatchFlags = (uint)dispatchDescription.flags; + + Constants.cameraNear = dispatchDescription.cameraNear; + Constants.cameraFar = dispatchDescription.cameraFar; + + Constants.interpolationRectBase = dispatchDescription.interpolationRect.position; + Constants.interpolationRectSize = dispatchDescription.interpolationRect.size; + + // Debug bar + Constants.debugBarColor.x = DebugBarColorSequence[_debugIndex * 3 + 0]; + Constants.debugBarColor.y = DebugBarColorSequence[_debugIndex * 3 + 1]; + Constants.debugBarColor.z = DebugBarColorSequence[_debugIndex * 3 + 2]; + _debugIndex = (_debugIndex + 1) % (DebugBarColorSequence.Length / 3); + + Constants.backBufferTransferFunction = (uint)dispatchDescription.backbufferTransferFunction; + Constants.minMaxLuminance = dispatchDescription.minMaxLuminance; + + float aspectRatio = dispatchDescription.renderSize.x / (float)dispatchDescription.renderSize.y; + float cameraAngleHorizontal = Mathf.Atan(Mathf.Tan(dispatchDescription.cameraFovAngleVertical / 2) * aspectRatio) * 2; + Constants.tanHalfFOV = Mathf.Tan(cameraAngleHorizontal * 0.5f); + Constants.deviceToViewDepth = SetupDeviceDepthToViewSpaceDepthParams(dispatchDescription); + + commandBuffer.SetBufferData(_frameInterpolationConstantsBuffer, _frameInterpolationConstantsArray); + + int doubleBufferId = _asyncSupported ? (int)(dispatchDescription.frameID & 1) : 0; + + int displayDispatchSizeX = (dispatchDescription.displaySize.x + 7) / 8; + int displayDispatchSizeY = (dispatchDescription.displaySize.y + 7) / 8; + + int renderDispatchSizeX = (dispatchDescription.renderSize.x + 7) / 8; + int renderDispatchSizeY = (dispatchDescription.renderSize.y + 7) / 8; + + int opticalFlowDispatchSizeX = (int)(dispatchDescription.displaySize.x / (float)dispatchDescription.opticalFlowBlockSize + 7) / 8; + int opticalFlowDispatchSizeY = (int)(dispatchDescription.displaySize.y / (float)dispatchDescription.opticalFlowBlockSize + 7) / 8; + + bool executePreparationPasses = (Constants.reset == 0); + + // Schedule work for the interpolation command list + // TODO commandBuffer.EndSample(_sampler); } + private Vector4 SetupDeviceDepthToViewSpaceDepthParams(FrameInterpolation.DispatchDescription dispatchParams) + { + bool inverted = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDepthInverted) != 0; + bool infinite = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDepthInfinite) != 0; + + // make sure it has no impact if near and far plane values are swapped in dispatch params + // the flags "inverted" and "infinite" will decide what transform to use + float min = Mathf.Min(dispatchParams.cameraNear, dispatchParams.cameraFar); + float max = Mathf.Max(dispatchParams.cameraNear, dispatchParams.cameraFar); + + if (inverted) + { + (min, max) = (max, min); + } + + float q = max / (min - max); + float d = -1.0f; + + Vector4 matrixElemC = new Vector4(q, -1.0f - Mathf.Epsilon, q, 0.0f + Mathf.Epsilon); + Vector4 matrixElemE = new Vector4(q * min, -min - Mathf.Epsilon, q * min, max); + + // Revert x and y coords + float aspect = (float)dispatchParams.renderSize.x / dispatchParams.renderSize.y; + float cotHalfFovY = Mathf.Cos(0.5f * dispatchParams.cameraFovAngleVertical) / Mathf.Sin(0.5f * dispatchParams.cameraFovAngleVertical); + + int matrixIndex = (inverted ? 2 : 0) + (infinite ? 1 : 0); + return new Vector4( + d * matrixElemC[matrixIndex], + matrixElemE[matrixIndex], + aspect / cotHalfFovY, + 1.0f / cotHalfFovY); + } + + private int _debugIndex = 0; + private static readonly float[] DebugBarColorSequence = + { + 0.0f, 1.0f, 1.0f, // teal + 1.0f, 0.42f, 0.0f, // orange + 0.0f, 0.16f, 1.0f, // blue + 0.74f, 1.0f, 0.0f, // lime + 0.68f, 0.0f, 1.0f, // purple + 0.0f, 1.0f, 0.1f, // green + 1.0f, 1.0f, 0.48f // bright yellow + }; + private static ComputeBuffer CreateConstantBuffer() where TConstants: struct { return new ComputeBuffer(1, Marshal.SizeOf(), ComputeBufferType.Constant); diff --git a/Runtime/FrameInterpolation/FrameInterpolationPass.cs b/Runtime/FrameInterpolation/FrameInterpolationPass.cs index 728c009..2a35e31 100644 --- a/Runtime/FrameInterpolation/FrameInterpolationPass.cs +++ b/Runtime/FrameInterpolation/FrameInterpolationPass.cs @@ -17,8 +17,6 @@ namespace FidelityFX.FrameGen protected CustomSampler Sampler; - protected bool AsyncSupported => (ContextDescription.flags & FrameInterpolation.InitializationFlags.EnableAsyncSupport) == FrameInterpolation.InitializationFlags.EnableAsyncSupport; - protected FrameInterpolationPass(FrameInterpolation.ContextDescription contextDescription, FrameInterpolationResources resources, ComputeBuffer constants) { ContextDescription = contextDescription; @@ -84,10 +82,8 @@ namespace FidelityFX.FrameGen { } - public void ScheduleDispatch(CommandBuffer commandBuffer, FrameInterpolation.PrepareDescription prepareParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ) + public void ScheduleDispatch(CommandBuffer commandBuffer, FrameInterpolation.PrepareDescription prepareParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ = 1) { - int doubleBufferId = AsyncSupported ? frameIndex : 0; - commandBuffer.BeginSample(Sampler); ref var mvs = ref prepareParams.motionVectors; @@ -95,9 +91,9 @@ namespace FidelityFX.FrameGen commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvInputMotionVectors, mvs.RenderTarget, mvs.MipLevel, mvs.SubElement); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvInputDepth, depth.RenderTarget, depth.MipLevel, depth.SubElement); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavReconstructedDepthPreviousFrame, Resources.ReconstructedDepth[doubleBufferId]); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavDilatedMotionVectors, Resources.DilatedMotionVectors[doubleBufferId]); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavDilatedDepth, Resources.DilatedDepth[doubleBufferId]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavReconstructedDepthPreviousFrame, Resources.ReconstructedDepth[frameIndex]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavDilatedMotionVectors, Resources.DilatedMotionVectors[frameIndex]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavDilatedDepth, Resources.DilatedDepth[frameIndex]); commandBuffer.SetComputeConstantBufferParam(ComputeShader, FrameInterpolationShaderIDs.CbFrameInterpolation, Constants, 0, Marshal.SizeOf()); @@ -143,11 +139,9 @@ namespace FidelityFX.FrameGen protected override void DoScheduleDispatch(CommandBuffer commandBuffer, FrameInterpolation.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ) { - int doubleBufferId = AsyncSupported ? frameIndex : 0; - // TODO: verify that we need the buffers from *this* frame (probably yes) - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedMotionVectors, Resources.DilatedMotionVectors[doubleBufferId]); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[doubleBufferId]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedMotionVectors, Resources.DilatedMotionVectors[frameIndex]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[frameIndex]); BindCurrentInterpolationSource(commandBuffer, dispatchParams); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.UavReconstructedDepthInterpolatedFrame, Resources.ReconstructedDepthInterpolatedFrame); @@ -168,10 +162,8 @@ namespace FidelityFX.FrameGen protected override void DoScheduleDispatch(CommandBuffer commandBuffer, FrameInterpolation.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ) { - int doubleBufferId = AsyncSupported ? frameIndex : 0; - - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedMotionVectors, Resources.DilatedMotionVectors[doubleBufferId]); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[doubleBufferId]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedMotionVectors, Resources.DilatedMotionVectors[frameIndex]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[frameIndex]); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvPreviousInterpolationSource, Resources.PreviousInterpolationSource); BindCurrentInterpolationSource(commandBuffer, dispatchParams); @@ -194,7 +186,6 @@ namespace FidelityFX.FrameGen protected override void DoScheduleDispatch(CommandBuffer commandBuffer, FrameInterpolation.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ) { - int doubleBufferId = AsyncSupported ? frameIndex : 0; ref var ofVector = ref dispatchParams.opticalFlowVector; if (dispatchParams.opticalFlowScale.x > 0f) @@ -204,7 +195,7 @@ namespace FidelityFX.FrameGen // TODO this might error... if so, bind an empty placeholder resource commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvOpticalFlowConfidence, BuiltinRenderTextureType.None); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[doubleBufferId]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[frameIndex]); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvPreviousInterpolationSource, Resources.PreviousInterpolationSource); BindCurrentInterpolationSource(commandBuffer, dispatchParams); @@ -227,12 +218,10 @@ namespace FidelityFX.FrameGen protected override void DoScheduleDispatch(CommandBuffer commandBuffer, FrameInterpolation.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ) { - int doubleBufferId = AsyncSupported ? frameIndex : 0; - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvGameMotionVectorFieldX, Resources.GameMotionVectorFieldX); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvGameMotionVectorFieldY, Resources.GameMotionVectorFieldY); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvReconstructedDepthPreviousFrame, Resources.ReconstructedDepth[doubleBufferId]); - commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[doubleBufferId]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvReconstructedDepthPreviousFrame, Resources.ReconstructedDepth[frameIndex]); + commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvDilatedDepth, Resources.DilatedDepth[frameIndex]); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvReconstructedDepthInterpolatedFrame, Resources.ReconstructedDepthInterpolatedFrame); commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, FrameInterpolationShaderIDs.SrvInpaintingPyramid, Resources.InpaintingPyramid);