using System; using System.Runtime.InteropServices; using UnityEngine; using UnityEngine.Assertions; using UnityEngine.Profiling; using UnityEngine.Rendering; namespace FidelityFX.FrameGen { public class FrameInterpolationContext: FfxContextBase { private FrameInterpolation.ContextDescription _contextDescription; private FrameInterpolationReconstructAndDilatePass _reconstructAndDilatePass; private FrameInterpolationSetupPass _setupPass; private FrameInterpolationReconstructPreviousDepthPass _reconstructPreviousDepthPass; private FrameInterpolationGameMotionVectorFieldPass _gameMotionVectorFieldPass; private FrameInterpolationOpticalFlowVectorFieldPass _opticalFlowVectorFieldPass; private FrameInterpolationDisocclusionMaskPass _disocclusionMaskPass; private FrameInterpolationInterpolationPass _interpolationPass; private FrameInterpolationInpaintingPyramidPass _inpaintingPyramidPass; private FrameInterpolationInpaintingPass _inpaintingPass; private FrameInterpolationGameVectorFieldInpaintingPyramidPass _gameVectorFieldInpaintingPyramidPass; private FrameInterpolationDebugViewPass _debugViewPass; private readonly FrameInterpolationResources _resources = new FrameInterpolationResources(); private readonly ConstantsBuffer _frameInterpolationConstants = new ConstantsBuffer(); private readonly ConstantsBuffer _spdConstants = new ConstantsBuffer(); private readonly CustomSampler _sampler = CustomSampler.Create("Frame Interpolation"); private readonly CustomSampler _prepareSampler = CustomSampler.Create("Frame Interpolation - Prepare"); private bool _firstExecution; private bool _asyncSupported; private ulong _previousFrameID; public void Create(in FrameInterpolation.ContextDescription contextDescription) { _contextDescription = contextDescription; _frameInterpolationConstants.Create(); _spdConstants.Create(); _firstExecution = true; _previousFrameID = 0; _asyncSupported = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableAsyncSupport) == FrameInterpolation.InitializationFlags.EnableAsyncSupport; ref var constants = ref _frameInterpolationConstants.Value; constants.maxRenderSize = _contextDescription.maxRenderSize; constants.displaySize = _contextDescription.displaySize; constants.displaySizeRcp.x = 1.0f / _contextDescription.displaySize.x; constants.displaySizeRcp.y = 1.0f / _contextDescription.displaySize.y; constants.interpolationRectBase = Vector2Int.zero; constants.interpolationRectSize = _contextDescription.displaySize; _resources.Create(_contextDescription); CreatePasses(); } private void CreatePasses() { _reconstructAndDilatePass = new FrameInterpolationReconstructAndDilatePass(_contextDescription, _resources, _frameInterpolationConstants); _setupPass = new FrameInterpolationSetupPass(_contextDescription, _resources, _frameInterpolationConstants); _reconstructPreviousDepthPass = new FrameInterpolationReconstructPreviousDepthPass(_contextDescription, _resources, _frameInterpolationConstants); _gameMotionVectorFieldPass = new FrameInterpolationGameMotionVectorFieldPass(_contextDescription, _resources, _frameInterpolationConstants); _opticalFlowVectorFieldPass = new FrameInterpolationOpticalFlowVectorFieldPass(_contextDescription, _resources, _frameInterpolationConstants); _disocclusionMaskPass = new FrameInterpolationDisocclusionMaskPass(_contextDescription, _resources, _frameInterpolationConstants); _interpolationPass = new FrameInterpolationInterpolationPass(_contextDescription, _resources, _frameInterpolationConstants); _inpaintingPyramidPass = new FrameInterpolationInpaintingPyramidPass(_contextDescription, _resources, _frameInterpolationConstants, _spdConstants); _inpaintingPass = new FrameInterpolationInpaintingPass(_contextDescription, _resources, _frameInterpolationConstants); _gameVectorFieldInpaintingPyramidPass = new FrameInterpolationGameVectorFieldInpaintingPyramidPass(_contextDescription, _resources, _frameInterpolationConstants, _spdConstants); _debugViewPass = new FrameInterpolationDebugViewPass(_contextDescription, _resources, _frameInterpolationConstants); } public void Destroy() { DestroyPass(ref _debugViewPass); DestroyPass(ref _gameVectorFieldInpaintingPyramidPass); DestroyPass(ref _inpaintingPass); DestroyPass(ref _inpaintingPyramidPass); DestroyPass(ref _interpolationPass); DestroyPass(ref _disocclusionMaskPass); DestroyPass(ref _opticalFlowVectorFieldPass); DestroyPass(ref _gameMotionVectorFieldPass); DestroyPass(ref _reconstructPreviousDepthPass); DestroyPass(ref _setupPass); DestroyPass(ref _reconstructAndDilatePass); _resources.Destroy(); _spdConstants.Destroy(); _frameInterpolationConstants.Destroy(); } public void Prepare(CommandBuffer commandBuffer, in FrameInterpolation.PrepareDescription prepareDescription) { commandBuffer.BeginSample(_prepareSampler); int doubleBufferId = _asyncSupported ? (int)(prepareDescription.frameID & 1) : 0; ref var constants = ref _frameInterpolationConstants.Value; constants.renderSize = prepareDescription.renderSize; constants.jitter = prepareDescription.jitterOffset; Vector2Int motionVectorsTargetSize = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDisplayResolutionMotionVectors) != 0 ? constants.displaySize : constants.renderSize; constants.motionVectorScale.x = prepareDescription.motionVectorScale.x / motionVectorsTargetSize.x; constants.motionVectorScale.y = prepareDescription.motionVectorScale.y / motionVectorsTargetSize.y; _frameInterpolationConstants.UpdateBufferData(commandBuffer); Assert.IsTrue(prepareDescription.depth.IsValid); Assert.IsTrue(prepareDescription.motionVectors.IsValid); // clear estimated depth resources { bool inverted = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDepthInverted) == FrameInterpolation.InitializationFlags.EnableDepthInverted; commandBuffer.SetRenderTarget(_resources.ReconstructedDepth[doubleBufferId]); commandBuffer.ClearRenderTarget(false, true, inverted ? Color.clear : Color.white); } int renderDispatchSizeX = (prepareDescription.renderSize.x + 7) / 8; int renderDispatchSizeY = (prepareDescription.renderSize.y + 7) / 8; _reconstructAndDilatePass.ScheduleDispatch(commandBuffer, prepareDescription, doubleBufferId, renderDispatchSizeX, renderDispatchSizeY); commandBuffer.EndSample(_prepareSampler); } public void Dispatch(CommandBuffer commandBuffer, in FrameInterpolation.DispatchDescription dispatchDescription) { commandBuffer.BeginSample(_sampler); bool reset = _firstExecution || dispatchDescription.reset; _firstExecution = false; Assert.IsTrue(!_asyncSupported || reset || dispatchDescription.frameID > _previousFrameID, "When async support is enabled, and the reset flag is not set, frame ID must increment in each dispatch."); bool frameIdDecreased = dispatchDescription.frameID < _previousFrameID; bool frameIdSkipped = (dispatchDescription.frameID - _previousFrameID) > 1; bool disjointFrameId = frameIdDecreased || frameIdSkipped; _previousFrameID = dispatchDescription.frameID; ref var constants = ref _frameInterpolationConstants.Value; constants.renderSize = dispatchDescription.renderSize; constants.displaySize = dispatchDescription.displaySize; constants.displaySizeRcp.x = 1.0f / dispatchDescription.displaySize.x; constants.displaySizeRcp.y = 1.0f / dispatchDescription.displaySize.y; constants.upscalerTargetSize = dispatchDescription.interpolationRect.size; constants.mode = 0; constants.reset = (reset || disjointFrameId) ? 1 : 0; constants.deltaTime = dispatchDescription.frameTimeDelta; constants.HUDLessAttachedFactor = dispatchDescription.currentBackBuffer_HUDLess.IsValid ? 1 : 0; constants.opticalFlowScale = dispatchDescription.opticalFlowScale; constants.opticalFlowBlockSize = dispatchDescription.opticalFlowBlockSize; constants.dispatchFlags = (uint)dispatchDescription.flags; constants.cameraNear = dispatchDescription.cameraNear; constants.cameraFar = dispatchDescription.cameraFar; constants.interpolationRectBase = dispatchDescription.interpolationRect.position; constants.interpolationRectSize = dispatchDescription.interpolationRect.size; // Debug bar constants.debugBarColor.x = DebugBarColorSequence[_debugIndex * 3 + 0]; constants.debugBarColor.y = DebugBarColorSequence[_debugIndex * 3 + 1]; constants.debugBarColor.z = DebugBarColorSequence[_debugIndex * 3 + 2]; _debugIndex = (_debugIndex + 1) % (DebugBarColorSequence.Length / 3); constants.backBufferTransferFunction = (uint)dispatchDescription.backbufferTransferFunction; constants.minMaxLuminance = dispatchDescription.minMaxLuminance; float aspectRatio = dispatchDescription.renderSize.x / (float)dispatchDescription.renderSize.y; float cameraAngleHorizontal = Mathf.Atan(Mathf.Tan(dispatchDescription.cameraFovAngleVertical / 2) * aspectRatio) * 2; constants.tanHalfFOV = Mathf.Tan(cameraAngleHorizontal * 0.5f); bool inverted = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDepthInverted) != 0; bool infinite = (_contextDescription.flags & FrameInterpolation.InitializationFlags.EnableDepthInfinite) != 0; constants.deviceToViewDepth = FfxUtils.SetupDeviceDepthToViewSpaceDepthParams( dispatchDescription.renderSize, dispatchDescription.cameraNear, dispatchDescription.cameraFar, dispatchDescription.cameraFovAngleVertical, inverted, infinite); _frameInterpolationConstants.UpdateBufferData(commandBuffer); int doubleBufferId = _asyncSupported ? (int)(dispatchDescription.frameID & 1) : 0; int displayDispatchSizeX = (dispatchDescription.displaySize.x + 7) / 8; int displayDispatchSizeY = (dispatchDescription.displaySize.y + 7) / 8; int renderDispatchSizeX = (dispatchDescription.renderSize.x + 7) / 8; int renderDispatchSizeY = (dispatchDescription.renderSize.y + 7) / 8; int opticalFlowDispatchSizeX = (int)(dispatchDescription.displaySize.x / (float)dispatchDescription.opticalFlowBlockSize + 7) / 8; int opticalFlowDispatchSizeY = (int)(dispatchDescription.displaySize.y / (float)dispatchDescription.opticalFlowBlockSize + 7) / 8; bool executePreparationPasses = (constants.reset == 0); // Schedule work for the interpolation command list _setupPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, renderDispatchSizeX, renderDispatchSizeY); // Only execute FG data preparation passes when reset wasn't triggered if (executePreparationPasses) { // Clear estimated depth resources commandBuffer.SetRenderTarget(_resources.ReconstructedDepthInterpolatedFrame); commandBuffer.ClearRenderTarget(false, true, inverted ? Color.clear : Color.white); _reconstructPreviousDepthPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, renderDispatchSizeX, renderDispatchSizeY); _gameMotionVectorFieldPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, renderDispatchSizeX, renderDispatchSizeY); DispatchGameVectorFieldInpaintingPyramid(commandBuffer, dispatchDescription, doubleBufferId); _opticalFlowVectorFieldPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, opticalFlowDispatchSizeX, opticalFlowDispatchSizeY); _disocclusionMaskPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, renderDispatchSizeX, renderDispatchSizeY); } _interpolationPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, displayDispatchSizeX, displayDispatchSizeY); // Inpainting pyramid SetupSpdConstants(dispatchDescription.displaySize, out var dispatchThreadGroupCount); _spdConstants.UpdateBufferData(commandBuffer); _inpaintingPyramidPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, dispatchThreadGroupCount.x, dispatchThreadGroupCount.y); _inpaintingPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, displayDispatchSizeX, displayDispatchSizeY); if ((dispatchDescription.flags & FrameInterpolation.DispatchFlags.DrawDebugView) != 0) { DispatchGameVectorFieldInpaintingPyramid(commandBuffer, dispatchDescription, doubleBufferId); _debugViewPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, displayDispatchSizeX, displayDispatchSizeY); } // Store current buffer commandBuffer.CopyTexture(dispatchDescription.InterpolationSource.RenderTarget, _resources.PreviousInterpolationSource); commandBuffer.EndSample(_sampler); } private void DispatchGameVectorFieldInpaintingPyramid(CommandBuffer commandBuffer, in FrameInterpolation.DispatchDescription dispatchDescription, int doubleBufferId) { SetupSpdConstants(dispatchDescription.renderSize, out var dispatchThreadGroupCount); _spdConstants.UpdateBufferData(commandBuffer); _gameVectorFieldInpaintingPyramidPass.ScheduleDispatch(commandBuffer, dispatchDescription, doubleBufferId, dispatchThreadGroupCount.x, dispatchThreadGroupCount.y); } private void SetupSpdConstants(Vector2Int resolution, out Vector2Int dispatchThreadGroupCount) { ref var spdConstants = ref _spdConstants.Value; FfxSpd.SetupSpdConstants(resolution, ref spdConstants, out dispatchThreadGroupCount); spdConstants.mips = Math.Min(spdConstants.mips, 7); } private int _debugIndex = 0; private static readonly float[] DebugBarColorSequence = { 0.0f, 1.0f, 1.0f, // teal 1.0f, 0.42f, 0.0f, // orange 0.0f, 0.16f, 1.0f, // blue 0.74f, 1.0f, 0.0f, // lime 0.68f, 0.0f, 1.0f, // purple 0.0f, 1.0f, 0.1f, // green 1.0f, 1.0f, 0.48f // bright yellow }; } }