ndepoel
/
FSRPackage
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Implemented the remaining optical flow passes and fixed a few small issues. 

Seems to work but it needs more extensive testing.
fsr3framegen
Nico de Poel 2 years ago
parent
c74320613b
commit
9a72a8f180
5 changed files with 144 additions and 41 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 34
      Runtime/OpticalFlow/OpticalFlow.cs
  2. 70
      Runtime/OpticalFlow/OpticalFlowContext.cs
  3. 75
      Runtime/OpticalFlow/OpticalFlowPass.cs
  4. 4
      Runtime/OpticalFlow/OpticalFlowResources.cs
  5. 2
      Runtime/OpticalFlow/OpticalFlowShaderIDs.cs

34
Runtime/OpticalFlow/OpticalFlow.cs
View File

@ -6,14 +6,25 @@ namespace FidelityFX.OpticalFlow
{
public static class OpticalFlow
{
internal const int MinBlockSize = 8;
internal const int OpticalFlowMaxPyramidLevels = 7;
internal const int HistogramBins = 256;
internal const int HistogramsPerDim = 3;
internal const int HistogramShifts = 3;
public static OpticalFlowContext CreateContext(in ContextDescription contextDescription)
public static OpticalFlowContext CreateContext(Vector2Int resolution, OpticalFlowShaders shaders)
{
throw new NotImplementedException();
Debug.Log($"Setting up Optical Flow with resolution: {resolution.x}x{resolution.y}");
var contextDescription = new ContextDescription
{
resolution = resolution,
shaders = shaders,
};
var context = new OpticalFlowContext();
context.Create(contextDescription);
return context;
}
public struct ContextDescription
@ -22,14 +33,19 @@ namespace FidelityFX.OpticalFlow
public OpticalFlowShaders shaders;
}
public struct DispatchDescription
public class DispatchDescription
{
public ResourceView color;
public ResourceView opticalFlowVector;
public ResourceView opticalFlowSCD;
public bool reset;
public int backbufferTransferFunction;
public Vector2 minMaxLuminance;
public ResourceView Color;
public ResourceView OpticalFlowVector;
public ResourceView OpticalFlowSCD;
public bool Reset;
public int BackbufferTransferFunction;
public Vector2 MinMaxLuminance;
}
public static Vector2Int GetOpticalFlowTextureSize(Vector2Int displaySize)
{
return GetOpticalFlowTextureSize(displaySize, MinBlockSize);
}
internal static Vector2Int GetOpticalFlowTextureSize(Vector2Int displaySize, int opticalFlowBlockSize)

70
Runtime/OpticalFlow/OpticalFlowContext.cs
View File

@ -2,6 +2,7 @@ using System;
using System.Runtime.InteropServices;
using UnityEngine;
using UnityEngine.Assertions;
using UnityEngine.Profiling;
using UnityEngine.Rendering;
namespace FidelityFX.OpticalFlow
@ -33,6 +34,7 @@ namespace FidelityFX.OpticalFlow
private bool _firstExecution;
private int _resourceFrameIndex;
private readonly Vector2Int[] _opticalFlowTextureSizes = new Vector2Int[OpticalFlow.OpticalFlowMaxPyramidLevels];
private readonly CustomSampler _sampler = CustomSampler.Create("Optical Flow");
public void Create(OpticalFlow.ContextDescription contextDescription)
{
@ -77,28 +79,30 @@ namespace FidelityFX.OpticalFlow
DestroyConstantBuffer(ref _opticalFlowConstantsBuffer);
}
public void Dispatch(CommandBuffer commandBuffer, in OpticalFlow.DispatchDescription dispatchDescription)
public void Dispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchDescription)
{
const int advancedAlgorithmIterations = 7;
const int opticalFlowBlockSize = 8;
Constants.backbufferTransferFunction = (uint)dispatchDescription.backbufferTransferFunction;
Constants.minMaxLuminance = dispatchDescription.minMaxLuminance;
Constants.backbufferTransferFunction = (uint)dispatchDescription.BackbufferTransferFunction;
Constants.minMaxLuminance = dispatchDescription.MinMaxLuminance;
int frameIndex = _resourceFrameIndex % 2;
bool resetAccumulation = dispatchDescription.reset || _firstExecution;
bool resetAccumulation = dispatchDescription.Reset || _firstExecution;
_firstExecution = false;
if (resetAccumulation)
Constants.frameIndex = 0;
else
Constants.frameIndex++;
commandBuffer.BeginSample(_sampler);
if (resetAccumulation)
{
commandBuffer.SetRenderTarget(_resources.OpticalFlowSCDTemp);
commandBuffer.ClearRenderTarget(false, true, Color.clear);
commandBuffer.SetRenderTarget(dispatchDescription.opticalFlowSCD.RenderTarget);
commandBuffer.SetRenderTarget(dispatchDescription.OpticalFlowSCD.RenderTarget);
commandBuffer.ClearRenderTarget(false, true, Color.clear);
commandBuffer.SetRenderTarget(_resources.OpticalFlowSCDHistogram);
commandBuffer.ClearRenderTarget(false, true, Color.clear);
@ -126,10 +130,10 @@ namespace FidelityFX.OpticalFlow
const int threadPixelsY = 2;
int dispatchX = ((_contextDescription.resolution.x + (threadPixelsX - 1)) / threadPixelsX + (threadGroupSizeX - 1)) / threadGroupSizeX;
int dispatchY = ((_contextDescription.resolution.y + (threadPixelsY - 1)) / threadPixelsY + (threadGroupSizeY - 1)) / threadGroupSizeY;
_prepareLumaPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, dispatchX, dispatchY);
_prepareLumaPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, 0, dispatchX, dispatchY);
}
_generateInputPyramidPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, threadGroupSizeOpticalFlowInputPyramid.x, threadGroupSizeOpticalFlowInputPyramid.y);
_generateInputPyramidPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, 0, threadGroupSizeOpticalFlowInputPyramid.x, threadGroupSizeOpticalFlowInputPyramid.y);
{
const int threadGroupSizeX = 32;
@ -139,12 +143,12 @@ namespace FidelityFX.OpticalFlow
int dispatchX = (strataWidth + threadGroupSizeX - 1) / threadGroupSizeX;
const int dispatchY = 16;
const int dispatchZ = OpticalFlow.HistogramsPerDim * OpticalFlow.HistogramsPerDim;
_generateScdHistogramPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, dispatchX, dispatchY, dispatchZ);
_generateScdHistogramPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, 0, dispatchX, dispatchY, dispatchZ);
}
{
const int dispatchX = OpticalFlow.HistogramsPerDim * OpticalFlow.HistogramsPerDim;
const int dispatchY = OpticalFlow.HistogramShifts;
_computeScdDivergencePass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, dispatchX, dispatchY);
_computeScdDivergencePass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, 0, dispatchX, dispatchY);
}
const int pyramidMaxIterations = advancedAlgorithmIterations;
@ -161,14 +165,54 @@ namespace FidelityFX.OpticalFlow
for (int level = pyramidMaxIterations - 1; level >= 0; --level)
{
// TODO: need to flip-flop Optical Flow output resources between levels as well (isOddFrame != isOddLevel)
Constants.opticalFlowPyramidLevel = (uint)level;
Constants.opticalFlowPyramidLevelCount = pyramidMaxIterations;
commandBuffer.SetBufferData(_opticalFlowConstantsBuffer, _opticalFlowConstantsArray);
// TODO: need to somehow address resources by level, not great when you've got a bazillion loose RT arrays... *sigh*
{
int inputLumaWidth = Math.Max(_contextDescription.resolution.x >> level, 1);
int inputLumaHeight = Math.Max(_contextDescription.resolution.y >> level, 1);
const int threadPixels = 4;
Assert.IsTrue(opticalFlowBlockSize >= threadPixels);
const int threadGroupSizeX = 4;
const int threadGroupSizeY = 16;
const int threadGroupSize = 64;
int dispatchX = ((inputLumaWidth + threadPixels - 1) / threadPixels * threadGroupSizeY + (threadGroupSize - 1)) / threadGroupSize;
int dispatchY = (inputLumaHeight + (threadGroupSizeY - 1)) / threadGroupSizeY;
_computeOpticalFlowPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, level, dispatchX, dispatchY);
}
{
int levelWidth = _opticalFlowTextureSizes[level].x;
int levelHeight = _opticalFlowTextureSizes[level].y;
const int threadGroupSizeX = 16;
const int threadGroupSizeY = 4;
int dispatchX = (levelWidth + threadGroupSizeX - 1) / threadGroupSizeX;
int dispatchY = (levelHeight + threadGroupSizeY - 1) / threadGroupSizeY;
_filterOpticalFlowPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, level, dispatchX, dispatchY);
}
if (level > 0)
{
int nextLevelWidth = _opticalFlowTextureSizes[level - 1].x;
int nextLevelHeight = _opticalFlowTextureSizes[level - 1].y;
const int threadGroupSizeX = opticalFlowBlockSize / 2;
const int threadGroupSizeY = opticalFlowBlockSize / 2;
const int threadGroupSizeZ = 4;
int dispatchX = (nextLevelWidth + threadGroupSizeX - 1) / threadGroupSizeX;
int dispatchY = (nextLevelHeight + threadGroupSizeY - 1) / threadGroupSizeY;
const int dispatchZ = 1;
_scaleOpticalFlowPass.ScheduleDispatch(commandBuffer, dispatchDescription, frameIndex, level, dispatchX, dispatchY, dispatchZ);
}
}
commandBuffer.EndSample(_sampler);
_resourceFrameIndex = (_resourceFrameIndex + 1) % MaxQueuedFrames;
}

75
Runtime/OpticalFlow/OpticalFlowPass.cs
View File

@ -28,14 +28,14 @@ namespace FidelityFX.OpticalFlow
{
}
public void ScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ = 1)
public void ScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ = 1)
{
commandBuffer.BeginSample(_sampler);
DoScheduleDispatch(commandBuffer, dispatchParams, frameIndex, dispatchX, dispatchY, dispatchZ);
DoScheduleDispatch(commandBuffer, dispatchParams, frameIndex, level, dispatchX, dispatchY, dispatchZ);
commandBuffer.EndSample(_sampler);
}
protected abstract void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ);
protected abstract void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ);
protected void InitComputeShader(string passName, ComputeShader shader)
{
@ -58,9 +58,9 @@ namespace FidelityFX.OpticalFlow
InitComputeShader("Prepare Luma", contextDescription.shaders.prepareLuma);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
ref var color = ref dispatchParams.color;
ref var color = ref dispatchParams.Color;
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvInputColor, color.RenderTarget, color.MipLevel, color.SubElement);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowInput, Resources.OpticalFlowInputLevels[0][frameIndex]);
@ -82,7 +82,7 @@ namespace FidelityFX.OpticalFlow
InitComputeShader("Generate Optical Flow Input Pyramid", contextDescription.shaders.generateOpticalFlowInputPyramid);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowInput, Resources.OpticalFlowInputLevels[0][frameIndex]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowInputLevel1, Resources.OpticalFlowInputLevels[1][frameIndex]);
@ -107,7 +107,7 @@ namespace FidelityFX.OpticalFlow
InitComputeShader("Generate SCD Histogram", contextDescription.shaders.generateScdHistogram);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
// TODO: probably needs to be input from this frame (result from previous passes), but double check to be sure
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlowInput, Resources.OpticalFlowInputLevels[0][frameIndex]);
@ -127,9 +127,9 @@ namespace FidelityFX.OpticalFlow
InitComputeShader("Compute SCD Divergence", contextDescription.shaders.computeScdDivergence);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
ref var scdOutput = ref dispatchParams.opticalFlowSCD;
ref var scdOutput = ref dispatchParams.OpticalFlowSCD;
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowScdHistogram, Resources.OpticalFlowSCDHistogram);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowScdPreviousHistogram, Resources.OpticalFlowSCDPreviousHistogram);
@ -147,11 +147,22 @@ namespace FidelityFX.OpticalFlow
public OpticalFlowComputePass(OpticalFlow.ContextDescription contextDescription, OpticalFlowResources resources, ComputeBuffer constants)
: base(contextDescription, resources, constants)
{
InitComputeShader("Optical Flow Search", contextDescription.shaders.computeOpticalFlow);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
throw new NotImplementedException();
int levelIndex = frameIndex ^ (level & 1);
ref var scdOutput = ref dispatchParams.OpticalFlowSCD;
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlowInput, Resources.OpticalFlowInputLevels[level][frameIndex]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlowPreviousInput, Resources.OpticalFlowInputLevels[level][frameIndex ^ 1]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlow, Resources.OpticalFlowLevels[level][levelIndex]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowScdOutput, scdOutput.RenderTarget, scdOutput.MipLevel, scdOutput.SubElement);
commandBuffer.SetComputeConstantBufferParam(ComputeShader, OpticalFlowShaderIDs.CbOpticalFlow, Constants, 0, Marshal.SizeOf<OpticalFlow.OpticalFlowConstants>());
commandBuffer.DispatchCompute(ComputeShader, KernelIndex, dispatchX, dispatchY, dispatchZ);
}
}
@ -160,11 +171,29 @@ namespace FidelityFX.OpticalFlow
public OpticalFlowFilterPass(OpticalFlow.ContextDescription contextDescription, OpticalFlowResources resources, ComputeBuffer constants)
: base(contextDescription, resources, constants)
{
InitComputeShader("Optical Flow Filter", contextDescription.shaders.filterOpticalFlow);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
throw new NotImplementedException();
int levelIndex = frameIndex ^ (level & 1);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlowPrevious, Resources.OpticalFlowLevels[level][levelIndex]);
if (level == 0)
{
// Final output (levels are counted in reverse)
ref var ofVector = ref dispatchParams.OpticalFlowVector;
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlow, ofVector.RenderTarget, ofVector.MipLevel, ofVector.SubElement);
}
else
{
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlow, Resources.OpticalFlowLevels[level][levelIndex ^ 1]);
}
commandBuffer.SetComputeConstantBufferParam(ComputeShader, OpticalFlowShaderIDs.CbOpticalFlow, Constants, 0, Marshal.SizeOf<OpticalFlow.OpticalFlowConstants>());
commandBuffer.DispatchCompute(ComputeShader, KernelIndex, dispatchX, dispatchY, dispatchZ);
}
}
@ -173,11 +202,27 @@ namespace FidelityFX.OpticalFlow
public OpticalFlowScalePass(OpticalFlow.ContextDescription contextDescription, OpticalFlowResources resources, ComputeBuffer constants)
: base(contextDescription, resources, constants)
{
InitComputeShader("Optical Flow Scale", contextDescription.shaders.scaleOpticalFlow);
}
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int dispatchX, int dispatchY, int dispatchZ)
protected override void DoScheduleDispatch(CommandBuffer commandBuffer, OpticalFlow.DispatchDescription dispatchParams, int frameIndex, int level, int dispatchX, int dispatchY, int dispatchZ)
{
throw new NotImplementedException();
if (level <= 0)
return;
int levelIndex = frameIndex ^ (level & 1);
ref var scdOutput = ref dispatchParams.OpticalFlowSCD;
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlowInput, Resources.OpticalFlowInputLevels[level][frameIndex]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlowPreviousInput, Resources.OpticalFlowInputLevels[level][frameIndex ^ 1]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.SrvOpticalFlow, Resources.OpticalFlowLevels[level][levelIndex ^ 1]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowNextLevel, Resources.OpticalFlowLevels[level - 1][levelIndex ^ 1]);
commandBuffer.SetComputeTextureParam(ComputeShader, KernelIndex, OpticalFlowShaderIDs.UavOpticalFlowScdOutput, scdOutput.RenderTarget, scdOutput.MipLevel, scdOutput.SubElement);
commandBuffer.SetComputeConstantBufferParam(ComputeShader, OpticalFlowShaderIDs.CbOpticalFlow, Constants, 0, Marshal.SizeOf<OpticalFlow.OpticalFlowConstants>());
commandBuffer.DispatchCompute(ComputeShader, KernelIndex, dispatchX, dispatchY, dispatchZ);
}
}
}

4
Runtime/OpticalFlow/OpticalFlowResources.cs
View File

@ -34,9 +34,7 @@ namespace FidelityFX.OpticalFlow
public void Create(OpticalFlow.ContextDescription contextDescription)
{
Vector2Int opticalFlowInputTextureSize = contextDescription.resolution;
const int minBlockSize = 8;
Vector2Int opticalFlowTextureSize = OpticalFlow.GetOpticalFlowTextureSize(contextDescription.resolution, minBlockSize);
Vector2Int opticalFlowTextureSize = OpticalFlow.GetOpticalFlowTextureSize(contextDescription.resolution, OpticalFlow.MinBlockSize);
// TODO: this seems useless, delete unless something comes up
Vector2Int opticalFlowHistogramTextureSize = OpticalFlow.GetOpticalFlowHistogramSize(0);

2
Runtime/OpticalFlow/OpticalFlowShaderIDs.cs
View File

@ -22,7 +22,7 @@ namespace FidelityFX.OpticalFlow
public static readonly int UavOpticalFlow = Shader.PropertyToID("rw_optical_flow");
public static readonly int UavOpticalFlowNextLevel = Shader.PropertyToID("rw_optical_flow_next_level");
public static readonly int UavOpticalFlowScdHistogram = Shader.PropertyToID("rw_optical_flow_scd_histogram"); // scene change detection histogram
public static readonly int UavOpticalFlowScdPreviousHistogram = Shader.PropertyToID("rw_optical_flow_previous_histogram");
public static readonly int UavOpticalFlowScdPreviousHistogram = Shader.PropertyToID("rw_optical_flow_scd_previous_histogram");
public static readonly int UavOpticalFlowScdTemp = Shader.PropertyToID("rw_optical_flow_scd_temp");
public static readonly int UavOpticalFlowScdOutput = Shader.PropertyToID("rw_optical_flow_scd_output");

Write Preview
Loading…
Cancel
Save