TAA improvements (#7441)

* TAA improvements - fix variance + AABB history clipping (we were computing the union of both AABB instead of the intersection). - added a setting for the variance parameter - added a setting for the jitter pattern - improved some default values - fixed a few comments - smaller code tweaks to facilitate future improvements
google · Dec 22, 2023 · 9d817a0 · 9d817a0
1 parent 95ecc3d
commit 9d817a0
Show file tree

Hide file tree

Showing 15 changed files with 255 additions and 120 deletions.
diff --git a/android/filament-android/src/main/java/com/google/android/filament/View.java b/android/filament-android/src/main/java/com/google/android/filament/View.java
@@ -1852,7 +1852,7 @@ public static class AmbientOcclusionOptions {
     }
 
     /**
-     * Options for Temporal Multi-Sample Anti-aliasing (MSAA)
+     * Options for Multi-Sample Anti-aliasing (MSAA)
      * @see setMultiSampleAntiAliasingOptions()
      */
     public static class MultiSampleAntiAliasingOptions {
@@ -1878,7 +1878,11 @@ public static class MultiSampleAntiAliasingOptions {
      * Options for Temporal Anti-aliasing (TAA)
      * Most TAA parameters are extremely costly to change, as they will trigger the TAA post-process
      * shaders to be recompiled. These options should be changed or set during initialization.
-     * `filterWidth` and `feedback`, however, could be changed an any time.
+     * `filterWidth`, `feedback` and `jitterPattern`, however, can be changed at any time.
+     *
+     * `feedback` of 0.1 effectively accumulates a maximum of 19 samples in steady state.
+     * see "A Survey of Temporal Antialiasing Techniques" by Lei Yang and all for more information.
+     *
      * @see setTemporalAntiAliasingOptions()
      */
     public static class TemporalAntiAliasingOptions {
@@ -1888,7 +1892,7 @@ public enum BoxType {
              */
             AABB,
             /**
-             * use the variance of the neighborhood
+             * use the variance of the neighborhood (not recommended)
              */
             VARIANCE,
             /**
@@ -1912,14 +1916,21 @@ public enum BoxClipping {
             NONE,
         }
 
+        public enum JitterPattern {
+            RGSS_X4,
+            UNIFORM_HELIX_X4,
+            HALTON_23_X8,
+            HALTON_23_X16,
+        }
+
         /**
-         * reconstruction filter width typically between 0.2 (sharper, aliased) and 1 (smoother)
+         * reconstruction filter width typically between 0.2 (sharper, aliased) and 1.5 (smoother)
          */
         public float filterWidth = 1.0f;
         /**
          * history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA).
          */
-        public float feedback = 0.04f;
+        public float feedback = 0.12f;
         /**
          * enables or disables temporal anti-aliasing
          */
@@ -1940,12 +1951,15 @@ public enum BoxClipping {
          * type of color gamut box
          */
         @NonNull
-        public TemporalAntiAliasingOptions.BoxType boxType = TemporalAntiAliasingOptions.BoxType.VARIANCE;
+        public TemporalAntiAliasingOptions.BoxType boxType = TemporalAntiAliasingOptions.BoxType.AABB;
         /**
          * clipping algorithm
          */
         @NonNull
         public TemporalAntiAliasingOptions.BoxClipping boxClipping = TemporalAntiAliasingOptions.BoxClipping.ACCURATE;
+        @NonNull
+        public TemporalAntiAliasingOptions.JitterPattern jitterPattern = TemporalAntiAliasingOptions.JitterPattern.HALTON_23_X16;
+        public float varianceGamma = 1.0f;
         /**
          * adjust the feedback dynamically to reduce flickering
          */

diff --git a/filament/include/filament/Options.h b/filament/include/filament/Options.h
@@ -401,7 +401,7 @@ struct AmbientOcclusionOptions {
 };
 
 /**
- * Options for Temporal Multi-Sample Anti-aliasing (MSAA)
+ * Options for Multi-Sample Anti-aliasing (MSAA)
  * @see setMultiSampleAntiAliasingOptions()
  */
 struct MultiSampleAntiAliasingOptions {
@@ -426,17 +426,21 @@ struct MultiSampleAntiAliasingOptions {
  * Options for Temporal Anti-aliasing (TAA)
  * Most TAA parameters are extremely costly to change, as they will trigger the TAA post-process
  * shaders to be recompiled. These options should be changed or set during initialization.
- * `filterWidth` and `feedback`, however, could be changed an any time.
+ * `filterWidth`, `feedback` and `jitterPattern`, however, could be changed an any time.
+ *
+ * `feedback` of 0.1 effectively accumulates a maximum of 19 samples in steady state.
+ * see "A Survey of Temporal Antialiasing Techniques" by Lei Yang and all for more information.
+ *
  * @see setTemporalAntiAliasingOptions()
  */
 struct TemporalAntiAliasingOptions {
-    float filterWidth = 1.0f;   //!< reconstruction filter width typically between 0.2 (sharper, aliased) and 1 (smoother)
-    float feedback = 0.04f;     //!< history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA).
+    float filterWidth = 1.0f;   //!< reconstruction filter width typically between 0.2 (sharper, aliased) and 1.5 (smoother)
+    float feedback = 0.12f;     //!< history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA).
     bool enabled = false;       //!< enables or disables temporal anti-aliasing
 
     enum class BoxType : uint8_t {
         AABB,           //!< use an AABB neighborhood
-        VARIANCE,       //!< use the variance of the neighborhood
+        VARIANCE,       //!< use the variance of the neighborhood (not recommended)
         AABB_VARIANCE   //!< use both AABB and variance
     };
 
@@ -446,11 +450,20 @@ struct TemporalAntiAliasingOptions {
         NONE            //!< no rejections (use for debugging)
     };
 
+    enum class JitterPattern : uint8_t {
+        RGSS_X4,             //!  4-samples, rotated grid sampling
+        UNIFORM_HELIX_X4,    //!  4-samples, uniform grid in helix sequence
+        HALTON_23_X8,        //!  8-samples of halton 2,3
+        HALTON_23_X16        //! 16-samples of halton 2,3
+    };
+
     bool filterHistory = true;      //!< whether to filter the history buffer
     bool filterInput = true;        //!< whether to apply the reconstruction filter to the input
     bool useYCoCg = false;          //!< whether to use the YcoCg color-space for history rejection
-    BoxType boxType = BoxType::VARIANCE;                //!< type of color gamut box
-    BoxClipping boxClipping = BoxClipping::ACCURATE;    //!< clipping algorithm
+    BoxType boxType = BoxType::AABB;            //!< type of color gamut box
+    BoxClipping boxClipping = BoxClipping::ACCURATE;     //!< clipping algorithm
+    JitterPattern jitterPattern = JitterPattern::HALTON_23_X16; //! Jitter Pattern
+    float varianceGamma = 1.0f; //! High values increases ghosting artefact, lower values increases jittering, range [0.75, 1.25]
 
     bool preventFlickering = false;     //!< adjust the feedback dynamically to reduce flickering
     bool historyReprojection = true;    //!< whether to apply history reprojection (debug option)

diff --git a/filament/src/PostProcessManager.cpp b/filament/src/PostProcessManager.cpp
@@ -169,7 +169,21 @@ FMaterialInstance* PostProcessManager::PostProcessMaterial::getMaterialInstance(
 
 // ------------------------------------------------------------------------------------------------
 
-const math::float2 PostProcessManager::sHaltonSamples[16] = {
+const PostProcessManager::JitterSequence<4> PostProcessManager::sRGSS4 = {{
+        { 0.625f, 0.125f },
+        { 0.125f, 0.375f },
+        { 0.875f, 0.625f },
+        { 0.375f, 0.875f }
+}};
+
+const PostProcessManager::JitterSequence<4> PostProcessManager::sUniformHelix4 = {{
+        { 0.25f, 0.25f },
+        { 0.75f, 0.75f },
+        { 0.25f, 0.75f },
+        { 0.75f, 0.25f },
+}};
+
+const PostProcessManager::JitterSequence<16> PostProcessManager::sHaltonSamples = {{
         { filament::halton( 0, 2), filament::halton( 0, 3) },
         { filament::halton( 1, 2), filament::halton( 1, 3) },
         { filament::halton( 2, 2), filament::halton( 2, 3) },
@@ -186,7 +200,7 @@ const math::float2 PostProcessManager::sHaltonSamples[16] = {
         { filament::halton(13, 2), filament::halton(13, 3) },
         { filament::halton(14, 2), filament::halton(14, 3) },
         { filament::halton(15, 2), filament::halton(15, 3) }
-};
+}};
 
 PostProcessManager::PostProcessManager(FEngine& engine) noexcept
         : mEngine(engine),
@@ -2500,7 +2514,9 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::fxaa(FrameGraph& fg,
     return ppFXAA->output;
 }
 
-void PostProcessManager::prepareTaa(FrameGraph& fg, filament::Viewport const& svp,
+void PostProcessManager::prepareTaa(FrameGraph& fg,
+        filament::Viewport const& svp,
+        TemporalAntiAliasingOptions const& taaOptions,
         FrameHistory& frameHistory,
         FrameHistoryEntry::TemporalAA FrameHistoryEntry::*pTaa,
         CameraInfo* inoutCameraInfo,
@@ -2512,9 +2528,33 @@ void PostProcessManager::prepareTaa(FrameGraph& fg, filament::Viewport const& sv
     current.projection = inoutCameraInfo->projection * inoutCameraInfo->getUserViewMatrix();
     current.frameId = previous.frameId + 1;
 
+    auto jitterPosition = [pattern = taaOptions.jitterPattern](size_t frameIndex){
+        using JitterPattern = TemporalAntiAliasingOptions::JitterPattern;
+        switch (pattern) {
+            case JitterPattern::RGSS_X4:
+                return sRGSS4(frameIndex);
+            case JitterPattern::UNIFORM_HELIX_X4:
+                return sUniformHelix4(frameIndex);
+            case JitterPattern::HALTON_23_X8:
+                return sHaltonSamples(frameIndex % 8);
+            case JitterPattern::HALTON_23_X16:
+                return sHaltonSamples(frameIndex);
+        }
+    };
+
     // sample position within a pixel [-0.5, 0.5]
-    float2 const jitter = halton(previous.frameId) - 0.5f;
-    current.jitter = jitter;
+    // for metal/vulkan we need to reverse the y-offset
+    current.jitter = jitterPosition(previous.frameId);
+    float2 jitter = current.jitter;
+    switch (mEngine.getBackend()) {
+        case Backend::VULKAN:
+        case Backend::METAL:
+            jitter.y = -jitter.y;
+            UTILS_FALLTHROUGH;
+        case Backend::OPENGL:
+        default:
+            break;
+    }
 
     float2 const jitterInClipSpace = jitter * (2.0f / float2{ svp.width, svp.height });
 
@@ -2554,6 +2594,7 @@ void PostProcessManager::configureTemporalAntiAliasingMaterial(
     setConstantParameter(ma, "preventFlickering", taaOptions.preventFlickering);
     setConstantParameter(ma, "boxType", (int32_t)taaOptions.boxType);
     setConstantParameter(ma, "boxClipping", (int32_t)taaOptions.boxClipping);
+    setConstantParameter(ma, "varianceGamma", taaOptions.varianceGamma);
     if (dirty) {
         ma->invalidate();
         // TODO: call Material::compile(), we can't si that now because it works only
@@ -2628,18 +2669,17 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
                 };
 
                 float sum = 0.0;
-                float weights[9];
+                float4 weights[9];
 
                 // this doesn't get vectorized (probably because of exp()), so don't bother
                 // unrolling it.
                 #pragma nounroll
                 for (size_t i = 0; i < 9; i++) {
-                    float2 d = sampleOffsets[i] - current.jitter;
-                    d *= 1.0f / taaOptions.filterWidth;
-                    // this is a gaussian fit of a 3.3 Blackman Harris window
+                    float2 const d = (sampleOffsets[i] - current.jitter) / taaOptions.filterWidth;
+                    // This is a gaussian fit of a 3.3-wide Blackman-Harris window
                     // see: "High Quality Temporal Supersampling" by Brian Karis
-                    weights[i] = std::exp2(-3.3f * (d.x * d.x + d.y * d.y));
-                    sum += weights[i];
+                    weights[i][0] = std::exp(-2.29f * (d.x * d.x + d.y * d.y));
+                    sum += weights[i][0];
                 }
                 for (auto& w : weights) {
                     w /= sum;
@@ -2660,6 +2700,7 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
                         .filterMin = SamplerMinFilter::LINEAR
                 });
                 mi->setParameter("filterWeights",  weights, 9);
+                mi->setParameter("jitter",  current.jitter);
                 mi->setParameter("reprojection",
                         mat4f{ historyProjection * inverse(current.projection) } *
                         normalizedToClip);

diff --git a/filament/src/PostProcessManager.h b/filament/src/PostProcessManager.h
@@ -211,7 +211,9 @@ class PostProcessManager {
             backend::TextureFormat outFormat, bool translucent) noexcept;
 
     // Temporal Anti-aliasing
-    void prepareTaa(FrameGraph& fg, filament::Viewport const& svp,
+    void prepareTaa(FrameGraph& fg,
+            filament::Viewport const& svp,
+            TemporalAntiAliasingOptions const& taaOptions,
             FrameHistory& frameHistory,
             FrameHistoryEntry::TemporalAA FrameHistoryEntry::*pTaa,
             CameraInfo* inoutCameraInfo,
@@ -269,10 +271,6 @@ class PostProcessManager {
     backend::Handle<backend::HwTexture> getOneTextureArray() const;
     backend::Handle<backend::HwTexture> getZeroTextureArray() const;
 
-    math::float2 halton(size_t index) const noexcept {
-        return mHaltonSamples[index & 0xFu];
-    }
-
     class PostProcessMaterial {
     public:
         PostProcessMaterial() noexcept;
@@ -357,8 +355,15 @@ class PostProcessManager {
 
     std::uniform_real_distribution<float> mUniformDistribution{0.0f, 1.0f};
 
-    static const math::float2 sHaltonSamples[16];
-    math::float2 const* mHaltonSamples = sHaltonSamples;
+    template<size_t SIZE>
+    struct JitterSequence {
+        auto operator()(size_t i) const noexcept { return positions[i % SIZE] - 0.5f; }
+        const math::float2 positions[SIZE];
+    };
+
+    static const JitterSequence<4> sRGSS4;
+    static const JitterSequence<4> sUniformHelix4;
+    static const JitterSequence<16> sHaltonSamples;
 
     bool mWorkaroundSplitEasu : 1;
     bool mWorkaroundAllowReadOnlyAncillaryFeedbackLoop : 1;

diff --git a/filament/src/details/Renderer.cpp b/filament/src/details/Renderer.cpp
@@ -614,7 +614,7 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) {
 
     view.prepare(engine, driver, arena, svp, cameraInfo, getShaderUserTime(), needsAlphaChannel);
 
-    view.prepareUpscaler(scale);
+    view.prepareUpscaler(scale, dsrOptions);
 
     /*
      * Allocate command buffer
@@ -818,14 +818,14 @@ void FRenderer::renderJob(ArenaScope& arena, FView& view) {
                 [=, &view](FrameGraphResources const& resources,
                         auto const&, DriverApi& driver) mutable {
                     auto out = resources.getRenderPassInfo();
-                    view.executePickingQueries(driver, out.target, aoOptions.resolution);
+                    view.executePickingQueries(driver, out.target, scale * aoOptions.resolution);
                 });
     }
 
     // Store this frame's camera projection in the frame history.
     if (UTILS_UNLIKELY(taaOptions.enabled)) {
         // Apply the TAA jitter to everything after the structure pass, starting with the color pass.
-        ppm.prepareTaa(fg, svp, view.getFrameHistory(), &FrameHistoryEntry::taa,
+        ppm.prepareTaa(fg, svp, taaOptions, view.getFrameHistory(), &FrameHistoryEntry::taa,
                 &cameraInfo, view.getPerViewUniforms());
     }
 

diff --git a/filament/src/details/View.cpp b/filament/src/details/View.cpp
@@ -725,9 +725,9 @@ UTILS_NOINLINE
     });
 }
 
-void FView::prepareUpscaler(float2 scale) const noexcept {
+void FView::prepareUpscaler(float2 scale, DynamicResolutionOptions const& options) const noexcept {
     SYSTRACE_CALL();
-    const float bias = (mDynamicResolution.quality >= QualityLevel::HIGH) ?
+    const float bias = (options.quality >= QualityLevel::HIGH) ?
             std::log2(std::min(scale.x, scale.y)) : 0.0f;
     mPerViewUniforms.prepareLodBias(bias);
 }
@@ -985,15 +985,15 @@ void FView::drainFrameHistory(FEngine& engine) noexcept {
 }
 
 void FView::executePickingQueries(backend::DriverApi& driver,
-        backend::RenderTargetHandle handle, float scale) noexcept {
+        backend::RenderTargetHandle handle, float2 scale) noexcept {
 
     while (mActivePickingQueriesList) {
         FPickingQuery* const pQuery = mActivePickingQueriesList;
         mActivePickingQueriesList = pQuery->next;
 
         // adjust for dynamic resolution and structure buffer scale
-        const uint32_t x = uint32_t(float(pQuery->x) * (scale * mScale.x));
-        const uint32_t y = uint32_t(float(pQuery->y) * (scale * mScale.y));
+        const uint32_t x = uint32_t(float(pQuery->x) * scale.x);
+        const uint32_t y = uint32_t(float(pQuery->y) * scale.y);
 
         if (UTILS_UNLIKELY(driver.getFeatureLevel() == FeatureLevel::FEATURE_LEVEL_0)) {
             driver.readPixels(handle, x, y, 1, 1, {

diff --git a/filament/src/details/View.h b/filament/src/details/View.h
@@ -133,7 +133,7 @@ class FView : public View {
         return mName.c_str_safe();
     }
 
-    void prepareUpscaler(math::float2 scale) const noexcept;
+    void prepareUpscaler(math::float2 scale, DynamicResolutionOptions const& options) const noexcept;
     void prepareCamera(FEngine& engine, const CameraInfo& cameraInfo) const noexcept;
 
     void prepareViewport(
@@ -417,7 +417,7 @@ class FView : public View {
             View::PickingQueryResultCallback callback) noexcept;
 
     void executePickingQueries(backend::DriverApi& driver,
-            backend::RenderTargetHandle handle, float scale) noexcept;
+            backend::RenderTargetHandle handle, math::float2 scale) noexcept;
 
     void setMaterialGlobal(uint32_t index, math::float4 const& value);