Add support for alpha to coverage dithering (#3069)

* Add support for alpha to coverage dithering

* Shader cache version bump

* Fix wrong alpha register

* Ensure support buffer is cleared

* New shader specialization based approach

Ryujinx.Graphics.GAL/IPipeline.cs

@@ -60,6 +60,8 @@ namespace Ryujinx.Graphics.GAL
 
         void SetLogicOpState(bool enable, LogicalOp op);
 
+        void SetMultisampleState(MultisampleDescriptor multisample);
+
         void SetPatchParameters(int vertices, ReadOnlySpan<float> defaultOuterLevel, ReadOnlySpan<float> defaultInnerLevel);
         void SetPointParameters(float size, bool isProgramPointSize, bool enablePointSprite, Origin origin);
 

Ryujinx.Graphics.GAL/MultisampleDescriptor.cs

@@ -0,0 +1,19 @@
+namespace Ryujinx.Graphics.GAL
+{
+    public struct MultisampleDescriptor
+    {
+        public bool AlphaToCoverageEnable { get; }
+        public bool AlphaToCoverageDitherEnable { get; }
+        public bool AlphaToOneEnable { get; }
+
+        public MultisampleDescriptor(
+            bool alphaToCoverageEnable,
+            bool alphaToCoverageDitherEnable,
+            bool alphaToOneEnable)
+        {
+            AlphaToCoverageEnable = alphaToCoverageEnable;
+            AlphaToCoverageDitherEnable = alphaToCoverageDitherEnable;
+            AlphaToOneEnable = alphaToOneEnable;
+        }
+    }
+}

Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs

@@ -179,6 +179,8 @@ namespace Ryujinx.Graphics.GAL.Multithreading
                 SetLineParametersCommand.Run(ref GetCommand<SetLineParametersCommand>(memory), threaded, renderer);
             _lookup[(int)CommandType.SetLogicOpState] = (Span<byte> memory, ThreadedRenderer threaded, IRenderer renderer) =>
                 SetLogicOpStateCommand.Run(ref GetCommand<SetLogicOpStateCommand>(memory), threaded, renderer);
+            _lookup[(int)CommandType.SetMultisampleState] = (Span<byte> memory, ThreadedRenderer threaded, IRenderer renderer) =>
+                SetMultisampleStateCommand.Run(ref GetCommand<SetMultisampleStateCommand>(memory), threaded, renderer);
             _lookup[(int)CommandType.SetPatchParameters] = (Span<byte> memory, ThreadedRenderer threaded, IRenderer renderer) =>
                 SetPatchParametersCommand.Run(ref GetCommand<SetPatchParametersCommand>(memory), threaded, renderer);
             _lookup[(int)CommandType.SetPointParameters] = (Span<byte> memory, ThreadedRenderer threaded, IRenderer renderer) =>

Ryujinx.Graphics.GAL/Multithreading/CommandType.cs

@@ -71,6 +71,7 @@
         SetIndexBuffer,
         SetLineParameters,
         SetLogicOpState,
+        SetMultisampleState,
         SetPatchParameters,
         SetPointParameters,
         SetPolygonMode,

Ryujinx.Graphics.GAL/Multithreading/Commands/SetMultisampleStateCommand.cs

@@ -0,0 +1,18 @@
+namespace Ryujinx.Graphics.GAL.Multithreading.Commands
+{
+    struct SetMultisampleStateCommand : IGALCommand
+    {
+        public CommandType CommandType => CommandType.SetMultisampleState;
+        private MultisampleDescriptor _multisample;
+
+        public void Set(MultisampleDescriptor multisample)
+        {
+            _multisample = multisample;
+        }
+
+        public static void Run(ref SetMultisampleStateCommand command, ThreadedRenderer threaded, IRenderer renderer)
+        {
+            renderer.Pipeline.SetMultisampleState(command._multisample);
+        }
+    }
+}

Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs

@@ -184,6 +184,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading
             _renderer.QueueCommand();
         }
 
+        public void SetMultisampleState(MultisampleDescriptor multisample)
+        {
+            _renderer.New<SetMultisampleStateCommand>().Set(multisample);
+            _renderer.QueueCommand();
+        }
+
         public void SetPatchParameters(int vertices, ReadOnlySpan<float> defaultOuterLevel, ReadOnlySpan<float> defaultInnerLevel)
         {
             _renderer.New<SetPatchParametersCommand>().Set(vertices, defaultOuterLevel, defaultInnerLevel);

Ryujinx.Graphics.GAL/SupportBufferUpdater.cs

@@ -18,6 +18,7 @@ namespace Ryujinx.Graphics.GAL
         {
             _renderer = renderer;
             Handle = renderer.CreateBuffer(SupportBuffer.RequiredSize);
+            renderer.Pipeline.ClearBuffer(Handle, 0, SupportBuffer.RequiredSize, 0);
         }
 
         private void MarkDirty(int startOffset, int byteSize)

Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs

@@ -166,7 +166,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                     nameof(ThreedClassState.BlendEnable),
                     nameof(ThreedClassState.BlendState)),
 
-                new StateUpdateCallbackEntry(UpdateLogicOpState, nameof(ThreedClassState.LogicOpState))
+                new StateUpdateCallbackEntry(UpdateLogicOpState, nameof(ThreedClassState.LogicOpState)),
+
+                new StateUpdateCallbackEntry(UpdateMultisampleState,
+                    nameof(ThreedClassState.AlphaToCoverageDitherEnable),
+                    nameof(ThreedClassState.MultisampleControl))
             });
         }
 
@@ -1092,6 +1096,20 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
             _context.Renderer.Pipeline.SetLogicOpState(logicOpState.Enable, logicOpState.LogicalOp);
         }
 
+        /// <summary>
+        /// Updates multisample state, based on guest state.
+        /// </summary>
+        private void UpdateMultisampleState()
+        {
+            bool alphaToCoverageEnable = (_state.State.MultisampleControl & 1) != 0;
+            bool alphaToOneEnable = (_state.State.MultisampleControl & 0x10) != 0;
+
+            _context.Renderer.Pipeline.SetMultisampleState(new MultisampleDescriptor(
+                alphaToCoverageEnable,
+                _state.State.AlphaToCoverageDitherEnable,
+                alphaToOneEnable));
+        }
+
         /// <summary>
         /// Updates host shaders based on the guest GPU state.
         /// </summary>
@@ -1231,7 +1249,9 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 _state.State.EarlyZForce,
                 _drawState.Topology,
                 _state.State.TessMode,
-                _state.State.ViewportTransformEnable == 0);
+                _state.State.ViewportTransformEnable == 0,
+                (_state.State.MultisampleControl & 1) != 0,
+                _state.State.AlphaToCoverageDitherEnable);
         }
 
         /// <summary>

Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClassState.cs

@@ -767,7 +767,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         public SamplerIndex SamplerIndex;
         public fixed uint Reserved1238[37];
         public Boolean32 DepthTestEnable;
-        public fixed uint Reserved12D0[5];
+        public fixed uint Reserved12D0[4];
+        public Boolean32 AlphaToCoverageDitherEnable;
         public Boolean32 BlendIndependent;
         public Boolean32 DepthWriteEnable;
         public Boolean32 AlphaTestEnable;
@@ -802,9 +803,10 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         public Boolean32 PointSpriteEnable;
         public fixed uint Reserved1524[3];
         public uint ResetCounter;
-        public uint Reserved1534;
+        public Boolean32 MultisampleEnable;
         public Boolean32 RtDepthStencilEnable;
-        public fixed uint Reserved153C[5];
+        public uint MultisampleControl;
+        public fixed uint Reserved1540[4];
         public GpuVa RenderEnableAddress;
         public Condition RenderEnableCondition;
         public PoolState SamplerPoolState;

Ryujinx.Graphics.Gpu/Shader/Cache/Migration.cs

@@ -167,6 +167,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
                             accessorHeader.StateFlags.HasFlag(GuestGpuStateFlags.EarlyZForce),
                             topology,
                             tessMode,
+                            false,
+                            false,
                             false);
 
                         TransformFeedbackDescriptor[] tfdNew = null;

Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheGpuAccessor.cs

@@ -67,6 +67,12 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
             return MemoryMarshal.Cast<byte, ulong>(_data.Span.Slice((int)address));
         }
 
+        /// <inheritdoc/>
+        public bool QueryAlphaToCoverageDitherEnable()
+        {
+            return _oldSpecState.GraphicsState.AlphaToCoverageEnable && _oldSpecState.GraphicsState.AlphaToCoverageDitherEnable;
+        }
+
         /// <inheritdoc/>
         public int QueryBindingConstantBuffer(int index)
         {

Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs

@@ -21,7 +21,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
         private const ushort FileFormatVersionMajor = 1;
         private const ushort FileFormatVersionMinor = 1;
         private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor;
-        private const uint CodeGenVersion = 3424;
+        private const uint CodeGenVersion = 3069;
 
         private const string SharedTocFileName = "shared.toc";
         private const string SharedDataFileName = "shared.data";

Ryujinx.Graphics.Gpu/Shader/GpuAccessor.cs

@@ -66,6 +66,12 @@ namespace Ryujinx.Graphics.Gpu.Shader
             return MemoryMarshal.Cast<byte, ulong>(_channel.MemoryManager.GetSpan(address, size));
         }
 
+        /// <inheritdoc/>
+        public bool QueryAlphaToCoverageDitherEnable()
+        {
+            return _state.GraphicsState.AlphaToCoverageEnable && _state.GraphicsState.AlphaToCoverageDitherEnable;
+        }
+
         /// <inheritdoc/>
         public int QueryBindingConstantBuffer(int index)
         {

Ryujinx.Graphics.Gpu/Shader/GpuChannelGraphicsState.cs

@@ -30,6 +30,16 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// </summary>
         public readonly bool ViewportTransformDisable;
 
+        /// <summary>
+        /// Indicates whenever alpha-to-coverage is enabled.
+        /// </summary>
+        public readonly bool AlphaToCoverageEnable;
+
+        /// <summary>
+        /// Indicates whenever alpha-to-coverage dithering is enabled.
+        /// </summary>
+        public readonly bool AlphaToCoverageDitherEnable;
+
         /// <summary>
         /// Creates a new GPU graphics state.
         /// </summary>
@@ -37,12 +47,22 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <param name="topology">Primitive topology</param>
         /// <param name="tessellationMode">Tessellation mode</param>
         /// <param name="viewportTransformDisable">Indicates whenever the viewport transform is disabled</param>
-        public GpuChannelGraphicsState(bool earlyZForce, PrimitiveTopology topology, TessMode tessellationMode, bool viewportTransformDisable)
+        /// <param name="alphaToCoverageEnable">Indicates whenever alpha-to-coverage is enabled</param>
+        /// <param name="alphaToCoverageDitherEnable">Indicates whenever alpha-to-coverage dithering is enabled</param>
+        public GpuChannelGraphicsState(
+            bool earlyZForce,
+            PrimitiveTopology topology,
+            TessMode tessellationMode,
+            bool viewportTransformDisable,
+            bool alphaToCoverageEnable,
+            bool alphaToCoverageDitherEnable)
         {
             EarlyZForce = earlyZForce;
             Topology = topology;
             TessellationMode = tessellationMode;
             ViewportTransformDisable = viewportTransformDisable;
+            AlphaToCoverageEnable = alphaToCoverageEnable;
+            AlphaToCoverageDitherEnable = alphaToCoverageDitherEnable;
         }
     }
 }

Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs

@@ -455,6 +455,14 @@ namespace Ryujinx.Graphics.Gpu.Shader
                 return false;
             }
 
+            bool thisA2cDitherEnable = GraphicsState.AlphaToCoverageEnable && GraphicsState.AlphaToCoverageDitherEnable;
+            bool otherA2cDitherEnable = graphicsState.AlphaToCoverageEnable && graphicsState.AlphaToCoverageDitherEnable;
+
+            if (otherA2cDitherEnable != thisA2cDitherEnable)
+            {
+                return false;
+            }
+
             return Matches(channel, poolState, checkTextures, isCompute: false);
         }
 

Ryujinx.Graphics.OpenGL/HwCapabilities.cs

@@ -5,6 +5,7 @@ namespace Ryujinx.Graphics.OpenGL
 {
     static class HwCapabilities
     {
+        private static readonly Lazy<bool> _supportsAlphaToCoverageDitherControl = new Lazy<bool>(() => HasExtension("GL_NV_alpha_to_coverage_dither_control"));
         private static readonly Lazy<bool> _supportsAstcCompression              = new Lazy<bool>(() => HasExtension("GL_KHR_texture_compression_astc_ldr"));
         private static readonly Lazy<bool> _supportsDrawTexture                  = new Lazy<bool>(() => HasExtension("GL_NV_draw_texture"));
         private static readonly Lazy<bool> _supportsFragmentShaderInterlock      = new Lazy<bool>(() => HasExtension("GL_ARB_fragment_shader_interlock"));
@@ -43,6 +44,7 @@ namespace Ryujinx.Graphics.OpenGL
 
         public static bool UsePersistentBufferForFlush       => _gpuVendor.Value == GpuVendor.AmdWindows || _gpuVendor.Value == GpuVendor.Nvidia;
 
+        public static bool SupportsAlphaToCoverageDitherControl => _supportsAlphaToCoverageDitherControl.Value;
         public static bool SupportsAstcCompression              => _supportsAstcCompression.Value;
         public static bool SupportsDrawTexture                  => _supportsDrawTexture.Value;
         public static bool SupportsFragmentShaderInterlock      => _supportsFragmentShaderInterlock.Value;

Ryujinx.Graphics.OpenGL/Pipeline.cs

@@ -918,6 +918,34 @@ namespace Ryujinx.Graphics.OpenGL
             }
         }
 
+        public void SetMultisampleState(MultisampleDescriptor multisample)
+        {
+            if (multisample.AlphaToCoverageEnable)
+            {
+                GL.Enable(EnableCap.SampleAlphaToCoverage);
+
+                if (multisample.AlphaToOneEnable)
+                {
+                    GL.Enable(EnableCap.SampleAlphaToOne);
+                }
+                else
+                {
+                    GL.Disable(EnableCap.SampleAlphaToOne);
+                }
+
+                if (HwCapabilities.SupportsAlphaToCoverageDitherControl)
+                {
+                    GL.NV.AlphaToCoverageDitherControl(multisample.AlphaToCoverageDitherEnable
+                        ? NvAlphaToCoverageDitherControl.AlphaToCoverageDitherEnableNv
+                        : NvAlphaToCoverageDitherControl.AlphaToCoverageDitherDisableNv);
+                }
+            }
+            else
+            {
+                GL.Disable(EnableCap.SampleAlphaToCoverage);
+            }
+        }
+
         public void SetLineParameters(float width, bool smooth)
         {
             if (smooth)

Ryujinx.Graphics.Shader/IGpuAccessor.cs

@@ -34,6 +34,15 @@ namespace Ryujinx.Graphics.Shader
         /// <returns>Span of the memory location</returns>
         ReadOnlySpan<ulong> GetCode(ulong address, int minimumSize);
 
+        /// <summary>
+        /// Queries whenever the alpha-to-coverage dithering feature is enabled.
+        /// </summary>
+        /// <returns>True if the feature is enabled, false otherwise</returns>
+        bool QueryAlphaToCoverageDitherEnable()
+        {
+            return false;
+        }
+
         /// <summary>
         /// Queries the binding number of a constant buffer.
         /// </summary>

Ryujinx.Graphics.Shader/Translation/EmitterContext.cs

@@ -205,6 +205,8 @@ namespace Ryujinx.Graphics.Shader.Translation
             }
             else if (Config.Stage == ShaderStage.Fragment)
             {
+                GenerateAlphaToCoverageDitherDiscard();
+
                 if (Config.OmapDepth)
                 {
                     Operand dest = Attribute(AttributeConsts.FragmentOutputDepth);
@@ -266,6 +268,35 @@ namespace Ryujinx.Graphics.Shader.Translation
             }
         }
 
+        private void GenerateAlphaToCoverageDitherDiscard()
+        {
+            // If the feature is disabled, or alpha is not written, then we're done.
+            if (!Config.GpuAccessor.QueryAlphaToCoverageDitherEnable() || (Config.OmapTargets & 8) == 0)
+            {
+                return;
+            }
+
+            // 11 11 11 10 10 10 10 00
+            // 11 01 01 01 01 00 00 00
+            Operand ditherMask = Const(unchecked((int)0xfbb99110u));
+
+            Operand x = this.BitwiseAnd(this.FP32ConvertToU32(Attribute(AttributeConsts.PositionX)), Const(1));
+            Operand y = this.BitwiseAnd(this.FP32ConvertToU32(Attribute(AttributeConsts.PositionY)), Const(1));
+            Operand xy = this.BitwiseOr(x, this.ShiftLeft(y, Const(1)));
+
+            Operand alpha = Register(3, RegisterType.Gpr);
+            Operand scaledAlpha = this.FPMultiply(this.FPSaturate(alpha), ConstF(8));
+            Operand quantizedAlpha = this.IMinimumU32(this.FP32ConvertToU32(scaledAlpha), Const(7));
+            Operand shift = this.BitwiseOr(this.ShiftLeft(quantizedAlpha, Const(2)), xy);
+            Operand opaque = this.BitwiseAnd(this.ShiftRightU32(ditherMask, shift), Const(1));
+
+            Operand a2cDitherEndLabel = Label();
+
+            this.BranchIfTrue(a2cDitherEndLabel, opaque);
+            this.Discard();
+            this.MarkLabel(a2cDitherEndLabel);
+        }
+
         public Operation[] GetOperations()
         {
             return _operations.ToArray();