From fd7567a6b56fcb82a52b85097582fc0a67038457 Mon Sep 17 00:00:00 2001
From: gdkchan <gab.dark.100@gmail.com>
Date: Tue, 28 Sep 2021 20:55:12 -0300
Subject: [PATCH] Only make render target 2D textures layered if needed (#2646)

* Only make render target 2D textures layered if needed

* Shader cache version bump

* Ensure topology is updated on channel swap
---
 .../Engine/Threed/DrawManager.cs              | 14 ++++++--
 .../Engine/Threed/StateUpdater.cs             |  9 ++++-
 .../Engine/Threed/ThreedClass.cs              |  1 +
 Ryujinx.Graphics.Gpu/Image/TextureCache.cs    | 13 +++++--
 .../Cache/Definition/HostShaderCacheEntry.cs  | 10 +++++-
 .../Definition/HostShaderCacheEntryHeader.cs  | 36 ++++++++++++++++---
 Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs    |  2 +-
 Ryujinx.Graphics.Shader/ShaderProgramInfo.cs  |  3 ++
 .../Translation/EmitterContext.cs             |  5 +++
 .../Translation/FeatureFlags.cs               |  7 ++--
 .../Translation/Translator.cs                 |  1 +
 11 files changed, 85 insertions(+), 16 deletions(-)

diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs
index e01938bd4a..82aff2049f 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs
@@ -13,6 +13,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         private readonly GpuChannel _channel;
         private readonly DeviceStateWithShadow<ThreedClassState> _state;
         private readonly DrawState _drawState;
+        private bool _topologySet;
 
         private bool _instancedDrawPending;
         private bool _instancedIndexed;
@@ -43,6 +44,14 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
             _drawState = drawState;
         }
 
+        /// <summary>
+        /// Marks the entire state as dirty, forcing a full host state update before the next draw.
+        /// </summary>
+        public void ForceStateDirty()
+        {
+            _topologySet = false;
+        }
+
         /// <summary>
         /// Pushes four 8-bit index buffer elements.
         /// </summary>
@@ -224,11 +233,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 _instanceIndex = 0;
             }
 
-            if (_drawState.Topology != topology)
+            if (_drawState.Topology != topology || !_topologySet)
             {
                 _context.Renderer.Pipeline.SetPrimitiveTopology(topology);
-
                 _drawState.Topology = topology;
+                _topologySet = true;
             }
         }
 
@@ -331,6 +340,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             _context.Renderer.Pipeline.SetPrimitiveTopology(topology);
             _drawState.Topology = topology;
+            _topologySet = true;
 
             ConditionalRenderEnabled renderEnable = ConditionalRendering.GetRenderEnable(
                 _context,
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
index f429ae9092..f9d16803e2 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs
@@ -6,7 +6,6 @@ using Ryujinx.Graphics.Gpu.Shader;
 using Ryujinx.Graphics.Shader;
 using Ryujinx.Graphics.Texture;
 using System;
-using System.Linq;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -31,6 +30,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
         private readonly ShaderProgramInfo[] _currentProgramInfo;
 
+        private bool _vtgWritesRtLayer;
         private byte _vsClipDistancesWritten;
 
         private bool _prevDrawIndexed;
@@ -334,6 +334,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
                 Image.Texture color = memoryManager.Physical.TextureCache.FindOrCreateTexture(
                     memoryManager,
                     colorState,
+                    _vtgWritesRtLayer,
                     samplesInX,
                     samplesInY,
                     sizeHint);
@@ -956,6 +957,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
             _drawState.VsUsesInstanceId = gs.Shaders[0]?.Info.UsesInstanceId ?? false;
             _vsClipDistancesWritten = gs.Shaders[0]?.Info.ClipDistancesWritten ?? 0;
+            _vtgWritesRtLayer = false;
 
             if (oldVsClipDistancesWritten != _vsClipDistancesWritten)
             {
@@ -979,6 +981,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
 
                 Span<TextureBindingInfo> textureBindings = _channel.TextureManager.RentGraphicsTextureBindings(stage, info.Textures.Count);
 
+                if (info.UsesRtLayer)
+                {
+                    _vtgWritesRtLayer = true;
+                }
+
                 for (int index = 0; index < info.Textures.Count; index++)
                 {
                     var descriptor = info.Textures[index];
diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
index 3d02af96b4..d4f228e9b9 100644
--- a/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
+++ b/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
@@ -139,6 +139,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
         /// </summary>
         public void ForceStateDirty()
         {
+            _drawManager.ForceStateDirty();
             _stateUpdater.SetAllDirty();
         }
 
diff --git a/Ryujinx.Graphics.Gpu/Image/TextureCache.cs b/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
index a6fa96526f..cc6867a645 100644
--- a/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
+++ b/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
@@ -244,11 +244,18 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         /// <param name="memoryManager">GPU memory manager where the texture is mapped</param>
         /// <param name="colorState">Color buffer texture to find or create</param>
+        /// <param name="layered">Indicates if the texture might be accessed with a non-zero layer index</param>
         /// <param name="samplesInX">Number of samples in the X direction, for MSAA</param>
         /// <param name="samplesInY">Number of samples in the Y direction, for MSAA</param>
         /// <param name="sizeHint">A hint indicating the minimum used size for the texture</param>
         /// <returns>The texture</returns>
-        public Texture FindOrCreateTexture(MemoryManager memoryManager, RtColorState colorState, int samplesInX, int samplesInY, Size sizeHint)
+        public Texture FindOrCreateTexture(
+            MemoryManager memoryManager,
+            RtColorState colorState,
+            bool layered,
+            int samplesInX,
+            int samplesInY,
+            Size sizeHint)
         {
             bool isLinear = colorState.MemoryLayout.UnpackIsLinear();
 
@@ -263,13 +270,13 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
             else if ((samplesInX | samplesInY) != 1)
             {
-                target = colorState.Depth > 1
+                target = colorState.Depth > 1 && layered
                     ? Target.Texture2DMultisampleArray
                     : Target.Texture2DMultisample;
             }
             else
             {
-                target = colorState.Depth > 1
+                target = colorState.Depth > 1 && layered
                     ? Target.Texture2DArray
                     : Target.Texture2D;
             }
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntry.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntry.cs
index b538e2de5a..68f6b3c1d9 100644
--- a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntry.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntry.cs
@@ -76,6 +76,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
                                                     programInfo.Textures.Count,
                                                     programInfo.Images.Count,
                                                     programInfo.UsesInstanceId,
+                                                    programInfo.UsesRtLayer,
                                                     programInfo.ClipDistancesWritten);
             CBuffers = programInfo.CBuffers.ToArray();
             SBuffers = programInfo.SBuffers.ToArray();
@@ -89,7 +90,14 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
         /// <returns>A new <see cref="ShaderProgramInfo"/> from this instance</returns>
         internal ShaderProgramInfo ToShaderProgramInfo()
         {
-            return new ShaderProgramInfo(CBuffers, SBuffers, Textures, Images, Header.UsesInstanceId, Header.ClipDistancesWritten);
+            return new ShaderProgramInfo(
+                CBuffers,
+                SBuffers,
+                Textures,
+                Images,
+                Header.UseFlags.HasFlag(UseFlags.InstanceId),
+                Header.UseFlags.HasFlag(UseFlags.RtLayer),
+                Header.ClipDistancesWritten);
         }
 
         /// <summary>
diff --git a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntryHeader.cs b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntryHeader.cs
index 7f27124ffd..4b8b15bc21 100644
--- a/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntryHeader.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/Cache/Definition/HostShaderCacheEntryHeader.cs
@@ -1,8 +1,28 @@
 using System.Runtime.InteropServices;
-using Ryujinx.Graphics.Shader;
 
 namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
 {
+    /// <summary>
+    /// Flags indicating if the shader accesses certain built-ins, such as the instance ID.
+    /// </summary>
+    enum UseFlags : byte
+    {
+        /// <summary>
+        /// None of the built-ins are used.
+        /// </summary>
+        None = 0,
+
+        /// <summary>
+        /// Indicates whenever the vertex shader reads the gl_InstanceID built-in.
+        /// </summary>
+        InstanceId = 1 << 0,
+
+        /// <summary>
+        /// Indicates whenever any of the VTG stages writes to the gl_Layer built-in.
+        /// </summary>
+        RtLayer = 1 << 1
+    }
+
     /// <summary>
     /// Host shader entry header used for binding information.
     /// </summary>
@@ -30,10 +50,9 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
         public int ImagesCount;
 
         /// <summary>
-        /// Set to true if the shader uses instance id.
+        /// Flags indicating if the shader accesses certain built-ins, such as the instance ID.
         /// </summary>
-        [MarshalAs(UnmanagedType.I1)]
-        public bool UsesInstanceId;
+        public UseFlags UseFlags;
 
         /// <summary>
         /// Set to true if this entry is in use.
@@ -65,15 +84,22 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache.Definition
             int texturesCount,
             int imagesCount,
             bool usesInstanceId,
+            bool usesRtLayer,
             byte clipDistancesWritten) : this()
         {
             CBuffersCount        = cBuffersCount;
             SBuffersCount        = sBuffersCount;
             TexturesCount        = texturesCount;
             ImagesCount          = imagesCount;
-            UsesInstanceId       = usesInstanceId;
             ClipDistancesWritten = clipDistancesWritten;
             InUse                = true;
+
+            UseFlags = usesInstanceId ? UseFlags.InstanceId : UseFlags.None;
+
+            if (usesRtLayer)
+            {
+                UseFlags |= UseFlags.RtLayer;
+            }
         }
     }
 }
diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
index 926a673aa8..af6f9462cb 100644
--- a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
+++ b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
@@ -40,7 +40,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
         /// <summary>
         /// Version of the codegen (to be changed when codegen or guest format change).
         /// </summary>
-        private const ulong ShaderCodeGenVersion = 2627;
+        private const ulong ShaderCodeGenVersion = 2646;
 
         // Progress reporting helpers
         private volatile int _shaderCount;
diff --git a/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs b/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs
index 9329442fe9..a9ce486b69 100644
--- a/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs
+++ b/Ryujinx.Graphics.Shader/ShaderProgramInfo.cs
@@ -11,6 +11,7 @@ namespace Ryujinx.Graphics.Shader
         public ReadOnlyCollection<TextureDescriptor> Images   { get; }
 
         public bool UsesInstanceId { get; }
+        public bool UsesRtLayer { get; }
         public byte ClipDistancesWritten { get; }
 
         public ShaderProgramInfo(
@@ -19,6 +20,7 @@ namespace Ryujinx.Graphics.Shader
             TextureDescriptor[] textures,
             TextureDescriptor[] images,
             bool                usesInstanceId,
+            bool                usesRtLayer,
             byte                clipDistancesWritten)
         {
             CBuffers = Array.AsReadOnly(cBuffers);
@@ -27,6 +29,7 @@ namespace Ryujinx.Graphics.Shader
             Images   = Array.AsReadOnly(images);
 
             UsesInstanceId = usesInstanceId;
+            UsesRtLayer = usesRtLayer;
             ClipDistancesWritten = clipDistancesWritten;
         }
     }
diff --git a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
index 5cdd5c0a6e..c96b75a5a0 100644
--- a/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
+++ b/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs
@@ -121,6 +121,11 @@ namespace Ryujinx.Graphics.Shader.Translation
                         break;
                 }
             }
+
+            if (Config.Stage != ShaderStage.Fragment && attribute == AttributeConsts.Layer)
+            {
+                Config.SetUsedFeature(FeatureFlags.RtLayer);
+            }
         }
 
         public void MarkLabel(Operand label)
diff --git a/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs b/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs
index 1636afd3af..f602ea64dd 100644
--- a/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs
+++ b/Ryujinx.Graphics.Shader/Translation/FeatureFlags.cs
@@ -17,8 +17,9 @@ namespace Ryujinx.Graphics.Shader.Translation
 
         Bindless = 1 << 2,
         InstanceId = 1 << 3,
-        CbIndexing = 1 << 4,
-        IaIndexing = 1 << 5,
-        OaIndexing = 1 << 6
+        RtLayer = 1 << 4,
+        CbIndexing = 1 << 5,
+        IaIndexing = 1 << 6,
+        OaIndexing = 1 << 7
     }
 }
diff --git a/Ryujinx.Graphics.Shader/Translation/Translator.cs b/Ryujinx.Graphics.Shader/Translation/Translator.cs
index f1e92d7c8c..1abf19d716 100644
--- a/Ryujinx.Graphics.Shader/Translation/Translator.cs
+++ b/Ryujinx.Graphics.Shader/Translation/Translator.cs
@@ -106,6 +106,7 @@ namespace Ryujinx.Graphics.Shader.Translation
                 config.GetTextureDescriptors(),
                 config.GetImageDescriptors(),
                 config.UsedFeatures.HasFlag(FeatureFlags.InstanceId),
+                config.UsedFeatures.HasFlag(FeatureFlags.RtLayer),
                 config.ClipDistancesWritten);
 
             return program;