From 99ffc061d30c92f224c6b839290e181b6179893d Mon Sep 17 00:00:00 2001 From: riperiperi Date: Fri, 17 Jun 2022 17:09:14 +0100 Subject: [PATCH] Optimize Texture Binding and Shader Specialization Checks (#3399) * Changes 1 * Changes 2 * Better ModifiedSequence handling This should handle PreciseEvents properly, and simplifies a few things. * Minor changes, remove debug log * Handle stage.Info being null Hopefully fixes Catherine crash * Fix shader specialization fast texture lookup * Fix some things. * Address Feedback Part 1 * Make method static. --- .../Engine/Compute/ComputeClass.cs | 19 +- .../Engine/Threed/StateUpdater.cs | 24 +- Ryujinx.Graphics.Gpu/Image/Pool.cs | 42 +++ Ryujinx.Graphics.Gpu/Image/Sampler.cs | 7 + Ryujinx.Graphics.Gpu/Image/SamplerPool.cs | 35 +++ Ryujinx.Graphics.Gpu/Image/Texture.cs | 8 + .../Image/TextureBindingsManager.cs | 292 +++++++++++++----- Ryujinx.Graphics.Gpu/Image/TextureManager.cs | 56 +++- Ryujinx.Graphics.Gpu/Image/TexturePool.cs | 144 ++++++--- Ryujinx.Graphics.Gpu/Memory/BufferManager.cs | 19 ++ .../Shader/CachedShaderProgram.cs | 2 + Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs | 4 +- .../Shader/ShaderSpecializationList.cs | 4 +- .../Shader/ShaderSpecializationState.cs | 231 +++++++++++--- Ryujinx.Graphics.Shader/TextureHandle.cs | 59 ++++ 15 files changed, 766 insertions(+), 180 deletions(-) diff --git a/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs b/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs index 87c14da8fd..a1a9b481fb 100644 --- a/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs +++ b/Ryujinx.Graphics.Gpu/Engine/Compute/ComputeClass.cs @@ -188,6 +188,9 @@ namespace Ryujinx.Graphics.Gpu.Engine.Compute _channel.BufferManager.SetComputeStorageBufferBindings(info.SBuffers); _channel.BufferManager.SetComputeUniformBufferBindings(info.CBuffers); + int maxTextureBinding = -1; + int maxImageBinding = -1; + TextureBindingInfo[] textureBindings = _channel.TextureManager.RentComputeTextureBindings(info.Textures.Count); for (int index = 0; index < info.Textures.Count; index++) @@ -202,6 +205,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Compute descriptor.CbufSlot, descriptor.HandleIndex, descriptor.Flags); + + if (descriptor.Binding > maxTextureBinding) + { + maxTextureBinding = descriptor.Binding; + } } TextureBindingInfo[] imageBindings = _channel.TextureManager.RentComputeImageBindings(info.Images.Count); @@ -220,9 +228,18 @@ namespace Ryujinx.Graphics.Gpu.Engine.Compute descriptor.CbufSlot, descriptor.HandleIndex, descriptor.Flags); + + if (descriptor.Binding > maxImageBinding) + { + maxImageBinding = descriptor.Binding; + } } - _channel.TextureManager.CommitComputeBindings(); + _channel.TextureManager.SetComputeMaxBindings(maxTextureBinding, maxImageBinding); + + // Should never return false for mismatching spec state, since the shader was fetched above. + _channel.TextureManager.CommitComputeBindings(cs.SpecializationState); + _channel.BufferManager.CommitComputeBindings(); _context.Renderer.Pipeline.DispatchCompute(qmd.CtaRasterWidth, qmd.CtaRasterHeight, qmd.CtaRasterDepth); diff --git a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs index f648479b66..c64c760ae2 100644 --- a/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs +++ b/Ryujinx.Graphics.Gpu/Engine/Threed/StateUpdater.cs @@ -201,7 +201,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed // of the shader for the new state. if (_shaderSpecState != null) { - if (!_shaderSpecState.MatchesGraphics(_channel, GetPoolState(), GetGraphicsState())) + if (!_shaderSpecState.MatchesGraphics(_channel, GetPoolState(), GetGraphicsState(), false)) { ForceShaderUpdate(); } @@ -275,7 +275,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed { UpdateStorageBuffers(); - _channel.TextureManager.CommitGraphicsBindings(); + if (!_channel.TextureManager.CommitGraphicsBindings(_shaderSpecState)) + { + // Shader must be reloaded. + UpdateShaderState(); + } + _channel.BufferManager.CommitGraphicsBindings(); } @@ -1150,6 +1155,9 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed return; } + int maxTextureBinding = -1; + int maxImageBinding = -1; + Span textureBindings = _channel.TextureManager.RentGraphicsTextureBindings(stage, info.Textures.Count); if (info.UsesRtLayer) @@ -1169,6 +1177,11 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed descriptor.CbufSlot, descriptor.HandleIndex, descriptor.Flags); + + if (descriptor.Binding > maxTextureBinding) + { + maxTextureBinding = descriptor.Binding; + } } TextureBindingInfo[] imageBindings = _channel.TextureManager.RentGraphicsImageBindings(stage, info.Images.Count); @@ -1187,8 +1200,15 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed descriptor.CbufSlot, descriptor.HandleIndex, descriptor.Flags); + + if (descriptor.Binding > maxImageBinding) + { + maxImageBinding = descriptor.Binding; + } } + _channel.TextureManager.SetGraphicsMaxBindings(maxTextureBinding, maxImageBinding); + _channel.BufferManager.SetGraphicsStorageBufferBindings(stage, info.SBuffers); _channel.BufferManager.SetGraphicsUniformBufferBindings(stage, info.CBuffers); } diff --git a/Ryujinx.Graphics.Gpu/Image/Pool.cs b/Ryujinx.Graphics.Gpu/Image/Pool.cs index f54ce1d705..8e21051344 100644 --- a/Ryujinx.Graphics.Gpu/Image/Pool.cs +++ b/Ryujinx.Graphics.Gpu/Image/Pool.cs @@ -1,6 +1,7 @@ using Ryujinx.Cpu.Tracking; using Ryujinx.Graphics.Gpu.Memory; using System; +using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Image { @@ -16,6 +17,7 @@ namespace Ryujinx.Graphics.Gpu.Image protected GpuContext Context; protected PhysicalMemory PhysicalMemory; protected int SequenceNumber; + protected int ModifiedSequenceNumber; protected T1[] Items; protected T2[] DescriptorCache; @@ -41,6 +43,9 @@ namespace Ryujinx.Graphics.Gpu.Image private readonly CpuMultiRegionHandle _memoryTracking; private readonly Action _modifiedDelegate; + private int _modifiedSequenceOffset; + private bool _modified; + /// /// Creates a new instance of the GPU resource pool. /// @@ -79,6 +84,16 @@ namespace Ryujinx.Graphics.Gpu.Image return PhysicalMemory.Read(Address + (ulong)id * DescriptorSize); } + /// + /// Gets a reference to the descriptor for a given ID. + /// + /// ID of the descriptor. This is effectively a zero-based index + /// A reference to the descriptor + public ref readonly T2 GetDescriptorRef(int id) + { + return ref MemoryMarshal.Cast(PhysicalMemory.GetSpan(Address + (ulong)id * DescriptorSize, DescriptorSize))[0]; + } + /// /// Gets the GPU resource with the given ID. /// @@ -93,7 +108,13 @@ namespace Ryujinx.Graphics.Gpu.Image /// public void SynchronizeMemory() { + _modified = false; _memoryTracking.QueryModified(_modifiedDelegate); + + if (_modified) + { + UpdateModifiedSequence(); + } } /// @@ -103,6 +124,8 @@ namespace Ryujinx.Graphics.Gpu.Image /// Size of the modified region private void RegionModified(ulong mAddress, ulong mSize) { + _modified = true; + if (mAddress < Address) { mAddress = Address; @@ -118,6 +141,15 @@ namespace Ryujinx.Graphics.Gpu.Image InvalidateRangeImpl(mAddress, mSize); } + /// + /// Updates the modified sequence number using the current sequence number and offset, + /// indicating that it has been modified. + /// + protected void UpdateModifiedSequence() + { + ModifiedSequenceNumber = SequenceNumber + _modifiedSequenceOffset; + } + /// /// An action to be performed when a precise memory access occurs to this resource. /// Makes sure that the dirty flags are checked. @@ -129,6 +161,16 @@ namespace Ryujinx.Graphics.Gpu.Image { if (write && Context.SequenceNumber == SequenceNumber) { + if (ModifiedSequenceNumber == SequenceNumber + _modifiedSequenceOffset) + { + // The modified sequence number is offset when PreciseActions occur so that + // users checking it will see an increment and know the pool has changed since + // their last look, even though the main SequenceNumber has not been changed. + + _modifiedSequenceOffset++; + } + + // Force the pool to be checked again the next time it is used. SequenceNumber--; } diff --git a/Ryujinx.Graphics.Gpu/Image/Sampler.cs b/Ryujinx.Graphics.Gpu/Image/Sampler.cs index f8923d3495..b70ac9eb98 100644 --- a/Ryujinx.Graphics.Gpu/Image/Sampler.cs +++ b/Ryujinx.Graphics.Gpu/Image/Sampler.cs @@ -8,6 +8,11 @@ namespace Ryujinx.Graphics.Gpu.Image /// class Sampler : IDisposable { + /// + /// True if the sampler is disposed, false otherwise. + /// + public bool IsDisposed { get; private set; } + /// /// Host sampler object. /// @@ -101,6 +106,8 @@ namespace Ryujinx.Graphics.Gpu.Image /// public void Dispose() { + IsDisposed = true; + _hostSampler.Dispose(); _anisoSampler?.Dispose(); } diff --git a/Ryujinx.Graphics.Gpu/Image/SamplerPool.cs b/Ryujinx.Graphics.Gpu/Image/SamplerPool.cs index e205ec4879..e95800ada8 100644 --- a/Ryujinx.Graphics.Gpu/Image/SamplerPool.cs +++ b/Ryujinx.Graphics.Gpu/Image/SamplerPool.cs @@ -48,6 +48,8 @@ namespace Ryujinx.Graphics.Gpu.Image Items[i] = null; } } + + UpdateModifiedSequence(); } SequenceNumber = Context.SequenceNumber; @@ -71,6 +73,39 @@ namespace Ryujinx.Graphics.Gpu.Image return sampler; } + /// + /// Checks if the pool was modified, and returns the last sequence number where a modification was detected. + /// + /// A number that increments each time a modification is detected + public int CheckModified() + { + if (SequenceNumber != Context.SequenceNumber) + { + SequenceNumber = Context.SequenceNumber; + + if (_forcedAnisotropy != GraphicsConfig.MaxAnisotropy) + { + _forcedAnisotropy = GraphicsConfig.MaxAnisotropy; + + for (int i = 0; i < Items.Length; i++) + { + if (Items[i] != null) + { + Items[i].Dispose(); + + Items[i] = null; + } + } + + UpdateModifiedSequence(); + } + + SynchronizeMemory(); + } + + return ModifiedSequenceNumber; + } + /// /// Implementation of the sampler pool range invalidation. /// diff --git a/Ryujinx.Graphics.Gpu/Image/Texture.cs b/Ryujinx.Graphics.Gpu/Image/Texture.cs index aadb4260bc..cb10f456b6 100644 --- a/Ryujinx.Graphics.Gpu/Image/Texture.cs +++ b/Ryujinx.Graphics.Gpu/Image/Texture.cs @@ -100,6 +100,11 @@ namespace Ryujinx.Graphics.Gpu.Image /// public bool AlwaysFlushOnOverlap { get; private set; } + /// + /// Increments when the host texture is swapped, or when the texture is removed from all pools. + /// + public int InvalidatedSequence { get; private set; } + private int _depth; private int _layers; public int FirstLayer { get; private set; } @@ -1407,6 +1412,7 @@ namespace Ryujinx.Graphics.Gpu.Image DisposeTextures(); HostTexture = hostTexture; + InvalidatedSequence++; } /// @@ -1535,6 +1541,8 @@ namespace Ryujinx.Graphics.Gpu.Image _poolOwners.Clear(); } + + InvalidatedSequence++; } /// diff --git a/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs b/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs index 7ac4e12e2a..f15f888505 100644 --- a/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs +++ b/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs @@ -1,8 +1,12 @@ using Ryujinx.Common.Logging; using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Engine.Types; +using Ryujinx.Graphics.Gpu.Memory; +using Ryujinx.Graphics.Gpu.Shader; using Ryujinx.Graphics.Shader; using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Image { @@ -35,17 +39,24 @@ namespace Ryujinx.Graphics.Gpu.Image { public ITexture Texture; public ISampler Sampler; + + public int TextureHandle; + public int SamplerHandle; + public int InvalidatedSequence; + public Texture CachedTexture; + public Sampler CachedSampler; } - private readonly TextureStatePerStage[][] _textureState; - private readonly TextureStatePerStage[][] _imageState; + private TextureStatePerStage[] _textureState; + private TextureStatePerStage[] _imageState; private int[] _textureBindingsCount; private int[] _imageBindingsCount; - private int _textureBufferIndex; + private int _texturePoolSequence; + private int _samplerPoolSequence; - private bool _rebind; + private int _textureBufferIndex; private readonly float[] _scales; private bool _scaleChanged; @@ -72,8 +83,8 @@ namespace Ryujinx.Graphics.Gpu.Image _textureBindings = new TextureBindingInfo[stages][]; _imageBindings = new TextureBindingInfo[stages][]; - _textureState = new TextureStatePerStage[stages][]; - _imageState = new TextureStatePerStage[stages][]; + _textureState = new TextureStatePerStage[InitialTextureStateSize]; + _imageState = new TextureStatePerStage[InitialImageStateSize]; _textureBindingsCount = new int[stages]; _imageBindingsCount = new int[stages]; @@ -82,9 +93,6 @@ namespace Ryujinx.Graphics.Gpu.Image { _textureBindings[stage] = new TextureBindingInfo[InitialTextureStateSize]; _imageBindings[stage] = new TextureBindingInfo[InitialImageStateSize]; - - _textureState[stage] = new TextureStatePerStage[InitialTextureStateSize]; - _imageState[stage] = new TextureStatePerStage[InitialImageStateSize]; } } @@ -99,15 +107,6 @@ namespace Ryujinx.Graphics.Gpu.Image if (count > _textureBindings[stage].Length) { Array.Resize(ref _textureBindings[stage], count); - Array.Resize(ref _textureState[stage], count); - } - - int toClear = Math.Max(_textureBindingsCount[stage], count); - TextureStatePerStage[] state = _textureState[stage]; - - for (int i = 0; i < toClear; i++) - { - state[i] = new TextureStatePerStage(); } _textureBindingsCount[stage] = count; @@ -126,15 +125,6 @@ namespace Ryujinx.Graphics.Gpu.Image if (count > _imageBindings[stage].Length) { Array.Resize(ref _imageBindings[stage], count); - Array.Resize(ref _imageState[stage], count); - } - - int toClear = Math.Max(_imageBindingsCount[stage], count); - TextureStatePerStage[] state = _imageState[stage]; - - for (int i = 0; i < toClear; i++) - { - state[i] = new TextureStatePerStage(); } _imageBindingsCount[stage] = count; @@ -142,6 +132,24 @@ namespace Ryujinx.Graphics.Gpu.Image return _imageBindings[stage]; } + /// + /// Sets the max binding indexes for textures and images. + /// + /// The maximum texture binding + /// The maximum image binding + public void SetMaxBindings(int maxTextureBinding, int maxImageBinding) + { + if (maxTextureBinding >= _textureState.Length) + { + Array.Resize(ref _textureState, maxTextureBinding + 1); + } + + if (maxImageBinding >= _imageState.Length) + { + Array.Resize(ref _imageState, maxImageBinding + 1); + } + } + /// /// Sets the textures constant buffer index. /// The constant buffer specified holds the texture handles. @@ -323,7 +331,9 @@ namespace Ryujinx.Graphics.Gpu.Image /// Ensures that the bindings are visible to the host GPU. /// Note: this actually performs the binding using the host graphics API. /// - public void CommitBindings() + /// Specialization state for the bound shader + /// True if all bound textures match the current shader specialiation state, false otherwise + public bool CommitBindings(ShaderSpecializationState specState) { ulong texturePoolAddress = _texturePoolAddress; @@ -331,10 +341,38 @@ namespace Ryujinx.Graphics.Gpu.Image ? _texturePoolCache.FindOrCreate(_channel, texturePoolAddress, _texturePoolMaximumId) : null; + // Check if the texture pool has been modified since bindings were last committed. + // If it wasn't, then it's possible to avoid looking up textures again when the handle remains the same. + bool poolModified = false; + + if (texturePool != null) + { + int texturePoolSequence = texturePool.CheckModified(); + + if (_texturePoolSequence != texturePoolSequence) + { + poolModified = true; + _texturePoolSequence = texturePoolSequence; + } + } + + if (_samplerPool != null) + { + int samplerPoolSequence = _samplerPool.CheckModified(); + + if (_samplerPoolSequence != samplerPoolSequence) + { + poolModified = true; + _samplerPoolSequence = samplerPoolSequence; + } + } + + bool specStateMatches = true; + if (_isCompute) { - CommitTextureBindings(texturePool, ShaderStage.Compute, 0); - CommitImageBindings (texturePool, ShaderStage.Compute, 0); + specStateMatches &= CommitTextureBindings(texturePool, ShaderStage.Compute, 0, poolModified, specState); + specStateMatches &= CommitImageBindings(texturePool, ShaderStage.Compute, 0, poolModified, specState); } else { @@ -342,14 +380,57 @@ namespace Ryujinx.Graphics.Gpu.Image { int stageIndex = (int)stage - 1; - CommitTextureBindings(texturePool, stage, stageIndex); - CommitImageBindings (texturePool, stage, stageIndex); + specStateMatches &= CommitTextureBindings(texturePool, stage, stageIndex, poolModified, specState); + specStateMatches &= CommitImageBindings(texturePool, stage, stageIndex, poolModified, specState); } } CommitRenderScale(); - _rebind = false; + return specStateMatches; + } + + /// + /// Fetch the constant buffers used for a texture to cache. + /// + /// Stage index of the constant buffer + /// The currently cached texture buffer index + /// The currently cached sampler buffer index + /// The currently cached texture buffer data + /// The currently cached sampler buffer data + /// The new texture buffer index + /// The new sampler buffer index + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void UpdateCachedBuffer( + int stageIndex, + ref int cachedTextureBufferIndex, + ref int cachedSamplerBufferIndex, + ref ReadOnlySpan cachedTextureBuffer, + ref ReadOnlySpan cachedSamplerBuffer, + int textureBufferIndex, + int samplerBufferIndex) + { + if (textureBufferIndex != cachedTextureBufferIndex) + { + ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex); + + cachedTextureBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); + cachedTextureBufferIndex = textureBufferIndex; + + if (samplerBufferIndex == textureBufferIndex) + { + cachedSamplerBuffer = cachedTextureBuffer; + cachedSamplerBufferIndex = samplerBufferIndex; + } + } + + if (samplerBufferIndex != cachedSamplerBufferIndex) + { + ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex); + + cachedSamplerBuffer = MemoryMarshal.Cast(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); + cachedSamplerBufferIndex = samplerBufferIndex; + } } /// @@ -358,13 +439,16 @@ namespace Ryujinx.Graphics.Gpu.Image /// /// The current texture pool /// The shader stage using the textures to be bound - /// The stage number of the specified shader stage - private void CommitTextureBindings(TexturePool pool, ShaderStage stage, int stageIndex) + /// The stage number of the specified shader stageTrue if either the texture or sampler pool was modified, false otherwise + /// Specialization state for the bound shader + /// True if all bound textures match the current shader specialiation state, false otherwise + private bool CommitTextureBindings(TexturePool pool, ShaderStage stage, int stageIndex, bool poolModified, ShaderSpecializationState specState) { int textureCount = _textureBindingsCount[stageIndex]; if (textureCount == 0) { - return; + return true; } var samplerPool = _samplerPool; @@ -372,17 +456,26 @@ namespace Ryujinx.Graphics.Gpu.Image if (pool == null) { Logger.Error?.Print(LogClass.Gpu, $"Shader stage \"{stage}\" uses textures, but texture pool was not set."); - return; + return true; } + bool specStateMatches = true; + + int cachedTextureBufferIndex = -1; + int cachedSamplerBufferIndex = -1; + ReadOnlySpan cachedTextureBuffer = Span.Empty; + ReadOnlySpan cachedSamplerBuffer = Span.Empty; + for (int index = 0; index < textureCount; index++) { TextureBindingInfo bindingInfo = _textureBindings[stageIndex][index]; (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, _textureBufferIndex); - int packedId = ReadPackedId(stageIndex, bindingInfo.Handle, textureBufferIndex, samplerBufferIndex); - int textureId = UnpackTextureId(packedId); + UpdateCachedBuffer(stageIndex, ref cachedTextureBufferIndex, ref cachedSamplerBufferIndex, ref cachedTextureBuffer, ref cachedSamplerBuffer, textureBufferIndex, samplerBufferIndex); + + int packedId = TextureHandle.ReadPackedId(bindingInfo.Handle, cachedTextureBuffer, cachedSamplerBuffer); + int textureId = TextureHandle.UnpackTextureId(packedId); int samplerId; if (_samplerIndex == SamplerIndex.ViaHeaderIndex) @@ -391,10 +484,30 @@ namespace Ryujinx.Graphics.Gpu.Image } else { - samplerId = UnpackSamplerId(packedId); + samplerId = TextureHandle.UnpackSamplerId(packedId); } - Texture texture = pool.Get(textureId); + ref TextureStatePerStage state = ref _textureState[bindingInfo.Binding]; + + if (!poolModified && + state.TextureHandle == textureId && + state.SamplerHandle == samplerId && + state.CachedTexture != null && + state.CachedTexture.InvalidatedSequence == state.InvalidatedSequence && + state.CachedSampler?.IsDisposed != true) + { + // The texture is already bound. + state.CachedTexture.SynchronizeMemory(); + + continue; + } + + state.TextureHandle = textureId; + state.SamplerHandle = samplerId; + + ref readonly TextureDescriptor descriptor = ref pool.GetForBinding(textureId, out Texture texture); + + specStateMatches &= specState.MatchesTexture(stage, index, descriptor); ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); @@ -407,30 +520,36 @@ namespace Ryujinx.Graphics.Gpu.Image } else { - if (_textureState[stageIndex][index].Texture != hostTexture || _rebind) + if (state.Texture != hostTexture) { if (UpdateScale(texture, bindingInfo, index, stage)) { hostTexture = texture?.GetTargetTexture(bindingInfo.Target); } - _textureState[stageIndex][index].Texture = hostTexture; + state.Texture = hostTexture; _context.Renderer.Pipeline.SetTexture(bindingInfo.Binding, hostTexture); } Sampler sampler = samplerPool?.Get(samplerId); + state.CachedSampler = sampler; ISampler hostSampler = sampler?.GetHostSampler(texture); - if (_textureState[stageIndex][index].Sampler != hostSampler || _rebind) + if (state.Sampler != hostSampler) { - _textureState[stageIndex][index].Sampler = hostSampler; + state.Sampler = hostSampler; _context.Renderer.Pipeline.SetSampler(bindingInfo.Binding, hostSampler); } + + state.CachedTexture = texture; + state.InvalidatedSequence = texture?.InvalidatedSequence ?? 0; } } + + return specStateMatches; } /// @@ -440,38 +559,72 @@ namespace Ryujinx.Graphics.Gpu.Image /// The current texture pool /// The shader stage using the textures to be bound /// The stage number of the specified shader stage - private void CommitImageBindings(TexturePool pool, ShaderStage stage, int stageIndex) + /// True if either the texture or sampler pool was modified, false otherwise + /// Specialization state for the bound shader + /// True if all bound images match the current shader specialiation state, false otherwise + private bool CommitImageBindings(TexturePool pool, ShaderStage stage, int stageIndex, bool poolModified, ShaderSpecializationState specState) { int imageCount = _imageBindingsCount[stageIndex]; if (imageCount == 0) { - return; + return true; } if (pool == null) { Logger.Error?.Print(LogClass.Gpu, $"Shader stage \"{stage}\" uses images, but texture pool was not set."); - return; + return true; } // Scales for images appear after the texture ones. int baseScaleIndex = _textureBindingsCount[stageIndex]; + int cachedTextureBufferIndex = -1; + int cachedSamplerBufferIndex = -1; + ReadOnlySpan cachedTextureBuffer = Span.Empty; + ReadOnlySpan cachedSamplerBuffer = Span.Empty; + + bool specStateMatches = true; + for (int index = 0; index < imageCount; index++) { TextureBindingInfo bindingInfo = _imageBindings[stageIndex][index]; (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, _textureBufferIndex); - int packedId = ReadPackedId(stageIndex, bindingInfo.Handle, textureBufferIndex, samplerBufferIndex); - int textureId = UnpackTextureId(packedId); + UpdateCachedBuffer(stageIndex, ref cachedTextureBufferIndex, ref cachedSamplerBufferIndex, ref cachedTextureBuffer, ref cachedSamplerBuffer, textureBufferIndex, samplerBufferIndex); - Texture texture = pool.Get(textureId); + int packedId = TextureHandle.ReadPackedId(bindingInfo.Handle, cachedTextureBuffer, cachedSamplerBuffer); + int textureId = TextureHandle.UnpackTextureId(packedId); - ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); + ref TextureStatePerStage state = ref _imageState[bindingInfo.Binding]; bool isStore = bindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore); + if (!poolModified && + state.TextureHandle == textureId && + state.CachedTexture != null && + state.CachedTexture.InvalidatedSequence == state.InvalidatedSequence) + { + // The texture is already bound. + state.CachedTexture.SynchronizeMemory(); + + if (isStore) + { + state.CachedTexture?.SignalModified(); + } + + continue; + } + + state.TextureHandle = textureId; + + ref readonly TextureDescriptor descriptor = ref pool.GetForBinding(textureId, out Texture texture); + + specStateMatches &= specState.MatchesImage(stage, index, descriptor); + + ITexture hostTexture = texture?.GetTargetTexture(bindingInfo.Target); + if (hostTexture != null && texture.Target == Target.TextureBuffer) { // Ensure that the buffer texture is using the correct buffer as storage. @@ -494,14 +647,14 @@ namespace Ryujinx.Graphics.Gpu.Image texture?.SignalModified(); } - if (_imageState[stageIndex][index].Texture != hostTexture || _rebind) + if (state.Texture != hostTexture) { if (UpdateScale(texture, bindingInfo, baseScaleIndex + index, stage)) { hostTexture = texture?.GetTargetTexture(bindingInfo.Target); } - _imageState[stageIndex][index].Texture = hostTexture; + state.Texture = hostTexture; Format format = bindingInfo.Format; @@ -512,8 +665,13 @@ namespace Ryujinx.Graphics.Gpu.Image _context.Renderer.Pipeline.SetImage(bindingInfo.Binding, hostTexture, format); } + + state.CachedTexture = texture; + state.InvalidatedSequence = texture?.InvalidatedSequence ?? 0; } } + + return specStateMatches; } /// @@ -537,7 +695,7 @@ namespace Ryujinx.Graphics.Gpu.Image (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(cbufSlot, bufferIndex); int packedId = ReadPackedId(stageIndex, handle, textureBufferIndex, samplerBufferIndex); - int textureId = UnpackTextureId(packedId); + int textureId = TextureHandle.UnpackTextureId(packedId); ulong poolAddress = _channel.MemoryManager.Translate(poolGpuVa); @@ -555,6 +713,7 @@ namespace Ryujinx.Graphics.Gpu.Image /// Index of the constant buffer holding the texture handles /// Index of the constant buffer holding the sampler handles /// The packed texture and sampler ID (the real texture handle) + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int ReadPackedId(int stageIndex, int wordOffset, int textureBufferIndex, int samplerBufferIndex) { (int textureWordOffset, int samplerWordOffset, TextureHandleType handleType) = TextureHandle.UnpackOffsets(wordOffset); @@ -590,32 +749,13 @@ namespace Ryujinx.Graphics.Gpu.Image return handle; } - /// - /// Unpacks the texture ID from the real texture handle. - /// - /// The real texture handle - /// The texture ID - private static int UnpackTextureId(int packedId) - { - return (packedId >> 0) & 0xfffff; - } - - /// - /// Unpacks the sampler ID from the real texture handle. - /// - /// The real texture handle - /// The sampler ID - private static int UnpackSamplerId(int packedId) - { - return (packedId >> 20) & 0xfff; - } - /// /// Force all bound textures and images to be rebound the next time CommitBindings is called. /// public void Rebind() { - _rebind = true; + Array.Clear(_textureState); + Array.Clear(_imageState); } /// diff --git a/Ryujinx.Graphics.Gpu/Image/TextureManager.cs b/Ryujinx.Graphics.Gpu/Image/TextureManager.cs index a1c292912d..628c31596a 100644 --- a/Ryujinx.Graphics.Gpu/Image/TextureManager.cs +++ b/Ryujinx.Graphics.Gpu/Image/TextureManager.cs @@ -1,5 +1,6 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Engine.Types; +using Ryujinx.Graphics.Gpu.Shader; using System; namespace Ryujinx.Graphics.Gpu.Image @@ -10,9 +11,11 @@ namespace Ryujinx.Graphics.Gpu.Image class TextureManager : IDisposable { private readonly GpuContext _context; + private readonly GpuChannel _channel; private readonly TextureBindingsManager _cpBindingsManager; private readonly TextureBindingsManager _gpBindingsManager; + private readonly TexturePoolCache _texturePoolCache; private readonly Texture[] _rtColors; private readonly ITexture[] _rtHostColors; @@ -35,6 +38,7 @@ namespace Ryujinx.Graphics.Gpu.Image public TextureManager(GpuContext context, GpuChannel channel) { _context = context; + _channel = channel; TexturePoolCache texturePoolCache = new TexturePoolCache(context); @@ -43,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Image _cpBindingsManager = new TextureBindingsManager(context, channel, texturePoolCache, scales, isCompute: true); _gpBindingsManager = new TextureBindingsManager(context, channel, texturePoolCache, scales, isCompute: false); + _texturePoolCache = texturePoolCache; _rtColors = new Texture[Constants.TotalRenderTargets]; _rtHostColors = new ITexture[Constants.TotalRenderTargets]; @@ -99,6 +104,16 @@ namespace Ryujinx.Graphics.Gpu.Image _cpBindingsManager.SetTextureBufferIndex(index); } + /// + /// Sets the max binding indexes on the compute pipeline. + /// + /// The maximum texture binding + /// The maximum image binding + public void SetComputeMaxBindings(int maxTextureBinding, int maxImageBinding) + { + _cpBindingsManager.SetMaxBindings(maxTextureBinding, maxImageBinding); + } + /// /// Sets the texture constant buffer index on the graphics pipeline. /// @@ -108,6 +123,16 @@ namespace Ryujinx.Graphics.Gpu.Image _gpBindingsManager.SetTextureBufferIndex(index); } + /// + /// Sets the max binding indexes on the graphics pipeline. + /// + /// The maximum texture binding + /// The maximum image binding + public void SetGraphicsMaxBindings(int maxTextureBinding, int maxImageBinding) + { + _gpBindingsManager.SetMaxBindings(maxTextureBinding, maxImageBinding); + } + /// /// Sets the current sampler pool on the compute pipeline. /// @@ -335,25 +360,48 @@ namespace Ryujinx.Graphics.Gpu.Image /// /// Commits bindings on the compute pipeline. /// - public void CommitComputeBindings() + /// Specialization state for the bound shader + /// True if all bound textures match the current shader specialization state, false otherwise + public bool CommitComputeBindings(ShaderSpecializationState specState) { // Every time we switch between graphics and compute work, // we must rebind everything. // Since compute work happens less often, we always do that // before and after the compute dispatch. _cpBindingsManager.Rebind(); - _cpBindingsManager.CommitBindings(); + bool result = _cpBindingsManager.CommitBindings(specState); _gpBindingsManager.Rebind(); + + return result; } /// /// Commits bindings on the graphics pipeline. /// - public void CommitGraphicsBindings() + /// Specialization state for the bound shader + /// True if all bound textures match the current shader specialization state, false otherwise + public bool CommitGraphicsBindings(ShaderSpecializationState specState) { - _gpBindingsManager.CommitBindings(); + bool result = _gpBindingsManager.CommitBindings(specState); UpdateRenderTargets(); + + return result; + } + + /// + /// Returns a texture pool from the cache, with the given address and maximum id. + /// + /// GPU virtual address of the texture pool + /// Maximum ID of the texture pool + /// The texture pool + public TexturePool GetTexturePool(ulong poolGpuVa, int maximumId) + { + ulong poolAddress = _channel.MemoryManager.Translate(poolGpuVa); + + TexturePool texturePool = _texturePoolCache.FindOrCreate(_channel, poolAddress, maximumId); + + return texturePool; } /// diff --git a/Ryujinx.Graphics.Gpu/Image/TexturePool.cs b/Ryujinx.Graphics.Gpu/Image/TexturePool.cs index 10a6ff82af..75974c43b0 100644 --- a/Ryujinx.Graphics.Gpu/Image/TexturePool.cs +++ b/Ryujinx.Graphics.Gpu/Image/TexturePool.cs @@ -14,6 +14,7 @@ namespace Ryujinx.Graphics.Gpu.Image { private readonly GpuChannel _channel; private readonly ConcurrentQueue _dereferenceQueue = new ConcurrentQueue(); + private TextureDescriptor _defaultDescriptor; /// /// Intrusive linked list node used on the texture pool cache. @@ -32,6 +33,62 @@ namespace Ryujinx.Graphics.Gpu.Image _channel = channel; } + /// + /// Gets the texture descripor and texture with the given ID with no bounds check or synchronization. + /// + /// ID of the texture. This is effectively a zero-based index + /// The texture with the given ID + /// The texture descriptor with the given ID + private ref readonly TextureDescriptor GetInternal(int id, out Texture texture) + { + texture = Items[id]; + + ref readonly TextureDescriptor descriptor = ref GetDescriptorRef(id); + + if (texture == null) + { + TextureInfo info = GetInfo(descriptor, out int layerSize); + + ProcessDereferenceQueue(); + + texture = PhysicalMemory.TextureCache.FindOrCreateTexture(_channel.MemoryManager, TextureSearchFlags.ForSampler, info, layerSize); + + // If this happens, then the texture address is invalid, we can't add it to the cache. + if (texture == null) + { + return ref descriptor; + } + + texture.IncrementReferenceCount(this, id); + + Items[id] = texture; + + DescriptorCache[id] = descriptor; + } + else + { + if (texture.ChangedSize) + { + // Texture changed size at one point - it may be a different size than the sampler expects. + // This can be triggered when the size is changed by a size hint on copy or draw, but the texture has been sampled before. + + int baseLevel = descriptor.UnpackBaseLevel(); + int width = Math.Max(1, descriptor.UnpackWidth() >> baseLevel); + int height = Math.Max(1, descriptor.UnpackHeight() >> baseLevel); + + if (texture.Info.Width != width || texture.Info.Height != height) + { + texture.ChangeSize(width, height, texture.Info.DepthOrLayers); + } + } + + // Memory is automatically synchronized on texture creation. + texture.SynchronizeMemory(); + } + + return ref descriptor; + } + /// /// Gets the texture with the given ID. /// @@ -51,56 +108,49 @@ namespace Ryujinx.Graphics.Gpu.Image SynchronizeMemory(); } - Texture texture = Items[id]; - - if (texture == null) - { - TextureDescriptor descriptor = GetDescriptor(id); - - TextureInfo info = GetInfo(descriptor, out int layerSize); - - ProcessDereferenceQueue(); - - texture = PhysicalMemory.TextureCache.FindOrCreateTexture(_channel.MemoryManager, TextureSearchFlags.ForSampler, info, layerSize); - - // If this happens, then the texture address is invalid, we can't add it to the cache. - if (texture == null) - { - return null; - } - - texture.IncrementReferenceCount(this, id); - - Items[id] = texture; - - DescriptorCache[id] = descriptor; - } - else - { - if (texture.ChangedSize) - { - // Texture changed size at one point - it may be a different size than the sampler expects. - // This can be triggered when the size is changed by a size hint on copy or draw, but the texture has been sampled before. - - TextureDescriptor descriptor = GetDescriptor(id); - - int baseLevel = descriptor.UnpackBaseLevel(); - int width = Math.Max(1, descriptor.UnpackWidth() >> baseLevel); - int height = Math.Max(1, descriptor.UnpackHeight() >> baseLevel); - - if (texture.Info.Width != width || texture.Info.Height != height) - { - texture.ChangeSize(width, height, texture.Info.DepthOrLayers); - } - } - - // Memory is automatically synchronized on texture creation. - texture.SynchronizeMemory(); - } + GetInternal(id, out Texture texture); return texture; } + /// + /// Gets the texture descriptor and texture with the given ID. + /// + /// + /// This method assumes that the pool has been manually synchronized before doing binding. + /// + /// ID of the texture. This is effectively a zero-based index + /// The texture with the given ID + /// The texture descriptor with the given ID + public ref readonly TextureDescriptor GetForBinding(int id, out Texture texture) + { + if ((uint)id >= Items.Length) + { + texture = null; + return ref _defaultDescriptor; + } + + // When getting for binding, assume the pool has already been synchronized. + + return ref GetInternal(id, out texture); + } + + /// + /// Checks if the pool was modified, and returns the last sequence number where a modification was detected. + /// + /// A number that increments each time a modification is detected + public int CheckModified() + { + if (SequenceNumber != Context.SequenceNumber) + { + SequenceNumber = Context.SequenceNumber; + + SynchronizeMemory(); + } + + return ModifiedSequenceNumber; + } + /// /// Forcibly remove a texture from this pool's items. /// If deferred, the dereference will be queued to occur on the render thread. @@ -175,7 +225,7 @@ namespace Ryujinx.Graphics.Gpu.Image /// The texture descriptor /// Layer size for textures using a sub-range of mipmap levels, otherwise 0 /// The texture information - private TextureInfo GetInfo(TextureDescriptor descriptor, out int layerSize) + private TextureInfo GetInfo(in TextureDescriptor descriptor, out int layerSize) { int depthOrLayers = descriptor.UnpackDepth(); int levels = descriptor.UnpackLevels(); diff --git a/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index 71f202aedc..9f5f39a92b 100644 --- a/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -378,6 +378,25 @@ namespace Ryujinx.Graphics.Gpu.Memory return _gpUniformBuffers[stage].Buffers[index].Address; } + /// + /// Gets the bounds of the uniform buffer currently bound at the given index. + /// + /// Indicates whenever the uniform is requested by the 3D or compute engine + /// Index of the shader stage, if the uniform is for the 3D engine + /// Index of the uniform buffer binding + /// The uniform buffer bounds, or an undefined value if the buffer is not currently bound + public ref BufferBounds GetUniformBufferBounds(bool isCompute, int stage, int index) + { + if (isCompute) + { + return ref _cpUniformBuffers.Buffers[index]; + } + else + { + return ref _gpUniformBuffers[stage].Buffers[index]; + } + } + /// /// Ensures that the compute engine bindings are visible to the host GPU. /// Note: this actually performs the binding using the host graphics API. diff --git a/Ryujinx.Graphics.Gpu/Shader/CachedShaderProgram.cs b/Ryujinx.Graphics.Gpu/Shader/CachedShaderProgram.cs index 3b4c65f3dc..69fcb27804 100644 --- a/Ryujinx.Graphics.Gpu/Shader/CachedShaderProgram.cs +++ b/Ryujinx.Graphics.Gpu/Shader/CachedShaderProgram.cs @@ -35,6 +35,8 @@ namespace Ryujinx.Graphics.Gpu.Shader HostProgram = hostProgram; SpecializationState = specializationState; Shaders = shaders; + + SpecializationState.Prepare(shaders); } /// diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs index df4b9d128a..0779bf2ce6 100644 --- a/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs +++ b/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs @@ -418,7 +418,7 @@ namespace Ryujinx.Graphics.Gpu.Shader { if (IsShaderEqual(channel.MemoryManager, cpShader.Shaders[0], gpuVa)) { - return cpShader.SpecializationState.MatchesCompute(channel, poolState); + return cpShader.SpecializationState.MatchesCompute(channel, poolState, true); } return false; @@ -454,7 +454,7 @@ namespace Ryujinx.Graphics.Gpu.Shader } } - return gpShaders.SpecializationState.MatchesGraphics(channel, poolState, graphicsState); + return gpShaders.SpecializationState.MatchesGraphics(channel, poolState, graphicsState, true); } /// diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationList.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationList.cs index e3e57d7452..43ccd892c3 100644 --- a/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationList.cs +++ b/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationList.cs @@ -35,7 +35,7 @@ namespace Ryujinx.Graphics.Gpu.Shader { foreach (var entry in _entries) { - if (entry.SpecializationState.MatchesGraphics(channel, poolState, graphicsState)) + if (entry.SpecializationState.MatchesGraphics(channel, poolState, graphicsState, true)) { program = entry; return true; @@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.Gpu.Shader { foreach (var entry in _entries) { - if (entry.SpecializationState.MatchesCompute(channel, poolState)) + if (entry.SpecializationState.MatchesCompute(channel, poolState, true)) { program = entry; return true; diff --git a/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs b/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs index 418c7b1a75..44ffd687d5 100644 --- a/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs +++ b/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs @@ -1,9 +1,14 @@ using Ryujinx.Common.Memory; +using Ryujinx.Graphics.Gpu.Image; +using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Gpu.Shader.DiskCache; using Ryujinx.Graphics.Shader; using System; using System.Collections.Generic; +using System.Linq; using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Shader { @@ -158,6 +163,9 @@ namespace Ryujinx.Graphics.Gpu.Shader } private readonly Dictionary> _textureSpecialization; + private KeyValuePair>[] _allTextures; + private Box[][] _textureByBinding; + private Box[][] _imageByBinding; /// /// Creates a new instance of the shader specialization state. @@ -194,6 +202,48 @@ namespace Ryujinx.Graphics.Gpu.Shader } } + /// + /// Prepare the shader specialization state for quick binding lookups. + /// + /// The shader stages + public void Prepare(CachedShaderStage[] stages) + { + _allTextures = _textureSpecialization.ToArray(); + + _textureByBinding = new Box[stages.Length][]; + _imageByBinding = new Box[stages.Length][]; + + for (int i = 0; i < stages.Length; i++) + { + CachedShaderStage stage = stages[i]; + if (stage?.Info != null) + { + var textures = stage.Info.Textures; + var images = stage.Info.Images; + + var texBindings = new Box[textures.Count]; + var imageBindings = new Box[images.Count]; + + int stageIndex = Math.Max(i - 1, 0); // Don't count VertexA for looking up spec state. No-Op for compute. + + for (int j = 0; j < textures.Count; j++) + { + var texture = textures[j]; + texBindings[j] = GetTextureSpecState(stageIndex, texture.HandleIndex, texture.CbufSlot); + } + + for (int j = 0; j < images.Count; j++) + { + var image = images[j]; + imageBindings[j] = GetTextureSpecState(stageIndex, image.HandleIndex, image.CbufSlot); + } + + _textureByBinding[i] = texBindings; + _imageByBinding[i] = imageBindings; + } + } + } + /// /// Indicates that the shader accesses the early Z force state. /// @@ -396,15 +446,16 @@ namespace Ryujinx.Graphics.Gpu.Shader /// GPU channel /// Texture pool state /// Graphics state + /// Indicates whether texture descriptors should be checked /// True if the state matches, false otherwise - public bool MatchesGraphics(GpuChannel channel, GpuChannelPoolState poolState, GpuChannelGraphicsState graphicsState) + public bool MatchesGraphics(GpuChannel channel, GpuChannelPoolState poolState, GpuChannelGraphicsState graphicsState, bool checkTextures) { if (graphicsState.ViewportTransformDisable != GraphicsState.ViewportTransformDisable) { return false; } - return Matches(channel, poolState, isCompute: false); + return Matches(channel, poolState, checkTextures, isCompute: false); } /// @@ -412,10 +463,64 @@ namespace Ryujinx.Graphics.Gpu.Shader /// /// GPU channel /// Texture pool state + /// Indicates whether texture descriptors should be checked /// True if the state matches, false otherwise - public bool MatchesCompute(GpuChannel channel, GpuChannelPoolState poolState) + public bool MatchesCompute(GpuChannel channel, GpuChannelPoolState poolState, bool checkTextures) { - return Matches(channel, poolState, isCompute: true); + return Matches(channel, poolState, checkTextures, isCompute: true); + } + + /// + /// Fetch the constant buffers used for a texture to cache. + /// + /// GPU channel + /// Indicates whenever the check is requested by the 3D or compute engine + /// The currently cached texture buffer index + /// The currently cached sampler buffer index + /// The currently cached texture buffer data + /// The currently cached sampler buffer data + /// The currently cached stage + /// The new texture buffer index + /// The new sampler buffer index + /// Stage index of the constant buffer + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void UpdateCachedBuffer( + GpuChannel channel, + bool isCompute, + ref int cachedTextureBufferIndex, + ref int cachedSamplerBufferIndex, + ref ReadOnlySpan cachedTextureBuffer, + ref ReadOnlySpan cachedSamplerBuffer, + ref int cachedStageIndex, + int textureBufferIndex, + int samplerBufferIndex, + int stageIndex) + { + bool stageChange = stageIndex != cachedStageIndex; + + if (stageChange || textureBufferIndex != cachedTextureBufferIndex) + { + ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex); + + cachedTextureBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); + cachedTextureBufferIndex = textureBufferIndex; + + if (samplerBufferIndex == textureBufferIndex) + { + cachedSamplerBuffer = cachedTextureBuffer; + cachedSamplerBufferIndex = samplerBufferIndex; + } + } + + if (stageChange || samplerBufferIndex != cachedSamplerBufferIndex) + { + ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex); + + cachedSamplerBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); + cachedSamplerBufferIndex = samplerBufferIndex; + } + + cachedStageIndex = stageIndex; } /// @@ -423,9 +528,10 @@ namespace Ryujinx.Graphics.Gpu.Shader /// /// GPU channel /// Texture pool state + /// Indicates whether texture descriptors should be checked /// Indicates whenever the check is requested by the 3D or compute engine /// True if the state matches, false otherwise - private bool Matches(GpuChannel channel, GpuChannelPoolState poolState, bool isCompute) + private bool Matches(GpuChannel channel, GpuChannelPoolState poolState, bool checkTextures, bool isCompute) { int constantBufferUsePerStageMask = _constantBufferUsePerStage; @@ -445,55 +551,60 @@ namespace Ryujinx.Graphics.Gpu.Shader constantBufferUsePerStageMask &= ~(1 << index); } - foreach (var kv in _textureSpecialization) + if (checkTextures) { - TextureKey textureKey = kv.Key; + TexturePool pool = channel.TextureManager.GetTexturePool(poolState.TexturePoolGpuVa, poolState.TexturePoolMaximumId); - (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(textureKey.CbufSlot, poolState.TextureBufferIndex); + int cachedTextureBufferIndex = -1; + int cachedSamplerBufferIndex = -1; + int cachedStageIndex = -1; + ReadOnlySpan cachedTextureBuffer = Span.Empty; + ReadOnlySpan cachedSamplerBuffer = Span.Empty; - ulong textureCbAddress; - ulong samplerCbAddress; - - if (isCompute) + foreach (var kv in _allTextures) { - textureCbAddress = channel.BufferManager.GetComputeUniformBufferAddress(textureBufferIndex); - samplerCbAddress = channel.BufferManager.GetComputeUniformBufferAddress(samplerBufferIndex); - } - else - { - textureCbAddress = channel.BufferManager.GetGraphicsUniformBufferAddress(textureKey.StageIndex, textureBufferIndex); - samplerCbAddress = channel.BufferManager.GetGraphicsUniformBufferAddress(textureKey.StageIndex, samplerBufferIndex); - } + TextureKey textureKey = kv.Key; - if (!channel.MemoryManager.Physical.IsMapped(textureCbAddress) || !channel.MemoryManager.Physical.IsMapped(samplerCbAddress)) - { - continue; + (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(textureKey.CbufSlot, poolState.TextureBufferIndex); + + UpdateCachedBuffer(channel, + isCompute, + ref cachedTextureBufferIndex, + ref cachedSamplerBufferIndex, + ref cachedTextureBuffer, + ref cachedSamplerBuffer, + ref cachedStageIndex, + textureBufferIndex, + samplerBufferIndex, + textureKey.StageIndex); + + int packedId = TextureHandle.ReadPackedId(textureKey.Handle, cachedTextureBuffer, cachedSamplerBuffer); + + int textureId = TextureHandle.UnpackTextureId(packedId); + + ref readonly Image.TextureDescriptor descriptor = ref pool.GetDescriptorRef(textureId); + + if (!MatchesTexture(kv.Value, descriptor)) + { + return false; + } } + } - Image.TextureDescriptor descriptor; - - if (isCompute) - { - descriptor = channel.TextureManager.GetComputeTextureDescriptor( - poolState.TexturePoolGpuVa, - poolState.TextureBufferIndex, - poolState.TexturePoolMaximumId, - textureKey.Handle, - textureKey.CbufSlot); - } - else - { - descriptor = channel.TextureManager.GetGraphicsTextureDescriptor( - poolState.TexturePoolGpuVa, - poolState.TextureBufferIndex, - poolState.TexturePoolMaximumId, - textureKey.StageIndex, - textureKey.Handle, - textureKey.CbufSlot); - } - - Box specializationState = kv.Value; + return true; + } + /// + /// Checks if the recorded texture state matches the given texture descriptor. + /// + /// Texture specialization state + /// Texture descriptor + /// True if the state matches, false otherwise + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool MatchesTexture(Box specializationState, in Image.TextureDescriptor descriptor) + { + if (specializationState != null) + { if (specializationState.Value.QueriedFlags.HasFlag(QueriedTextureStateFlags.CoordNormalized) && specializationState.Value.CoordNormalized != descriptor.UnpackTextureCoordNormalized()) { @@ -504,6 +615,34 @@ namespace Ryujinx.Graphics.Gpu.Shader return true; } + /// + /// Checks if the recorded texture state for a given texture binding matches a texture descriptor. + /// + /// The shader stage + /// The texture index + /// Texture descriptor + /// True if the state matches, false otherwise + public bool MatchesTexture(ShaderStage stage, int index, in Image.TextureDescriptor descriptor) + { + Box specializationState = _textureByBinding[(int)stage][index]; + + return MatchesTexture(specializationState, descriptor); + } + + /// + /// Checks if the recorded texture state for a given image binding matches a texture descriptor. + /// + /// The shader stage + /// The texture index + /// Texture descriptor + /// True if the state matches, false otherwise + public bool MatchesImage(ShaderStage stage, int index, in Image.TextureDescriptor descriptor) + { + Box specializationState = _imageByBinding[(int)stage][index]; + + return MatchesTexture(specializationState, descriptor); + } + /// /// Reads shader specialization state that has been serialized. /// diff --git a/Ryujinx.Graphics.Shader/TextureHandle.cs b/Ryujinx.Graphics.Shader/TextureHandle.cs index b3712e6bf2..d468188b87 100644 --- a/Ryujinx.Graphics.Shader/TextureHandle.cs +++ b/Ryujinx.Graphics.Shader/TextureHandle.cs @@ -1,3 +1,4 @@ +using System; using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Shader @@ -50,5 +51,63 @@ namespace Ryujinx.Graphics.Shader { return (handle & 0x3fff, (handle >> 14) & 0x3fff, (TextureHandleType)((uint)handle >> 28)); } + + /// + /// Unpacks the texture ID from the real texture handle. + /// + /// The real texture handle + /// The texture ID + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int UnpackTextureId(int packedId) + { + return (packedId >> 0) & 0xfffff; + } + + /// + /// Unpacks the sampler ID from the real texture handle. + /// + /// The real texture handle + /// The sampler ID + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int UnpackSamplerId(int packedId) + { + return (packedId >> 20) & 0xfff; + } + + /// + /// Reads a packed texture and sampler ID (basically, the real texture handle) + /// from a given texture/sampler constant buffer. + /// + /// A word offset of the handle on the buffer (the "fake" shader handle) + /// The constant buffer to fetch texture IDs from + /// The constant buffer to fetch sampler IDs from + /// The packed texture and sampler ID (the real texture handle) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int ReadPackedId(int wordOffset, ReadOnlySpan cachedTextureBuffer, ReadOnlySpan cachedSamplerBuffer) + { + (int textureWordOffset, int samplerWordOffset, TextureHandleType handleType) = UnpackOffsets(wordOffset); + + int handle = cachedTextureBuffer[textureWordOffset]; + + // The "wordOffset" (which is really the immediate value used on texture instructions on the shader) + // is a 13-bit value. However, in order to also support separate samplers and textures (which uses + // bindless textures on the shader), we extend it with another value on the higher 16 bits with + // another offset for the sampler. + // The shader translator has code to detect separate texture and sampler uses with a bindless texture, + // turn that into a regular texture access and produce those special handles with values on the higher 16 bits. + if (handleType != TextureHandleType.CombinedSampler) + { + int samplerHandle = cachedSamplerBuffer[samplerWordOffset]; + + if (handleType == TextureHandleType.SeparateSamplerId) + { + samplerHandle <<= 20; + } + + handle |= samplerHandle; + } + + return handle; + } } }