Return mapped buffer pointer directly for flush, WriteableRegion for textures (#2494)

* Return mapped buffer pointer directly for flush, WriteableRegion for textures

A few changes here to generally improve performance, even for platforms not using the persistent buffer flush.

- Texture and buffer flush now return a ReadOnlySpan<byte>. It's guaranteed that this span is pinned in memory, but it will be overwritten on the next flush from that thread, so it is expected that the data is used before calling again.
- As a result, persistent mappings no longer copy to a new array - rather the persistent map is returned directly as a Span<>. A similar host array is used for the glGet flushes instead of allocating new arrays each time.
- Texture flushes now do their layout conversion into a WriteableRegion when the texture is not MultiRange, which allows the flush to happen directly into guest memory rather than into a temporary span, then copied over. This avoids another copy when doing layout conversion.

Overall, this saves 1 data copy for buffer flush, 1 copy for linear textures with matching source/target stride, and 2 copies for block textures or linear textures with mismatching strides.

* Fix tests

* Fix array pointer for Mesa/Intel path

* Address some feedback

* Update method for getting array pointer.

Ryujinx.Cpu/MemoryManager.cs

@@ -237,7 +237,7 @@ namespace Ryujinx.Cpu
         }
 
         /// <inheritdoc/>
-        public unsafe WritableRegion GetWritableRegion(ulong va, int size)
+        public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
         {
             if (size == 0)
             {
@@ -246,6 +246,11 @@ namespace Ryujinx.Cpu
 
             if (IsContiguousAndMapped(va, size))
             {
+                if (tracked)
+                {
+                    SignalMemoryTracking(va, (ulong)size, true);
+                }
+
                 return new WritableRegion(null, va, new NativeMemoryManager<byte>((byte*)GetHostAddress(va), size).Memory);
             }
             else
@@ -254,7 +259,7 @@ namespace Ryujinx.Cpu
 
                 GetSpan(va, size).CopyTo(memory.Span);
 
-                return new WritableRegion(this, va, memory);
+                return new WritableRegion(this, va, memory, tracked);
             }
         }
 

Ryujinx.Cpu/MemoryManagerHostMapped.cs

@@ -285,9 +285,16 @@ namespace Ryujinx.Cpu
         }
 
         /// <inheritdoc/>
-        public WritableRegion GetWritableRegion(ulong va, int size)
+        public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
         {
-            AssertMapped(va, (ulong)size);
+            if (tracked)
+            {
+                SignalMemoryTracking(va, (ulong)size, true);
+            }
+            else
+            {
+                AssertMapped(va, (ulong)size);
+            }
 
             return _addressSpaceMirror.GetWritableRegion(va, size);
         }

Ryujinx.Graphics.GAL/IRenderer.cs

@@ -27,7 +27,7 @@ namespace Ryujinx.Graphics.GAL
 
         void DeleteBuffer(BufferHandle buffer);
 
-        byte[] GetBufferData(BufferHandle buffer, int offset, int size);
+        ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
 
         Capabilities GetCapabilities();
 

Ryujinx.Graphics.GAL/ITexture.cs

@@ -14,7 +14,7 @@ namespace Ryujinx.Graphics.GAL
 
         ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel);
 
-        byte[] GetData();
+        ReadOnlySpan<byte> GetData();
 
         void SetData(ReadOnlySpan<byte> data);
         void SetData(ReadOnlySpan<byte> data, int layer, int level);

Ryujinx.Graphics.Gpu/Image/Texture.cs

@@ -4,6 +4,7 @@ using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu.Memory;
 using Ryujinx.Graphics.Texture;
 using Ryujinx.Graphics.Texture.Astc;
+using Ryujinx.Memory;
 using Ryujinx.Memory.Range;
 using System;
 using System.Collections.Generic;
@@ -821,14 +822,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 return; // Flushing this format is not supported, as it may have been converted to another host format.
             }
 
-            if (tracked)
+            FlushTextureDataToGuest(tracked);
-            {
-                _physicalMemory.Write(Range, GetTextureDataFromGpu(tracked));
-            }
-            else
-            {
-                _physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(tracked));
-            }
         }
 
         /// <summary>
@@ -864,10 +858,44 @@ namespace Ryujinx.Graphics.Gpu.Image
                     texture = _flushHostTexture = GetScaledHostTexture(1f, _flushHostTexture);
                 }
 
-                _physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(false, texture));
+                FlushTextureDataToGuest(false, texture);
             });
         }
 
+        /// <summary>
+        /// Gets data from the host GPU, and flushes it to guest memory.
+        /// </summary>
+        /// <remarks>
+        /// This method should be used to retrieve data that was modified by the host GPU.
+        /// This is not cheap, avoid doing that unless strictly needed.
+        /// When possible, the data is written directly into guest memory, rather than copied.
+        /// </remarks>
+        /// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
+        /// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
+        private void FlushTextureDataToGuest(bool tracked, ITexture texture = null)
+        {
+            if (Range.Count == 1)
+            {
+                MemoryRange subrange = Range.GetSubRange(0);
+
+                using (WritableRegion region = _physicalMemory.GetWritableRegion(subrange.Address, (int)subrange.Size, tracked))
+                {
+                    GetTextureDataFromGpu(region.Memory.Span, tracked, texture);
+                }
+            }
+            else
+            {
+                if (tracked)
+                {
+                    _physicalMemory.Write(Range, GetTextureDataFromGpu(Span<byte>.Empty, true, texture));
+                }
+                else 
+                {
+                    _physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(Span<byte>.Empty, false, texture));
+                }
+            }
+        }
+
         /// <summary>
         /// Gets data from the host GPU.
         /// </summary>
@@ -875,8 +903,11 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// This method should be used to retrieve data that was modified by the host GPU.
         /// This is not cheap, avoid doing that unless strictly needed.
         /// </remarks>
-        /// <returns>Host texture data</returns>
+        /// <param name="output">An output span to place the texture data into. If empty, one is generated</param>
-        private ReadOnlySpan<byte> GetTextureDataFromGpu(bool blacklist, ITexture texture = null)
+        /// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
+        /// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
+        /// <returns>The span containing the texture data</returns>
+        private ReadOnlySpan<byte> GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
         {
             ReadOnlySpan<byte> data;
 
@@ -909,6 +940,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 if (Info.IsLinear)
                 {
                     data = LayoutConverter.ConvertLinearToLinearStrided(
+                        output,
                         Info.Width,
                         Info.Height,
                         Info.FormatInfo.BlockWidth,
@@ -920,6 +952,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 else
                 {
                     data = LayoutConverter.ConvertLinearToBlockLinear(
+                        output,
                         Info.Width,
                         Info.Height,
                         _depth,

Ryujinx.Graphics.Gpu/Memory/Buffer.cs

@@ -412,7 +412,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         {
             int offset = (int)(address - Address);
 
-            byte[] data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
+            ReadOnlySpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
 
             // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
             _physicalMemory.WriteUntracked(address, data);

Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs

@@ -128,10 +128,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         /// <param name="address">Start address of the range</param>
         /// <param name="size">Size in bytes to be range</param>
+        /// <param name="tracked">True if write tracking is triggered on the span</param>
         /// <returns>A writable region with the data at the specified memory location</returns>
-        public WritableRegion GetWritableRegion(ulong address, int size)
+        public WritableRegion GetWritableRegion(ulong address, int size, bool tracked = false)
         {
-            return _cpuMemory.GetWritableRegion(address, size);
+            return _cpuMemory.GetWritableRegion(address, size, tracked);
         }
 
         /// <summary>

Ryujinx.Graphics.OpenGL/Buffer.cs

@@ -55,15 +55,22 @@ namespace Ryujinx.Graphics.OpenGL
                 (IntPtr)size);
         }
 
-        public static byte[] GetData(BufferHandle buffer, int offset, int size)
+        public static unsafe ReadOnlySpan<byte> GetData(Renderer renderer, BufferHandle buffer, int offset, int size)
         {
-            GL.BindBuffer(BufferTarget.CopyReadBuffer, buffer.ToInt32());
+            if (HwCapabilities.UsePersistentBufferForFlush)
+            {
+                return renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size);
+            }
+            else
+            {
+                IntPtr target = renderer.PersistentBuffers.Default.GetHostArray(size);
 
-            byte[] data = new byte[size];
+                GL.BindBuffer(BufferTarget.CopyReadBuffer, buffer.ToInt32());
 
-            GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, data);
+                GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target);
 
-            return data;
+                return new ReadOnlySpan<byte>(target.ToPointer(), size);
+            }
         }
 
         public static void Resize(BufferHandle handle, int size)

Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs

@@ -38,9 +38,9 @@ namespace Ryujinx.Graphics.OpenGL.Image
             throw new NotSupportedException();
         }
 
-        public byte[] GetData()
+        public ReadOnlySpan<byte> GetData()
         {
-            return Buffer.GetData(_buffer, _bufferOffset, _bufferSize);
+            return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize);
         }
 
         public void SetData(ReadOnlySpan<byte> data)

Ryujinx.Graphics.OpenGL/Image/TextureView.cs

@@ -119,7 +119,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
             _renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter);
         }
 
-        public byte[] GetData()
+        public unsafe ReadOnlySpan<byte> GetData()
         {
             int size = 0;
 
@@ -134,17 +134,11 @@ namespace Ryujinx.Graphics.OpenGL.Image
             }
             else
             {
-                byte[] data = new byte[size];
+                IntPtr target = _renderer.PersistentBuffers.Default.GetHostArray(size);
 
-                unsafe
+                WriteTo(target);
-                {
-                    fixed (byte* ptr = data)
-                    {
-                        WriteTo((IntPtr)ptr);
-                    }
-                }
 
-                return data;
+                return new ReadOnlySpan<byte>(target.ToPointer(), size);
             }
         }
 

Ryujinx.Graphics.OpenGL/PersistentBuffers.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using OpenTK.Graphics.OpenGL;
 using Ryujinx.Common.Logging;
@@ -27,6 +28,9 @@ namespace Ryujinx.Graphics.OpenGL
         private int _copyBufferHandle;
         private int _copyBufferSize;
 
+        private byte[] _data;
+        private IntPtr _dataMap;
+
         private void EnsureBuffer(int requiredSize)
         {
             if (_copyBufferSize < requiredSize && _copyBufferHandle != 0)
@@ -48,6 +52,18 @@ namespace Ryujinx.Graphics.OpenGL
             }
         }
 
+        public unsafe IntPtr GetHostArray(int requiredSize)
+        {
+            if (_data == null || _data.Length < requiredSize)
+            {
+                _data = GC.AllocateUninitializedArray<byte>(requiredSize, true);
+
+                _dataMap = (IntPtr)Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(_data));
+            }
+
+            return _dataMap;
+        }
+
         private void Sync()
         {
             GL.MemoryBarrier(MemoryBarrierFlags.ClientMappedBufferBarrierBit);
@@ -63,7 +79,7 @@ namespace Ryujinx.Graphics.OpenGL
             GL.DeleteSync(sync);
         }
 
-        public byte[] GetTextureData(TextureView view, int size)
+        public unsafe ReadOnlySpan<byte> GetTextureData(TextureView view, int size)
         {
             EnsureBuffer(size);
 
@@ -73,16 +89,12 @@ namespace Ryujinx.Graphics.OpenGL
 
             GL.BindBuffer(BufferTarget.PixelPackBuffer, 0);
 
-            byte[] data = new byte[size];
-
             Sync();
 
-            Marshal.Copy(_bufferMap, data, 0, size);
+            return new ReadOnlySpan<byte>(_bufferMap.ToPointer(), size);
-
-            return data;
         }
 
-        public byte[] GetBufferData(BufferHandle buffer, int offset, int size)
+        public unsafe ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
         {
             EnsureBuffer(size);
 
@@ -93,13 +105,9 @@ namespace Ryujinx.Graphics.OpenGL
 
             GL.BindBuffer(BufferTarget.CopyWriteBuffer, 0);
 
-            byte[] data = new byte[size];
-
             Sync();
 
-            Marshal.Copy(_bufferMap, data, 0, size);
+            return new ReadOnlySpan<byte>(_bufferMap.ToPointer(), size);
-
-            return data;
         }
 
         public void Dispose()

Ryujinx.Graphics.OpenGL/Renderer.cs

@@ -91,16 +91,9 @@ namespace Ryujinx.Graphics.OpenGL
             Buffer.Delete(buffer);
         }
 
-        public byte[] GetBufferData(BufferHandle buffer, int offset, int size)
+        public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
         {
-            if (HwCapabilities.UsePersistentBufferForFlush)
+            return Buffer.GetData(this, buffer, offset, size);
-            {
-                return PersistentBuffers.Default.GetBufferData(buffer, offset, size);
-            }
-            else
-            {
-                return Buffer.GetData(buffer, offset, size);
-            }
         }
 
         public Capabilities GetCapabilities()

Ryujinx.Graphics.Texture/LayoutConverter.cs

@@ -359,6 +359,7 @@ namespace Ryujinx.Graphics.Texture
         }
 
         public static ReadOnlySpan<byte> ConvertLinearToBlockLinear(
+            Span<byte> output,
             int width,
             int height,
             int depth,
@@ -373,7 +374,10 @@ namespace Ryujinx.Graphics.Texture
             SizeInfo sizeInfo,
             ReadOnlySpan<byte> data)
         {
-            Span<byte> output = new byte[sizeInfo.TotalSize];
+            if (output.Length == 0)
+            {
+                output = new byte[sizeInfo.TotalSize];
+            }
 
             int inOffs = 0;
 
@@ -500,6 +504,7 @@ namespace Ryujinx.Graphics.Texture
         }
 
         public static ReadOnlySpan<byte> ConvertLinearToLinearStrided(
+            Span<byte> output,
             int width,
             int height,
             int blockWidth,
@@ -516,10 +521,21 @@ namespace Ryujinx.Graphics.Texture
 
             if (inStride == stride)
             {
-                return data;
+                if (output.Length != 0)
+                {
+                    data.CopyTo(output);
+                    return output;
+                }
+                else
+                {
+                    return data;
+                }
             }
 
-            Span<byte> output = new byte[h * stride];
+            if (output.Length == 0)
+            {
+                output = new byte[h * stride];
+            }
 
             int inOffs = 0;
             int outOffs = 0;

Ryujinx.Memory.Tests/MockVirtualMemoryManager.cs

@@ -49,7 +49,7 @@ namespace Ryujinx.Memory.Tests
             throw new NotImplementedException();
         }
 
-        public WritableRegion GetWritableRegion(ulong va, int size)
+        public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
         {
             throw new NotImplementedException();
         }

Ryujinx.Memory/AddressSpaceManager.cs

@@ -207,9 +207,10 @@ namespace Ryujinx.Memory
         /// </remarks>
         /// <param name="va">Virtual address of the data</param>
         /// <param name="size">Size of the data</param>
+        /// <param name="tracked">True if write tracking is triggered on the span</param>
         /// <returns>A writable region of memory containing the data</returns>
         /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
-        public unsafe WritableRegion GetWritableRegion(ulong va, int size)
+        public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
         {
             if (size == 0)
             {

Ryujinx.Memory/IVirtualMemoryManager.cs

@@ -87,9 +87,10 @@ namespace Ryujinx.Memory
         /// </summary>
         /// <param name="va">Virtual address of the data</param>
         /// <param name="size">Size of the data</param>
+        /// <param name="tracked">True if write tracking is triggered on the span</param>
         /// <returns>A writable region of memory containing the data</returns>
         /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
-        WritableRegion GetWritableRegion(ulong va, int size);
+        WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false);
 
         /// <summary>
         /// Gets a reference for the given type at the specified virtual memory address.

Ryujinx.Memory/IWritableBlock.cs

@@ -5,5 +5,7 @@ namespace Ryujinx.Memory
     public interface IWritableBlock
     {
         void Write(ulong va, ReadOnlySpan<byte> data);
+
+        void WriteUntracked(ulong va, ReadOnlySpan<byte> data) => Write(va, data);
     }
 }

Ryujinx.Memory/WritableRegion.cs

@@ -6,15 +6,17 @@ namespace Ryujinx.Memory
     {
         private readonly IWritableBlock _block;
         private readonly ulong _va;
+        private readonly bool _tracked;
 
         private bool NeedsWriteback => _block != null;
 
         public Memory<byte> Memory { get; }
 
-        public WritableRegion(IWritableBlock block, ulong va, Memory<byte> memory)
+        public WritableRegion(IWritableBlock block, ulong va, Memory<byte> memory, bool tracked = false)
         {
             _block = block;
             _va = va;
+            _tracked = tracked;
             Memory = memory;
         }
 
@@ -22,7 +24,14 @@ namespace Ryujinx.Memory
         {
             if (NeedsWriteback)
             {
-                _block.Write(_va, Memory.Span);
+                if (_tracked)
+                {
+                    _block.Write(_va, Memory.Span);
+                }
+                else
+                {
+                    _block.WriteUntracked(_va, Memory.Span);
+                }
             }
         }
     }