From 167f50bbcd5b2378a038e540877be4d9b71a12f6 Mon Sep 17 00:00:00 2001 From: gdkchan Date: Thu, 22 Feb 2024 11:03:07 -0300 Subject: [PATCH] Implement virtual buffer dependencies (#6190) * Implement virtual buffer copies * Introduce TranslateAndCreateMultiBuffersPhysicalOnly, use it for copy and clear * Rename VirtualBufferCache to VirtualRangeCache * Fix potential issue where virtual range could exist in the cache, without a physical buffer * Fix bug that could cause copy with negative size on CopyToDependantVirtualBuffer * Remove virtual copy back for SyncAction * GetData XML docs * Make field readonly * Fix virtual buffer modification tracking * Remove CopyFromDependantVirtualBuffers from ExternalFlush * Move things around a little to avoid perf impact - Inline null check for CopyFromDependantVirtualBuffers - Remove extra method call for SynchronizeMemoryWithVirtualCopyBack, prefer calling CopyFromDependantVirtualBuffers separately * Fix up XML doc --------- Co-authored-by: riperiperi --- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 215 +++++++++++++++++- .../Memory/BufferCache.cs | 192 +++++++++++++--- .../Memory/MemoryManager.cs | 8 +- .../Memory/MultiRangeBuffer.cs | 187 ++++++++++++++- ...ualBufferCache.cs => VirtualRangeCache.cs} | 20 +- 5 files changed, 573 insertions(+), 49 deletions(-) rename src/Ryujinx.Graphics.Gpu/Memory/{VirtualBufferCache.cs => VirtualRangeCache.cs} (92%) diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index 12461e96e1..e01e5142c8 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -5,6 +5,8 @@ using Ryujinx.Memory.Tracking; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { @@ -65,6 +67,9 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly Action _loadDelegate; private readonly Action _modifiedDelegate; + private HashSet _virtualDependencies; + private readonly ReaderWriterLockSlim _virtualDependenciesLock; + private int _sequenceNumber; private readonly bool _useGranular; @@ -152,6 +157,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _externalFlushDelegate = new RegionSignal(ExternalFlush); _loadDelegate = new Action(LoadRegion); _modifiedDelegate = new Action(RegionModified); + + _virtualDependenciesLock = new ReaderWriterLockSlim(); } /// @@ -220,6 +227,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the range to synchronize /// Size in bytes of the range to synchronize + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SynchronizeMemory(ulong address, ulong size) { if (_useGranular) @@ -239,6 +247,7 @@ namespace Ryujinx.Graphics.Gpu.Memory else { _context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size)); + CopyToDependantVirtualBuffers(); } _sequenceNumber = _context.SequenceNumber; @@ -460,6 +469,8 @@ namespace Ryujinx.Graphics.Gpu.Memory int offset = (int)(mAddress - Address); _context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize)); + + CopyToDependantVirtualBuffers(mAddress, mSize); } /// @@ -520,6 +531,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The offset of the destination buffer to copy into public void CopyTo(Buffer destination, int dstOffset) { + CopyFromDependantVirtualBuffers(); _context.Renderer.Pipeline.CopyBuffer(Handle, destination.Handle, 0, dstOffset, (int)Size); } @@ -536,7 +548,7 @@ namespace Ryujinx.Graphics.Gpu.Memory using PinnedSpan data = _context.Renderer.GetBufferData(Handle, offset, (int)size); // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. - _physicalMemory.WriteUntracked(address, data.Get()); + _physicalMemory.WriteUntracked(address, CopyFromDependantVirtualBuffers(data.Get(), address, size)); } /// @@ -617,6 +629,207 @@ namespace Ryujinx.Graphics.Gpu.Memory UnmappedSequence++; } + /// + /// Adds a virtual buffer dependency, indicating that a virtual buffer depends on data from this buffer. + /// + /// Dependant virtual buffer + public void AddVirtualDependency(MultiRangeBuffer virtualBuffer) + { + _virtualDependenciesLock.EnterWriteLock(); + + try + { + (_virtualDependencies ??= new()).Add(virtualBuffer); + } + finally + { + _virtualDependenciesLock.ExitWriteLock(); + } + } + + /// + /// Removes a virtual buffer dependency, indicating that a virtual buffer no longer depends on data from this buffer. + /// + /// Dependant virtual buffer + public void RemoveVirtualDependency(MultiRangeBuffer virtualBuffer) + { + _virtualDependenciesLock.EnterWriteLock(); + + try + { + if (_virtualDependencies != null) + { + _virtualDependencies.Remove(virtualBuffer); + + if (_virtualDependencies.Count == 0) + { + _virtualDependencies = null; + } + } + } + finally + { + _virtualDependenciesLock.ExitWriteLock(); + } + } + + /// + /// Copies the buffer data to all virtual buffers that depends on it. + /// + public void CopyToDependantVirtualBuffers() + { + CopyToDependantVirtualBuffers(Address, Size); + } + + /// + /// Copies the buffer data inside the specifide range to all virtual buffers that depends on it. + /// + /// Address of the range + /// Size of the range in bytes + public void CopyToDependantVirtualBuffers(ulong address, ulong size) + { + if (_virtualDependencies != null) + { + foreach (var virtualBuffer in _virtualDependencies) + { + CopyToDependantVirtualBuffer(virtualBuffer, address, size); + } + } + } + + /// + /// Copies all modified ranges from all virtual buffers back into this buffer. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void CopyFromDependantVirtualBuffers() + { + if (_virtualDependencies != null) + { + CopyFromDependantVirtualBuffersImpl(); + } + } + + /// + /// Copies all modified ranges from all virtual buffers back into this buffer. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private void CopyFromDependantVirtualBuffersImpl() + { + foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber)) + { + virtualBuffer.ConsumeModifiedRegion(this, (mAddress, mSize) => + { + // Get offset inside both this and the virtual buffer. + // Note that sometimes there is no right answer for the virtual offset, + // as the same physical range might be mapped multiple times inside a virtual buffer. + // We just assume it does not happen in practice as it can only be implemented correctly + // when the host has support for proper sparse mapping. + + ulong mEndAddress = mAddress + mSize; + mAddress = Math.Max(mAddress, Address); + mSize = Math.Min(mEndAddress, EndAddress) - mAddress; + + int physicalOffset = (int)(mAddress - Address); + int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize)); + + _context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)mSize); + }); + } + } + + /// + /// Copies all overlapping modified ranges from all virtual buffers back into this buffer, and returns an updated span with the data. + /// + /// Span where the unmodified data will be taken from for the output + /// Address of the region to copy + /// Size of the region to copy in bytes + /// A span with , and the data for all modified ranges if any + private ReadOnlySpan CopyFromDependantVirtualBuffers(ReadOnlySpan dataSpan, ulong address, ulong size) + { + _virtualDependenciesLock.EnterReadLock(); + + try + { + if (_virtualDependencies != null) + { + byte[] storage = dataSpan.ToArray(); + + foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber)) + { + virtualBuffer.ConsumeModifiedRegion(address, size, (mAddress, mSize) => + { + // Get offset inside both this and the virtual buffer. + // Note that sometimes there is no right answer for the virtual offset, + // as the same physical range might be mapped multiple times inside a virtual buffer. + // We just assume it does not happen in practice as it can only be implemented correctly + // when the host has support for proper sparse mapping. + + ulong mEndAddress = mAddress + mSize; + mAddress = Math.Max(mAddress, address); + mSize = Math.Min(mEndAddress, address + size) - mAddress; + + int physicalOffset = (int)(mAddress - Address); + int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize)); + + _context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)size); + virtualBuffer.GetData(storage.AsSpan().Slice((int)(mAddress - address), (int)mSize), virtualOffset, (int)mSize); + }); + } + + dataSpan = storage; + } + } + finally + { + _virtualDependenciesLock.ExitReadLock(); + } + + return dataSpan; + } + + /// + /// Copies the buffer data to the specified virtual buffer. + /// + /// Virtual buffer to copy the data into + public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer) + { + CopyToDependantVirtualBuffer(virtualBuffer, Address, Size); + } + + /// + /// Copies the buffer data inside the given range to the specified virtual buffer. + /// + /// Virtual buffer to copy the data into + /// Address of the range + /// Size of the range in bytes + public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer, ulong address, ulong size) + { + // Broadcast data to all ranges of the virtual buffer that are contained inside this buffer. + + ulong lastOffset = 0; + + while (virtualBuffer.TryGetPhysicalOffset(this, lastOffset, out ulong srcOffset, out ulong dstOffset, out ulong copySize)) + { + ulong innerOffset = address - Address; + ulong innerEndOffset = (address + size) - Address; + + lastOffset = dstOffset + copySize; + + // Clamp range to the specified range. + ulong copySrcOffset = Math.Max(srcOffset, innerOffset); + ulong copySrcEndOffset = Math.Min(innerEndOffset, srcOffset + copySize); + + if (copySrcEndOffset > copySrcOffset) + { + copySize = copySrcEndOffset - copySrcOffset; + dstOffset += copySrcOffset - srcOffset; + srcOffset = copySrcOffset; + + _context.Renderer.Pipeline.CopyBuffer(Handle, virtualBuffer.Handle, (int)srcOffset, (int)dstOffset, (int)copySize); + } + } + } + /// /// Increments the buffer reference count. /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index bd9aa39c82..c6284780d3 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -3,6 +3,7 @@ using Ryujinx.Memory.Range; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; namespace Ryujinx.Graphics.Gpu.Memory { @@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory private readonly Dictionary _dirtyCache; private readonly Dictionary _modifiedCache; private bool _pruneCaches; + private int _virtualModifiedSequenceNumber; public event Action NotifyBuffersModified; @@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Performs address translation of the GPU virtual address, and creates - /// new buffers, if needed, for the specified range. + /// new physical and virtual buffers, if needed, for the specified range. /// /// GPU memory manager where the buffer is mapped /// Start GPU virtual address of the buffer @@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory return new MultiRange(MemoryManager.PteUnmapped, size); } - bool supportsSparse = _context.Capabilities.SupportsSparseBuffer; - // Fast path not taken for non-contiguous ranges, // since multi-range buffers are not coalesced, so a buffer that covers // the entire cached range might not actually exist. - if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) && + if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) && range.Count == 1) { return range; @@ -154,6 +154,50 @@ namespace Ryujinx.Graphics.Gpu.Memory return range; } + /// + /// Performs address translation of the GPU virtual address, and creates + /// new physical buffers, if needed, for the specified range. + /// + /// GPU memory manager where the buffer is mapped + /// Start GPU virtual address of the buffer + /// Size in bytes of the buffer + /// Physical ranges of the buffer, after address translation + public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size) + { + if (gpuVa == 0) + { + return new MultiRange(MemoryManager.PteUnmapped, size); + } + + // Fast path not taken for non-contiguous ranges, + // since multi-range buffers are not coalesced, so a buffer that covers + // the entire cached range might not actually exist. + if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) && + range.Count == 1) + { + return range; + } + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + if (range.Count > 1) + { + CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize); + } + else + { + CreateBuffer(subRange.Address, subRange.Size); + } + } + } + + return range; + } + /// /// Creates a new buffer for the specified range, if it does not yet exist. /// This can be used to ensure the existance of a buffer. @@ -263,41 +307,108 @@ namespace Ryujinx.Graphics.Gpu.Memory } } - BufferRange[] storages = new BufferRange[range.Count]; + MultiRangeBuffer multiRangeBuffer; + MemoryRange[] alignedSubRanges = new MemoryRange[range.Count]; ulong alignmentMask = SparseBufferAlignmentSize - 1; - for (int i = 0; i < range.Count; i++) + if (_context.Capabilities.SupportsSparseBuffer) { - MemoryRange subRange = range.GetSubRange(i); + BufferRange[] storages = new BufferRange[range.Count]; + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + ulong endAddress = subRange.Address + subRange.Size; + + ulong alignedAddress = subRange.Address & ~alignmentMask; + ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; + ulong alignedSize = alignedEndAddress - alignedAddress; + + Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); + BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); + + alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + storages[i] = bufferRange; + } + else + { + ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; + + alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); + } + } + + multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); + } + else + { + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + if (subRange.Address != MemoryManager.PteUnmapped) + { + ulong endAddress = subRange.Address + subRange.Size; + + ulong alignedAddress = subRange.Address & ~alignmentMask; + ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; + ulong alignedSize = alignedEndAddress - alignedAddress; + + alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + } + else + { + ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; + + alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); + } + } + + multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges)); + + UpdateVirtualBufferDependencies(multiRangeBuffer); + } + + _multiRangeBuffers.Add(multiRangeBuffer); + } + + /// + /// Adds two-way dependencies to all physical buffers contained within a given virtual buffer. + /// + /// Virtual buffer to have dependencies added + private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer) + { + virtualBuffer.ClearPhysicalDependencies(); + + ulong dstOffset = 0; + + HashSet physicalBuffers = new(); + + for (int i = 0; i < virtualBuffer.Range.Count; i++) + { + MemoryRange subRange = virtualBuffer.Range.GetSubRange(i); if (subRange.Address != MemoryManager.PteUnmapped) { - ulong endAddress = subRange.Address + subRange.Size; + Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size); - ulong alignedAddress = subRange.Address & ~alignmentMask; - ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; - ulong alignedSize = alignedEndAddress - alignedAddress; - - Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize); - BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false); - - storages[i] = bufferRange; - alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize); + virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size); + physicalBuffers.Add(buffer); } - else - { - ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask; - storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); - alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize); - } + dstOffset += subRange.Size; } - MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); - - _multiRangeBuffers.Add(multiRangeBuffer); + foreach (var buffer in physicalBuffers) + { + buffer.CopyToDependantVirtualBuffer(virtualBuffer); + } } /// @@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Size in bytes of the copy public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size) { - MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size); - MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size); + MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size); + MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size); if (srcRange.Count == 1 && dstRange.Count == 1) { @@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory dstBuffer.ClearModified(dstAddress, size); memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer); } + + dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size); } /// @@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Value to be written into the buffer public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value) { - MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size); + MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size); for (int index = 0; index < range.Count; index++) { @@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value); memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer); + + buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size); } } @@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer) + { + buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber); + } + return buffer; } @@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { buffer = _buffers.FindFirstOverlap(address, size); + buffer.CopyFromDependantVirtualBuffers(); buffer.SynchronizeMemory(address, size); if (write) @@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory if (range.Count == 1) { MemoryRange subRange = range.GetSubRange(0); - SynchronizeBufferRange(subRange.Address, subRange.Size); + SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true); } else { for (int index = 0; index < range.Count; index++) { MemoryRange subRange = range.GetSubRange(index); - SynchronizeBufferRange(subRange.Address, subRange.Size); + SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false); } } } @@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the memory range /// Size in bytes of the memory range - private void SynchronizeBufferRange(ulong address, ulong size) + /// Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual) { if (size != 0) { Buffer buffer = _buffers.FindFirstOverlap(address, size); + if (copyBackVirtual) + { + buffer.CopyFromDependantVirtualBuffers(); + } + buffer.SynchronizeMemory(address, size); } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs index 74d527051d..30f87813f3 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs @@ -40,9 +40,9 @@ namespace Ryujinx.Graphics.Gpu.Memory internal PhysicalMemory Physical { get; } /// - /// Virtual buffer cache. + /// Virtual range cache. /// - internal VirtualBufferCache VirtualBufferCache { get; } + internal VirtualRangeCache VirtualRangeCache { get; } /// /// Cache of GPU counters. @@ -56,12 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory internal MemoryManager(PhysicalMemory physicalMemory) { Physical = physicalMemory; - VirtualBufferCache = new VirtualBufferCache(this); + VirtualRangeCache = new VirtualRangeCache(this); CounterCache = new CounterCache(); _pageTable = new ulong[PtLvl0Size][]; MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler; - MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler; + MemoryUnmapped += VirtualRangeCache.MemoryUnmappedHandler; MemoryUnmapped += CounterCache.MemoryUnmappedHandler; } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs index e039a7a43b..d92b0836e9 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs @@ -1,6 +1,7 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using System; +using System.Collections.Generic; namespace Ryujinx.Graphics.Gpu.Memory { @@ -21,12 +22,73 @@ namespace Ryujinx.Graphics.Gpu.Memory /// public MultiRange Range { get; } + /// + /// Ever increasing counter value indicating when the buffer was modified relative to other buffers. + /// + public int ModificationSequenceNumber { get; private set; } + + /// + /// Physical buffer dependency entry. + /// + private readonly struct PhysicalDependency + { + /// + /// Physical buffer. + /// + public readonly Buffer PhysicalBuffer; + + /// + /// Offset of the range on the physical buffer. + /// + public readonly ulong PhysicalOffset; + + /// + /// Offset of the range on the virtual buffer. + /// + public readonly ulong VirtualOffset; + + /// + /// Size of the range. + /// + public readonly ulong Size; + + /// + /// Creates a new physical dependency. + /// + /// Physical buffer + /// Offset of the range on the physical buffer + /// Offset of the range on the virtual buffer + /// Size of the range + public PhysicalDependency(Buffer physicalBuffer, ulong physicalOffset, ulong virtualOffset, ulong size) + { + PhysicalBuffer = physicalBuffer; + PhysicalOffset = physicalOffset; + VirtualOffset = virtualOffset; + Size = size; + } + } + + private List _dependencies; + private BufferModifiedRangeList _modifiedRanges = null; + /// /// Creates a new instance of the buffer. /// /// GPU context that the buffer belongs to /// Range of memory where the data is mapped - /// Backing memory for the buffers + public MultiRangeBuffer(GpuContext context, MultiRange range) + { + _context = context; + Range = range; + Handle = context.Renderer.CreateBuffer((int)range.GetSize()); + } + + /// + /// Creates a new instance of the buffer. + /// + /// GPU context that the buffer belongs to + /// Range of memory where the data is mapped + /// Backing memory for the buffer public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan storages) { _context = context; @@ -49,11 +111,134 @@ namespace Ryujinx.Graphics.Gpu.Memory return new BufferRange(Handle, offset, (int)range.GetSize()); } + /// + /// Removes all physical buffer dependencies. + /// + public void ClearPhysicalDependencies() + { + _dependencies?.Clear(); + } + + /// + /// Adds a physical buffer dependency. + /// + /// Physical buffer to be added + /// Address inside the physical buffer where the virtual buffer range is located + /// Offset inside the virtual buffer where the physical range is located + /// Size of the range in bytes + public void AddPhysicalDependency(Buffer buffer, ulong rangeAddress, ulong dstOffset, ulong rangeSize) + { + (_dependencies ??= new()).Add(new(buffer, rangeAddress - buffer.Address, dstOffset, rangeSize)); + buffer.AddVirtualDependency(this); + } + + /// + /// Tries to get the physical range corresponding to the given physical buffer. + /// + /// Physical buffer + /// Minimum virtual offset that a range match can have + /// Physical offset of the match + /// Virtual offset of the match, always greater than or equal + /// Size of the range match + /// True if a match was found for the given parameters, false otherwise + public bool TryGetPhysicalOffset(Buffer buffer, ulong minimumVirtOffset, out ulong physicalOffset, out ulong virtualOffset, out ulong size) + { + physicalOffset = 0; + virtualOffset = 0; + size = 0; + + if (_dependencies != null) + { + foreach (var dependency in _dependencies) + { + if (dependency.PhysicalBuffer == buffer && dependency.VirtualOffset >= minimumVirtOffset) + { + physicalOffset = dependency.PhysicalOffset; + virtualOffset = dependency.VirtualOffset; + size = dependency.Size; + + return true; + } + } + } + + return false; + } + + /// + /// Adds a modified virtual memory range. + /// + /// + /// This is only required when the host does not support sparse buffers, otherwise only physical buffers need to track modification. + /// + /// Modified range + /// ModificationSequenceNumber + public void AddModifiedRegion(MultiRange range, int modifiedSequenceNumber) + { + _modifiedRanges ??= new(_context, null, null); + + for (int i = 0; i < range.Count; i++) + { + MemoryRange subRange = range.GetSubRange(i); + + _modifiedRanges.SignalModified(subRange.Address, subRange.Size); + } + + ModificationSequenceNumber = modifiedSequenceNumber; + } + + /// + /// Calls the specified for all modified ranges that overlaps with . + /// + /// Buffer to have its range checked + /// Action to perform for modified ranges + public void ConsumeModifiedRegion(Buffer buffer, Action rangeAction) + { + ConsumeModifiedRegion(buffer.Address, buffer.Size, rangeAction); + } + + /// + /// Calls the specified for all modified ranges that overlaps with and . + /// + /// Address of the region to consume + /// Size of the region to consume + /// Action to perform for modified ranges + public void ConsumeModifiedRegion(ulong address, ulong size, Action rangeAction) + { + if (_modifiedRanges != null) + { + _modifiedRanges.GetRanges(address, size, rangeAction); + _modifiedRanges.Clear(address, size); + } + } + + /// + /// Gets data from the specified region of the buffer, and places it on . + /// + /// Span to put the data into + /// Offset of the buffer to get the data from + /// Size of the data in bytes + public void GetData(Span output, int offset, int size) + { + using PinnedSpan data = _context.Renderer.GetBufferData(Handle, offset, size); + data.Get().CopyTo(output); + } + /// /// Disposes the host buffer. /// public void Dispose() { + if (_dependencies != null) + { + foreach (var dependency in _dependencies) + { + dependency.PhysicalBuffer.RemoveVirtualDependency(this); + } + + _dependencies = null; + } + _context.Renderer.DeleteBuffer(Handle); } } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs similarity index 92% rename from src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs rename to src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs index 858c5e3b0f..889f5c9caa 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs @@ -6,9 +6,9 @@ using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { /// - /// Virtual buffer cache. + /// Virtual range cache. /// - class VirtualBufferCache + class VirtualRangeCache { private readonly MemoryManager _memoryManager; @@ -68,10 +68,10 @@ namespace Ryujinx.Graphics.Gpu.Memory private int _hasDeferredUnmaps; /// - /// Creates a new instance of the virtual buffer cache. + /// Creates a new instance of the virtual range cache. /// - /// Memory manager that the virtual buffer cache belongs to - public VirtualBufferCache(MemoryManager memoryManager) + /// Memory manager that the virtual range cache belongs to + public VirtualRangeCache(MemoryManager memoryManager) { _memoryManager = memoryManager; _virtualRanges = new RangeList(); @@ -102,10 +102,9 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// GPU virtual address to get the physical range from /// Size in bytes of the region - /// Indicates host support for sparse buffer mapping of non-contiguous ranges /// Physical range for the specified GPU virtual region /// True if the range already existed, false if a new one was created and added - public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range) + public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range) { VirtualRange[] overlaps = _virtualRangeOverlaps; int overlapsCount; @@ -158,7 +157,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } else { - found = true; + found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range); range = overlap0.Range.Slice(gpuVa - overlap0.Address, size); } } @@ -175,11 +174,10 @@ namespace Ryujinx.Graphics.Gpu.Memory ShrinkOverlapsBufferIfNeeded(); // If the the range is not properly aligned for sparse mapping, - // or if the host does not support sparse mapping, let's just - // force it to a single range. + // let's just force it to a single range. // This might cause issues in some applications that uses sparse // mappings. - if (!IsSparseAligned(range) || !supportsSparse) + if (!IsSparseAligned(range)) { range = new MultiRange(range.GetSubRange(0).Address, size); }