forked from Mirror/Ryujinx
Implement support for multi-range buffers using Vulkan sparse mappings (#5427)
* Pass MultiRange to BufferManager * Implement support for multi-range buffers using Vulkan sparse mappings * Use multi-range for remaining buffers, delete old methods * Assume that more buffers are contiguous * Dispose multi-range buffers after they are removed from the list * Properly init BufferBounds for constant and storage buffers * Do not try reading zero bytes data from an unmapped address on the shader cache + PR feedback * Fix misaligned sparse buffer offsets * Null check can be simplified * PR feedback
This commit is contained in:
parent
0531c16326
commit
1df6c07f78
33 changed files with 1241 additions and 233 deletions
|
@ -1,9 +1,13 @@
|
||||||
|
using System;
|
||||||
|
|
||||||
namespace Ryujinx.Graphics.GAL
|
namespace Ryujinx.Graphics.GAL
|
||||||
{
|
{
|
||||||
|
[Flags]
|
||||||
public enum BufferAccess
|
public enum BufferAccess
|
||||||
{
|
{
|
||||||
Default,
|
Default = 0,
|
||||||
FlushPersistent,
|
FlushPersistent = 1 << 0,
|
||||||
Stream
|
Stream = 1 << 1,
|
||||||
|
SparseCompatible = 1 << 2,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ namespace Ryujinx.Graphics.GAL
|
||||||
public readonly bool SupportsR4G4B4A4Format;
|
public readonly bool SupportsR4G4B4A4Format;
|
||||||
public readonly bool SupportsScaledVertexFormats;
|
public readonly bool SupportsScaledVertexFormats;
|
||||||
public readonly bool SupportsSnormBufferTextureFormat;
|
public readonly bool SupportsSnormBufferTextureFormat;
|
||||||
|
public readonly bool SupportsSparseBuffer;
|
||||||
public readonly bool Supports5BitComponentFormat;
|
public readonly bool Supports5BitComponentFormat;
|
||||||
public readonly bool SupportsBlendEquationAdvanced;
|
public readonly bool SupportsBlendEquationAdvanced;
|
||||||
public readonly bool SupportsFragmentShaderInterlock;
|
public readonly bool SupportsFragmentShaderInterlock;
|
||||||
|
@ -79,6 +80,7 @@ namespace Ryujinx.Graphics.GAL
|
||||||
bool supportsScaledVertexFormats,
|
bool supportsScaledVertexFormats,
|
||||||
bool supportsSnormBufferTextureFormat,
|
bool supportsSnormBufferTextureFormat,
|
||||||
bool supports5BitComponentFormat,
|
bool supports5BitComponentFormat,
|
||||||
|
bool supportsSparseBuffer,
|
||||||
bool supportsBlendEquationAdvanced,
|
bool supportsBlendEquationAdvanced,
|
||||||
bool supportsFragmentShaderInterlock,
|
bool supportsFragmentShaderInterlock,
|
||||||
bool supportsFragmentShaderOrderingIntel,
|
bool supportsFragmentShaderOrderingIntel,
|
||||||
|
@ -130,6 +132,7 @@ namespace Ryujinx.Graphics.GAL
|
||||||
SupportsScaledVertexFormats = supportsScaledVertexFormats;
|
SupportsScaledVertexFormats = supportsScaledVertexFormats;
|
||||||
SupportsSnormBufferTextureFormat = supportsSnormBufferTextureFormat;
|
SupportsSnormBufferTextureFormat = supportsSnormBufferTextureFormat;
|
||||||
Supports5BitComponentFormat = supports5BitComponentFormat;
|
Supports5BitComponentFormat = supports5BitComponentFormat;
|
||||||
|
SupportsSparseBuffer = supportsSparseBuffer;
|
||||||
SupportsBlendEquationAdvanced = supportsBlendEquationAdvanced;
|
SupportsBlendEquationAdvanced = supportsBlendEquationAdvanced;
|
||||||
SupportsFragmentShaderInterlock = supportsFragmentShaderInterlock;
|
SupportsFragmentShaderInterlock = supportsFragmentShaderInterlock;
|
||||||
SupportsFragmentShaderOrderingIntel = supportsFragmentShaderOrderingIntel;
|
SupportsFragmentShaderOrderingIntel = supportsFragmentShaderOrderingIntel;
|
||||||
|
|
|
@ -16,13 +16,10 @@ namespace Ryujinx.Graphics.GAL
|
||||||
|
|
||||||
void BackgroundContextAction(Action action, bool alwaysBackground = false);
|
void BackgroundContextAction(Action action, bool alwaysBackground = false);
|
||||||
|
|
||||||
BufferHandle CreateBuffer(int size, BufferHandle storageHint);
|
BufferHandle CreateBuffer(int size, BufferAccess access = BufferAccess.Default);
|
||||||
BufferHandle CreateBuffer(int size)
|
BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint);
|
||||||
{
|
|
||||||
return CreateBuffer(size, BufferHandle.Null);
|
|
||||||
}
|
|
||||||
BufferHandle CreateBuffer(nint pointer, int size);
|
BufferHandle CreateBuffer(nint pointer, int size);
|
||||||
BufferHandle CreateBuffer(int size, BufferAccess access);
|
BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers);
|
||||||
|
|
||||||
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
|
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
|
||||||
|
|
||||||
|
|
|
@ -44,6 +44,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
|
||||||
Register<ActionCommand>(CommandType.Action);
|
Register<ActionCommand>(CommandType.Action);
|
||||||
Register<CreateBufferCommand>(CommandType.CreateBuffer);
|
Register<CreateBufferCommand>(CommandType.CreateBuffer);
|
||||||
Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess);
|
Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess);
|
||||||
|
Register<CreateBufferSparseCommand>(CommandType.CreateBufferSparse);
|
||||||
Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer);
|
Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer);
|
||||||
Register<CreateProgramCommand>(CommandType.CreateProgram);
|
Register<CreateProgramCommand>(CommandType.CreateProgram);
|
||||||
Register<CreateSamplerCommand>(CommandType.CreateSampler);
|
Register<CreateSamplerCommand>(CommandType.CreateSampler);
|
||||||
|
|
|
@ -5,6 +5,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
|
||||||
Action,
|
Action,
|
||||||
CreateBuffer,
|
CreateBuffer,
|
||||||
CreateBufferAccess,
|
CreateBufferAccess,
|
||||||
|
CreateBufferSparse,
|
||||||
CreateHostBuffer,
|
CreateHostBuffer,
|
||||||
CreateProgram,
|
CreateProgram,
|
||||||
CreateSampler,
|
CreateSampler,
|
||||||
|
|
|
@ -5,12 +5,14 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
|
||||||
public readonly CommandType CommandType => CommandType.CreateBuffer;
|
public readonly CommandType CommandType => CommandType.CreateBuffer;
|
||||||
private BufferHandle _threadedHandle;
|
private BufferHandle _threadedHandle;
|
||||||
private int _size;
|
private int _size;
|
||||||
|
private BufferAccess _access;
|
||||||
private BufferHandle _storageHint;
|
private BufferHandle _storageHint;
|
||||||
|
|
||||||
public void Set(BufferHandle threadedHandle, int size, BufferHandle storageHint)
|
public void Set(BufferHandle threadedHandle, int size, BufferAccess access, BufferHandle storageHint)
|
||||||
{
|
{
|
||||||
_threadedHandle = threadedHandle;
|
_threadedHandle = threadedHandle;
|
||||||
_size = size;
|
_size = size;
|
||||||
|
_access = access;
|
||||||
_storageHint = storageHint;
|
_storageHint = storageHint;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -23,7 +25,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
|
||||||
hint = threaded.Buffers.MapBuffer(command._storageHint);
|
hint = threaded.Buffers.MapBuffer(command._storageHint);
|
||||||
}
|
}
|
||||||
|
|
||||||
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, hint));
|
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, command._access, hint));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,25 @@
|
||||||
|
using Ryujinx.Graphics.GAL.Multithreading.Model;
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
|
||||||
|
{
|
||||||
|
struct CreateBufferSparseCommand : IGALCommand, IGALCommand<CreateBufferSparseCommand>
|
||||||
|
{
|
||||||
|
public readonly CommandType CommandType => CommandType.CreateBufferSparse;
|
||||||
|
private BufferHandle _threadedHandle;
|
||||||
|
private SpanRef<BufferRange> _buffers;
|
||||||
|
|
||||||
|
public void Set(BufferHandle threadedHandle, SpanRef<BufferRange> buffers)
|
||||||
|
{
|
||||||
|
_threadedHandle = threadedHandle;
|
||||||
|
_buffers = buffers;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void Run(ref CreateBufferSparseCommand command, ThreadedRenderer threaded, IRenderer renderer)
|
||||||
|
{
|
||||||
|
Span<BufferRange> buffers = command._buffers.Get(threaded);
|
||||||
|
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBufferSparse(threaded.Buffers.MapBufferRanges(buffers)));
|
||||||
|
command._buffers.Dispose(threaded);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -263,10 +263,19 @@ namespace Ryujinx.Graphics.GAL.Multithreading
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
|
public BufferHandle CreateBuffer(int size, BufferAccess access)
|
||||||
{
|
{
|
||||||
BufferHandle handle = Buffers.CreateBufferHandle();
|
BufferHandle handle = Buffers.CreateBufferHandle();
|
||||||
New<CreateBufferCommand>().Set(handle, size, storageHint);
|
New<CreateBufferAccessCommand>().Set(handle, size, access);
|
||||||
|
QueueCommand();
|
||||||
|
|
||||||
|
return handle;
|
||||||
|
}
|
||||||
|
|
||||||
|
public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
|
||||||
|
{
|
||||||
|
BufferHandle handle = Buffers.CreateBufferHandle();
|
||||||
|
New<CreateBufferCommand>().Set(handle, size, access, storageHint);
|
||||||
QueueCommand();
|
QueueCommand();
|
||||||
|
|
||||||
return handle;
|
return handle;
|
||||||
|
@ -281,10 +290,10 @@ namespace Ryujinx.Graphics.GAL.Multithreading
|
||||||
return handle;
|
return handle;
|
||||||
}
|
}
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(int size, BufferAccess access)
|
public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
|
||||||
{
|
{
|
||||||
BufferHandle handle = Buffers.CreateBufferHandle();
|
BufferHandle handle = Buffers.CreateBufferHandle();
|
||||||
New<CreateBufferAccessCommand>().Set(handle, size, access);
|
New<CreateBufferSparseCommand>().Set(handle, CopySpan(storageBuffers));
|
||||||
QueueCommand();
|
QueueCommand();
|
||||||
|
|
||||||
return handle;
|
return handle;
|
||||||
|
|
|
@ -5,6 +5,7 @@ using Ryujinx.Graphics.GAL;
|
||||||
using Ryujinx.Graphics.Gpu.Engine.GPFifo;
|
using Ryujinx.Graphics.Gpu.Engine.GPFifo;
|
||||||
using Ryujinx.Graphics.Gpu.Engine.Threed;
|
using Ryujinx.Graphics.Gpu.Engine.Threed;
|
||||||
using Ryujinx.Graphics.Gpu.Engine.Types;
|
using Ryujinx.Graphics.Gpu.Engine.Types;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
@ -392,12 +393,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.MME
|
||||||
|
|
||||||
_processor.ThreedClass.DrawIndirect(
|
_processor.ThreedClass.DrawIndirect(
|
||||||
topology,
|
topology,
|
||||||
indirectBufferAddress,
|
new MultiRange(indirectBufferAddress, IndirectIndexedDataEntrySize),
|
||||||
0,
|
default,
|
||||||
1,
|
1,
|
||||||
IndirectIndexedDataEntrySize,
|
IndirectIndexedDataEntrySize,
|
||||||
indexCount,
|
indexCount,
|
||||||
Threed.IndirectDrawType.DrawIndexedIndirect);
|
IndirectDrawType.DrawIndexedIndirect);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -494,13 +495,13 @@ namespace Ryujinx.Graphics.Gpu.Engine.MME
|
||||||
|
|
||||||
ulong indirectBufferSize = (ulong)maxDrawCount * (ulong)stride;
|
ulong indirectBufferSize = (ulong)maxDrawCount * (ulong)stride;
|
||||||
|
|
||||||
ulong indirectBufferAddress = bufferCache.TranslateAndCreateBuffer(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize);
|
MultiRange indirectBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize);
|
||||||
ulong parameterBufferAddress = bufferCache.TranslateAndCreateBuffer(_processor.MemoryManager, parameterBufferGpuVa, 4);
|
MultiRange parameterBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, parameterBufferGpuVa, 4);
|
||||||
|
|
||||||
_processor.ThreedClass.DrawIndirect(
|
_processor.ThreedClass.DrawIndirect(
|
||||||
topology,
|
topology,
|
||||||
indirectBufferAddress,
|
indirectBufferRange,
|
||||||
parameterBufferAddress,
|
parameterBufferRange,
|
||||||
maxDrawCount,
|
maxDrawCount,
|
||||||
stride,
|
stride,
|
||||||
indexCount,
|
indexCount,
|
||||||
|
|
|
@ -370,8 +370,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw
|
||||||
{
|
{
|
||||||
var memoryManager = _channel.MemoryManager;
|
var memoryManager = _channel.MemoryManager;
|
||||||
|
|
||||||
address = memoryManager.Translate(address);
|
BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address, size));
|
||||||
BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(address, size);
|
|
||||||
|
|
||||||
ITexture bufferTexture = _vacContext.EnsureBufferTexture(index + 2, format);
|
ITexture bufferTexture = _vacContext.EnsureBufferTexture(index + 2, format);
|
||||||
bufferTexture.SetStorage(range);
|
bufferTexture.SetStorage(range);
|
||||||
|
@ -412,9 +411,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw
|
||||||
|
|
||||||
var memoryManager = _channel.MemoryManager;
|
var memoryManager = _channel.MemoryManager;
|
||||||
|
|
||||||
address = memoryManager.Translate(address + indexOffset);
|
|
||||||
ulong misalign = address & ((ulong)_context.Capabilities.TextureBufferOffsetAlignment - 1);
|
ulong misalign = address & ((ulong)_context.Capabilities.TextureBufferOffsetAlignment - 1);
|
||||||
BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(address - misalign, size + misalign);
|
BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address + indexOffset - misalign, size + misalign));
|
||||||
misalignedOffset = (int)misalign >> shift;
|
misalignedOffset = (int)misalign >> shift;
|
||||||
|
|
||||||
SetIndexBufferTexture(reservations, range, format);
|
SetIndexBufferTexture(reservations, range, format);
|
||||||
|
|
|
@ -3,6 +3,7 @@ using Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw;
|
||||||
using Ryujinx.Graphics.Gpu.Engine.Types;
|
using Ryujinx.Graphics.Gpu.Engine.Types;
|
||||||
using Ryujinx.Graphics.Gpu.Image;
|
using Ryujinx.Graphics.Gpu.Image;
|
||||||
using Ryujinx.Graphics.Gpu.Memory;
|
using Ryujinx.Graphics.Gpu.Memory;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
using System;
|
using System;
|
||||||
|
|
||||||
namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
||||||
|
@ -630,8 +631,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="engine">3D engine where this method is being called</param>
|
/// <param name="engine">3D engine where this method is being called</param>
|
||||||
/// <param name="topology">Primitive topology</param>
|
/// <param name="topology">Primitive topology</param>
|
||||||
/// <param name="indirectBufferAddress">Address of the buffer with the draw parameters, such as count, first index, etc</param>
|
/// <param name="indirectBufferRange">Memory range of the buffer with the draw parameters, such as count, first index, etc</param>
|
||||||
/// <param name="parameterBufferAddress">Address of the buffer with the draw count</param>
|
/// <param name="parameterBufferRange">Memory range of the buffer with the draw count</param>
|
||||||
/// <param name="maxDrawCount">Maximum number of draws that can be made</param>
|
/// <param name="maxDrawCount">Maximum number of draws that can be made</param>
|
||||||
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param>
|
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param>
|
||||||
/// <param name="indexCount">Maximum number of indices that the draw can consume</param>
|
/// <param name="indexCount">Maximum number of indices that the draw can consume</param>
|
||||||
|
@ -639,8 +640,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
||||||
public void DrawIndirect(
|
public void DrawIndirect(
|
||||||
ThreedClass engine,
|
ThreedClass engine,
|
||||||
PrimitiveTopology topology,
|
PrimitiveTopology topology,
|
||||||
ulong indirectBufferAddress,
|
MultiRange indirectBufferRange,
|
||||||
ulong parameterBufferAddress,
|
MultiRange parameterBufferRange,
|
||||||
int maxDrawCount,
|
int maxDrawCount,
|
||||||
int stride,
|
int stride,
|
||||||
int indexCount,
|
int indexCount,
|
||||||
|
@ -681,8 +682,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
||||||
|
|
||||||
if (hasCount)
|
if (hasCount)
|
||||||
{
|
{
|
||||||
var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferAddress, (ulong)maxDrawCount * (ulong)stride);
|
var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
|
||||||
var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferAddress, 4);
|
var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferRange);
|
||||||
|
|
||||||
if (indexed)
|
if (indexed)
|
||||||
{
|
{
|
||||||
|
@ -695,7 +696,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferAddress, (ulong)stride);
|
var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
|
||||||
|
|
||||||
if (indexed)
|
if (indexed)
|
||||||
{
|
{
|
||||||
|
|
|
@ -6,6 +6,7 @@ using Ryujinx.Graphics.Gpu.Engine.InlineToMemory;
|
||||||
using Ryujinx.Graphics.Gpu.Engine.Threed.Blender;
|
using Ryujinx.Graphics.Gpu.Engine.Threed.Blender;
|
||||||
using Ryujinx.Graphics.Gpu.Engine.Types;
|
using Ryujinx.Graphics.Gpu.Engine.Types;
|
||||||
using Ryujinx.Graphics.Gpu.Synchronization;
|
using Ryujinx.Graphics.Gpu.Synchronization;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
|
@ -803,22 +804,22 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
|
||||||
/// Performs a indirect draw, with parameters from a GPU buffer.
|
/// Performs a indirect draw, with parameters from a GPU buffer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="topology">Primitive topology</param>
|
/// <param name="topology">Primitive topology</param>
|
||||||
/// <param name="indirectBufferAddress">Address of the buffer with the draw parameters, such as count, first index, etc</param>
|
/// <param name="indirectBufferRange">Memory range of the buffer with the draw parameters, such as count, first index, etc</param>
|
||||||
/// <param name="parameterBufferAddress">Address of the buffer with the draw count</param>
|
/// <param name="parameterBufferRange">Memory range of the buffer with the draw count</param>
|
||||||
/// <param name="maxDrawCount">Maximum number of draws that can be made</param>
|
/// <param name="maxDrawCount">Maximum number of draws that can be made</param>
|
||||||
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param>
|
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferRange"/></param>
|
||||||
/// <param name="indexCount">Maximum number of indices that the draw can consume</param>
|
/// <param name="indexCount">Maximum number of indices that the draw can consume</param>
|
||||||
/// <param name="drawType">Type of the indirect draw, which can be indexed or non-indexed, with or without a draw count</param>
|
/// <param name="drawType">Type of the indirect draw, which can be indexed or non-indexed, with or without a draw count</param>
|
||||||
public void DrawIndirect(
|
public void DrawIndirect(
|
||||||
PrimitiveTopology topology,
|
PrimitiveTopology topology,
|
||||||
ulong indirectBufferAddress,
|
MultiRange indirectBufferRange,
|
||||||
ulong parameterBufferAddress,
|
MultiRange parameterBufferRange,
|
||||||
int maxDrawCount,
|
int maxDrawCount,
|
||||||
int stride,
|
int stride,
|
||||||
int indexCount,
|
int indexCount,
|
||||||
IndirectDrawType drawType)
|
IndirectDrawType drawType)
|
||||||
{
|
{
|
||||||
_drawManager.DrawIndirect(this, topology, indirectBufferAddress, parameterBufferAddress, maxDrawCount, stride, indexCount, drawType);
|
_drawManager.DrawIndirect(this, topology, indirectBufferRange, parameterBufferRange, maxDrawCount, stride, indexCount, drawType);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
|
@ -382,7 +382,7 @@ namespace Ryujinx.Graphics.Gpu.Image
|
||||||
{
|
{
|
||||||
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex);
|
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex);
|
||||||
|
|
||||||
cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
|
cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Range));
|
||||||
cachedTextureBufferIndex = textureBufferIndex;
|
cachedTextureBufferIndex = textureBufferIndex;
|
||||||
|
|
||||||
if (samplerBufferIndex == textureBufferIndex)
|
if (samplerBufferIndex == textureBufferIndex)
|
||||||
|
@ -396,7 +396,7 @@ namespace Ryujinx.Graphics.Gpu.Image
|
||||||
{
|
{
|
||||||
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex);
|
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex);
|
||||||
|
|
||||||
cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
|
cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Range));
|
||||||
cachedSamplerBufferIndex = samplerBufferIndex;
|
cachedSamplerBufferIndex = samplerBufferIndex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -524,7 +524,7 @@ namespace Ryujinx.Graphics.Gpu.Image
|
||||||
// Ensure that the buffer texture is using the correct buffer as storage.
|
// Ensure that the buffer texture is using the correct buffer as storage.
|
||||||
// Buffers are frequently re-created to accommodate larger data, so we need to re-bind
|
// Buffers are frequently re-created to accommodate larger data, so we need to re-bind
|
||||||
// to ensure we're not using a old buffer that was already deleted.
|
// to ensure we're not using a old buffer that was already deleted.
|
||||||
_channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, bindingInfo.Format, false);
|
_channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range, bindingInfo, bindingInfo.Format, false);
|
||||||
|
|
||||||
// Cache is not used for buffer texture, it must always rebind.
|
// Cache is not used for buffer texture, it must always rebind.
|
||||||
state.CachedTexture = null;
|
state.CachedTexture = null;
|
||||||
|
@ -661,7 +661,7 @@ namespace Ryujinx.Graphics.Gpu.Image
|
||||||
format = texture.Format;
|
format = texture.Format;
|
||||||
}
|
}
|
||||||
|
|
||||||
_channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, format, true);
|
_channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range, bindingInfo, format, true);
|
||||||
|
|
||||||
// Cache is not used for buffer texture, it must always rebind.
|
// Cache is not used for buffer texture, it must always rebind.
|
||||||
state.CachedTexture = null;
|
state.CachedTexture = null;
|
||||||
|
|
|
@ -43,6 +43,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int UnmappedSequence { get; private set; }
|
public int UnmappedSequence { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates if the buffer can be used in a sparse buffer mapping.
|
||||||
|
/// </summary>
|
||||||
|
public bool SparseCompatible { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Ranges of the buffer that have been modified on the GPU.
|
/// Ranges of the buffer that have been modified on the GPU.
|
||||||
/// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached.
|
/// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached.
|
||||||
|
@ -77,15 +82,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="physicalMemory">Physical memory where the buffer is mapped</param>
|
/// <param name="physicalMemory">Physical memory where the buffer is mapped</param>
|
||||||
/// <param name="address">Start address of the buffer</param>
|
/// <param name="address">Start address of the buffer</param>
|
||||||
/// <param name="size">Size of the buffer in bytes</param>
|
/// <param name="size">Size of the buffer in bytes</param>
|
||||||
|
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
|
||||||
/// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
|
/// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
|
||||||
public Buffer(GpuContext context, PhysicalMemory physicalMemory, ulong address, ulong size, IEnumerable<Buffer> baseBuffers = null)
|
public Buffer(
|
||||||
|
GpuContext context,
|
||||||
|
PhysicalMemory physicalMemory,
|
||||||
|
ulong address,
|
||||||
|
ulong size,
|
||||||
|
bool sparseCompatible,
|
||||||
|
IEnumerable<Buffer> baseBuffers = null)
|
||||||
{
|
{
|
||||||
_context = context;
|
_context = context;
|
||||||
_physicalMemory = physicalMemory;
|
_physicalMemory = physicalMemory;
|
||||||
Address = address;
|
Address = address;
|
||||||
Size = size;
|
Size = size;
|
||||||
|
SparseCompatible = sparseCompatible;
|
||||||
|
|
||||||
Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
|
BufferAccess access = sparseCompatible ? BufferAccess.SparseCompatible : BufferAccess.Default;
|
||||||
|
|
||||||
|
Handle = context.Renderer.CreateBuffer((int)size, access, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
|
||||||
|
|
||||||
_useGranular = size > GranularBufferThreshold;
|
_useGranular = size > GranularBufferThreshold;
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
using Ryujinx.Graphics.Shader;
|
using Ryujinx.Graphics.Shader;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
|
|
||||||
namespace Ryujinx.Graphics.Gpu.Memory
|
namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
|
@ -8,30 +9,28 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
readonly struct BufferBounds
|
readonly struct BufferBounds
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Region virtual address.
|
/// Physical memory ranges where the buffer is mapped.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ulong Address { get; }
|
public MultiRange Range { get; }
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Region size in bytes.
|
|
||||||
/// </summary>
|
|
||||||
public ulong Size { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Buffer usage flags.
|
/// Buffer usage flags.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public BufferUsageFlags Flags { get; }
|
public BufferUsageFlags Flags { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates that the backing memory for the buffer does not exist.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsUnmapped => Range.IsUnmapped;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new buffer region.
|
/// Creates a new buffer region.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="address">Region address</param>
|
/// <param name="range">Physical memory ranges where the buffer is mapped</param>
|
||||||
/// <param name="size">Region size</param>
|
|
||||||
/// <param name="flags">Buffer usage flags</param>
|
/// <param name="flags">Buffer usage flags</param>
|
||||||
public BufferBounds(ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
|
public BufferBounds(MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
|
||||||
{
|
{
|
||||||
Address = address;
|
Range = range;
|
||||||
Size = size;
|
|
||||||
Flags = flags;
|
Flags = flags;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,12 +11,24 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
class BufferCache : IDisposable
|
class BufferCache : IDisposable
|
||||||
{
|
{
|
||||||
private const int OverlapsBufferInitialCapacity = 10;
|
/// <summary>
|
||||||
private const int OverlapsBufferMaxCapacity = 10000;
|
/// Initial size for the array holding overlaps.
|
||||||
|
/// </summary>
|
||||||
|
public const int OverlapsBufferInitialCapacity = 10;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Maximum size that an array holding overlaps may have after trimming.
|
||||||
|
/// </summary>
|
||||||
|
public const int OverlapsBufferMaxCapacity = 10000;
|
||||||
|
|
||||||
private const ulong BufferAlignmentSize = 0x1000;
|
private const ulong BufferAlignmentSize = 0x1000;
|
||||||
private const ulong BufferAlignmentMask = BufferAlignmentSize - 1;
|
private const ulong BufferAlignmentMask = BufferAlignmentSize - 1;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Alignment required for sparse buffer mappings.
|
||||||
|
/// </summary>
|
||||||
|
public const ulong SparseBufferAlignmentSize = 0x10000;
|
||||||
|
|
||||||
private const ulong MaxDynamicGrowthSize = 0x100000;
|
private const ulong MaxDynamicGrowthSize = 0x100000;
|
||||||
|
|
||||||
private readonly GpuContext _context;
|
private readonly GpuContext _context;
|
||||||
|
@ -27,6 +39,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// Must lock for any access from other threads.
|
/// Must lock for any access from other threads.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
private readonly RangeList<Buffer> _buffers;
|
private readonly RangeList<Buffer> _buffers;
|
||||||
|
private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
|
||||||
|
|
||||||
private Buffer[] _bufferOverlaps;
|
private Buffer[] _bufferOverlaps;
|
||||||
|
|
||||||
|
@ -47,6 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
_physicalMemory = physicalMemory;
|
_physicalMemory = physicalMemory;
|
||||||
|
|
||||||
_buffers = new RangeList<Buffer>();
|
_buffers = new RangeList<Buffer>();
|
||||||
|
_multiRangeBuffers = new MultiRangeList<MultiRangeBuffer>();
|
||||||
|
|
||||||
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
|
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
|
||||||
|
|
||||||
|
@ -66,45 +80,100 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
Buffer[] overlaps = new Buffer[10];
|
Buffer[] overlaps = new Buffer[10];
|
||||||
int overlapCount;
|
int overlapCount;
|
||||||
|
|
||||||
ulong address = ((MemoryManager)sender).Translate(e.Address);
|
MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
|
||||||
ulong size = e.Size;
|
|
||||||
|
for (int index = 0; index < range.Count; index++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(index);
|
||||||
|
|
||||||
lock (_buffers)
|
lock (_buffers)
|
||||||
{
|
{
|
||||||
overlapCount = _buffers.FindOverlaps(address, size, ref overlaps);
|
overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = 0; i < overlapCount; i++)
|
for (int i = 0; i < overlapCount; i++)
|
||||||
{
|
{
|
||||||
overlaps[i].Unmapped(address, size);
|
overlaps[i].Unmapped(subRange.Address, subRange.Size);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs address translation of the GPU virtual address, and creates a
|
/// Performs address translation of the GPU virtual address, and creates a
|
||||||
/// new buffer, if needed, for the specified range.
|
/// new buffer, if needed, for the specified contiguous range.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
|
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
|
||||||
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
||||||
/// <param name="size">Size in bytes of the buffer</param>
|
/// <param name="size">Size in bytes of the buffer</param>
|
||||||
/// <returns>CPU virtual address of the buffer, after address translation</returns>
|
/// <returns>Contiguous physical range of the buffer, after address translation</returns>
|
||||||
public ulong TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
|
public MultiRange TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
|
||||||
{
|
{
|
||||||
if (gpuVa == 0)
|
if (gpuVa == 0)
|
||||||
{
|
{
|
||||||
return 0;
|
return new MultiRange(MemoryManager.PteUnmapped, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
ulong address = memoryManager.Translate(gpuVa);
|
ulong address = memoryManager.Translate(gpuVa);
|
||||||
|
|
||||||
if (address == MemoryManager.PteUnmapped)
|
if (address != MemoryManager.PteUnmapped)
|
||||||
{
|
{
|
||||||
return 0;
|
CreateBuffer(address, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
CreateBuffer(address, size);
|
return new MultiRange(address, size);
|
||||||
|
}
|
||||||
|
|
||||||
return address;
|
/// <summary>
|
||||||
|
/// Performs address translation of the GPU virtual address, and creates
|
||||||
|
/// new buffers, if needed, for the specified range.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
|
||||||
|
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
|
||||||
|
/// <param name="size">Size in bytes of the buffer</param>
|
||||||
|
/// <returns>Physical ranges of the buffer, after address translation</returns>
|
||||||
|
public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size)
|
||||||
|
{
|
||||||
|
if (gpuVa == 0)
|
||||||
|
{
|
||||||
|
return new MultiRange(MemoryManager.PteUnmapped, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
|
||||||
|
|
||||||
|
// Fast path not taken for non-contiguous ranges,
|
||||||
|
// since multi-range buffers are not coalesced, so a buffer that covers
|
||||||
|
// the entire cached range might not actually exist.
|
||||||
|
if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
|
||||||
|
range.Count == 1)
|
||||||
|
{
|
||||||
|
return range;
|
||||||
|
}
|
||||||
|
|
||||||
|
CreateBuffer(range);
|
||||||
|
|
||||||
|
return range;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new buffer for the specified range, if it does not yet exist.
|
||||||
|
/// This can be used to ensure the existance of a buffer.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="range">Physical ranges of memory where the buffer data is located</param>
|
||||||
|
public void CreateBuffer(MultiRange range)
|
||||||
|
{
|
||||||
|
if (range.Count > 1)
|
||||||
|
{
|
||||||
|
CreateMultiRangeBuffer(range);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(0);
|
||||||
|
|
||||||
|
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||||
|
{
|
||||||
|
CreateBuffer(subRange.Address, subRange.Size);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -118,7 +187,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
ulong endAddress = address + size;
|
ulong endAddress = address + size;
|
||||||
|
|
||||||
ulong alignedAddress = address & ~BufferAlignmentMask;
|
ulong alignedAddress = address & ~BufferAlignmentMask;
|
||||||
|
|
||||||
ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask;
|
ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask;
|
||||||
|
|
||||||
// The buffer must have the size of at least one page.
|
// The buffer must have the size of at least one page.
|
||||||
|
@ -130,6 +198,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress);
|
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new buffer for the specified range, if it does not yet exist.
|
||||||
|
/// This can be used to ensure the existance of a buffer.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="address">Address of the buffer in memory</param>
|
||||||
|
/// <param name="size">Size of the buffer in bytes</param>
|
||||||
|
/// <param name="alignment">Alignment of the start address of the buffer in bytes</param>
|
||||||
|
public void CreateBuffer(ulong address, ulong size, ulong alignment)
|
||||||
|
{
|
||||||
|
ulong alignmentMask = alignment - 1;
|
||||||
|
ulong pageAlignmentMask = BufferAlignmentMask;
|
||||||
|
ulong endAddress = address + size;
|
||||||
|
|
||||||
|
ulong alignedAddress = address & ~alignmentMask;
|
||||||
|
ulong alignedEndAddress = (endAddress + pageAlignmentMask) & ~pageAlignmentMask;
|
||||||
|
|
||||||
|
// The buffer must have the size of at least one page.
|
||||||
|
if (alignedEndAddress == alignedAddress)
|
||||||
|
{
|
||||||
|
alignedEndAddress += pageAlignmentMask;
|
||||||
|
}
|
||||||
|
|
||||||
|
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress, alignment);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a buffer for a memory region composed of multiple physical ranges,
|
||||||
|
/// if it does not exist yet.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="range">Physical ranges of memory</param>
|
||||||
|
private void CreateMultiRangeBuffer(MultiRange range)
|
||||||
|
{
|
||||||
|
// Ensure all non-contiguous buffer we might use are sparse aligned.
|
||||||
|
for (int i = 0; i < range.Count; i++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(i);
|
||||||
|
|
||||||
|
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||||
|
{
|
||||||
|
CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create sparse buffer.
|
||||||
|
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
|
||||||
|
|
||||||
|
int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
|
||||||
|
|
||||||
|
for (int index = 0; index < overlapCount; index++)
|
||||||
|
{
|
||||||
|
if (overlaps[index].Range.Contains(range))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int index = 0; index < overlapCount; index++)
|
||||||
|
{
|
||||||
|
if (range.Contains(overlaps[index].Range))
|
||||||
|
{
|
||||||
|
_multiRangeBuffers.Remove(overlaps[index]);
|
||||||
|
overlaps[index].Dispose();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
BufferRange[] storages = new BufferRange[range.Count];
|
||||||
|
MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
|
||||||
|
|
||||||
|
ulong alignmentMask = SparseBufferAlignmentSize - 1;
|
||||||
|
|
||||||
|
for (int i = 0; i < range.Count; i++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(i);
|
||||||
|
|
||||||
|
if (subRange.Address != MemoryManager.PteUnmapped)
|
||||||
|
{
|
||||||
|
ulong endAddress = subRange.Address + subRange.Size;
|
||||||
|
|
||||||
|
ulong alignedAddress = subRange.Address & ~alignmentMask;
|
||||||
|
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
|
||||||
|
ulong alignedSize = alignedEndAddress - alignedAddress;
|
||||||
|
|
||||||
|
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
|
||||||
|
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
|
||||||
|
|
||||||
|
storages[i] = bufferRange;
|
||||||
|
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
|
||||||
|
|
||||||
|
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
|
||||||
|
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
|
||||||
|
|
||||||
|
_multiRangeBuffers.Add(multiRangeBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs address translation of the GPU virtual address, and attempts to force
|
/// Performs address translation of the GPU virtual address, and attempts to force
|
||||||
/// the buffer in the region as dirty.
|
/// the buffer in the region as dirty.
|
||||||
|
@ -150,7 +320,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
result.EndGpuAddress < gpuVa + size ||
|
result.EndGpuAddress < gpuVa + size ||
|
||||||
result.UnmappedSequence != result.Buffer.UnmappedSequence)
|
result.UnmappedSequence != result.Buffer.UnmappedSequence)
|
||||||
{
|
{
|
||||||
ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
|
MultiRange range = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
|
||||||
|
ulong address = range.GetSubRange(0).Address;
|
||||||
result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size));
|
result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size));
|
||||||
|
|
||||||
_dirtyCache[gpuVa] = result;
|
_dirtyCache[gpuVa] = result;
|
||||||
|
@ -184,7 +355,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
result.EndGpuAddress < alignedEndGpuVa ||
|
result.EndGpuAddress < alignedEndGpuVa ||
|
||||||
result.UnmappedSequence != result.Buffer.UnmappedSequence)
|
result.UnmappedSequence != result.Buffer.UnmappedSequence)
|
||||||
{
|
{
|
||||||
ulong address = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
|
MultiRange range = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
|
||||||
|
ulong address = range.GetSubRange(0).Address;
|
||||||
result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size));
|
result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size));
|
||||||
|
|
||||||
_modifiedCache[alignedGpuVa] = result;
|
_modifiedCache[alignedGpuVa] = result;
|
||||||
|
@ -204,7 +376,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="size">Size in bytes of the buffer</param>
|
/// <param name="size">Size in bytes of the buffer</param>
|
||||||
private void CreateBufferAligned(ulong address, ulong size)
|
private void CreateBufferAligned(ulong address, ulong size)
|
||||||
{
|
{
|
||||||
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
|
Buffer[] overlaps = _bufferOverlaps;
|
||||||
|
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
|
||||||
|
|
||||||
if (overlapsCount != 0)
|
if (overlapsCount != 0)
|
||||||
{
|
{
|
||||||
|
@ -215,9 +388,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
// old buffer(s) to the new buffer.
|
// old buffer(s) to the new buffer.
|
||||||
|
|
||||||
ulong endAddress = address + size;
|
ulong endAddress = address + size;
|
||||||
|
Buffer overlap0 = overlaps[0];
|
||||||
|
|
||||||
if (_bufferOverlaps[0].Address > address || _bufferOverlaps[0].EndAddress < endAddress)
|
if (overlap0.Address > address || overlap0.EndAddress < endAddress)
|
||||||
{
|
{
|
||||||
|
bool anySparseCompatible = false;
|
||||||
|
|
||||||
// Check if the following conditions are met:
|
// Check if the following conditions are met:
|
||||||
// - We have a single overlap.
|
// - We have a single overlap.
|
||||||
// - The overlap starts at or before the requested range. That is, the overlap happens at the end.
|
// - The overlap starts at or before the requested range. That is, the overlap happens at the end.
|
||||||
|
@ -228,23 +404,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the
|
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the
|
||||||
// range crosses a page, and after alignment, ends having a size of 2 pages.
|
// range crosses a page, and after alignment, ends having a size of 2 pages.
|
||||||
if (overlapsCount == 1 &&
|
if (overlapsCount == 1 &&
|
||||||
address >= _bufferOverlaps[0].Address &&
|
address >= overlap0.Address &&
|
||||||
endAddress - _bufferOverlaps[0].EndAddress <= BufferAlignmentSize * 2)
|
endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
|
||||||
{
|
{
|
||||||
// Try to grow the buffer by 1.5x of its current size.
|
// Try to grow the buffer by 1.5x of its current size.
|
||||||
// This improves performance in the cases where the buffer is resized often by small amounts.
|
// This improves performance in the cases where the buffer is resized often by small amounts.
|
||||||
ulong existingSize = _bufferOverlaps[0].Size;
|
ulong existingSize = overlap0.Size;
|
||||||
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
|
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
|
||||||
|
|
||||||
size = Math.Max(size, growthSize);
|
size = Math.Max(size, growthSize);
|
||||||
endAddress = address + size;
|
endAddress = address + size;
|
||||||
|
|
||||||
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
|
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int index = 0; index < overlapsCount; index++)
|
for (int index = 0; index < overlapsCount; index++)
|
||||||
{
|
{
|
||||||
Buffer buffer = _bufferOverlaps[index];
|
Buffer buffer = overlaps[index];
|
||||||
|
|
||||||
|
anySparseCompatible |= buffer.SparseCompatible;
|
||||||
|
|
||||||
address = Math.Min(address, buffer.Address);
|
address = Math.Min(address, buffer.Address);
|
||||||
endAddress = Math.Max(endAddress, buffer.EndAddress);
|
endAddress = Math.Max(endAddress, buffer.EndAddress);
|
||||||
|
@ -257,7 +435,114 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
ulong newSize = endAddress - address;
|
ulong newSize = endAddress - address;
|
||||||
|
|
||||||
Buffer newBuffer = new(_context, _physicalMemory, address, newSize, _bufferOverlaps.Take(overlapsCount));
|
CreateBufferAligned(address, newSize, anySparseCompatible, overlaps, overlapsCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// No overlap, just create a new buffer.
|
||||||
|
Buffer buffer = new(_context, _physicalMemory, address, size, sparseCompatible: false);
|
||||||
|
|
||||||
|
lock (_buffers)
|
||||||
|
{
|
||||||
|
_buffers.Add(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ShrinkOverlapsBufferIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new buffer for the specified range, if needed.
|
||||||
|
/// If a buffer where this range can be fully contained already exists,
|
||||||
|
/// then the creation of a new buffer is not necessary.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="address">Address of the buffer in guest memory</param>
|
||||||
|
/// <param name="size">Size in bytes of the buffer</param>
|
||||||
|
/// <param name="alignment">Alignment of the start address of the buffer</param>
|
||||||
|
private void CreateBufferAligned(ulong address, ulong size, ulong alignment)
|
||||||
|
{
|
||||||
|
Buffer[] overlaps = _bufferOverlaps;
|
||||||
|
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
|
||||||
|
bool sparseAligned = alignment >= SparseBufferAlignmentSize;
|
||||||
|
|
||||||
|
if (overlapsCount != 0)
|
||||||
|
{
|
||||||
|
// If the buffer already exists, make sure if covers the entire range,
|
||||||
|
// and make sure it is properly aligned, otherwise sparse mapping may fail.
|
||||||
|
|
||||||
|
ulong endAddress = address + size;
|
||||||
|
Buffer overlap0 = overlaps[0];
|
||||||
|
|
||||||
|
if (overlap0.Address > address ||
|
||||||
|
overlap0.EndAddress < endAddress ||
|
||||||
|
(overlap0.Address & (alignment - 1)) != 0 ||
|
||||||
|
(!overlap0.SparseCompatible && sparseAligned))
|
||||||
|
{
|
||||||
|
// We need to make sure the new buffer is properly aligned.
|
||||||
|
// However, after the range is aligned, it is possible that it
|
||||||
|
// overlaps more buffers, so try again after each extension
|
||||||
|
// and ensure we cover all overlaps.
|
||||||
|
|
||||||
|
int oldOverlapsCount;
|
||||||
|
|
||||||
|
do
|
||||||
|
{
|
||||||
|
for (int index = 0; index < overlapsCount; index++)
|
||||||
|
{
|
||||||
|
Buffer buffer = overlaps[index];
|
||||||
|
|
||||||
|
address = Math.Min(address, buffer.Address);
|
||||||
|
endAddress = Math.Max(endAddress, buffer.EndAddress);
|
||||||
|
}
|
||||||
|
|
||||||
|
address &= ~(alignment - 1);
|
||||||
|
|
||||||
|
oldOverlapsCount = overlapsCount;
|
||||||
|
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps);
|
||||||
|
}
|
||||||
|
while (oldOverlapsCount != overlapsCount);
|
||||||
|
|
||||||
|
lock (_buffers)
|
||||||
|
{
|
||||||
|
for (int index = 0; index < overlapsCount; index++)
|
||||||
|
{
|
||||||
|
_buffers.Remove(overlaps[index]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ulong newSize = endAddress - address;
|
||||||
|
|
||||||
|
CreateBufferAligned(address, newSize, sparseAligned, overlaps, overlapsCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// No overlap, just create a new buffer.
|
||||||
|
Buffer buffer = new(_context, _physicalMemory, address, size, sparseAligned);
|
||||||
|
|
||||||
|
lock (_buffers)
|
||||||
|
{
|
||||||
|
_buffers.Add(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ShrinkOverlapsBufferIfNeeded();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new buffer for the specified range, if needed.
|
||||||
|
/// If a buffer where this range can be fully contained already exists,
|
||||||
|
/// then the creation of a new buffer is not necessary.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="address">Address of the buffer in guest memory</param>
|
||||||
|
/// <param name="size">Size in bytes of the buffer</param>
|
||||||
|
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
|
||||||
|
/// <param name="overlaps">Buffers overlapping the range</param>
|
||||||
|
/// <param name="overlapsCount">Total of overlaps</param>
|
||||||
|
private void CreateBufferAligned(ulong address, ulong size, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
|
||||||
|
{
|
||||||
|
Buffer newBuffer = new Buffer(_context, _physicalMemory, address, size, sparseCompatible, overlaps.Take(overlapsCount));
|
||||||
|
|
||||||
lock (_buffers)
|
lock (_buffers)
|
||||||
{
|
{
|
||||||
|
@ -266,7 +551,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
for (int index = 0; index < overlapsCount; index++)
|
for (int index = 0; index < overlapsCount; index++)
|
||||||
{
|
{
|
||||||
Buffer buffer = _bufferOverlaps[index];
|
Buffer buffer = overlaps[index];
|
||||||
|
|
||||||
int dstOffset = (int)(buffer.Address - newBuffer.Address);
|
int dstOffset = (int)(buffer.Address - newBuffer.Address);
|
||||||
|
|
||||||
|
@ -276,24 +561,40 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
buffer.DecrementReferenceCount();
|
buffer.DecrementReferenceCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
newBuffer.SynchronizeMemory(address, newSize);
|
newBuffer.SynchronizeMemory(address, size);
|
||||||
|
|
||||||
// Existing buffers were modified, we need to rebind everything.
|
// Existing buffers were modified, we need to rebind everything.
|
||||||
NotifyBuffersModified?.Invoke();
|
NotifyBuffersModified?.Invoke();
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// No overlap, just create a new buffer.
|
|
||||||
Buffer buffer = new(_context, _physicalMemory, address, size);
|
|
||||||
|
|
||||||
lock (_buffers)
|
RecreateMultiRangeBuffers(address, size);
|
||||||
{
|
|
||||||
_buffers.Add(buffer);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ShrinkOverlapsBufferIfNeeded();
|
/// <summary>
|
||||||
|
/// Recreates all the multi-range buffers that overlaps a given physical memory range.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="address">Start address of the range</param>
|
||||||
|
/// <param name="size">Size of the range in bytes</param>
|
||||||
|
private void RecreateMultiRangeBuffers(ulong address, ulong size)
|
||||||
|
{
|
||||||
|
if ((address & (SparseBufferAlignmentSize - 1)) != 0 || (size & (SparseBufferAlignmentSize - 1)) != 0)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
|
||||||
|
|
||||||
|
int overlapCount = _multiRangeBuffers.FindOverlaps(address, size, ref overlaps);
|
||||||
|
|
||||||
|
for (int index = 0; index < overlapCount; index++)
|
||||||
|
{
|
||||||
|
_multiRangeBuffers.Remove(overlaps[index]);
|
||||||
|
overlaps[index].Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int index = 0; index < overlapCount; index++)
|
||||||
|
{
|
||||||
|
CreateMultiRangeBuffer(overlaps[index].Range);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -319,9 +620,63 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="size">Size in bytes of the copy</param>
|
/// <param name="size">Size in bytes of the copy</param>
|
||||||
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
|
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
|
||||||
{
|
{
|
||||||
ulong srcAddress = TranslateAndCreateBuffer(memoryManager, srcVa, size);
|
MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
|
||||||
ulong dstAddress = TranslateAndCreateBuffer(memoryManager, dstVa, size);
|
MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
|
||||||
|
|
||||||
|
if (srcRange.Count == 1 && dstRange.Count == 1)
|
||||||
|
{
|
||||||
|
CopyBufferSingleRange(memoryManager, srcRange.GetSubRange(0).Address, dstRange.GetSubRange(0).Address, size);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ulong copiedSize = 0;
|
||||||
|
ulong srcOffset = 0;
|
||||||
|
ulong dstOffset = 0;
|
||||||
|
int srcRangeIndex = 0;
|
||||||
|
int dstRangeIndex = 0;
|
||||||
|
|
||||||
|
while (copiedSize < size)
|
||||||
|
{
|
||||||
|
if (srcRange.GetSubRange(srcRangeIndex).Size == srcOffset)
|
||||||
|
{
|
||||||
|
srcRangeIndex++;
|
||||||
|
srcOffset = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dstRange.GetSubRange(dstRangeIndex).Size == dstOffset)
|
||||||
|
{
|
||||||
|
dstRangeIndex++;
|
||||||
|
dstOffset = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
MemoryRange srcSubRange = srcRange.GetSubRange(srcRangeIndex);
|
||||||
|
MemoryRange dstSubRange = dstRange.GetSubRange(dstRangeIndex);
|
||||||
|
|
||||||
|
ulong srcSize = srcSubRange.Size - srcOffset;
|
||||||
|
ulong dstSize = dstSubRange.Size - dstOffset;
|
||||||
|
ulong copySize = Math.Min(srcSize, dstSize);
|
||||||
|
|
||||||
|
CopyBufferSingleRange(memoryManager, srcSubRange.Address + srcOffset, dstSubRange.Address + dstOffset, copySize);
|
||||||
|
|
||||||
|
srcOffset += copySize;
|
||||||
|
dstOffset += copySize;
|
||||||
|
copiedSize += copySize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Copy a buffer data from a given address to another.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// This does a GPU side copy.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
|
||||||
|
/// <param name="srcAddress">Physical address of the copy source</param>
|
||||||
|
/// <param name="dstAddress">Physical address of the copy destination</param>
|
||||||
|
/// <param name="size">Size in bytes of the copy</param>
|
||||||
|
private void CopyBufferSingleRange(MemoryManager memoryManager, ulong srcAddress, ulong dstAddress, ulong size)
|
||||||
|
{
|
||||||
Buffer srcBuffer = GetBuffer(srcAddress, size);
|
Buffer srcBuffer = GetBuffer(srcAddress, size);
|
||||||
Buffer dstBuffer = GetBuffer(dstAddress, size);
|
Buffer dstBuffer = GetBuffer(dstAddress, size);
|
||||||
|
|
||||||
|
@ -360,39 +715,98 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="value">Value to be written into the buffer</param>
|
/// <param name="value">Value to be written into the buffer</param>
|
||||||
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
|
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
|
||||||
{
|
{
|
||||||
ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
|
MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
|
||||||
|
|
||||||
Buffer buffer = GetBuffer(address, size);
|
for (int index = 0; index < range.Count; index++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(index);
|
||||||
|
Buffer buffer = GetBuffer(subRange.Address, subRange.Size);
|
||||||
|
|
||||||
int offset = (int)(address - buffer.Address);
|
int offset = (int)(subRange.Address - buffer.Address);
|
||||||
|
|
||||||
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)size, value);
|
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
|
||||||
|
|
||||||
memoryManager.Physical.FillTrackedResource(address, size, value, ResourceKind.Buffer);
|
memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a buffer sub-range from a start address til a page boundary after the given size.
|
/// Gets a buffer sub-range starting at a given memory address, aligned to the next page boundary.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="address">Start address of the memory range</param>
|
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
|
||||||
/// <param name="size">Size in bytes of the memory range</param>
|
|
||||||
/// <param name="write">Whether the buffer will be written to by this use</param>
|
/// <param name="write">Whether the buffer will be written to by this use</param>
|
||||||
/// <returns>The buffer sub-range starting at the given memory address</returns>
|
/// <returns>The buffer sub-range starting at the given memory address</returns>
|
||||||
public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false)
|
public BufferRange GetBufferRangeAligned(MultiRange range, bool write = false)
|
||||||
{
|
{
|
||||||
return GetBuffer(address, size, write).GetRangeAligned(address, size, write);
|
if (range.Count > 1)
|
||||||
|
{
|
||||||
|
return GetBuffer(range, write).GetRange(range);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(0);
|
||||||
|
return GetBuffer(subRange.Address, subRange.Size, write).GetRangeAligned(subRange.Address, subRange.Size, write);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a buffer sub-range for a given memory range.
|
/// Gets a buffer sub-range for a given memory range.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="address">Start address of the memory range</param>
|
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
|
||||||
/// <param name="size">Size in bytes of the memory range</param>
|
|
||||||
/// <param name="write">Whether the buffer will be written to by this use</param>
|
/// <param name="write">Whether the buffer will be written to by this use</param>
|
||||||
/// <returns>The buffer sub-range for the given range</returns>
|
/// <returns>The buffer sub-range for the given range</returns>
|
||||||
public BufferRange GetBufferRange(ulong address, ulong size, bool write = false)
|
public BufferRange GetBufferRange(MultiRange range, bool write = false)
|
||||||
{
|
{
|
||||||
return GetBuffer(address, size, write).GetRange(address, size, write);
|
if (range.Count > 1)
|
||||||
|
{
|
||||||
|
return GetBuffer(range, write).GetRange(range);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(0);
|
||||||
|
return GetBuffer(subRange.Address, subRange.Size, write).GetRange(subRange.Address, subRange.Size, write);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a buffer for a given memory range.
|
||||||
|
/// A buffer overlapping with the specified range is assumed to already exist on the cache.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
|
||||||
|
/// <param name="write">Whether the buffer will be written to by this use</param>
|
||||||
|
/// <returns>The buffer where the range is fully contained</returns>
|
||||||
|
private MultiRangeBuffer GetBuffer(MultiRange range, bool write = false)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < range.Count; i++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(i);
|
||||||
|
|
||||||
|
Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
|
||||||
|
|
||||||
|
subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
|
||||||
|
|
||||||
|
if (write)
|
||||||
|
{
|
||||||
|
subBuffer.SignalModified(subRange.Address, subRange.Size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
|
||||||
|
|
||||||
|
int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
|
||||||
|
|
||||||
|
MultiRangeBuffer buffer = null;
|
||||||
|
|
||||||
|
for (int i = 0; i < overlapCount; i++)
|
||||||
|
{
|
||||||
|
if (overlaps[i].Range.Contains(range))
|
||||||
|
{
|
||||||
|
buffer = overlaps[i];
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -426,12 +840,33 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
return buffer;
|
return buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Performs guest to host memory synchronization of a given memory range.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
|
||||||
|
public void SynchronizeBufferRange(MultiRange range)
|
||||||
|
{
|
||||||
|
if (range.Count == 1)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(0);
|
||||||
|
SynchronizeBufferRange(subRange.Address, subRange.Size);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
for (int index = 0; index < range.Count; index++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(index);
|
||||||
|
SynchronizeBufferRange(subRange.Address, subRange.Size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs guest to host memory synchronization of a given memory range.
|
/// Performs guest to host memory synchronization of a given memory range.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="address">Start address of the memory range</param>
|
/// <param name="address">Start address of the memory range</param>
|
||||||
/// <param name="size">Size in bytes of the memory range</param>
|
/// <param name="size">Size in bytes of the memory range</param>
|
||||||
public void SynchronizeBufferRange(ulong address, ulong size)
|
private void SynchronizeBufferRange(ulong address, ulong size)
|
||||||
{
|
{
|
||||||
if (size != 0)
|
if (size != 0)
|
||||||
{
|
{
|
||||||
|
@ -491,7 +926,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Disposes all buffers in the cache.
|
/// Disposes all buffers in the cache.
|
||||||
/// It's an error to use the buffer manager after disposal.
|
/// It's an error to use the buffer cache after disposal.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
|
|
|
@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL;
|
||||||
using Ryujinx.Graphics.Gpu.Image;
|
using Ryujinx.Graphics.Gpu.Image;
|
||||||
using Ryujinx.Graphics.Gpu.Shader;
|
using Ryujinx.Graphics.Gpu.Shader;
|
||||||
using Ryujinx.Graphics.Shader;
|
using Ryujinx.Graphics.Shader;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
using System;
|
using System;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
|
@ -62,18 +63,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
Bindings = new BufferDescriptor[count];
|
Bindings = new BufferDescriptor[count];
|
||||||
Buffers = new BufferBounds[count];
|
Buffers = new BufferBounds[count];
|
||||||
Unaligned = new bool[count];
|
Unaligned = new bool[count];
|
||||||
|
|
||||||
|
Buffers.AsSpan().Fill(new BufferBounds(new MultiRange(MemoryManager.PteUnmapped, 0UL)));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Sets the region of a buffer at a given slot.
|
/// Sets the region of a buffer at a given slot.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="index">Buffer slot</param>
|
/// <param name="index">Buffer slot</param>
|
||||||
/// <param name="address">Region virtual address</param>
|
/// <param name="range">Physical memory regions where the buffer is mapped</param>
|
||||||
/// <param name="size">Region size in bytes</param>
|
|
||||||
/// <param name="flags">Buffer usage flags</param>
|
/// <param name="flags">Buffer usage flags</param>
|
||||||
public void SetBounds(int index, ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
|
public void SetBounds(int index, MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
|
||||||
{
|
{
|
||||||
Buffers[index] = new BufferBounds(address, size, flags);
|
Buffers[index] = new BufferBounds(range, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -120,6 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
_context = context;
|
_context = context;
|
||||||
_channel = channel;
|
_channel = channel;
|
||||||
|
|
||||||
|
_indexBuffer.Range = new MultiRange(MemoryManager.PteUnmapped, 0UL);
|
||||||
_vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers];
|
_vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers];
|
||||||
|
|
||||||
_transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers];
|
_transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers];
|
||||||
|
@ -150,10 +153,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="type">Type of each index buffer element</param>
|
/// <param name="type">Type of each index buffer element</param>
|
||||||
public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type)
|
public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type)
|
||||||
{
|
{
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
_indexBuffer.Address = address;
|
_indexBuffer.Range = range;
|
||||||
_indexBuffer.Size = size;
|
|
||||||
_indexBuffer.Type = type;
|
_indexBuffer.Type = type;
|
||||||
|
|
||||||
_indexBufferDirty = true;
|
_indexBufferDirty = true;
|
||||||
|
@ -181,16 +183,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="divisor">Vertex divisor of the buffer, for instanced draws</param>
|
/// <param name="divisor">Vertex divisor of the buffer, for instanced draws</param>
|
||||||
public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor)
|
public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor)
|
||||||
{
|
{
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
_vertexBuffers[index].Address = address;
|
_vertexBuffers[index].Range = range;
|
||||||
_vertexBuffers[index].Size = size;
|
|
||||||
_vertexBuffers[index].Stride = stride;
|
_vertexBuffers[index].Stride = stride;
|
||||||
_vertexBuffers[index].Divisor = divisor;
|
_vertexBuffers[index].Divisor = divisor;
|
||||||
|
|
||||||
_vertexBuffersDirty = true;
|
_vertexBuffersDirty = true;
|
||||||
|
|
||||||
if (address != 0)
|
if (!range.IsUnmapped)
|
||||||
{
|
{
|
||||||
_vertexBuffersEnableMask |= 1u << index;
|
_vertexBuffersEnableMask |= 1u << index;
|
||||||
}
|
}
|
||||||
|
@ -209,9 +210,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="size">Size in bytes of the transform feedback buffer</param>
|
/// <param name="size">Size in bytes of the transform feedback buffer</param>
|
||||||
public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size)
|
public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size)
|
||||||
{
|
{
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
_transformFeedbackBuffers[index] = new BufferBounds(address, size);
|
_transformFeedbackBuffers[index] = new BufferBounds(range);
|
||||||
_transformFeedbackBuffersDirty = true;
|
_transformFeedbackBuffersDirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -256,9 +257,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
|
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
|
||||||
|
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
_cpStorageBuffers.SetBounds(index, address, size, flags);
|
_cpStorageBuffers.SetBounds(index, range, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -280,15 +281,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
|
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
|
||||||
|
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
if (buffers.Buffers[index].Address != address ||
|
if (!buffers.Buffers[index].Range.Equals(range))
|
||||||
buffers.Buffers[index].Size != size)
|
|
||||||
{
|
{
|
||||||
_gpStorageBuffersDirty = true;
|
_gpStorageBuffersDirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
buffers.SetBounds(index, address, size, flags);
|
buffers.SetBounds(index, range, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -300,9 +300,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="size">Size in bytes of the storage buffer</param>
|
/// <param name="size">Size in bytes of the storage buffer</param>
|
||||||
public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size)
|
public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size)
|
||||||
{
|
{
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
_cpUniformBuffers.SetBounds(index, address, size);
|
_cpUniformBuffers.SetBounds(index, range);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -315,9 +315,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <param name="size">Size in bytes of the storage buffer</param>
|
/// <param name="size">Size in bytes of the storage buffer</param>
|
||||||
public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size)
|
public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size)
|
||||||
{
|
{
|
||||||
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
|
||||||
|
|
||||||
_gpUniformBuffers[stage].SetBounds(index, address, size);
|
_gpUniformBuffers[stage].SetBounds(index, range);
|
||||||
_gpUniformBuffersDirty = true;
|
_gpUniformBuffersDirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -379,7 +379,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++)
|
for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++)
|
||||||
{
|
{
|
||||||
if (_cpUniformBuffers.Buffers[i].Address != 0)
|
if (!_cpUniformBuffers.Buffers[i].IsUnmapped)
|
||||||
{
|
{
|
||||||
mask |= 1u << i;
|
mask |= 1u << i;
|
||||||
}
|
}
|
||||||
|
@ -399,7 +399,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++)
|
for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++)
|
||||||
{
|
{
|
||||||
if (_gpUniformBuffers[stage].Buffers[i].Address != 0)
|
if (!_gpUniformBuffers[stage].Buffers[i].IsUnmapped)
|
||||||
{
|
{
|
||||||
mask |= 1u << i;
|
mask |= 1u << i;
|
||||||
}
|
}
|
||||||
|
@ -415,7 +415,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
|
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
|
||||||
public ulong GetComputeUniformBufferAddress(int index)
|
public ulong GetComputeUniformBufferAddress(int index)
|
||||||
{
|
{
|
||||||
return _cpUniformBuffers.Buffers[index].Address;
|
return _cpUniformBuffers.Buffers[index].Range.GetSubRange(0).Address;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -426,7 +426,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
|
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
|
||||||
public ulong GetGraphicsUniformBufferAddress(int stage, int index)
|
public ulong GetGraphicsUniformBufferAddress(int stage, int index)
|
||||||
{
|
{
|
||||||
return _gpUniformBuffers[stage].Buffers[index].Address;
|
return _gpUniformBuffers[stage].Buffers[index].Range.GetSubRange(0).Address;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -477,7 +477,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
foreach (var binding in _bufferTextures)
|
foreach (var binding in _bufferTextures)
|
||||||
{
|
{
|
||||||
var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore);
|
var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore);
|
||||||
var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Address, binding.Size, isStore);
|
var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Range, isStore);
|
||||||
binding.Texture.SetStorage(range);
|
binding.Texture.SetStorage(range);
|
||||||
|
|
||||||
// The texture must be rebound to use the new storage if it was updated.
|
// The texture must be rebound to use the new storage if it was updated.
|
||||||
|
@ -511,16 +511,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
_indexBufferDirty = false;
|
_indexBufferDirty = false;
|
||||||
|
|
||||||
if (_indexBuffer.Address != 0)
|
if (!_indexBuffer.Range.IsUnmapped)
|
||||||
{
|
{
|
||||||
BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Address, _indexBuffer.Size);
|
BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Range);
|
||||||
|
|
||||||
_context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type);
|
_context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (_indexBuffer.Address != 0)
|
else if (!_indexBuffer.Range.IsUnmapped)
|
||||||
{
|
{
|
||||||
bufferCache.SynchronizeBufferRange(_indexBuffer.Address, _indexBuffer.Size);
|
bufferCache.SynchronizeBufferRange(_indexBuffer.Range);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (_rebind)
|
else if (_rebind)
|
||||||
|
@ -540,12 +540,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
VertexBuffer vb = _vertexBuffers[index];
|
VertexBuffer vb = _vertexBuffers[index];
|
||||||
|
|
||||||
if (vb.Address == 0)
|
if (vb.Range.IsUnmapped)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
BufferRange buffer = bufferCache.GetBufferRange(vb.Address, vb.Size);
|
BufferRange buffer = bufferCache.GetBufferRange(vb.Range);
|
||||||
|
|
||||||
vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor);
|
vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor);
|
||||||
}
|
}
|
||||||
|
@ -558,12 +558,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
VertexBuffer vb = _vertexBuffers[index];
|
VertexBuffer vb = _vertexBuffers[index];
|
||||||
|
|
||||||
if (vb.Address == 0)
|
if (vb.Range.IsUnmapped)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
bufferCache.SynchronizeBufferRange(vb.Address, vb.Size);
|
bufferCache.SynchronizeBufferRange(vb.Range);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -579,13 +579,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
BufferBounds tfb = _transformFeedbackBuffers[index];
|
BufferBounds tfb = _transformFeedbackBuffers[index];
|
||||||
|
|
||||||
if (tfb.Address == 0)
|
if (tfb.IsUnmapped)
|
||||||
{
|
{
|
||||||
tfbs[index] = BufferRange.Empty;
|
tfbs[index] = BufferRange.Empty;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
tfbs[index] = bufferCache.GetBufferRange(tfb.Address, tfb.Size, write: true);
|
tfbs[index] = bufferCache.GetBufferRange(tfb.Range, write: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
_context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs);
|
_context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs);
|
||||||
|
@ -600,21 +600,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
BufferBounds tfb = _transformFeedbackBuffers[index];
|
BufferBounds tfb = _transformFeedbackBuffers[index];
|
||||||
|
|
||||||
if (tfb.Address == 0)
|
if (tfb.IsUnmapped)
|
||||||
{
|
{
|
||||||
buffers[index] = new BufferAssignment(index, BufferRange.Empty);
|
buffers[index] = new BufferAssignment(index, BufferRange.Empty);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
ulong endAddress = tfb.Address + tfb.Size;
|
MultiRange range = tfb.Range;
|
||||||
ulong address = BitUtils.AlignDown(tfb.Address, (ulong)alignment);
|
ulong address0 = range.GetSubRange(0).Address;
|
||||||
ulong size = endAddress - address;
|
ulong address = BitUtils.AlignDown(address0, (ulong)alignment);
|
||||||
|
|
||||||
int tfeOffset = ((int)tfb.Address & (alignment - 1)) / 4;
|
if (range.Count == 1)
|
||||||
|
{
|
||||||
|
range = new MultiRange(address, range.GetSubRange(0).Size + (address0 - address));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
MemoryRange[] subRanges = new MemoryRange[range.Count];
|
||||||
|
|
||||||
|
subRanges[0] = new MemoryRange(address, range.GetSubRange(0).Size + (address0 - address));
|
||||||
|
|
||||||
|
for (int i = 1; i < range.Count; i++)
|
||||||
|
{
|
||||||
|
subRanges[i] = range.GetSubRange(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
range = new MultiRange(subRanges);
|
||||||
|
}
|
||||||
|
|
||||||
|
int tfeOffset = ((int)address0 & (alignment - 1)) / 4;
|
||||||
|
|
||||||
_context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset);
|
_context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset);
|
||||||
|
|
||||||
buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(address, size, write: true));
|
buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(range, write: true));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -627,12 +645,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
BufferBounds tfb = _transformFeedbackBuffers[index];
|
BufferBounds tfb = _transformFeedbackBuffers[index];
|
||||||
|
|
||||||
if (tfb.Address == 0)
|
if (tfb.IsUnmapped)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
bufferCache.SynchronizeBufferRange(tfb.Address, tfb.Size);
|
bufferCache.SynchronizeBufferRange(tfb.Range);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -688,12 +706,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
|
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
|
||||||
|
|
||||||
if (bounds.Address != 0)
|
if (!bounds.IsUnmapped)
|
||||||
{
|
{
|
||||||
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
|
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
|
||||||
var range = isStorage
|
var range = isStorage
|
||||||
? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
|
? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
|
||||||
: bufferCache.GetBufferRange(bounds.Address, bounds.Size);
|
: bufferCache.GetBufferRange(bounds.Range);
|
||||||
|
|
||||||
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
|
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
|
||||||
}
|
}
|
||||||
|
@ -725,12 +743,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
|
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
|
||||||
|
|
||||||
if (bounds.Address != 0)
|
if (!bounds.IsUnmapped)
|
||||||
{
|
{
|
||||||
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
|
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
|
||||||
var range = isStorage
|
var range = isStorage
|
||||||
? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
|
? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
|
||||||
: bufferCache.GetBufferRange(bounds.Address, bounds.Size);
|
: bufferCache.GetBufferRange(bounds.Range);
|
||||||
|
|
||||||
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
|
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
|
||||||
}
|
}
|
||||||
|
@ -778,12 +796,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
|
||||||
BufferBounds bounds = buffers.Buffers[binding.Slot];
|
BufferBounds bounds = buffers.Buffers[binding.Slot];
|
||||||
|
|
||||||
if (bounds.Address == 0)
|
if (bounds.IsUnmapped)
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
_channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Address, bounds.Size);
|
_channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Range);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -793,23 +811,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="stage">Shader stage accessing the texture</param>
|
/// <param name="stage">Shader stage accessing the texture</param>
|
||||||
/// <param name="texture">Buffer texture</param>
|
/// <param name="texture">Buffer texture</param>
|
||||||
/// <param name="address">Address of the buffer in memory</param>
|
/// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
|
||||||
/// <param name="size">Size of the buffer in bytes</param>
|
|
||||||
/// <param name="bindingInfo">Binding info for the buffer texture</param>
|
/// <param name="bindingInfo">Binding info for the buffer texture</param>
|
||||||
/// <param name="format">Format of the buffer texture</param>
|
/// <param name="format">Format of the buffer texture</param>
|
||||||
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
|
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
|
||||||
public void SetBufferTextureStorage(
|
public void SetBufferTextureStorage(
|
||||||
ShaderStage stage,
|
ShaderStage stage,
|
||||||
ITexture texture,
|
ITexture texture,
|
||||||
ulong address,
|
MultiRange range,
|
||||||
ulong size,
|
|
||||||
TextureBindingInfo bindingInfo,
|
TextureBindingInfo bindingInfo,
|
||||||
Format format,
|
Format format,
|
||||||
bool isImage)
|
bool isImage)
|
||||||
{
|
{
|
||||||
_channel.MemoryManager.Physical.BufferCache.CreateBuffer(address, size);
|
_channel.MemoryManager.Physical.BufferCache.CreateBuffer(range);
|
||||||
|
|
||||||
_bufferTextures.Add(new BufferTextureBinding(stage, texture, address, size, bindingInfo, format, isImage));
|
_bufferTextures.Add(new BufferTextureBinding(stage, texture, range, bindingInfo, format, isImage));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
using Ryujinx.Graphics.GAL;
|
using Ryujinx.Graphics.GAL;
|
||||||
using Ryujinx.Graphics.Gpu.Image;
|
using Ryujinx.Graphics.Gpu.Image;
|
||||||
using Ryujinx.Graphics.Shader;
|
using Ryujinx.Graphics.Shader;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
|
|
||||||
namespace Ryujinx.Graphics.Gpu.Memory
|
namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
|
@ -20,14 +21,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
public ITexture Texture { get; }
|
public ITexture Texture { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The base address of the buffer binding.
|
/// Physical ranges of memory where the buffer texture data is located.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ulong Address { get; }
|
public MultiRange Range { get; }
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// The size of the buffer binding in bytes.
|
|
||||||
/// </summary>
|
|
||||||
public ulong Size { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The image or sampler binding info for the buffer texture.
|
/// The image or sampler binding info for the buffer texture.
|
||||||
|
@ -49,24 +45,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="stage">Shader stage accessing the texture</param>
|
/// <param name="stage">Shader stage accessing the texture</param>
|
||||||
/// <param name="texture">Buffer texture</param>
|
/// <param name="texture">Buffer texture</param>
|
||||||
/// <param name="address">Base address</param>
|
/// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
|
||||||
/// <param name="size">Size in bytes</param>
|
|
||||||
/// <param name="bindingInfo">Binding info</param>
|
/// <param name="bindingInfo">Binding info</param>
|
||||||
/// <param name="format">Binding format</param>
|
/// <param name="format">Binding format</param>
|
||||||
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
|
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
|
||||||
public BufferTextureBinding(
|
public BufferTextureBinding(
|
||||||
ShaderStage stage,
|
ShaderStage stage,
|
||||||
ITexture texture,
|
ITexture texture,
|
||||||
ulong address,
|
MultiRange range,
|
||||||
ulong size,
|
|
||||||
TextureBindingInfo bindingInfo,
|
TextureBindingInfo bindingInfo,
|
||||||
Format format,
|
Format format,
|
||||||
bool isImage)
|
bool isImage)
|
||||||
{
|
{
|
||||||
Stage = stage;
|
Stage = stage;
|
||||||
Texture = texture;
|
Texture = texture;
|
||||||
Address = address;
|
Range = range;
|
||||||
Size = size;
|
|
||||||
BindingInfo = bindingInfo;
|
BindingInfo = bindingInfo;
|
||||||
Format = format;
|
Format = format;
|
||||||
IsImage = isImage;
|
IsImage = isImage;
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
using Ryujinx.Graphics.GAL;
|
using Ryujinx.Graphics.GAL;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
|
|
||||||
namespace Ryujinx.Graphics.Gpu.Memory
|
namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
|
@ -7,9 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
struct IndexBuffer
|
struct IndexBuffer
|
||||||
{
|
{
|
||||||
public ulong Address;
|
public MultiRange Range;
|
||||||
public ulong Size;
|
|
||||||
|
|
||||||
public IndexType Type;
|
public IndexType Type;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,6 +39,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal PhysicalMemory Physical { get; }
|
internal PhysicalMemory Physical { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Virtual buffer cache.
|
||||||
|
/// </summary>
|
||||||
|
internal VirtualBufferCache VirtualBufferCache { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Cache of GPU counters.
|
/// Cache of GPU counters.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
@ -51,10 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
internal MemoryManager(PhysicalMemory physicalMemory)
|
internal MemoryManager(PhysicalMemory physicalMemory)
|
||||||
{
|
{
|
||||||
Physical = physicalMemory;
|
Physical = physicalMemory;
|
||||||
|
VirtualBufferCache = new VirtualBufferCache(this);
|
||||||
CounterCache = new CounterCache();
|
CounterCache = new CounterCache();
|
||||||
_pageTable = new ulong[PtLvl0Size][];
|
_pageTable = new ulong[PtLvl0Size][];
|
||||||
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
|
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
|
||||||
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
|
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
|
||||||
|
MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler;
|
||||||
MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
|
MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -508,6 +515,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
regionSize += Math.Min(endVa - va, PageSize);
|
regionSize += Math.Min(endVa - va, PageSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (regions.Count == 0)
|
||||||
|
{
|
||||||
|
return new MultiRange(regionStart, regionSize);
|
||||||
|
}
|
||||||
|
|
||||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||||
|
|
||||||
return new MultiRange(regions.ToArray());
|
return new MultiRange(regions.ToArray());
|
||||||
|
|
60
src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
Normal file
60
src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
Normal file
|
@ -0,0 +1,60 @@
|
||||||
|
using Ryujinx.Graphics.GAL;
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
|
using System;
|
||||||
|
|
||||||
|
namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Buffer, used to store vertex and index data, uniform and storage buffers, and others.
|
||||||
|
/// </summary>
|
||||||
|
class MultiRangeBuffer : IMultiRangeItem, IDisposable
|
||||||
|
{
|
||||||
|
private readonly GpuContext _context;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Host buffer handle.
|
||||||
|
/// </summary>
|
||||||
|
public BufferHandle Handle { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Range of memory where the data is located.
|
||||||
|
/// </summary>
|
||||||
|
public MultiRange Range { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new instance of the buffer.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="context">GPU context that the buffer belongs to</param>
|
||||||
|
/// <param name="range">Range of memory where the data is mapped</param>
|
||||||
|
/// <param name="storages">Backing memory for the buffers</param>
|
||||||
|
public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
|
||||||
|
{
|
||||||
|
_context = context;
|
||||||
|
Range = range;
|
||||||
|
Handle = context.Renderer.CreateBufferSparse(storages);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a sub-range from the buffer.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// This can be used to bind and use sub-ranges of the buffer on the host API.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="range">Range of memory where the data is mapped</param>
|
||||||
|
/// <returns>The buffer sub-range</returns>
|
||||||
|
public BufferRange GetRange(MultiRange range)
|
||||||
|
{
|
||||||
|
int offset = Range.FindOffset(range);
|
||||||
|
|
||||||
|
return new BufferRange(Handle, offset, (int)range.GetSize());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Disposes the host buffer.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
_context.Renderer.DeleteBuffer(Handle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,3 +1,5 @@
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
|
|
||||||
namespace Ryujinx.Graphics.Gpu.Memory
|
namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -5,8 +7,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
struct VertexBuffer
|
struct VertexBuffer
|
||||||
{
|
{
|
||||||
public ulong Address;
|
public MultiRange Range;
|
||||||
public ulong Size;
|
|
||||||
public int Stride;
|
public int Stride;
|
||||||
public int Divisor;
|
public int Divisor;
|
||||||
}
|
}
|
||||||
|
|
238
src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs
Normal file
238
src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs
Normal file
|
@ -0,0 +1,238 @@
|
||||||
|
using Ryujinx.Memory.Range;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Threading;
|
||||||
|
|
||||||
|
namespace Ryujinx.Graphics.Gpu.Memory
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Virtual buffer cache.
|
||||||
|
/// </summary>
|
||||||
|
class VirtualBufferCache
|
||||||
|
{
|
||||||
|
private readonly MemoryManager _memoryManager;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Represents a GPU virtual memory range.
|
||||||
|
/// </summary>
|
||||||
|
private readonly struct VirtualRange : IRange
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// GPU virtual address where the range starts.
|
||||||
|
/// </summary>
|
||||||
|
public ulong Address { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Size of the range in bytes.
|
||||||
|
/// </summary>
|
||||||
|
public ulong Size { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// GPU virtual address where the range ends.
|
||||||
|
/// </summary>
|
||||||
|
public ulong EndAddress => Address + Size;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Physical regions where the GPU virtual region is mapped.
|
||||||
|
/// </summary>
|
||||||
|
public MultiRange Range { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new virtual memory range.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="address">GPU virtual address where the range starts</param>
|
||||||
|
/// <param name="size">Size of the range in bytes</param>
|
||||||
|
/// <param name="range">Physical regions where the GPU virtual region is mapped</param>
|
||||||
|
public VirtualRange(ulong address, ulong size, MultiRange range)
|
||||||
|
{
|
||||||
|
Address = address;
|
||||||
|
Size = size;
|
||||||
|
Range = range;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks if a given range overlaps with the buffer.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="address">Start address of the range</param>
|
||||||
|
/// <param name="size">Size in bytes of the range</param>
|
||||||
|
/// <returns>True if the range overlaps, false otherwise</returns>
|
||||||
|
public bool OverlapsWith(ulong address, ulong size)
|
||||||
|
{
|
||||||
|
return Address < address + size && address < EndAddress;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private readonly RangeList<VirtualRange> _virtualRanges;
|
||||||
|
private VirtualRange[] _virtualRangeOverlaps;
|
||||||
|
private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps;
|
||||||
|
private int _hasDeferredUnmaps;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a new instance of the virtual buffer cache.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param>
|
||||||
|
public VirtualBufferCache(MemoryManager memoryManager)
|
||||||
|
{
|
||||||
|
_memoryManager = memoryManager;
|
||||||
|
_virtualRanges = new RangeList<VirtualRange>();
|
||||||
|
_virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity];
|
||||||
|
_deferredUnmaps = new ConcurrentQueue<VirtualRange>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Handles removal of buffers written to a memory region being unmapped.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="sender">Sender object</param>
|
||||||
|
/// <param name="e">Event arguments</param>
|
||||||
|
public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
|
||||||
|
{
|
||||||
|
void EnqueueUnmap()
|
||||||
|
{
|
||||||
|
_deferredUnmaps.Enqueue(new VirtualRange(e.Address, e.Size, default));
|
||||||
|
|
||||||
|
Interlocked.Exchange(ref _hasDeferredUnmaps, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
e.AddRemapAction(EnqueueUnmap);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Tries to get a existing, cached physical range for the specified virtual region.
|
||||||
|
/// If no cached range is found, a new one is created and added.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="gpuVa">GPU virtual address to get the physical range from</param>
|
||||||
|
/// <param name="size">Size in bytes of the region</param>
|
||||||
|
/// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
|
||||||
|
/// <param name="range">Physical range for the specified GPU virtual region</param>
|
||||||
|
/// <returns>True if the range already existed, false if a new one was created and added</returns>
|
||||||
|
public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range)
|
||||||
|
{
|
||||||
|
VirtualRange[] overlaps = _virtualRangeOverlaps;
|
||||||
|
int overlapsCount;
|
||||||
|
|
||||||
|
if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0)
|
||||||
|
{
|
||||||
|
while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange))
|
||||||
|
{
|
||||||
|
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps);
|
||||||
|
|
||||||
|
for (int index = 0; index < overlapsCount; index++)
|
||||||
|
{
|
||||||
|
_virtualRanges.Remove(overlaps[index]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool found = false;
|
||||||
|
|
||||||
|
ulong originalVa = gpuVa;
|
||||||
|
|
||||||
|
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps);
|
||||||
|
|
||||||
|
if (overlapsCount != 0)
|
||||||
|
{
|
||||||
|
// The virtual range already exists. We just need to check if our range fits inside
|
||||||
|
// the existing one, and if not, we must extend the existing one.
|
||||||
|
|
||||||
|
ulong endAddress = gpuVa + size;
|
||||||
|
VirtualRange overlap0 = overlaps[0];
|
||||||
|
|
||||||
|
if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress)
|
||||||
|
{
|
||||||
|
for (int index = 0; index < overlapsCount; index++)
|
||||||
|
{
|
||||||
|
VirtualRange virtualRange = overlaps[index];
|
||||||
|
|
||||||
|
gpuVa = Math.Min(gpuVa, virtualRange.Address);
|
||||||
|
endAddress = Math.Max(endAddress, virtualRange.EndAddress);
|
||||||
|
|
||||||
|
_virtualRanges.Remove(virtualRange);
|
||||||
|
}
|
||||||
|
|
||||||
|
ulong newSize = endAddress - gpuVa;
|
||||||
|
MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize);
|
||||||
|
|
||||||
|
_virtualRanges.Add(new(gpuVa, newSize, newRange));
|
||||||
|
|
||||||
|
range = newRange.Slice(originalVa - gpuVa, size);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
found = true;
|
||||||
|
range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// No overlap, just create a new virtual range.
|
||||||
|
range = _memoryManager.GetPhysicalRegions(gpuVa, size);
|
||||||
|
|
||||||
|
VirtualRange virtualRange = new(gpuVa, size, range);
|
||||||
|
|
||||||
|
_virtualRanges.Add(virtualRange);
|
||||||
|
}
|
||||||
|
|
||||||
|
ShrinkOverlapsBufferIfNeeded();
|
||||||
|
|
||||||
|
// If the the range is not properly aligned for sparse mapping,
|
||||||
|
// or if the host does not support sparse mapping, let's just
|
||||||
|
// force it to a single range.
|
||||||
|
// This might cause issues in some applications that uses sparse
|
||||||
|
// mappings.
|
||||||
|
if (!IsSparseAligned(range) || !supportsSparse)
|
||||||
|
{
|
||||||
|
range = new MultiRange(range.GetSubRange(0).Address, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Checks if the physical memory ranges are valid for sparse mapping,
|
||||||
|
/// which requires all sub-ranges to be 64KB aligned.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="range">Range to check</param>
|
||||||
|
/// <returns>True if the range is valid for sparse mapping, false otherwise</returns>
|
||||||
|
private static bool IsSparseAligned(MultiRange range)
|
||||||
|
{
|
||||||
|
if (range.Count == 1)
|
||||||
|
{
|
||||||
|
return (range.GetSubRange(0).Address & (BufferCache.SparseBufferAlignmentSize - 1)) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < range.Count; i++)
|
||||||
|
{
|
||||||
|
MemoryRange subRange = range.GetSubRange(i);
|
||||||
|
|
||||||
|
// Check if address is aligned. The address of the first sub-range can
|
||||||
|
// be misaligned as it is at the start.
|
||||||
|
if (i > 0 &&
|
||||||
|
subRange.Address != MemoryManager.PteUnmapped &&
|
||||||
|
(subRange.Address & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the size is aligned. The size of the last sub-range can
|
||||||
|
// be misaligned as it is at the end.
|
||||||
|
if (i < range.Count - 1 && (subRange.Size & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
|
||||||
|
/// </summary>
|
||||||
|
private void ShrinkOverlapsBufferIfNeeded()
|
||||||
|
{
|
||||||
|
if (_virtualRangeOverlaps.Length > BufferCache.OverlapsBufferMaxCapacity)
|
||||||
|
{
|
||||||
|
Array.Resize(ref _virtualRangeOverlaps, BufferCache.OverlapsBufferMaxCapacity);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -730,8 +730,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
|
||||||
|
|
||||||
codeA ??= memoryManager.GetSpan(vertexA.Address, vertexA.Size).ToArray();
|
codeA ??= memoryManager.GetSpan(vertexA.Address, vertexA.Size).ToArray();
|
||||||
codeB ??= memoryManager.GetSpan(currentStage.Address, currentStage.Size).ToArray();
|
codeB ??= memoryManager.GetSpan(currentStage.Address, currentStage.Size).ToArray();
|
||||||
byte[] cb1DataA = memoryManager.Physical.GetSpan(cb1DataAddress, vertexA.Cb1DataSize).ToArray();
|
byte[] cb1DataA = ReadArray(memoryManager, cb1DataAddress, vertexA.Cb1DataSize);
|
||||||
byte[] cb1DataB = memoryManager.Physical.GetSpan(cb1DataAddress, currentStage.Cb1DataSize).ToArray();
|
byte[] cb1DataB = ReadArray(memoryManager, cb1DataAddress, currentStage.Cb1DataSize);
|
||||||
|
|
||||||
ShaderDumpPaths pathsA = default;
|
ShaderDumpPaths pathsA = default;
|
||||||
ShaderDumpPaths pathsB = default;
|
ShaderDumpPaths pathsB = default;
|
||||||
|
@ -770,7 +770,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
|
||||||
? channel.BufferManager.GetComputeUniformBufferAddress(1)
|
? channel.BufferManager.GetComputeUniformBufferAddress(1)
|
||||||
: channel.BufferManager.GetGraphicsUniformBufferAddress(StageToStageIndex(context.Stage), 1);
|
: channel.BufferManager.GetGraphicsUniformBufferAddress(StageToStageIndex(context.Stage), 1);
|
||||||
|
|
||||||
byte[] cb1Data = memoryManager.Physical.GetSpan(cb1DataAddress, context.Cb1DataSize).ToArray();
|
byte[] cb1Data = ReadArray(memoryManager, cb1DataAddress, context.Cb1DataSize);
|
||||||
code ??= memoryManager.GetSpan(context.Address, context.Size).ToArray();
|
code ??= memoryManager.GetSpan(context.Address, context.Size).ToArray();
|
||||||
|
|
||||||
ShaderDumpPaths paths = dumper?.Dump(code, context.Stage == ShaderStage.Compute) ?? default;
|
ShaderDumpPaths paths = dumper?.Dump(code, context.Stage == ShaderStage.Compute) ?? default;
|
||||||
|
@ -781,6 +781,23 @@ namespace Ryujinx.Graphics.Gpu.Shader
|
||||||
return new TranslatedShader(new CachedShaderStage(program.Info, code, cb1Data), program);
|
return new TranslatedShader(new CachedShaderStage(program.Info, code, cb1Data), program);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads data from physical memory, returns an empty array if the memory is unmapped or size is 0.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="memoryManager">Memory manager with the physical memory to read from</param>
|
||||||
|
/// <param name="address">Physical address of the region to read</param>
|
||||||
|
/// <param name="size">Size in bytes of the data</param>
|
||||||
|
/// <returns>An array with the data at the specified memory location</returns>
|
||||||
|
private static byte[] ReadArray(MemoryManager memoryManager, ulong address, int size)
|
||||||
|
{
|
||||||
|
if (address == MemoryManager.PteUnmapped || size == 0)
|
||||||
|
{
|
||||||
|
return Array.Empty<byte>();
|
||||||
|
}
|
||||||
|
|
||||||
|
return memoryManager.Physical.GetSpan(address, size).ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the index of a stage from a <see cref="ShaderStage"/>.
|
/// Gets the index of a stage from a <see cref="ShaderStage"/>.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|
|
@ -611,7 +611,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
|
||||||
{
|
{
|
||||||
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex);
|
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex);
|
||||||
|
|
||||||
cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
|
cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Range));
|
||||||
cachedTextureBufferIndex = textureBufferIndex;
|
cachedTextureBufferIndex = textureBufferIndex;
|
||||||
|
|
||||||
if (samplerBufferIndex == textureBufferIndex)
|
if (samplerBufferIndex == textureBufferIndex)
|
||||||
|
@ -625,7 +625,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
|
||||||
{
|
{
|
||||||
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex);
|
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex);
|
||||||
|
|
||||||
cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
|
cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Range));
|
||||||
cachedSamplerBufferIndex = samplerBufferIndex;
|
cachedSamplerBufferIndex = samplerBufferIndex;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -57,16 +57,11 @@ namespace Ryujinx.Graphics.OpenGL
|
||||||
ResourcePool = new ResourcePool();
|
ResourcePool = new ResourcePool();
|
||||||
}
|
}
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
|
|
||||||
{
|
|
||||||
return CreateBuffer(size, GAL.BufferAccess.Default);
|
|
||||||
}
|
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access)
|
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access)
|
||||||
{
|
{
|
||||||
BufferCount++;
|
BufferCount++;
|
||||||
|
|
||||||
if (access == GAL.BufferAccess.FlushPersistent)
|
if (access.HasFlag(GAL.BufferAccess.FlushPersistent))
|
||||||
{
|
{
|
||||||
BufferHandle handle = Buffer.CreatePersistent(size);
|
BufferHandle handle = Buffer.CreatePersistent(size);
|
||||||
|
|
||||||
|
@ -80,11 +75,21 @@ namespace Ryujinx.Graphics.OpenGL
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access, BufferHandle storageHint)
|
||||||
|
{
|
||||||
|
return CreateBuffer(size, access);
|
||||||
|
}
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(nint pointer, int size)
|
public BufferHandle CreateBuffer(nint pointer, int size)
|
||||||
{
|
{
|
||||||
throw new NotSupportedException();
|
throw new NotSupportedException();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
|
||||||
|
{
|
||||||
|
throw new NotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info)
|
public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info)
|
||||||
{
|
{
|
||||||
return new Program(shaders, info.FragmentOutputMap);
|
return new Program(shaders, info.FragmentOutputMap);
|
||||||
|
@ -148,6 +153,7 @@ namespace Ryujinx.Graphics.OpenGL
|
||||||
supportsR4G4B4A4Format: true,
|
supportsR4G4B4A4Format: true,
|
||||||
supportsSnormBufferTextureFormat: false,
|
supportsSnormBufferTextureFormat: false,
|
||||||
supports5BitComponentFormat: true,
|
supports5BitComponentFormat: true,
|
||||||
|
supportsSparseBuffer: false,
|
||||||
supportsBlendEquationAdvanced: HwCapabilities.SupportsBlendEquationAdvanced,
|
supportsBlendEquationAdvanced: HwCapabilities.SupportsBlendEquationAdvanced,
|
||||||
supportsFragmentShaderInterlock: HwCapabilities.SupportsFragmentShaderInterlock,
|
supportsFragmentShaderInterlock: HwCapabilities.SupportsFragmentShaderInterlock,
|
||||||
supportsFragmentShaderOrderingIntel: HwCapabilities.SupportsFragmentShaderOrdering,
|
supportsFragmentShaderOrderingIntel: HwCapabilities.SupportsFragmentShaderOrdering,
|
||||||
|
|
|
@ -8,5 +8,6 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
HostMapped,
|
HostMapped,
|
||||||
DeviceLocal,
|
DeviceLocal,
|
||||||
DeviceLocalMapped,
|
DeviceLocalMapped,
|
||||||
|
Sparse,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,7 +46,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
|
|
||||||
private bool _lastAccessIsWrite;
|
private bool _lastAccessIsWrite;
|
||||||
|
|
||||||
private readonly BufferAllocationType _baseType;
|
private BufferAllocationType _baseType;
|
||||||
private BufferAllocationType _currentType;
|
private BufferAllocationType _currentType;
|
||||||
private bool _swapQueued;
|
private bool _swapQueued;
|
||||||
|
|
||||||
|
@ -109,6 +109,22 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
_flushLock = new ReaderWriterLockSlim();
|
_flushLock = new ReaderWriterLockSlim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, int size, Auto<MemoryAllocation>[] storageAllocations)
|
||||||
|
{
|
||||||
|
_gd = gd;
|
||||||
|
_device = device;
|
||||||
|
_waitable = new MultiFenceHolder(size);
|
||||||
|
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, storageAllocations);
|
||||||
|
_bufferHandle = buffer.Handle;
|
||||||
|
Size = size;
|
||||||
|
|
||||||
|
_baseType = BufferAllocationType.Sparse;
|
||||||
|
_currentType = BufferAllocationType.Sparse;
|
||||||
|
DesiredType = BufferAllocationType.Sparse;
|
||||||
|
|
||||||
|
_flushLock = new ReaderWriterLockSlim();
|
||||||
|
}
|
||||||
|
|
||||||
public bool TryBackingSwap(ref CommandBufferScoped? cbs)
|
public bool TryBackingSwap(ref CommandBufferScoped? cbs)
|
||||||
{
|
{
|
||||||
if (_swapQueued && DesiredType != _currentType)
|
if (_swapQueued && DesiredType != _currentType)
|
||||||
|
@ -122,7 +138,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
var currentBuffer = _buffer;
|
var currentBuffer = _buffer;
|
||||||
IntPtr currentMap = _map;
|
IntPtr currentMap = _map;
|
||||||
|
|
||||||
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, _currentType);
|
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, false, _currentType);
|
||||||
|
|
||||||
if (buffer.Handle != 0)
|
if (buffer.Handle != 0)
|
||||||
{
|
{
|
||||||
|
@ -253,6 +269,14 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void Pin()
|
||||||
|
{
|
||||||
|
if (_baseType == BufferAllocationType.Auto)
|
||||||
|
{
|
||||||
|
_baseType = _currentType;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView)
|
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView)
|
||||||
{
|
{
|
||||||
var bufferViewCreateInfo = new BufferViewCreateInfo
|
var bufferViewCreateInfo = new BufferViewCreateInfo
|
||||||
|
@ -506,6 +530,16 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Auto<MemoryAllocation> GetAllocation()
|
||||||
|
{
|
||||||
|
return _allocationAuto;
|
||||||
|
}
|
||||||
|
|
||||||
|
public (DeviceMemory, ulong) GetDeviceMemoryAndOffset()
|
||||||
|
{
|
||||||
|
return (_allocation.Memory, _allocation.Offset);
|
||||||
|
}
|
||||||
|
|
||||||
public void SignalWrite(int offset, int size)
|
public void SignalWrite(int offset, int size)
|
||||||
{
|
{
|
||||||
ConsiderBackingSwap();
|
ConsiderBackingSwap();
|
||||||
|
@ -1072,7 +1106,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_allocationAuto.Dispose();
|
_allocationAuto?.Dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
_flushLock.EnterWriteLock();
|
_flushLock.EnterWriteLock();
|
||||||
|
|
|
@ -96,25 +96,131 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
return Unsafe.As<ulong, BufferHandle>(ref handle64);
|
return Unsafe.As<ulong, BufferHandle>(ref handle64);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public unsafe BufferHandle CreateSparse(VulkanRenderer gd, ReadOnlySpan<BufferRange> storageBuffers)
|
||||||
|
{
|
||||||
|
var usage = DefaultBufferUsageFlags;
|
||||||
|
|
||||||
|
if (gd.Capabilities.SupportsIndirectParameters)
|
||||||
|
{
|
||||||
|
usage |= BufferUsageFlags.IndirectBufferBit;
|
||||||
|
}
|
||||||
|
|
||||||
|
ulong size = 0;
|
||||||
|
|
||||||
|
foreach (BufferRange range in storageBuffers)
|
||||||
|
{
|
||||||
|
size += (ulong)range.Size;
|
||||||
|
}
|
||||||
|
|
||||||
|
var bufferCreateInfo = new BufferCreateInfo()
|
||||||
|
{
|
||||||
|
SType = StructureType.BufferCreateInfo,
|
||||||
|
Size = size,
|
||||||
|
Usage = usage,
|
||||||
|
SharingMode = SharingMode.Exclusive,
|
||||||
|
Flags = BufferCreateFlags.SparseBindingBit | BufferCreateFlags.SparseAliasedBit
|
||||||
|
};
|
||||||
|
|
||||||
|
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
|
||||||
|
|
||||||
|
var memoryBinds = new SparseMemoryBind[storageBuffers.Length];
|
||||||
|
var storageAllocations = new Auto<MemoryAllocation>[storageBuffers.Length];
|
||||||
|
int storageAllocationsCount = 0;
|
||||||
|
|
||||||
|
ulong dstOffset = 0;
|
||||||
|
|
||||||
|
for (int index = 0; index < storageBuffers.Length; index++)
|
||||||
|
{
|
||||||
|
BufferRange range = storageBuffers[index];
|
||||||
|
|
||||||
|
if (TryGetBuffer(range.Handle, out var existingHolder))
|
||||||
|
{
|
||||||
|
// Since this buffer now also owns the memory from the referenced buffer,
|
||||||
|
// we pin it to ensure the memory location will not change.
|
||||||
|
existingHolder.Pin();
|
||||||
|
|
||||||
|
(var memory, var offset) = existingHolder.GetDeviceMemoryAndOffset();
|
||||||
|
|
||||||
|
memoryBinds[index] = new SparseMemoryBind()
|
||||||
|
{
|
||||||
|
ResourceOffset = dstOffset,
|
||||||
|
Size = (ulong)range.Size,
|
||||||
|
Memory = memory,
|
||||||
|
MemoryOffset = offset + (ulong)range.Offset,
|
||||||
|
Flags = SparseMemoryBindFlags.None
|
||||||
|
};
|
||||||
|
|
||||||
|
storageAllocations[storageAllocationsCount++] = existingHolder.GetAllocation();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
memoryBinds[index] = new SparseMemoryBind()
|
||||||
|
{
|
||||||
|
ResourceOffset = dstOffset,
|
||||||
|
Size = (ulong)range.Size,
|
||||||
|
Memory = default,
|
||||||
|
MemoryOffset = 0UL,
|
||||||
|
Flags = SparseMemoryBindFlags.None
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
dstOffset += (ulong)range.Size;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (storageAllocations.Length != storageAllocationsCount)
|
||||||
|
{
|
||||||
|
Array.Resize(ref storageAllocations, storageAllocationsCount);
|
||||||
|
}
|
||||||
|
|
||||||
|
fixed (SparseMemoryBind* pMemoryBinds = memoryBinds)
|
||||||
|
{
|
||||||
|
SparseBufferMemoryBindInfo bufferBind = new SparseBufferMemoryBindInfo()
|
||||||
|
{
|
||||||
|
Buffer = buffer,
|
||||||
|
BindCount = (uint)memoryBinds.Length,
|
||||||
|
PBinds = pMemoryBinds
|
||||||
|
};
|
||||||
|
|
||||||
|
BindSparseInfo bindSparseInfo = new BindSparseInfo()
|
||||||
|
{
|
||||||
|
SType = StructureType.BindSparseInfo,
|
||||||
|
BufferBindCount = 1,
|
||||||
|
PBufferBinds = &bufferBind
|
||||||
|
};
|
||||||
|
|
||||||
|
gd.Api.QueueBindSparse(gd.Queue, 1, bindSparseInfo, default).ThrowOnError();
|
||||||
|
}
|
||||||
|
|
||||||
|
var holder = new BufferHolder(gd, _device, buffer, (int)size, storageAllocations);
|
||||||
|
|
||||||
|
BufferCount++;
|
||||||
|
|
||||||
|
ulong handle64 = (uint)_buffers.Add(holder);
|
||||||
|
|
||||||
|
return Unsafe.As<ulong, BufferHandle>(ref handle64);
|
||||||
|
}
|
||||||
|
|
||||||
public BufferHandle CreateWithHandle(
|
public BufferHandle CreateWithHandle(
|
||||||
VulkanRenderer gd,
|
VulkanRenderer gd,
|
||||||
int size,
|
int size,
|
||||||
|
bool sparseCompatible = false,
|
||||||
BufferAllocationType baseType = BufferAllocationType.HostMapped,
|
BufferAllocationType baseType = BufferAllocationType.HostMapped,
|
||||||
BufferHandle storageHint = default,
|
BufferHandle storageHint = default,
|
||||||
bool forceMirrors = false)
|
bool forceMirrors = false)
|
||||||
{
|
{
|
||||||
return CreateWithHandle(gd, size, out _, baseType, storageHint, forceMirrors);
|
return CreateWithHandle(gd, size, out _, sparseCompatible, baseType, storageHint, forceMirrors);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BufferHandle CreateWithHandle(
|
public BufferHandle CreateWithHandle(
|
||||||
VulkanRenderer gd,
|
VulkanRenderer gd,
|
||||||
int size,
|
int size,
|
||||||
out BufferHolder holder,
|
out BufferHolder holder,
|
||||||
|
bool sparseCompatible = false,
|
||||||
BufferAllocationType baseType = BufferAllocationType.HostMapped,
|
BufferAllocationType baseType = BufferAllocationType.HostMapped,
|
||||||
BufferHandle storageHint = default,
|
BufferHandle storageHint = default,
|
||||||
bool forceMirrors = false)
|
bool forceMirrors = false)
|
||||||
{
|
{
|
||||||
holder = Create(gd, size, baseType: baseType, storageHint: storageHint);
|
holder = Create(gd, size, forConditionalRendering: false, sparseCompatible, baseType, storageHint);
|
||||||
if (holder == null)
|
if (holder == null)
|
||||||
{
|
{
|
||||||
return BufferHandle.Null;
|
return BufferHandle.Null;
|
||||||
|
@ -163,6 +269,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
int size,
|
int size,
|
||||||
BufferAllocationType type,
|
BufferAllocationType type,
|
||||||
bool forConditionalRendering = false,
|
bool forConditionalRendering = false,
|
||||||
|
bool sparseCompatible = false,
|
||||||
BufferAllocationType fallbackType = BufferAllocationType.Auto)
|
BufferAllocationType fallbackType = BufferAllocationType.Auto)
|
||||||
{
|
{
|
||||||
var usage = DefaultBufferUsageFlags;
|
var usage = DefaultBufferUsageFlags;
|
||||||
|
@ -187,6 +294,11 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
|
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
|
||||||
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
|
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
|
||||||
|
|
||||||
|
if (sparseCompatible)
|
||||||
|
{
|
||||||
|
requirements.Alignment = Math.Max(requirements.Alignment, Constants.SparseBufferAlignment);
|
||||||
|
}
|
||||||
|
|
||||||
MemoryAllocation allocation;
|
MemoryAllocation allocation;
|
||||||
|
|
||||||
do
|
do
|
||||||
|
@ -227,6 +339,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
VulkanRenderer gd,
|
VulkanRenderer gd,
|
||||||
int size,
|
int size,
|
||||||
bool forConditionalRendering = false,
|
bool forConditionalRendering = false,
|
||||||
|
bool sparseCompatible = false,
|
||||||
BufferAllocationType baseType = BufferAllocationType.HostMapped,
|
BufferAllocationType baseType = BufferAllocationType.HostMapped,
|
||||||
BufferHandle storageHint = default)
|
BufferHandle storageHint = default)
|
||||||
{
|
{
|
||||||
|
@ -255,7 +368,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
}
|
}
|
||||||
|
|
||||||
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) =
|
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) =
|
||||||
CreateBacking(gd, size, type, forConditionalRendering);
|
CreateBacking(gd, size, type, forConditionalRendering, sparseCompatible);
|
||||||
|
|
||||||
if (buffer.Handle != 0)
|
if (buffer.Handle != 0)
|
||||||
{
|
{
|
||||||
|
|
|
@ -16,5 +16,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
public const int MaxStorageBufferBindings = MaxStorageBuffersPerStage * MaxShaderStages;
|
public const int MaxStorageBufferBindings = MaxStorageBuffersPerStage * MaxShaderStages;
|
||||||
public const int MaxTextureBindings = MaxTexturesPerStage * MaxShaderStages;
|
public const int MaxTextureBindings = MaxTexturesPerStage * MaxShaderStages;
|
||||||
public const int MaxImageBindings = MaxImagesPerStage * MaxShaderStages;
|
public const int MaxImageBindings = MaxImagesPerStage * MaxShaderStages;
|
||||||
|
|
||||||
|
public const ulong SparseBufferAlignment = 0x10000;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -424,12 +424,12 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
|
|
||||||
public static BufferAllocationType Convert(this BufferAccess access)
|
public static BufferAllocationType Convert(this BufferAccess access)
|
||||||
{
|
{
|
||||||
return access switch
|
if (access.HasFlag(BufferAccess.FlushPersistent) || access.HasFlag(BufferAccess.Stream))
|
||||||
{
|
{
|
||||||
BufferAccess.FlushPersistent => BufferAllocationType.HostMapped,
|
return BufferAllocationType.HostMapped;
|
||||||
BufferAccess.Stream => BufferAllocationType.HostMapped,
|
}
|
||||||
_ => BufferAllocationType.Auto,
|
|
||||||
};
|
return BufferAllocationType.Auto;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static T2 LogInvalidAndReturn<T1, T2>(T1 value, string name, T2 defaultValue = default)
|
private static T2 LogInvalidAndReturn<T1, T2>(T1 value, string name, T2 defaultValue = default)
|
||||||
|
|
|
@ -392,6 +392,8 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
|
|
||||||
LoadFeatures(maxQueueCount, queueFamilyIndex);
|
LoadFeatures(maxQueueCount, queueFamilyIndex);
|
||||||
|
|
||||||
|
QueueFamilyIndex = queueFamilyIndex;
|
||||||
|
|
||||||
_window = new Window(this, _surface, _physicalDevice.PhysicalDevice, _device);
|
_window = new Window(this, _surface, _physicalDevice.PhysicalDevice, _device);
|
||||||
|
|
||||||
_initialized = true;
|
_initialized = true;
|
||||||
|
@ -399,12 +401,12 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(int size, BufferAccess access)
|
public BufferHandle CreateBuffer(int size, BufferAccess access)
|
||||||
{
|
{
|
||||||
return BufferManager.CreateWithHandle(this, size, access.Convert(), default, access == BufferAccess.Stream);
|
return BufferManager.CreateWithHandle(this, size, access.HasFlag(BufferAccess.SparseCompatible), access.Convert(), default, access == BufferAccess.Stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
|
public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
|
||||||
{
|
{
|
||||||
return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint);
|
return BufferManager.CreateWithHandle(this, size, access.HasFlag(BufferAccess.SparseCompatible), access.Convert(), storageHint);
|
||||||
}
|
}
|
||||||
|
|
||||||
public BufferHandle CreateBuffer(nint pointer, int size)
|
public BufferHandle CreateBuffer(nint pointer, int size)
|
||||||
|
@ -412,6 +414,11 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
return BufferManager.CreateHostImported(this, pointer, size);
|
return BufferManager.CreateHostImported(this, pointer, size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
|
||||||
|
{
|
||||||
|
return BufferManager.CreateSparse(this, storageBuffers);
|
||||||
|
}
|
||||||
|
|
||||||
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
|
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
|
||||||
{
|
{
|
||||||
bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute;
|
bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute;
|
||||||
|
@ -571,6 +578,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
Api.GetPhysicalDeviceFeatures2(_physicalDevice.PhysicalDevice, &features2);
|
Api.GetPhysicalDeviceFeatures2(_physicalDevice.PhysicalDevice, &features2);
|
||||||
|
|
||||||
var limits = _physicalDevice.PhysicalDeviceProperties.Limits;
|
var limits = _physicalDevice.PhysicalDeviceProperties.Limits;
|
||||||
|
var mainQueueProperties = _physicalDevice.QueueFamilyProperties[QueueFamilyIndex];
|
||||||
|
|
||||||
return new Capabilities(
|
return new Capabilities(
|
||||||
api: TargetApi.Vulkan,
|
api: TargetApi.Vulkan,
|
||||||
|
@ -590,6 +598,7 @@ namespace Ryujinx.Graphics.Vulkan
|
||||||
supportsR4G4B4A4Format: supportsR4G4B4A4Format,
|
supportsR4G4B4A4Format: supportsR4G4B4A4Format,
|
||||||
supportsSnormBufferTextureFormat: true,
|
supportsSnormBufferTextureFormat: true,
|
||||||
supports5BitComponentFormat: supports5BitComponentFormat,
|
supports5BitComponentFormat: supports5BitComponentFormat,
|
||||||
|
supportsSparseBuffer: features2.Features.SparseBinding && mainQueueProperties.QueueFlags.HasFlag(QueueFlags.SparseBindingBit),
|
||||||
supportsBlendEquationAdvanced: Capabilities.SupportsBlendEquationAdvanced,
|
supportsBlendEquationAdvanced: Capabilities.SupportsBlendEquationAdvanced,
|
||||||
supportsFragmentShaderInterlock: Capabilities.SupportsFragmentShaderInterlock,
|
supportsFragmentShaderInterlock: Capabilities.SupportsFragmentShaderInterlock,
|
||||||
supportsFragmentShaderOrderingIntel: false,
|
supportsFragmentShaderOrderingIntel: false,
|
||||||
|
|
|
@ -15,6 +15,11 @@ namespace Ryujinx.Memory.Range
|
||||||
|
|
||||||
private bool HasSingleRange => _ranges == null;
|
private bool HasSingleRange => _ranges == null;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Indicates that the range is fully unmapped.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsUnmapped => HasSingleRange && _singleRange.Address == InvalidAddress;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Total of physical sub-ranges on the virtual memory region.
|
/// Total of physical sub-ranges on the virtual memory region.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
@ -37,9 +42,19 @@ namespace Ryujinx.Memory.Range
|
||||||
/// <param name="ranges">Array of physical regions</param>
|
/// <param name="ranges">Array of physical regions</param>
|
||||||
/// <exception cref="ArgumentNullException"><paramref name="ranges"/> is null</exception>
|
/// <exception cref="ArgumentNullException"><paramref name="ranges"/> is null</exception>
|
||||||
public MultiRange(MemoryRange[] ranges)
|
public MultiRange(MemoryRange[] ranges)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(ranges);
|
||||||
|
|
||||||
|
if (ranges.Length == 1)
|
||||||
|
{
|
||||||
|
_singleRange = ranges[0];
|
||||||
|
_ranges = null;
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
_singleRange = MemoryRange.Empty;
|
_singleRange = MemoryRange.Empty;
|
||||||
_ranges = ranges ?? throw new ArgumentNullException(nameof(ranges));
|
_ranges = ranges;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
@ -91,7 +106,7 @@ namespace Ryujinx.Memory.Range
|
||||||
offset -= range.Size;
|
offset -= range.Size;
|
||||||
}
|
}
|
||||||
|
|
||||||
return new MultiRange(ranges.ToArray());
|
return ranges.Count == 1 ? new MultiRange(ranges[0].Address, ranges[0].Size) : new MultiRange(ranges.ToArray());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Reference in a new issue