Implement support for multi-range buffers using Vulkan sparse mappings (#5427)

* Pass MultiRange to BufferManager

* Implement support for multi-range buffers using Vulkan sparse mappings

* Use multi-range for remaining buffers, delete old methods

* Assume that more buffers are contiguous

* Dispose multi-range buffers after they are removed from the list

* Properly init BufferBounds for constant and storage buffers

* Do not try reading zero bytes data from an unmapped address on the shader cache + PR feedback

* Fix misaligned sparse buffer offsets

* Null check can be simplified

* PR feedback
This commit is contained in:
gdkchan 2023-12-04 16:30:19 -03:00 committed by GitHub
parent 0531c16326
commit 1df6c07f78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 1241 additions and 233 deletions

View file

@ -1,9 +1,13 @@
using System;
namespace Ryujinx.Graphics.GAL namespace Ryujinx.Graphics.GAL
{ {
[Flags]
public enum BufferAccess public enum BufferAccess
{ {
Default, Default = 0,
FlushPersistent, FlushPersistent = 1 << 0,
Stream Stream = 1 << 1,
SparseCompatible = 1 << 2,
} }
} }

View file

@ -23,6 +23,7 @@ namespace Ryujinx.Graphics.GAL
public readonly bool SupportsR4G4B4A4Format; public readonly bool SupportsR4G4B4A4Format;
public readonly bool SupportsScaledVertexFormats; public readonly bool SupportsScaledVertexFormats;
public readonly bool SupportsSnormBufferTextureFormat; public readonly bool SupportsSnormBufferTextureFormat;
public readonly bool SupportsSparseBuffer;
public readonly bool Supports5BitComponentFormat; public readonly bool Supports5BitComponentFormat;
public readonly bool SupportsBlendEquationAdvanced; public readonly bool SupportsBlendEquationAdvanced;
public readonly bool SupportsFragmentShaderInterlock; public readonly bool SupportsFragmentShaderInterlock;
@ -79,6 +80,7 @@ namespace Ryujinx.Graphics.GAL
bool supportsScaledVertexFormats, bool supportsScaledVertexFormats,
bool supportsSnormBufferTextureFormat, bool supportsSnormBufferTextureFormat,
bool supports5BitComponentFormat, bool supports5BitComponentFormat,
bool supportsSparseBuffer,
bool supportsBlendEquationAdvanced, bool supportsBlendEquationAdvanced,
bool supportsFragmentShaderInterlock, bool supportsFragmentShaderInterlock,
bool supportsFragmentShaderOrderingIntel, bool supportsFragmentShaderOrderingIntel,
@ -130,6 +132,7 @@ namespace Ryujinx.Graphics.GAL
SupportsScaledVertexFormats = supportsScaledVertexFormats; SupportsScaledVertexFormats = supportsScaledVertexFormats;
SupportsSnormBufferTextureFormat = supportsSnormBufferTextureFormat; SupportsSnormBufferTextureFormat = supportsSnormBufferTextureFormat;
Supports5BitComponentFormat = supports5BitComponentFormat; Supports5BitComponentFormat = supports5BitComponentFormat;
SupportsSparseBuffer = supportsSparseBuffer;
SupportsBlendEquationAdvanced = supportsBlendEquationAdvanced; SupportsBlendEquationAdvanced = supportsBlendEquationAdvanced;
SupportsFragmentShaderInterlock = supportsFragmentShaderInterlock; SupportsFragmentShaderInterlock = supportsFragmentShaderInterlock;
SupportsFragmentShaderOrderingIntel = supportsFragmentShaderOrderingIntel; SupportsFragmentShaderOrderingIntel = supportsFragmentShaderOrderingIntel;

View file

@ -16,13 +16,10 @@ namespace Ryujinx.Graphics.GAL
void BackgroundContextAction(Action action, bool alwaysBackground = false); void BackgroundContextAction(Action action, bool alwaysBackground = false);
BufferHandle CreateBuffer(int size, BufferHandle storageHint); BufferHandle CreateBuffer(int size, BufferAccess access = BufferAccess.Default);
BufferHandle CreateBuffer(int size) BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint);
{
return CreateBuffer(size, BufferHandle.Null);
}
BufferHandle CreateBuffer(nint pointer, int size); BufferHandle CreateBuffer(nint pointer, int size);
BufferHandle CreateBuffer(int size, BufferAccess access); BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers);
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info); IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);

View file

@ -44,6 +44,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
Register<ActionCommand>(CommandType.Action); Register<ActionCommand>(CommandType.Action);
Register<CreateBufferCommand>(CommandType.CreateBuffer); Register<CreateBufferCommand>(CommandType.CreateBuffer);
Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess); Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess);
Register<CreateBufferSparseCommand>(CommandType.CreateBufferSparse);
Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer); Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer);
Register<CreateProgramCommand>(CommandType.CreateProgram); Register<CreateProgramCommand>(CommandType.CreateProgram);
Register<CreateSamplerCommand>(CommandType.CreateSampler); Register<CreateSamplerCommand>(CommandType.CreateSampler);

View file

@ -5,6 +5,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
Action, Action,
CreateBuffer, CreateBuffer,
CreateBufferAccess, CreateBufferAccess,
CreateBufferSparse,
CreateHostBuffer, CreateHostBuffer,
CreateProgram, CreateProgram,
CreateSampler, CreateSampler,

View file

@ -5,12 +5,14 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
public readonly CommandType CommandType => CommandType.CreateBuffer; public readonly CommandType CommandType => CommandType.CreateBuffer;
private BufferHandle _threadedHandle; private BufferHandle _threadedHandle;
private int _size; private int _size;
private BufferAccess _access;
private BufferHandle _storageHint; private BufferHandle _storageHint;
public void Set(BufferHandle threadedHandle, int size, BufferHandle storageHint) public void Set(BufferHandle threadedHandle, int size, BufferAccess access, BufferHandle storageHint)
{ {
_threadedHandle = threadedHandle; _threadedHandle = threadedHandle;
_size = size; _size = size;
_access = access;
_storageHint = storageHint; _storageHint = storageHint;
} }
@ -23,7 +25,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
hint = threaded.Buffers.MapBuffer(command._storageHint); hint = threaded.Buffers.MapBuffer(command._storageHint);
} }
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, hint)); threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, command._access, hint));
} }
} }
} }

View file

@ -0,0 +1,25 @@
using Ryujinx.Graphics.GAL.Multithreading.Model;
using System;
namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
{
struct CreateBufferSparseCommand : IGALCommand, IGALCommand<CreateBufferSparseCommand>
{
public readonly CommandType CommandType => CommandType.CreateBufferSparse;
private BufferHandle _threadedHandle;
private SpanRef<BufferRange> _buffers;
public void Set(BufferHandle threadedHandle, SpanRef<BufferRange> buffers)
{
_threadedHandle = threadedHandle;
_buffers = buffers;
}
public static void Run(ref CreateBufferSparseCommand command, ThreadedRenderer threaded, IRenderer renderer)
{
Span<BufferRange> buffers = command._buffers.Get(threaded);
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBufferSparse(threaded.Buffers.MapBufferRanges(buffers)));
command._buffers.Dispose(threaded);
}
}
}

View file

@ -263,10 +263,19 @@ namespace Ryujinx.Graphics.GAL.Multithreading
} }
} }
public BufferHandle CreateBuffer(int size, BufferHandle storageHint) public BufferHandle CreateBuffer(int size, BufferAccess access)
{ {
BufferHandle handle = Buffers.CreateBufferHandle(); BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateBufferCommand>().Set(handle, size, storageHint); New<CreateBufferAccessCommand>().Set(handle, size, access);
QueueCommand();
return handle;
}
public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
{
BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateBufferCommand>().Set(handle, size, access, storageHint);
QueueCommand(); QueueCommand();
return handle; return handle;
@ -281,10 +290,10 @@ namespace Ryujinx.Graphics.GAL.Multithreading
return handle; return handle;
} }
public BufferHandle CreateBuffer(int size, BufferAccess access) public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
{ {
BufferHandle handle = Buffers.CreateBufferHandle(); BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateBufferAccessCommand>().Set(handle, size, access); New<CreateBufferSparseCommand>().Set(handle, CopySpan(storageBuffers));
QueueCommand(); QueueCommand();
return handle; return handle;

View file

@ -5,6 +5,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Engine.GPFifo; using Ryujinx.Graphics.Gpu.Engine.GPFifo;
using Ryujinx.Graphics.Gpu.Engine.Threed; using Ryujinx.Graphics.Gpu.Engine.Threed;
using Ryujinx.Graphics.Gpu.Engine.Types; using Ryujinx.Graphics.Gpu.Engine.Types;
using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@ -392,12 +393,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.MME
_processor.ThreedClass.DrawIndirect( _processor.ThreedClass.DrawIndirect(
topology, topology,
indirectBufferAddress, new MultiRange(indirectBufferAddress, IndirectIndexedDataEntrySize),
0, default,
1, 1,
IndirectIndexedDataEntrySize, IndirectIndexedDataEntrySize,
indexCount, indexCount,
Threed.IndirectDrawType.DrawIndexedIndirect); IndirectDrawType.DrawIndexedIndirect);
} }
else else
{ {
@ -494,13 +495,13 @@ namespace Ryujinx.Graphics.Gpu.Engine.MME
ulong indirectBufferSize = (ulong)maxDrawCount * (ulong)stride; ulong indirectBufferSize = (ulong)maxDrawCount * (ulong)stride;
ulong indirectBufferAddress = bufferCache.TranslateAndCreateBuffer(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize); MultiRange indirectBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize);
ulong parameterBufferAddress = bufferCache.TranslateAndCreateBuffer(_processor.MemoryManager, parameterBufferGpuVa, 4); MultiRange parameterBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, parameterBufferGpuVa, 4);
_processor.ThreedClass.DrawIndirect( _processor.ThreedClass.DrawIndirect(
topology, topology,
indirectBufferAddress, indirectBufferRange,
parameterBufferAddress, parameterBufferRange,
maxDrawCount, maxDrawCount,
stride, stride,
indexCount, indexCount,

View file

@ -370,8 +370,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw
{ {
var memoryManager = _channel.MemoryManager; var memoryManager = _channel.MemoryManager;
address = memoryManager.Translate(address); BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address, size));
BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(address, size);
ITexture bufferTexture = _vacContext.EnsureBufferTexture(index + 2, format); ITexture bufferTexture = _vacContext.EnsureBufferTexture(index + 2, format);
bufferTexture.SetStorage(range); bufferTexture.SetStorage(range);
@ -412,9 +411,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw
var memoryManager = _channel.MemoryManager; var memoryManager = _channel.MemoryManager;
address = memoryManager.Translate(address + indexOffset);
ulong misalign = address & ((ulong)_context.Capabilities.TextureBufferOffsetAlignment - 1); ulong misalign = address & ((ulong)_context.Capabilities.TextureBufferOffsetAlignment - 1);
BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(address - misalign, size + misalign); BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address + indexOffset - misalign, size + misalign));
misalignedOffset = (int)misalign >> shift; misalignedOffset = (int)misalign >> shift;
SetIndexBufferTexture(reservations, range, format); SetIndexBufferTexture(reservations, range, format);

View file

@ -3,6 +3,7 @@ using Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw;
using Ryujinx.Graphics.Gpu.Engine.Types; using Ryujinx.Graphics.Gpu.Engine.Types;
using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Gpu.Memory;
using Ryujinx.Memory.Range;
using System; using System;
namespace Ryujinx.Graphics.Gpu.Engine.Threed namespace Ryujinx.Graphics.Gpu.Engine.Threed
@ -630,8 +631,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
/// </summary> /// </summary>
/// <param name="engine">3D engine where this method is being called</param> /// <param name="engine">3D engine where this method is being called</param>
/// <param name="topology">Primitive topology</param> /// <param name="topology">Primitive topology</param>
/// <param name="indirectBufferAddress">Address of the buffer with the draw parameters, such as count, first index, etc</param> /// <param name="indirectBufferRange">Memory range of the buffer with the draw parameters, such as count, first index, etc</param>
/// <param name="parameterBufferAddress">Address of the buffer with the draw count</param> /// <param name="parameterBufferRange">Memory range of the buffer with the draw count</param>
/// <param name="maxDrawCount">Maximum number of draws that can be made</param> /// <param name="maxDrawCount">Maximum number of draws that can be made</param>
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param> /// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param>
/// <param name="indexCount">Maximum number of indices that the draw can consume</param> /// <param name="indexCount">Maximum number of indices that the draw can consume</param>
@ -639,8 +640,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
public void DrawIndirect( public void DrawIndirect(
ThreedClass engine, ThreedClass engine,
PrimitiveTopology topology, PrimitiveTopology topology,
ulong indirectBufferAddress, MultiRange indirectBufferRange,
ulong parameterBufferAddress, MultiRange parameterBufferRange,
int maxDrawCount, int maxDrawCount,
int stride, int stride,
int indexCount, int indexCount,
@ -681,8 +682,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
if (hasCount) if (hasCount)
{ {
var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferAddress, (ulong)maxDrawCount * (ulong)stride); var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferAddress, 4); var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferRange);
if (indexed) if (indexed)
{ {
@ -695,7 +696,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
} }
else else
{ {
var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferAddress, (ulong)stride); var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
if (indexed) if (indexed)
{ {

View file

@ -6,6 +6,7 @@ using Ryujinx.Graphics.Gpu.Engine.InlineToMemory;
using Ryujinx.Graphics.Gpu.Engine.Threed.Blender; using Ryujinx.Graphics.Gpu.Engine.Threed.Blender;
using Ryujinx.Graphics.Gpu.Engine.Types; using Ryujinx.Graphics.Gpu.Engine.Types;
using Ryujinx.Graphics.Gpu.Synchronization; using Ryujinx.Graphics.Gpu.Synchronization;
using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
@ -803,22 +804,22 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
/// Performs a indirect draw, with parameters from a GPU buffer. /// Performs a indirect draw, with parameters from a GPU buffer.
/// </summary> /// </summary>
/// <param name="topology">Primitive topology</param> /// <param name="topology">Primitive topology</param>
/// <param name="indirectBufferAddress">Address of the buffer with the draw parameters, such as count, first index, etc</param> /// <param name="indirectBufferRange">Memory range of the buffer with the draw parameters, such as count, first index, etc</param>
/// <param name="parameterBufferAddress">Address of the buffer with the draw count</param> /// <param name="parameterBufferRange">Memory range of the buffer with the draw count</param>
/// <param name="maxDrawCount">Maximum number of draws that can be made</param> /// <param name="maxDrawCount">Maximum number of draws that can be made</param>
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param> /// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferRange"/></param>
/// <param name="indexCount">Maximum number of indices that the draw can consume</param> /// <param name="indexCount">Maximum number of indices that the draw can consume</param>
/// <param name="drawType">Type of the indirect draw, which can be indexed or non-indexed, with or without a draw count</param> /// <param name="drawType">Type of the indirect draw, which can be indexed or non-indexed, with or without a draw count</param>
public void DrawIndirect( public void DrawIndirect(
PrimitiveTopology topology, PrimitiveTopology topology,
ulong indirectBufferAddress, MultiRange indirectBufferRange,
ulong parameterBufferAddress, MultiRange parameterBufferRange,
int maxDrawCount, int maxDrawCount,
int stride, int stride,
int indexCount, int indexCount,
IndirectDrawType drawType) IndirectDrawType drawType)
{ {
_drawManager.DrawIndirect(this, topology, indirectBufferAddress, parameterBufferAddress, maxDrawCount, stride, indexCount, drawType); _drawManager.DrawIndirect(this, topology, indirectBufferRange, parameterBufferRange, maxDrawCount, stride, indexCount, drawType);
} }
/// <summary> /// <summary>

View file

@ -382,7 +382,7 @@ namespace Ryujinx.Graphics.Gpu.Image
{ {
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex); ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex);
cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedTextureBufferIndex = textureBufferIndex; cachedTextureBufferIndex = textureBufferIndex;
if (samplerBufferIndex == textureBufferIndex) if (samplerBufferIndex == textureBufferIndex)
@ -396,7 +396,7 @@ namespace Ryujinx.Graphics.Gpu.Image
{ {
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex); ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex);
cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedSamplerBufferIndex = samplerBufferIndex; cachedSamplerBufferIndex = samplerBufferIndex;
} }
} }
@ -524,7 +524,7 @@ namespace Ryujinx.Graphics.Gpu.Image
// Ensure that the buffer texture is using the correct buffer as storage. // Ensure that the buffer texture is using the correct buffer as storage.
// Buffers are frequently re-created to accommodate larger data, so we need to re-bind // Buffers are frequently re-created to accommodate larger data, so we need to re-bind
// to ensure we're not using a old buffer that was already deleted. // to ensure we're not using a old buffer that was already deleted.
_channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, bindingInfo.Format, false); _channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range, bindingInfo, bindingInfo.Format, false);
// Cache is not used for buffer texture, it must always rebind. // Cache is not used for buffer texture, it must always rebind.
state.CachedTexture = null; state.CachedTexture = null;
@ -661,7 +661,7 @@ namespace Ryujinx.Graphics.Gpu.Image
format = texture.Format; format = texture.Format;
} }
_channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, format, true); _channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range, bindingInfo, format, true);
// Cache is not used for buffer texture, it must always rebind. // Cache is not used for buffer texture, it must always rebind.
state.CachedTexture = null; state.CachedTexture = null;

View file

@ -43,6 +43,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
public int UnmappedSequence { get; private set; } public int UnmappedSequence { get; private set; }
/// <summary>
/// Indicates if the buffer can be used in a sparse buffer mapping.
/// </summary>
public bool SparseCompatible { get; }
/// <summary> /// <summary>
/// Ranges of the buffer that have been modified on the GPU. /// Ranges of the buffer that have been modified on the GPU.
/// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached. /// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached.
@ -77,15 +82,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="physicalMemory">Physical memory where the buffer is mapped</param> /// <param name="physicalMemory">Physical memory where the buffer is mapped</param>
/// <param name="address">Start address of the buffer</param> /// <param name="address">Start address of the buffer</param>
/// <param name="size">Size of the buffer in bytes</param> /// <param name="size">Size of the buffer in bytes</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param> /// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
public Buffer(GpuContext context, PhysicalMemory physicalMemory, ulong address, ulong size, IEnumerable<Buffer> baseBuffers = null) public Buffer(
GpuContext context,
PhysicalMemory physicalMemory,
ulong address,
ulong size,
bool sparseCompatible,
IEnumerable<Buffer> baseBuffers = null)
{ {
_context = context; _context = context;
_physicalMemory = physicalMemory; _physicalMemory = physicalMemory;
Address = address; Address = address;
Size = size; Size = size;
SparseCompatible = sparseCompatible;
Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null); BufferAccess access = sparseCompatible ? BufferAccess.SparseCompatible : BufferAccess.Default;
Handle = context.Renderer.CreateBuffer((int)size, access, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
_useGranular = size > GranularBufferThreshold; _useGranular = size > GranularBufferThreshold;

View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.Shader; using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -8,30 +9,28 @@ namespace Ryujinx.Graphics.Gpu.Memory
readonly struct BufferBounds readonly struct BufferBounds
{ {
/// <summary> /// <summary>
/// Region virtual address. /// Physical memory ranges where the buffer is mapped.
/// </summary> /// </summary>
public ulong Address { get; } public MultiRange Range { get; }
/// <summary>
/// Region size in bytes.
/// </summary>
public ulong Size { get; }
/// <summary> /// <summary>
/// Buffer usage flags. /// Buffer usage flags.
/// </summary> /// </summary>
public BufferUsageFlags Flags { get; } public BufferUsageFlags Flags { get; }
/// <summary>
/// Indicates that the backing memory for the buffer does not exist.
/// </summary>
public bool IsUnmapped => Range.IsUnmapped;
/// <summary> /// <summary>
/// Creates a new buffer region. /// Creates a new buffer region.
/// </summary> /// </summary>
/// <param name="address">Region address</param> /// <param name="range">Physical memory ranges where the buffer is mapped</param>
/// <param name="size">Region size</param>
/// <param name="flags">Buffer usage flags</param> /// <param name="flags">Buffer usage flags</param>
public BufferBounds(ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None) public BufferBounds(MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{ {
Address = address; Range = range;
Size = size;
Flags = flags; Flags = flags;
} }
} }

View file

@ -11,12 +11,24 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
class BufferCache : IDisposable class BufferCache : IDisposable
{ {
private const int OverlapsBufferInitialCapacity = 10; /// <summary>
private const int OverlapsBufferMaxCapacity = 10000; /// Initial size for the array holding overlaps.
/// </summary>
public const int OverlapsBufferInitialCapacity = 10;
/// <summary>
/// Maximum size that an array holding overlaps may have after trimming.
/// </summary>
public const int OverlapsBufferMaxCapacity = 10000;
private const ulong BufferAlignmentSize = 0x1000; private const ulong BufferAlignmentSize = 0x1000;
private const ulong BufferAlignmentMask = BufferAlignmentSize - 1; private const ulong BufferAlignmentMask = BufferAlignmentSize - 1;
/// <summary>
/// Alignment required for sparse buffer mappings.
/// </summary>
public const ulong SparseBufferAlignmentSize = 0x10000;
private const ulong MaxDynamicGrowthSize = 0x100000; private const ulong MaxDynamicGrowthSize = 0x100000;
private readonly GpuContext _context; private readonly GpuContext _context;
@ -27,6 +39,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// Must lock for any access from other threads. /// Must lock for any access from other threads.
/// </remarks> /// </remarks>
private readonly RangeList<Buffer> _buffers; private readonly RangeList<Buffer> _buffers;
private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
private Buffer[] _bufferOverlaps; private Buffer[] _bufferOverlaps;
@ -47,6 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_physicalMemory = physicalMemory; _physicalMemory = physicalMemory;
_buffers = new RangeList<Buffer>(); _buffers = new RangeList<Buffer>();
_multiRangeBuffers = new MultiRangeList<MultiRangeBuffer>();
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity]; _bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
@ -66,45 +80,100 @@ namespace Ryujinx.Graphics.Gpu.Memory
Buffer[] overlaps = new Buffer[10]; Buffer[] overlaps = new Buffer[10];
int overlapCount; int overlapCount;
ulong address = ((MemoryManager)sender).Translate(e.Address); MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
ulong size = e.Size;
lock (_buffers) for (int index = 0; index < range.Count; index++)
{ {
overlapCount = _buffers.FindOverlaps(address, size, ref overlaps); MemoryRange subRange = range.GetSubRange(index);
}
for (int i = 0; i < overlapCount; i++) lock (_buffers)
{ {
overlaps[i].Unmapped(address, size); overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps);
}
for (int i = 0; i < overlapCount; i++)
{
overlaps[i].Unmapped(subRange.Address, subRange.Size);
}
} }
} }
/// <summary> /// <summary>
/// Performs address translation of the GPU virtual address, and creates a /// Performs address translation of the GPU virtual address, and creates a
/// new buffer, if needed, for the specified range. /// new buffer, if needed, for the specified contiguous range.
/// </summary> /// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param> /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param> /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
/// <param name="size">Size in bytes of the buffer</param> /// <param name="size">Size in bytes of the buffer</param>
/// <returns>CPU virtual address of the buffer, after address translation</returns> /// <returns>Contiguous physical range of the buffer, after address translation</returns>
public ulong TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size) public MultiRange TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
{ {
if (gpuVa == 0) if (gpuVa == 0)
{ {
return 0; return new MultiRange(MemoryManager.PteUnmapped, size);
} }
ulong address = memoryManager.Translate(gpuVa); ulong address = memoryManager.Translate(gpuVa);
if (address == MemoryManager.PteUnmapped) if (address != MemoryManager.PteUnmapped)
{ {
return 0; CreateBuffer(address, size);
} }
CreateBuffer(address, size); return new MultiRange(address, size);
}
return address; /// <summary>
/// Performs address translation of the GPU virtual address, and creates
/// new buffers, if needed, for the specified range.
/// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <returns>Physical ranges of the buffer, after address translation</returns>
public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size)
{
if (gpuVa == 0)
{
return new MultiRange(MemoryManager.PteUnmapped, size);
}
bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
// Fast path not taken for non-contiguous ranges,
// since multi-range buffers are not coalesced, so a buffer that covers
// the entire cached range might not actually exist.
if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
range.Count == 1)
{
return range;
}
CreateBuffer(range);
return range;
}
/// <summary>
/// Creates a new buffer for the specified range, if it does not yet exist.
/// This can be used to ensure the existance of a buffer.
/// </summary>
/// <param name="range">Physical ranges of memory where the buffer data is located</param>
public void CreateBuffer(MultiRange range)
{
if (range.Count > 1)
{
CreateMultiRangeBuffer(range);
}
else
{
MemoryRange subRange = range.GetSubRange(0);
if (subRange.Address != MemoryManager.PteUnmapped)
{
CreateBuffer(subRange.Address, subRange.Size);
}
}
} }
/// <summary> /// <summary>
@ -118,7 +187,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size; ulong endAddress = address + size;
ulong alignedAddress = address & ~BufferAlignmentMask; ulong alignedAddress = address & ~BufferAlignmentMask;
ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask; ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask;
// The buffer must have the size of at least one page. // The buffer must have the size of at least one page.
@ -130,6 +198,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress); CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress);
} }
/// <summary>
/// Creates a new buffer for the specified range, if it does not yet exist.
/// This can be used to ensure the existance of a buffer.
/// </summary>
/// <param name="address">Address of the buffer in memory</param>
/// <param name="size">Size of the buffer in bytes</param>
/// <param name="alignment">Alignment of the start address of the buffer in bytes</param>
public void CreateBuffer(ulong address, ulong size, ulong alignment)
{
ulong alignmentMask = alignment - 1;
ulong pageAlignmentMask = BufferAlignmentMask;
ulong endAddress = address + size;
ulong alignedAddress = address & ~alignmentMask;
ulong alignedEndAddress = (endAddress + pageAlignmentMask) & ~pageAlignmentMask;
// The buffer must have the size of at least one page.
if (alignedEndAddress == alignedAddress)
{
alignedEndAddress += pageAlignmentMask;
}
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress, alignment);
}
/// <summary>
/// Creates a buffer for a memory region composed of multiple physical ranges,
/// if it does not exist yet.
/// </summary>
/// <param name="range">Physical ranges of memory</param>
private void CreateMultiRangeBuffer(MultiRange range)
{
// Ensure all non-contiguous buffer we might use are sparse aligned.
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
}
}
// Create sparse buffer.
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
for (int index = 0; index < overlapCount; index++)
{
if (overlaps[index].Range.Contains(range))
{
return;
}
}
for (int index = 0; index < overlapCount; index++)
{
if (range.Contains(overlaps[index].Range))
{
_multiRangeBuffers.Remove(overlaps[index]);
overlaps[index].Dispose();
}
}
BufferRange[] storages = new BufferRange[range.Count];
MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
ulong alignmentMask = SparseBufferAlignmentSize - 1;
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
ulong endAddress = subRange.Address + subRange.Size;
ulong alignedAddress = subRange.Address & ~alignmentMask;
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
ulong alignedSize = alignedEndAddress - alignedAddress;
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
storages[i] = bufferRange;
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
}
else
{
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
}
}
MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
_multiRangeBuffers.Add(multiRangeBuffer);
}
/// <summary> /// <summary>
/// Performs address translation of the GPU virtual address, and attempts to force /// Performs address translation of the GPU virtual address, and attempts to force
/// the buffer in the region as dirty. /// the buffer in the region as dirty.
@ -150,7 +320,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
result.EndGpuAddress < gpuVa + size || result.EndGpuAddress < gpuVa + size ||
result.UnmappedSequence != result.Buffer.UnmappedSequence) result.UnmappedSequence != result.Buffer.UnmappedSequence)
{ {
ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size); MultiRange range = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
ulong address = range.GetSubRange(0).Address;
result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size)); result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size));
_dirtyCache[gpuVa] = result; _dirtyCache[gpuVa] = result;
@ -184,7 +355,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
result.EndGpuAddress < alignedEndGpuVa || result.EndGpuAddress < alignedEndGpuVa ||
result.UnmappedSequence != result.Buffer.UnmappedSequence) result.UnmappedSequence != result.Buffer.UnmappedSequence)
{ {
ulong address = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size); MultiRange range = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
ulong address = range.GetSubRange(0).Address;
result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size)); result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size));
_modifiedCache[alignedGpuVa] = result; _modifiedCache[alignedGpuVa] = result;
@ -204,7 +376,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the buffer</param> /// <param name="size">Size in bytes of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size) private void CreateBufferAligned(ulong address, ulong size)
{ {
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps); Buffer[] overlaps = _bufferOverlaps;
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
if (overlapsCount != 0) if (overlapsCount != 0)
{ {
@ -215,9 +388,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
// old buffer(s) to the new buffer. // old buffer(s) to the new buffer.
ulong endAddress = address + size; ulong endAddress = address + size;
Buffer overlap0 = overlaps[0];
if (_bufferOverlaps[0].Address > address || _bufferOverlaps[0].EndAddress < endAddress) if (overlap0.Address > address || overlap0.EndAddress < endAddress)
{ {
bool anySparseCompatible = false;
// Check if the following conditions are met: // Check if the following conditions are met:
// - We have a single overlap. // - We have a single overlap.
// - The overlap starts at or before the requested range. That is, the overlap happens at the end. // - The overlap starts at or before the requested range. That is, the overlap happens at the end.
@ -228,23 +404,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the // Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages. // range crosses a page, and after alignment, ends having a size of 2 pages.
if (overlapsCount == 1 && if (overlapsCount == 1 &&
address >= _bufferOverlaps[0].Address && address >= overlap0.Address &&
endAddress - _bufferOverlaps[0].EndAddress <= BufferAlignmentSize * 2) endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
{ {
// Try to grow the buffer by 1.5x of its current size. // Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts. // This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = _bufferOverlaps[0].Size; ulong existingSize = overlap0.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask; ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize); size = Math.Max(size, growthSize);
endAddress = address + size; endAddress = address + size;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps); overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
} }
for (int index = 0; index < overlapsCount; index++) for (int index = 0; index < overlapsCount; index++)
{ {
Buffer buffer = _bufferOverlaps[index]; Buffer buffer = overlaps[index];
anySparseCompatible |= buffer.SparseCompatible;
address = Math.Min(address, buffer.Address); address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress); endAddress = Math.Max(endAddress, buffer.EndAddress);
@ -257,35 +435,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong newSize = endAddress - address; ulong newSize = endAddress - address;
Buffer newBuffer = new(_context, _physicalMemory, address, newSize, _bufferOverlaps.Take(overlapsCount)); CreateBufferAligned(address, newSize, anySparseCompatible, overlaps, overlapsCount);
lock (_buffers)
{
_buffers.Add(newBuffer);
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = _bufferOverlaps[index];
int dstOffset = (int)(buffer.Address - newBuffer.Address);
buffer.CopyTo(newBuffer, dstOffset);
newBuffer.InheritModifiedRanges(buffer);
buffer.DecrementReferenceCount();
}
newBuffer.SynchronizeMemory(address, newSize);
// Existing buffers were modified, we need to rebind everything.
NotifyBuffersModified?.Invoke();
} }
} }
else else
{ {
// No overlap, just create a new buffer. // No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size); Buffer buffer = new(_context, _physicalMemory, address, size, sparseCompatible: false);
lock (_buffers) lock (_buffers)
{ {
@ -296,6 +452,151 @@ namespace Ryujinx.Graphics.Gpu.Memory
ShrinkOverlapsBufferIfNeeded(); ShrinkOverlapsBufferIfNeeded();
} }
/// <summary>
/// Creates a new buffer for the specified range, if needed.
/// If a buffer where this range can be fully contained already exists,
/// then the creation of a new buffer is not necessary.
/// </summary>
/// <param name="address">Address of the buffer in guest memory</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <param name="alignment">Alignment of the start address of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size, ulong alignment)
{
Buffer[] overlaps = _bufferOverlaps;
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
bool sparseAligned = alignment >= SparseBufferAlignmentSize;
if (overlapsCount != 0)
{
// If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size;
Buffer overlap0 = overlaps[0];
if (overlap0.Address > address ||
overlap0.EndAddress < endAddress ||
(overlap0.Address & (alignment - 1)) != 0 ||
(!overlap0.SparseCompatible && sparseAligned))
{
// We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it
// overlaps more buffers, so try again after each extension
// and ensure we cover all overlaps.
int oldOverlapsCount;
do
{
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
}
address &= ~(alignment - 1);
oldOverlapsCount = overlapsCount;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps);
}
while (oldOverlapsCount != overlapsCount);
lock (_buffers)
{
for (int index = 0; index < overlapsCount; index++)
{
_buffers.Remove(overlaps[index]);
}
}
ulong newSize = endAddress - address;
CreateBufferAligned(address, newSize, sparseAligned, overlaps, overlapsCount);
}
}
else
{
// No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size, sparseAligned);
lock (_buffers)
{
_buffers.Add(buffer);
}
}
ShrinkOverlapsBufferIfNeeded();
}
/// <summary>
/// Creates a new buffer for the specified range, if needed.
/// If a buffer where this range can be fully contained already exists,
/// then the creation of a new buffer is not necessary.
/// </summary>
/// <param name="address">Address of the buffer in guest memory</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param>
/// <param name="overlapsCount">Total of overlaps</param>
private void CreateBufferAligned(ulong address, ulong size, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
{
Buffer newBuffer = new Buffer(_context, _physicalMemory, address, size, sparseCompatible, overlaps.Take(overlapsCount));
lock (_buffers)
{
_buffers.Add(newBuffer);
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
int dstOffset = (int)(buffer.Address - newBuffer.Address);
buffer.CopyTo(newBuffer, dstOffset);
newBuffer.InheritModifiedRanges(buffer);
buffer.DecrementReferenceCount();
}
newBuffer.SynchronizeMemory(address, size);
// Existing buffers were modified, we need to rebind everything.
NotifyBuffersModified?.Invoke();
RecreateMultiRangeBuffers(address, size);
}
/// <summary>
/// Recreates all the multi-range buffers that overlaps a given physical memory range.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size of the range in bytes</param>
private void RecreateMultiRangeBuffers(ulong address, ulong size)
{
if ((address & (SparseBufferAlignmentSize - 1)) != 0 || (size & (SparseBufferAlignmentSize - 1)) != 0)
{
return;
}
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
int overlapCount = _multiRangeBuffers.FindOverlaps(address, size, ref overlaps);
for (int index = 0; index < overlapCount; index++)
{
_multiRangeBuffers.Remove(overlaps[index]);
overlaps[index].Dispose();
}
for (int index = 0; index < overlapCount; index++)
{
CreateMultiRangeBuffer(overlaps[index].Range);
}
}
/// <summary> /// <summary>
/// Resizes the temporary buffer used for range list intersection results, if it has grown too much. /// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
/// </summary> /// </summary>
@ -319,9 +620,63 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the copy</param> /// <param name="size">Size in bytes of the copy</param>
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size) public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
{ {
ulong srcAddress = TranslateAndCreateBuffer(memoryManager, srcVa, size); MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
ulong dstAddress = TranslateAndCreateBuffer(memoryManager, dstVa, size); MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
if (srcRange.Count == 1 && dstRange.Count == 1)
{
CopyBufferSingleRange(memoryManager, srcRange.GetSubRange(0).Address, dstRange.GetSubRange(0).Address, size);
}
else
{
ulong copiedSize = 0;
ulong srcOffset = 0;
ulong dstOffset = 0;
int srcRangeIndex = 0;
int dstRangeIndex = 0;
while (copiedSize < size)
{
if (srcRange.GetSubRange(srcRangeIndex).Size == srcOffset)
{
srcRangeIndex++;
srcOffset = 0;
}
if (dstRange.GetSubRange(dstRangeIndex).Size == dstOffset)
{
dstRangeIndex++;
dstOffset = 0;
}
MemoryRange srcSubRange = srcRange.GetSubRange(srcRangeIndex);
MemoryRange dstSubRange = dstRange.GetSubRange(dstRangeIndex);
ulong srcSize = srcSubRange.Size - srcOffset;
ulong dstSize = dstSubRange.Size - dstOffset;
ulong copySize = Math.Min(srcSize, dstSize);
CopyBufferSingleRange(memoryManager, srcSubRange.Address + srcOffset, dstSubRange.Address + dstOffset, copySize);
srcOffset += copySize;
dstOffset += copySize;
copiedSize += copySize;
}
}
}
/// <summary>
/// Copy a buffer data from a given address to another.
/// </summary>
/// <remarks>
/// This does a GPU side copy.
/// </remarks>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="srcAddress">Physical address of the copy source</param>
/// <param name="dstAddress">Physical address of the copy destination</param>
/// <param name="size">Size in bytes of the copy</param>
private void CopyBufferSingleRange(MemoryManager memoryManager, ulong srcAddress, ulong dstAddress, ulong size)
{
Buffer srcBuffer = GetBuffer(srcAddress, size); Buffer srcBuffer = GetBuffer(srcAddress, size);
Buffer dstBuffer = GetBuffer(dstAddress, size); Buffer dstBuffer = GetBuffer(dstAddress, size);
@ -360,39 +715,98 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="value">Value to be written into the buffer</param> /// <param name="value">Value to be written into the buffer</param>
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value) public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
{ {
ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size); MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
Buffer buffer = GetBuffer(address, size); for (int index = 0; index < range.Count; index++)
{
MemoryRange subRange = range.GetSubRange(index);
Buffer buffer = GetBuffer(subRange.Address, subRange.Size);
int offset = (int)(address - buffer.Address); int offset = (int)(subRange.Address - buffer.Address);
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)size, value); _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
memoryManager.Physical.FillTrackedResource(address, size, value, ResourceKind.Buffer); memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
}
} }
/// <summary> /// <summary>
/// Gets a buffer sub-range from a start address til a page boundary after the given size. /// Gets a buffer sub-range starting at a given memory address, aligned to the next page boundary.
/// </summary> /// </summary>
/// <param name="address">Start address of the memory range</param> /// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="size">Size in bytes of the memory range</param>
/// <param name="write">Whether the buffer will be written to by this use</param> /// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range starting at the given memory address</returns> /// <returns>The buffer sub-range starting at the given memory address</returns>
public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false) public BufferRange GetBufferRangeAligned(MultiRange range, bool write = false)
{ {
return GetBuffer(address, size, write).GetRangeAligned(address, size, write); if (range.Count > 1)
{
return GetBuffer(range, write).GetRange(range);
}
else
{
MemoryRange subRange = range.GetSubRange(0);
return GetBuffer(subRange.Address, subRange.Size, write).GetRangeAligned(subRange.Address, subRange.Size, write);
}
} }
/// <summary> /// <summary>
/// Gets a buffer sub-range for a given memory range. /// Gets a buffer sub-range for a given memory range.
/// </summary> /// </summary>
/// <param name="address">Start address of the memory range</param> /// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="size">Size in bytes of the memory range</param>
/// <param name="write">Whether the buffer will be written to by this use</param> /// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range for the given range</returns> /// <returns>The buffer sub-range for the given range</returns>
public BufferRange GetBufferRange(ulong address, ulong size, bool write = false) public BufferRange GetBufferRange(MultiRange range, bool write = false)
{ {
return GetBuffer(address, size, write).GetRange(address, size, write); if (range.Count > 1)
{
return GetBuffer(range, write).GetRange(range);
}
else
{
MemoryRange subRange = range.GetSubRange(0);
return GetBuffer(subRange.Address, subRange.Size, write).GetRange(subRange.Address, subRange.Size, write);
}
}
/// <summary>
/// Gets a buffer for a given memory range.
/// A buffer overlapping with the specified range is assumed to already exist on the cache.
/// </summary>
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer where the range is fully contained</returns>
private MultiRangeBuffer GetBuffer(MultiRange range, bool write = false)
{
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
if (write)
{
subBuffer.SignalModified(subRange.Address, subRange.Size);
}
}
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
MultiRangeBuffer buffer = null;
for (int i = 0; i < overlapCount; i++)
{
if (overlaps[i].Range.Contains(range))
{
buffer = overlaps[i];
break;
}
}
return buffer;
} }
/// <summary> /// <summary>
@ -426,12 +840,33 @@ namespace Ryujinx.Graphics.Gpu.Memory
return buffer; return buffer;
} }
/// <summary>
/// Performs guest to host memory synchronization of a given memory range.
/// </summary>
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
public void SynchronizeBufferRange(MultiRange range)
{
if (range.Count == 1)
{
MemoryRange subRange = range.GetSubRange(0);
SynchronizeBufferRange(subRange.Address, subRange.Size);
}
else
{
for (int index = 0; index < range.Count; index++)
{
MemoryRange subRange = range.GetSubRange(index);
SynchronizeBufferRange(subRange.Address, subRange.Size);
}
}
}
/// <summary> /// <summary>
/// Performs guest to host memory synchronization of a given memory range. /// Performs guest to host memory synchronization of a given memory range.
/// </summary> /// </summary>
/// <param name="address">Start address of the memory range</param> /// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param> /// <param name="size">Size in bytes of the memory range</param>
public void SynchronizeBufferRange(ulong address, ulong size) private void SynchronizeBufferRange(ulong address, ulong size)
{ {
if (size != 0) if (size != 0)
{ {
@ -491,7 +926,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary> /// <summary>
/// Disposes all buffers in the cache. /// Disposes all buffers in the cache.
/// It's an error to use the buffer manager after disposal. /// It's an error to use the buffer cache after disposal.
/// </summary> /// </summary>
public void Dispose() public void Dispose()
{ {

View file

@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Shader; using Ryujinx.Graphics.Gpu.Shader;
using Ryujinx.Graphics.Shader; using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
@ -62,18 +63,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
Bindings = new BufferDescriptor[count]; Bindings = new BufferDescriptor[count];
Buffers = new BufferBounds[count]; Buffers = new BufferBounds[count];
Unaligned = new bool[count]; Unaligned = new bool[count];
Buffers.AsSpan().Fill(new BufferBounds(new MultiRange(MemoryManager.PteUnmapped, 0UL)));
} }
/// <summary> /// <summary>
/// Sets the region of a buffer at a given slot. /// Sets the region of a buffer at a given slot.
/// </summary> /// </summary>
/// <param name="index">Buffer slot</param> /// <param name="index">Buffer slot</param>
/// <param name="address">Region virtual address</param> /// <param name="range">Physical memory regions where the buffer is mapped</param>
/// <param name="size">Region size in bytes</param>
/// <param name="flags">Buffer usage flags</param> /// <param name="flags">Buffer usage flags</param>
public void SetBounds(int index, ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None) public void SetBounds(int index, MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{ {
Buffers[index] = new BufferBounds(address, size, flags); Buffers[index] = new BufferBounds(range, flags);
} }
/// <summary> /// <summary>
@ -120,6 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context = context; _context = context;
_channel = channel; _channel = channel;
_indexBuffer.Range = new MultiRange(MemoryManager.PteUnmapped, 0UL);
_vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers]; _vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers];
_transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers]; _transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers];
@ -150,10 +153,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="type">Type of each index buffer element</param> /// <param name="type">Type of each index buffer element</param>
public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type) public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type)
{ {
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_indexBuffer.Address = address; _indexBuffer.Range = range;
_indexBuffer.Size = size;
_indexBuffer.Type = type; _indexBuffer.Type = type;
_indexBufferDirty = true; _indexBufferDirty = true;
@ -181,16 +183,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="divisor">Vertex divisor of the buffer, for instanced draws</param> /// <param name="divisor">Vertex divisor of the buffer, for instanced draws</param>
public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor) public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor)
{ {
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_vertexBuffers[index].Address = address; _vertexBuffers[index].Range = range;
_vertexBuffers[index].Size = size;
_vertexBuffers[index].Stride = stride; _vertexBuffers[index].Stride = stride;
_vertexBuffers[index].Divisor = divisor; _vertexBuffers[index].Divisor = divisor;
_vertexBuffersDirty = true; _vertexBuffersDirty = true;
if (address != 0) if (!range.IsUnmapped)
{ {
_vertexBuffersEnableMask |= 1u << index; _vertexBuffersEnableMask |= 1u << index;
} }
@ -209,9 +210,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the transform feedback buffer</param> /// <param name="size">Size in bytes of the transform feedback buffer</param>
public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size) public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size)
{ {
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
_transformFeedbackBuffers[index] = new BufferBounds(address, size); _transformFeedbackBuffers[index] = new BufferBounds(range);
_transformFeedbackBuffersDirty = true; _transformFeedbackBuffersDirty = true;
} }
@ -256,9 +257,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
_cpStorageBuffers.SetBounds(index, address, size, flags); _cpStorageBuffers.SetBounds(index, range, flags);
} }
/// <summary> /// <summary>
@ -280,15 +281,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment); gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
if (buffers.Buffers[index].Address != address || if (!buffers.Buffers[index].Range.Equals(range))
buffers.Buffers[index].Size != size)
{ {
_gpStorageBuffersDirty = true; _gpStorageBuffersDirty = true;
} }
buffers.SetBounds(index, address, size, flags); buffers.SetBounds(index, range, flags);
} }
/// <summary> /// <summary>
@ -300,9 +300,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the storage buffer</param> /// <param name="size">Size in bytes of the storage buffer</param>
public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size) public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size)
{ {
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_cpUniformBuffers.SetBounds(index, address, size); _cpUniformBuffers.SetBounds(index, range);
} }
/// <summary> /// <summary>
@ -315,9 +315,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the storage buffer</param> /// <param name="size">Size in bytes of the storage buffer</param>
public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size) public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size)
{ {
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size); MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_gpUniformBuffers[stage].SetBounds(index, address, size); _gpUniformBuffers[stage].SetBounds(index, range);
_gpUniformBuffersDirty = true; _gpUniformBuffersDirty = true;
} }
@ -379,7 +379,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++) for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++)
{ {
if (_cpUniformBuffers.Buffers[i].Address != 0) if (!_cpUniformBuffers.Buffers[i].IsUnmapped)
{ {
mask |= 1u << i; mask |= 1u << i;
} }
@ -399,7 +399,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++) for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++)
{ {
if (_gpUniformBuffers[stage].Buffers[i].Address != 0) if (!_gpUniformBuffers[stage].Buffers[i].IsUnmapped)
{ {
mask |= 1u << i; mask |= 1u << i;
} }
@ -415,7 +415,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns> /// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
public ulong GetComputeUniformBufferAddress(int index) public ulong GetComputeUniformBufferAddress(int index)
{ {
return _cpUniformBuffers.Buffers[index].Address; return _cpUniformBuffers.Buffers[index].Range.GetSubRange(0).Address;
} }
/// <summary> /// <summary>
@ -426,7 +426,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns> /// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
public ulong GetGraphicsUniformBufferAddress(int stage, int index) public ulong GetGraphicsUniformBufferAddress(int stage, int index)
{ {
return _gpUniformBuffers[stage].Buffers[index].Address; return _gpUniformBuffers[stage].Buffers[index].Range.GetSubRange(0).Address;
} }
/// <summary> /// <summary>
@ -477,7 +477,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
foreach (var binding in _bufferTextures) foreach (var binding in _bufferTextures)
{ {
var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore); var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore);
var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Address, binding.Size, isStore); var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Range, isStore);
binding.Texture.SetStorage(range); binding.Texture.SetStorage(range);
// The texture must be rebound to use the new storage if it was updated. // The texture must be rebound to use the new storage if it was updated.
@ -511,16 +511,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
_indexBufferDirty = false; _indexBufferDirty = false;
if (_indexBuffer.Address != 0) if (!_indexBuffer.Range.IsUnmapped)
{ {
BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Address, _indexBuffer.Size); BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Range);
_context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type); _context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type);
} }
} }
else if (_indexBuffer.Address != 0) else if (!_indexBuffer.Range.IsUnmapped)
{ {
bufferCache.SynchronizeBufferRange(_indexBuffer.Address, _indexBuffer.Size); bufferCache.SynchronizeBufferRange(_indexBuffer.Range);
} }
} }
else if (_rebind) else if (_rebind)
@ -540,12 +540,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
VertexBuffer vb = _vertexBuffers[index]; VertexBuffer vb = _vertexBuffers[index];
if (vb.Address == 0) if (vb.Range.IsUnmapped)
{ {
continue; continue;
} }
BufferRange buffer = bufferCache.GetBufferRange(vb.Address, vb.Size); BufferRange buffer = bufferCache.GetBufferRange(vb.Range);
vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor); vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor);
} }
@ -558,12 +558,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
VertexBuffer vb = _vertexBuffers[index]; VertexBuffer vb = _vertexBuffers[index];
if (vb.Address == 0) if (vb.Range.IsUnmapped)
{ {
continue; continue;
} }
bufferCache.SynchronizeBufferRange(vb.Address, vb.Size); bufferCache.SynchronizeBufferRange(vb.Range);
} }
} }
@ -579,13 +579,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
BufferBounds tfb = _transformFeedbackBuffers[index]; BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.Address == 0) if (tfb.IsUnmapped)
{ {
tfbs[index] = BufferRange.Empty; tfbs[index] = BufferRange.Empty;
continue; continue;
} }
tfbs[index] = bufferCache.GetBufferRange(tfb.Address, tfb.Size, write: true); tfbs[index] = bufferCache.GetBufferRange(tfb.Range, write: true);
} }
_context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs); _context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs);
@ -600,21 +600,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
BufferBounds tfb = _transformFeedbackBuffers[index]; BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.Address == 0) if (tfb.IsUnmapped)
{ {
buffers[index] = new BufferAssignment(index, BufferRange.Empty); buffers[index] = new BufferAssignment(index, BufferRange.Empty);
} }
else else
{ {
ulong endAddress = tfb.Address + tfb.Size; MultiRange range = tfb.Range;
ulong address = BitUtils.AlignDown(tfb.Address, (ulong)alignment); ulong address0 = range.GetSubRange(0).Address;
ulong size = endAddress - address; ulong address = BitUtils.AlignDown(address0, (ulong)alignment);
int tfeOffset = ((int)tfb.Address & (alignment - 1)) / 4; if (range.Count == 1)
{
range = new MultiRange(address, range.GetSubRange(0).Size + (address0 - address));
}
else
{
MemoryRange[] subRanges = new MemoryRange[range.Count];
subRanges[0] = new MemoryRange(address, range.GetSubRange(0).Size + (address0 - address));
for (int i = 1; i < range.Count; i++)
{
subRanges[i] = range.GetSubRange(i);
}
range = new MultiRange(subRanges);
}
int tfeOffset = ((int)address0 & (alignment - 1)) / 4;
_context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset); _context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset);
buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(address, size, write: true)); buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(range, write: true));
} }
} }
@ -627,12 +645,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
BufferBounds tfb = _transformFeedbackBuffers[index]; BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.Address == 0) if (tfb.IsUnmapped)
{ {
continue; continue;
} }
bufferCache.SynchronizeBufferRange(tfb.Address, tfb.Size); bufferCache.SynchronizeBufferRange(tfb.Range);
} }
} }
@ -688,12 +706,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot]; BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
if (bounds.Address != 0) if (!bounds.IsUnmapped)
{ {
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage var range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) ? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
: bufferCache.GetBufferRange(bounds.Address, bounds.Size); : bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
} }
@ -725,12 +743,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot]; BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
if (bounds.Address != 0) if (!bounds.IsUnmapped)
{ {
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage var range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) ? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
: bufferCache.GetBufferRange(bounds.Address, bounds.Size); : bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
} }
@ -778,12 +796,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[binding.Slot]; BufferBounds bounds = buffers.Buffers[binding.Slot];
if (bounds.Address == 0) if (bounds.IsUnmapped)
{ {
continue; continue;
} }
_channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Address, bounds.Size); _channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Range);
} }
} }
} }
@ -793,23 +811,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
/// <param name="stage">Shader stage accessing the texture</param> /// <param name="stage">Shader stage accessing the texture</param>
/// <param name="texture">Buffer texture</param> /// <param name="texture">Buffer texture</param>
/// <param name="address">Address of the buffer in memory</param> /// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
/// <param name="size">Size of the buffer in bytes</param>
/// <param name="bindingInfo">Binding info for the buffer texture</param> /// <param name="bindingInfo">Binding info for the buffer texture</param>
/// <param name="format">Format of the buffer texture</param> /// <param name="format">Format of the buffer texture</param>
/// <param name="isImage">Whether the binding is for an image or a sampler</param> /// <param name="isImage">Whether the binding is for an image or a sampler</param>
public void SetBufferTextureStorage( public void SetBufferTextureStorage(
ShaderStage stage, ShaderStage stage,
ITexture texture, ITexture texture,
ulong address, MultiRange range,
ulong size,
TextureBindingInfo bindingInfo, TextureBindingInfo bindingInfo,
Format format, Format format,
bool isImage) bool isImage)
{ {
_channel.MemoryManager.Physical.BufferCache.CreateBuffer(address, size); _channel.MemoryManager.Physical.BufferCache.CreateBuffer(range);
_bufferTextures.Add(new BufferTextureBinding(stage, texture, address, size, bindingInfo, format, isImage)); _bufferTextures.Add(new BufferTextureBinding(stage, texture, range, bindingInfo, format, isImage));
} }
/// <summary> /// <summary>

View file

@ -1,6 +1,7 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Shader; using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -20,14 +21,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
public ITexture Texture { get; } public ITexture Texture { get; }
/// <summary> /// <summary>
/// The base address of the buffer binding. /// Physical ranges of memory where the buffer texture data is located.
/// </summary> /// </summary>
public ulong Address { get; } public MultiRange Range { get; }
/// <summary>
/// The size of the buffer binding in bytes.
/// </summary>
public ulong Size { get; }
/// <summary> /// <summary>
/// The image or sampler binding info for the buffer texture. /// The image or sampler binding info for the buffer texture.
@ -49,24 +45,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
/// <param name="stage">Shader stage accessing the texture</param> /// <param name="stage">Shader stage accessing the texture</param>
/// <param name="texture">Buffer texture</param> /// <param name="texture">Buffer texture</param>
/// <param name="address">Base address</param> /// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
/// <param name="size">Size in bytes</param>
/// <param name="bindingInfo">Binding info</param> /// <param name="bindingInfo">Binding info</param>
/// <param name="format">Binding format</param> /// <param name="format">Binding format</param>
/// <param name="isImage">Whether the binding is for an image or a sampler</param> /// <param name="isImage">Whether the binding is for an image or a sampler</param>
public BufferTextureBinding( public BufferTextureBinding(
ShaderStage stage, ShaderStage stage,
ITexture texture, ITexture texture,
ulong address, MultiRange range,
ulong size,
TextureBindingInfo bindingInfo, TextureBindingInfo bindingInfo,
Format format, Format format,
bool isImage) bool isImage)
{ {
Stage = stage; Stage = stage;
Texture = texture; Texture = texture;
Address = address; Range = range;
Size = size;
BindingInfo = bindingInfo; BindingInfo = bindingInfo;
Format = format; Format = format;
IsImage = isImage; IsImage = isImage;

View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -7,9 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
struct IndexBuffer struct IndexBuffer
{ {
public ulong Address; public MultiRange Range;
public ulong Size;
public IndexType Type; public IndexType Type;
} }
} }

View file

@ -39,6 +39,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
internal PhysicalMemory Physical { get; } internal PhysicalMemory Physical { get; }
/// <summary>
/// Virtual buffer cache.
/// </summary>
internal VirtualBufferCache VirtualBufferCache { get; }
/// <summary> /// <summary>
/// Cache of GPU counters. /// Cache of GPU counters.
/// </summary> /// </summary>
@ -51,10 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
internal MemoryManager(PhysicalMemory physicalMemory) internal MemoryManager(PhysicalMemory physicalMemory)
{ {
Physical = physicalMemory; Physical = physicalMemory;
VirtualBufferCache = new VirtualBufferCache(this);
CounterCache = new CounterCache(); CounterCache = new CounterCache();
_pageTable = new ulong[PtLvl0Size][]; _pageTable = new ulong[PtLvl0Size][];
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler;
MemoryUnmapped += CounterCache.MemoryUnmappedHandler; MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
} }
@ -508,6 +515,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
regionSize += Math.Min(endVa - va, PageSize); regionSize += Math.Min(endVa - va, PageSize);
} }
if (regions.Count == 0)
{
return new MultiRange(regionStart, regionSize);
}
regions.Add(new MemoryRange(regionStart, regionSize)); regions.Add(new MemoryRange(regionStart, regionSize));
return new MultiRange(regions.ToArray()); return new MultiRange(regions.ToArray());

View file

@ -0,0 +1,60 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System;
namespace Ryujinx.Graphics.Gpu.Memory
{
/// <summary>
/// Buffer, used to store vertex and index data, uniform and storage buffers, and others.
/// </summary>
class MultiRangeBuffer : IMultiRangeItem, IDisposable
{
private readonly GpuContext _context;
/// <summary>
/// Host buffer handle.
/// </summary>
public BufferHandle Handle { get; }
/// <summary>
/// Range of memory where the data is located.
/// </summary>
public MultiRange Range { get; }
/// <summary>
/// Creates a new instance of the buffer.
/// </summary>
/// <param name="context">GPU context that the buffer belongs to</param>
/// <param name="range">Range of memory where the data is mapped</param>
/// <param name="storages">Backing memory for the buffers</param>
public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
{
_context = context;
Range = range;
Handle = context.Renderer.CreateBufferSparse(storages);
}
/// <summary>
/// Gets a sub-range from the buffer.
/// </summary>
/// <remarks>
/// This can be used to bind and use sub-ranges of the buffer on the host API.
/// </remarks>
/// <param name="range">Range of memory where the data is mapped</param>
/// <returns>The buffer sub-range</returns>
public BufferRange GetRange(MultiRange range)
{
int offset = Range.FindOffset(range);
return new BufferRange(Handle, offset, (int)range.GetSize());
}
/// <summary>
/// Disposes the host buffer.
/// </summary>
public void Dispose()
{
_context.Renderer.DeleteBuffer(Handle);
}
}
}

View file

@ -1,3 +1,5 @@
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
/// <summary> /// <summary>
@ -5,8 +7,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
struct VertexBuffer struct VertexBuffer
{ {
public ulong Address; public MultiRange Range;
public ulong Size;
public int Stride; public int Stride;
public int Divisor; public int Divisor;
} }

View file

@ -0,0 +1,238 @@
using Ryujinx.Memory.Range;
using System;
using System.Collections.Concurrent;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory
{
/// <summary>
/// Virtual buffer cache.
/// </summary>
class VirtualBufferCache
{
private readonly MemoryManager _memoryManager;
/// <summary>
/// Represents a GPU virtual memory range.
/// </summary>
private readonly struct VirtualRange : IRange
{
/// <summary>
/// GPU virtual address where the range starts.
/// </summary>
public ulong Address { get; }
/// <summary>
/// Size of the range in bytes.
/// </summary>
public ulong Size { get; }
/// <summary>
/// GPU virtual address where the range ends.
/// </summary>
public ulong EndAddress => Address + Size;
/// <summary>
/// Physical regions where the GPU virtual region is mapped.
/// </summary>
public MultiRange Range { get; }
/// <summary>
/// Creates a new virtual memory range.
/// </summary>
/// <param name="address">GPU virtual address where the range starts</param>
/// <param name="size">Size of the range in bytes</param>
/// <param name="range">Physical regions where the GPU virtual region is mapped</param>
public VirtualRange(ulong address, ulong size, MultiRange range)
{
Address = address;
Size = size;
Range = range;
}
/// <summary>
/// Checks if a given range overlaps with the buffer.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <returns>True if the range overlaps, false otherwise</returns>
public bool OverlapsWith(ulong address, ulong size)
{
return Address < address + size && address < EndAddress;
}
}
private readonly RangeList<VirtualRange> _virtualRanges;
private VirtualRange[] _virtualRangeOverlaps;
private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps;
private int _hasDeferredUnmaps;
/// <summary>
/// Creates a new instance of the virtual buffer cache.
/// </summary>
/// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param>
public VirtualBufferCache(MemoryManager memoryManager)
{
_memoryManager = memoryManager;
_virtualRanges = new RangeList<VirtualRange>();
_virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity];
_deferredUnmaps = new ConcurrentQueue<VirtualRange>();
}
/// <summary>
/// Handles removal of buffers written to a memory region being unmapped.
/// </summary>
/// <param name="sender">Sender object</param>
/// <param name="e">Event arguments</param>
public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
{
void EnqueueUnmap()
{
_deferredUnmaps.Enqueue(new VirtualRange(e.Address, e.Size, default));
Interlocked.Exchange(ref _hasDeferredUnmaps, 1);
}
e.AddRemapAction(EnqueueUnmap);
}
/// <summary>
/// Tries to get a existing, cached physical range for the specified virtual region.
/// If no cached range is found, a new one is created and added.
/// </summary>
/// <param name="gpuVa">GPU virtual address to get the physical range from</param>
/// <param name="size">Size in bytes of the region</param>
/// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
/// <param name="range">Physical range for the specified GPU virtual region</param>
/// <returns>True if the range already existed, false if a new one was created and added</returns>
public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range)
{
VirtualRange[] overlaps = _virtualRangeOverlaps;
int overlapsCount;
if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0)
{
while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange))
{
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps);
for (int index = 0; index < overlapsCount; index++)
{
_virtualRanges.Remove(overlaps[index]);
}
}
}
bool found = false;
ulong originalVa = gpuVa;
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps);
if (overlapsCount != 0)
{
// The virtual range already exists. We just need to check if our range fits inside
// the existing one, and if not, we must extend the existing one.
ulong endAddress = gpuVa + size;
VirtualRange overlap0 = overlaps[0];
if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress)
{
for (int index = 0; index < overlapsCount; index++)
{
VirtualRange virtualRange = overlaps[index];
gpuVa = Math.Min(gpuVa, virtualRange.Address);
endAddress = Math.Max(endAddress, virtualRange.EndAddress);
_virtualRanges.Remove(virtualRange);
}
ulong newSize = endAddress - gpuVa;
MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize);
_virtualRanges.Add(new(gpuVa, newSize, newRange));
range = newRange.Slice(originalVa - gpuVa, size);
}
else
{
found = true;
range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
}
}
else
{
// No overlap, just create a new virtual range.
range = _memoryManager.GetPhysicalRegions(gpuVa, size);
VirtualRange virtualRange = new(gpuVa, size, range);
_virtualRanges.Add(virtualRange);
}
ShrinkOverlapsBufferIfNeeded();
// If the the range is not properly aligned for sparse mapping,
// or if the host does not support sparse mapping, let's just
// force it to a single range.
// This might cause issues in some applications that uses sparse
// mappings.
if (!IsSparseAligned(range) || !supportsSparse)
{
range = new MultiRange(range.GetSubRange(0).Address, size);
}
return found;
}
/// <summary>
/// Checks if the physical memory ranges are valid for sparse mapping,
/// which requires all sub-ranges to be 64KB aligned.
/// </summary>
/// <param name="range">Range to check</param>
/// <returns>True if the range is valid for sparse mapping, false otherwise</returns>
private static bool IsSparseAligned(MultiRange range)
{
if (range.Count == 1)
{
return (range.GetSubRange(0).Address & (BufferCache.SparseBufferAlignmentSize - 1)) == 0;
}
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
// Check if address is aligned. The address of the first sub-range can
// be misaligned as it is at the start.
if (i > 0 &&
subRange.Address != MemoryManager.PteUnmapped &&
(subRange.Address & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
{
return false;
}
// Check if the size is aligned. The size of the last sub-range can
// be misaligned as it is at the end.
if (i < range.Count - 1 && (subRange.Size & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
{
return false;
}
}
return true;
}
/// <summary>
/// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
/// </summary>
private void ShrinkOverlapsBufferIfNeeded()
{
if (_virtualRangeOverlaps.Length > BufferCache.OverlapsBufferMaxCapacity)
{
Array.Resize(ref _virtualRangeOverlaps, BufferCache.OverlapsBufferMaxCapacity);
}
}
}
}

View file

@ -730,8 +730,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
codeA ??= memoryManager.GetSpan(vertexA.Address, vertexA.Size).ToArray(); codeA ??= memoryManager.GetSpan(vertexA.Address, vertexA.Size).ToArray();
codeB ??= memoryManager.GetSpan(currentStage.Address, currentStage.Size).ToArray(); codeB ??= memoryManager.GetSpan(currentStage.Address, currentStage.Size).ToArray();
byte[] cb1DataA = memoryManager.Physical.GetSpan(cb1DataAddress, vertexA.Cb1DataSize).ToArray(); byte[] cb1DataA = ReadArray(memoryManager, cb1DataAddress, vertexA.Cb1DataSize);
byte[] cb1DataB = memoryManager.Physical.GetSpan(cb1DataAddress, currentStage.Cb1DataSize).ToArray(); byte[] cb1DataB = ReadArray(memoryManager, cb1DataAddress, currentStage.Cb1DataSize);
ShaderDumpPaths pathsA = default; ShaderDumpPaths pathsA = default;
ShaderDumpPaths pathsB = default; ShaderDumpPaths pathsB = default;
@ -770,7 +770,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
? channel.BufferManager.GetComputeUniformBufferAddress(1) ? channel.BufferManager.GetComputeUniformBufferAddress(1)
: channel.BufferManager.GetGraphicsUniformBufferAddress(StageToStageIndex(context.Stage), 1); : channel.BufferManager.GetGraphicsUniformBufferAddress(StageToStageIndex(context.Stage), 1);
byte[] cb1Data = memoryManager.Physical.GetSpan(cb1DataAddress, context.Cb1DataSize).ToArray(); byte[] cb1Data = ReadArray(memoryManager, cb1DataAddress, context.Cb1DataSize);
code ??= memoryManager.GetSpan(context.Address, context.Size).ToArray(); code ??= memoryManager.GetSpan(context.Address, context.Size).ToArray();
ShaderDumpPaths paths = dumper?.Dump(code, context.Stage == ShaderStage.Compute) ?? default; ShaderDumpPaths paths = dumper?.Dump(code, context.Stage == ShaderStage.Compute) ?? default;
@ -781,6 +781,23 @@ namespace Ryujinx.Graphics.Gpu.Shader
return new TranslatedShader(new CachedShaderStage(program.Info, code, cb1Data), program); return new TranslatedShader(new CachedShaderStage(program.Info, code, cb1Data), program);
} }
/// <summary>
/// Reads data from physical memory, returns an empty array if the memory is unmapped or size is 0.
/// </summary>
/// <param name="memoryManager">Memory manager with the physical memory to read from</param>
/// <param name="address">Physical address of the region to read</param>
/// <param name="size">Size in bytes of the data</param>
/// <returns>An array with the data at the specified memory location</returns>
private static byte[] ReadArray(MemoryManager memoryManager, ulong address, int size)
{
if (address == MemoryManager.PteUnmapped || size == 0)
{
return Array.Empty<byte>();
}
return memoryManager.Physical.GetSpan(address, size).ToArray();
}
/// <summary> /// <summary>
/// Gets the index of a stage from a <see cref="ShaderStage"/>. /// Gets the index of a stage from a <see cref="ShaderStage"/>.
/// </summary> /// </summary>

View file

@ -611,7 +611,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
{ {
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex); ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex);
cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedTextureBufferIndex = textureBufferIndex; cachedTextureBufferIndex = textureBufferIndex;
if (samplerBufferIndex == textureBufferIndex) if (samplerBufferIndex == textureBufferIndex)
@ -625,7 +625,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
{ {
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex); ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex);
cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedSamplerBufferIndex = samplerBufferIndex; cachedSamplerBufferIndex = samplerBufferIndex;
} }

View file

@ -57,16 +57,11 @@ namespace Ryujinx.Graphics.OpenGL
ResourcePool = new ResourcePool(); ResourcePool = new ResourcePool();
} }
public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{
return CreateBuffer(size, GAL.BufferAccess.Default);
}
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access) public BufferHandle CreateBuffer(int size, GAL.BufferAccess access)
{ {
BufferCount++; BufferCount++;
if (access == GAL.BufferAccess.FlushPersistent) if (access.HasFlag(GAL.BufferAccess.FlushPersistent))
{ {
BufferHandle handle = Buffer.CreatePersistent(size); BufferHandle handle = Buffer.CreatePersistent(size);
@ -80,11 +75,21 @@ namespace Ryujinx.Graphics.OpenGL
} }
} }
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access, BufferHandle storageHint)
{
return CreateBuffer(size, access);
}
public BufferHandle CreateBuffer(nint pointer, int size) public BufferHandle CreateBuffer(nint pointer, int size)
{ {
throw new NotSupportedException(); throw new NotSupportedException();
} }
public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
{
throw new NotSupportedException();
}
public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info) public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info)
{ {
return new Program(shaders, info.FragmentOutputMap); return new Program(shaders, info.FragmentOutputMap);
@ -148,6 +153,7 @@ namespace Ryujinx.Graphics.OpenGL
supportsR4G4B4A4Format: true, supportsR4G4B4A4Format: true,
supportsSnormBufferTextureFormat: false, supportsSnormBufferTextureFormat: false,
supports5BitComponentFormat: true, supports5BitComponentFormat: true,
supportsSparseBuffer: false,
supportsBlendEquationAdvanced: HwCapabilities.SupportsBlendEquationAdvanced, supportsBlendEquationAdvanced: HwCapabilities.SupportsBlendEquationAdvanced,
supportsFragmentShaderInterlock: HwCapabilities.SupportsFragmentShaderInterlock, supportsFragmentShaderInterlock: HwCapabilities.SupportsFragmentShaderInterlock,
supportsFragmentShaderOrderingIntel: HwCapabilities.SupportsFragmentShaderOrdering, supportsFragmentShaderOrderingIntel: HwCapabilities.SupportsFragmentShaderOrdering,

View file

@ -8,5 +8,6 @@ namespace Ryujinx.Graphics.Vulkan
HostMapped, HostMapped,
DeviceLocal, DeviceLocal,
DeviceLocalMapped, DeviceLocalMapped,
Sparse,
} }
} }

View file

@ -46,7 +46,7 @@ namespace Ryujinx.Graphics.Vulkan
private bool _lastAccessIsWrite; private bool _lastAccessIsWrite;
private readonly BufferAllocationType _baseType; private BufferAllocationType _baseType;
private BufferAllocationType _currentType; private BufferAllocationType _currentType;
private bool _swapQueued; private bool _swapQueued;
@ -109,6 +109,22 @@ namespace Ryujinx.Graphics.Vulkan
_flushLock = new ReaderWriterLockSlim(); _flushLock = new ReaderWriterLockSlim();
} }
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, int size, Auto<MemoryAllocation>[] storageAllocations)
{
_gd = gd;
_device = device;
_waitable = new MultiFenceHolder(size);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, storageAllocations);
_bufferHandle = buffer.Handle;
Size = size;
_baseType = BufferAllocationType.Sparse;
_currentType = BufferAllocationType.Sparse;
DesiredType = BufferAllocationType.Sparse;
_flushLock = new ReaderWriterLockSlim();
}
public bool TryBackingSwap(ref CommandBufferScoped? cbs) public bool TryBackingSwap(ref CommandBufferScoped? cbs)
{ {
if (_swapQueued && DesiredType != _currentType) if (_swapQueued && DesiredType != _currentType)
@ -122,7 +138,7 @@ namespace Ryujinx.Graphics.Vulkan
var currentBuffer = _buffer; var currentBuffer = _buffer;
IntPtr currentMap = _map; IntPtr currentMap = _map;
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, _currentType); (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, false, _currentType);
if (buffer.Handle != 0) if (buffer.Handle != 0)
{ {
@ -253,6 +269,14 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
public void Pin()
{
if (_baseType == BufferAllocationType.Auto)
{
_baseType = _currentType;
}
}
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView) public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView)
{ {
var bufferViewCreateInfo = new BufferViewCreateInfo var bufferViewCreateInfo = new BufferViewCreateInfo
@ -506,6 +530,16 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
public Auto<MemoryAllocation> GetAllocation()
{
return _allocationAuto;
}
public (DeviceMemory, ulong) GetDeviceMemoryAndOffset()
{
return (_allocation.Memory, _allocation.Offset);
}
public void SignalWrite(int offset, int size) public void SignalWrite(int offset, int size)
{ {
ConsiderBackingSwap(); ConsiderBackingSwap();
@ -1072,7 +1106,7 @@ namespace Ryujinx.Graphics.Vulkan
} }
else else
{ {
_allocationAuto.Dispose(); _allocationAuto?.Dispose();
} }
_flushLock.EnterWriteLock(); _flushLock.EnterWriteLock();

View file

@ -96,25 +96,131 @@ namespace Ryujinx.Graphics.Vulkan
return Unsafe.As<ulong, BufferHandle>(ref handle64); return Unsafe.As<ulong, BufferHandle>(ref handle64);
} }
public unsafe BufferHandle CreateSparse(VulkanRenderer gd, ReadOnlySpan<BufferRange> storageBuffers)
{
var usage = DefaultBufferUsageFlags;
if (gd.Capabilities.SupportsIndirectParameters)
{
usage |= BufferUsageFlags.IndirectBufferBit;
}
ulong size = 0;
foreach (BufferRange range in storageBuffers)
{
size += (ulong)range.Size;
}
var bufferCreateInfo = new BufferCreateInfo()
{
SType = StructureType.BufferCreateInfo,
Size = size,
Usage = usage,
SharingMode = SharingMode.Exclusive,
Flags = BufferCreateFlags.SparseBindingBit | BufferCreateFlags.SparseAliasedBit
};
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
var memoryBinds = new SparseMemoryBind[storageBuffers.Length];
var storageAllocations = new Auto<MemoryAllocation>[storageBuffers.Length];
int storageAllocationsCount = 0;
ulong dstOffset = 0;
for (int index = 0; index < storageBuffers.Length; index++)
{
BufferRange range = storageBuffers[index];
if (TryGetBuffer(range.Handle, out var existingHolder))
{
// Since this buffer now also owns the memory from the referenced buffer,
// we pin it to ensure the memory location will not change.
existingHolder.Pin();
(var memory, var offset) = existingHolder.GetDeviceMemoryAndOffset();
memoryBinds[index] = new SparseMemoryBind()
{
ResourceOffset = dstOffset,
Size = (ulong)range.Size,
Memory = memory,
MemoryOffset = offset + (ulong)range.Offset,
Flags = SparseMemoryBindFlags.None
};
storageAllocations[storageAllocationsCount++] = existingHolder.GetAllocation();
}
else
{
memoryBinds[index] = new SparseMemoryBind()
{
ResourceOffset = dstOffset,
Size = (ulong)range.Size,
Memory = default,
MemoryOffset = 0UL,
Flags = SparseMemoryBindFlags.None
};
}
dstOffset += (ulong)range.Size;
}
if (storageAllocations.Length != storageAllocationsCount)
{
Array.Resize(ref storageAllocations, storageAllocationsCount);
}
fixed (SparseMemoryBind* pMemoryBinds = memoryBinds)
{
SparseBufferMemoryBindInfo bufferBind = new SparseBufferMemoryBindInfo()
{
Buffer = buffer,
BindCount = (uint)memoryBinds.Length,
PBinds = pMemoryBinds
};
BindSparseInfo bindSparseInfo = new BindSparseInfo()
{
SType = StructureType.BindSparseInfo,
BufferBindCount = 1,
PBufferBinds = &bufferBind
};
gd.Api.QueueBindSparse(gd.Queue, 1, bindSparseInfo, default).ThrowOnError();
}
var holder = new BufferHolder(gd, _device, buffer, (int)size, storageAllocations);
BufferCount++;
ulong handle64 = (uint)_buffers.Add(holder);
return Unsafe.As<ulong, BufferHandle>(ref handle64);
}
public BufferHandle CreateWithHandle( public BufferHandle CreateWithHandle(
VulkanRenderer gd, VulkanRenderer gd,
int size, int size,
bool sparseCompatible = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default, BufferHandle storageHint = default,
bool forceMirrors = false) bool forceMirrors = false)
{ {
return CreateWithHandle(gd, size, out _, baseType, storageHint, forceMirrors); return CreateWithHandle(gd, size, out _, sparseCompatible, baseType, storageHint, forceMirrors);
} }
public BufferHandle CreateWithHandle( public BufferHandle CreateWithHandle(
VulkanRenderer gd, VulkanRenderer gd,
int size, int size,
out BufferHolder holder, out BufferHolder holder,
bool sparseCompatible = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default, BufferHandle storageHint = default,
bool forceMirrors = false) bool forceMirrors = false)
{ {
holder = Create(gd, size, baseType: baseType, storageHint: storageHint); holder = Create(gd, size, forConditionalRendering: false, sparseCompatible, baseType, storageHint);
if (holder == null) if (holder == null)
{ {
return BufferHandle.Null; return BufferHandle.Null;
@ -163,6 +269,7 @@ namespace Ryujinx.Graphics.Vulkan
int size, int size,
BufferAllocationType type, BufferAllocationType type,
bool forConditionalRendering = false, bool forConditionalRendering = false,
bool sparseCompatible = false,
BufferAllocationType fallbackType = BufferAllocationType.Auto) BufferAllocationType fallbackType = BufferAllocationType.Auto)
{ {
var usage = DefaultBufferUsageFlags; var usage = DefaultBufferUsageFlags;
@ -187,6 +294,11 @@ namespace Ryujinx.Graphics.Vulkan
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
if (sparseCompatible)
{
requirements.Alignment = Math.Max(requirements.Alignment, Constants.SparseBufferAlignment);
}
MemoryAllocation allocation; MemoryAllocation allocation;
do do
@ -227,6 +339,7 @@ namespace Ryujinx.Graphics.Vulkan
VulkanRenderer gd, VulkanRenderer gd,
int size, int size,
bool forConditionalRendering = false, bool forConditionalRendering = false,
bool sparseCompatible = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default) BufferHandle storageHint = default)
{ {
@ -255,7 +368,7 @@ namespace Ryujinx.Graphics.Vulkan
} }
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) =
CreateBacking(gd, size, type, forConditionalRendering); CreateBacking(gd, size, type, forConditionalRendering, sparseCompatible);
if (buffer.Handle != 0) if (buffer.Handle != 0)
{ {

View file

@ -16,5 +16,7 @@ namespace Ryujinx.Graphics.Vulkan
public const int MaxStorageBufferBindings = MaxStorageBuffersPerStage * MaxShaderStages; public const int MaxStorageBufferBindings = MaxStorageBuffersPerStage * MaxShaderStages;
public const int MaxTextureBindings = MaxTexturesPerStage * MaxShaderStages; public const int MaxTextureBindings = MaxTexturesPerStage * MaxShaderStages;
public const int MaxImageBindings = MaxImagesPerStage * MaxShaderStages; public const int MaxImageBindings = MaxImagesPerStage * MaxShaderStages;
public const ulong SparseBufferAlignment = 0x10000;
} }
} }

View file

@ -424,12 +424,12 @@ namespace Ryujinx.Graphics.Vulkan
public static BufferAllocationType Convert(this BufferAccess access) public static BufferAllocationType Convert(this BufferAccess access)
{ {
return access switch if (access.HasFlag(BufferAccess.FlushPersistent) || access.HasFlag(BufferAccess.Stream))
{ {
BufferAccess.FlushPersistent => BufferAllocationType.HostMapped, return BufferAllocationType.HostMapped;
BufferAccess.Stream => BufferAllocationType.HostMapped, }
_ => BufferAllocationType.Auto,
}; return BufferAllocationType.Auto;
} }
private static T2 LogInvalidAndReturn<T1, T2>(T1 value, string name, T2 defaultValue = default) private static T2 LogInvalidAndReturn<T1, T2>(T1 value, string name, T2 defaultValue = default)

View file

@ -392,6 +392,8 @@ namespace Ryujinx.Graphics.Vulkan
LoadFeatures(maxQueueCount, queueFamilyIndex); LoadFeatures(maxQueueCount, queueFamilyIndex);
QueueFamilyIndex = queueFamilyIndex;
_window = new Window(this, _surface, _physicalDevice.PhysicalDevice, _device); _window = new Window(this, _surface, _physicalDevice.PhysicalDevice, _device);
_initialized = true; _initialized = true;
@ -399,12 +401,12 @@ namespace Ryujinx.Graphics.Vulkan
public BufferHandle CreateBuffer(int size, BufferAccess access) public BufferHandle CreateBuffer(int size, BufferAccess access)
{ {
return BufferManager.CreateWithHandle(this, size, access.Convert(), default, access == BufferAccess.Stream); return BufferManager.CreateWithHandle(this, size, access.HasFlag(BufferAccess.SparseCompatible), access.Convert(), default, access == BufferAccess.Stream);
} }
public BufferHandle CreateBuffer(int size, BufferHandle storageHint) public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
{ {
return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint); return BufferManager.CreateWithHandle(this, size, access.HasFlag(BufferAccess.SparseCompatible), access.Convert(), storageHint);
} }
public BufferHandle CreateBuffer(nint pointer, int size) public BufferHandle CreateBuffer(nint pointer, int size)
@ -412,6 +414,11 @@ namespace Ryujinx.Graphics.Vulkan
return BufferManager.CreateHostImported(this, pointer, size); return BufferManager.CreateHostImported(this, pointer, size);
} }
public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
{
return BufferManager.CreateSparse(this, storageBuffers);
}
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info) public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
{ {
bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute; bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute;
@ -571,6 +578,7 @@ namespace Ryujinx.Graphics.Vulkan
Api.GetPhysicalDeviceFeatures2(_physicalDevice.PhysicalDevice, &features2); Api.GetPhysicalDeviceFeatures2(_physicalDevice.PhysicalDevice, &features2);
var limits = _physicalDevice.PhysicalDeviceProperties.Limits; var limits = _physicalDevice.PhysicalDeviceProperties.Limits;
var mainQueueProperties = _physicalDevice.QueueFamilyProperties[QueueFamilyIndex];
return new Capabilities( return new Capabilities(
api: TargetApi.Vulkan, api: TargetApi.Vulkan,
@ -590,6 +598,7 @@ namespace Ryujinx.Graphics.Vulkan
supportsR4G4B4A4Format: supportsR4G4B4A4Format, supportsR4G4B4A4Format: supportsR4G4B4A4Format,
supportsSnormBufferTextureFormat: true, supportsSnormBufferTextureFormat: true,
supports5BitComponentFormat: supports5BitComponentFormat, supports5BitComponentFormat: supports5BitComponentFormat,
supportsSparseBuffer: features2.Features.SparseBinding && mainQueueProperties.QueueFlags.HasFlag(QueueFlags.SparseBindingBit),
supportsBlendEquationAdvanced: Capabilities.SupportsBlendEquationAdvanced, supportsBlendEquationAdvanced: Capabilities.SupportsBlendEquationAdvanced,
supportsFragmentShaderInterlock: Capabilities.SupportsFragmentShaderInterlock, supportsFragmentShaderInterlock: Capabilities.SupportsFragmentShaderInterlock,
supportsFragmentShaderOrderingIntel: false, supportsFragmentShaderOrderingIntel: false,

View file

@ -15,6 +15,11 @@ namespace Ryujinx.Memory.Range
private bool HasSingleRange => _ranges == null; private bool HasSingleRange => _ranges == null;
/// <summary>
/// Indicates that the range is fully unmapped.
/// </summary>
public bool IsUnmapped => HasSingleRange && _singleRange.Address == InvalidAddress;
/// <summary> /// <summary>
/// Total of physical sub-ranges on the virtual memory region. /// Total of physical sub-ranges on the virtual memory region.
/// </summary> /// </summary>
@ -38,8 +43,18 @@ namespace Ryujinx.Memory.Range
/// <exception cref="ArgumentNullException"><paramref name="ranges"/> is null</exception> /// <exception cref="ArgumentNullException"><paramref name="ranges"/> is null</exception>
public MultiRange(MemoryRange[] ranges) public MultiRange(MemoryRange[] ranges)
{ {
_singleRange = MemoryRange.Empty; ArgumentNullException.ThrowIfNull(ranges);
_ranges = ranges ?? throw new ArgumentNullException(nameof(ranges));
if (ranges.Length == 1)
{
_singleRange = ranges[0];
_ranges = null;
}
else
{
_singleRange = MemoryRange.Empty;
_ranges = ranges;
}
} }
/// <summary> /// <summary>
@ -91,7 +106,7 @@ namespace Ryujinx.Memory.Range
offset -= range.Size; offset -= range.Size;
} }
return new MultiRange(ranges.ToArray()); return ranges.Count == 1 ? new MultiRange(ranges[0].Address, ranges[0].Size) : new MultiRange(ranges.ToArray());
} }
} }