From 492a0463358e7706e0fb34537d55810d833ae695 Mon Sep 17 00:00:00 2001 From: riperiperi Date: Mon, 14 Aug 2023 18:18:47 +0100 Subject: [PATCH] Vulkan: Buffer Mirrors for MacOS performance (#4899) * Initial implementation of buffer mirrors Generally slower right now, goal is to reduce render passes in games that do inline updates Fix support buffer mirrors Reintroduce vertex buffer mirror Add storage buffer support Optimisation part 1 More optimisation Avoid useless data copies. Remove unused cbIndex stuff Properly set write flag for storage buffers. Fix minor issues Not sure why this was here. Fix BufferRangeList Fix some big issues Align storage buffers rather than getting full buffer as a range Improves mirrorability of read-only storage buffers Increase staging buffer size, as it now contains mirrors Fix some issues with buffers not updating Fix buffer SetDataUnchecked offset for one of the paths when using mirrors Fix buffer mirrors interaction with buffer textures Fix mirror rebinding Move GetBuffer calls on indirect draws before BeginRenderPass to avoid draws without render pass Fix mirrors rebase Fix rebase 2023 * Fix crash when using stale vertex buffer Similar to `Get` with a size that's too large, just treat it as a clamp. * Explicitly set support buffer as mirrorable * Address feedback * Remove unused fragment of MVK workaround * Replace logging for staging buffer OOM * Address format issues * Address more format issues * Mini cleanup * Address more things * Rename BufferRangeList * Support bounding range for ClearMirrors and UploadPendingData * Add maximum size for vertex buffer mirrors * Enable index buffer mirrors Enabled on all platforms for the IbStreamer. * Feedback * Remove mystery BufferCache change Probably macos related? * Fix mirrors not creating when staging buffer is empty. * Change log level to debug --- src/Ryujinx.Graphics.GAL/BufferAccess.cs | 1 + src/Ryujinx.Graphics.GAL/BufferRange.cs | 4 +- .../Multithreading/BufferMap.cs | 8 +- .../Engine/Threed/IbStreamer.cs | 4 +- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 14 +- .../Memory/BufferCache.cs | 8 +- .../Memory/BufferManager.cs | 6 +- .../Memory/SupportBufferUpdater.cs | 2 +- src/Ryujinx.Graphics.Vulkan/Auto.cs | 24 +- src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs | 263 +++++++++++++++ src/Ryujinx.Graphics.Vulkan/BufferHolder.cs | 283 +++++++++++++++- src/Ryujinx.Graphics.Vulkan/BufferManager.cs | 13 +- .../BufferMirrorRangeList.cs | 305 ++++++++++++++++++ src/Ryujinx.Graphics.Vulkan/BufferState.cs | 7 +- .../BufferUsageBitmap.cs | 19 +- .../DescriptorSetUpdater.cs | 211 +++++++++--- src/Ryujinx.Graphics.Vulkan/EnumConversion.cs | 1 + .../HardwareCapabilities.cs | 5 +- .../IndexBufferState.cs | 14 +- .../MultiFenceHolder.cs | 17 +- src/Ryujinx.Graphics.Vulkan/PipelineBase.cs | 70 ++-- src/Ryujinx.Graphics.Vulkan/PipelineFull.cs | 13 + .../Queries/BufferedQuery.cs | 2 +- src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs | 90 +++++- src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs | 12 +- .../VertexBufferState.cs | 13 +- src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs | 12 +- 27 files changed, 1285 insertions(+), 136 deletions(-) create mode 100644 src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs create mode 100644 src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs diff --git a/src/Ryujinx.Graphics.GAL/BufferAccess.cs b/src/Ryujinx.Graphics.GAL/BufferAccess.cs index 2f7d994fe..e7d7ceb0a 100644 --- a/src/Ryujinx.Graphics.GAL/BufferAccess.cs +++ b/src/Ryujinx.Graphics.GAL/BufferAccess.cs @@ -4,5 +4,6 @@ namespace Ryujinx.Graphics.GAL { Default, FlushPersistent, + Stream } } diff --git a/src/Ryujinx.Graphics.GAL/BufferRange.cs b/src/Ryujinx.Graphics.GAL/BufferRange.cs index 483747f10..fec82de24 100644 --- a/src/Ryujinx.Graphics.GAL/BufferRange.cs +++ b/src/Ryujinx.Graphics.GAL/BufferRange.cs @@ -10,12 +10,14 @@ namespace Ryujinx.Graphics.GAL public int Offset { get; } public int Size { get; } + public bool Write { get; } - public BufferRange(BufferHandle handle, int offset, int size) + public BufferRange(BufferHandle handle, int offset, int size, bool write = false) { Handle = handle; Offset = offset; Size = size; + Write = write; } } } diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs b/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs index c30df0465..6377e5eac 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs @@ -113,7 +113,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading internal BufferRange MapBufferRange(BufferRange range) { - return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size); + return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size, range.Write); } internal Span MapBufferRanges(Span ranges) @@ -131,7 +131,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading result = BufferHandle.Null; } - range = new BufferRange(result, range.Offset, range.Size); + range = new BufferRange(result, range.Offset, range.Size, range.Write); } } @@ -154,7 +154,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading result = BufferHandle.Null; } - assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size)); + assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size, range.Write)); } } @@ -176,7 +176,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading result = BufferHandle.Null; } - range = new BufferRange(result, range.Offset, range.Size); + range = new BufferRange(result, range.Offset, range.Size, range.Write); ranges[i] = new VertexBufferDescriptor(range, ranges[i].Stride, ranges[i].Divisor); } diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs index 022e12f57..6e15fcfa3 100644 --- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs +++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs @@ -171,7 +171,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed if (_inlineIndexBuffer == BufferHandle.Null) { - _inlineIndexBuffer = renderer.CreateBuffer(size); + _inlineIndexBuffer = renderer.CreateBuffer(size, BufferAccess.Stream); _inlineIndexBufferSize = size; } else if (_inlineIndexBufferSize < size) @@ -179,7 +179,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed BufferHandle oldBuffer = _inlineIndexBuffer; int oldSize = _inlineIndexBufferSize; - _inlineIndexBuffer = renderer.CreateBuffer(size); + _inlineIndexBuffer = renderer.CreateBuffer(size, BufferAccess.Stream); _inlineIndexBufferSize = size; renderer.Pipeline.CopyBuffer(oldBuffer, _inlineIndexBuffer, 0, 0, oldSize); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index e27c14a16..c9286a619 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -140,18 +140,21 @@ namespace Ryujinx.Graphics.Gpu.Memory } /// - /// Gets a sub-range from the buffer, from a start address till the end of the buffer. + /// Gets a sub-range from the buffer, from a start address til a page boundary after the given size. /// /// /// This can be used to bind and use sub-ranges of the buffer on the host API. /// /// Start address of the sub-range, must be greater than or equal to the buffer address + /// Size in bytes of the sub-range, must be less than or equal to the buffer size + /// Whether the buffer will be written to by this use /// The buffer sub-range - public BufferRange GetRange(ulong address) + public BufferRange GetRangeAligned(ulong address, ulong size, bool write) { + ulong end = ((address + size + MemoryManager.PageMask) & ~MemoryManager.PageMask) - Address; ulong offset = address - Address; - return new BufferRange(Handle, (int)offset, (int)(Size - offset)); + return new BufferRange(Handle, (int)offset, (int)(end - offset), write); } /// @@ -162,12 +165,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Start address of the sub-range, must be greater than or equal to the buffer address /// Size in bytes of the sub-range, must be less than or equal to the buffer size + /// Whether the buffer will be written to by this use /// The buffer sub-range - public BufferRange GetRange(ulong address, ulong size) + public BufferRange GetRange(ulong address, ulong size, bool write) { int offset = (int)(address - Address); - return new BufferRange(Handle, offset, (int)size); + return new BufferRange(Handle, offset, (int)size, write); } /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index f8f572c6a..05cc312c7 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -372,15 +372,15 @@ namespace Ryujinx.Graphics.Gpu.Memory } /// - /// Gets a buffer sub-range starting at a given memory address. + /// Gets a buffer sub-range from a start address til a page boundary after the given size. /// /// Start address of the memory range /// Size in bytes of the memory range /// Whether the buffer will be written to by this use /// The buffer sub-range starting at the given memory address - public BufferRange GetBufferRangeTillEnd(ulong address, ulong size, bool write = false) + public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false) { - return GetBuffer(address, size, write).GetRange(address); + return GetBuffer(address, size, write).GetRangeAligned(address, size, write); } /// @@ -392,7 +392,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// The buffer sub-range for the given range public BufferRange GetBufferRange(ulong address, ulong size, bool write = false) { - return GetBuffer(address, size, write).GetRange(address, size); + return GetBuffer(address, size, write).GetRange(address, size, write); } /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index c656b0f64..bf4cb5d05 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -614,7 +614,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_tfInfoBuffer == BufferHandle.Null) { - _tfInfoBuffer = _context.Renderer.CreateBuffer(TfInfoBufferSize); + _tfInfoBuffer = _context.Renderer.CreateBuffer(TfInfoBufferSize, BufferAccess.Stream); } buffers[0] = new BufferAssignment(0, new BufferRange(_tfInfoBuffer, 0, TfInfoBufferSize)); @@ -727,7 +727,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage - ? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) + ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) : bufferCache.GetBufferRange(bounds.Address, bounds.Size); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); @@ -764,7 +764,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage - ? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) + ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) : bufferCache.GetBufferRange(bounds.Address, bounds.Size); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs b/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs index 409c7a786..c1e91c54b 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs @@ -228,7 +228,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (_handle == BufferHandle.Null) { - _handle = _renderer.CreateBuffer(SupportBuffer.RequiredSize); + _handle = _renderer.CreateBuffer(SupportBuffer.RequiredSize, BufferAccess.Stream); _renderer.Pipeline.ClearBuffer(_handle, 0, SupportBuffer.RequiredSize, 0); var range = new BufferRange(_handle, 0, SupportBuffer.RequiredSize); diff --git a/src/Ryujinx.Graphics.Vulkan/Auto.cs b/src/Ryujinx.Graphics.Vulkan/Auto.cs index fdce7232c..026dd2b60 100644 --- a/src/Ryujinx.Graphics.Vulkan/Auto.cs +++ b/src/Ryujinx.Graphics.Vulkan/Auto.cs @@ -18,6 +18,12 @@ namespace Ryujinx.Graphics.Vulkan void AddCommandBufferDependencies(CommandBufferScoped cbs); } + interface IMirrorable where T : IDisposable + { + Auto GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored); + void ClearMirrors(CommandBufferScoped cbs, int offset, int size); + } + class Auto : IAutoPrivate, IDisposable where T : IDisposable { private int _referenceCount; @@ -26,6 +32,7 @@ namespace Ryujinx.Graphics.Vulkan private readonly BitMap _cbOwnership; private readonly MultiFenceHolder _waitable; private readonly IAutoPrivate[] _referencedObjs; + private readonly IMirrorable _mirrorable; private bool _disposed; private bool _destroyed; @@ -37,6 +44,11 @@ namespace Ryujinx.Graphics.Vulkan _cbOwnership = new BitMap(CommandBufferPool.MaxCommandBuffers); } + public Auto(T value, IMirrorable mirrorable, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value, waitable, referencedObjs) + { + _mirrorable = mirrorable; + } + public Auto(T value, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value) { _waitable = waitable; @@ -48,9 +60,17 @@ namespace Ryujinx.Graphics.Vulkan } } - public T Get(CommandBufferScoped cbs, int offset, int size) + public T GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) { - _waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size); + var mirror = _mirrorable.GetMirrorable(cbs, ref offset, size, out mirrored); + mirror._waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, false); + return mirror.Get(cbs); + } + + public T Get(CommandBufferScoped cbs, int offset, int size, bool write = false) + { + _mirrorable?.ClearMirrors(cbs, offset, size); + _waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, write); return Get(cbs); } diff --git a/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs b/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs new file mode 100644 index 000000000..15672e9cb --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs @@ -0,0 +1,263 @@ +using Ryujinx.Common.Memory; +using System; +using System.Numerics; + +namespace Ryujinx.Graphics.Vulkan +{ + interface IBitMapListener + { + void BitMapSignal(int index, int count); + } + + struct BitMapStruct where T : IArray + { + public const int IntSize = 64; + + private const int IntShift = 6; + private const int IntMask = IntSize - 1; + + private T _masks; + + public BitMapStruct() + { + _masks = default; + } + + public bool BecomesUnsetFrom(in BitMapStruct from, ref BitMapStruct into) + { + bool result = false; + + int masks = _masks.Length; + for (int i = 0; i < masks; i++) + { + long fromMask = from._masks[i]; + long unsetMask = (~fromMask) & (fromMask ^ _masks[i]); + into._masks[i] = unsetMask; + + result |= unsetMask != 0; + } + + return result; + } + + public void SetAndSignalUnset(in BitMapStruct from, ref T2 listener) where T2 : struct, IBitMapListener + { + BitMapStruct result = new(); + + if (BecomesUnsetFrom(from, ref result)) + { + // Iterate the set bits in the result, and signal them. + + int offset = 0; + int masks = _masks.Length; + ref T resultMasks = ref result._masks; + for (int i = 0; i < masks; i++) + { + long value = resultMasks[i]; + while (value != 0) + { + int bit = BitOperations.TrailingZeroCount((ulong)value); + + listener.BitMapSignal(offset + bit, 1); + + value &= ~(1L << bit); + } + + offset += IntSize; + } + } + + _masks = from._masks; + } + + public void SignalSet(Action action) + { + // Iterate the set bits in the result, and signal them. + + int offset = 0; + int masks = _masks.Length; + for (int i = 0; i < masks; i++) + { + long value = _masks[i]; + while (value != 0) + { + int bit = BitOperations.TrailingZeroCount((ulong)value); + + action(offset + bit, 1); + + value &= ~(1L << bit); + } + + offset += IntSize; + } + } + + public bool AnySet() + { + for (int i = 0; i < _masks.Length; i++) + { + if (_masks[i] != 0) + { + return true; + } + } + + return false; + } + + public bool IsSet(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + return (_masks[wordIndex] & wordMask) != 0; + } + + public bool IsSet(int start, int end) + { + if (start == end) + { + return IsSet(start); + } + + int startIndex = start >> IntShift; + int startBit = start & IntMask; + long startMask = -1L << startBit; + + int endIndex = end >> IntShift; + int endBit = end & IntMask; + long endMask = (long)(ulong.MaxValue >> (IntMask - endBit)); + + if (startIndex == endIndex) + { + return (_masks[startIndex] & startMask & endMask) != 0; + } + + if ((_masks[startIndex] & startMask) != 0) + { + return true; + } + + for (int i = startIndex + 1; i < endIndex; i++) + { + if (_masks[i] != 0) + { + return true; + } + } + + if ((_masks[endIndex] & endMask) != 0) + { + return true; + } + + return false; + } + + public bool Set(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + if ((_masks[wordIndex] & wordMask) != 0) + { + return false; + } + + _masks[wordIndex] |= wordMask; + + return true; + } + + public void Set(int bit, bool value) + { + if (value) + { + Set(bit); + } + else + { + Clear(bit); + } + } + + public void SetRange(int start, int end) + { + if (start == end) + { + Set(start); + return; + } + + int startIndex = start >> IntShift; + int startBit = start & IntMask; + long startMask = -1L << startBit; + + int endIndex = end >> IntShift; + int endBit = end & IntMask; + long endMask = (long)(ulong.MaxValue >> (IntMask - endBit)); + + if (startIndex == endIndex) + { + _masks[startIndex] |= startMask & endMask; + } + else + { + _masks[startIndex] |= startMask; + + for (int i = startIndex + 1; i < endIndex; i++) + { + _masks[i] |= -1L; + } + + _masks[endIndex] |= endMask; + } + } + + public BitMapStruct Union(BitMapStruct other) + { + var result = new BitMapStruct(); + + ref var masks = ref _masks; + ref var otherMasks = ref other._masks; + ref var newMasks = ref result._masks; + + for (int i = 0; i < masks.Length; i++) + { + newMasks[i] = masks[i] | otherMasks[i]; + } + + return result; + } + + public void Clear(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + _masks[wordIndex] &= ~wordMask; + } + + public void Clear() + { + for (int i = 0; i < _masks.Length; i++) + { + _masks[i] = 0; + } + } + + public void ClearInt(int start, int end) + { + for (int i = start; i <= end; i++) + { + _masks[i] = 0; + } + } + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs index 54635631a..c767a57a7 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs @@ -10,7 +10,7 @@ using VkFormat = Silk.NET.Vulkan.Format; namespace Ryujinx.Graphics.Vulkan { - class BufferHolder : IDisposable + class BufferHolder : IDisposable, IMirrorable, IMirrorable { private const int MaxUpdateBufferSize = 0x10000; @@ -64,6 +64,11 @@ namespace Ryujinx.Graphics.Vulkan private List _swapActions; + private byte[] _pendingData; + private BufferMirrorRangeList _pendingDataRanges; + private Dictionary _mirrors; + private bool _useMirrors; + public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType) { _gd = gd; @@ -71,7 +76,7 @@ namespace Ryujinx.Graphics.Vulkan _allocation = allocation; _allocationAuto = new Auto(allocation); _waitable = new MultiFenceHolder(size); - _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; Size = size; _map = allocation.HostPointer; @@ -81,6 +86,7 @@ namespace Ryujinx.Graphics.Vulkan DesiredType = currentType; _flushLock = new ReaderWriterLock(); + _useMirrors = gd.IsTBDR; } public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto allocation, int size, BufferAllocationType type, BufferAllocationType currentType, int offset) @@ -91,7 +97,7 @@ namespace Ryujinx.Graphics.Vulkan _allocationAuto = allocation; _allocationImported = true; _waitable = new MultiFenceHolder(size); - _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; Size = size; _map = _allocation.HostPointer + offset; @@ -110,7 +116,7 @@ namespace Ryujinx.Graphics.Vulkan // Only swap if the buffer is not used in any queued command buffer. bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); - if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld) + if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld && (_pendingData == null || cbs != null)) { var currentAllocation = _allocationAuto; var currentBuffer = _buffer; @@ -120,6 +126,11 @@ namespace Ryujinx.Graphics.Vulkan if (buffer.Handle != 0) { + if (cbs != null) + { + ClearMirrors(cbs.Value, 0, Size); + } + _flushLock.AcquireWriterLock(Timeout.Infinite); ClearFlushFence(); @@ -128,7 +139,7 @@ namespace Ryujinx.Graphics.Vulkan _allocation = allocation; _allocationAuto = new Auto(allocation); - _buffer = new Auto(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto); + _buffer = new Auto(new DisposableBuffer(_gd.Api, _device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; _map = allocation.HostPointer; @@ -257,7 +268,7 @@ namespace Ryujinx.Graphics.Vulkan (_swapActions ??= new List()).Add(invalidateView); - return new Auto(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer); + return new Auto(new DisposableBufferView(_gd.Api, _device, bufferView), this, _waitable, _buffer); } public void InheritMetrics(BufferHolder other) @@ -302,6 +313,82 @@ namespace Ryujinx.Graphics.Vulkan } } + private static ulong ToMirrorKey(int offset, int size) + { + return ((ulong)offset << 32) | (uint)size; + } + + private static (int offset, int size) FromMirrorKey(ulong key) + { + return ((int)(key >> 32), (int)key); + } + + private unsafe bool TryGetMirror(CommandBufferScoped cbs, ref int offset, int size, out Auto buffer) + { + size = Math.Min(size, Size - offset); + + // Does this binding need to be mirrored? + + if (!_pendingDataRanges.OverlapsWith(offset, size)) + { + buffer = null; + return false; + } + + var key = ToMirrorKey(offset, size); + + if (_mirrors.TryGetValue(key, out StagingBufferReserved reserved)) + { + buffer = reserved.Buffer.GetBuffer(); + offset = reserved.Offset; + + return true; + } + + // Is this mirror allowed to exist? Can't be used for write in any in-flight write. + if (_waitable.IsBufferRangeInUse(offset, size, true)) + { + // Some of the data is not mirrorable, so upload the whole range. + ClearMirrors(cbs, offset, size); + + buffer = null; + return false; + } + + // Build data for the new mirror. + + var baseData = new Span((void*)(_map + offset), size); + var modData = _pendingData.AsSpan(offset, size); + + StagingBufferReserved? newMirror = _gd.BufferManager.StagingBuffer.TryReserveData(cbs, size, (int)_gd.Capabilities.MinResourceAlignment); + + if (newMirror != null) + { + var mirror = newMirror.Value; + _pendingDataRanges.FillData(baseData, modData, offset, new Span((void*)(mirror.Buffer._map + mirror.Offset), size)); + + if (_mirrors.Count == 0) + { + _gd.PipelineInternal.RegisterActiveMirror(this); + } + + _mirrors.Add(key, mirror); + + buffer = mirror.Buffer.GetBuffer(); + offset = mirror.Offset; + + return true; + } + else + { + // Data could not be placed on the mirror, likely out of space. Force the data to flush. + ClearMirrors(cbs, offset, size); + + buffer = null; + return false; + } + } + public Auto GetBuffer() { return _buffer; @@ -339,6 +426,86 @@ namespace Ryujinx.Graphics.Vulkan return _buffer; } + public Auto GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + if (_pendingData != null && TryGetMirror(cbs, ref offset, size, out Auto result)) + { + mirrored = true; + return result; + } + + mirrored = false; + return _buffer; + } + + Auto IMirrorable.GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + // Cannot mirror buffer views right now. + + throw new NotImplementedException(); + } + + public void ClearMirrors() + { + // Clear mirrors without forcing a flush. This happens when the command buffer is switched, + // as all reserved areas on the staging buffer are released. + + if (_pendingData != null) + { + _mirrors.Clear(); + }; + } + + public void ClearMirrors(CommandBufferScoped cbs, int offset, int size) + { + // Clear mirrors in the given range, and submit overlapping pending data. + + if (_pendingData != null) + { + bool hadMirrors = _mirrors.Count > 0 && RemoveOverlappingMirrors(offset, size); + + if (_pendingDataRanges.Count() != 0) + { + UploadPendingData(cbs, offset, size); + } + + if (hadMirrors) + { + _gd.PipelineInternal.Rebind(_buffer, offset, size); + } + }; + } + + public void UseMirrors() + { + _useMirrors = true; + } + + private void UploadPendingData(CommandBufferScoped cbs, int offset, int size) + { + var ranges = _pendingDataRanges.FindOverlaps(offset, size); + + if (ranges != null) + { + _pendingDataRanges.Remove(offset, size); + + foreach (var range in ranges) + { + int rangeOffset = Math.Max(offset, range.Offset); + int rangeSize = Math.Min(offset + size, range.End) - rangeOffset; + + if (_gd.PipelineInternal.CurrentCommandBuffer.CommandBuffer.Handle == cbs.CommandBuffer.Handle) + { + SetData(rangeOffset, _pendingData.AsSpan(rangeOffset, rangeSize), cbs, _gd.PipelineInternal.EndRenderPass, false); + } + else + { + SetData(rangeOffset, _pendingData.AsSpan(rangeOffset, rangeSize), cbs, null, false); + } + } + } + } + public void SignalWrite(int offset, int size) { ConsiderBackingSwap(); @@ -472,7 +639,34 @@ namespace Ryujinx.Graphics.Vulkan throw new InvalidOperationException("The buffer is not host mapped."); } - public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferScoped? cbs = null, Action endRenderPass = null) + public bool RemoveOverlappingMirrors(int offset, int size) + { + List toRemove = null; + foreach (var key in _mirrors.Keys) + { + (int keyOffset, int keySize) = FromMirrorKey(key); + if (!(offset + size <= keyOffset || offset >= keyOffset + keySize)) + { + toRemove ??= new List(); + + toRemove.Add(key); + } + } + + if (toRemove != null) + { + foreach (var key in toRemove) + { + _mirrors.Remove(key); + } + + return true; + } + + return false; + } + + public unsafe void SetData(int offset, ReadOnlySpan data, CommandBufferScoped? cbs = null, Action endRenderPass = null, bool allowCbsWait = true) { int dataSize = Math.Min(data.Length, Size - offset); if (dataSize == 0) @@ -481,6 +675,7 @@ namespace Ryujinx.Graphics.Vulkan } _setCount++; + bool allowMirror = _useMirrors && allowCbsWait && cbs != null && _currentType <= BufferAllocationType.HostMapped; if (_map != IntPtr.Zero) { @@ -488,7 +683,7 @@ namespace Ryujinx.Graphics.Vulkan bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); // If the buffer is rented, take a little more time and check if the use overlaps this handle. - bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize); + bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize, false); if (!needsFlush) { @@ -496,12 +691,48 @@ namespace Ryujinx.Graphics.Vulkan data[..dataSize].CopyTo(new Span((void*)(_map + offset), dataSize)); + if (_pendingData != null) + { + bool removed = _pendingDataRanges.Remove(offset, dataSize); + if (RemoveOverlappingMirrors(offset, dataSize) || removed) + { + // If any mirrors were removed, rebind the buffer range. + _gd.PipelineInternal.Rebind(_buffer, offset, dataSize); + } + } + SignalWrite(offset, dataSize); return; } } + // If the buffer does not have an in-flight write (including an inline update), then upload data to a pendingCopy. + if (allowMirror && !_waitable.IsBufferRangeInUse(offset, dataSize, true)) + { + if (_pendingData == null) + { + _pendingData = new byte[Size]; + _mirrors = new Dictionary(); + } + + data[..dataSize].CopyTo(_pendingData.AsSpan(offset, dataSize)); + _pendingDataRanges.Add(offset, dataSize); + + // Remove any overlapping mirrors. + RemoveOverlappingMirrors(offset, dataSize); + + // Tell the graphics device to rebind any constant buffer that overlaps the newly modified range, as it should access a mirror. + _gd.PipelineInternal.Rebind(_buffer, offset, dataSize); + + return; + } + + if (_pendingData != null) + { + _pendingDataRanges.Remove(offset, dataSize); + } + if (cbs != null && _gd.PipelineInternal.RenderPassActive && !(_buffer.HasCommandBufferDependency(cbs.Value) && @@ -519,7 +750,37 @@ namespace Ryujinx.Graphics.Vulkan data.Length > MaxUpdateBufferSize || !TryPushData(cbs.Value, endRenderPass, offset, data)) { - _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); + if (allowCbsWait) + { + _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); + } + else + { + bool rentCbs = cbs == null; + if (rentCbs) + { + cbs = _gd.CommandBufferPool.Rent(); + } + + if (!_gd.BufferManager.StagingBuffer.TryPushData(cbs.Value, endRenderPass, this, offset, data)) + { + // Need to do a slow upload. + BufferHolder srcHolder = _gd.BufferManager.Create(_gd, dataSize, baseType: BufferAllocationType.HostMapped); + srcHolder.SetDataUnchecked(0, data); + + var srcBuffer = srcHolder.GetBuffer(); + var dstBuffer = this.GetBuffer(cbs.Value.CommandBuffer, true); + + Copy(_gd, cbs.Value, srcBuffer, dstBuffer, 0, offset, dataSize); + + srcHolder.Dispose(); + } + + if (rentCbs) + { + cbs.Value.Dispose(); + } + } } } @@ -558,7 +819,7 @@ namespace Ryujinx.Graphics.Vulkan endRenderPass?.Invoke(); - var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value; + var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length, true).Value; _writeCount--; @@ -608,7 +869,7 @@ namespace Ryujinx.Graphics.Vulkan bool registerSrcUsage = true) { var srcBuffer = registerSrcUsage ? src.Get(cbs, srcOffset, size).Value : src.GetUnsafe().Value; - var dstBuffer = dst.Get(cbs, dstOffset, size).Value; + var dstBuffer = dst.Get(cbs, dstOffset, size, true).Value; InsertBufferBarrier( gd, diff --git a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs index b916a1ef2..380967022 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs @@ -100,9 +100,10 @@ namespace Ryujinx.Graphics.Vulkan VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, - BufferHandle storageHint = default) + BufferHandle storageHint = default, + bool forceMirrors = false) { - return CreateWithHandle(gd, size, out _, baseType, storageHint); + return CreateWithHandle(gd, size, out _, baseType, storageHint, forceMirrors); } public BufferHandle CreateWithHandle( @@ -110,7 +111,8 @@ namespace Ryujinx.Graphics.Vulkan int size, out BufferHolder holder, BufferAllocationType baseType = BufferAllocationType.HostMapped, - BufferHandle storageHint = default) + BufferHandle storageHint = default, + bool forceMirrors = false) { holder = Create(gd, size, baseType: baseType, storageHint: storageHint); if (holder == null) @@ -118,6 +120,11 @@ namespace Ryujinx.Graphics.Vulkan return BufferHandle.Null; } + if (forceMirrors) + { + holder.UseMirrors(); + } + BufferCount++; ulong handle64 = (uint)_buffers.Add(holder); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs b/src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs new file mode 100644 index 000000000..9e0b7244a --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs @@ -0,0 +1,305 @@ +using System; +using System.Collections.Generic; + +namespace Ryujinx.Graphics.Vulkan +{ + /// + /// A structure tracking pending upload ranges for buffers. + /// Where a range is present, pending data exists that can either be used to build mirrors + /// or upload directly to the buffer. + /// + struct BufferMirrorRangeList + { + internal readonly struct Range + { + public int Offset { get; } + public int Size { get; } + + public int End => Offset + Size; + + public Range(int offset, int size) + { + Offset = offset; + Size = size; + } + + public bool OverlapsWith(int offset, int size) + { + return Offset < offset + size && offset < Offset + Size; + } + } + + private List _ranges; + + public readonly IEnumerable All() + { + return _ranges; + } + + public readonly bool Remove(int offset, int size) + { + var list = _ranges; + bool removedAny = false; + if (list != null) + { + int overlapIndex = BinarySearch(list, offset, size); + + if (overlapIndex >= 0) + { + // Overlaps with a range. Search back to find the first one it doesn't overlap with. + + while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size)) + { + overlapIndex--; + } + + int endOffset = offset + size; + int startIndex = overlapIndex; + + var currentOverlap = list[overlapIndex]; + + // Orphan the start of the overlap. + if (currentOverlap.Offset < offset) + { + list[overlapIndex] = new Range(currentOverlap.Offset, offset - currentOverlap.Offset); + currentOverlap = new Range(offset, currentOverlap.End - offset); + list.Insert(++overlapIndex, currentOverlap); + startIndex++; + + removedAny = true; + } + + // Remove any middle overlaps. + while (currentOverlap.Offset < endOffset) + { + if (currentOverlap.End > endOffset) + { + // Update the end overlap instead of removing it, if it spans beyond the removed range. + list[overlapIndex] = new Range(endOffset, currentOverlap.End - endOffset); + + removedAny = true; + break; + } + + if (++overlapIndex >= list.Count) + { + break; + } + + currentOverlap = list[overlapIndex]; + } + + int count = overlapIndex - startIndex; + + list.RemoveRange(startIndex, count); + + removedAny |= count > 0; + } + } + + return removedAny; + } + + public void Add(int offset, int size) + { + var list = _ranges; + if (list != null) + { + int overlapIndex = BinarySearch(list, offset, size); + if (overlapIndex >= 0) + { + while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size)) + { + overlapIndex--; + } + + int endOffset = offset + size; + int startIndex = overlapIndex; + + while (overlapIndex < list.Count && list[overlapIndex].OverlapsWith(offset, size)) + { + var currentOverlap = list[overlapIndex]; + var currentOverlapEndOffset = currentOverlap.Offset + currentOverlap.Size; + + if (offset > currentOverlap.Offset) + { + offset = currentOverlap.Offset; + } + + if (endOffset < currentOverlapEndOffset) + { + endOffset = currentOverlapEndOffset; + } + + overlapIndex++; + size = endOffset - offset; + } + + int count = overlapIndex - startIndex; + + list.RemoveRange(startIndex, count); + + overlapIndex = startIndex; + } + else + { + overlapIndex = ~overlapIndex; + } + + list.Insert(overlapIndex, new Range(offset, size)); + } + else + { + _ranges = new List + { + new Range(offset, size) + }; + } + } + + public readonly bool OverlapsWith(int offset, int size) + { + var list = _ranges; + if (list == null) + { + return false; + } + + return BinarySearch(list, offset, size) >= 0; + } + + public readonly List FindOverlaps(int offset, int size) + { + var list = _ranges; + if (list == null) + { + return null; + } + + List result = null; + + int index = BinarySearch(list, offset, size); + + if (index >= 0) + { + while (index > 0 && list[index - 1].OverlapsWith(offset, size)) + { + index--; + } + + do + { + (result ??= new List()).Add(list[index++]); + } + while (index < list.Count && list[index].OverlapsWith(offset, size)); + } + + return result; + } + + private static int BinarySearch(List list, int offset, int size) + { + int left = 0; + int right = list.Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + var item = list[middle]; + + if (item.OverlapsWith(offset, size)) + { + return middle; + } + + if (offset < item.Offset) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + public readonly void FillData(Span baseData, Span modData, int offset, Span result) + { + int size = baseData.Length; + int endOffset = offset + size; + + var list = _ranges; + if (list == null) + { + baseData.CopyTo(result); + } + + int srcOffset = offset; + int dstOffset = 0; + bool activeRange = false; + + for (int i = 0; i < list.Count; i++) + { + var range = list[i]; + + int rangeEnd = range.Offset + range.Size; + + if (activeRange) + { + if (range.Offset >= endOffset) + { + break; + } + } + else + { + if (rangeEnd <= offset) + { + continue; + } + + activeRange = true; + } + + int baseSize = range.Offset - srcOffset; + + if (baseSize > 0) + { + baseData.Slice(dstOffset, baseSize).CopyTo(result.Slice(dstOffset, baseSize)); + srcOffset += baseSize; + dstOffset += baseSize; + } + + int modSize = Math.Min(rangeEnd - srcOffset, endOffset - srcOffset); + if (modSize != 0) + { + modData.Slice(dstOffset, modSize).CopyTo(result.Slice(dstOffset, modSize)); + srcOffset += modSize; + dstOffset += modSize; + } + } + + int baseSizeEnd = endOffset - srcOffset; + + if (baseSizeEnd > 0) + { + baseData.Slice(dstOffset, baseSizeEnd).CopyTo(result.Slice(dstOffset, baseSizeEnd)); + } + } + + public readonly int Count() + { + return _ranges?.Count ?? 0; + } + + public void Clear() + { + _ranges = null; + } + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/BufferState.cs b/src/Ryujinx.Graphics.Vulkan/BufferState.cs index ee4badd2f..198ee54d4 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferState.cs @@ -23,7 +23,7 @@ namespace Ryujinx.Graphics.Vulkan { if (_buffer != null) { - var buffer = _buffer.Get(cbs, _offset, _size).Value; + var buffer = _buffer.Get(cbs, _offset, _size, true).Value; gd.TransformFeedbackApi.CmdBindTransformFeedbackBuffers(cbs.CommandBuffer, binding, 1, buffer, (ulong)_offset, (ulong)_size); } @@ -40,6 +40,11 @@ namespace Ryujinx.Graphics.Vulkan } } + public readonly bool Overlaps(Auto buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } + public readonly void Dispose() { _buffer?.DecrementReferenceCount(); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs b/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs index a8ff7c286..19dcaccd9 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs @@ -6,6 +6,7 @@ private readonly int _size; private readonly int _granularity; private readonly int _bits; + private readonly int _writeBitOffset; private readonly int _intsPerCb; private readonly int _bitsPerCb; @@ -14,7 +15,11 @@ { _size = size; _granularity = granularity; - _bits = (size + (granularity - 1)) / granularity; + + // There are two sets of bits - one for read tracking, and the other for write. + int bits = (size + (granularity - 1)) / granularity; + _writeBitOffset = bits; + _bits = bits << 1; _intsPerCb = (_bits + (BitMap.IntSize - 1)) / BitMap.IntSize; _bitsPerCb = _intsPerCb * BitMap.IntSize; @@ -22,7 +27,7 @@ _bitmap = new BitMap(_bitsPerCb * CommandBufferPool.MaxCommandBuffers); } - public void Add(int cbIndex, int offset, int size) + public void Add(int cbIndex, int offset, int size, bool write) { if (size == 0) { @@ -35,32 +40,32 @@ size = _size - offset; } - int cbBase = cbIndex * _bitsPerCb; + int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0); int start = cbBase + offset / _granularity; int end = cbBase + (offset + size - 1) / _granularity; _bitmap.SetRange(start, end); } - public bool OverlapsWith(int cbIndex, int offset, int size) + public bool OverlapsWith(int cbIndex, int offset, int size, bool write = false) { if (size == 0) { return false; } - int cbBase = cbIndex * _bitsPerCb; + int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0); int start = cbBase + offset / _granularity; int end = cbBase + (offset + size - 1) / _granularity; return _bitmap.IsSet(start, end); } - public bool OverlapsWith(int offset, int size) + public bool OverlapsWith(int offset, int size, bool write) { for (int i = 0; i < CommandBufferPool.MaxCommandBuffers; i++) { - if (OverlapsWith(i, offset, size)) + if (OverlapsWith(i, offset, size, write)) { return true; } diff --git a/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs b/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs index 45392b642..14e4c02f0 100644 --- a/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs +++ b/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs @@ -1,9 +1,9 @@ -using Ryujinx.Graphics.GAL; +using Ryujinx.Common.Memory; +using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Shader; using Silk.NET.Vulkan; using System; using System.Runtime.CompilerServices; -using Buffer = Silk.NET.Vulkan.Buffer; using CompareOp = Ryujinx.Graphics.GAL.CompareOp; using Format = Ryujinx.Graphics.GAL.Format; using SamplerCreateInfo = Ryujinx.Graphics.GAL.SamplerCreateInfo; @@ -12,13 +12,34 @@ namespace Ryujinx.Graphics.Vulkan { class DescriptorSetUpdater { + private const ulong StorageBufferMaxMirrorable = 0x2000; + private record struct BufferRef + { + public Auto Buffer; + public int Offset; + public bool Write; + + public BufferRef(Auto buffer) + { + Buffer = buffer; + Offset = 0; + Write = true; + } + + public BufferRef(Auto buffer, ref BufferRange range) + { + Buffer = buffer; + Offset = range.Offset; + Write = range.Write; + } + } + private readonly VulkanRenderer _gd; private readonly PipelineBase _pipeline; - private ShaderCollection _program; - private readonly Auto[] _uniformBufferRefs; - private readonly Auto[] _storageBufferRefs; + private readonly BufferRef[] _uniformBufferRefs; + private readonly BufferRef[] _storageBufferRefs; private readonly Auto[] _textureRefs; private readonly Auto[] _samplerRefs; private readonly Auto[] _imageRefs; @@ -33,8 +54,10 @@ namespace Ryujinx.Graphics.Vulkan private readonly BufferView[] _bufferTextures; private readonly BufferView[] _bufferImages; - private readonly bool[] _uniformSet; - private readonly bool[] _storageSet; + private BitMapStruct> _uniformSet; + private BitMapStruct> _storageSet; + private BitMapStruct> _uniformMirrored; + private BitMapStruct> _storageMirrored; [Flags] private enum DirtyFlags @@ -61,8 +84,8 @@ namespace Ryujinx.Graphics.Vulkan // Some of the bindings counts needs to be multiplied by 2 because we have buffer and // regular textures/images interleaved on the same descriptor set. - _uniformBufferRefs = new Auto[Constants.MaxUniformBufferBindings]; - _storageBufferRefs = new Auto[Constants.MaxStorageBufferBindings]; + _uniformBufferRefs = new BufferRef[Constants.MaxUniformBufferBindings]; + _storageBufferRefs = new BufferRef[Constants.MaxStorageBufferBindings]; _textureRefs = new Auto[Constants.MaxTextureBindings * 2]; _samplerRefs = new Auto[Constants.MaxTextureBindings * 2]; _imageRefs = new Auto[Constants.MaxImageBindings * 2]; @@ -85,9 +108,6 @@ namespace Ryujinx.Graphics.Vulkan _textures.AsSpan().Fill(initialImageInfo); _images.AsSpan().Fill(initialImageInfo); - _uniformSet = new bool[Constants.MaxUniformBufferBindings]; - _storageSet = new bool[Constants.MaxStorageBufferBindings]; - if (gd.Capabilities.SupportsNullDescriptors) { // If null descriptors are supported, we can pass null as the handle. @@ -138,6 +158,63 @@ namespace Ryujinx.Graphics.Vulkan _dummyTexture.SetData(dummyTextureData); } + private static bool BindingOverlaps(ref DescriptorBufferInfo info, int bindingOffset, int offset, int size) + { + return offset < bindingOffset + (int)info.Range && (offset + size) > bindingOffset; + } + + internal void Rebind(Auto buffer, int offset, int size) + { + if (_program == null) + { + return; + } + + // Check stage bindings + + _uniformMirrored.Union(_uniformSet).SignalSet((int binding, int count) => + { + for (int i = 0; i < count; i++) + { + ref BufferRef bufferRef = ref _uniformBufferRefs[binding]; + if (bufferRef.Buffer == buffer) + { + ref DescriptorBufferInfo info = ref _uniformBuffers[binding]; + int bindingOffset = bufferRef.Offset; + + if (BindingOverlaps(ref info, bindingOffset, offset, size)) + { + _uniformSet.Clear(binding); + SignalDirty(DirtyFlags.Uniform); + } + } + + binding++; + } + }); + + _storageMirrored.Union(_storageSet).SignalSet((int binding, int count) => + { + for (int i = 0; i < count; i++) + { + ref BufferRef bufferRef = ref _storageBufferRefs[binding]; + if (bufferRef.Buffer == buffer) + { + ref DescriptorBufferInfo info = ref _storageBuffers[binding]; + int bindingOffset = bufferRef.Offset; + + if (BindingOverlaps(ref info, bindingOffset, offset, size)) + { + _storageSet.Clear(binding); + SignalDirty(DirtyFlags.Storage); + } + } + + binding++; + } + }); + } + public void SetProgram(ShaderCollection program) { _program = program; @@ -180,22 +257,28 @@ namespace Ryujinx.Graphics.Vulkan var buffer = assignment.Range; int index = assignment.Binding; - Auto vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true); - ref Auto currentVkBuffer = ref _storageBufferRefs[index]; + Auto vkBuffer = buffer.Handle == BufferHandle.Null + ? null + : _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, buffer.Write, isSSBO: true); + + ref BufferRef currentBufferRef = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new() { Offset = (ulong)buffer.Offset, Range = (ulong)buffer.Size, }; + + var newRef = new BufferRef(vkBuffer, ref buffer); + ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _storageSet[index] = false; + _storageSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -209,21 +292,24 @@ namespace Ryujinx.Graphics.Vulkan var vkBuffer = buffers[i]; int index = first + i; - ref Auto currentVkBuffer = ref _storageBufferRefs[index]; + ref BufferRef currentBufferRef = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new() { Offset = 0, Range = Vk.WholeSize, }; + + BufferRef newRef = new(vkBuffer); + ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _storageSet[index] = false; + _storageSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -288,22 +374,28 @@ namespace Ryujinx.Graphics.Vulkan var buffer = assignment.Range; int index = assignment.Binding; - Auto vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); - ref Auto currentVkBuffer = ref _uniformBufferRefs[index]; + Auto vkBuffer = buffer.Handle == BufferHandle.Null + ? null + : _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); + + ref BufferRef currentBufferRef = ref _uniformBufferRefs[index]; DescriptorBufferInfo info = new() { Offset = (ulong)buffer.Offset, Range = (ulong)buffer.Size, }; + + BufferRef newRef = new(vkBuffer, ref buffer); + ref DescriptorBufferInfo currentInfo = ref _uniformBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _uniformSet[index] = false; + _uniformSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -353,13 +445,26 @@ namespace Ryujinx.Graphics.Vulkan } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void UpdateBuffer( + private static bool UpdateBuffer( CommandBufferScoped cbs, ref DescriptorBufferInfo info, - Auto buffer, - Auto dummyBuffer) + ref BufferRef buffer, + Auto dummyBuffer, + bool mirrorable) { - info.Buffer = buffer?.Get(cbs, (int)info.Offset, (int)info.Range).Value ?? default; + int offset = buffer.Offset; + bool mirrored = false; + + if (mirrorable) + { + info.Buffer = buffer.Buffer?.GetMirrorable(cbs, ref offset, (int)info.Range, out mirrored).Value ?? default; + } + else + { + info.Buffer = buffer.Buffer?.Get(cbs, offset, (int)info.Range, buffer.Write).Value ?? default; + } + + info.Offset = (ulong)offset; // The spec requires that buffers with null handle have offset as 0 and range as VK_WHOLE_SIZE. if (info.Buffer.Handle == 0) @@ -368,6 +473,8 @@ namespace Ryujinx.Graphics.Vulkan info.Offset = 0; info.Range = Vk.WholeSize; } + + return mirrored; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -404,11 +511,13 @@ namespace Ryujinx.Graphics.Vulkan { int index = binding + i; - if (!_uniformSet[index]) + if (_uniformSet.Set(index)) { - UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); + ref BufferRef buffer = ref _uniformBufferRefs[index]; - _uniformSet[index] = true; + bool mirrored = UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true); + + _uniformMirrored.Set(index, mirrored); } } @@ -421,11 +530,19 @@ namespace Ryujinx.Graphics.Vulkan { int index = binding + i; - if (!_storageSet[index]) - { - UpdateBuffer(cbs, ref _storageBuffers[index], _storageBufferRefs[index], dummyBuffer); + ref BufferRef buffer = ref _storageBufferRefs[index]; - _storageSet[index] = true; + if (_storageSet.Set(index)) + { + ref var info = ref _storageBuffers[index]; + + bool mirrored = UpdateBuffer(cbs, + ref info, + ref _storageBufferRefs[index], + dummyBuffer, + !buffer.Write && info.Range <= StorageBufferMaxMirrorable); + + _storageMirrored.Set(index, mirrored); } } @@ -464,7 +581,7 @@ namespace Ryujinx.Graphics.Vulkan for (int i = 0; i < count; i++) { - bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs) ?? default; + bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs, false) ?? default; } dsc.UpdateBufferImages(0, binding, bufferTextures[..count], DescriptorType.UniformTexelBuffer); @@ -489,7 +606,7 @@ namespace Ryujinx.Graphics.Vulkan for (int i = 0; i < count; i++) { - bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i]) ?? default; + bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i], true) ?? default; } dsc.UpdateBufferImages(0, binding, bufferImages[..count], DescriptorType.StorageTexelBuffer); @@ -546,10 +663,10 @@ namespace Ryujinx.Graphics.Vulkan { int index = binding + i; - if (!_uniformSet[index]) + if (_uniformSet.Set(index)) { - UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); - _uniformSet[index] = true; + ref BufferRef buffer = ref _uniformBufferRefs[index]; + UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true); doUpdate = true; } } @@ -582,17 +699,17 @@ namespace Ryujinx.Graphics.Vulkan { _dirty = DirtyFlags.All; - Array.Clear(_uniformSet); - Array.Clear(_storageSet); + _uniformSet.Clear(); + _storageSet.Clear(); } - private static void SwapBuffer(Auto[] list, Auto from, Auto to) + private static void SwapBuffer(BufferRef[] list, Auto from, Auto to) { for (int i = 0; i < list.Length; i++) { - if (list[i] == from) + if (list[i].Buffer == from) { - list[i] = to; + list[i].Buffer = to; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs index 9323fcf97..f478c58e2 100644 --- a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs +++ b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs @@ -427,6 +427,7 @@ namespace Ryujinx.Graphics.Vulkan return access switch { BufferAccess.FlushPersistent => BufferAllocationType.HostMapped, + BufferAccess.Stream => BufferAllocationType.HostMapped, _ => BufferAllocationType.Auto, }; } diff --git a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs index 11d9c4fb4..e76a332f4 100644 --- a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs +++ b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs @@ -52,6 +52,7 @@ namespace Ryujinx.Graphics.Vulkan public readonly PortabilitySubsetFlags PortabilitySubset; public readonly uint VertexBufferAlignment; public readonly uint SubTexelPrecisionBits; + public readonly ulong MinResourceAlignment; public HardwareCapabilities( bool supportsIndexTypeUint8, @@ -89,7 +90,8 @@ namespace Ryujinx.Graphics.Vulkan SampleCountFlags supportedSampleCounts, PortabilitySubsetFlags portabilitySubset, uint vertexBufferAlignment, - uint subTexelPrecisionBits) + uint subTexelPrecisionBits, + ulong minResourceAlignment) { SupportsIndexTypeUint8 = supportsIndexTypeUint8; SupportsCustomBorderColor = supportsCustomBorderColor; @@ -127,6 +129,7 @@ namespace Ryujinx.Graphics.Vulkan PortabilitySubset = portabilitySubset; VertexBufferAlignment = vertexBufferAlignment; SubTexelPrecisionBits = subTexelPrecisionBits; + MinResourceAlignment = minResourceAlignment; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs b/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs index b839619e9..b85f0c7f0 100644 --- a/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs @@ -5,6 +5,8 @@ namespace Ryujinx.Graphics.Vulkan { internal struct IndexBufferState { + private const int IndexBufferMaxMirrorable = 0x20000; + public static IndexBufferState Null => new(BufferHandle.Null, 0, 0); private readonly int _offset; @@ -37,6 +39,7 @@ namespace Ryujinx.Graphics.Vulkan Auto autoBuffer; int offset, size; IndexType type = _type; + bool mirrorable = false; if (_type == IndexType.Uint8Ext && !gd.Capabilities.SupportsIndexTypeUint8) { @@ -56,6 +59,8 @@ namespace Ryujinx.Graphics.Vulkan autoBuffer = null; } + mirrorable = _size < IndexBufferMaxMirrorable; + offset = _offset; size = _size; } @@ -64,7 +69,9 @@ namespace Ryujinx.Graphics.Vulkan if (autoBuffer != null) { - gd.Api.CmdBindIndexBuffer(cbs.CommandBuffer, autoBuffer.Get(cbs, offset, size).Value, (ulong)offset, type); + DisposableBuffer buffer = mirrorable ? autoBuffer.GetMirrorable(cbs, ref offset, size, out _) : autoBuffer.Get(cbs, offset, size); + + gd.Api.CmdBindIndexBuffer(cbs.CommandBuffer, buffer.Value, (ulong)offset, type); } } @@ -155,5 +162,10 @@ namespace Ryujinx.Graphics.Vulkan _buffer = to; } } + + public readonly bool Overlaps(Auto buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } } } diff --git a/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs b/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs index 71769c5e1..4d2d312fe 100644 --- a/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs @@ -32,14 +32,20 @@ namespace Ryujinx.Graphics.Vulkan } /// - /// Adds buffer usage information to the uses list. + /// Adds read/write buffer usage information to the uses list. /// /// Index of the command buffer where the buffer is used /// Offset of the buffer being used /// Size of the buffer region being used, in bytes - public void AddBufferUse(int cbIndex, int offset, int size) + /// Whether the access is a write or not + public void AddBufferUse(int cbIndex, int offset, int size, bool write) { - _bufferUsageBitmap.Add(cbIndex, offset, size); + _bufferUsageBitmap.Add(cbIndex, offset, size, false); + + if (write) + { + _bufferUsageBitmap.Add(cbIndex, offset, size, true); + } } /// @@ -68,10 +74,11 @@ namespace Ryujinx.Graphics.Vulkan /// /// Offset of the buffer being used /// Size of the buffer region being used, in bytes + /// True if only write usages should count /// True if in use, false otherwise - public bool IsBufferRangeInUse(int offset, int size) + public bool IsBufferRangeInUse(int offset, int size, bool write) { - return _bufferUsageBitmap.OverlapsWith(offset, size); + return _bufferUsageBitmap.OverlapsWith(offset, size, write); } /// diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs index 8b931e526..67b16ec96 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs @@ -193,7 +193,7 @@ namespace Ryujinx.Graphics.Vulkan { EndRenderPass(); - var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size).Value; + var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size, true).Value; BufferHolder.InsertBufferBarrier( Gd, @@ -469,6 +469,10 @@ namespace Ryujinx.Graphics.Vulkan return; } + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; + UpdateIndexBufferPattern(); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); @@ -498,10 +502,6 @@ namespace Ryujinx.Graphics.Vulkan } else { - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - ResumeTransformFeedbackInternal(); Gd.Api.CmdDrawIndexedIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); @@ -515,15 +515,19 @@ namespace Ryujinx.Graphics.Vulkan return; } + var countBuffer = Gd.BufferManager + .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) + .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; + + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; + UpdateIndexBufferPattern(); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); DrawCount++; - var countBuffer = Gd.BufferManager - .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) - .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; - if (_indexBufferPattern != null) { // Convert the index buffer into a supported topology. @@ -570,10 +574,6 @@ namespace Ryujinx.Graphics.Vulkan } else { - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - ResumeTransformFeedbackInternal(); if (Gd.Capabilities.SupportsIndirectParameters) @@ -609,15 +609,15 @@ namespace Ryujinx.Graphics.Vulkan // TODO: Support quads and other unsupported topologies. + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value; + RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); ResumeTransformFeedbackInternal(); DrawCount++; - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - Gd.Api.CmdDrawIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); } @@ -634,6 +634,14 @@ namespace Ryujinx.Graphics.Vulkan return; } + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value; + + var countBuffer = Gd.BufferManager + .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) + .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size, false).Value; + // TODO: Support quads and other unsupported topologies. RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); @@ -641,14 +649,6 @@ namespace Ryujinx.Graphics.Vulkan ResumeTransformFeedbackInternal(); DrawCount++; - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - - var countBuffer = Gd.BufferManager - .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) - .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; - Gd.DrawIndirectCountApi.CmdDrawIndirectCount( CommandBuffer, buffer, @@ -709,6 +709,26 @@ namespace Ryujinx.Graphics.Vulkan return CommandBuffer.Handle == cb.Handle; } + internal void Rebind(Auto buffer, int offset, int size) + { + _descriptorSetUpdater.Rebind(buffer, offset, size); + + if (_indexBuffer.Overlaps(buffer, offset, size)) + { + _indexBuffer.BindIndexBuffer(Gd, Cbs); + } + + for (int i = 0; i < _vertexBuffers.Length; i++) + { + if (_vertexBuffers[i].Overlaps(buffer, offset, size)) + { + _vertexBuffers[i].BindVertexBuffer(Gd, Cbs, (uint)i, ref _newState, _vertexBufferUpdater); + } + } + + _vertexBufferUpdater.Commit(Cbs); + } + #pragma warning disable CA1822 // Mark member as static public void SetAlphaTest(bool enable, float reference, CompareOp op) { diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs b/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs index dfdac52fd..dcc6c5300 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs @@ -14,6 +14,7 @@ namespace Ryujinx.Graphics.Vulkan private CounterQueueEvent _activeConditionalRender; private readonly List _pendingQueryCopies; + private readonly List _activeBufferMirrors; private ulong _byteWeight; @@ -24,6 +25,7 @@ namespace Ryujinx.Graphics.Vulkan _activeQueries = new List<(QueryPool, bool)>(); _pendingQueryCopies = new(); _backingSwaps = new(); + _activeBufferMirrors = new(); CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer; } @@ -233,6 +235,12 @@ namespace Ryujinx.Graphics.Vulkan Gd.RegisterFlush(); // Restore per-command buffer state. + foreach (BufferHolder buffer in _activeBufferMirrors) + { + buffer.ClearMirrors(); + } + + _activeBufferMirrors.Clear(); foreach ((var queryPool, var isOcclusion) in _activeQueries) { @@ -249,6 +257,11 @@ namespace Ryujinx.Graphics.Vulkan Restore(); } + public void RegisterActiveMirror(BufferHolder buffer) + { + _activeBufferMirrors.Add(buffer); + } + public void BeginQuery(BufferedQuery query, QueryPool pool, bool needsReset, bool isOcclusion, bool fromSamplePool) { if (needsReset) diff --git a/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs b/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs index 8d4fdc196..2a85429fb 100644 --- a/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs +++ b/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs @@ -183,7 +183,7 @@ namespace Ryujinx.Graphics.Vulkan.Queries public void PoolCopy(CommandBufferScoped cbs) { - var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long)).Value; + var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long), true).Value; QueryResultFlags flags = QueryResultFlags.ResultWaitBit; diff --git a/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs b/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs index 00fa6477d..32ec8c7c6 100644 --- a/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs @@ -1,12 +1,28 @@ +using Ryujinx.Common; +using Ryujinx.Common.Logging; using System; using System.Collections.Generic; using System.Diagnostics; namespace Ryujinx.Graphics.Vulkan { + readonly struct StagingBufferReserved + { + public readonly BufferHolder Buffer; + public readonly int Offset; + public readonly int Size; + + public StagingBufferReserved(BufferHolder buffer, int offset, int size) + { + Buffer = buffer; + Offset = offset; + Size = size; + } + } + class StagingBuffer : IDisposable { - private const int BufferSize = 16 * 1024 * 1024; + private const int BufferSize = 32 * 1024 * 1024; private int _freeOffset; private int _freeSize; @@ -130,13 +146,83 @@ namespace Ryujinx.Graphics.Vulkan } } - endRenderPass(); + endRenderPass?.Invoke(); PushDataImpl(cbs, dst, dstOffset, data); return true; } + private StagingBufferReserved ReserveDataImpl(CommandBufferScoped cbs, int size, int alignment) + { + // Assumes the caller has already determined that there is enough space. + int offset = BitUtils.AlignUp(_freeOffset, alignment); + int padding = offset - _freeOffset; + + int capacity = Math.Min(_freeSize, BufferSize - offset); + int reservedLength = size + padding; + if (capacity < size) + { + offset = 0; // Place at start. + reservedLength += capacity; + } + + _freeOffset = (_freeOffset + reservedLength) & (BufferSize - 1); + _freeSize -= reservedLength; + Debug.Assert(_freeSize >= 0); + + _pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), reservedLength)); + + return new StagingBufferReserved(_buffer, offset, size); + } + + private int GetContiguousFreeSize(int alignment) + { + int alignedFreeOffset = BitUtils.AlignUp(_freeOffset, alignment); + int padding = alignedFreeOffset - _freeOffset; + + // Free regions: + // - Aligned free offset to end (minimum free size - padding) + // - 0 to _freeOffset + freeSize wrapped (only if free area contains 0) + + int endOffset = (_freeOffset + _freeSize) & (BufferSize - 1); + + return Math.Max( + Math.Min(_freeSize - padding, BufferSize - alignedFreeOffset), + endOffset <= _freeOffset ? Math.Min(_freeSize, endOffset) : 0 + ); + } + + /// + /// Reserve a range on the staging buffer for the current command buffer and upload data to it. + /// + /// Command buffer to reserve the data on + /// The data to upload + /// The required alignment for the buffer offset + /// The reserved range of the staging buffer + public unsafe StagingBufferReserved? TryReserveData(CommandBufferScoped cbs, int size, int alignment) + { + if (size > BufferSize) + { + return null; + } + + // Temporary reserved data cannot be fragmented. + + if (GetContiguousFreeSize(alignment) < size) + { + FreeCompleted(); + + if (GetContiguousFreeSize(alignment) < size) + { + Logger.Debug?.PrintMsg(LogClass.Gpu, $"Staging buffer out of space to reserve data of size {size}."); + return null; + } + } + + return ReserveDataImpl(cbs, size, alignment); + } + private bool WaitFreeCompleted(CommandBufferPool cbp) { if (_pendingCopies.TryPeek(out var pc)) diff --git a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs index ddcf51f69..285a56498 100644 --- a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs @@ -127,24 +127,24 @@ namespace Ryujinx.Graphics.Vulkan ReleaseImpl(); } - public BufferView GetBufferView(CommandBufferScoped cbs) + public BufferView GetBufferView(CommandBufferScoped cbs, bool write) { _bufferView ??= _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl); - return _bufferView?.Get(cbs, _offset, _size).Value ?? default; + return _bufferView?.Get(cbs, _offset, _size, write).Value ?? default; } - public BufferView GetBufferView(CommandBufferScoped cbs, Format format) + public BufferView GetBufferView(CommandBufferScoped cbs, Format format, bool write) { var vkFormat = FormatTable.GetFormat(format); if (vkFormat == VkFormat) { - return GetBufferView(cbs); + return GetBufferView(cbs, write); } if (_selfManagedViews != null && _selfManagedViews.TryGetValue(format, out var bufferView)) { - return bufferView.Get(cbs, _offset, _size).Value; + return bufferView.Get(cbs, _offset, _size, write).Value; } bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl); @@ -154,7 +154,7 @@ namespace Ryujinx.Graphics.Vulkan (_selfManagedViews ??= new Dictionary>()).Add(format, bufferView); } - return bufferView?.Get(cbs, _offset, _size).Value ?? default; + return bufferView?.Get(cbs, _offset, _size, write).Value ?? default; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs b/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs index cbbd829ab..9a943bf98 100644 --- a/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs @@ -4,6 +4,8 @@ namespace Ryujinx.Graphics.Vulkan { internal struct VertexBufferState { + private const int VertexBufferMaxMirrorable = 0x20000; + public static VertexBufferState Null => new(null, 0, 0, 0); private readonly int _offset; @@ -88,9 +90,11 @@ namespace Ryujinx.Graphics.Vulkan if (autoBuffer != null) { - var buffer = autoBuffer.Get(cbs, _offset, _size).Value; + int offset = _offset; + bool mirrorable = _size <= VertexBufferMaxMirrorable; + var buffer = mirrorable ? autoBuffer.GetMirrorable(cbs, ref offset, _size, out _).Value : autoBuffer.Get(cbs, offset, _size).Value; - updater.BindVertexBuffer(cbs, binding, buffer, (ulong)_offset, (ulong)_size, (ulong)_stride); + updater.BindVertexBuffer(cbs, binding, buffer, (ulong)offset, (ulong)_size, (ulong)_stride); } } @@ -99,6 +103,11 @@ namespace Ryujinx.Graphics.Vulkan return _buffer == buffer; } + public readonly bool Overlaps(Auto buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } + public readonly bool Matches(Auto buffer, int descriptorIndex, int offset, int size, int stride = 0) { return _buffer == buffer && DescriptorIndex == descriptorIndex && _offset == offset && _size == size && _stride == stride; diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs index 11c3bfe4e..20b32c70b 100644 --- a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs +++ b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs @@ -293,6 +293,13 @@ namespace Ryujinx.Graphics.Vulkan ref var properties = ref properties2.Properties; + ulong minResourceAlignment = Math.Max( + Math.Max( + properties.Limits.MinStorageBufferOffsetAlignment, + properties.Limits.MinUniformBufferOffsetAlignment), + properties.Limits.MinTexelBufferOffsetAlignment + ); + SampleCountFlags supportedSampleCounts = properties.Limits.FramebufferColorSampleCounts & properties.Limits.FramebufferDepthSampleCounts & @@ -334,7 +341,8 @@ namespace Ryujinx.Graphics.Vulkan supportedSampleCounts, portabilityFlags, vertexBufferAlignment, - properties.Limits.SubTexelPrecisionBits); + properties.Limits.SubTexelPrecisionBits, + minResourceAlignment); IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(_physicalDevice); @@ -397,7 +405,7 @@ namespace Ryujinx.Graphics.Vulkan public BufferHandle CreateBuffer(int size, BufferAccess access) { - return BufferManager.CreateWithHandle(this, size, access.Convert()); + return BufferManager.CreateWithHandle(this, size, access.Convert(), default, access == BufferAccess.Stream); } public BufferHandle CreateBuffer(int size, BufferHandle storageHint)