Vulkan: Migrate buffers between memory types to improve GPU performance (#4540)

* Initial implementation of migration between memory heaps

- Missing OOM handling
- Missing `_map` data safety when remapping
  - Copy may not have completed yet (needs some kind of fence)
  - Map may be unmapped before it is done being used. (needs scoped access)
- SSBO accesses are all "writes" - maybe pass info in another way.
- Missing keeping map type when resizing buffers (should this be done?)

* Ensure migrated data is in place before flushing.

* Fix issue where old waitable would be signalled.

- There is a real issue where existing Auto<> references need to be replaced.

* Swap bound Auto<> instances when swapping buffer backing

* Fix conversion buffers

* Don't try move buffers if the host has shared memory.

* Make GPU methods return PinnedSpan with scope

* Storage Hint

* Fix stupidity

* Fix rebase

* Tweak rules

Attempt to sidestep BOTW slowdown

* Remove line

* Migrate only when command buffers flush

* Change backing swap log to debug

* Address some feedback

* Disallow backing swap when the flush lock is held by the current thread

* Make PinnedSpan from ReadOnlySpan explicitly unsafe

* Fix some small issues

- Index buffer swap fixed
- Allocate DeviceLocal buffers using a separate block list to images.

* Remove alternative flags

* Address feedback
This commit is contained in:
riperiperi 2023-03-19 20:56:48 +00:00 committed by GitHub
parent 67b4e63cff
commit 9f1cf6458c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 660 additions and 167 deletions

View file

@ -15,7 +15,12 @@ namespace Ryujinx.Graphics.GAL
void BackgroundContextAction(Action action, bool alwaysBackground = false); void BackgroundContextAction(Action action, bool alwaysBackground = false);
BufferHandle CreateBuffer(int size); BufferHandle CreateBuffer(int size, BufferHandle storageHint);
BufferHandle CreateBuffer(int size)
{
return CreateBuffer(size, BufferHandle.Null);
}
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info); IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
@ -26,7 +31,7 @@ namespace Ryujinx.Graphics.GAL
void DeleteBuffer(BufferHandle buffer); void DeleteBuffer(BufferHandle buffer);
ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size); PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
Capabilities GetCapabilities(); Capabilities GetCapabilities();
ulong GetCurrentSync(); ulong GetCurrentSync();

View file

@ -15,8 +15,8 @@ namespace Ryujinx.Graphics.GAL
ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel); ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel);
ReadOnlySpan<byte> GetData(); PinnedSpan<byte> GetData();
ReadOnlySpan<byte> GetData(int layer, int level); PinnedSpan<byte> GetData(int layer, int level);
void SetData(SpanOrArray<byte> data); void SetData(SpanOrArray<byte> data);
void SetData(SpanOrArray<byte> data, int layer, int level); void SetData(SpanOrArray<byte> data, int layer, int level);

View file

@ -21,9 +21,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Buffer
public static void Run(ref BufferGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer) public static void Run(ref BufferGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer)
{ {
ReadOnlySpan<byte> result = renderer.GetBufferData(threaded.Buffers.MapBuffer(command._buffer), command._offset, command._size); PinnedSpan<byte> result = renderer.GetBufferData(threaded.Buffers.MapBuffer(command._buffer), command._offset, command._size);
command._result.Get(threaded).Result = new PinnedSpan<byte>(result); command._result.Get(threaded).Result = result;
} }
} }
} }

View file

@ -5,16 +5,25 @@
public CommandType CommandType => CommandType.CreateBuffer; public CommandType CommandType => CommandType.CreateBuffer;
private BufferHandle _threadedHandle; private BufferHandle _threadedHandle;
private int _size; private int _size;
private BufferHandle _storageHint;
public void Set(BufferHandle threadedHandle, int size) public void Set(BufferHandle threadedHandle, int size, BufferHandle storageHint)
{ {
_threadedHandle = threadedHandle; _threadedHandle = threadedHandle;
_size = size; _size = size;
_storageHint = storageHint;
} }
public static void Run(ref CreateBufferCommand command, ThreadedRenderer threaded, IRenderer renderer) public static void Run(ref CreateBufferCommand command, ThreadedRenderer threaded, IRenderer renderer)
{ {
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size)); BufferHandle hint = BufferHandle.Null;
if (command._storageHint != BufferHandle.Null)
{
hint = threaded.Buffers.MapBuffer(command._storageHint);
}
threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, hint));
} }
} }
} }

View file

@ -18,9 +18,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture
public static void Run(ref TextureGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer) public static void Run(ref TextureGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer)
{ {
ReadOnlySpan<byte> result = command._texture.Get(threaded).Base.GetData(); PinnedSpan<byte> result = command._texture.Get(threaded).Base.GetData();
command._result.Get(threaded).Result = new PinnedSpan<byte>(result); command._result.Get(threaded).Result = result;
} }
} }
} }

View file

@ -22,9 +22,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture
public static void Run(ref TextureGetDataSliceCommand command, ThreadedRenderer threaded, IRenderer renderer) public static void Run(ref TextureGetDataSliceCommand command, ThreadedRenderer threaded, IRenderer renderer)
{ {
ReadOnlySpan<byte> result = command._texture.Get(threaded).Base.GetData(command._layer, command._level); PinnedSpan<byte> result = command._texture.Get(threaded).Base.GetData(command._layer, command._level);
command._result.Get(threaded).Result = new PinnedSpan<byte>(result); command._result.Get(threaded).Result = result;
} }
} }
} }

View file

@ -1,23 +0,0 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.GAL.Multithreading.Model
{
unsafe struct PinnedSpan<T> where T : unmanaged
{
private void* _ptr;
private int _size;
public PinnedSpan(ReadOnlySpan<T> span)
{
_ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(span));
_size = span.Length;
}
public ReadOnlySpan<T> Get()
{
return new ReadOnlySpan<T>(_ptr, _size * Unsafe.SizeOf<T>());
}
}
}

View file

@ -72,7 +72,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
return newTex; return newTex;
} }
public ReadOnlySpan<byte> GetData() public PinnedSpan<byte> GetData()
{ {
if (_renderer.IsGpuThread()) if (_renderer.IsGpuThread())
{ {
@ -80,7 +80,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
_renderer.New<TextureGetDataCommand>().Set(Ref(this), Ref(box)); _renderer.New<TextureGetDataCommand>().Set(Ref(this), Ref(box));
_renderer.InvokeCommand(); _renderer.InvokeCommand();
return box.Result.Get(); return box.Result;
} }
else else
{ {
@ -90,7 +90,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
} }
} }
public ReadOnlySpan<byte> GetData(int layer, int level) public PinnedSpan<byte> GetData(int layer, int level)
{ {
if (_renderer.IsGpuThread()) if (_renderer.IsGpuThread())
{ {
@ -98,7 +98,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources
_renderer.New<TextureGetDataSliceCommand>().Set(Ref(this), Ref(box), layer, level); _renderer.New<TextureGetDataSliceCommand>().Set(Ref(this), Ref(box), layer, level);
_renderer.InvokeCommand(); _renderer.InvokeCommand();
return box.Result.Get(); return box.Result;
} }
else else
{ {

View file

@ -265,10 +265,10 @@ namespace Ryujinx.Graphics.GAL.Multithreading
} }
} }
public BufferHandle CreateBuffer(int size) public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{ {
BufferHandle handle = Buffers.CreateBufferHandle(); BufferHandle handle = Buffers.CreateBufferHandle();
New<CreateBufferCommand>().Set(handle, size); New<CreateBufferCommand>().Set(handle, size, storageHint);
QueueCommand(); QueueCommand();
return handle; return handle;
@ -329,7 +329,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
QueueCommand(); QueueCommand();
} }
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{ {
if (IsGpuThread()) if (IsGpuThread())
{ {
@ -337,7 +337,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
New<BufferGetDataCommand>().Set(buffer, offset, size, Ref(box)); New<BufferGetDataCommand>().Set(buffer, offset, size, Ref(box));
InvokeCommand(); InvokeCommand();
return box.Result.Get(); return box.Result;
} }
else else
{ {

View file

@ -0,0 +1,53 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.GAL
{
public unsafe struct PinnedSpan<T> : IDisposable where T : unmanaged
{
private void* _ptr;
private int _size;
private Action _disposeAction;
/// <summary>
/// Creates a new PinnedSpan from an existing ReadOnlySpan. The span *must* be pinned in memory.
/// The data must be guaranteed to live until disposeAction is called.
/// </summary>
/// <param name="span">Existing span</param>
/// <param name="disposeAction">Action to call on dispose</param>
/// <remarks>
/// If a dispose action is not provided, it is safe to assume the resource will be available until the next call.
/// </remarks>
public static PinnedSpan<T> UnsafeFromSpan(ReadOnlySpan<T> span, Action disposeAction = null)
{
return new PinnedSpan<T>(Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)), span.Length, disposeAction);
}
/// <summary>
/// Creates a new PinnedSpan from an existing unsafe region. The data must be guaranteed to live until disposeAction is called.
/// </summary>
/// <param name="ptr">Pointer to the region</param>
/// <param name="size">The total items of T the region contains</param>
/// <param name="disposeAction">Action to call on dispose</param>
/// <remarks>
/// If a dispose action is not provided, it is safe to assume the resource will be available until the next call.
/// </remarks>
public PinnedSpan(void* ptr, int size, Action disposeAction = null)
{
_ptr = ptr;
_size = size;
_disposeAction = disposeAction;
}
public ReadOnlySpan<T> Get()
{
return new ReadOnlySpan<T>(_ptr, _size * Unsafe.SizeOf<T>());
}
public void Dispose()
{
_disposeAction?.Invoke();
}
}
}

View file

@ -1022,13 +1022,12 @@ namespace Ryujinx.Graphics.Gpu.Image
/// This method should be used to retrieve data that was modified by the host GPU. /// This method should be used to retrieve data that was modified by the host GPU.
/// This is not cheap, avoid doing that unless strictly needed. /// This is not cheap, avoid doing that unless strictly needed.
/// </remarks> /// </remarks>
/// <param name="output">An output span to place the texture data into. If empty, one is generated</param> /// <param name="output">An output span to place the texture data into</param>
/// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param> /// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param> /// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
/// <returns>The span containing the texture data</returns> private void GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
private ReadOnlySpan<byte> GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
{ {
ReadOnlySpan<byte> data; PinnedSpan<byte> data;
if (texture != null) if (texture != null)
{ {
@ -1054,9 +1053,9 @@ namespace Ryujinx.Graphics.Gpu.Image
} }
} }
data = ConvertFromHostCompatibleFormat(output, data); ConvertFromHostCompatibleFormat(output, data.Get());
return data; data.Dispose();
} }
/// <summary> /// <summary>
@ -1071,10 +1070,9 @@ namespace Ryujinx.Graphics.Gpu.Image
/// <param name="level">The level of the texture to flush</param> /// <param name="level">The level of the texture to flush</param>
/// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param> /// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param> /// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
/// <returns>The span containing the texture data</returns> public void GetTextureDataSliceFromGpu(Span<byte> output, int layer, int level, bool blacklist, ITexture texture = null)
public ReadOnlySpan<byte> GetTextureDataSliceFromGpu(Span<byte> output, int layer, int level, bool blacklist, ITexture texture = null)
{ {
ReadOnlySpan<byte> data; PinnedSpan<byte> data;
if (texture != null) if (texture != null)
{ {
@ -1100,9 +1098,9 @@ namespace Ryujinx.Graphics.Gpu.Image
} }
} }
data = ConvertFromHostCompatibleFormat(output, data, level, true); ConvertFromHostCompatibleFormat(output, data.Get(), level, true);
return data; data.Dispose();
} }
/// <summary> /// <summary>

View file

@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
Address = address; Address = address;
Size = size; Size = size;
Handle = context.Renderer.CreateBuffer((int)size); Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
_useGranular = size > GranularBufferThreshold; _useGranular = size > GranularBufferThreshold;
@ -415,10 +415,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
int offset = (int)(address - Address); int offset = (int)(address - Address);
ReadOnlySpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size); using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
// TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
_physicalMemory.WriteUntracked(address, data); _physicalMemory.WriteUntracked(address, data.Get());
} }
/// <summary> /// <summary>

View file

@ -55,11 +55,14 @@ namespace Ryujinx.Graphics.OpenGL
(IntPtr)size); (IntPtr)size);
} }
public static unsafe ReadOnlySpan<byte> GetData(OpenGLRenderer renderer, BufferHandle buffer, int offset, int size) public static unsafe PinnedSpan<byte> GetData(OpenGLRenderer renderer, BufferHandle buffer, int offset, int size)
{ {
// Data in the persistent buffer and host array is guaranteed to be available
// until the next time the host thread requests data.
if (HwCapabilities.UsePersistentBufferForFlush) if (HwCapabilities.UsePersistentBufferForFlush)
{ {
return renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size); return PinnedSpan<byte>.UnsafeFromSpan(renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size));
} }
else else
{ {
@ -69,7 +72,7 @@ namespace Ryujinx.Graphics.OpenGL
GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target); GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target);
return new ReadOnlySpan<byte>(target.ToPointer(), size); return new PinnedSpan<byte>(target.ToPointer(), size);
} }
} }

View file

@ -39,12 +39,12 @@ namespace Ryujinx.Graphics.OpenGL.Image
throw new NotSupportedException(); throw new NotSupportedException();
} }
public ReadOnlySpan<byte> GetData() public PinnedSpan<byte> GetData()
{ {
return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize); return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize);
} }
public ReadOnlySpan<byte> GetData(int layer, int level) public PinnedSpan<byte> GetData(int layer, int level)
{ {
return GetData(); return GetData();
} }

View file

@ -166,7 +166,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
_renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter); _renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter);
} }
public unsafe ReadOnlySpan<byte> GetData() public unsafe PinnedSpan<byte> GetData()
{ {
int size = 0; int size = 0;
int levels = Info.GetLevelsClamped(); int levels = Info.GetLevelsClamped();
@ -196,16 +196,16 @@ namespace Ryujinx.Graphics.OpenGL.Image
data = FormatConverter.ConvertD24S8ToS8D24(data); data = FormatConverter.ConvertD24S8ToS8D24(data);
} }
return data; return PinnedSpan<byte>.UnsafeFromSpan(data);
} }
public unsafe ReadOnlySpan<byte> GetData(int layer, int level) public unsafe PinnedSpan<byte> GetData(int layer, int level)
{ {
int size = Info.GetMipSize(level); int size = Info.GetMipSize(level);
if (HwCapabilities.UsePersistentBufferForFlush) if (HwCapabilities.UsePersistentBufferForFlush)
{ {
return _renderer.PersistentBuffers.Default.GetTextureData(this, size, layer, level); return PinnedSpan<byte>.UnsafeFromSpan(_renderer.PersistentBuffers.Default.GetTextureData(this, size, layer, level));
} }
else else
{ {
@ -213,7 +213,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
int offset = WriteTo2D(target, layer, level); int offset = WriteTo2D(target, layer, level);
return new ReadOnlySpan<byte>(target.ToPointer(), size).Slice(offset); return new PinnedSpan<byte>((byte*)target.ToPointer() + offset, size);
} }
} }

View file

@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.OpenGL
ResourcePool = new ResourcePool(); ResourcePool = new ResourcePool();
} }
public BufferHandle CreateBuffer(int size) public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{ {
BufferCount++; BufferCount++;
@ -96,7 +96,7 @@ namespace Ryujinx.Graphics.OpenGL
return new HardwareInfo(GpuVendor, GpuRenderer); return new HardwareInfo(GpuVendor, GpuRenderer);
} }
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{ {
return Buffer.GetData(this, buffer, offset, size); return Buffer.GetData(this, buffer, offset, size);
} }

View file

@ -0,0 +1,12 @@
namespace Ryujinx.Graphics.Vulkan
{
internal enum BufferAllocationType
{
Auto = 0,
HostMappedNoCache,
HostMapped,
DeviceLocal,
DeviceLocalMapped
}
}

View file

@ -1,7 +1,10 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Common.Logging;
using Ryujinx.Graphics.GAL;
using Silk.NET.Vulkan; using Silk.NET.Vulkan;
using System; using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Threading;
using VkBuffer = Silk.NET.Vulkan.Buffer; using VkBuffer = Silk.NET.Vulkan.Buffer;
using VkFormat = Silk.NET.Vulkan.Format; using VkFormat = Silk.NET.Vulkan.Format;
@ -11,6 +14,12 @@ namespace Ryujinx.Graphics.Vulkan
{ {
private const int MaxUpdateBufferSize = 0x10000; private const int MaxUpdateBufferSize = 0x10000;
private const int SetCountThreshold = 100;
private const int WriteCountThreshold = 50;
private const int FlushCountThreshold = 5;
public const int DeviceLocalSizeThreshold = 256 * 1024; // 256kb
public const AccessFlags DefaultAccessFlags = public const AccessFlags DefaultAccessFlags =
AccessFlags.IndirectCommandReadBit | AccessFlags.IndirectCommandReadBit |
AccessFlags.ShaderReadBit | AccessFlags.ShaderReadBit |
@ -21,10 +30,10 @@ namespace Ryujinx.Graphics.Vulkan
private readonly VulkanRenderer _gd; private readonly VulkanRenderer _gd;
private readonly Device _device; private readonly Device _device;
private readonly MemoryAllocation _allocation; private MemoryAllocation _allocation;
private readonly Auto<DisposableBuffer> _buffer; private Auto<DisposableBuffer> _buffer;
private readonly Auto<MemoryAllocation> _allocationAuto; private Auto<MemoryAllocation> _allocationAuto;
private readonly ulong _bufferHandle; private ulong _bufferHandle;
private CacheByRange<BufferHolder> _cachedConvertedBuffers; private CacheByRange<BufferHolder> _cachedConvertedBuffers;
@ -32,11 +41,28 @@ namespace Ryujinx.Graphics.Vulkan
private IntPtr _map; private IntPtr _map;
private readonly MultiFenceHolder _waitable; private MultiFenceHolder _waitable;
private bool _lastAccessIsWrite; private bool _lastAccessIsWrite;
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size) private BufferAllocationType _baseType;
private BufferAllocationType _currentType;
private bool _swapQueued;
public BufferAllocationType DesiredType { get; private set; }
private int _setCount;
private int _writeCount;
private int _flushCount;
private int _flushTemp;
private ReaderWriterLock _flushLock;
private FenceHolder _flushFence;
private int _flushWaiting;
private List<Action> _swapActions;
public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType)
{ {
_gd = gd; _gd = gd;
_device = device; _device = device;
@ -47,9 +73,153 @@ namespace Ryujinx.Graphics.Vulkan
_bufferHandle = buffer.Handle; _bufferHandle = buffer.Handle;
Size = size; Size = size;
_map = allocation.HostPointer; _map = allocation.HostPointer;
_baseType = type;
_currentType = currentType;
DesiredType = currentType;
_flushLock = new ReaderWriterLock();
} }
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size) public bool TryBackingSwap(ref CommandBufferScoped? cbs)
{
if (_swapQueued && DesiredType != _currentType)
{
// Only swap if the buffer is not used in any queued command buffer.
bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool);
if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld)
{
var currentAllocation = _allocationAuto;
var currentBuffer = _buffer;
IntPtr currentMap = _map;
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, _currentType);
if (buffer.Handle != 0)
{
_flushLock.AcquireWriterLock(Timeout.Infinite);
ClearFlushFence();
_waitable = new MultiFenceHolder(Size);
_allocation = allocation;
_allocationAuto = new Auto<MemoryAllocation>(allocation);
_buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto);
_bufferHandle = buffer.Handle;
_map = allocation.HostPointer;
if (_map != IntPtr.Zero && currentMap != IntPtr.Zero)
{
// Copy data directly. Readbacks don't have to wait if this is done.
unsafe
{
new Span<byte>((void*)currentMap, Size).CopyTo(new Span<byte>((void*)_map, Size));
}
}
else
{
if (cbs == null)
{
cbs = _gd.CommandBufferPool.Rent();
}
CommandBufferScoped cbsV = cbs.Value;
Copy(_gd, cbsV, currentBuffer, _buffer, 0, 0, Size);
// Need to wait for the data to reach the new buffer before data can be flushed.
_flushFence = _gd.CommandBufferPool.GetFence(cbsV.CommandBufferIndex);
_flushFence.Get();
}
Logger.Debug?.PrintMsg(LogClass.Gpu, $"Converted {Size} buffer {_currentType} to {resultType}");
_currentType = resultType;
if (_swapActions != null)
{
foreach (var action in _swapActions)
{
action();
}
_swapActions.Clear();
}
currentBuffer.Dispose();
currentAllocation.Dispose();
_gd.PipelineInternal.SwapBuffer(currentBuffer, _buffer);
_flushLock.ReleaseWriterLock();
}
_swapQueued = false;
return true;
}
else
{
return false;
}
}
else
{
_swapQueued = false;
return true;
}
}
private void ConsiderBackingSwap()
{
if (_baseType == BufferAllocationType.Auto)
{
if (_writeCount >= WriteCountThreshold || _setCount >= SetCountThreshold || _flushCount >= FlushCountThreshold)
{
if (_flushCount > 0 || _flushTemp-- > 0)
{
// Buffers that flush should ideally be mapped in host address space for easy copies.
// If the buffer is large it will do better on GPU memory, as there will be more writes than data flushes (typically individual pages).
// If it is small, then it's likely most of the buffer will be flushed so we want it on host memory, as access is cached.
DesiredType = Size > DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocalMapped : BufferAllocationType.HostMapped;
// It's harder for a buffer that is flushed to revert to another type of mapping.
if (_flushCount > 0)
{
_flushTemp = 1000;
}
}
else if (_writeCount >= WriteCountThreshold)
{
// Buffers that are written often should ideally be in the device local heap. (Storage buffers)
DesiredType = BufferAllocationType.DeviceLocal;
}
else if (_setCount > SetCountThreshold)
{
// Buffers that have their data set often should ideally be host mapped. (Constant buffers)
DesiredType = BufferAllocationType.HostMapped;
}
_flushCount = 0;
_writeCount = 0;
_setCount = 0;
}
if (!_swapQueued && DesiredType != _currentType)
{
_swapQueued = true;
_gd.PipelineInternal.AddBackingSwap(this);
}
}
}
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView)
{ {
var bufferViewCreateInfo = new BufferViewCreateInfo() var bufferViewCreateInfo = new BufferViewCreateInfo()
{ {
@ -62,9 +232,19 @@ namespace Ryujinx.Graphics.Vulkan
_gd.Api.CreateBufferView(_device, bufferViewCreateInfo, null, out var bufferView).ThrowOnError(); _gd.Api.CreateBufferView(_device, bufferViewCreateInfo, null, out var bufferView).ThrowOnError();
(_swapActions ??= new List<Action>()).Add(invalidateView);
return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer); return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer);
} }
public void InheritMetrics(BufferHolder other)
{
_setCount = other._setCount;
_writeCount = other._writeCount;
_flushCount = other._flushCount;
_flushTemp = other._flushTemp;
}
public unsafe void InsertBarrier(CommandBuffer commandBuffer, bool isWrite) public unsafe void InsertBarrier(CommandBuffer commandBuffer, bool isWrite)
{ {
// If the last access is write, we always need a barrier to be sure we will read or modify // If the last access is write, we always need a barrier to be sure we will read or modify
@ -104,12 +284,22 @@ namespace Ryujinx.Graphics.Vulkan
return _buffer; return _buffer;
} }
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, bool isWrite = false) public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, bool isWrite = false, bool isSSBO = false)
{ {
if (isWrite) if (isWrite)
{ {
_writeCount++;
SignalWrite(0, Size); SignalWrite(0, Size);
} }
else if (isSSBO)
{
// Always consider SSBO access for swapping to device local memory.
_writeCount++;
ConsiderBackingSwap();
}
return _buffer; return _buffer;
} }
@ -118,6 +308,8 @@ namespace Ryujinx.Graphics.Vulkan
{ {
if (isWrite) if (isWrite)
{ {
_writeCount++;
SignalWrite(offset, size); SignalWrite(offset, size);
} }
@ -126,6 +318,8 @@ namespace Ryujinx.Graphics.Vulkan
public void SignalWrite(int offset, int size) public void SignalWrite(int offset, int size)
{ {
ConsiderBackingSwap();
if (offset == 0 && size == Size) if (offset == 0 && size == Size)
{ {
_cachedConvertedBuffers.Clear(); _cachedConvertedBuffers.Clear();
@ -147,11 +341,76 @@ namespace Ryujinx.Graphics.Vulkan
return _map; return _map;
} }
public unsafe ReadOnlySpan<byte> GetData(int offset, int size) private void ClearFlushFence()
{ {
// Asusmes _flushLock is held as writer.
if (_flushFence != null)
{
if (_flushWaiting == 0)
{
_flushFence.Put();
}
_flushFence = null;
}
}
private void WaitForFlushFence()
{
// Assumes the _flushLock is held as reader, returns in same state.
if (_flushFence != null)
{
// If storage has changed, make sure the fence has been reached so that the data is in place.
var cookie = _flushLock.UpgradeToWriterLock(Timeout.Infinite);
if (_flushFence != null)
{
var fence = _flushFence;
Interlocked.Increment(ref _flushWaiting);
// Don't wait in the lock.
var restoreCookie = _flushLock.ReleaseLock();
fence.Wait();
_flushLock.RestoreLock(ref restoreCookie);
if (Interlocked.Decrement(ref _flushWaiting) == 0)
{
fence.Put();
}
_flushFence = null;
}
_flushLock.DowngradeFromWriterLock(ref cookie);
}
}
public unsafe PinnedSpan<byte> GetData(int offset, int size)
{
_flushLock.AcquireReaderLock(Timeout.Infinite);
WaitForFlushFence();
_flushCount++;
Span<byte> result;
if (_map != IntPtr.Zero) if (_map != IntPtr.Zero)
{ {
return GetDataStorage(offset, size); result = GetDataStorage(offset, size);
// Need to be careful here, the buffer can't be unmapped while the data is being used.
_buffer.IncrementReferenceCount();
_flushLock.ReleaseReaderLock();
return PinnedSpan<byte>.UnsafeFromSpan(result, _buffer.DecrementReferenceCount);
} }
else else
{ {
@ -161,12 +420,17 @@ namespace Ryujinx.Graphics.Vulkan
{ {
_gd.FlushAllCommands(); _gd.FlushAllCommands();
return resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size); result = resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size);
} }
else else
{ {
return resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size); result = resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size);
} }
_flushLock.ReleaseReaderLock();
// Flush buffer is pinned until the next GetBufferData on the thread, which is fine for current uses.
return PinnedSpan<byte>.UnsafeFromSpan(result);
} }
} }
@ -190,6 +454,8 @@ namespace Ryujinx.Graphics.Vulkan
return; return;
} }
_setCount++;
if (_map != IntPtr.Zero) if (_map != IntPtr.Zero)
{ {
// If persistently mapped, set the data directly if the buffer is not currently in use. // If persistently mapped, set the data directly if the buffer is not currently in use.
@ -268,6 +534,8 @@ namespace Ryujinx.Graphics.Vulkan
var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value; var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value;
_writeCount--;
InsertBufferBarrier( InsertBufferBarrier(
_gd, _gd,
cbs.CommandBuffer, cbs.CommandBuffer,
@ -502,11 +770,19 @@ namespace Ryujinx.Graphics.Vulkan
public void Dispose() public void Dispose()
{ {
_swapQueued = false;
_gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_buffer, (ulong)Size); _gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_buffer, (ulong)Size);
_buffer.Dispose(); _buffer.Dispose();
_allocationAuto.Dispose(); _allocationAuto.Dispose();
_cachedConvertedBuffers.Dispose(); _cachedConvertedBuffers.Dispose();
_flushLock.AcquireWriterLock(Timeout.Infinite);
ClearFlushFence();
_flushLock.ReleaseWriterLock();
} }
} }
} }

View file

@ -4,6 +4,7 @@ using System;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using VkFormat = Silk.NET.Vulkan.Format; using VkFormat = Silk.NET.Vulkan.Format;
using VkBuffer = Silk.NET.Vulkan.Buffer;
namespace Ryujinx.Graphics.Vulkan namespace Ryujinx.Graphics.Vulkan
{ {
@ -16,17 +17,17 @@ namespace Ryujinx.Graphics.Vulkan
// Some drivers don't expose a "HostCached" memory type, // Some drivers don't expose a "HostCached" memory type,
// so we need those alternative flags for the allocation to succeed there. // so we need those alternative flags for the allocation to succeed there.
private const MemoryPropertyFlags DefaultBufferMemoryAltFlags = private const MemoryPropertyFlags DefaultBufferMemoryNoCacheFlags =
MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostVisibleBit |
MemoryPropertyFlags.HostCoherentBit; MemoryPropertyFlags.HostCoherentBit;
private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags = private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags =
MemoryPropertyFlags.DeviceLocalBit; MemoryPropertyFlags.DeviceLocalBit;
private const MemoryPropertyFlags FlushableDeviceLocalBufferMemoryFlags = private const MemoryPropertyFlags DeviceLocalMappedBufferMemoryFlags =
MemoryPropertyFlags.DeviceLocalBit |
MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostVisibleBit |
MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.HostCoherentBit;
MemoryPropertyFlags.DeviceLocalBit;
private const BufferUsageFlags DefaultBufferUsageFlags = private const BufferUsageFlags DefaultBufferUsageFlags =
BufferUsageFlags.TransferSrcBit | BufferUsageFlags.TransferSrcBit |
@ -54,14 +55,14 @@ namespace Ryujinx.Graphics.Vulkan
StagingBuffer = new StagingBuffer(gd, this); StagingBuffer = new StagingBuffer(gd, this);
} }
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal) public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default)
{ {
return CreateWithHandle(gd, size, deviceLocal, out _); return CreateWithHandle(gd, size, out _, baseType, storageHint);
} }
public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal, out BufferHolder holder) public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, out BufferHolder holder, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default)
{ {
holder = Create(gd, size, deviceLocal: deviceLocal); holder = Create(gd, size, baseType: baseType, storageHint: storageHint);
if (holder == null) if (holder == null)
{ {
return BufferHandle.Null; return BufferHandle.Null;
@ -74,7 +75,12 @@ namespace Ryujinx.Graphics.Vulkan
return Unsafe.As<ulong, BufferHandle>(ref handle64); return Unsafe.As<ulong, BufferHandle>(ref handle64);
} }
public unsafe BufferHolder Create(VulkanRenderer gd, int size, bool forConditionalRendering = false, bool deviceLocal = false) public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking(
VulkanRenderer gd,
int size,
BufferAllocationType type,
bool forConditionalRendering = false,
BufferAllocationType fallbackType = BufferAllocationType.Auto)
{ {
var usage = DefaultBufferUsageFlags; var usage = DefaultBufferUsageFlags;
@ -98,48 +104,106 @@ namespace Ryujinx.Graphics.Vulkan
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
MemoryPropertyFlags allocateFlags; MemoryAllocation allocation;
MemoryPropertyFlags allocateFlagsAlt;
if (deviceLocal) do
{ {
allocateFlags = DeviceLocalBufferMemoryFlags; var allocateFlags = type switch
allocateFlagsAlt = DeviceLocalBufferMemoryFlags;
}
else
{ {
allocateFlags = DefaultBufferMemoryFlags; BufferAllocationType.HostMappedNoCache => DefaultBufferMemoryNoCacheFlags,
allocateFlagsAlt = DefaultBufferMemoryAltFlags; BufferAllocationType.HostMapped => DefaultBufferMemoryFlags,
} BufferAllocationType.DeviceLocal => DeviceLocalBufferMemoryFlags,
BufferAllocationType.DeviceLocalMapped => DeviceLocalMappedBufferMemoryFlags,
_ => DefaultBufferMemoryFlags
};
var allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, allocateFlagsAlt); // If an allocation with this memory type fails, fall back to the previous one.
try
{
allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, true);
}
catch (VulkanException)
{
allocation = default;
}
}
while (allocation.Memory.Handle == 0 && (--type != fallbackType));
if (allocation.Memory.Handle == 0UL) if (allocation.Memory.Handle == 0UL)
{ {
gd.Api.DestroyBuffer(_device, buffer, null); gd.Api.DestroyBuffer(_device, buffer, null);
return null; return default;
} }
gd.Api.BindBufferMemory(_device, buffer, allocation.Memory, allocation.Offset); gd.Api.BindBufferMemory(_device, buffer, allocation.Memory, allocation.Offset);
return new BufferHolder(gd, _device, buffer, allocation, size); return (buffer, allocation, type);
} }
public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size) public unsafe BufferHolder Create(
VulkanRenderer gd,
int size,
bool forConditionalRendering = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default)
{ {
if (TryGetBuffer(handle, out var holder)) BufferAllocationType type = baseType;
BufferHolder storageHintHolder = null;
if (baseType == BufferAllocationType.Auto)
{ {
return holder.CreateView(format, offset, size); if (gd.IsSharedMemory)
{
baseType = BufferAllocationType.HostMapped;
type = baseType;
}
else
{
type = size >= BufferHolder.DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocal : BufferAllocationType.HostMapped;
}
if (storageHint != BufferHandle.Null)
{
if (TryGetBuffer(storageHint, out storageHintHolder))
{
type = storageHintHolder.DesiredType;
}
}
}
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) =
CreateBacking(gd, size, type, forConditionalRendering);
if (buffer.Handle != 0)
{
var holder = new BufferHolder(gd, _device, buffer, allocation, size, baseType, resultType);
if (storageHintHolder != null)
{
holder.InheritMetrics(storageHintHolder);
}
return holder;
} }
return null; return null;
} }
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite) public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size, Action invalidateView)
{ {
if (TryGetBuffer(handle, out var holder)) if (TryGetBuffer(handle, out var holder))
{ {
return holder.GetBuffer(commandBuffer, isWrite); return holder.CreateView(format, offset, size, invalidateView);
}
return null;
}
public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite, bool isSSBO = false)
{
if (TryGetBuffer(handle, out var holder))
{
return holder.GetBuffer(commandBuffer, isWrite, isSSBO);
} }
return null; return null;
@ -332,14 +396,14 @@ namespace Ryujinx.Graphics.Vulkan
return null; return null;
} }
public ReadOnlySpan<byte> GetData(BufferHandle handle, int offset, int size) public PinnedSpan<byte> GetData(BufferHandle handle, int offset, int size)
{ {
if (TryGetBuffer(handle, out var holder)) if (TryGetBuffer(handle, out var holder))
{ {
return holder.GetData(offset, size); return holder.GetData(offset, size);
} }
return ReadOnlySpan<byte>.Empty; return new PinnedSpan<byte>();
} }
public void SetData<T>(BufferHandle handle, int offset, ReadOnlySpan<T> data) where T : unmanaged public void SetData<T>(BufferHandle handle, int offset, ReadOnlySpan<T> data) where T : unmanaged

View file

@ -2,14 +2,14 @@
namespace Ryujinx.Graphics.Vulkan namespace Ryujinx.Graphics.Vulkan
{ {
readonly struct BufferState : IDisposable struct BufferState : IDisposable
{ {
public static BufferState Null => new BufferState(null, 0, 0); public static BufferState Null => new BufferState(null, 0, 0);
private readonly int _offset; private readonly int _offset;
private readonly int _size; private readonly int _size;
private readonly Auto<DisposableBuffer> _buffer; private Auto<DisposableBuffer> _buffer;
public BufferState(Auto<DisposableBuffer> buffer, int offset, int size) public BufferState(Auto<DisposableBuffer> buffer, int offset, int size)
{ {
@ -29,6 +29,17 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
if (_buffer == from)
{
_buffer.DecrementReferenceCount();
to.IncrementReferenceCount();
_buffer = to;
}
}
public void Dispose() public void Dispose()
{ {
_buffer?.DecrementReferenceCount(); _buffer?.DecrementReferenceCount();

View file

@ -94,7 +94,7 @@ namespace Ryujinx.Graphics.Vulkan
else else
{ {
// If null descriptors are not supported, we need to pass the handle of a dummy buffer on unused bindings. // If null descriptors are not supported, we need to pass the handle of a dummy buffer on unused bindings.
_dummyBuffer = gd.BufferManager.Create(gd, 0x10000, forConditionalRendering: false, deviceLocal: true); _dummyBuffer = gd.BufferManager.Create(gd, 0x10000, forConditionalRendering: false, baseType: BufferAllocationType.DeviceLocal);
} }
_dummyTexture = gd.CreateTextureView(new TextureCreateInfo( _dummyTexture = gd.CreateTextureView(new TextureCreateInfo(
@ -178,7 +178,7 @@ namespace Ryujinx.Graphics.Vulkan
var buffer = assignment.Range; var buffer = assignment.Range;
int index = assignment.Binding; int index = assignment.Binding;
Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true);
ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index]; ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index];
DescriptorBufferInfo info = new DescriptorBufferInfo() DescriptorBufferInfo info = new DescriptorBufferInfo()
@ -640,6 +640,23 @@ namespace Ryujinx.Graphics.Vulkan
Array.Clear(_storageSet); Array.Clear(_storageSet);
} }
private void SwapBuffer(Auto<DisposableBuffer>[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
for (int i = 0; i < list.Length; i++)
{
if (list[i] == from)
{
list[i] = to;
}
}
}
public void SwapBuffer(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
SwapBuffer(_uniformBufferRefs, from, to);
SwapBuffer(_storageBufferRefs, from, to);
}
protected virtual void Dispose(bool disposing) protected virtual void Dispose(bool disposing)
{ {
if (disposing) if (disposing)

View file

@ -156,11 +156,11 @@ namespace Ryujinx.Graphics.Vulkan.Effects
}; };
int rangeSize = dimensionsBuffer.Length * sizeof(float); int rangeSize = dimensionsBuffer.Length * sizeof(float);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false); var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize);
_renderer.BufferManager.SetData(bufferHandle, 0, dimensionsBuffer); _renderer.BufferManager.SetData(bufferHandle, 0, dimensionsBuffer);
ReadOnlySpan<float> sharpeningBuffer = stackalloc float[] { 1.5f - (Level * 0.01f * 1.5f)}; ReadOnlySpan<float> sharpeningBuffer = stackalloc float[] { 1.5f - (Level * 0.01f * 1.5f)};
var sharpeningBufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, sizeof(float), false); var sharpeningBufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, sizeof(float));
_renderer.BufferManager.SetData(sharpeningBufferHandle, 0, sharpeningBuffer); _renderer.BufferManager.SetData(sharpeningBufferHandle, 0, sharpeningBuffer);
int threadGroupWorkRegionDim = 16; int threadGroupWorkRegionDim = 16;

View file

@ -87,7 +87,7 @@ namespace Ryujinx.Graphics.Vulkan.Effects
ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height }; ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height };
int rangeSize = resolutionBuffer.Length * sizeof(float); int rangeSize = resolutionBuffer.Length * sizeof(float);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false); var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize);
_renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer); _renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer);

View file

@ -266,7 +266,7 @@ namespace Ryujinx.Graphics.Vulkan.Effects
ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height }; ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height };
int rangeSize = resolutionBuffer.Length * sizeof(float); int rangeSize = resolutionBuffer.Length * sizeof(float);
var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false); var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize);
_renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer); _renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer);
var bufferRanges = new BufferRange(bufferHandle, 0, rangeSize); var bufferRanges = new BufferRange(bufferHandle, 0, rangeSize);

View file

@ -394,7 +394,7 @@ namespace Ryujinx.Graphics.Vulkan
(region[2], region[3]) = (region[3], region[2]); (region[2], region[3]) = (region[3], region[2]);
} }
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, region); gd.BufferManager.SetData<float>(bufferHandle, 0, region);
@ -495,7 +495,7 @@ namespace Ryujinx.Graphics.Vulkan
(region[2], region[3]) = (region[3], region[2]); (region[2], region[3]) = (region[3], region[2]);
} }
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, region); gd.BufferManager.SetData<float>(bufferHandle, 0, region);
@ -649,7 +649,7 @@ namespace Ryujinx.Graphics.Vulkan
_pipeline.SetCommandBuffer(cbs); _pipeline.SetCommandBuffer(cbs);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, clearColor); gd.BufferManager.SetData<float>(bufferHandle, 0, clearColor);
@ -726,7 +726,7 @@ namespace Ryujinx.Graphics.Vulkan
(region[2], region[3]) = (region[3], region[2]); (region[2], region[3]) = (region[3], region[2]);
} }
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize);
gd.BufferManager.SetData<float>(bufferHandle, 0, region); gd.BufferManager.SetData<float>(bufferHandle, 0, region);
@ -802,7 +802,7 @@ namespace Ryujinx.Graphics.Vulkan
shaderParams[2] = size; shaderParams[2] = size;
shaderParams[3] = srcOffset; shaderParams[3] = srcOffset;
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -958,7 +958,7 @@ namespace Ryujinx.Graphics.Vulkan
shaderParams[0] = BitOperations.Log2((uint)ratio); shaderParams[0] = BitOperations.Log2((uint)ratio);
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -1050,7 +1050,7 @@ namespace Ryujinx.Graphics.Vulkan
(shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples); (shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples);
(shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples)); (shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples));
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -1133,7 +1133,7 @@ namespace Ryujinx.Graphics.Vulkan
(shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples); (shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples);
(shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples)); (shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples));
var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize);
gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams);
@ -1407,7 +1407,7 @@ namespace Ryujinx.Graphics.Vulkan
pattern.OffsetIndex.CopyTo(shaderParams.Slice(0, pattern.OffsetIndex.Length)); pattern.OffsetIndex.CopyTo(shaderParams.Slice(0, pattern.OffsetIndex.Length));
var patternBufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false, out var patternBuffer); var patternBufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, out var patternBuffer);
var patternBufferAuto = patternBuffer.GetBuffer(); var patternBufferAuto = patternBuffer.GetBuffer();
gd.BufferManager.SetData<int>(patternBufferHandle, 0, shaderParams); gd.BufferManager.SetData<int>(patternBufferHandle, 0, shaderParams);

View file

@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Vulkan
} }
// Expand the repeating pattern to the number of requested primitives. // Expand the repeating pattern to the number of requested primitives.
BufferHandle newBuffer = _gd.CreateBuffer(expectedSize * sizeof(int)); BufferHandle newBuffer = _gd.BufferManager.CreateWithHandle(_gd, expectedSize * sizeof(int));
// Copy the old data to the new one. // Copy the old data to the new one.
if (_repeatingBuffer != BufferHandle.Null) if (_repeatingBuffer != BufferHandle.Null)

View file

@ -146,5 +146,16 @@ namespace Ryujinx.Graphics.Vulkan
{ {
return _buffer == buffer; return _buffer == buffer;
} }
public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
if (_buffer == from)
{
_buffer.DecrementReferenceCount();
to.IncrementReferenceCount();
_buffer = to;
}
}
} }
} }

View file

@ -28,32 +28,25 @@ namespace Ryujinx.Graphics.Vulkan
public MemoryAllocation AllocateDeviceMemory( public MemoryAllocation AllocateDeviceMemory(
MemoryRequirements requirements, MemoryRequirements requirements,
MemoryPropertyFlags flags = 0) MemoryPropertyFlags flags = 0,
bool isBuffer = false)
{ {
return AllocateDeviceMemory(requirements, flags, flags); int memoryTypeIndex = FindSuitableMemoryTypeIndex(requirements.MemoryTypeBits, flags);
}
public MemoryAllocation AllocateDeviceMemory(
MemoryRequirements requirements,
MemoryPropertyFlags flags,
MemoryPropertyFlags alternativeFlags)
{
int memoryTypeIndex = FindSuitableMemoryTypeIndex(requirements.MemoryTypeBits, flags, alternativeFlags);
if (memoryTypeIndex < 0) if (memoryTypeIndex < 0)
{ {
return default; return default;
} }
bool map = flags.HasFlag(MemoryPropertyFlags.HostVisibleBit); bool map = flags.HasFlag(MemoryPropertyFlags.HostVisibleBit);
return Allocate(memoryTypeIndex, requirements.Size, requirements.Alignment, map); return Allocate(memoryTypeIndex, requirements.Size, requirements.Alignment, map, isBuffer);
} }
private MemoryAllocation Allocate(int memoryTypeIndex, ulong size, ulong alignment, bool map) private MemoryAllocation Allocate(int memoryTypeIndex, ulong size, ulong alignment, bool map, bool isBuffer)
{ {
for (int i = 0; i < _blockLists.Count; i++) for (int i = 0; i < _blockLists.Count; i++)
{ {
var bl = _blockLists[i]; var bl = _blockLists[i];
if (bl.MemoryTypeIndex == memoryTypeIndex) if (bl.MemoryTypeIndex == memoryTypeIndex && bl.ForBuffer == isBuffer)
{ {
lock (bl) lock (bl)
{ {
@ -62,18 +55,15 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
var newBl = new MemoryAllocatorBlockList(_api, _device, memoryTypeIndex, _blockAlignment); var newBl = new MemoryAllocatorBlockList(_api, _device, memoryTypeIndex, _blockAlignment, isBuffer);
_blockLists.Add(newBl); _blockLists.Add(newBl);
return newBl.Allocate(size, alignment, map); return newBl.Allocate(size, alignment, map);
} }
private int FindSuitableMemoryTypeIndex( private int FindSuitableMemoryTypeIndex(
uint memoryTypeBits, uint memoryTypeBits,
MemoryPropertyFlags flags, MemoryPropertyFlags flags)
MemoryPropertyFlags alternativeFlags)
{ {
int bestCandidateIndex = -1;
for (int i = 0; i < _physicalDeviceMemoryProperties.MemoryTypeCount; i++) for (int i = 0; i < _physicalDeviceMemoryProperties.MemoryTypeCount; i++)
{ {
var type = _physicalDeviceMemoryProperties.MemoryTypes[i]; var type = _physicalDeviceMemoryProperties.MemoryTypes[i];
@ -84,14 +74,27 @@ namespace Ryujinx.Graphics.Vulkan
{ {
return i; return i;
} }
else if (type.PropertyFlags.HasFlag(alternativeFlags))
{
bestCandidateIndex = i;
}
} }
} }
return bestCandidateIndex; return -1;
}
public static bool IsDeviceMemoryShared(Vk api, PhysicalDevice physicalDevice)
{
// The device is regarded as having shared memory if all heaps have the device local bit.
api.GetPhysicalDeviceMemoryProperties(physicalDevice, out var properties);
for (int i = 0; i < properties.MemoryHeapCount; i++)
{
if (!properties.MemoryHeaps[i].Flags.HasFlag(MemoryHeapFlags.DeviceLocalBit))
{
return false;
}
}
return true;
} }
public void Dispose() public void Dispose()

View file

@ -162,15 +162,17 @@ namespace Ryujinx.Graphics.Vulkan
private readonly Device _device; private readonly Device _device;
public int MemoryTypeIndex { get; } public int MemoryTypeIndex { get; }
public bool ForBuffer { get; }
private readonly int _blockAlignment; private readonly int _blockAlignment;
public MemoryAllocatorBlockList(Vk api, Device device, int memoryTypeIndex, int blockAlignment) public MemoryAllocatorBlockList(Vk api, Device device, int memoryTypeIndex, int blockAlignment, bool forBuffer)
{ {
_blocks = new List<Block>(); _blocks = new List<Block>();
_api = api; _api = api;
_device = device; _device = device;
MemoryTypeIndex = memoryTypeIndex; MemoryTypeIndex = memoryTypeIndex;
ForBuffer = forBuffer;
_blockAlignment = blockAlignment; _blockAlignment = blockAlignment;
} }

View file

@ -1297,6 +1297,25 @@ namespace Ryujinx.Graphics.Vulkan
SignalStateChange(); SignalStateChange();
} }
public void SwapBuffer(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
_indexBuffer.Swap(from, to);
for (int i = 0; i < _vertexBuffers.Length; i++)
{
_vertexBuffers[i].Swap(from, to);
}
for (int i = 0; i < _transformFeedbackBuffers.Length; i++)
{
_transformFeedbackBuffers[i].Swap(from, to);
}
_descriptorSetUpdater.SwapBuffer(from, to);
SignalCommandBufferChange();
}
public unsafe void TextureBarrier() public unsafe void TextureBarrier()
{ {
MemoryBarrier memoryBarrier = new MemoryBarrier() MemoryBarrier memoryBarrier = new MemoryBarrier()

View file

@ -17,10 +17,13 @@ namespace Ryujinx.Graphics.Vulkan
private ulong _byteWeight; private ulong _byteWeight;
private List<BufferHolder> _backingSwaps;
public PipelineFull(VulkanRenderer gd, Device device) : base(gd, device) public PipelineFull(VulkanRenderer gd, Device device) : base(gd, device)
{ {
_activeQueries = new List<(QueryPool, bool)>(); _activeQueries = new List<(QueryPool, bool)>();
_pendingQueryCopies = new(); _pendingQueryCopies = new();
_backingSwaps = new();
CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer; CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer;
} }
@ -185,6 +188,20 @@ namespace Ryujinx.Graphics.Vulkan
} }
} }
private void TryBackingSwaps()
{
CommandBufferScoped? cbs = null;
_backingSwaps.RemoveAll((holder) => holder.TryBackingSwap(ref cbs));
cbs?.Dispose();
}
public void AddBackingSwap(BufferHolder holder)
{
_backingSwaps.Add(holder);
}
public void Restore() public void Restore()
{ {
if (Pipeline != null) if (Pipeline != null)
@ -230,6 +247,8 @@ namespace Ryujinx.Graphics.Vulkan
Gd.ResetCounterPool(); Gd.ResetCounterPool();
TryBackingSwaps();
Restore(); Restore();
} }

View file

@ -57,12 +57,12 @@ namespace Ryujinx.Graphics.Vulkan
throw new NotSupportedException(); throw new NotSupportedException();
} }
public ReadOnlySpan<byte> GetData() public PinnedSpan<byte> GetData()
{ {
return _gd.GetBufferData(_bufferHandle, _offset, _size); return _gd.GetBufferData(_bufferHandle, _offset, _size);
} }
public ReadOnlySpan<byte> GetData(int layer, int level) public PinnedSpan<byte> GetData(int layer, int level)
{ {
return GetData(); return GetData();
} }
@ -128,7 +128,7 @@ namespace Ryujinx.Graphics.Vulkan
{ {
if (_bufferView == null) if (_bufferView == null)
{ {
_bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size); _bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl);
} }
return _bufferView?.Get(cbs, _offset, _size).Value ?? default; return _bufferView?.Get(cbs, _offset, _size).Value ?? default;
@ -147,7 +147,7 @@ namespace Ryujinx.Graphics.Vulkan
return bufferView.Get(cbs, _offset, _size).Value; return bufferView.Get(cbs, _offset, _size).Value;
} }
bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size); bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl);
if (bufferView != null) if (bufferView != null)
{ {

View file

@ -531,7 +531,7 @@ namespace Ryujinx.Graphics.Vulkan
return bitmap; return bitmap;
} }
public ReadOnlySpan<byte> GetData() public PinnedSpan<byte> GetData()
{ {
BackgroundResource resources = _gd.BackgroundResources.Get(); BackgroundResource resources = _gd.BackgroundResources.Get();
@ -539,15 +539,15 @@ namespace Ryujinx.Graphics.Vulkan
{ {
_gd.FlushAllCommands(); _gd.FlushAllCommands();
return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer()); return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer()));
} }
else else
{ {
return GetData(resources.GetPool(), resources.GetFlushBuffer()); return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer()));
} }
} }
public ReadOnlySpan<byte> GetData(int layer, int level) public PinnedSpan<byte> GetData(int layer, int level)
{ {
BackgroundResource resources = _gd.BackgroundResources.Get(); BackgroundResource resources = _gd.BackgroundResources.Get();
@ -555,11 +555,11 @@ namespace Ryujinx.Graphics.Vulkan
{ {
_gd.FlushAllCommands(); _gd.FlushAllCommands();
return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level); return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level));
} }
else else
{ {
return GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level); return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level));
} }
} }

View file

@ -129,6 +129,17 @@ namespace Ryujinx.Graphics.Vulkan
return _buffer == buffer; return _buffer == buffer;
} }
public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to)
{
if (_buffer == from)
{
_buffer.DecrementReferenceCount();
to.IncrementReferenceCount();
_buffer = to;
}
}
public void Dispose() public void Dispose()
{ {
// Only dispose if this buffer is not refetched on each bind. // Only dispose if this buffer is not refetched on each bind.

View file

@ -80,6 +80,7 @@ namespace Ryujinx.Graphics.Vulkan
internal bool IsAmdGcn { get; private set; } internal bool IsAmdGcn { get; private set; }
internal bool IsMoltenVk { get; private set; } internal bool IsMoltenVk { get; private set; }
internal bool IsTBDR { get; private set; } internal bool IsTBDR { get; private set; }
internal bool IsSharedMemory { get; private set; }
public string GpuVendor { get; private set; } public string GpuVendor { get; private set; }
public string GpuRenderer { get; private set; } public string GpuRenderer { get; private set; }
public string GpuVersion { get; private set; } public string GpuVersion { get; private set; }
@ -313,6 +314,8 @@ namespace Ryujinx.Graphics.Vulkan
portabilityFlags, portabilityFlags,
vertexBufferAlignment); vertexBufferAlignment);
IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(Api, _physicalDevice);
MemoryAllocator = new MemoryAllocator(Api, _physicalDevice, _device, properties.Limits.MaxMemoryAllocationCount); MemoryAllocator = new MemoryAllocator(Api, _physicalDevice, _device, properties.Limits.MaxMemoryAllocationCount);
CommandBufferPool = VulkanInitialization.CreateCommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex); CommandBufferPool = VulkanInitialization.CreateCommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex);
@ -373,9 +376,9 @@ namespace Ryujinx.Graphics.Vulkan
_initialized = true; _initialized = true;
} }
public BufferHandle CreateBuffer(int size) public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
{ {
return BufferManager.CreateWithHandle(this, size, false); return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint);
} }
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info) public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
@ -439,7 +442,7 @@ namespace Ryujinx.Graphics.Vulkan
_syncManager.RegisterFlush(); _syncManager.RegisterFlush();
} }
public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{ {
return BufferManager.GetData(buffer, offset, size); return BufferManager.GetData(buffer, offset, size);
} }