Implement virtual buffer dependencies (#6190)

* Implement virtual buffer copies

* Introduce TranslateAndCreateMultiBuffersPhysicalOnly, use it for copy and clear

* Rename VirtualBufferCache to VirtualRangeCache

* Fix potential issue where virtual range could exist in the cache, without a physical buffer

* Fix bug that could cause copy with negative size on CopyToDependantVirtualBuffer

* Remove virtual copy back for SyncAction

* GetData XML docs

* Make field readonly

* Fix virtual buffer modification tracking

* Remove CopyFromDependantVirtualBuffers from ExternalFlush

* Move things around a little to avoid perf impact

- Inline null check for CopyFromDependantVirtualBuffers
- Remove extra method call for SynchronizeMemoryWithVirtualCopyBack, prefer calling CopyFromDependantVirtualBuffers separately

* Fix up XML doc

---------

Co-authored-by: riperiperi <rhy3756547@hotmail.com>
This commit is contained in:
gdkchan 2024-02-22 11:03:07 -03:00 committed by GitHub
parent ba91f5d401
commit 167f50bbcd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 573 additions and 49 deletions

View file

@ -5,6 +5,8 @@ using Ryujinx.Memory.Tracking;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -65,6 +67,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
private readonly Action<ulong, ulong> _loadDelegate; private readonly Action<ulong, ulong> _loadDelegate;
private readonly Action<ulong, ulong> _modifiedDelegate; private readonly Action<ulong, ulong> _modifiedDelegate;
private HashSet<MultiRangeBuffer> _virtualDependencies;
private readonly ReaderWriterLockSlim _virtualDependenciesLock;
private int _sequenceNumber; private int _sequenceNumber;
private readonly bool _useGranular; private readonly bool _useGranular;
@ -152,6 +157,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
_externalFlushDelegate = new RegionSignal(ExternalFlush); _externalFlushDelegate = new RegionSignal(ExternalFlush);
_loadDelegate = new Action<ulong, ulong>(LoadRegion); _loadDelegate = new Action<ulong, ulong>(LoadRegion);
_modifiedDelegate = new Action<ulong, ulong>(RegionModified); _modifiedDelegate = new Action<ulong, ulong>(RegionModified);
_virtualDependenciesLock = new ReaderWriterLockSlim();
} }
/// <summary> /// <summary>
@ -220,6 +227,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </remarks> /// </remarks>
/// <param name="address">Start address of the range to synchronize</param> /// <param name="address">Start address of the range to synchronize</param>
/// <param name="size">Size in bytes of the range to synchronize</param> /// <param name="size">Size in bytes of the range to synchronize</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void SynchronizeMemory(ulong address, ulong size) public void SynchronizeMemory(ulong address, ulong size)
{ {
if (_useGranular) if (_useGranular)
@ -239,6 +247,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
else else
{ {
_context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size)); _context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size));
CopyToDependantVirtualBuffers();
} }
_sequenceNumber = _context.SequenceNumber; _sequenceNumber = _context.SequenceNumber;
@ -460,6 +469,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
int offset = (int)(mAddress - Address); int offset = (int)(mAddress - Address);
_context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize)); _context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize));
CopyToDependantVirtualBuffers(mAddress, mSize);
} }
/// <summary> /// <summary>
@ -520,6 +531,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="dstOffset">The offset of the destination buffer to copy into</param> /// <param name="dstOffset">The offset of the destination buffer to copy into</param>
public void CopyTo(Buffer destination, int dstOffset) public void CopyTo(Buffer destination, int dstOffset)
{ {
CopyFromDependantVirtualBuffers();
_context.Renderer.Pipeline.CopyBuffer(Handle, destination.Handle, 0, dstOffset, (int)Size); _context.Renderer.Pipeline.CopyBuffer(Handle, destination.Handle, 0, dstOffset, (int)Size);
} }
@ -536,7 +548,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size); using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
// TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
_physicalMemory.WriteUntracked(address, data.Get()); _physicalMemory.WriteUntracked(address, CopyFromDependantVirtualBuffers(data.Get(), address, size));
} }
/// <summary> /// <summary>
@ -617,6 +629,207 @@ namespace Ryujinx.Graphics.Gpu.Memory
UnmappedSequence++; UnmappedSequence++;
} }
/// <summary>
/// Adds a virtual buffer dependency, indicating that a virtual buffer depends on data from this buffer.
/// </summary>
/// <param name="virtualBuffer">Dependant virtual buffer</param>
public void AddVirtualDependency(MultiRangeBuffer virtualBuffer)
{
_virtualDependenciesLock.EnterWriteLock();
try
{
(_virtualDependencies ??= new()).Add(virtualBuffer);
}
finally
{
_virtualDependenciesLock.ExitWriteLock();
}
}
/// <summary>
/// Removes a virtual buffer dependency, indicating that a virtual buffer no longer depends on data from this buffer.
/// </summary>
/// <param name="virtualBuffer">Dependant virtual buffer</param>
public void RemoveVirtualDependency(MultiRangeBuffer virtualBuffer)
{
_virtualDependenciesLock.EnterWriteLock();
try
{
if (_virtualDependencies != null)
{
_virtualDependencies.Remove(virtualBuffer);
if (_virtualDependencies.Count == 0)
{
_virtualDependencies = null;
}
}
}
finally
{
_virtualDependenciesLock.ExitWriteLock();
}
}
/// <summary>
/// Copies the buffer data to all virtual buffers that depends on it.
/// </summary>
public void CopyToDependantVirtualBuffers()
{
CopyToDependantVirtualBuffers(Address, Size);
}
/// <summary>
/// Copies the buffer data inside the specifide range to all virtual buffers that depends on it.
/// </summary>
/// <param name="address">Address of the range</param>
/// <param name="size">Size of the range in bytes</param>
public void CopyToDependantVirtualBuffers(ulong address, ulong size)
{
if (_virtualDependencies != null)
{
foreach (var virtualBuffer in _virtualDependencies)
{
CopyToDependantVirtualBuffer(virtualBuffer, address, size);
}
}
}
/// <summary>
/// Copies all modified ranges from all virtual buffers back into this buffer.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CopyFromDependantVirtualBuffers()
{
if (_virtualDependencies != null)
{
CopyFromDependantVirtualBuffersImpl();
}
}
/// <summary>
/// Copies all modified ranges from all virtual buffers back into this buffer.
/// </summary>
[MethodImpl(MethodImplOptions.NoInlining)]
private void CopyFromDependantVirtualBuffersImpl()
{
foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber))
{
virtualBuffer.ConsumeModifiedRegion(this, (mAddress, mSize) =>
{
// Get offset inside both this and the virtual buffer.
// Note that sometimes there is no right answer for the virtual offset,
// as the same physical range might be mapped multiple times inside a virtual buffer.
// We just assume it does not happen in practice as it can only be implemented correctly
// when the host has support for proper sparse mapping.
ulong mEndAddress = mAddress + mSize;
mAddress = Math.Max(mAddress, Address);
mSize = Math.Min(mEndAddress, EndAddress) - mAddress;
int physicalOffset = (int)(mAddress - Address);
int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize));
_context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)mSize);
});
}
}
/// <summary>
/// Copies all overlapping modified ranges from all virtual buffers back into this buffer, and returns an updated span with the data.
/// </summary>
/// <param name="dataSpan">Span where the unmodified data will be taken from for the output</param>
/// <param name="address">Address of the region to copy</param>
/// <param name="size">Size of the region to copy in bytes</param>
/// <returns>A span with <paramref name="dataSpan"/>, and the data for all modified ranges if any</returns>
private ReadOnlySpan<byte> CopyFromDependantVirtualBuffers(ReadOnlySpan<byte> dataSpan, ulong address, ulong size)
{
_virtualDependenciesLock.EnterReadLock();
try
{
if (_virtualDependencies != null)
{
byte[] storage = dataSpan.ToArray();
foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber))
{
virtualBuffer.ConsumeModifiedRegion(address, size, (mAddress, mSize) =>
{
// Get offset inside both this and the virtual buffer.
// Note that sometimes there is no right answer for the virtual offset,
// as the same physical range might be mapped multiple times inside a virtual buffer.
// We just assume it does not happen in practice as it can only be implemented correctly
// when the host has support for proper sparse mapping.
ulong mEndAddress = mAddress + mSize;
mAddress = Math.Max(mAddress, address);
mSize = Math.Min(mEndAddress, address + size) - mAddress;
int physicalOffset = (int)(mAddress - Address);
int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize));
_context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)size);
virtualBuffer.GetData(storage.AsSpan().Slice((int)(mAddress - address), (int)mSize), virtualOffset, (int)mSize);
});
}
dataSpan = storage;
}
}
finally
{
_virtualDependenciesLock.ExitReadLock();
}
return dataSpan;
}
/// <summary>
/// Copies the buffer data to the specified virtual buffer.
/// </summary>
/// <param name="virtualBuffer">Virtual buffer to copy the data into</param>
public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer)
{
CopyToDependantVirtualBuffer(virtualBuffer, Address, Size);
}
/// <summary>
/// Copies the buffer data inside the given range to the specified virtual buffer.
/// </summary>
/// <param name="virtualBuffer">Virtual buffer to copy the data into</param>
/// <param name="address">Address of the range</param>
/// <param name="size">Size of the range in bytes</param>
public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer, ulong address, ulong size)
{
// Broadcast data to all ranges of the virtual buffer that are contained inside this buffer.
ulong lastOffset = 0;
while (virtualBuffer.TryGetPhysicalOffset(this, lastOffset, out ulong srcOffset, out ulong dstOffset, out ulong copySize))
{
ulong innerOffset = address - Address;
ulong innerEndOffset = (address + size) - Address;
lastOffset = dstOffset + copySize;
// Clamp range to the specified range.
ulong copySrcOffset = Math.Max(srcOffset, innerOffset);
ulong copySrcEndOffset = Math.Min(innerEndOffset, srcOffset + copySize);
if (copySrcEndOffset > copySrcOffset)
{
copySize = copySrcEndOffset - copySrcOffset;
dstOffset += copySrcOffset - srcOffset;
srcOffset = copySrcOffset;
_context.Renderer.Pipeline.CopyBuffer(Handle, virtualBuffer.Handle, (int)srcOffset, (int)dstOffset, (int)copySize);
}
}
}
/// <summary> /// <summary>
/// Increments the buffer reference count. /// Increments the buffer reference count.
/// </summary> /// </summary>

View file

@ -3,6 +3,7 @@ using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Runtime.CompilerServices;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache; private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache;
private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache; private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache;
private bool _pruneCaches; private bool _pruneCaches;
private int _virtualModifiedSequenceNumber;
public event Action NotifyBuffersModified; public event Action NotifyBuffersModified;
@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary> /// <summary>
/// Performs address translation of the GPU virtual address, and creates /// Performs address translation of the GPU virtual address, and creates
/// new buffers, if needed, for the specified range. /// new physical and virtual buffers, if needed, for the specified range.
/// </summary> /// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param> /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param> /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
return new MultiRange(MemoryManager.PteUnmapped, size); return new MultiRange(MemoryManager.PteUnmapped, size);
} }
bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
// Fast path not taken for non-contiguous ranges, // Fast path not taken for non-contiguous ranges,
// since multi-range buffers are not coalesced, so a buffer that covers // since multi-range buffers are not coalesced, so a buffer that covers
// the entire cached range might not actually exist. // the entire cached range might not actually exist.
if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) && if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
range.Count == 1) range.Count == 1)
{ {
return range; return range;
@ -154,6 +154,50 @@ namespace Ryujinx.Graphics.Gpu.Memory
return range; return range;
} }
/// <summary>
/// Performs address translation of the GPU virtual address, and creates
/// new physical buffers, if needed, for the specified range.
/// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <returns>Physical ranges of the buffer, after address translation</returns>
public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size)
{
if (gpuVa == 0)
{
return new MultiRange(MemoryManager.PteUnmapped, size);
}
// Fast path not taken for non-contiguous ranges,
// since multi-range buffers are not coalesced, so a buffer that covers
// the entire cached range might not actually exist.
if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
range.Count == 1)
{
return range;
}
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
if (range.Count > 1)
{
CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
}
else
{
CreateBuffer(subRange.Address, subRange.Size);
}
}
}
return range;
}
/// <summary> /// <summary>
/// Creates a new buffer for the specified range, if it does not yet exist. /// Creates a new buffer for the specified range, if it does not yet exist.
/// This can be used to ensure the existance of a buffer. /// This can be used to ensure the existance of a buffer.
@ -263,41 +307,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
} }
BufferRange[] storages = new BufferRange[range.Count]; MultiRangeBuffer multiRangeBuffer;
MemoryRange[] alignedSubRanges = new MemoryRange[range.Count]; MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
ulong alignmentMask = SparseBufferAlignmentSize - 1; ulong alignmentMask = SparseBufferAlignmentSize - 1;
for (int i = 0; i < range.Count; i++) if (_context.Capabilities.SupportsSparseBuffer)
{ {
MemoryRange subRange = range.GetSubRange(i); BufferRange[] storages = new BufferRange[range.Count];
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
ulong endAddress = subRange.Address + subRange.Size;
ulong alignedAddress = subRange.Address & ~alignmentMask;
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
ulong alignedSize = alignedEndAddress - alignedAddress;
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
storages[i] = bufferRange;
}
else
{
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
}
}
multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
}
else
{
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
ulong endAddress = subRange.Address + subRange.Size;
ulong alignedAddress = subRange.Address & ~alignmentMask;
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
ulong alignedSize = alignedEndAddress - alignedAddress;
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
}
else
{
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
}
}
multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges));
UpdateVirtualBufferDependencies(multiRangeBuffer);
}
_multiRangeBuffers.Add(multiRangeBuffer);
}
/// <summary>
/// Adds two-way dependencies to all physical buffers contained within a given virtual buffer.
/// </summary>
/// <param name="virtualBuffer">Virtual buffer to have dependencies added</param>
private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer)
{
virtualBuffer.ClearPhysicalDependencies();
ulong dstOffset = 0;
HashSet<Buffer> physicalBuffers = new();
for (int i = 0; i < virtualBuffer.Range.Count; i++)
{
MemoryRange subRange = virtualBuffer.Range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped) if (subRange.Address != MemoryManager.PteUnmapped)
{ {
ulong endAddress = subRange.Address + subRange.Size; Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
ulong alignedAddress = subRange.Address & ~alignmentMask; virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size);
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask; physicalBuffers.Add(buffer);
ulong alignedSize = alignedEndAddress - alignedAddress;
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
storages[i] = bufferRange;
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
} }
else
{
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize); dstOffset += subRange.Size;
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
}
} }
MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages); foreach (var buffer in physicalBuffers)
{
_multiRangeBuffers.Add(multiRangeBuffer); buffer.CopyToDependantVirtualBuffer(virtualBuffer);
}
} }
/// <summary> /// <summary>
@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the copy</param> /// <param name="size">Size in bytes of the copy</param>
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size) public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
{ {
MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size); MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size);
MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size); MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size);
if (srcRange.Count == 1 && dstRange.Count == 1) if (srcRange.Count == 1 && dstRange.Count == 1)
{ {
@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
dstBuffer.ClearModified(dstAddress, size); dstBuffer.ClearModified(dstAddress, size);
memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer); memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer);
} }
dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size);
} }
/// <summary> /// <summary>
@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="value">Value to be written into the buffer</param> /// <param name="value">Value to be written into the buffer</param>
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value) public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
{ {
MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size); MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size);
for (int index = 0; index < range.Count; index++) for (int index = 0; index < range.Count; index++)
{ {
@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value); _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer); memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size);
} }
} }
@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
} }
if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer)
{
buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber);
}
return buffer; return buffer;
} }
@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
buffer = _buffers.FindFirstOverlap(address, size); buffer = _buffers.FindFirstOverlap(address, size);
buffer.CopyFromDependantVirtualBuffers();
buffer.SynchronizeMemory(address, size); buffer.SynchronizeMemory(address, size);
if (write) if (write)
@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
if (range.Count == 1) if (range.Count == 1)
{ {
MemoryRange subRange = range.GetSubRange(0); MemoryRange subRange = range.GetSubRange(0);
SynchronizeBufferRange(subRange.Address, subRange.Size); SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true);
} }
else else
{ {
for (int index = 0; index < range.Count; index++) for (int index = 0; index < range.Count; index++)
{ {
MemoryRange subRange = range.GetSubRange(index); MemoryRange subRange = range.GetSubRange(index);
SynchronizeBufferRange(subRange.Address, subRange.Size); SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false);
} }
} }
} }
@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
/// <param name="address">Start address of the memory range</param> /// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param> /// <param name="size">Size in bytes of the memory range</param>
private void SynchronizeBufferRange(ulong address, ulong size) /// <param name="copyBackVirtual">Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual)
{ {
if (size != 0) if (size != 0)
{ {
Buffer buffer = _buffers.FindFirstOverlap(address, size); Buffer buffer = _buffers.FindFirstOverlap(address, size);
if (copyBackVirtual)
{
buffer.CopyFromDependantVirtualBuffers();
}
buffer.SynchronizeMemory(address, size); buffer.SynchronizeMemory(address, size);
} }
} }

View file

@ -40,9 +40,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
internal PhysicalMemory Physical { get; } internal PhysicalMemory Physical { get; }
/// <summary> /// <summary>
/// Virtual buffer cache. /// Virtual range cache.
/// </summary> /// </summary>
internal VirtualBufferCache VirtualBufferCache { get; } internal VirtualRangeCache VirtualRangeCache { get; }
/// <summary> /// <summary>
/// Cache of GPU counters. /// Cache of GPU counters.
@ -56,12 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
internal MemoryManager(PhysicalMemory physicalMemory) internal MemoryManager(PhysicalMemory physicalMemory)
{ {
Physical = physicalMemory; Physical = physicalMemory;
VirtualBufferCache = new VirtualBufferCache(this); VirtualRangeCache = new VirtualRangeCache(this);
CounterCache = new CounterCache(); CounterCache = new CounterCache();
_pageTable = new ulong[PtLvl0Size][]; _pageTable = new ulong[PtLvl0Size][];
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler; MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler; MemoryUnmapped += VirtualRangeCache.MemoryUnmappedHandler;
MemoryUnmapped += CounterCache.MemoryUnmappedHandler; MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
} }

View file

@ -1,6 +1,7 @@
using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
@ -21,12 +22,73 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
public MultiRange Range { get; } public MultiRange Range { get; }
/// <summary>
/// Ever increasing counter value indicating when the buffer was modified relative to other buffers.
/// </summary>
public int ModificationSequenceNumber { get; private set; }
/// <summary>
/// Physical buffer dependency entry.
/// </summary>
private readonly struct PhysicalDependency
{
/// <summary>
/// Physical buffer.
/// </summary>
public readonly Buffer PhysicalBuffer;
/// <summary>
/// Offset of the range on the physical buffer.
/// </summary>
public readonly ulong PhysicalOffset;
/// <summary>
/// Offset of the range on the virtual buffer.
/// </summary>
public readonly ulong VirtualOffset;
/// <summary>
/// Size of the range.
/// </summary>
public readonly ulong Size;
/// <summary>
/// Creates a new physical dependency.
/// </summary>
/// <param name="physicalBuffer">Physical buffer</param>
/// <param name="physicalOffset">Offset of the range on the physical buffer</param>
/// <param name="virtualOffset">Offset of the range on the virtual buffer</param>
/// <param name="size">Size of the range</param>
public PhysicalDependency(Buffer physicalBuffer, ulong physicalOffset, ulong virtualOffset, ulong size)
{
PhysicalBuffer = physicalBuffer;
PhysicalOffset = physicalOffset;
VirtualOffset = virtualOffset;
Size = size;
}
}
private List<PhysicalDependency> _dependencies;
private BufferModifiedRangeList _modifiedRanges = null;
/// <summary> /// <summary>
/// Creates a new instance of the buffer. /// Creates a new instance of the buffer.
/// </summary> /// </summary>
/// <param name="context">GPU context that the buffer belongs to</param> /// <param name="context">GPU context that the buffer belongs to</param>
/// <param name="range">Range of memory where the data is mapped</param> /// <param name="range">Range of memory where the data is mapped</param>
/// <param name="storages">Backing memory for the buffers</param> public MultiRangeBuffer(GpuContext context, MultiRange range)
{
_context = context;
Range = range;
Handle = context.Renderer.CreateBuffer((int)range.GetSize());
}
/// <summary>
/// Creates a new instance of the buffer.
/// </summary>
/// <param name="context">GPU context that the buffer belongs to</param>
/// <param name="range">Range of memory where the data is mapped</param>
/// <param name="storages">Backing memory for the buffer</param>
public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages) public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
{ {
_context = context; _context = context;
@ -49,11 +111,134 @@ namespace Ryujinx.Graphics.Gpu.Memory
return new BufferRange(Handle, offset, (int)range.GetSize()); return new BufferRange(Handle, offset, (int)range.GetSize());
} }
/// <summary>
/// Removes all physical buffer dependencies.
/// </summary>
public void ClearPhysicalDependencies()
{
_dependencies?.Clear();
}
/// <summary>
/// Adds a physical buffer dependency.
/// </summary>
/// <param name="buffer">Physical buffer to be added</param>
/// <param name="rangeAddress">Address inside the physical buffer where the virtual buffer range is located</param>
/// <param name="dstOffset">Offset inside the virtual buffer where the physical range is located</param>
/// <param name="rangeSize">Size of the range in bytes</param>
public void AddPhysicalDependency(Buffer buffer, ulong rangeAddress, ulong dstOffset, ulong rangeSize)
{
(_dependencies ??= new()).Add(new(buffer, rangeAddress - buffer.Address, dstOffset, rangeSize));
buffer.AddVirtualDependency(this);
}
/// <summary>
/// Tries to get the physical range corresponding to the given physical buffer.
/// </summary>
/// <param name="buffer">Physical buffer</param>
/// <param name="minimumVirtOffset">Minimum virtual offset that a range match can have</param>
/// <param name="physicalOffset">Physical offset of the match</param>
/// <param name="virtualOffset">Virtual offset of the match, always greater than or equal <paramref name="minimumVirtOffset"/></param>
/// <param name="size">Size of the range match</param>
/// <returns>True if a match was found for the given parameters, false otherwise</returns>
public bool TryGetPhysicalOffset(Buffer buffer, ulong minimumVirtOffset, out ulong physicalOffset, out ulong virtualOffset, out ulong size)
{
physicalOffset = 0;
virtualOffset = 0;
size = 0;
if (_dependencies != null)
{
foreach (var dependency in _dependencies)
{
if (dependency.PhysicalBuffer == buffer && dependency.VirtualOffset >= minimumVirtOffset)
{
physicalOffset = dependency.PhysicalOffset;
virtualOffset = dependency.VirtualOffset;
size = dependency.Size;
return true;
}
}
}
return false;
}
/// <summary>
/// Adds a modified virtual memory range.
/// </summary>
/// <remarks>
/// This is only required when the host does not support sparse buffers, otherwise only physical buffers need to track modification.
/// </remarks>
/// <param name="range">Modified range</param>
/// <param name="modifiedSequenceNumber">ModificationSequenceNumber</param>
public void AddModifiedRegion(MultiRange range, int modifiedSequenceNumber)
{
_modifiedRanges ??= new(_context, null, null);
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
_modifiedRanges.SignalModified(subRange.Address, subRange.Size);
}
ModificationSequenceNumber = modifiedSequenceNumber;
}
/// <summary>
/// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="buffer"/>.
/// </summary>
/// <param name="buffer">Buffer to have its range checked</param>
/// <param name="rangeAction">Action to perform for modified ranges</param>
public void ConsumeModifiedRegion(Buffer buffer, Action<ulong, ulong> rangeAction)
{
ConsumeModifiedRegion(buffer.Address, buffer.Size, rangeAction);
}
/// <summary>
/// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="address"/> and <paramref name="size"/>.
/// </summary>
/// <param name="address">Address of the region to consume</param>
/// <param name="size">Size of the region to consume</param>
/// <param name="rangeAction">Action to perform for modified ranges</param>
public void ConsumeModifiedRegion(ulong address, ulong size, Action<ulong, ulong> rangeAction)
{
if (_modifiedRanges != null)
{
_modifiedRanges.GetRanges(address, size, rangeAction);
_modifiedRanges.Clear(address, size);
}
}
/// <summary>
/// Gets data from the specified region of the buffer, and places it on <paramref name="output"/>.
/// </summary>
/// <param name="output">Span to put the data into</param>
/// <param name="offset">Offset of the buffer to get the data from</param>
/// <param name="size">Size of the data in bytes</param>
public void GetData(Span<byte> output, int offset, int size)
{
using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, size);
data.Get().CopyTo(output);
}
/// <summary> /// <summary>
/// Disposes the host buffer. /// Disposes the host buffer.
/// </summary> /// </summary>
public void Dispose() public void Dispose()
{ {
if (_dependencies != null)
{
foreach (var dependency in _dependencies)
{
dependency.PhysicalBuffer.RemoveVirtualDependency(this);
}
_dependencies = null;
}
_context.Renderer.DeleteBuffer(Handle); _context.Renderer.DeleteBuffer(Handle);
} }
} }

View file

@ -6,9 +6,9 @@ using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory namespace Ryujinx.Graphics.Gpu.Memory
{ {
/// <summary> /// <summary>
/// Virtual buffer cache. /// Virtual range cache.
/// </summary> /// </summary>
class VirtualBufferCache class VirtualRangeCache
{ {
private readonly MemoryManager _memoryManager; private readonly MemoryManager _memoryManager;
@ -68,10 +68,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
private int _hasDeferredUnmaps; private int _hasDeferredUnmaps;
/// <summary> /// <summary>
/// Creates a new instance of the virtual buffer cache. /// Creates a new instance of the virtual range cache.
/// </summary> /// </summary>
/// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param> /// <param name="memoryManager">Memory manager that the virtual range cache belongs to</param>
public VirtualBufferCache(MemoryManager memoryManager) public VirtualRangeCache(MemoryManager memoryManager)
{ {
_memoryManager = memoryManager; _memoryManager = memoryManager;
_virtualRanges = new RangeList<VirtualRange>(); _virtualRanges = new RangeList<VirtualRange>();
@ -102,10 +102,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
/// <param name="gpuVa">GPU virtual address to get the physical range from</param> /// <param name="gpuVa">GPU virtual address to get the physical range from</param>
/// <param name="size">Size in bytes of the region</param> /// <param name="size">Size in bytes of the region</param>
/// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
/// <param name="range">Physical range for the specified GPU virtual region</param> /// <param name="range">Physical range for the specified GPU virtual region</param>
/// <returns>True if the range already existed, false if a new one was created and added</returns> /// <returns>True if the range already existed, false if a new one was created and added</returns>
public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range) public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range)
{ {
VirtualRange[] overlaps = _virtualRangeOverlaps; VirtualRange[] overlaps = _virtualRangeOverlaps;
int overlapsCount; int overlapsCount;
@ -158,7 +157,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
} }
else else
{ {
found = true; found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range);
range = overlap0.Range.Slice(gpuVa - overlap0.Address, size); range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
} }
} }
@ -175,11 +174,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
ShrinkOverlapsBufferIfNeeded(); ShrinkOverlapsBufferIfNeeded();
// If the the range is not properly aligned for sparse mapping, // If the the range is not properly aligned for sparse mapping,
// or if the host does not support sparse mapping, let's just // let's just force it to a single range.
// force it to a single range.
// This might cause issues in some applications that uses sparse // This might cause issues in some applications that uses sparse
// mappings. // mappings.
if (!IsSparseAligned(range) || !supportsSparse) if (!IsSparseAligned(range))
{ {
range = new MultiRange(range.GetSubRange(0).Address, size); range = new MultiRange(range.GetSubRange(0).Address, size);
} }