From 12a7a2ead812d46deb9d978b6758731157be1cbc Mon Sep 17 00:00:00 2001 From: riperiperi Date: Thu, 24 Jun 2021 00:31:26 +0100 Subject: [PATCH] Inherit buffer tracking handles rather than recreating on resize (#2330) This greatly speeds up games that constantly resize buffers, and removes stuttering on games that resize large buffers occasionally: - Large improvement on Super Mario 3D All-Stars (#1663 needed for best performance) - Improvement to Hyrule Warriors: AoC, and UE4 games. These games can still stutter due to texture creation/loading. - Small improvement to other games, potential 1-frame stutters avoided. `ForceSynchronizeMemory`, which was added with POWER, is no longer needed. Some tests have been added for the MultiRegionHandle. --- Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs | 5 +- Ryujinx.Cpu/MemoryManager.cs | 4 +- Ryujinx.Cpu/MemoryManagerHostMapped.cs | 4 +- Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs | 2 + Ryujinx.Cpu/Tracking/CpuRegionHandle.cs | 1 + Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 90 +++++++------ Ryujinx.Graphics.Gpu/Memory/BufferManager.cs | 9 +- Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs | 7 +- .../MultiRegionTrackingTests.cs | 122 +++++++++++++++++- Ryujinx.Memory/Tracking/MemoryTracking.cs | 5 +- Ryujinx.Memory/Tracking/MultiRegionHandle.cs | 64 ++++++++- 11 files changed, 255 insertions(+), 58 deletions(-) diff --git a/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs b/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs index 818743397..23b4119f1 100644 --- a/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs +++ b/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs @@ -1,6 +1,8 @@ using Ryujinx.Cpu.Tracking; using Ryujinx.Memory; +using Ryujinx.Memory.Tracking; using System; +using System.Collections.Generic; namespace Ryujinx.Cpu { @@ -26,9 +28,10 @@ namespace Ryujinx.Cpu /// /// CPU virtual address of the region /// Size of the region + /// Handles to inherit state from or reuse. When none are present, provide null /// Desired granularity of write tracking /// The memory tracking handle - CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity); + CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity); /// /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. diff --git a/Ryujinx.Cpu/MemoryManager.cs b/Ryujinx.Cpu/MemoryManager.cs index dbc2f736a..9eb27c4f4 100644 --- a/Ryujinx.Cpu/MemoryManager.cs +++ b/Ryujinx.Cpu/MemoryManager.cs @@ -550,9 +550,9 @@ namespace Ryujinx.Cpu } /// - public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity) + public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity) { - return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, granularity)); + return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity)); } /// diff --git a/Ryujinx.Cpu/MemoryManagerHostMapped.cs b/Ryujinx.Cpu/MemoryManagerHostMapped.cs index da81d04f1..0b4e0e4cb 100644 --- a/Ryujinx.Cpu/MemoryManagerHostMapped.cs +++ b/Ryujinx.Cpu/MemoryManagerHostMapped.cs @@ -586,9 +586,9 @@ namespace Ryujinx.Cpu } /// - public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity) + public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity) { - return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, granularity)); + return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity)); } /// diff --git a/Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs b/Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs index 344b1a789..78c1b2404 100644 --- a/Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs +++ b/Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs @@ -1,5 +1,6 @@ using Ryujinx.Memory.Tracking; using System; +using System.Collections.Generic; namespace Ryujinx.Cpu.Tracking { @@ -16,6 +17,7 @@ namespace Ryujinx.Cpu.Tracking public void Dispose() => _impl.Dispose(); public void ForceDirty(ulong address, ulong size) => _impl.ForceDirty(address, size); + public IEnumerable GetHandles() => _impl.GetHandles(); public void QueryModified(Action modifiedAction) => _impl.QueryModified(modifiedAction); public void QueryModified(ulong address, ulong size, Action modifiedAction) => _impl.QueryModified(address, size, modifiedAction); public void QueryModified(ulong address, ulong size, Action modifiedAction, int sequenceNumber) => _impl.QueryModified(address, size, modifiedAction, sequenceNumber); diff --git a/Ryujinx.Cpu/Tracking/CpuRegionHandle.cs b/Ryujinx.Cpu/Tracking/CpuRegionHandle.cs index acb27b401..dd122288d 100644 --- a/Ryujinx.Cpu/Tracking/CpuRegionHandle.cs +++ b/Ryujinx.Cpu/Tracking/CpuRegionHandle.cs @@ -21,6 +21,7 @@ namespace Ryujinx.Cpu.Tracking public void Dispose() => _impl.Dispose(); public bool DirtyOrVolatile() => _impl.DirtyOrVolatile(); public void ForceDirty() => _impl.ForceDirty(); + public IRegionHandle GetHandle() => _impl; public void RegisterAction(RegionSignal action) => _impl.RegisterAction(action); public void RegisterDirtyEvent(Action action) => _impl.RegisterDirtyEvent(action); public void Reprotect(bool asDirty = false) => _impl.Reprotect(asDirty); diff --git a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index c567e30c1..b4854d816 100644 --- a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -3,6 +3,8 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; +using System.Collections.Generic; +using System.Linq; namespace Ryujinx.Graphics.Gpu.Memory { @@ -68,7 +70,8 @@ namespace Ryujinx.Graphics.Gpu.Memory /// GPU context that the buffer belongs to /// Start address of the buffer /// Size of the buffer in bytes - public Buffer(GpuContext context, ulong address, ulong size) + /// Buffers which this buffer contains, and will inherit tracking handles from + public Buffer(GpuContext context, ulong address, ulong size, IEnumerable baseBuffers = null) { _context = context; Address = address; @@ -78,13 +81,45 @@ namespace Ryujinx.Graphics.Gpu.Memory _useGranular = size > GranularBufferThreshold; + IEnumerable baseHandles = null; + + if (baseBuffers != null) + { + baseHandles = baseBuffers.SelectMany(buffer => + { + if (buffer._useGranular) + { + return buffer._memoryTrackingGranular.GetHandles(); + } + else + { + return Enumerable.Repeat(buffer._memoryTracking.GetHandle(), 1); + } + }); + } + if (_useGranular) { - _memoryTrackingGranular = context.PhysicalMemory.BeginGranularTracking(address, size); + _memoryTrackingGranular = context.PhysicalMemory.BeginGranularTracking(address, size, baseHandles); } else { _memoryTracking = context.PhysicalMemory.BeginTracking(address, size); + + if (baseHandles != null) + { + _memoryTracking.Reprotect(false); + + foreach (IRegionHandle handle in baseHandles) + { + if (handle.Dirty) + { + _memoryTracking.Reprotect(true); + } + + handle.Dispose(); + } + } } _externalFlushDelegate = new RegionSignal(ExternalFlush); @@ -180,39 +215,6 @@ namespace Ryujinx.Graphics.Gpu.Memory } } - /// - /// Performs guest to host memory synchronization of the buffer data, regardless of sequence number. - /// - /// - /// This causes the buffer data to be overwritten if a write was detected from the CPU, - /// since the last call to this method. - /// - /// Start address of the range to synchronize - /// Size in bytes of the range to synchronize - public void ForceSynchronizeMemory(ulong address, ulong size) - { - if (_useGranular) - { - _memoryTrackingGranular.QueryModified(address, size, _modifiedDelegate); - } - else - { - if (_memoryTracking.DirtyOrVolatile()) - { - _memoryTracking.Reprotect(); - - if (_modifiedRanges != null) - { - _modifiedRanges.ExcludeModifiedRegions(Address, Size, _loadDelegate); - } - else - { - _context.Renderer.SetBufferData(Handle, 0, _context.PhysicalMemory.GetSpan(Address, (int)Size)); - } - } - } - } - /// /// Ensure that the modified range list exists. /// @@ -461,18 +463,26 @@ namespace Ryujinx.Graphics.Gpu.Memory } /// - /// Disposes the host buffer. + /// Disposes the host buffer's data, not its tracking handles. /// - public void Dispose() + public void DisposeData() { _modifiedRanges?.Clear(); - _memoryTrackingGranular?.Dispose(); - _memoryTracking?.Dispose(); - _context.Renderer.DeleteBuffer(Handle); UnmappedSequence++; } + + /// + /// Disposes the host buffer. + /// + public void Dispose() + { + _memoryTrackingGranular?.Dispose(); + _memoryTracking?.Dispose(); + + DisposeData(); + } } } \ No newline at end of file diff --git a/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index 4a794b19d..20fa1f3a9 100644 --- a/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -533,8 +533,7 @@ namespace Ryujinx.Graphics.Gpu.Memory } } - Buffer newBuffer = new Buffer(_context, address, endAddress - address); - newBuffer.SynchronizeMemory(address, endAddress - address); + Buffer newBuffer = new Buffer(_context, address, endAddress - address, _bufferOverlaps.Take(overlapsCount)); lock (_buffers) { @@ -547,14 +546,14 @@ namespace Ryujinx.Graphics.Gpu.Memory int dstOffset = (int)(buffer.Address - newBuffer.Address); - buffer.ForceSynchronizeMemory(buffer.Address, buffer.Size); - buffer.CopyTo(newBuffer, dstOffset); newBuffer.InheritModifiedRanges(buffer); - buffer.Dispose(); + buffer.DisposeData(); } + newBuffer.SynchronizeMemory(address, endAddress - address); + // Existing buffers were modified, we need to rebind everything. _rebind = true; } diff --git a/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs b/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs index 3d2af5329..6463b932b 100644 --- a/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs +++ b/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs @@ -2,7 +2,9 @@ using Ryujinx.Cpu; using Ryujinx.Cpu.Tracking; using Ryujinx.Memory; using Ryujinx.Memory.Range; +using Ryujinx.Memory.Tracking; using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -200,11 +202,12 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// CPU virtual address of the region /// Size of the region + /// Handles to inherit state from or reuse /// Desired granularity of write tracking /// The memory tracking handle - public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity = 4096) + public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles = null, ulong granularity = 4096) { - return _cpuMemory.BeginGranularTracking(address, size, granularity); + return _cpuMemory.BeginGranularTracking(address, size, handles, granularity); } /// diff --git a/Ryujinx.Memory.Tests/MultiRegionTrackingTests.cs b/Ryujinx.Memory.Tests/MultiRegionTrackingTests.cs index 22e198c51..057565031 100644 --- a/Ryujinx.Memory.Tests/MultiRegionTrackingTests.cs +++ b/Ryujinx.Memory.Tests/MultiRegionTrackingTests.cs @@ -35,7 +35,7 @@ namespace Ryujinx.Memory.Tests { return smart ? _tracking.BeginSmartGranularTracking(address, size, granularity) : - (IMultiRegionHandle)_tracking.BeginGranularTracking(address, size, granularity); + (IMultiRegionHandle)_tracking.BeginGranularTracking(address, size, null, granularity); } private void RandomOrder(Random random, List indices, Action action) @@ -279,5 +279,125 @@ namespace Ryujinx.Memory.Tests Assert.AreEqual(0, _tracking.GetRegionCount()); } + + [Test] + public void InheritHandles() + { + // Test merging the following into a granular region handle: + // - 3x gap (creates new granular handles) + // - 3x from multiregion: not dirty, dirty and with action + // - 2x gap + // - 3x single page: not dirty, dirty and with action + // - 3x two page: not dirty, dirty and with action (handle is not reused, but its state is copied to the granular handles) + // - 1x gap + // For a total of 18 pages. + + bool[] actionsTriggered = new bool[3]; + + MultiRegionHandle granular = _tracking.BeginGranularTracking(PageSize * 3, PageSize * 3, null, PageSize); + PreparePages(granular, 3, PageSize * 3); + + // Write to the second handle in the multiregion. + _tracking.VirtualMemoryEvent(PageSize * 4, PageSize, true); + + // Add an action to the third handle in the multiregion. + granular.RegisterAction(PageSize * 5, PageSize, (_, _) => { actionsTriggered[0] = true; }); + + RegionHandle[] singlePages = new RegionHandle[3]; + + for (int i = 0; i < 3; i++) + { + singlePages[i] = _tracking.BeginTracking(PageSize * (8 + (ulong)i), PageSize); + singlePages[i].Reprotect(); + } + + // Write to the second handle. + _tracking.VirtualMemoryEvent(PageSize * 9, PageSize, true); + + // Add an action to the third handle. + singlePages[2].RegisterAction((_, _) => { actionsTriggered[1] = true; }); + + RegionHandle[] doublePages = new RegionHandle[3]; + + for (int i = 0; i < 3; i++) + { + doublePages[i] = _tracking.BeginTracking(PageSize * (11 + (ulong)i * 2), PageSize * 2); + doublePages[i].Reprotect(); + } + + // Write to the second handle. + _tracking.VirtualMemoryEvent(PageSize * 13, PageSize * 2, true); + + // Add an action to the third handle. + doublePages[2].RegisterAction((_, _) => { actionsTriggered[2] = true; }); + + // Finally, create a granular handle that inherits all these handles. + + IEnumerable[] handleGroups = new IEnumerable[] + { + granular.GetHandles(), + singlePages, + doublePages + }; + + MultiRegionHandle combined = _tracking.BeginGranularTracking(0, PageSize * 18, handleGroups.SelectMany((handles) => handles), PageSize); + + bool[] expectedDirty = new bool[] + { + true, true, true, // Gap. + false, true, false, // Multi-region. + true, true, // Gap. + false, true, false, // Individual handles. + false, false, true, true, false, false, // Double size handles. + true // Gap. + }; + + for (int i = 0; i < 18; i++) + { + bool modified = false; + combined.QueryModified(PageSize * (ulong)i, PageSize, (_, _) => { modified = true; }); + + Assert.AreEqual(expectedDirty[i], modified); + } + + Assert.AreEqual(new bool[3], actionsTriggered); + + _tracking.VirtualMemoryEvent(PageSize * 5, PageSize, false); + Assert.IsTrue(actionsTriggered[0]); + + _tracking.VirtualMemoryEvent(PageSize * 10, PageSize, false); + Assert.IsTrue(actionsTriggered[1]); + + _tracking.VirtualMemoryEvent(PageSize * 15, PageSize, false); + Assert.IsTrue(actionsTriggered[2]); + + // The double page handles should be disposed, as they were split into granular handles. + foreach (RegionHandle doublePage in doublePages) + { + // These should have been disposed. + bool throws = false; + + try + { + doublePage.Dispose(); + } + catch (ObjectDisposedException) + { + throws = true; + } + + Assert.IsTrue(throws); + } + + IEnumerable combinedHandles = combined.GetHandles(); + + Assert.AreEqual(handleGroups[0].ElementAt(0), combinedHandles.ElementAt(3)); + Assert.AreEqual(handleGroups[0].ElementAt(1), combinedHandles.ElementAt(4)); + Assert.AreEqual(handleGroups[0].ElementAt(2), combinedHandles.ElementAt(5)); + + Assert.AreEqual(singlePages[0], combinedHandles.ElementAt(8)); + Assert.AreEqual(singlePages[1], combinedHandles.ElementAt(9)); + Assert.AreEqual(singlePages[2], combinedHandles.ElementAt(10)); + } } } diff --git a/Ryujinx.Memory/Tracking/MemoryTracking.cs b/Ryujinx.Memory/Tracking/MemoryTracking.cs index 70951e8c9..aafb418dc 100644 --- a/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -134,13 +134,14 @@ namespace Ryujinx.Memory.Tracking /// /// CPU virtual address of the region /// Size of the region + /// Handles to inherit state from or reuse. When none are present, provide null /// Desired granularity of write tracking /// The memory tracking handle - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, ulong granularity) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity) { (address, size) = PageAlign(address, size); - return new MultiRegionHandle(this, address, size, granularity); + return new MultiRegionHandle(this, address, size, handles, granularity); } /// diff --git a/Ryujinx.Memory/Tracking/MultiRegionHandle.cs b/Ryujinx.Memory/Tracking/MultiRegionHandle.cs index 1f09807a6..638e7290f 100644 --- a/Ryujinx.Memory/Tracking/MultiRegionHandle.cs +++ b/Ryujinx.Memory/Tracking/MultiRegionHandle.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; namespace Ryujinx.Memory.Tracking { @@ -18,16 +19,68 @@ namespace Ryujinx.Memory.Tracking public bool Dirty { get; private set; } = true; - internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, ulong granularity) + internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, IEnumerable handles, ulong granularity) { _handles = new RegionHandle[size / granularity]; Granularity = granularity; - for (int i = 0; i < _handles.Length; i++) + int i = 0; + + if (handles != null) + { + // Inherit from the handles we were given. Any gaps must be filled with new handles, + // and old handles larger than our granularity must copy their state onto new granular handles and dispose. + // It is assumed that the provided handles do not overlap, in order, are on page boundaries, + // and don't extend past the requested range. + + foreach (RegionHandle handle in handles) + { + int startIndex = (int)((handle.Address - address) / granularity); + + // Fill any gap left before this handle. + while (i < startIndex) + { + RegionHandle fillHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity); + fillHandle.Parent = this; + _handles[i++] = fillHandle; + } + + if (handle.Size == granularity) + { + handle.Parent = this; + _handles[i++] = handle; + } + else + { + int endIndex = (int)((handle.EndAddress - address) / granularity); + + while (i < endIndex) + { + RegionHandle splitHandle = tracking.BeginTracking(address + (ulong)i * granularity, granularity); + splitHandle.Parent = this; + + splitHandle.Reprotect(handle.Dirty); + + RegionSignal signal = handle.PreAction; + if (signal != null) + { + splitHandle.RegisterAction(signal); + } + + _handles[i++] = splitHandle; + } + + handle.Dispose(); + } + } + } + + // Fill any remaining space with new handles. + while (i < _handles.Length) { RegionHandle handle = tracking.BeginTracking(address + (ulong)i * granularity, granularity); handle.Parent = this; - _handles[i] = handle; + _handles[i++] = handle; } Address = address; @@ -48,6 +101,11 @@ namespace Ryujinx.Memory.Tracking } } + public IEnumerable GetHandles() + { + return _handles; + } + public void SignalWrite() { Dirty = true;