From 5def0429f82c795d820b2307a0301b78dfb1e6b7 Mon Sep 17 00:00:00 2001 From: jhorv <38920027+jhorv@users.noreply.github.com> Date: Thu, 4 Apr 2024 21:23:03 -0400 Subject: [PATCH] Add support to IVirtualMemoryManager for zero-copy reads (#6251) * - WritableRegion: enable wrapping IMemoryOwner - IVirtualMemoryManager impls of GetWritableRegion() use pooled memory when region is non-contiguous. - IVirtualMemoryManager: add GetReadOnlySequence() and impls - ByteMemoryPool: add new method RentCopy() - ByteMemoryPool: make class static, remove ctor and singleton field from earlier impl * - BytesReadOnlySequenceSegment: move from Ryujinx.Common.Memory to Ryujinx.Memory - BytesReadOnlySequenceSegment: add IsContiguousWith() and Replace() methods - VirtualMemoryManagerBase: - remove generic type parameters, instead use ulong for virtual addresses and nuint for host/physical addresses - implement IWritableBlock - add virtual GetReadOnlySequence() with coalescing of contiguous segments - add virtual GetSpan() - add virtual GetWritableRegion() - add abstract IsMapped() - add virtual MapForeign(ulong, nuint, ulong) - add virtual Read() - add virtual Read(ulong, Span) - add virtual ReadTracked() - add virtual SignalMemoryTracking() - add virtual Write() - add virtual Write() - add virtual WriteUntracked() - add virtual WriteWithRedundancyCheck() - VirtualMemoryManagerRefCountedBase: remove generic type parameters - AddressSpaceManager: remove redundant methods, add required overrides - HvMemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManager: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - MemoryManagerHostMapped: remove redundant methods, add required overrides, add overrides for _invalidAccessHandler handling - NativeMemoryManager: add get properties for Pointer and Length - throughout: removed invalid comments * make HvMemoryManager class sealed * remove unused method * adjust MemoryManagerHostTracked * let MemoryManagerHostTracked override WriteImpl() --- .../ByteMemoryPool.ByteMemoryPoolBuffer.cs | 2 +- src/Ryujinx.Common/Memory/ByteMemoryPool.cs | 32 +- src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs | 249 ++----------- src/Ryujinx.Cpu/Jit/MemoryManager.cs | 266 +++---------- .../Jit/MemoryManagerHostMapped.cs | 89 ++--- .../Jit/MemoryManagerHostTracked.cs | 197 ++++------ .../VirtualMemoryManagerRefCountedBase.cs | 5 +- src/Ryujinx.Memory/AddressSpaceManager.cs | 176 +-------- .../BytesReadOnlySequenceSegment.cs | 60 +++ src/Ryujinx.Memory/IVirtualMemoryManager.cs | 10 + src/Ryujinx.Memory/NativeMemoryManager.cs | 4 + .../VirtualMemoryManagerBase.cs | 348 +++++++++++++++++- src/Ryujinx.Memory/WritableRegion.cs | 10 + .../MockVirtualMemoryManager.cs | 6 + 14 files changed, 635 insertions(+), 819 deletions(-) create mode 100644 src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs diff --git a/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs b/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs index df3f8dc93e..05fb29ac71 100644 --- a/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs +++ b/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs @@ -4,7 +4,7 @@ using System.Threading; namespace Ryujinx.Common.Memory { - public sealed partial class ByteMemoryPool + public partial class ByteMemoryPool { /// /// Represents a that wraps an array rented from diff --git a/src/Ryujinx.Common/Memory/ByteMemoryPool.cs b/src/Ryujinx.Common/Memory/ByteMemoryPool.cs index 071f56b136..6fd6a98aa7 100644 --- a/src/Ryujinx.Common/Memory/ByteMemoryPool.cs +++ b/src/Ryujinx.Common/Memory/ByteMemoryPool.cs @@ -6,24 +6,8 @@ namespace Ryujinx.Common.Memory /// /// Provides a pool of re-usable byte array instances. /// - public sealed partial class ByteMemoryPool + public static partial class ByteMemoryPool { - private static readonly ByteMemoryPool _shared = new(); - - /// - /// Constructs a instance. Private to force access through - /// the instance. - /// - private ByteMemoryPool() - { - // No implementation - } - - /// - /// Retrieves a shared instance. - /// - public static ByteMemoryPool Shared => _shared; - /// /// Returns the maximum buffer size supported by this pool. /// @@ -95,6 +79,20 @@ namespace Ryujinx.Common.Memory return buffer; } + /// + /// Copies into a newly rented byte memory buffer. + /// + /// The byte buffer to copy + /// A wrapping the rented memory with copied to it + public static IMemoryOwner RentCopy(ReadOnlySpan buffer) + { + var copy = RentImpl(buffer.Length); + + buffer.CopyTo(copy.Memory.Span); + + return copy; + } + private static ByteMemoryPoolBuffer RentImpl(int length) { if ((uint)length > Array.MaxLength) diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs index 80f7c8a1f8..0c2e5f33a0 100644 --- a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs +++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs @@ -3,10 +3,10 @@ using Ryujinx.Memory; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Runtime.Versioning; namespace Ryujinx.Cpu.AppleHv @@ -15,7 +15,7 @@ namespace Ryujinx.Cpu.AppleHv /// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table. /// [SupportedOSPlatform("macos")] - public class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public sealed class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private readonly InvalidAccessHandler _invalidAccessHandler; @@ -96,12 +96,6 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -126,20 +120,11 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -152,7 +137,6 @@ namespace Ryujinx.Cpu.AppleHv } } - /// public override void Read(ulong va, Span data) { try @@ -168,101 +152,11 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - SignalMemoryTracking(va, (ulong)data.Length, true); - - WriteImpl(va, data); - } - - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - WriteImpl(va, data); - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return false; - } - - SignalMemoryTracking(va, (ulong)data.Length, false); - - if (IsContiguousAndMapped(va, data.Length)) - { - var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); - - bool changed = !data.SequenceEqual(target); - - if (changed) - { - data.CopyTo(target); - } - - return changed; - } - else - { - WriteImpl(va, data); - - return true; - } - } - - private void WriteImpl(ulong va, ReadOnlySpan data) + public override void Write(ulong va, ReadOnlySpan data) { try { - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressChecked(va); - - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(_backingMemory.GetSpan(pa, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); - - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); - } - } + base.Write(va, data); } catch (InvalidMemoryRegionException) { @@ -273,61 +167,38 @@ namespace Ryujinx.Cpu.AppleHv } } - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + public override void WriteUntracked(ulong va, ReadOnlySpan data) { - if (size == 0) + try { - return ReadOnlySpan.Empty; + base.WriteUntracked(va, data); } - - if (tracked) + catch (InvalidMemoryRegionException) { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); - } - else - { - Span data = new byte[size]; - - base.Read(va, data); - - return data; + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } } } - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) { - if (size == 0) + try { - return new WritableRegion(null, va, Memory.Empty); + return base.GetReadOnlySequence(va, size, tracked); } - - if (tracked) + catch (InvalidMemoryRegionException) { - SignalMemoryTracking(va, (ulong)size, true); - } + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } - if (IsContiguousAndMapped(va, size)) - { - return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - Memory memory = new byte[size]; - - base.Read(va, memory.Span); - - return new WritableRegion(this, va, memory); + return ReadOnlySequence.Empty; } } - /// public ref T GetRef(ulong va) where T : unmanaged { if (!IsContiguous(va, Unsafe.SizeOf())) @@ -340,9 +211,8 @@ namespace Ryujinx.Cpu.AppleHv return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { return ValidateAddress(va) && _pages.IsMapped(va); } @@ -355,39 +225,6 @@ namespace Ryujinx.Cpu.AppleHv return _pages.IsRangeMapped(va, size); } - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - /// public IEnumerable GetHostRegions(ulong va, ulong size) { @@ -464,11 +301,10 @@ namespace Ryujinx.Cpu.AppleHv return regions; } - /// /// /// This function also validates that the given range is both valid and mapped, and will throw if it is not. /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -481,24 +317,6 @@ namespace Ryujinx.Cpu.AppleHv _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - /// public void Reprotect(ulong va, ulong size, MemoryPermission protection) { // TODO @@ -535,7 +353,7 @@ namespace Ryujinx.Cpu.AppleHv return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - private ulong GetPhysicalAddressChecked(ulong va) + private nuint GetPhysicalAddressChecked(ulong va) { if (!IsMapped(va)) { @@ -545,9 +363,9 @@ namespace Ryujinx.Cpu.AppleHv return GetPhysicalAddressInternal(va); } - private ulong GetPhysicalAddressInternal(ulong va) + private nuint GetPhysicalAddressInternal(ulong va) { - return _pageTable.Read(va) + (va & PageMask); + return (nuint)(_pageTable.Read(va) + (va & PageMask)); } /// @@ -558,10 +376,17 @@ namespace Ryujinx.Cpu.AppleHv _addressSpace.Dispose(); } - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _backingMemory.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) + protected override nuint TranslateVirtualAddressChecked(ulong va) => GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => GetPhysicalAddressInternal(va); + } } diff --git a/src/Ryujinx.Cpu/Jit/MemoryManager.cs b/src/Ryujinx.Cpu/Jit/MemoryManager.cs index c87c8b8cc5..dfa5b93539 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManager.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManager.cs @@ -3,6 +3,7 @@ using Ryujinx.Memory; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -14,7 +15,7 @@ namespace Ryujinx.Cpu.Jit /// /// Represents a CPU memory manager. /// - public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private const int PteSize = 8; @@ -97,12 +98,6 @@ namespace Ryujinx.Cpu.Jit Tracking.Map(oVa, size); } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -128,20 +123,11 @@ namespace Ryujinx.Cpu.Jit } } - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -190,117 +176,11 @@ namespace Ryujinx.Cpu.Jit } } - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - SignalMemoryTracking(va, (ulong)data.Length, true); - - WriteImpl(va, data); - } - - /// - public void WriteGuest(ulong va, T value) where T : unmanaged - { - Span data = MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)); - - SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true); - - WriteImpl(va, data); - } - - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - WriteImpl(va, data); - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return false; - } - - SignalMemoryTracking(va, (ulong)data.Length, false); - - if (IsContiguousAndMapped(va, data.Length)) - { - var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); - - bool changed = !data.SequenceEqual(target); - - if (changed) - { - data.CopyTo(target); - } - - return changed; - } - else - { - WriteImpl(va, data); - - return true; - } - } - - /// - /// Writes data to CPU mapped memory. - /// - /// Virtual address to write the data into - /// Data to be written - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void WriteImpl(ulong va, ReadOnlySpan data) + public override void Write(ulong va, ReadOnlySpan data) { try { - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressInternal(va); - - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(_backingMemory.GetSpan(pa, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - ulong pa = GetPhysicalAddressInternal(va + (ulong)offset); - - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); - } - } + base.Write(va, data); } catch (InvalidMemoryRegionException) { @@ -312,60 +192,47 @@ namespace Ryujinx.Cpu.Jit } /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + public void WriteGuest(ulong va, T value) where T : unmanaged { - if (size == 0) + Span data = MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)); + + SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true); + + Write(va, data); + } + + public override void WriteUntracked(ulong va, ReadOnlySpan data) + { + try { - return ReadOnlySpan.Empty; + base.WriteUntracked(va, data); } - - if (tracked) + catch (InvalidMemoryRegionException) { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); - } - else - { - Span data = new byte[size]; - - base.Read(va, data); - - return data; + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } } } - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) { - if (size == 0) + try { - return new WritableRegion(null, va, Memory.Empty); + return base.GetReadOnlySequence(va, size, tracked); } - - if (IsContiguousAndMapped(va, size)) + catch (InvalidMemoryRegionException) { - if (tracked) + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) { - SignalMemoryTracking(va, (ulong)size, true); + throw; } - return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - Memory memory = new byte[size]; - - GetSpan(va, size).CopyTo(memory.Span); - - return new WritableRegion(this, va, memory, tracked); + return ReadOnlySequence.Empty; } } - /// public ref T GetRef(ulong va) where T : unmanaged { if (!IsContiguous(va, Unsafe.SizeOf())) @@ -378,56 +245,6 @@ namespace Ryujinx.Cpu.Jit return ref _backingMemory.GetRef(GetPhysicalAddressInternal(va)); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, uint size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - /// public IEnumerable GetHostRegions(ulong va, ulong size) { @@ -532,9 +349,8 @@ namespace Ryujinx.Cpu.Jit return true; } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { if (!ValidateAddress(va)) { @@ -544,9 +360,9 @@ namespace Ryujinx.Cpu.Jit return _pageTable.Read((va / PageSize) * PteSize) != 0; } - private ulong GetPhysicalAddressInternal(ulong va) + private nuint GetPhysicalAddressInternal(ulong va) { - return PteToPa(_pageTable.Read((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask); + return (nuint)(PteToPa(_pageTable.Read((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask)); } /// @@ -643,9 +459,7 @@ namespace Ryujinx.Cpu.Jit { ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize); - long pte; - - pte = Volatile.Read(ref pageRef); + long pte = Volatile.Read(ref pageRef); if ((pte & tag) != 0) { @@ -663,7 +477,7 @@ namespace Ryujinx.Cpu.Jit } /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { SignalMemoryTrackingImpl(va, size, write, false, precise, exemptId); } @@ -683,10 +497,16 @@ namespace Ryujinx.Cpu.Jit /// protected override void Destroy() => _pageTable.Dispose(); - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _backingMemory.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) + protected override nuint TranslateVirtualAddressChecked(ulong va) + => GetPhysicalAddressInternal(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) => GetPhysicalAddressInternal(va); } } diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs index f410d02e96..c60ab6b246 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs @@ -3,6 +3,7 @@ using Ryujinx.Memory; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -12,7 +13,7 @@ namespace Ryujinx.Cpu.Jit /// /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. /// - public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private readonly InvalidAccessHandler _invalidAccessHandler; private readonly bool _unsafeMode; @@ -96,12 +97,6 @@ namespace Ryujinx.Cpu.Jit Tracking.Map(va, size); } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -138,8 +133,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public T Read(ulong va) where T : unmanaged + public override T Read(ulong va) { try { @@ -158,14 +152,11 @@ namespace Ryujinx.Cpu.Jit } } - /// - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -178,7 +169,6 @@ namespace Ryujinx.Cpu.Jit } } - /// public override void Read(ulong va, Span data) { try @@ -196,9 +186,7 @@ namespace Ryujinx.Cpu.Jit } } - - /// - public void Write(ulong va, T value) where T : unmanaged + public override void Write(ulong va, T value) { try { @@ -215,8 +203,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public void Write(ulong va, ReadOnlySpan data) + public override void Write(ulong va, ReadOnlySpan data) { try { @@ -233,8 +220,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) + public override void WriteUntracked(ulong va, ReadOnlySpan data) { try { @@ -251,8 +237,7 @@ namespace Ryujinx.Cpu.Jit } } - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) { try { @@ -279,8 +264,21 @@ namespace Ryujinx.Cpu.Jit } } - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, write: false); + } + else + { + AssertMapped(va, (ulong)size); + } + + return new ReadOnlySequence(_addressSpace.Mirror.GetMemory(va, size)); + } + + public override ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) { if (tracked) { @@ -294,8 +292,7 @@ namespace Ryujinx.Cpu.Jit return _addressSpace.Mirror.GetSpan(va, size); } - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) { if (tracked) { @@ -309,7 +306,6 @@ namespace Ryujinx.Cpu.Jit return _addressSpace.Mirror.GetWritableRegion(va, size); } - /// public ref T GetRef(ulong va) where T : unmanaged { SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); @@ -317,9 +313,8 @@ namespace Ryujinx.Cpu.Jit return ref _addressSpace.Mirror.GetRef(va); } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { return ValidateAddress(va) && _pages.IsMapped(va); } @@ -390,11 +385,10 @@ namespace Ryujinx.Cpu.Jit return _pageTable.Read(va) + (va & PageMask); } - /// /// /// This function also validates that the given range is both valid and mapped, and will throw if it is not. /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -407,23 +401,6 @@ namespace Ryujinx.Cpu.Jit _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - /// public void Reprotect(ulong va, ulong size, MemoryPermission protection) { @@ -470,10 +447,16 @@ namespace Ryujinx.Cpu.Jit _memoryEh.Dispose(); } - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _addressSpace.Mirror.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _addressSpace.Mirror.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) - => va; + protected override nuint TranslateVirtualAddressChecked(ulong va) + => (nuint)GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => (nuint)GetPhysicalAddressInternal(va); } } diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs index 18404bcc74..b2964cd29c 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs @@ -8,14 +8,13 @@ using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace Ryujinx.Cpu.Jit { /// /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. /// - public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase, IWritableBlock, IMemoryManager, IVirtualMemoryManagerTracked + public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked { private readonly InvalidAccessHandler _invalidAccessHandler; private readonly bool _unsafeMode; @@ -100,12 +99,6 @@ namespace Ryujinx.Cpu.Jit Tracking.Map(va, size); } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) - { - throw new NotSupportedException(); - } - /// public void Unmap(ulong va, ulong size) { @@ -120,18 +113,11 @@ namespace Ryujinx.Cpu.Jit _nativePageTable.Unmap(va, size); } - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - public T ReadTracked(ulong va) where T : unmanaged + public override T ReadTracked(ulong va) { try { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); + return base.ReadTracked(va); } catch (InvalidMemoryRegionException) { @@ -145,38 +131,39 @@ namespace Ryujinx.Cpu.Jit } public override void Read(ulong va, Span data) - { - ReadImpl(va, data); - } - - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - public void Write(ulong va, ReadOnlySpan data) { if (data.Length == 0) { return; } - SignalMemoryTracking(va, (ulong)data.Length, true); - - WriteImpl(va, data); - } - - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) + try { - return; - } + AssertValidAddressAndSize(va, (ulong)data.Length); - WriteImpl(va, data); + ulong endVa = va + (ulong)data.Length; + int offset = 0; + + while (va < endVa) + { + (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); + + memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize)); + + va += copySize; + offset += (int)copySize; + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } } - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) { if (data.Length == 0) { @@ -206,35 +193,7 @@ namespace Ryujinx.Cpu.Jit } } - private void WriteImpl(ulong va, ReadOnlySpan data) - { - try - { - AssertValidAddressAndSize(va, (ulong)data.Length); - - ulong endVa = va + (ulong)data.Length; - int offset = 0; - - while (va < endVa) - { - (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); - - data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize)); - - va += copySize; - offset += (int)copySize; - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + public override ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) { if (size == 0) { @@ -254,13 +213,13 @@ namespace Ryujinx.Cpu.Jit { Span data = new byte[size]; - ReadImpl(va, data); + Read(va, data); return data; } } - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) { if (size == 0) { @@ -280,7 +239,7 @@ namespace Ryujinx.Cpu.Jit { Memory memory = new byte[size]; - ReadImpl(va, memory.Span); + Read(va, memory.Span); return new WritableRegion(this, va, memory); } @@ -299,7 +258,7 @@ namespace Ryujinx.Cpu.Jit } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { return ValidateAddress(va) && _pages.IsMapped(va); } @@ -311,8 +270,6 @@ namespace Ryujinx.Cpu.Jit return _pages.IsRangeMapped(va, size); } - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - private bool TryGetVirtualContiguous(ulong va, int size, out MemoryBlock memory, out ulong offset) { if (_addressSpace.HasAnyPrivateAllocation(va, (ulong)size, out PrivateRange range)) @@ -491,44 +448,11 @@ namespace Ryujinx.Cpu.Jit return regions; } - private void ReadImpl(ulong va, Span data) - { - if (data.Length == 0) - { - return; - } - - try - { - AssertValidAddressAndSize(va, (ulong)data.Length); - - ulong endVa = va + (ulong)data.Length; - int offset = 0; - - while (va < endVa) - { - (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); - - memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize)); - - va += copySize; - offset += (int)copySize; - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - /// /// /// This function also validates that the given range is both valid and mapped, and will throw if it is not. /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -543,23 +467,6 @@ namespace Ryujinx.Cpu.Jit _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { return Tracking.BeginTracking(address, size, id, flags); @@ -618,10 +525,44 @@ namespace Ryujinx.Cpu.Jit _nativePageTable.Dispose(); } - protected override Span GetPhysicalAddressSpan(ulong pa, int size) + protected override Memory GetPhysicalAddressMemory(nuint pa, int size) + => _backingMemory.GetMemory(pa, size); + + protected override Span GetPhysicalAddressSpan(nuint pa, int size) => _backingMemory.GetSpan(pa, size); - protected override ulong TranslateVirtualAddressForRead(ulong va) - => GetPhysicalAddressInternal(va); + protected override void WriteImpl(ulong va, ReadOnlySpan data) + { + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + ulong endVa = va + (ulong)data.Length; + int offset = 0; + + while (va < endVa) + { + (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset)); + + data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize)); + + va += copySize; + offset += (int)copySize; + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + protected override nuint TranslateVirtualAddressChecked(ulong va) + => (nuint)GetPhysicalAddressChecked(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) + => (nuint)GetPhysicalAddressInternal(va); } } diff --git a/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs b/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs index c2d8cfb1a0..3c7b338055 100644 --- a/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs +++ b/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs @@ -1,13 +1,10 @@ using Ryujinx.Memory; using System.Diagnostics; -using System.Numerics; using System.Threading; namespace Ryujinx.Cpu { - public abstract class VirtualMemoryManagerRefCountedBase : VirtualMemoryManagerBase, IRefCounted - where TVirtual : IBinaryInteger - where TPhysical : IBinaryInteger + public abstract class VirtualMemoryManagerRefCountedBase : VirtualMemoryManagerBase, IRefCounted { private int _referenceCount; diff --git a/src/Ryujinx.Memory/AddressSpaceManager.cs b/src/Ryujinx.Memory/AddressSpaceManager.cs index f19b45b659..f089c85736 100644 --- a/src/Ryujinx.Memory/AddressSpaceManager.cs +++ b/src/Ryujinx.Memory/AddressSpaceManager.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace Ryujinx.Memory { @@ -11,7 +10,7 @@ namespace Ryujinx.Memory /// Represents a address space manager. /// Supports virtual memory region mapping, address translation and read/write access to mapped regions. /// - public sealed class AddressSpaceManager : VirtualMemoryManagerBase, IVirtualMemoryManager, IWritableBlock + public sealed class AddressSpaceManager : VirtualMemoryManagerBase, IVirtualMemoryManager { /// public bool Supports4KBPages => true; @@ -63,8 +62,7 @@ namespace Ryujinx.Memory } } - /// - public void MapForeign(ulong va, nuint hostPointer, ulong size) + public override void MapForeign(ulong va, nuint hostPointer, ulong size) { AssertValidAddressAndSize(va, size); @@ -92,106 +90,6 @@ namespace Ryujinx.Memory } } - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(GetHostSpanContiguous(va, data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(GetHostSpanContiguous(va, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(GetHostSpanContiguous(va + (ulong)offset, size)); - } - } - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - Write(va, data); - - return true; - } - - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return ReadOnlySpan.Empty; - } - - if (IsContiguousAndMapped(va, size)) - { - return GetHostSpanContiguous(va, size); - } - else - { - Span data = new byte[size]; - - Read(va, data); - - return data; - } - } - - /// - public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return new WritableRegion(null, va, Memory.Empty); - } - - if (IsContiguousAndMapped(va, size)) - { - return new WritableRegion(null, va, new NativeMemoryManager((byte*)GetHostAddress(va), size).Memory); - } - else - { - Memory memory = new byte[size]; - - GetSpan(va, size).CopyTo(memory.Span); - - return new WritableRegion(this, va, memory); - } - } - /// public unsafe ref T GetRef(ulong va) where T : unmanaged { @@ -203,50 +101,6 @@ namespace Ryujinx.Memory return ref *(T*)GetHostAddress(va); } - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, uint size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetHostAddress(va) + PageSize != GetHostAddress(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - /// public IEnumerable GetHostRegions(ulong va, ulong size) { @@ -304,7 +158,7 @@ namespace Ryujinx.Memory return null; } - int pages = GetPagesCount(va, (uint)size, out va); + int pages = GetPagesCount(va, size, out va); var regions = new List(); @@ -336,9 +190,8 @@ namespace Ryujinx.Memory return regions; } - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) + public override bool IsMapped(ulong va) { if (!ValidateAddress(va)) { @@ -351,7 +204,7 @@ namespace Ryujinx.Memory /// public bool IsRangeMapped(ulong va, ulong size) { - if (size == 0UL) + if (size == 0) { return true; } @@ -376,11 +229,6 @@ namespace Ryujinx.Memory return true; } - private unsafe Span GetHostSpanContiguous(ulong va, int size) - { - return new Span((void*)GetHostAddress(va), size); - } - private nuint GetHostAddress(ulong va) { return _pageTable.Read(va) + (nuint)(va & PageMask); @@ -397,16 +245,16 @@ namespace Ryujinx.Memory throw new NotImplementedException(); } - /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) - { - // Only the ARM Memory Manager has tracking for now. - } + protected unsafe override Memory GetPhysicalAddressMemory(nuint pa, int size) + => new NativeMemoryManager((byte*)pa, size).Memory; protected override unsafe Span GetPhysicalAddressSpan(nuint pa, int size) - => new((void*)pa, size); + => new Span((void*)pa, size); - protected override nuint TranslateVirtualAddressForRead(ulong va) + protected override nuint TranslateVirtualAddressChecked(ulong va) + => GetHostAddress(va); + + protected override nuint TranslateVirtualAddressUnchecked(ulong va) => GetHostAddress(va); } } diff --git a/src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs b/src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs new file mode 100644 index 0000000000..5fe8d936c3 --- /dev/null +++ b/src/Ryujinx.Memory/BytesReadOnlySequenceSegment.cs @@ -0,0 +1,60 @@ +using System; +using System.Buffers; +using System.Runtime.InteropServices; + +namespace Ryujinx.Memory +{ + /// + /// A concrete implementation of , + /// with methods to help build a full sequence. + /// + public sealed class BytesReadOnlySequenceSegment : ReadOnlySequenceSegment + { + public BytesReadOnlySequenceSegment(Memory memory) => Memory = memory; + + public BytesReadOnlySequenceSegment Append(Memory memory) + { + var nextSegment = new BytesReadOnlySequenceSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length + }; + + Next = nextSegment; + + return nextSegment; + } + + /// + /// Attempts to determine if the current and are contiguous. + /// Only works if both were created by a . + /// + /// The segment to check if continuous with the current one + /// The starting address of the contiguous segment + /// The size of the contiguous segment + /// True if the segments are contiguous, otherwise false + public unsafe bool IsContiguousWith(Memory other, out nuint contiguousStart, out int contiguousSize) + { + if (MemoryMarshal.TryGetMemoryManager>(Memory, out var thisMemoryManager) && + MemoryMarshal.TryGetMemoryManager>(other, out var otherMemoryManager) && + thisMemoryManager.Pointer + thisMemoryManager.Length == otherMemoryManager.Pointer) + { + contiguousStart = (nuint)thisMemoryManager.Pointer; + contiguousSize = thisMemoryManager.Length + otherMemoryManager.Length; + return true; + } + else + { + contiguousStart = 0; + contiguousSize = 0; + return false; + } + } + + /// + /// Replaces the current value with the one provided. + /// + /// The new segment to hold in this + public void Replace(Memory memory) + => Memory = memory; + } +} diff --git a/src/Ryujinx.Memory/IVirtualMemoryManager.cs b/src/Ryujinx.Memory/IVirtualMemoryManager.cs index 557da2f261..96d3e85797 100644 --- a/src/Ryujinx.Memory/IVirtualMemoryManager.cs +++ b/src/Ryujinx.Memory/IVirtualMemoryManager.cs @@ -124,6 +124,16 @@ namespace Ryujinx.Memory } } + /// + /// Gets a read-only sequence of read-only memory blocks from CPU mapped memory. + /// + /// Virtual address of the data + /// Size of the data + /// True if read tracking is triggered on the memory + /// A read-only sequence of read-only memory of the data + /// Throw for unhandled invalid or unmapped memory accesses + ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false); + /// /// Gets a read-only span of data from CPU mapped memory. /// diff --git a/src/Ryujinx.Memory/NativeMemoryManager.cs b/src/Ryujinx.Memory/NativeMemoryManager.cs index fe718bda81..9ca6329382 100644 --- a/src/Ryujinx.Memory/NativeMemoryManager.cs +++ b/src/Ryujinx.Memory/NativeMemoryManager.cs @@ -14,6 +14,10 @@ namespace Ryujinx.Memory _length = length; } + public unsafe T* Pointer => _pointer; + + public int Length => _length; + public override Span GetSpan() { return new Span((void*)_pointer, _length); diff --git a/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs b/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs index cbec88cc56..506e25f668 100644 --- a/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs +++ b/src/Ryujinx.Memory/VirtualMemoryManagerBase.cs @@ -1,34 +1,171 @@ +using Ryujinx.Common.Memory; using System; -using System.Numerics; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Ryujinx.Memory { - public abstract class VirtualMemoryManagerBase - where TVirtual : IBinaryInteger - where TPhysical : IBinaryInteger + public abstract class VirtualMemoryManagerBase : IWritableBlock { public const int PageBits = 12; public const int PageSize = 1 << PageBits; public const int PageMask = PageSize - 1; - protected abstract TVirtual AddressSpaceSize { get; } + protected abstract ulong AddressSpaceSize { get; } - public virtual void Read(TVirtual va, Span data) + public virtual ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySequence.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return new ReadOnlySequence(GetPhysicalAddressMemory(pa, size)); + } + else + { + AssertValidAddressAndSize(va, size); + + int offset = 0, segmentSize; + + BytesReadOnlySequenceSegment first = null, last = null; + + if ((va & PageMask) != 0) + { + nuint pa = TranslateVirtualAddressChecked(va); + + segmentSize = Math.Min(size, PageSize - (int)(va & PageMask)); + + Memory memory = GetPhysicalAddressMemory(pa, segmentSize); + + first = last = new BytesReadOnlySequenceSegment(memory); + + offset += segmentSize; + } + + for (; offset < size; offset += segmentSize) + { + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); + + segmentSize = Math.Min(size - offset, PageSize); + + Memory memory = GetPhysicalAddressMemory(pa, segmentSize); + + if (first is null) + { + first = last = new BytesReadOnlySequenceSegment(memory); + } + else + { + if (last.IsContiguousWith(memory, out nuint contiguousStart, out int contiguousSize)) + { + last.Replace(GetPhysicalAddressMemory(contiguousStart, contiguousSize)); + } + else + { + last = last.Append(memory); + } + } + } + + return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex)); + } + } + + public virtual ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySpan.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return GetPhysicalAddressSpan(pa, size); + } + else + { + Span data = new byte[size]; + + Read(va, data); + + return data; + } + } + + public virtual WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return new WritableRegion(null, va, Memory.Empty); + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, true); + } + + if (IsContiguousAndMapped(va, size)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + return new WritableRegion(null, va, GetPhysicalAddressMemory(pa, size)); + } + else + { + IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size); + + Read(va, memoryOwner.Memory.Span); + + return new WritableRegion(this, va, memoryOwner); + } + } + + public abstract bool IsMapped(ulong va); + + public virtual void MapForeign(ulong va, nuint hostPointer, ulong size) + { + throw new NotSupportedException(); + } + + public virtual T Read(ulong va) where T : unmanaged + { + return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; + } + + public virtual void Read(ulong va, Span data) { if (data.Length == 0) { return; } - AssertValidAddressAndSize(va, TVirtual.CreateChecked(data.Length)); + AssertValidAddressAndSize(va, data.Length); int offset = 0, size; - if ((int.CreateTruncating(va) & PageMask) != 0) + if ((va & PageMask) != 0) { - TPhysical pa = TranslateVirtualAddressForRead(va); + nuint pa = TranslateVirtualAddressChecked(va); - size = Math.Min(data.Length, PageSize - ((int.CreateTruncating(va) & PageMask))); + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); GetPhysicalAddressSpan(pa, size).CopyTo(data[..size]); @@ -37,7 +174,7 @@ namespace Ryujinx.Memory for (; offset < data.Length; offset += size) { - TPhysical pa = TranslateVirtualAddressForRead(va + TVirtual.CreateChecked(offset)); + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); size = Math.Min(data.Length - offset, PageSize); @@ -45,13 +182,84 @@ namespace Ryujinx.Memory } } + public virtual T ReadTracked(ulong va) where T : unmanaged + { + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); + + return Read(va); + } + + public virtual void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + // No default implementation + } + + public virtual void Write(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + SignalMemoryTracking(va, (ulong)data.Length, true); + + WriteImpl(va, data); + } + + public virtual void Write(ulong va, T value) where T : unmanaged + { + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + } + + public virtual void WriteUntracked(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + WriteImpl(va, data); + } + + public virtual bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return false; + } + + if (IsContiguousAndMapped(va, data.Length)) + { + SignalMemoryTracking(va, (ulong)data.Length, false); + + nuint pa = TranslateVirtualAddressChecked(va); + + var target = GetPhysicalAddressSpan(pa, data.Length); + + bool changed = !data.SequenceEqual(target); + + if (changed) + { + data.CopyTo(target); + } + + return changed; + } + else + { + Write(va, data); + + return true; + } + } + /// /// Ensures the combination of virtual address and size is part of the addressable space. /// /// Virtual address of the range /// Size of the range in bytes /// Throw when the memory region specified outside the addressable space - protected void AssertValidAddressAndSize(TVirtual va, TVirtual size) + protected void AssertValidAddressAndSize(ulong va, ulong size) { if (!ValidateAddressAndSize(va, size)) { @@ -59,16 +267,82 @@ namespace Ryujinx.Memory } } - protected abstract Span GetPhysicalAddressSpan(TPhysical pa, int size); + /// + /// Ensures the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// Throw when the memory region specified outside the addressable space + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected void AssertValidAddressAndSize(ulong va, int size) + => AssertValidAddressAndSize(va, (ulong)size); - protected abstract TPhysical TranslateVirtualAddressForRead(TVirtual va); + /// + /// Computes the number of pages in a virtual address range. + /// + /// Virtual address of the range + /// Size of the range + /// The virtual address of the beginning of the first page + /// This function does not differentiate between allocated and unallocated pages. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + protected abstract Memory GetPhysicalAddressMemory(nuint pa, int size); + + protected abstract Span GetPhysicalAddressSpan(nuint pa, int size); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguous(ulong va, int size) => IsContiguous(va, (ulong)size); + + protected virtual bool IsContiguous(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return false; + } + + int pages = GetPagesCount(va, size, out va); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return false; + } + + if (TranslateVirtualAddressUnchecked(va) + PageSize != TranslateVirtualAddressUnchecked(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool IsContiguousAndMapped(ulong va, int size) + => IsContiguous(va, size) && IsMapped(va); + + protected abstract nuint TranslateVirtualAddressChecked(ulong va); + + protected abstract nuint TranslateVirtualAddressUnchecked(ulong va); /// /// Checks if the virtual address is part of the addressable space. /// /// Virtual address /// True if the virtual address is part of the addressable space - protected bool ValidateAddress(TVirtual va) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected bool ValidateAddress(ulong va) { return va < AddressSpaceSize; } @@ -79,13 +353,53 @@ namespace Ryujinx.Memory /// Virtual address of the range /// Size of the range in bytes /// True if the combination of virtual address and size is part of the addressable space - protected bool ValidateAddressAndSize(TVirtual va, TVirtual size) + protected bool ValidateAddressAndSize(ulong va, ulong size) { - TVirtual endVa = va + size; + ulong endVa = va + size; return endVa >= va && endVa >= size && endVa <= AddressSpaceSize; } protected static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + + protected static void ThrowMemoryNotContiguous() + => throw new MemoryNotContiguousException(); + + protected virtual void WriteImpl(ulong va, ReadOnlySpan data) + { + AssertValidAddressAndSize(va, data.Length); + + if (IsContiguousAndMapped(va, data.Length)) + { + nuint pa = TranslateVirtualAddressUnchecked(va); + + data.CopyTo(GetPhysicalAddressSpan(pa, data.Length)); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + nuint pa = TranslateVirtualAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + data[..size].CopyTo(GetPhysicalAddressSpan(pa, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + nuint pa = TranslateVirtualAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + data.Slice(offset, size).CopyTo(GetPhysicalAddressSpan(pa, size)); + } + } + } + } } diff --git a/src/Ryujinx.Memory/WritableRegion.cs b/src/Ryujinx.Memory/WritableRegion.cs index 666c8a99b2..2c21ef4e80 100644 --- a/src/Ryujinx.Memory/WritableRegion.cs +++ b/src/Ryujinx.Memory/WritableRegion.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; namespace Ryujinx.Memory { @@ -6,6 +7,7 @@ namespace Ryujinx.Memory { private readonly IWritableBlock _block; private readonly ulong _va; + private readonly IMemoryOwner _memoryOwner; private readonly bool _tracked; private bool NeedsWriteback => _block != null; @@ -20,6 +22,12 @@ namespace Ryujinx.Memory Memory = memory; } + public WritableRegion(IWritableBlock block, ulong va, IMemoryOwner memoryOwner, bool tracked = false) + : this(block, va, memoryOwner.Memory, tracked) + { + _memoryOwner = memoryOwner; + } + public void Dispose() { if (NeedsWriteback) @@ -33,6 +41,8 @@ namespace Ryujinx.Memory _block.WriteUntracked(_va, Memory.Span); } } + + _memoryOwner?.Dispose(); } } } diff --git a/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs b/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs index 85a1ac02bc..15e7d9b89a 100644 --- a/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs +++ b/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs @@ -1,6 +1,7 @@ using Ryujinx.Memory; using Ryujinx.Memory.Range; using System; +using System.Buffers; using System.Collections.Generic; namespace Ryujinx.Tests.Memory @@ -57,6 +58,11 @@ namespace Ryujinx.Tests.Memory throw new NotImplementedException(); } + public ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false) + { + throw new NotImplementedException(); + } + public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) { throw new NotImplementedException();