From fdd3263e31f8bf352a21e05703d0a6a82c800995 Mon Sep 17 00:00:00 2001 From: riperiperi Date: Thu, 14 Mar 2024 22:38:27 +0000 Subject: [PATCH] Separate guest/host tracking + unaligned protection (#6486) * WIP: Separate guest/host tracking + unaligned protection Allow memory manager to define support for single byte guest tracking * Formatting * Improve docs * Properly handle cases where the address space bits are too low * Address feedback --- .../Instructions/NativeInterface.cs | 20 +- src/ARMeilleure/Memory/IMemoryManager.cs | 22 + src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs | 336 +-------------- .../IVirtualMemoryManagerTracked.cs | 6 +- src/Ryujinx.Cpu/Jit/MemoryManager.cs | 152 ++++--- .../Jit/MemoryManagerHostMapped.cs | 338 +-------------- src/Ryujinx.Cpu/ManagedPageFlags.cs | 389 ++++++++++++++++++ src/Ryujinx.Graphics.Gpu/Image/Pool.cs | 2 +- src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 4 +- .../Memory/PhysicalMemory.cs | 16 +- src/Ryujinx.Memory/AddressSpaceManager.cs | 2 +- src/Ryujinx.Memory/IVirtualMemoryManager.cs | 3 +- src/Ryujinx.Memory/Tracking/MemoryTracking.cs | 137 ++++-- .../Tracking/MultiRegionHandle.cs | 9 +- src/Ryujinx.Memory/Tracking/RegionFlags.cs | 21 + src/Ryujinx.Memory/Tracking/RegionHandle.cs | 68 ++- src/Ryujinx.Memory/Tracking/VirtualRegion.cs | 10 +- .../MockVirtualMemoryManager.cs | 2 +- 18 files changed, 774 insertions(+), 763 deletions(-) create mode 100644 src/Ryujinx.Cpu/ManagedPageFlags.cs create mode 100644 src/Ryujinx.Memory/Tracking/RegionFlags.cs diff --git a/src/ARMeilleure/Instructions/NativeInterface.cs b/src/ARMeilleure/Instructions/NativeInterface.cs index d1b2e353c..0cd3754f7 100644 --- a/src/ARMeilleure/Instructions/NativeInterface.cs +++ b/src/ARMeilleure/Instructions/NativeInterface.cs @@ -91,54 +91,54 @@ namespace ARMeilleure.Instructions #region "Read" public static byte ReadByte(ulong address) { - return GetMemoryManager().ReadTracked(address); + return GetMemoryManager().ReadGuest(address); } public static ushort ReadUInt16(ulong address) { - return GetMemoryManager().ReadTracked(address); + return GetMemoryManager().ReadGuest(address); } public static uint ReadUInt32(ulong address) { - return GetMemoryManager().ReadTracked(address); + return GetMemoryManager().ReadGuest(address); } public static ulong ReadUInt64(ulong address) { - return GetMemoryManager().ReadTracked(address); + return GetMemoryManager().ReadGuest(address); } public static V128 ReadVector128(ulong address) { - return GetMemoryManager().ReadTracked(address); + return GetMemoryManager().ReadGuest(address); } #endregion #region "Write" public static void WriteByte(ulong address, byte value) { - GetMemoryManager().Write(address, value); + GetMemoryManager().WriteGuest(address, value); } public static void WriteUInt16(ulong address, ushort value) { - GetMemoryManager().Write(address, value); + GetMemoryManager().WriteGuest(address, value); } public static void WriteUInt32(ulong address, uint value) { - GetMemoryManager().Write(address, value); + GetMemoryManager().WriteGuest(address, value); } public static void WriteUInt64(ulong address, ulong value) { - GetMemoryManager().Write(address, value); + GetMemoryManager().WriteGuest(address, value); } public static void WriteVector128(ulong address, V128 value) { - GetMemoryManager().Write(address, value); + GetMemoryManager().WriteGuest(address, value); } #endregion diff --git a/src/ARMeilleure/Memory/IMemoryManager.cs b/src/ARMeilleure/Memory/IMemoryManager.cs index 952cd2b4f..46d442655 100644 --- a/src/ARMeilleure/Memory/IMemoryManager.cs +++ b/src/ARMeilleure/Memory/IMemoryManager.cs @@ -28,6 +28,17 @@ namespace ARMeilleure.Memory /// The data T ReadTracked(ulong va) where T : unmanaged; + /// + /// Reads data from CPU mapped memory, from guest code. (with read tracking) + /// + /// Type of the data being read + /// Virtual address of the data in memory + /// The data + T ReadGuest(ulong va) where T : unmanaged + { + return ReadTracked(va); + } + /// /// Writes data to CPU mapped memory. /// @@ -36,6 +47,17 @@ namespace ARMeilleure.Memory /// Data to be written void Write(ulong va, T value) where T : unmanaged; + /// + /// Writes data to CPU mapped memory, from guest code. + /// + /// Type of the data being written + /// Virtual address to write the data into + /// Data to be written + void WriteGuest(ulong va, T value) where T : unmanaged + { + Write(va, value); + } + /// /// Gets a read-only span of data from CPU mapped memory. /// diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs index 6e864f4ca..80f7c8a1f 100644 --- a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs +++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; -using System.Threading; namespace Ryujinx.Cpu.AppleHv { @@ -18,21 +17,6 @@ namespace Ryujinx.Cpu.AppleHv [SupportedOSPlatform("macos")] public class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock { - public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. - public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. - - private enum HostMappedPtBits : ulong - { - Unmapped = 0, - Mapped, - WriteTracked, - ReadWriteTracked, - - MappedReplicated = 0x5555555555555555, - WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, - ReadWriteTrackedReplicated = ulong.MaxValue, - } - private readonly InvalidAccessHandler _invalidAccessHandler; private readonly HvAddressSpace _addressSpace; @@ -42,7 +26,7 @@ namespace Ryujinx.Cpu.AppleHv private readonly MemoryBlock _backingMemory; private readonly PageTable _pageTable; - private readonly ulong[] _pageBitmap; + private readonly ManagedPageFlags _pages; public bool Supports4KBPages => true; @@ -84,7 +68,7 @@ namespace Ryujinx.Cpu.AppleHv AddressSpaceBits = asBits; - _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + _pages = new ManagedPageFlags(AddressSpaceBits); Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler); } @@ -95,7 +79,7 @@ namespace Ryujinx.Cpu.AppleHv PtMap(va, pa, size); _addressSpace.MapUser(va, pa, size, MemoryPermission.ReadWriteExecute); - AddMapping(va, size); + _pages.AddMapping(va, size); Tracking.Map(va, size); } @@ -126,7 +110,7 @@ namespace Ryujinx.Cpu.AppleHv UnmapEvent?.Invoke(va, size); Tracking.Unmap(va, size); - RemoveMapping(va, size); + _pages.RemoveMapping(va, size); _addressSpace.UnmapUser(va, size); PtUnmap(va, size); } @@ -360,22 +344,7 @@ namespace Ryujinx.Cpu.AppleHv [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsMapped(ulong va) { - return ValidateAddress(va) && IsMappedImpl(va); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsMappedImpl(ulong va) - { - ulong page = va >> PageBits; - - int bit = (int)((page & 31) << 1); - - int pageIndex = (int)(page >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - - return ((pte >> bit) & 3) != 0; + return ValidateAddress(va) && _pages.IsMapped(va); } /// @@ -383,58 +352,7 @@ namespace Ryujinx.Cpu.AppleHv { AssertValidAddressAndSize(va, size); - return IsRangeMappedImpl(va, size); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) - { - startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); - endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); - - pageIndex = (int)(pageStart >> PageToPteShift); - pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); - } - - private bool IsRangeMappedImpl(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - - if (pages == 1) - { - return IsMappedImpl(va); - } - - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - // Check if either bit in each 2 bit page entry is set. - // OR the block with itself shifted down by 1, and check the first bit of each entry. - - ulong mask = BlockMappedMask & startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte = Volatile.Read(ref pageRef); - - pte |= pte >> 1; - if ((pte & mask) != mask) - { - return false; - } - - mask = BlockMappedMask; - } - - return true; + return _pages.IsRangeMapped(va, size); } private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); @@ -560,76 +478,7 @@ namespace Ryujinx.Cpu.AppleHv return; } - // Software table, used for managed memory tracking. - - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); - - int bit = (int)((pageStart & 31) << 1); - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - ulong state = ((pte >> bit) & 3); - - if (state >= tag) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - return; - } - else if (state == 0) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte = Volatile.Read(ref pageRef); - ulong mappedMask = mask & BlockMappedMask; - - ulong mappedPte = pte | (pte >> 1); - if ((mappedPte & mappedMask) != mappedMask) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - - pte &= mask; - if ((pte & anyTrackingTag) != 0) // Search for any tracking. - { - // Writes trigger any tracking. - // Only trigger tracking from reads if both bits are set on any page. - if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } - } - - mask = ulong.MaxValue; - } - } + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } /// @@ -656,103 +505,28 @@ namespace Ryujinx.Cpu.AppleHv } /// - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - int pages = GetPagesCount(va, size, out va); - ulong pageStart = va >> PageBits; - - if (pages == 1) + if (guest) { - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, - _ => (ulong)HostMappedPtBits.ReadWriteTracked, - }; - - int bit = (int)((pageStart & 31) << 1); - - ulong tagMask = 3UL << bit; - ulong invTagMask = ~tagMask; - - ulong tag = protTag << bit; - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + _addressSpace.ReprotectUser(va, size, protection); } else { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, - _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, - }; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Change the protection of all 2 bit entries that are mapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask &= mask; // Only update mapped pages within the given range. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); - - mask = ulong.MaxValue; - } + _pages.TrackingReprotect(va, size, protection); } - - protection = protection switch - { - MemoryPermission.None => MemoryPermission.ReadAndWrite, - MemoryPermission.Write => MemoryPermission.Read, - _ => MemoryPermission.None, - }; - - _addressSpace.ReprotectUser(va, size, protection); } /// - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginTracking(address, size, id); + return Tracking.BeginTracking(address, size, id, flags); } /// - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); } /// @@ -761,86 +535,6 @@ namespace Ryujinx.Cpu.AppleHv return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - /// - /// Adds the given address mapping to the page table. - /// - /// Virtual memory address - /// Size to be mapped - private void AddMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Map all 2-bit entries that are unmapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); - - mask = ulong.MaxValue; - } - } - - /// - /// Removes the given address mapping from the page table. - /// - /// Virtual memory address - /// Size to be unmapped - private void RemoveMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - startMask = ~startMask; - endMask = ~endMask; - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask |= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); - - mask = 0; - } - } - private ulong GetPhysicalAddressChecked(ulong va) { if (!IsMapped(va)) diff --git a/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs b/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs index 199bff240..e8d11ede5 100644 --- a/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs +++ b/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs @@ -28,8 +28,9 @@ namespace Ryujinx.Cpu /// CPU virtual address of the region /// Size of the region /// Handle ID + /// Region flags /// The memory tracking handle - RegionHandle BeginTracking(ulong address, ulong size, int id); + RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None); /// /// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. @@ -39,8 +40,9 @@ namespace Ryujinx.Cpu /// Handles to inherit state from or reuse. When none are present, provide null /// Desired granularity of write tracking /// Handle ID + /// Region flags /// The memory tracking handle - MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id); + MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None); /// /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. diff --git a/src/Ryujinx.Cpu/Jit/MemoryManager.cs b/src/Ryujinx.Cpu/Jit/MemoryManager.cs index bbfdf536e..c87c8b8cc 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManager.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManager.cs @@ -33,6 +33,8 @@ namespace Ryujinx.Cpu.Jit private readonly MemoryBlock _pageTable; + private readonly ManagedPageFlags _pages; + /// /// Page table base pointer. /// @@ -70,6 +72,8 @@ namespace Ryujinx.Cpu.Jit AddressSpaceSize = asSize; _pageTable = new MemoryBlock((asSize / PageSize) * PteSize); + _pages = new ManagedPageFlags(AddressSpaceBits); + Tracking = new MemoryTracking(this, PageSize); } @@ -89,6 +93,7 @@ namespace Ryujinx.Cpu.Jit remainingSize -= PageSize; } + _pages.AddMapping(oVa, size); Tracking.Map(oVa, size); } @@ -111,6 +116,7 @@ namespace Ryujinx.Cpu.Jit UnmapEvent?.Invoke(va, size); Tracking.Unmap(va, size); + _pages.RemoveMapping(va, size); ulong remainingSize = size; while (remainingSize != 0) @@ -148,6 +154,26 @@ namespace Ryujinx.Cpu.Jit } } + /// + public T ReadGuest(ulong va) where T : unmanaged + { + try + { + SignalMemoryTrackingImpl(va, (ulong)Unsafe.SizeOf(), false, true); + + return Read(va); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + + return default; + } + } + /// public override void Read(ulong va, Span data) { @@ -183,6 +209,16 @@ namespace Ryujinx.Cpu.Jit WriteImpl(va, data); } + /// + public void WriteGuest(ulong va, T value) where T : unmanaged + { + Span data = MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)); + + SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true); + + WriteImpl(va, data); + } + /// public void WriteUntracked(ulong va, ReadOnlySpan data) { @@ -520,50 +556,57 @@ namespace Ryujinx.Cpu.Jit } /// - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { AssertValidAddressAndSize(va, size); - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - long tag = protection switch + if (guest) { - MemoryPermission.None => 0L, - MemoryPermission.Write => 2L << PointerTagBit, - _ => 3L << PointerTagBit, - }; + // Protection is inverted on software pages, since the default value is 0. + protection = (~protection) & MemoryPermission.ReadAndWrite; - int pages = GetPagesCount(va, (uint)size, out va); - ulong pageStart = va >> PageBits; - long invTagMask = ~(0xffffL << 48); - - for (int page = 0; page < pages; page++) - { - ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize); - - long pte; - - do + long tag = protection switch { - pte = Volatile.Read(ref pageRef); - } - while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + MemoryPermission.None => 0L, + MemoryPermission.Write => 2L << PointerTagBit, + _ => 3L << PointerTagBit, + }; - pageStart++; + int pages = GetPagesCount(va, (uint)size, out va); + ulong pageStart = va >> PageBits; + long invTagMask = ~(0xffffL << 48); + + for (int page = 0; page < pages; page++) + { + ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize); + + long pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + + pageStart++; + } + } + else + { + _pages.TrackingReprotect(va, size, protection); } } /// - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginTracking(address, size, id); + return Tracking.BeginTracking(address, size, id, flags); } /// - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); } /// @@ -572,8 +615,7 @@ namespace Ryujinx.Cpu.Jit return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + private void SignalMemoryTrackingImpl(ulong va, ulong size, bool write, bool guest, bool precise = false, int? exemptId = null) { AssertValidAddressAndSize(va, size); @@ -583,31 +625,47 @@ namespace Ryujinx.Cpu.Jit return; } - // We emulate guard pages for software memory access. This makes for an easy transition to - // tracking using host guard pages in future, but also supporting platforms where this is not possible. + // If the memory tracking is coming from the guest, use the tag bits in the page table entry. + // Otherwise, use the managed page flags. - // Write tag includes read protection, since we don't have any read actions that aren't performed before write too. - long tag = (write ? 3L : 1L) << PointerTagBit; - - int pages = GetPagesCount(va, (uint)size, out _); - ulong pageStart = va >> PageBits; - - for (int page = 0; page < pages; page++) + if (guest) { - ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize); + // We emulate guard pages for software memory access. This makes for an easy transition to + // tracking using host guard pages in future, but also supporting platforms where this is not possible. - long pte; + // Write tag includes read protection, since we don't have any read actions that aren't performed before write too. + long tag = (write ? 3L : 1L) << PointerTagBit; - pte = Volatile.Read(ref pageRef); + int pages = GetPagesCount(va, (uint)size, out _); + ulong pageStart = va >> PageBits; - if ((pte & tag) != 0) + for (int page = 0; page < pages; page++) { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } + ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize); - pageStart++; + long pte; + + pte = Volatile.Read(ref pageRef); + + if ((pte & tag) != 0) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId, true); + break; + } + + pageStart++; + } } + else + { + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); + } + } + + /// + public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + SignalMemoryTrackingImpl(va, size, write, false, precise, exemptId); } private ulong PaToPte(ulong pa) diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs index 0b6ba260a..f410d02e9 100644 --- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; -using System.Threading; namespace Ryujinx.Cpu.Jit { @@ -15,21 +14,6 @@ namespace Ryujinx.Cpu.Jit /// public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock { - public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. - public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. - - private enum HostMappedPtBits : ulong - { - Unmapped = 0, - Mapped, - WriteTracked, - ReadWriteTracked, - - MappedReplicated = 0x5555555555555555, - WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, - ReadWriteTrackedReplicated = ulong.MaxValue, - } - private readonly InvalidAccessHandler _invalidAccessHandler; private readonly bool _unsafeMode; @@ -39,7 +23,7 @@ namespace Ryujinx.Cpu.Jit private readonly MemoryEhMeilleure _memoryEh; - private readonly ulong[] _pageBitmap; + private readonly ManagedPageFlags _pages; /// public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize; @@ -81,7 +65,7 @@ namespace Ryujinx.Cpu.Jit AddressSpaceBits = asBits; - _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + _pages = new ManagedPageFlags(AddressSpaceBits); Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler); _memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking); @@ -94,7 +78,7 @@ namespace Ryujinx.Cpu.Jit /// Size of the range in bytes private void AssertMapped(ulong va, ulong size) { - if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size)) + if (!ValidateAddressAndSize(va, size) || !_pages.IsRangeMapped(va, size)) { throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); } @@ -106,7 +90,7 @@ namespace Ryujinx.Cpu.Jit AssertValidAddressAndSize(va, size); _addressSpace.Map(va, pa, size, flags); - AddMapping(va, size); + _pages.AddMapping(va, size); PtMap(va, pa, size); Tracking.Map(va, size); @@ -126,7 +110,7 @@ namespace Ryujinx.Cpu.Jit UnmapEvent?.Invoke(va, size); Tracking.Unmap(va, size); - RemoveMapping(va, size); + _pages.RemoveMapping(va, size); PtUnmap(va, size); _addressSpace.Unmap(va, size); } @@ -337,22 +321,7 @@ namespace Ryujinx.Cpu.Jit [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool IsMapped(ulong va) { - return ValidateAddress(va) && IsMappedImpl(va); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsMappedImpl(ulong va) - { - ulong page = va >> PageBits; - - int bit = (int)((page & 31) << 1); - - int pageIndex = (int)(page >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - - return ((pte >> bit) & 3) != 0; + return ValidateAddress(va) && _pages.IsMapped(va); } /// @@ -360,58 +329,7 @@ namespace Ryujinx.Cpu.Jit { AssertValidAddressAndSize(va, size); - return IsRangeMappedImpl(va, size); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) - { - startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); - endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); - - pageIndex = (int)(pageStart >> PageToPteShift); - pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); - } - - private bool IsRangeMappedImpl(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - - if (pages == 1) - { - return IsMappedImpl(va); - } - - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - // Check if either bit in each 2 bit page entry is set. - // OR the block with itself shifted down by 1, and check the first bit of each entry. - - ulong mask = BlockMappedMask & startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte = Volatile.Read(ref pageRef); - - pte |= pte >> 1; - if ((pte & mask) != mask) - { - return false; - } - - mask = BlockMappedMask; - } - - return true; + return _pages.IsRangeMapped(va, size); } /// @@ -486,76 +404,7 @@ namespace Ryujinx.Cpu.Jit return; } - // Software table, used for managed memory tracking. - - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); - - int bit = (int)((pageStart & 31) << 1); - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - ulong state = ((pte >> bit) & 3); - - if (state >= tag) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - return; - } - else if (state == 0) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte = Volatile.Read(ref pageRef); - ulong mappedMask = mask & BlockMappedMask; - - ulong mappedPte = pte | (pte >> 1); - if ((mappedPte & mappedMask) != mappedMask) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - - pte &= mask; - if ((pte & anyTrackingTag) != 0) // Search for any tracking. - { - // Writes trigger any tracking. - // Only trigger tracking from reads if both bits are set on any page. - if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } - } - - mask = ulong.MaxValue; - } - } + _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId); } /// @@ -582,103 +431,28 @@ namespace Ryujinx.Cpu.Jit } /// - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - int pages = GetPagesCount(va, size, out va); - ulong pageStart = va >> PageBits; - - if (pages == 1) + if (guest) { - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, - _ => (ulong)HostMappedPtBits.ReadWriteTracked, - }; - - int bit = (int)((pageStart & 31) << 1); - - ulong tagMask = 3UL << bit; - ulong invTagMask = ~tagMask; - - ulong tag = protTag << bit; - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + _addressSpace.Base.Reprotect(va, size, protection, false); } else { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, - _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, - }; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Change the protection of all 2 bit entries that are mapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask &= mask; // Only update mapped pages within the given range. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); - - mask = ulong.MaxValue; - } + _pages.TrackingReprotect(va, size, protection); } - - protection = protection switch - { - MemoryPermission.None => MemoryPermission.ReadAndWrite, - MemoryPermission.Write => MemoryPermission.Read, - _ => MemoryPermission.None, - }; - - _addressSpace.Base.Reprotect(va, size, protection, false); } /// - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginTracking(address, size, id); + return Tracking.BeginTracking(address, size, id, flags); } /// - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags); } /// @@ -687,86 +461,6 @@ namespace Ryujinx.Cpu.Jit return Tracking.BeginSmartGranularTracking(address, size, granularity, id); } - /// - /// Adds the given address mapping to the page table. - /// - /// Virtual memory address - /// Size to be mapped - private void AddMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Map all 2-bit entries that are unmapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); - - mask = ulong.MaxValue; - } - } - - /// - /// Removes the given address mapping from the page table. - /// - /// Virtual memory address - /// Size to be unmapped - private void RemoveMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - startMask = ~startMask; - endMask = ~endMask; - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask |= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); - - mask = 0; - } - } - /// /// Disposes of resources used by the memory manager. /// diff --git a/src/Ryujinx.Cpu/ManagedPageFlags.cs b/src/Ryujinx.Cpu/ManagedPageFlags.cs new file mode 100644 index 000000000..a839dae67 --- /dev/null +++ b/src/Ryujinx.Cpu/ManagedPageFlags.cs @@ -0,0 +1,389 @@ +using Ryujinx.Memory; +using Ryujinx.Memory.Tracking; +using System; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Ryujinx.Cpu +{ + /// + /// A page bitmap that keeps track of mapped state and tracking protection + /// for managed memory accesses (not using host page protection). + /// + internal readonly struct ManagedPageFlags + { + public const int PageBits = 12; + public const int PageSize = 1 << PageBits; + public const int PageMask = PageSize - 1; + + private readonly ulong[] _pageBitmap; + + public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. + public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. + + private enum ManagedPtBits : ulong + { + Unmapped = 0, + Mapped, + WriteTracked, + ReadWriteTracked, + + MappedReplicated = 0x5555555555555555, + WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, + ReadWriteTrackedReplicated = ulong.MaxValue, + } + + public ManagedPageFlags(int addressSpaceBits) + { + int bits = Math.Max(0, addressSpaceBits - (PageBits + PageToPteShift)); + _pageBitmap = new ulong[1 << bits]; + } + + /// + /// Computes the number of pages in a virtual address range. + /// + /// Virtual address of the range + /// Size of the range + /// The virtual address of the beginning of the first page + /// This function does not differentiate between allocated and unallocated pages. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + /// + /// Checks if the page at a given CPU virtual address is mapped. + /// + /// Virtual address to check + /// True if the address is mapped, false otherwise + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool IsMapped(ulong va) + { + ulong page = va >> PageBits; + + int bit = (int)((page & 31) << 1); + + int pageIndex = (int)(page >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + + return ((pte >> bit) & 3) != 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) + { + startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); + endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); + + pageIndex = (int)(pageStart >> PageToPteShift); + pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); + } + + /// + /// Checks if a memory range is mapped. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// True if the entire range is mapped, false otherwise + public readonly bool IsRangeMapped(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + + if (pages == 1) + { + return IsMapped(va); + } + + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + // Check if either bit in each 2 bit page entry is set. + // OR the block with itself shifted down by 1, and check the first bit of each entry. + + ulong mask = BlockMappedMask & startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte = Volatile.Read(ref pageRef); + + pte |= pte >> 1; + if ((pte & mask) != mask) + { + return false; + } + + mask = BlockMappedMask; + } + + return true; + } + + /// + /// Reprotect a region of virtual memory for tracking. + /// + /// Virtual address base + /// Size of the region to protect + /// Memory protection to set + public readonly void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + { + // Protection is inverted on software pages, since the default value is 0. + protection = (~protection) & MemoryPermission.ReadAndWrite; + + int pages = GetPagesCount(va, size, out va); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)ManagedPtBits.Mapped, + MemoryPermission.Write => (ulong)ManagedPtBits.WriteTracked, + _ => (ulong)ManagedPtBits.ReadWriteTracked, + }; + + int bit = (int)((pageStart & 31) << 1); + + ulong tagMask = 3UL << bit; + ulong invTagMask = ~tagMask; + + ulong tag = protTag << bit; + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)ManagedPtBits.MappedReplicated, + MemoryPermission.Write => (ulong)ManagedPtBits.WriteTrackedReplicated, + _ => (ulong)ManagedPtBits.ReadWriteTrackedReplicated, + }; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Change the protection of all 2 bit entries that are mapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask &= mask; // Only update mapped pages within the given range. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); + + mask = ulong.MaxValue; + } + } + } + + /// + /// Alerts the memory tracking that a given region has been read from or written to. + /// This should be called before read/write is performed. + /// + /// Memory tracking structure to call when pages are protected + /// Virtual address of the region + /// Size of the region + /// True if the region was written, false if read + /// Optional ID of the handles that should not be signalled + /// + /// This function also validates that the given range is both valid and mapped, and will throw if it is not. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void SignalMemoryTracking(MemoryTracking tracking, ulong va, ulong size, bool write, int? exemptId = null) + { + // Software table, used for managed memory tracking. + + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong tag = (ulong)(write ? ManagedPtBits.WriteTracked : ManagedPtBits.ReadWriteTracked); + + int bit = (int)((pageStart & 31) << 1); + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + ulong state = ((pte >> bit) & 3); + + if (state >= tag) + { + tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); + return; + } + else if (state == 0) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong anyTrackingTag = (ulong)ManagedPtBits.WriteTrackedReplicated; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte = Volatile.Read(ref pageRef); + ulong mappedMask = mask & BlockMappedMask; + + ulong mappedPte = pte | (pte >> 1); + if ((mappedPte & mappedMask) != mappedMask) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + + pte &= mask; + if ((pte & anyTrackingTag) != 0) // Search for any tracking. + { + // Writes trigger any tracking. + // Only trigger tracking from reads if both bits are set on any page. + if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) + { + tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); + break; + } + } + + mask = ulong.MaxValue; + } + } + } + + /// + /// Adds the given address mapping to the page table. + /// + /// Virtual memory address + /// Size to be mapped + public readonly void AddMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Map all 2-bit entries that are unmapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); + + mask = ulong.MaxValue; + } + } + + /// + /// Removes the given address mapping from the page table. + /// + /// Virtual memory address + /// Size to be unmapped + public readonly void RemoveMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + startMask = ~startMask; + endMask = ~endMask; + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask |= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); + + mask = 0; + } + } + + private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + } +} diff --git a/src/Ryujinx.Graphics.Gpu/Image/Pool.cs b/src/Ryujinx.Graphics.Gpu/Image/Pool.cs index 0c3a219de..6ede01971 100644 --- a/src/Ryujinx.Graphics.Gpu/Image/Pool.cs +++ b/src/Ryujinx.Graphics.Gpu/Image/Pool.cs @@ -69,7 +69,7 @@ namespace Ryujinx.Graphics.Gpu.Image Address = address; Size = size; - _memoryTracking = physicalMemory.BeginGranularTracking(address, size, ResourceKind.Pool); + _memoryTracking = physicalMemory.BeginGranularTracking(address, size, ResourceKind.Pool, RegionFlags.None); _memoryTracking.RegisterPreciseAction(address, size, PreciseAction); _modifiedDelegate = RegionModified; } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index e01e5142c..d293060b5 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -128,13 +128,13 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_useGranular) { - _memoryTrackingGranular = physicalMemory.BeginGranularTracking(address, size, ResourceKind.Buffer, baseHandles); + _memoryTrackingGranular = physicalMemory.BeginGranularTracking(address, size, ResourceKind.Buffer, RegionFlags.UnalignedAccess, baseHandles); _memoryTrackingGranular.RegisterPreciseAction(address, size, PreciseAction); } else { - _memoryTracking = physicalMemory.BeginTracking(address, size, ResourceKind.Buffer); + _memoryTracking = physicalMemory.BeginTracking(address, size, ResourceKind.Buffer, RegionFlags.UnalignedAccess); if (baseHandles != null) { diff --git a/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs b/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs index 69a3054a4..cca02bb15 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs @@ -368,10 +368,11 @@ namespace Ryujinx.Graphics.Gpu.Memory /// CPU virtual address of the region /// Size of the region /// Kind of the resource being tracked + /// Region flags /// The memory tracking handle - public RegionHandle BeginTracking(ulong address, ulong size, ResourceKind kind) + public RegionHandle BeginTracking(ulong address, ulong size, ResourceKind kind, RegionFlags flags = RegionFlags.None) { - return _cpuMemory.BeginTracking(address, size, (int)kind); + return _cpuMemory.BeginTracking(address, size, (int)kind, flags); } /// @@ -408,12 +409,19 @@ namespace Ryujinx.Graphics.Gpu.Memory /// CPU virtual address of the region /// Size of the region /// Kind of the resource being tracked + /// Region flags /// Handles to inherit state from or reuse /// Desired granularity of write tracking /// The memory tracking handle - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, ResourceKind kind, IEnumerable handles = null, ulong granularity = 4096) + public MultiRegionHandle BeginGranularTracking( + ulong address, + ulong size, + ResourceKind kind, + RegionFlags flags = RegionFlags.None, + IEnumerable handles = null, + ulong granularity = 4096) { - return _cpuMemory.BeginGranularTracking(address, size, handles, granularity, (int)kind); + return _cpuMemory.BeginGranularTracking(address, size, handles, granularity, (int)kind, flags); } /// diff --git a/src/Ryujinx.Memory/AddressSpaceManager.cs b/src/Ryujinx.Memory/AddressSpaceManager.cs index b953eb306..0a4a95143 100644 --- a/src/Ryujinx.Memory/AddressSpaceManager.cs +++ b/src/Ryujinx.Memory/AddressSpaceManager.cs @@ -392,7 +392,7 @@ namespace Ryujinx.Memory } /// - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest = false) { throw new NotImplementedException(); } diff --git a/src/Ryujinx.Memory/IVirtualMemoryManager.cs b/src/Ryujinx.Memory/IVirtualMemoryManager.cs index 9cf3663cf..557da2f26 100644 --- a/src/Ryujinx.Memory/IVirtualMemoryManager.cs +++ b/src/Ryujinx.Memory/IVirtualMemoryManager.cs @@ -214,6 +214,7 @@ namespace Ryujinx.Memory /// Virtual address base /// Size of the region to protect /// Memory protection to set - void TrackingReprotect(ulong va, ulong size, MemoryPermission protection); + /// True if the protection is for guest access, false otherwise + void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest); } } diff --git a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs index 6febcbbb3..96cb2c5f5 100644 --- a/src/Ryujinx.Memory/Tracking/MemoryTracking.cs +++ b/src/Ryujinx.Memory/Tracking/MemoryTracking.cs @@ -14,9 +14,14 @@ namespace Ryujinx.Memory.Tracking // Only use these from within the lock. private readonly NonOverlappingRangeList _virtualRegions; + // Guest virtual regions are a subset of the normal virtual regions, with potentially different protection + // and expanded area of effect on platforms that don't support misaligned page protection. + private readonly NonOverlappingRangeList _guestVirtualRegions; private readonly int _pageSize; + private readonly bool _singleByteGuestTracking; + /// /// This lock must be obtained when traversing or updating the region-handle hierarchy. /// It is not required when reading dirty flags. @@ -27,16 +32,27 @@ namespace Ryujinx.Memory.Tracking /// Create a new tracking structure for the given "physical" memory block, /// with a given "virtual" memory manager that will provide mappings and virtual memory protection. /// + /// + /// If is true, the memory manager must also support protection on partially + /// unmapped regions without throwing exceptions or dropping protection on the mapped portion. + /// /// Virtual memory manager - /// Physical memory block /// Page size of the virtual memory space - public MemoryTracking(IVirtualMemoryManager memoryManager, int pageSize, InvalidAccessHandler invalidAccessHandler = null) + /// Method to call for invalid memory accesses + /// True if the guest only signals writes for the first byte + public MemoryTracking( + IVirtualMemoryManager memoryManager, + int pageSize, + InvalidAccessHandler invalidAccessHandler = null, + bool singleByteGuestTracking = false) { _memoryManager = memoryManager; _pageSize = pageSize; _invalidAccessHandler = invalidAccessHandler; + _singleByteGuestTracking = singleByteGuestTracking; _virtualRegions = new NonOverlappingRangeList(); + _guestVirtualRegions = new NonOverlappingRangeList(); } private (ulong address, ulong size) PageAlign(ulong address, ulong size) @@ -62,20 +78,25 @@ namespace Ryujinx.Memory.Tracking { ref var overlaps = ref ThreadStaticArray.Get(); - int count = _virtualRegions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + for (int type = 0; type < 2; type++) { - VirtualRegion region = overlaps[i]; + NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - // If the region has been fully remapped, signal that it has been mapped again. - bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); - if (remapped) + int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); + + for (int i = 0; i < count; i++) { - region.SignalMappingChanged(true); - } + VirtualRegion region = overlaps[i]; - region.UpdateProtection(); + // If the region has been fully remapped, signal that it has been mapped again. + bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); + if (remapped) + { + region.SignalMappingChanged(true); + } + + region.UpdateProtection(); + } } } } @@ -95,27 +116,58 @@ namespace Ryujinx.Memory.Tracking { ref var overlaps = ref ThreadStaticArray.Get(); - int count = _virtualRegions.FindOverlapsNonOverlapping(va, size, ref overlaps); - - for (int i = 0; i < count; i++) + for (int type = 0; type < 2; type++) { - VirtualRegion region = overlaps[i]; + NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; - region.SignalMappingChanged(false); + int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); + + for (int i = 0; i < count; i++) + { + VirtualRegion region = overlaps[i]; + + region.SignalMappingChanged(false); + } } } } + /// + /// Alter a tracked memory region to properly capture unaligned accesses. + /// For most memory manager modes, this does nothing. + /// + /// Original region address + /// Original region size + /// A new address and size for tracking unaligned accesses + internal (ulong newAddress, ulong newSize) GetUnalignedSafeRegion(ulong address, ulong size) + { + if (_singleByteGuestTracking) + { + // The guest only signals the first byte of each memory access with the current memory manager. + // To catch unaligned access properly, we need to also protect the page before the address. + + // Assume that the address and size are already aligned. + + return (address - (ulong)_pageSize, size + (ulong)_pageSize); + } + else + { + return (address, size); + } + } + /// /// Get a list of virtual regions that a handle covers. /// /// Starting virtual memory address of the handle /// Size of the handle's memory region + /// True if getting handles for guest protection, false otherwise /// A list of virtual regions within the given range - internal List GetVirtualRegionsForHandle(ulong va, ulong size) + internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) { List result = new(); - _virtualRegions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size)); + NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; + regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); return result; } @@ -126,7 +178,14 @@ namespace Ryujinx.Memory.Tracking /// Region to remove internal void RemoveVirtual(VirtualRegion region) { - _virtualRegions.Remove(region); + if (region.Guest) + { + _guestVirtualRegions.Remove(region); + } + else + { + _virtualRegions.Remove(region); + } } /// @@ -137,10 +196,11 @@ namespace Ryujinx.Memory.Tracking /// Handles to inherit state from or reuse. When none are present, provide null /// Desired granularity of write tracking /// Handle ID + /// Region flags /// The memory tracking handle - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { - return new MultiRegionHandle(this, address, size, handles, granularity, id); + return new MultiRegionHandle(this, address, size, handles, granularity, id, flags); } /// @@ -164,15 +224,16 @@ namespace Ryujinx.Memory.Tracking /// CPU virtual address of the region /// Size of the region /// Handle ID + /// Region flags /// The memory tracking handle - public RegionHandle BeginTracking(ulong address, ulong size, int id) + public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { var (paAddress, paSize) = PageAlign(address, size); lock (TrackingLock) { bool mapped = _memoryManager.IsRangeMapped(address, size); - RegionHandle handle = new(this, paAddress, paSize, address, size, id, mapped); + RegionHandle handle = new(this, paAddress, paSize, address, size, id, flags, mapped); return handle; } @@ -186,15 +247,16 @@ namespace Ryujinx.Memory.Tracking /// The bitmap owning the dirty flag for this handle /// The bit of this handle within the dirty flag /// Handle ID + /// Region flags /// The memory tracking handle - internal RegionHandle BeginTrackingBitmap(ulong address, ulong size, ConcurrentBitmap bitmap, int bit, int id) + internal RegionHandle BeginTrackingBitmap(ulong address, ulong size, ConcurrentBitmap bitmap, int bit, int id, RegionFlags flags = RegionFlags.None) { var (paAddress, paSize) = PageAlign(address, size); lock (TrackingLock) { bool mapped = _memoryManager.IsRangeMapped(address, size); - RegionHandle handle = new(this, paAddress, paSize, address, size, bitmap, bit, id, mapped); + RegionHandle handle = new(this, paAddress, paSize, address, size, bitmap, bit, id, flags, mapped); return handle; } @@ -202,6 +264,7 @@ namespace Ryujinx.Memory.Tracking /// /// Signal that a virtual memory event happened at the given location. + /// The memory event is assumed to be triggered by guest code. /// /// Virtual address accessed /// Size of the region affected in bytes @@ -209,7 +272,7 @@ namespace Ryujinx.Memory.Tracking /// True if the event triggered any tracking regions, false otherwise public bool VirtualMemoryEvent(ulong address, ulong size, bool write) { - return VirtualMemoryEvent(address, size, write, precise: false, null); + return VirtualMemoryEvent(address, size, write, precise: false, exemptId: null, guest: true); } /// @@ -222,8 +285,9 @@ namespace Ryujinx.Memory.Tracking /// Whether the region was written to or read /// True if the access is precise, false otherwise /// Optional ID that of the handles that should not be signalled + /// True if the access is from the guest, false otherwise /// True if the event triggered any tracking regions, false otherwise - public bool VirtualMemoryEvent(ulong address, ulong size, bool write, bool precise, int? exemptId = null) + public bool VirtualMemoryEvent(ulong address, ulong size, bool write, bool precise, int? exemptId = null, bool guest = false) { // Look up the virtual region using the region list. // Signal up the chain to relevant handles. @@ -234,7 +298,9 @@ namespace Ryujinx.Memory.Tracking { ref var overlaps = ref ThreadStaticArray.Get(); - int count = _virtualRegions.FindOverlapsNonOverlapping(address, size, ref overlaps); + NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; + + int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); if (count == 0 && !precise) { @@ -242,7 +308,7 @@ namespace Ryujinx.Memory.Tracking { // TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // This code handles that case when it happens, but it would be better to find out how this happens. - _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite); + _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); return true; // This memory _should_ be mapped, so we need to try again. } else @@ -252,6 +318,12 @@ namespace Ryujinx.Memory.Tracking } else { + if (guest && _singleByteGuestTracking) + { + // Increase the access size to trigger handles with misaligned accesses. + size += (ulong)_pageSize; + } + for (int i = 0; i < count; i++) { VirtualRegion region = overlaps[i]; @@ -285,9 +357,10 @@ namespace Ryujinx.Memory.Tracking /// /// Region to reprotect /// Memory permission to protect with - internal void ProtectVirtualRegion(VirtualRegion region, MemoryPermission permission) + /// True if the protection is for guest access, false otherwise + internal void ProtectVirtualRegion(VirtualRegion region, MemoryPermission permission, bool guest) { - _memoryManager.TrackingReprotect(region.Address, region.Size, permission); + _memoryManager.TrackingReprotect(region.Address, region.Size, permission, guest); } /// diff --git a/src/Ryujinx.Memory/Tracking/MultiRegionHandle.cs b/src/Ryujinx.Memory/Tracking/MultiRegionHandle.cs index 1c1b48b3c..6fdca69f5 100644 --- a/src/Ryujinx.Memory/Tracking/MultiRegionHandle.cs +++ b/src/Ryujinx.Memory/Tracking/MultiRegionHandle.cs @@ -37,7 +37,8 @@ namespace Ryujinx.Memory.Tracking ulong size, IEnumerable handles, ulong granularity, - int id) + int id, + RegionFlags flags) { _handles = new RegionHandle[(size + granularity - 1) / granularity]; Granularity = granularity; @@ -62,7 +63,7 @@ namespace Ryujinx.Memory.Tracking // Fill any gap left before this handle. while (i < startIndex) { - RegionHandle fillHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i, id); + RegionHandle fillHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i, id, flags); fillHandle.Parent = this; _handles[i++] = fillHandle; } @@ -83,7 +84,7 @@ namespace Ryujinx.Memory.Tracking while (i < endIndex) { - RegionHandle splitHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i, id); + RegionHandle splitHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i, id, flags); splitHandle.Parent = this; splitHandle.Reprotect(handle.Dirty); @@ -106,7 +107,7 @@ namespace Ryujinx.Memory.Tracking // Fill any remaining space with new handles. while (i < _handles.Length) { - RegionHandle handle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i, id); + RegionHandle handle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i, id, flags); handle.Parent = this; _handles[i++] = handle; } diff --git a/src/Ryujinx.Memory/Tracking/RegionFlags.cs b/src/Ryujinx.Memory/Tracking/RegionFlags.cs new file mode 100644 index 000000000..ceb8e56a6 --- /dev/null +++ b/src/Ryujinx.Memory/Tracking/RegionFlags.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Ryujinx.Memory.Tracking +{ + [Flags] + public enum RegionFlags + { + None = 0, + + /// + /// Access to the resource is expected to occasionally be unaligned. + /// With some memory managers, guest protection must extend into the previous page to cover unaligned access. + /// If this is not expected, protection is not altered, which can avoid unintended resource dirty/flush. + /// + UnalignedAccess = 1, + } +} diff --git a/src/Ryujinx.Memory/Tracking/RegionHandle.cs b/src/Ryujinx.Memory/Tracking/RegionHandle.cs index df3d9c311..a94ffa43c 100644 --- a/src/Ryujinx.Memory/Tracking/RegionHandle.cs +++ b/src/Ryujinx.Memory/Tracking/RegionHandle.cs @@ -55,6 +55,8 @@ namespace Ryujinx.Memory.Tracking private RegionSignal _preAction; // Action to perform before a read or write. This will block the memory access. private PreciseRegionSignal _preciseAction; // Action to perform on a precise read or write. private readonly List _regions; + private readonly List _guestRegions; + private readonly List _allRegions; private readonly MemoryTracking _tracking; private bool _disposed; @@ -99,6 +101,7 @@ namespace Ryujinx.Memory.Tracking /// The bitmap the dirty flag for this handle is stored in /// The bit index representing the dirty flag for this handle /// Handle ID + /// Region flags /// True if the region handle starts mapped internal RegionHandle( MemoryTracking tracking, @@ -109,6 +112,7 @@ namespace Ryujinx.Memory.Tracking ConcurrentBitmap bitmap, int bit, int id, + RegionFlags flags, bool mapped = true) { Bitmap = bitmap; @@ -128,11 +132,12 @@ namespace Ryujinx.Memory.Tracking RealEndAddress = realAddress + realSize; _tracking = tracking; - _regions = tracking.GetVirtualRegionsForHandle(address, size); - foreach (var region in _regions) - { - region.Handles.Add(this); - } + + _regions = tracking.GetVirtualRegionsForHandle(address, size, false); + _guestRegions = GetGuestRegions(tracking, address, size, flags); + _allRegions = new List(_regions.Count + _guestRegions.Count); + + InitializeRegions(); } /// @@ -145,8 +150,9 @@ namespace Ryujinx.Memory.Tracking /// The real, unaligned address of the handle /// The real, unaligned size of the handle /// Handle ID + /// Region flags /// True if the region handle starts mapped - internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, ulong realAddress, ulong realSize, int id, bool mapped = true) + internal RegionHandle(MemoryTracking tracking, ulong address, ulong size, ulong realAddress, ulong realSize, int id, RegionFlags flags, bool mapped = true) { Bitmap = new ConcurrentBitmap(1, mapped); @@ -163,8 +169,37 @@ namespace Ryujinx.Memory.Tracking RealEndAddress = realAddress + realSize; _tracking = tracking; - _regions = tracking.GetVirtualRegionsForHandle(address, size); - foreach (var region in _regions) + + _regions = tracking.GetVirtualRegionsForHandle(address, size, false); + _guestRegions = GetGuestRegions(tracking, address, size, flags); + _allRegions = new List(_regions.Count + _guestRegions.Count); + + InitializeRegions(); + } + + private List GetGuestRegions(MemoryTracking tracking, ulong address, ulong size, RegionFlags flags) + { + ulong guestAddress; + ulong guestSize; + + if (flags.HasFlag(RegionFlags.UnalignedAccess)) + { + (guestAddress, guestSize) = tracking.GetUnalignedSafeRegion(address, size); + } + else + { + (guestAddress, guestSize) = (address, size); + } + + return tracking.GetVirtualRegionsForHandle(guestAddress, guestSize, true); + } + + private void InitializeRegions() + { + _allRegions.AddRange(_regions); + _allRegions.AddRange(_guestRegions); + + foreach (var region in _allRegions) { region.Handles.Add(this); } @@ -321,7 +356,7 @@ namespace Ryujinx.Memory.Tracking lock (_tracking.TrackingLock) { - foreach (VirtualRegion region in _regions) + foreach (VirtualRegion region in _allRegions) { protectionChanged |= region.UpdateProtection(); } @@ -379,7 +414,7 @@ namespace Ryujinx.Memory.Tracking { lock (_tracking.TrackingLock) { - foreach (VirtualRegion region in _regions) + foreach (VirtualRegion region in _allRegions) { region.UpdateProtection(); } @@ -414,7 +449,16 @@ namespace Ryujinx.Memory.Tracking /// Virtual region to add as a child internal void AddChild(VirtualRegion region) { - _regions.Add(region); + if (region.Guest) + { + _guestRegions.Add(region); + } + else + { + _regions.Add(region); + } + + _allRegions.Add(region); } /// @@ -469,7 +513,7 @@ namespace Ryujinx.Memory.Tracking lock (_tracking.TrackingLock) { - foreach (VirtualRegion region in _regions) + foreach (VirtualRegion region in _allRegions) { region.RemoveHandle(this); } diff --git a/src/Ryujinx.Memory/Tracking/VirtualRegion.cs b/src/Ryujinx.Memory/Tracking/VirtualRegion.cs index 538e94fef..bb087e9af 100644 --- a/src/Ryujinx.Memory/Tracking/VirtualRegion.cs +++ b/src/Ryujinx.Memory/Tracking/VirtualRegion.cs @@ -13,10 +13,14 @@ namespace Ryujinx.Memory.Tracking private readonly MemoryTracking _tracking; private MemoryPermission _lastPermission; - public VirtualRegion(MemoryTracking tracking, ulong address, ulong size, MemoryPermission lastPermission = MemoryPermission.Invalid) : base(address, size) + public bool Guest { get; } + + public VirtualRegion(MemoryTracking tracking, ulong address, ulong size, bool guest, MemoryPermission lastPermission = MemoryPermission.Invalid) : base(address, size) { _lastPermission = lastPermission; _tracking = tracking; + + Guest = guest; } /// @@ -103,7 +107,7 @@ namespace Ryujinx.Memory.Tracking if (_lastPermission != permission) { - _tracking.ProtectVirtualRegion(this, permission); + _tracking.ProtectVirtualRegion(this, permission, Guest); _lastPermission = permission; return true; @@ -131,7 +135,7 @@ namespace Ryujinx.Memory.Tracking public override INonOverlappingRange Split(ulong splitAddress) { - VirtualRegion newRegion = new(_tracking, splitAddress, EndAddress - splitAddress, _lastPermission); + VirtualRegion newRegion = new(_tracking, splitAddress, EndAddress - splitAddress, Guest, _lastPermission); Size = splitAddress - Address; // The new region inherits all of our parents. diff --git a/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs b/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs index 5c8396e6d..85a1ac02b 100644 --- a/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs +++ b/src/Ryujinx.Tests.Memory/MockVirtualMemoryManager.cs @@ -107,7 +107,7 @@ namespace Ryujinx.Tests.Memory throw new NotImplementedException(); } - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest) { OnProtect?.Invoke(va, size, protection); }