diff --git a/Ryujinx.Cpu/Jit/MappingTree.cs b/Ryujinx.Cpu/Jit/MappingTree.cs new file mode 100644 index 000000000..278e2ebe5 --- /dev/null +++ b/Ryujinx.Cpu/Jit/MappingTree.cs @@ -0,0 +1,353 @@ +using Ryujinx.Common.Collections; +using Ryujinx.Memory; +using Ryujinx.Memory.Range; +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Ryujinx.Cpu.Jit +{ + class MappingTree + { + private const ulong PageSize = 0x1000; + + private enum MappingState : byte + { + Unmapped, + Mapped, + MappedWithMirror + } + + private class Mapping : IntrusiveRedBlackTreeNode, IComparable + { + public ulong Address { get; private set; } + public ulong Size { get; private set; } + public ulong EndAddress => Address + Size; + public ulong BackingOffset { get; private set; } + public MappingState State { get; private set; } + + public Mapping(ulong address, ulong size, ulong backingOffset, MappingState state) + { + Address = address; + Size = size; + BackingOffset = backingOffset; + State = state; + } + + public Mapping Split(ulong splitAddress) + { + ulong leftSize = splitAddress - Address; + ulong rightSize = EndAddress - splitAddress; + + Mapping left = new Mapping(Address, leftSize, BackingOffset, State); + + Address = splitAddress; + Size = rightSize; + + if (State != MappingState.Unmapped) + { + BackingOffset += leftSize; + } + + return left; + } + + public void UpdateState(ulong newBackingOffset, MappingState newState) + { + BackingOffset = newBackingOffset; + State = newState; + } + + public void Extend(ulong sizeDelta) + { + Size += sizeDelta; + } + + public int CompareTo(Mapping other) + { + if (Address < other.Address) + { + return -1; + } + else if (Address <= other.EndAddress - 1UL) + { + return 0; + } + else + { + return 1; + } + } + } + + private readonly IntrusiveRedBlackTree _tree; + private readonly ReaderWriterLock _treeLock; + + public MappingTree(ulong addressSpaceSize) + { + _tree = new IntrusiveRedBlackTree(); + _treeLock = new ReaderWriterLock(); + + _tree.Add(new Mapping(0UL, addressSpaceSize, 0UL, MappingState.Unmapped)); + } + + public void Map(ulong va, ulong pa, ulong size) + { + _treeLock.AcquireWriterLock(Timeout.Infinite); + Update(va, pa, size, MappingState.Mapped); + _treeLock.ReleaseWriterLock(); + } + + public void Unmap(ulong va, ulong size) + { + _treeLock.AcquireWriterLock(Timeout.Infinite); + Update(va, 0UL, size, MappingState.Unmapped); + _treeLock.ReleaseWriterLock(); + } + + public IEnumerable GetPhysicalRegions(ulong va, ulong size) + { + _treeLock.AcquireReaderLock(Timeout.Infinite); + var regions = GetPhysicalRegionsImpl(va, size); + _treeLock.ReleaseReaderLock(); + + return regions; + } + + public bool TryGetContiguousPa(ulong va, ulong size, out ulong pa) + { + _treeLock.AcquireReaderLock(Timeout.Infinite); + bool result = TryGetContiguousPaImpl(va, size, out pa); + _treeLock.ReleaseReaderLock(); + + return result; + } + + public (MemoryBlock, ulong) GetContiguousBlock(MemoryBlock backingMemory, MemoryBlock mirror, ulong va, ulong size) + { + _treeLock.AcquireReaderLock(Timeout.Infinite); + var result = GetContiguousBlockImpl(backingMemory, mirror, va, size); + _treeLock.ReleaseReaderLock(); + + return result; + } + + private void Update(ulong va, ulong pa, ulong size, MappingState state) + { + Mapping map = _tree.GetNode(new Mapping(va, 1UL, 0UL, MappingState.Unmapped)); + + Update(map, va, pa, size, state); + } + + private Mapping Update(Mapping map, ulong va, ulong pa, ulong size, MappingState state) + { + ulong endAddress = va + size; + + for (; map != null; map = map.Successor) + { + if (map.Address < va) + { + _tree.Add(map.Split(va)); + } + + if (map.EndAddress > endAddress) + { + Mapping newMap = map.Split(endAddress); + _tree.Add(newMap); + map = newMap; + } + + map.UpdateState(pa, state); + map = TryCoalesce(map); + + if (map.EndAddress >= endAddress) + { + break; + } + } + + return map; + } + + private Mapping TryCoalesce(Mapping map) + { + Mapping previousMap = map.Predecessor; + Mapping nextMap = map.Successor; + + if (previousMap != null && CanCoalesce(previousMap, map)) + { + previousMap.Extend(map.Size); + _tree.Remove(map); + map = previousMap; + } + + if (nextMap != null && CanCoalesce(map, nextMap)) + { + map.Extend(nextMap.Size); + _tree.Remove(nextMap); + } + + return map; + } + + private static bool CanCoalesce(Mapping left, Mapping right) + { + if (left.State != right.State) + { + return false; + } + + return left.State == MappingState.Unmapped || (left.BackingOffset + left.Size == right.BackingOffset); + } + + private IEnumerable GetPhysicalRegionsImpl(ulong va, ulong size) + { + Mapping map = _tree.GetNode(new Mapping(va, 1UL, 0UL, MappingState.Unmapped)); + + if (map == null) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + + var regions = new List(); + + ulong endAddress = va + size; + ulong regionStart = 0; + ulong regionSize = 0; + + for (; map != null; map = map.Successor) + { + if (map.State == MappingState.Unmapped) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + + ulong clampedAddress = Math.Max(map.Address, va); + ulong clampedEndAddress = Math.Min(map.EndAddress, endAddress); + ulong clampedSize = clampedEndAddress - clampedAddress; + + ulong pa = map.BackingOffset + (clampedAddress - map.Address); + + if (pa != regionStart + regionSize) + { + if (regionSize != 0) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + } + + regionStart = pa; + regionSize = clampedSize; + } + else + { + regionSize += clampedSize; + } + + if (map.EndAddress >= endAddress) + { + break; + } + } + + if (regionSize != 0) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + } + + return regions; + } + + private bool TryGetContiguousPaImpl(ulong va, ulong size, out ulong pa) + { + Mapping map = _tree.GetNode(new Mapping(va, 1UL, 0UL, MappingState.Unmapped)); + + ulong endAddress = va + size; + + if (map != null && map.Address <= va && map.EndAddress >= endAddress) + { + pa = map.BackingOffset + (va - map.Address); + return true; + } + + pa = 0; + return false; + } + + private (MemoryBlock, ulong) GetContiguousBlockImpl(MemoryBlock backingMemory, MemoryBlock mirror, ulong va, ulong size) + { + Mapping map = _tree.GetNode(new Mapping(va, 1UL, 0UL, MappingState.Unmapped)); + + ulong endAddress = va + size; + + if (map != null && map.Address <= va && map.EndAddress >= endAddress) + { + ulong pa = map.BackingOffset + (va - map.Address); + return (backingMemory, pa); + } + + if (map != null) + { + Mapping firstMap = map; + + bool contiguous = true; + ulong expectedPa = map.BackingOffset + map.Size; + + while ((map = map.Successor) != null && map.Address < endAddress) + { + if (map.State == MappingState.Unmapped || map.BackingOffset != expectedPa) + { + contiguous = false; + break; + } + + if (map.EndAddress >= endAddress) + { + break; + } + + expectedPa = map.BackingOffset + map.Size; + } + + if (contiguous && map != null) + { + ulong pa = firstMap.BackingOffset + (va - firstMap.Address); + return (backingMemory, pa); + } + + map = firstMap; + } + + ulong endVaAligned = (endAddress + PageSize - 1) & ~(PageSize - 1); + ulong vaAligned = va & ~(PageSize - 1); + + // Make sure the range that will be accessed on the mirror is fully mapped. + for (; map != null; map = map.Successor) + { + if (map.State == MappingState.Mapped) + { + ulong clampedAddress = Math.Max(map.Address, vaAligned); + ulong clampedEndAddress = Math.Min(map.EndAddress, endVaAligned); + ulong clampedSize = clampedEndAddress - clampedAddress; + ulong backingOffset = map.BackingOffset + (clampedAddress - map.Address); + + LockCookie lockCookie = _treeLock.UpgradeToWriterLock(Timeout.Infinite); + + mirror.MapView(backingMemory, backingOffset, clampedAddress, clampedSize); + + map = Update(map, clampedAddress, backingOffset, clampedSize, MappingState.MappedWithMirror); + + _treeLock.DowngradeFromWriterLock(ref lockCookie); + } + + if (map.EndAddress >= endAddress) + { + break; + } + } + + return (mirror, va); + } + + private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs index 4df29699d..a183f95e3 100644 --- a/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs +++ b/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs @@ -42,7 +42,7 @@ namespace Ryujinx.Cpu.Jit private readonly ulong _addressSpaceSize; private readonly MemoryBlock _backingMemory; - private readonly PageTable _pageTable; + private readonly MappingTree _mappingTree; private readonly MemoryEhMeilleure _memoryEh; @@ -68,7 +68,6 @@ namespace Ryujinx.Cpu.Jit public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize, bool unsafeMode, InvalidAccessHandler invalidAccessHandler = null) { _backingMemory = backingMemory; - _pageTable = new PageTable(); _invalidAccessHandler = invalidAccessHandler; _unsafeMode = unsafeMode; _addressSpaceSize = addressSpaceSize; @@ -84,6 +83,8 @@ namespace Ryujinx.Cpu.Jit AddressSpaceBits = asBits; + _mappingTree = new MappingTree(asSize); + _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; MemoryAllocationFlags asFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible; @@ -150,9 +151,8 @@ namespace Ryujinx.Cpu.Jit AssertValidAddressAndSize(va, size); _addressSpace.MapView(_backingMemory, pa, va, size); - _addressSpaceMirror.MapView(_backingMemory, pa, va, size); AddMapping(va, size); - PtMap(va, pa, size); + _mappingTree.Map(va, pa, size); Tracking.Map(va, size); } @@ -166,34 +166,11 @@ namespace Ryujinx.Cpu.Jit Tracking.Unmap(va, size); RemoveMapping(va, size); - PtUnmap(va, size); + _mappingTree.Unmap(va, size); _addressSpace.UnmapView(_backingMemory, va, size); _addressSpaceMirror.UnmapView(_backingMemory, va, size); } - private void PtMap(ulong va, ulong pa, ulong size) - { - while (size != 0) - { - _pageTable.Map(va, pa); - - va += PageSize; - pa += PageSize; - size -= PageSize; - } - } - - private void PtUnmap(ulong va, ulong size) - { - while (size != 0) - { - _pageTable.Unmap(va); - - va += PageSize; - size -= PageSize; - } - } - /// public T Read(ulong va) where T : unmanaged { @@ -201,7 +178,8 @@ namespace Ryujinx.Cpu.Jit { AssertMapped(va, (ulong)Unsafe.SizeOf()); - return _addressSpaceMirror.Read(va); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)Unsafe.SizeOf()); + return block.Read(offset); } catch (InvalidMemoryRegionException) { @@ -241,7 +219,8 @@ namespace Ryujinx.Cpu.Jit { AssertMapped(va, (ulong)data.Length); - _addressSpaceMirror.Read(va, data); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)data.Length); + block.Read(offset, data); } catch (InvalidMemoryRegionException) { @@ -260,7 +239,8 @@ namespace Ryujinx.Cpu.Jit { SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), write: true); - _addressSpaceMirror.Write(va, value); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)Unsafe.SizeOf()); + block.Write(offset, value); } catch (InvalidMemoryRegionException) { @@ -274,10 +254,12 @@ namespace Ryujinx.Cpu.Jit /// public void Write(ulong va, ReadOnlySpan data) { - try { + try + { SignalMemoryTracking(va, (ulong)data.Length, write: true); - _addressSpaceMirror.Write(va, data); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)data.Length); + block.Write(offset, data); } catch (InvalidMemoryRegionException) { @@ -295,7 +277,8 @@ namespace Ryujinx.Cpu.Jit { AssertMapped(va, (ulong)data.Length); - _addressSpaceMirror.Write(va, data); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)data.Length); + block.Write(offset, data); } catch (InvalidMemoryRegionException) { @@ -318,7 +301,8 @@ namespace Ryujinx.Cpu.Jit AssertMapped(va, (ulong)size); } - return _addressSpaceMirror.GetSpan(va, size); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)size); + return block.GetSpan(offset, size); } /// @@ -333,7 +317,8 @@ namespace Ryujinx.Cpu.Jit AssertMapped(va, (ulong)size); } - return _addressSpaceMirror.GetWritableRegion(va, size); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)size); + return block.GetWritableRegion(offset, size); } /// @@ -341,7 +326,8 @@ namespace Ryujinx.Cpu.Jit { SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); - return ref _addressSpaceMirror.GetRef(va); + (MemoryBlock block, ulong offset) = GetContiguousBlock(va, (ulong)Unsafe.SizeOf()); + return ref block.GetRef(offset); } /// @@ -428,51 +414,7 @@ namespace Ryujinx.Cpu.Jit /// public IEnumerable GetPhysicalRegions(ulong va, ulong size) { - int pages = GetPagesCount(va, (uint)size, out va); - - var regions = new List(); - - ulong regionStart = GetPhysicalAddressChecked(va); - ulong regionSize = PageSize; - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return null; - } - - ulong newPa = GetPhysicalAddressChecked(va + PageSize); - - if (GetPhysicalAddressChecked(va) + PageSize != newPa) - { - regions.Add(new MemoryRange(regionStart, regionSize)); - regionStart = newPa; - regionSize = 0; - } - - va += PageSize; - regionSize += PageSize; - } - - regions.Add(new MemoryRange(regionStart, regionSize)); - - return regions; - } - - private ulong GetPhysicalAddressChecked(ulong va) - { - if (!IsMapped(va)) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); - } - - return GetPhysicalAddressInternal(va); - } - - private ulong GetPhysicalAddressInternal(ulong va) - { - return _pageTable.Read(va) + (va & PageMask); + return _mappingTree.GetPhysicalRegions(va, size); } /// @@ -684,6 +626,11 @@ namespace Ryujinx.Cpu.Jit return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity)); } + private (MemoryBlock, ulong) GetContiguousBlock(ulong va, ulong size) + { + return _mappingTree.GetContiguousBlock(_backingMemory, _addressSpaceMirror, va, size); + } + /// /// Adds the given address mapping to the page table. ///