using Ryujinx.Cpu.AppleHv.Arm; using System; using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading; namespace Ryujinx.Cpu.AppleHv { class HvAddressSpaceRange : IDisposable { private const ulong AllocationGranule = 1UL << 14; private const ulong AttributesMask = (0x3ffUL << 2) | (0x3fffUL << 50); private const ulong BaseAttributes = (1UL << 10) | (3UL << 8); // Access flag set, inner shareable. private const int LevelBits = 9; private const int LevelCount = 1 << LevelBits; private const int LevelMask = LevelCount - 1; private const int PageBits = 12; private const int PageSize = 1 << PageBits; private const int PageMask = PageSize - 1; private const int AllLevelsMask = PageMask | (LevelMask << PageBits) | (LevelMask << (PageBits + LevelBits)); private class PtLevel { public ulong Address => Allocation.Ipa + Allocation.Offset; public int EntriesCount; public readonly HvMemoryBlockAllocation Allocation; public readonly PtLevel[] Next; public PtLevel(HvMemoryBlockAllocator blockAllocator, int count, bool hasNext) { ulong size = (ulong)count * sizeof(ulong); Allocation = blockAllocator.Allocate(size, PageSize); AsSpan().Fill(0UL); if (hasNext) { Next = new PtLevel[count]; } } public unsafe Span AsSpan() { return MemoryMarshal.Cast(Allocation.Memory.GetSpan(Allocation.Offset, (int)Allocation.Size)); } } private PtLevel _level0; private int _tlbInvalidationPending; private readonly HvIpaAllocator _ipaAllocator; private readonly HvMemoryBlockAllocator _blockAllocator; public HvAddressSpaceRange(HvIpaAllocator ipaAllocator) { _ipaAllocator = ipaAllocator; _blockAllocator = new HvMemoryBlockAllocator(ipaAllocator, (int)AllocationGranule); } public ulong GetIpaBase() { return EnsureLevel0().Address; } public bool GetAndClearTlbInvalidationPending() { return Interlocked.Exchange(ref _tlbInvalidationPending, 0) != 0; } public void Map(ulong va, ulong pa, ulong size, ApFlags accessPermission) { MapImpl(va, pa, size, (ulong)accessPermission | BaseAttributes); } public void Unmap(ulong va, ulong size) { UnmapImpl(EnsureLevel0(), 0, va, size); Interlocked.Exchange(ref _tlbInvalidationPending, 1); } public void Reprotect(ulong va, ulong size, ApFlags accessPermission) { UpdateAttributes(va, size, (ulong)accessPermission | BaseAttributes); } private void MapImpl(ulong va, ulong pa, ulong size, ulong attr) { PtLevel level0 = EnsureLevel0(); ulong endVa = va + size; while (va < endVa) { (ulong mapSize, int depth) = GetMapSizeAndDepth(va, pa, endVa); PtLevel currentLevel = level0; for (int i = 0; i < depth; i++) { int l = (int)(va >> (PageBits + (2 - i) * LevelBits)) & LevelMask; EnsureTable(currentLevel, l, i == 0); currentLevel = currentLevel.Next[l]; } (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); for (ulong i = 0; i < mapSize; i += blockSize) { if ((va >> blockShift) << blockShift != va || (pa >> blockShift) << blockShift != pa) { Debug.Fail($"Block size 0x{blockSize:X} (log2: {blockShift}) is invalid for VA 0x{va:X} or PA 0x{pa:X}."); } WriteBlock(currentLevel, (int)(va >> blockShift) & LevelMask, depth, pa, attr); va += blockSize; pa += blockSize; } } } private void UnmapImpl(PtLevel level, int depth, ulong va, ulong size) { ulong endVa = (va + size + PageMask) & ~((ulong)PageMask); va &= ~((ulong)PageMask); (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); while (va < endVa) { ulong nextEntryVa = GetNextAddress(va, blockSize); ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va); int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask; PtLevel nextTable = level.Next != null ? level.Next[l] : null; if (nextTable != null) { // Entry is a table, visit it and update attributes as required. UnmapImpl(nextTable, depth + 1, va, chunckSize); } else if (chunckSize != blockSize) { // Entry is a block but is not aligned, we need to turn it into a table. ref ulong pte = ref level.AsSpan()[l]; nextTable = CreateTable(pte, depth + 1); level.Next[l] = nextTable; // Now that we have a table, we can handle it like the first case. UnmapImpl(nextTable, depth + 1, va, chunckSize); // Update PTE to point to the new table. pte = (nextTable.Address & ~(ulong)PageMask) | 3UL; } // If entry is a block, or if entry is a table but it is empty, we can remove it. if (nextTable == null || nextTable.EntriesCount == 0) { // Entry is a block and is fully aligned, so we can just set it to 0. if (nextTable != null) { nextTable.Allocation.Dispose(); level.Next[l] = null; } level.AsSpan()[l] = 0UL; level.EntriesCount--; ValidateEntriesCount(level.EntriesCount); } va += chunckSize; } } private void UpdateAttributes(ulong va, ulong size, ulong newAttr) { UpdateAttributes(EnsureLevel0(), 0, va, size, newAttr); Interlocked.Exchange(ref _tlbInvalidationPending, 1); } private void UpdateAttributes(PtLevel level, int depth, ulong va, ulong size, ulong newAttr) { ulong endVa = (va + size + PageSize - 1) & ~((ulong)PageSize - 1); va &= ~((ulong)PageSize - 1); (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); while (va < endVa) { ulong nextEntryVa = GetNextAddress(va, blockSize); ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va); int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask; ref ulong pte = ref level.AsSpan()[l]; // First check if the region is mapped. if ((pte & 3) != 0) { PtLevel nextTable = level.Next != null ? level.Next[l] : null; if (nextTable != null) { // Entry is a table, visit it and update attributes as required. UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr); } else if (chunckSize != blockSize) { // Entry is a block but is not aligned, we need to turn it into a table. nextTable = CreateTable(pte, depth + 1); level.Next[l] = nextTable; // Now that we have a table, we can handle it like the first case. UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr); // Update PTE to point to the new table. pte = (nextTable.Address & ~(ulong)PageMask) | 3UL; } else { // Entry is a block and is fully aligned, so we can just update the attributes. // Update PTE with the new attributes. pte = (pte & ~AttributesMask) | newAttr; } } va += chunckSize; } } private PtLevel CreateTable(ulong pte, int depth) { pte &= ~3UL; pte |= (depth == 2 ? 3UL : 1UL); PtLevel level = new PtLevel(_blockAllocator, LevelCount, depth < 2); Span currentLevel = level.AsSpan(); (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); // Fill in the blocks. for (int i = 0; i < LevelCount; i++) { ulong offset = (ulong)i << blockShift; currentLevel[i] = pte + offset; } level.EntriesCount = LevelCount; return level; } private static (ulong, int) GetBlockSizeAndShift(int depth) { int blockShift = PageBits + (2 - depth) * LevelBits; ulong blockSize = 1UL << blockShift; return (blockSize, blockShift); } private static (ulong, int) GetMapSizeAndDepth(ulong va, ulong pa, ulong endVa) { // Both virtual and physical addresses must be aligned to the block size. ulong combinedAddress = va | pa; ulong l0Alignment = 1UL << (PageBits + LevelBits * 2); ulong l1Alignment = 1UL << (PageBits + LevelBits); if ((combinedAddress & (l0Alignment - 1)) == 0 && AlignDown(endVa, l0Alignment) > va) { return (AlignDown(endVa, l0Alignment) - va, 0); } else if ((combinedAddress & (l1Alignment - 1)) == 0 && AlignDown(endVa, l1Alignment) > va) { ulong nextOrderVa = GetNextAddress(va, l0Alignment); if (nextOrderVa <= endVa) { return (nextOrderVa - va, 1); } else { return (AlignDown(endVa, l1Alignment) - va, 1); } } else { ulong nextOrderVa = GetNextAddress(va, l1Alignment); if (nextOrderVa <= endVa) { return (nextOrderVa - va, 2); } else { return (endVa - va, 2); } } } private static ulong AlignDown(ulong va, ulong alignment) { return va & ~(alignment - 1); } private static ulong GetNextAddress(ulong va, ulong alignment) { return (va + alignment) & ~(alignment - 1); } private PtLevel EnsureLevel0() { PtLevel level0 = _level0; if (level0 == null) { level0 = new PtLevel(_blockAllocator, LevelCount, true); _level0 = level0; } return level0; } private void EnsureTable(PtLevel level, int index, bool hasNext) { Span currentTable = level.AsSpan(); if ((currentTable[index] & 1) == 0) { PtLevel nextLevel = new PtLevel(_blockAllocator, LevelCount, hasNext); currentTable[index] = (nextLevel.Address & ~(ulong)PageMask) | 3UL; level.Next[index] = nextLevel; level.EntriesCount++; ValidateEntriesCount(level.EntriesCount); } else if (level.Next[index] == null) { Debug.Fail($"Index {index} is block, expected a table."); } } private void WriteBlock(PtLevel level, int index, int depth, ulong pa, ulong attr) { Span currentTable = level.AsSpan(); currentTable[index] = (pa & ~((ulong)AllLevelsMask >> (depth * LevelBits))) | (depth == 2 ? 3UL : 1UL) | attr; level.EntriesCount++; ValidateEntriesCount(level.EntriesCount); } private static void ValidateEntriesCount(int count) { Debug.Assert(count >= 0 && count <= LevelCount, $"Entries count {count} is invalid."); } public void Dispose() { _blockAllocator.Dispose(); } } }