Ryujinx/Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs

370 lines
13 KiB
C#
Raw Normal View History

using Ryujinx.Cpu.AppleHv.Arm;
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
namespace Ryujinx.Cpu.AppleHv
{
class HvAddressSpaceRange : IDisposable
{
private const ulong AllocationGranule = 1UL << 14;
private const ulong AttributesMask = (0x3ffUL << 2) | (0x3fffUL << 50);
private const ulong BaseAttributes = (1UL << 10) | (3UL << 8); // Access flag set, inner shareable.
private const int LevelBits = 9;
private const int LevelCount = 1 << LevelBits;
private const int LevelMask = LevelCount - 1;
private const int PageBits = 12;
private const int PageSize = 1 << PageBits;
private const int PageMask = PageSize - 1;
private const int AllLevelsMask = PageMask | (LevelMask << PageBits) | (LevelMask << (PageBits + LevelBits));
private class PtLevel
{
public ulong Address => Allocation.Ipa + Allocation.Offset;
public int EntriesCount;
public readonly HvMemoryBlockAllocation Allocation;
public readonly PtLevel[] Next;
public PtLevel(HvMemoryBlockAllocator blockAllocator, int count, bool hasNext)
{
ulong size = (ulong)count * sizeof(ulong);
Allocation = blockAllocator.Allocate(size, PageSize);
AsSpan().Fill(0UL);
if (hasNext)
{
Next = new PtLevel[count];
}
}
public unsafe Span<ulong> AsSpan()
{
return MemoryMarshal.Cast<byte, ulong>(Allocation.Memory.GetSpan(Allocation.Offset, (int)Allocation.Size));
}
}
private PtLevel _level0;
private int _tlbInvalidationPending;
private readonly HvIpaAllocator _ipaAllocator;
private readonly HvMemoryBlockAllocator _blockAllocator;
public HvAddressSpaceRange(HvIpaAllocator ipaAllocator)
{
_ipaAllocator = ipaAllocator;
_blockAllocator = new HvMemoryBlockAllocator(ipaAllocator, (int)AllocationGranule);
}
public ulong GetIpaBase()
{
return EnsureLevel0().Address;
}
public bool GetAndClearTlbInvalidationPending()
{
return Interlocked.Exchange(ref _tlbInvalidationPending, 0) != 0;
}
public void Map(ulong va, ulong pa, ulong size, ApFlags accessPermission)
{
MapImpl(va, pa, size, (ulong)accessPermission | BaseAttributes);
}
public void Unmap(ulong va, ulong size)
{
UnmapImpl(EnsureLevel0(), 0, va, size);
Interlocked.Exchange(ref _tlbInvalidationPending, 1);
}
public void Reprotect(ulong va, ulong size, ApFlags accessPermission)
{
UpdateAttributes(va, size, (ulong)accessPermission | BaseAttributes);
}
private void MapImpl(ulong va, ulong pa, ulong size, ulong attr)
{
PtLevel level0 = EnsureLevel0();
ulong endVa = va + size;
while (va < endVa)
{
(ulong mapSize, int depth) = GetMapSizeAndDepth(va, pa, endVa);
PtLevel currentLevel = level0;
for (int i = 0; i < depth; i++)
{
int l = (int)(va >> (PageBits + (2 - i) * LevelBits)) & LevelMask;
EnsureTable(currentLevel, l, i == 0);
currentLevel = currentLevel.Next[l];
}
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
for (ulong i = 0; i < mapSize; i += blockSize)
{
if ((va >> blockShift) << blockShift != va ||
(pa >> blockShift) << blockShift != pa)
{
Debug.Fail($"Block size 0x{blockSize:X} (log2: {blockShift}) is invalid for VA 0x{va:X} or PA 0x{pa:X}.");
}
WriteBlock(currentLevel, (int)(va >> blockShift) & LevelMask, depth, pa, attr);
va += blockSize;
pa += blockSize;
}
}
}
private void UnmapImpl(PtLevel level, int depth, ulong va, ulong size)
{
ulong endVa = (va + size + PageMask) & ~((ulong)PageMask);
va &= ~((ulong)PageMask);
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
while (va < endVa)
{
ulong nextEntryVa = GetNextAddress(va, blockSize);
ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va);
int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask;
PtLevel nextTable = level.Next != null ? level.Next[l] : null;
if (nextTable != null)
{
// Entry is a table, visit it and update attributes as required.
UnmapImpl(nextTable, depth + 1, va, chunckSize);
}
else if (chunckSize != blockSize)
{
// Entry is a block but is not aligned, we need to turn it into a table.
ref ulong pte = ref level.AsSpan()[l];
nextTable = CreateTable(pte, depth + 1);
level.Next[l] = nextTable;
// Now that we have a table, we can handle it like the first case.
UnmapImpl(nextTable, depth + 1, va, chunckSize);
// Update PTE to point to the new table.
pte = (nextTable.Address & ~(ulong)PageMask) | 3UL;
}
// If entry is a block, or if entry is a table but it is empty, we can remove it.
if (nextTable == null || nextTable.EntriesCount == 0)
{
// Entry is a block and is fully aligned, so we can just set it to 0.
if (nextTable != null)
{
nextTable.Allocation.Dispose();
level.Next[l] = null;
}
level.AsSpan()[l] = 0UL;
level.EntriesCount--;
ValidateEntriesCount(level.EntriesCount);
}
va += chunckSize;
}
}
private void UpdateAttributes(ulong va, ulong size, ulong newAttr)
{
UpdateAttributes(EnsureLevel0(), 0, va, size, newAttr);
Interlocked.Exchange(ref _tlbInvalidationPending, 1);
}
private void UpdateAttributes(PtLevel level, int depth, ulong va, ulong size, ulong newAttr)
{
ulong endVa = (va + size + PageSize - 1) & ~((ulong)PageSize - 1);
va &= ~((ulong)PageSize - 1);
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
while (va < endVa)
{
ulong nextEntryVa = GetNextAddress(va, blockSize);
ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va);
int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask;
ref ulong pte = ref level.AsSpan()[l];
// First check if the region is mapped.
if ((pte & 3) != 0)
{
PtLevel nextTable = level.Next != null ? level.Next[l] : null;
if (nextTable != null)
{
// Entry is a table, visit it and update attributes as required.
UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr);
}
else if (chunckSize != blockSize)
{
// Entry is a block but is not aligned, we need to turn it into a table.
nextTable = CreateTable(pte, depth + 1);
level.Next[l] = nextTable;
// Now that we have a table, we can handle it like the first case.
UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr);
// Update PTE to point to the new table.
pte = (nextTable.Address & ~(ulong)PageMask) | 3UL;
}
else
{
// Entry is a block and is fully aligned, so we can just update the attributes.
// Update PTE with the new attributes.
pte = (pte & ~AttributesMask) | newAttr;
}
}
va += chunckSize;
}
}
private PtLevel CreateTable(ulong pte, int depth)
{
pte &= ~3UL;
pte |= (depth == 2 ? 3UL : 1UL);
PtLevel level = new PtLevel(_blockAllocator, LevelCount, depth < 2);
Span<ulong> currentLevel = level.AsSpan();
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
// Fill in the blocks.
for (int i = 0; i < LevelCount; i++)
{
ulong offset = (ulong)i << blockShift;
currentLevel[i] = pte + offset;
}
level.EntriesCount = LevelCount;
return level;
}
private static (ulong, int) GetBlockSizeAndShift(int depth)
{
int blockShift = PageBits + (2 - depth) * LevelBits;
ulong blockSize = 1UL << blockShift;
return (blockSize, blockShift);
}
private static (ulong, int) GetMapSizeAndDepth(ulong va, ulong pa, ulong endVa)
{
// Both virtual and physical addresses must be aligned to the block size.
ulong combinedAddress = va | pa;
ulong l0Alignment = 1UL << (PageBits + LevelBits * 2);
ulong l1Alignment = 1UL << (PageBits + LevelBits);
if ((combinedAddress & (l0Alignment - 1)) == 0 && AlignDown(endVa, l0Alignment) > va)
{
return (AlignDown(endVa, l0Alignment) - va, 0);
}
else if ((combinedAddress & (l1Alignment - 1)) == 0 && AlignDown(endVa, l1Alignment) > va)
{
ulong nextOrderVa = GetNextAddress(va, l0Alignment);
if (nextOrderVa <= endVa)
{
return (nextOrderVa - va, 1);
}
else
{
return (AlignDown(endVa, l1Alignment) - va, 1);
}
}
else
{
ulong nextOrderVa = GetNextAddress(va, l1Alignment);
if (nextOrderVa <= endVa)
{
return (nextOrderVa - va, 2);
}
else
{
return (endVa - va, 2);
}
}
}
private static ulong AlignDown(ulong va, ulong alignment)
{
return va & ~(alignment - 1);
}
private static ulong GetNextAddress(ulong va, ulong alignment)
{
return (va + alignment) & ~(alignment - 1);
}
private PtLevel EnsureLevel0()
{
PtLevel level0 = _level0;
if (level0 == null)
{
level0 = new PtLevel(_blockAllocator, LevelCount, true);
_level0 = level0;
}
return level0;
}
private void EnsureTable(PtLevel level, int index, bool hasNext)
{
Span<ulong> currentTable = level.AsSpan();
if ((currentTable[index] & 1) == 0)
{
PtLevel nextLevel = new PtLevel(_blockAllocator, LevelCount, hasNext);
currentTable[index] = (nextLevel.Address & ~(ulong)PageMask) | 3UL;
level.Next[index] = nextLevel;
level.EntriesCount++;
ValidateEntriesCount(level.EntriesCount);
}
else if (level.Next[index] == null)
{
Debug.Fail($"Index {index} is block, expected a table.");
}
}
private void WriteBlock(PtLevel level, int index, int depth, ulong pa, ulong attr)
{
Span<ulong> currentTable = level.AsSpan();
currentTable[index] = (pa & ~((ulong)AllLevelsMask >> (depth * LevelBits))) | (depth == 2 ? 3UL : 1UL) | attr;
level.EntriesCount++;
ValidateEntriesCount(level.EntriesCount);
}
private static void ValidateEntriesCount(int count)
{
Debug.Assert(count >= 0 && count <= LevelCount, $"Entries count {count} is invalid.");
}
public void Dispose()
{
_blockAllocator.Dispose();
}
}
}