using Ryujinx.Cpu; using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Memory { /// /// GPU memory manager. /// public class MemoryManager { private const ulong AddressSpaceSize = 1UL << 40; public const ulong BadAddress = ulong.MaxValue; private const int PtLvl0Bits = 14; private const int PtLvl1Bits = 14; public const int PtPageBits = 12; private const ulong PtLvl0Size = 1UL << PtLvl0Bits; private const ulong PtLvl1Size = 1UL << PtLvl1Bits; public const ulong PageSize = 1UL << PtPageBits; private const ulong PtLvl0Mask = PtLvl0Size - 1; private const ulong PtLvl1Mask = PtLvl1Size - 1; public const ulong PageMask = PageSize - 1; private const int PtLvl0Bit = PtPageBits + PtLvl1Bits; private const int PtLvl1Bit = PtPageBits; private const ulong PteUnmapped = 0xffffffff_ffffffff; private const ulong PteReserved = 0xffffffff_fffffffe; private readonly ulong[][] _pageTable; public event EventHandler MemoryUnmapped; private GpuContext _context; /// /// Creates a new instance of the GPU memory manager. /// public MemoryManager(GpuContext context) { _context = context; _pageTable = new ulong[PtLvl0Size][]; } /// /// Reads data from GPU mapped memory. /// /// Type of the data /// GPU virtual address where the data is located /// The data at the specified memory location public T Read(ulong gpuVa) where T : unmanaged { ulong processVa = Translate(gpuVa); return MemoryMarshal.Cast(_context.PhysicalMemory.GetSpan(processVa, Unsafe.SizeOf()))[0]; } /// /// Gets a read-only span of data from GPU mapped memory. /// This reads as much data as possible, up to the specified maximum size. /// /// GPU virtual address where the data is located /// Size of the data /// The span of the data at the specified memory location public ReadOnlySpan GetSpan(ulong gpuVa, int size) { ulong processVa = Translate(gpuVa); return _context.PhysicalMemory.GetSpan(processVa, size); } /// /// Gets a writable region from GPU mapped memory. /// /// Start address of the range /// Size in bytes to be range /// A writable region with the data at the specified memory location public WritableRegion GetWritableRegion(ulong gpuVa, int size) { ulong processVa = Translate(gpuVa); return _context.PhysicalMemory.GetWritableRegion(processVa, size); } /// /// Writes data to GPU mapped memory. /// /// GPU virtual address to write the data into /// The data to be written public void Write(ulong gpuVa, ReadOnlySpan data) { ulong processVa = Translate(gpuVa); _context.PhysicalMemory.Write(processVa, data); } /// /// Maps a given range of pages to the specified CPU virtual address. /// /// /// All addresses and sizes must be page aligned. /// /// CPU virtual address to map into /// GPU virtual address to be mapped /// Size in bytes of the mapping /// GPU virtual address of the mapping public ulong Map(ulong pa, ulong va, ulong size) { lock (_pageTable) { for (ulong offset = 0; offset < size; offset += PageSize) { SetPte(va + offset, pa + offset); } } return va; } /// /// Maps a given range of pages to an allocated GPU virtual address. /// The memory is automatically allocated by the memory manager. /// /// CPU virtual address to map into /// Size in bytes of the mapping /// Required alignment of the GPU virtual address in bytes /// GPU virtual address where the range was mapped, or an all ones mask in case of failure public ulong MapAllocate(ulong pa, ulong size, ulong alignment) { lock (_pageTable) { ulong va = GetFreePosition(size, alignment); if (va != PteUnmapped) { for (ulong offset = 0; offset < size; offset += PageSize) { SetPte(va + offset, pa + offset); } } return va; } } /// /// Maps a given range of pages to an allocated GPU virtual address. /// The memory is automatically allocated by the memory manager. /// This also ensures that the mapping is always done in the first 4GB of GPU address space. /// /// CPU virtual address to map into /// Size in bytes of the mapping /// GPU virtual address where the range was mapped, or an all ones mask in case of failure public ulong MapLow(ulong pa, ulong size) { lock (_pageTable) { ulong va = GetFreePosition(size, 1, PageSize); if (va != PteUnmapped && va <= uint.MaxValue && (va + size) <= uint.MaxValue) { for (ulong offset = 0; offset < size; offset += PageSize) { SetPte(va + offset, pa + offset); } } else { va = PteUnmapped; } return va; } } /// /// Reserves memory at a fixed GPU memory location. /// This prevents the reserved region from being used for memory allocation for map. /// /// GPU virtual address to reserve /// Size in bytes of the reservation /// GPU virtual address of the reservation, or an all ones mask in case of failure public ulong ReserveFixed(ulong va, ulong size) { lock (_pageTable) { for (ulong offset = 0; offset < size; offset += PageSize) { if (IsPageInUse(va + offset)) { return PteUnmapped; } } for (ulong offset = 0; offset < size; offset += PageSize) { SetPte(va + offset, PteReserved); } } return va; } /// /// Reserves memory at any GPU memory location. /// /// Size in bytes of the reservation /// Reservation address alignment in bytes /// GPU virtual address of the reservation, or an all ones mask in case of failure public ulong Reserve(ulong size, ulong alignment) { lock (_pageTable) { ulong address = GetFreePosition(size, alignment); if (address != PteUnmapped) { for (ulong offset = 0; offset < size; offset += PageSize) { SetPte(address + offset, PteReserved); } } return address; } } /// /// Frees memory that was previously allocated by a map or reserved. /// /// GPU virtual address to free /// Size in bytes of the region being freed public void Free(ulong va, ulong size) { lock (_pageTable) { // Event handlers are not expected to be thread safe. MemoryUnmapped?.Invoke(this, new UnmapEventArgs(va, size)); for (ulong offset = 0; offset < size; offset += PageSize) { SetPte(va + offset, PteUnmapped); } } } /// /// Gets the address of an unused (free) region of the specified size. /// /// Size of the region in bytes /// Required alignment of the region address in bytes /// Start address of the search on the address space /// GPU virtual address of the allocation, or an all ones mask in case of failure private ulong GetFreePosition(ulong size, ulong alignment = 1, ulong start = 1UL << 32) { // Note: Address 0 is not considered valid by the driver, // when 0 is returned it's considered a mapping error. ulong address = start; ulong freeSize = 0; if (alignment == 0) { alignment = 1; } alignment = (alignment + PageMask) & ~PageMask; while (address + freeSize < AddressSpaceSize) { if (!IsPageInUse(address + freeSize)) { freeSize += PageSize; if (freeSize >= size) { return address; } } else { address += freeSize + PageSize; freeSize = 0; ulong remainder = address % alignment; if (remainder != 0) { address = (address - remainder) + alignment; } } } return PteUnmapped; } /// /// Translates a GPU virtual address to a CPU virtual address. /// /// GPU virtual address to be translated /// CPU virtual address public ulong Translate(ulong gpuVa) { ulong baseAddress = GetPte(gpuVa); if (baseAddress == PteUnmapped || baseAddress == PteReserved) { return PteUnmapped; } return baseAddress + (gpuVa & PageMask); } /// /// Checks if a given memory page is mapped or reserved. /// /// GPU virtual address of the page /// True if the page is mapped or reserved, false otherwise private bool IsPageInUse(ulong gpuVa) { if (gpuVa >> PtLvl0Bits + PtLvl1Bits + PtPageBits != 0) { return false; } ulong l0 = (gpuVa >> PtLvl0Bit) & PtLvl0Mask; ulong l1 = (gpuVa >> PtLvl1Bit) & PtLvl1Mask; if (_pageTable[l0] == null) { return false; } return _pageTable[l0][l1] != PteUnmapped; } /// /// Gets the Page Table entry for a given GPU virtual address. /// /// GPU virtual address /// Page table entry (CPU virtual address) private ulong GetPte(ulong gpuVa) { ulong l0 = (gpuVa >> PtLvl0Bit) & PtLvl0Mask; ulong l1 = (gpuVa >> PtLvl1Bit) & PtLvl1Mask; if (_pageTable[l0] == null) { return PteUnmapped; } return _pageTable[l0][l1]; } /// /// Sets a Page Table entry at a given GPU virtual address. /// /// GPU virtual address /// Page table entry (CPU virtual address) private void SetPte(ulong gpuVa, ulong pte) { ulong l0 = (gpuVa >> PtLvl0Bit) & PtLvl0Mask; ulong l1 = (gpuVa >> PtLvl1Bit) & PtLvl1Mask; if (_pageTable[l0] == null) { _pageTable[l0] = new ulong[PtLvl1Size]; for (ulong index = 0; index < PtLvl1Size; index++) { _pageTable[l0][index] = PteUnmapped; } } _pageTable[l0][l1] = pte; } } }