From a53cfdab78c382677eb826bd5bedb58b3b838796 Mon Sep 17 00:00:00 2001 From: gdkchan Date: Sun, 29 Jan 2023 08:37:52 -0300 Subject: [PATCH] Initial Apple Hypervisor based CPU emulation (#4332) * Initial Apple Hypervisor based CPU emulation implementation * Add UseHypervisor Setting * Add basic MacOS support to Avalonia * Fix initialization * Fix GTK build * Fix/silence warnings * Change exceptions to asserts on HvAddressSpaceRange * Replace DllImport with LibraryImport * Fix LibraryImport * Remove unneeded usings * Revert outdated change * Set DiskCacheLoadState when using hypervisor too * Fix HvExecutionContext PC value * Address PR feedback * Use existing entitlements.xml file on distribution folder --------- Co-authored-by: riperiperi --- Ryujinx.Ava/AppHost.cs | 7 +- Ryujinx.Ava/Assets/Locales/en_US.json | 4 +- Ryujinx.Ava/Ryujinx.Ava.csproj | 7 +- .../UI/ViewModels/SettingsViewModel.cs | 14 +- .../UI/Views/Settings/SettingsCPUView.axaml | 10 +- Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs | 27 + Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs | 47 + .../AppleHv/DummyDiskCacheLoadState.cs | 17 + Ryujinx.Cpu/AppleHv/HvAddressSpace.cs | 129 +++ Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs | 370 +++++++ Ryujinx.Cpu/AppleHv/HvApi.cs | 320 ++++++ Ryujinx.Cpu/AppleHv/HvCpuContext.cs | 47 + Ryujinx.Cpu/AppleHv/HvEngine.cs | 20 + Ryujinx.Cpu/AppleHv/HvExecutionContext.cs | 284 ++++++ .../AppleHv/HvExecutionContextShadow.cs | 59 ++ Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs | 196 ++++ Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs | 34 + .../AppleHv/HvMemoryBlockAllocation.cs | 34 + Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs | 59 ++ Ryujinx.Cpu/AppleHv/HvMemoryManager.cs | 947 ++++++++++++++++++ Ryujinx.Cpu/AppleHv/HvVcpu.cs | 25 + Ryujinx.Cpu/AppleHv/HvVcpuPool.cs | 103 ++ Ryujinx.Cpu/AppleHv/HvVm.cs | 68 ++ Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs | 46 + Ryujinx.HLE/HLEConfiguration.cs | 9 +- Ryujinx.HLE/HOS/ArmProcessContextFactory.cs | 59 +- Ryujinx.HLE/HOS/Horizon.cs | 3 - Ryujinx.HLE/HOS/ProgramLoader.cs | 4 +- Ryujinx.Headless.SDL2/Options.cs | 3 + Ryujinx.Headless.SDL2/Program.cs | 3 +- .../Configuration/ConfigurationFileFormat.cs | 7 +- .../Configuration/ConfigurationState.cs | 19 +- Ryujinx/Ui/MainWindow.cs | 3 +- 33 files changed, 2939 insertions(+), 45 deletions(-) create mode 100644 Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs create mode 100644 Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs create mode 100644 Ryujinx.Cpu/AppleHv/DummyDiskCacheLoadState.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvAddressSpace.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvApi.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvCpuContext.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvEngine.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvExecutionContext.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocation.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvMemoryManager.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvVcpu.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvVcpuPool.cs create mode 100644 Ryujinx.Cpu/AppleHv/HvVm.cs create mode 100644 Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs diff --git a/Ryujinx.Ava/AppHost.cs b/Ryujinx.Ava/AppHost.cs index ad33d08c8d..6146a7d9e2 100644 --- a/Ryujinx.Ava/AppHost.cs +++ b/Ryujinx.Ava/AppHost.cs @@ -241,7 +241,7 @@ namespace Ryujinx.Ava { DateTime currentTime = DateTime.Now; string filename = $"ryujinx_capture_{currentTime.Year}-{currentTime.Month:D2}-{currentTime.Day:D2}_{currentTime.Hour:D2}-{currentTime.Minute:D2}-{currentTime.Second:D2}.png"; - + string directory = AppDataManager.Mode switch { AppDataManager.LaunchMode.Portable => Path.Combine(AppDataManager.BaseDirPath, "screenshots"), @@ -678,7 +678,8 @@ namespace Ryujinx.Ava ConfigurationState.Instance.System.MemoryManagerMode, ConfigurationState.Instance.System.IgnoreMissingServices, ConfigurationState.Instance.Graphics.AspectRatio, - ConfigurationState.Instance.System.AudioVolume); + ConfigurationState.Instance.System.AudioVolume, + ConfigurationState.Instance.System.UseHypervisor); Device = new Switch(configuration); } @@ -839,7 +840,7 @@ namespace Ryujinx.Ava { // Run a status update only when a frame is to be drawn. This prevents from updating the ui and wasting a render when no frame is queued. string dockedMode = ConfigurationState.Instance.System.EnableDockedMode ? LocaleManager.Instance[LocaleKeys.Docked] : LocaleManager.Instance[LocaleKeys.Handheld]; - + if (GraphicsConfig.ResScale != 1) { dockedMode += $" ({GraphicsConfig.ResScale}x)"; diff --git a/Ryujinx.Ava/Assets/Locales/en_US.json b/Ryujinx.Ava/Assets/Locales/en_US.json index 9bc6b58177..b7d1e02bf7 100644 --- a/Ryujinx.Ava/Assets/Locales/en_US.json +++ b/Ryujinx.Ava/Assets/Locales/en_US.json @@ -7,6 +7,7 @@ "SettingsTabSystemMemoryManagerModeSoftware": "Software", "SettingsTabSystemMemoryManagerModeHost": "Host (fast)", "SettingsTabSystemMemoryManagerModeHostUnchecked": "Host Unchecked (fastest, unsafe)", + "SettingsTabSystemUseHypervisor": "Use Hypervisor", "MenuBarFile": "_File", "MenuBarFileOpenFromFile": "_Load Application From File", "MenuBarFileOpenUnpacked": "Load _Unpacked Game", @@ -457,6 +458,7 @@ "MemoryManagerSoftwareTooltip": "Use a software page table for address translation. Highest accuracy but slowest performance.", "MemoryManagerHostTooltip": "Directly map memory in the host address space. Much faster JIT compilation and execution.", "MemoryManagerUnsafeTooltip": "Directly map memory, but do not mask the address within the guest address space before access. Faster, but at the cost of safety. The guest application can access memory from anywhere in Ryujinx, so only run programs you trust with this mode.", + "UseHypervisorTooltip": "Use Hypervisor instead of JIT. Greatly improves performance when available, but can be unstable in its current state.", "DRamTooltip": "Utilizes an alternative MemoryMode layout to mimic a Switch development model.\n\nThis is only useful for higher-resolution texture packs or 4k resolution mods. Does NOT improve performance.\n\nLeave OFF if unsure.", "IgnoreMissingServicesTooltip": "Ignores unimplemented Horizon OS services. This may help in bypassing crashes when booting certain games.\n\nLeave OFF if unsure.", "GraphicsBackendThreadingTooltip": "Executes graphics backend commands on a second thread.\n\nSpeeds up shader compilation, reduces stuttering, and improves performance on GPU drivers without multithreading support of their own. Slightly better performance on drivers with multithreading.\n\nSet to AUTO if unsure.", @@ -507,7 +509,7 @@ "SettingsTabNetwork": "Network", "SettingsTabNetworkConnection": "Network Connection", "SettingsTabCpuCache": "CPU Cache", - "SettingsTabCpuMemory": "CPU Memory", + "SettingsTabCpuMemory": "CPU Mode", "DialogUpdaterFlatpakNotSupportedMessage": "Please update Ryujinx via FlatHub.", "UpdaterDisabledWarningTitle": "Updater Disabled!", "GameListContextMenuOpenSdModsDirectory": "Open Atmosphere Mods Directory", diff --git a/Ryujinx.Ava/Ryujinx.Ava.csproj b/Ryujinx.Ava/Ryujinx.Ava.csproj index 88b60d0ba2..61a2bcb7e0 100644 --- a/Ryujinx.Ava/Ryujinx.Ava.csproj +++ b/Ryujinx.Ava/Ryujinx.Ava.csproj @@ -1,4 +1,4 @@ - + net7.0 win10-x64;osx-x64;linux-x64 @@ -6,11 +6,16 @@ true 1.0.0-dirty $(DefineConstants);$(ExtraDefineConstants) + - Ryujinx.Ava Ryujinx.ico true + + + + true true diff --git a/Ryujinx.Ava/UI/ViewModels/SettingsViewModel.cs b/Ryujinx.Ava/UI/ViewModels/SettingsViewModel.cs index e6a0111e88..36b37b0f52 100644 --- a/Ryujinx.Ava/UI/ViewModels/SettingsViewModel.cs +++ b/Ryujinx.Ava/UI/ViewModels/SettingsViewModel.cs @@ -22,6 +22,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; +using System.Runtime.InteropServices; using TimeZone = Ryujinx.Ava.UI.Models.TimeZone; namespace Ryujinx.Ava.UI.ViewModels @@ -59,6 +60,7 @@ namespace Ryujinx.Ava.UI.ViewModels OnPropertyChanged(nameof(IsCustomResolutionScaleActive)); } } + public int GraphicsBackendMultithreadingIndex { get => _graphicsBackendMultithreadingIndex; @@ -106,6 +108,8 @@ namespace Ryujinx.Ava.UI.ViewModels public bool IsOpenGLAvailable => !OperatingSystem.IsMacOS(); + public bool IsHypervisorAvailable => OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64; + public bool DirectoryChanged { get => _directoryChanged; @@ -117,10 +121,7 @@ namespace Ryujinx.Ava.UI.ViewModels } } - public bool IsMacOS - { - get => OperatingSystem.IsMacOS(); - } + public bool IsMacOS => OperatingSystem.IsMacOS(); public bool EnableDiscordIntegration { get; set; } public bool CheckUpdatesOnStart { get; set; } @@ -153,6 +154,7 @@ namespace Ryujinx.Ava.UI.ViewModels public bool EnableCustomTheme { get; set; } public bool IsCustomResolutionScaleActive => _resolutionScale == 4; public bool IsVulkanSelected => GraphicsBackendIndex == 0; + public bool UseHypervisor { get; set; } public string TimeZone { get; set; } public string ShaderDumpPath { get; set; } @@ -349,6 +351,7 @@ namespace Ryujinx.Ava.UI.ViewModels // CPU EnablePptc = config.System.EnablePtc; MemoryMode = (int)config.System.MemoryManagerMode.Value; + UseHypervisor = config.System.UseHypervisor; // Graphics GraphicsBackendIndex = (int)config.Graphics.GraphicsBackend.Value; @@ -369,7 +372,7 @@ namespace Ryujinx.Ava.UI.ViewModels // Network EnableInternetAccess = config.System.EnableInternetAccess; - + // Logging EnableFileLog = config.Logger.EnableFileLog; EnableStub = config.Logger.EnableStub; @@ -432,6 +435,7 @@ namespace Ryujinx.Ava.UI.ViewModels // CPU config.System.EnablePtc.Value = EnablePptc; config.System.MemoryManagerMode.Value = (MemoryManagerMode)MemoryMode; + config.System.UseHypervisor.Value = UseHypervisor; // Graphics config.Graphics.GraphicsBackend.Value = (GraphicsBackend)GraphicsBackendIndex; diff --git a/Ryujinx.Ava/UI/Views/Settings/SettingsCPUView.axaml b/Ryujinx.Ava/UI/Views/Settings/SettingsCPUView.axaml index a0cf34529a..e98b963c19 100644 --- a/Ryujinx.Ava/UI/Views/Settings/SettingsCPUView.axaml +++ b/Ryujinx.Ava/UI/Views/Settings/SettingsCPUView.axaml @@ -1,4 +1,4 @@ - + + + - \ No newline at end of file + diff --git a/Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs b/Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs new file mode 100644 index 0000000000..95e6743276 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs @@ -0,0 +1,27 @@ +namespace Ryujinx.Cpu.AppleHv.Arm +{ + enum ApFlags : ulong + { + ApShift = 6, + PxnShift = 53, + UxnShift = 54, + + UserExecuteKernelReadWriteExecute = (0UL << (int)ApShift), + UserReadWriteExecuteKernelReadWrite = (1UL << (int)ApShift), + UserExecuteKernelReadExecute = (2UL << (int)ApShift), + UserReadExecuteKernelReadExecute = (3UL << (int)ApShift), + + UserExecuteKernelReadWrite = (1UL << (int)PxnShift) | (0UL << (int)ApShift), + UserExecuteKernelRead = (1UL << (int)PxnShift) | (2UL << (int)ApShift), + UserReadExecuteKernelRead = (1UL << (int)PxnShift) | (3UL << (int)ApShift), + + UserNoneKernelReadWriteExecute = (1UL << (int)UxnShift) | (0UL << (int)ApShift), + UserReadWriteKernelReadWrite = (1UL << (int)UxnShift) | (1UL << (int)ApShift), + UserNoneKernelReadExecute = (1UL << (int)UxnShift) | (2UL << (int)ApShift), + UserReadKernelReadExecute = (1UL << (int)UxnShift) | (3UL << (int)ApShift), + + UserNoneKernelReadWrite = (1UL << (int)PxnShift) | (1UL << (int)UxnShift) | (0UL << (int)ApShift), + UserNoneKernelRead = (1UL << (int)PxnShift) | (1UL << (int)UxnShift) | (2UL << (int)ApShift), + UserReadKernelRead = (1UL << (int)PxnShift) | (1UL << (int)UxnShift) | (3UL << (int)ApShift) + } +} diff --git a/Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs b/Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs new file mode 100644 index 0000000000..18152f2545 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs @@ -0,0 +1,47 @@ +namespace Ryujinx.Cpu.AppleHv.Arm +{ + enum ExceptionClass + { + Unknown = 0b000000, + TrappedWfeWfiWfetWfit = 0b000001, + TrappedMcrMrcCp15 = 0b000011, + TrappedMcrrMrrcCp15 = 0b000100, + TrappedMcrMrcCp14 = 0b000101, + TrappedLdcStc = 0b000110, + TrappedSveFpSimd = 0b000111, + TrappedVmrs = 0b001000, + TrappedPAuth = 0b001001, + TrappedLd64bSt64bSt64bvSt64bv0 = 0b001010, + TrappedMrrcCp14 = 0b001100, + IllegalExecutionState = 0b001110, + SvcAarch32 = 0b010001, + HvcAarch32 = 0b010010, + SmcAarch32 = 0b010011, + SvcAarch64 = 0b010101, + HvcAarch64 = 0b010110, + SmcAarch64 = 0b010111, + TrappedMsrMrsSystem = 0b011000, + TrappedSve = 0b011001, + TrappedEretEretaaEretab = 0b011010, + PointerAuthenticationFailure = 0b011100, + ImplementationDefinedEl3 = 0b011111, + InstructionAbortLowerEl = 0b100000, + InstructionAbortSameEl = 0b100001, + PcAlignmentFault = 0b100010, + DataAbortLowerEl = 0b100100, + DataAbortSameEl = 0b100101, + SpAlignmentFault = 0b100110, + TrappedFpExceptionAarch32 = 0b101000, + TrappedFpExceptionAarch64 = 0b101100, + SErrorInterrupt = 0b101111, + BreakpointLowerEl = 0b110000, + BreakpointSameEl = 0b110001, + SoftwareStepLowerEl = 0b110010, + SoftwareStepSameEl = 0b110011, + WatchpointLowerEl = 0b110100, + WatchpointSameEl = 0b110101, + BkptAarch32 = 0b111000, + VectorCatchAarch32 = 0b111010, + BrkAarch64 = 0b111100 + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/DummyDiskCacheLoadState.cs b/Ryujinx.Cpu/AppleHv/DummyDiskCacheLoadState.cs new file mode 100644 index 0000000000..6a692e7408 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/DummyDiskCacheLoadState.cs @@ -0,0 +1,17 @@ +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + public class DummyDiskCacheLoadState : IDiskCacheLoadState + { +#pragma warning disable CS0067 + /// + public event Action StateChanged; +#pragma warning restore CS0067 + + /// + public void Cancel() + { + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvAddressSpace.cs b/Ryujinx.Cpu/AppleHv/HvAddressSpace.cs new file mode 100644 index 0000000000..78f4c46402 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvAddressSpace.cs @@ -0,0 +1,129 @@ +using Ryujinx.Cpu.AppleHv.Arm; +using Ryujinx.Memory; +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvAddressSpace : IDisposable + { + private const ulong KernelRegionBase = unchecked((ulong)-(1L << 39)); + private const ulong KernelRegionCodeOffset = 0UL; + private const ulong KernelRegionCodeSize = 0x2000UL; + private const ulong KernelRegionTlbiEretOffset = KernelRegionCodeOffset + 0x1000UL; + private const ulong KernelRegionEretOffset = KernelRegionTlbiEretOffset + 4UL; + + public const ulong KernelRegionEretAddress = KernelRegionBase + KernelRegionEretOffset; + public const ulong KernelRegionTlbiEretAddress = KernelRegionBase + KernelRegionTlbiEretOffset; + + private const ulong AllocationGranule = 1UL << 14; + + private readonly ulong _asBase; + private readonly ulong _asSize; + private readonly ulong _backingSize; + + private readonly HvAddressSpaceRange _userRange; + private readonly HvAddressSpaceRange _kernelRange; + + private MemoryBlock _kernelCodeBlock; + + public HvAddressSpace(MemoryBlock backingMemory, ulong asSize) + { + (_asBase, var ipaAllocator) = HvVm.CreateAddressSpace(backingMemory); + _asSize = asSize; + _backingSize = backingMemory.Size; + + _userRange = new HvAddressSpaceRange(ipaAllocator); + _kernelRange = new HvAddressSpaceRange(ipaAllocator); + + _kernelCodeBlock = new MemoryBlock(AllocationGranule); + + InitializeKernelCode(ipaAllocator); + } + + private void InitializeKernelCode(HvIpaAllocator ipaAllocator) + { + // Write exception handlers. + for (ulong offset = 0; offset < 0x800; offset += 0x80) + { + // Offsets: + // 0x0: Synchronous + // 0x80: IRQ + // 0x100: FIQ + // 0x180: SError + _kernelCodeBlock.Write(KernelRegionCodeOffset + offset, 0xD41FFFE2u); // HVC #0xFFFF + _kernelCodeBlock.Write(KernelRegionCodeOffset + offset + 4, 0xD69F03E0u); // ERET + } + + _kernelCodeBlock.Write(KernelRegionTlbiEretOffset, 0xD508831Fu); // TLBI VMALLE1IS + _kernelCodeBlock.Write(KernelRegionEretOffset, 0xD69F03E0u); // ERET + + ulong kernelCodePa = ipaAllocator.Allocate(AllocationGranule); + HvApi.hv_vm_map((ulong)_kernelCodeBlock.Pointer, kernelCodePa, AllocationGranule, hv_memory_flags_t.HV_MEMORY_READ | hv_memory_flags_t.HV_MEMORY_EXEC).ThrowOnError(); + + _kernelRange.Map(KernelRegionCodeOffset, kernelCodePa, KernelRegionCodeSize, ApFlags.UserNoneKernelReadExecute); + } + + public void InitializeMmu(ulong vcpu) + { + HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_VBAR_EL1, KernelRegionBase + KernelRegionCodeOffset); + + HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_TTBR0_EL1, _userRange.GetIpaBase()); + HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_TTBR1_EL1, _kernelRange.GetIpaBase()); + HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_MAIR_EL1, 0xffUL); + HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_TCR_EL1, 0x00000011B5193519UL); + HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_SCTLR_EL1, 0x0000000034D5D925UL); + } + + public bool GetAndClearUserTlbInvalidationPending() + { + return _userRange.GetAndClearTlbInvalidationPending(); + } + + public void MapUser(ulong va, ulong pa, ulong size, MemoryPermission permission) + { + pa += _asBase; + + lock (_userRange) + { + _userRange.Map(va, pa, size, GetApFlags(permission)); + } + } + + public void UnmapUser(ulong va, ulong size) + { + lock (_userRange) + { + _userRange.Unmap(va, size); + } + } + + public void ReprotectUser(ulong va, ulong size, MemoryPermission permission) + { + lock (_userRange) + { + _userRange.Reprotect(va, size, GetApFlags(permission)); + } + } + + private static ApFlags GetApFlags(MemoryPermission permission) + { + return permission switch + { + MemoryPermission.None => ApFlags.UserNoneKernelRead, + MemoryPermission.Execute => ApFlags.UserExecuteKernelRead, + MemoryPermission.Read => ApFlags.UserReadKernelRead, + MemoryPermission.ReadAndWrite => ApFlags.UserReadWriteKernelReadWrite, + MemoryPermission.ReadAndExecute => ApFlags.UserReadExecuteKernelRead, + MemoryPermission.ReadWriteExecute => ApFlags.UserReadWriteExecuteKernelReadWrite, + _ => throw new ArgumentException($"Permission \"{permission}\" is invalid.") + }; + } + + public void Dispose() + { + _userRange.Dispose(); + _kernelRange.Dispose(); + HvVm.DestroyAddressSpace(_asBase, _backingSize); + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs b/Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs new file mode 100644 index 0000000000..ca30bb6811 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs @@ -0,0 +1,370 @@ +using Ryujinx.Cpu.AppleHv.Arm; +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvAddressSpaceRange : IDisposable + { + private const ulong AllocationGranule = 1UL << 14; + + private const ulong AttributesMask = (0x3ffUL << 2) | (0x3fffUL << 50); + + private const ulong BaseAttributes = (1UL << 10) | (3UL << 8); // Access flag set, inner shareable. + + private const int LevelBits = 9; + private const int LevelCount = 1 << LevelBits; + private const int LevelMask = LevelCount - 1; + private const int PageBits = 12; + private const int PageSize = 1 << PageBits; + private const int PageMask = PageSize - 1; + private const int AllLevelsMask = PageMask | (LevelMask << PageBits) | (LevelMask << (PageBits + LevelBits)); + + private class PtLevel + { + public ulong Address => Allocation.Ipa + Allocation.Offset; + public int EntriesCount; + public readonly HvMemoryBlockAllocation Allocation; + public readonly PtLevel[] Next; + + public PtLevel(HvMemoryBlockAllocator blockAllocator, int count, bool hasNext) + { + ulong size = (ulong)count * sizeof(ulong); + Allocation = blockAllocator.Allocate(size, PageSize); + + AsSpan().Fill(0UL); + + if (hasNext) + { + Next = new PtLevel[count]; + } + } + + public unsafe Span AsSpan() + { + return MemoryMarshal.Cast(Allocation.Memory.GetSpan(Allocation.Offset, (int)Allocation.Size)); + } + } + + private PtLevel _level0; + + private int _tlbInvalidationPending; + + private readonly HvIpaAllocator _ipaAllocator; + private readonly HvMemoryBlockAllocator _blockAllocator; + + public HvAddressSpaceRange(HvIpaAllocator ipaAllocator) + { + _ipaAllocator = ipaAllocator; + _blockAllocator = new HvMemoryBlockAllocator(ipaAllocator, (int)AllocationGranule); + } + + public ulong GetIpaBase() + { + return EnsureLevel0().Address; + } + + public bool GetAndClearTlbInvalidationPending() + { + return Interlocked.Exchange(ref _tlbInvalidationPending, 0) != 0; + } + + public void Map(ulong va, ulong pa, ulong size, ApFlags accessPermission) + { + MapImpl(va, pa, size, (ulong)accessPermission | BaseAttributes); + } + + public void Unmap(ulong va, ulong size) + { + UnmapImpl(EnsureLevel0(), 0, va, size); + Interlocked.Exchange(ref _tlbInvalidationPending, 1); + } + + public void Reprotect(ulong va, ulong size, ApFlags accessPermission) + { + UpdateAttributes(va, size, (ulong)accessPermission | BaseAttributes); + } + + private void MapImpl(ulong va, ulong pa, ulong size, ulong attr) + { + PtLevel level0 = EnsureLevel0(); + + ulong endVa = va + size; + + while (va < endVa) + { + (ulong mapSize, int depth) = GetMapSizeAndDepth(va, pa, endVa); + + PtLevel currentLevel = level0; + + for (int i = 0; i < depth; i++) + { + int l = (int)(va >> (PageBits + (2 - i) * LevelBits)) & LevelMask; + EnsureTable(currentLevel, l, i == 0); + currentLevel = currentLevel.Next[l]; + } + + (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); + + for (ulong i = 0; i < mapSize; i += blockSize) + { + if ((va >> blockShift) << blockShift != va || + (pa >> blockShift) << blockShift != pa) + { + Debug.Fail($"Block size 0x{blockSize:X} (log2: {blockShift}) is invalid for VA 0x{va:X} or PA 0x{pa:X}."); + } + + WriteBlock(currentLevel, (int)(va >> blockShift) & LevelMask, depth, pa, attr); + + va += blockSize; + pa += blockSize; + } + } + } + + private void UnmapImpl(PtLevel level, int depth, ulong va, ulong size) + { + ulong endVa = (va + size + PageMask) & ~((ulong)PageMask); + va &= ~((ulong)PageMask); + + (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); + + while (va < endVa) + { + ulong nextEntryVa = GetNextAddress(va, blockSize); + ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va); + + int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask; + + PtLevel nextTable = level.Next != null ? level.Next[l] : null; + + if (nextTable != null) + { + // Entry is a table, visit it and update attributes as required. + UnmapImpl(nextTable, depth + 1, va, chunckSize); + } + else if (chunckSize != blockSize) + { + // Entry is a block but is not aligned, we need to turn it into a table. + ref ulong pte = ref level.AsSpan()[l]; + nextTable = CreateTable(pte, depth + 1); + level.Next[l] = nextTable; + + // Now that we have a table, we can handle it like the first case. + UnmapImpl(nextTable, depth + 1, va, chunckSize); + + // Update PTE to point to the new table. + pte = (nextTable.Address & ~(ulong)PageMask) | 3UL; + } + + // If entry is a block, or if entry is a table but it is empty, we can remove it. + if (nextTable == null || nextTable.EntriesCount == 0) + { + // Entry is a block and is fully aligned, so we can just set it to 0. + if (nextTable != null) + { + nextTable.Allocation.Dispose(); + level.Next[l] = null; + } + + level.AsSpan()[l] = 0UL; + level.EntriesCount--; + ValidateEntriesCount(level.EntriesCount); + } + + va += chunckSize; + } + } + + private void UpdateAttributes(ulong va, ulong size, ulong newAttr) + { + UpdateAttributes(EnsureLevel0(), 0, va, size, newAttr); + + Interlocked.Exchange(ref _tlbInvalidationPending, 1); + } + + private void UpdateAttributes(PtLevel level, int depth, ulong va, ulong size, ulong newAttr) + { + ulong endVa = (va + size + PageSize - 1) & ~((ulong)PageSize - 1); + va &= ~((ulong)PageSize - 1); + + (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); + + while (va < endVa) + { + ulong nextEntryVa = GetNextAddress(va, blockSize); + ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va); + + int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask; + + ref ulong pte = ref level.AsSpan()[l]; + + // First check if the region is mapped. + if ((pte & 3) != 0) + { + PtLevel nextTable = level.Next != null ? level.Next[l] : null; + + if (nextTable != null) + { + // Entry is a table, visit it and update attributes as required. + UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr); + } + else if (chunckSize != blockSize) + { + // Entry is a block but is not aligned, we need to turn it into a table. + nextTable = CreateTable(pte, depth + 1); + level.Next[l] = nextTable; + + // Now that we have a table, we can handle it like the first case. + UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr); + + // Update PTE to point to the new table. + pte = (nextTable.Address & ~(ulong)PageMask) | 3UL; + } + else + { + // Entry is a block and is fully aligned, so we can just update the attributes. + // Update PTE with the new attributes. + pte = (pte & ~AttributesMask) | newAttr; + } + } + + va += chunckSize; + } + } + + private PtLevel CreateTable(ulong pte, int depth) + { + pte &= ~3UL; + pte |= (depth == 2 ? 3UL : 1UL); + + PtLevel level = new PtLevel(_blockAllocator, LevelCount, depth < 2); + Span currentLevel = level.AsSpan(); + + (ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth); + + // Fill in the blocks. + for (int i = 0; i < LevelCount; i++) + { + ulong offset = (ulong)i << blockShift; + currentLevel[i] = pte + offset; + } + + level.EntriesCount = LevelCount; + + return level; + } + + private static (ulong, int) GetBlockSizeAndShift(int depth) + { + int blockShift = PageBits + (2 - depth) * LevelBits; + ulong blockSize = 1UL << blockShift; + + return (blockSize, blockShift); + } + + private static (ulong, int) GetMapSizeAndDepth(ulong va, ulong pa, ulong endVa) + { + // Both virtual and physical addresses must be aligned to the block size. + ulong combinedAddress = va | pa; + + ulong l0Alignment = 1UL << (PageBits + LevelBits * 2); + ulong l1Alignment = 1UL << (PageBits + LevelBits); + + if ((combinedAddress & (l0Alignment - 1)) == 0 && AlignDown(endVa, l0Alignment) > va) + { + return (AlignDown(endVa, l0Alignment) - va, 0); + } + else if ((combinedAddress & (l1Alignment - 1)) == 0 && AlignDown(endVa, l1Alignment) > va) + { + ulong nextOrderVa = GetNextAddress(va, l0Alignment); + + if (nextOrderVa <= endVa) + { + return (nextOrderVa - va, 1); + } + else + { + return (AlignDown(endVa, l1Alignment) - va, 1); + } + } + else + { + ulong nextOrderVa = GetNextAddress(va, l1Alignment); + + if (nextOrderVa <= endVa) + { + return (nextOrderVa - va, 2); + } + else + { + return (endVa - va, 2); + } + } + } + + private static ulong AlignDown(ulong va, ulong alignment) + { + return va & ~(alignment - 1); + } + + private static ulong GetNextAddress(ulong va, ulong alignment) + { + return (va + alignment) & ~(alignment - 1); + } + + private PtLevel EnsureLevel0() + { + PtLevel level0 = _level0; + + if (level0 == null) + { + level0 = new PtLevel(_blockAllocator, LevelCount, true); + _level0 = level0; + } + + return level0; + } + + private void EnsureTable(PtLevel level, int index, bool hasNext) + { + Span currentTable = level.AsSpan(); + + if ((currentTable[index] & 1) == 0) + { + PtLevel nextLevel = new PtLevel(_blockAllocator, LevelCount, hasNext); + + currentTable[index] = (nextLevel.Address & ~(ulong)PageMask) | 3UL; + level.Next[index] = nextLevel; + level.EntriesCount++; + ValidateEntriesCount(level.EntriesCount); + } + else if (level.Next[index] == null) + { + Debug.Fail($"Index {index} is block, expected a table."); + } + } + + private void WriteBlock(PtLevel level, int index, int depth, ulong pa, ulong attr) + { + Span currentTable = level.AsSpan(); + + currentTable[index] = (pa & ~((ulong)AllLevelsMask >> (depth * LevelBits))) | (depth == 2 ? 3UL : 1UL) | attr; + + level.EntriesCount++; + ValidateEntriesCount(level.EntriesCount); + } + + private static void ValidateEntriesCount(int count) + { + Debug.Assert(count >= 0 && count <= LevelCount, $"Entries count {count} is invalid."); + } + + public void Dispose() + { + _blockAllocator.Dispose(); + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvApi.cs b/Ryujinx.Cpu/AppleHv/HvApi.cs new file mode 100644 index 0000000000..d7628bb523 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvApi.cs @@ -0,0 +1,320 @@ +using ARMeilleure.State; +using System; +using System.Runtime.InteropServices; + +namespace Ryujinx.Cpu.AppleHv +{ + struct hv_vcpu_exit_exception_t + { +#pragma warning disable CS0649 + public ulong syndrome; + public ulong virtual_address; + public ulong physical_address; +#pragma warning restore CS0649 + } + + struct hv_vcpu_exit_t + { +#pragma warning disable CS0649 + public uint reason; + public hv_vcpu_exit_exception_t exception; +#pragma warning restore CS0649 + } + + enum hv_reg_t : uint + { + HV_REG_X0, + HV_REG_X1, + HV_REG_X2, + HV_REG_X3, + HV_REG_X4, + HV_REG_X5, + HV_REG_X6, + HV_REG_X7, + HV_REG_X8, + HV_REG_X9, + HV_REG_X10, + HV_REG_X11, + HV_REG_X12, + HV_REG_X13, + HV_REG_X14, + HV_REG_X15, + HV_REG_X16, + HV_REG_X17, + HV_REG_X18, + HV_REG_X19, + HV_REG_X20, + HV_REG_X21, + HV_REG_X22, + HV_REG_X23, + HV_REG_X24, + HV_REG_X25, + HV_REG_X26, + HV_REG_X27, + HV_REG_X28, + HV_REG_X29, + HV_REG_FP = HV_REG_X29, + HV_REG_X30, + HV_REG_LR = HV_REG_X30, + HV_REG_PC, + HV_REG_FPCR, + HV_REG_FPSR, + HV_REG_CPSR, + } + + enum hv_simd_fp_reg_t : uint + { + HV_SIMD_FP_REG_Q0, + HV_SIMD_FP_REG_Q1, + HV_SIMD_FP_REG_Q2, + HV_SIMD_FP_REG_Q3, + HV_SIMD_FP_REG_Q4, + HV_SIMD_FP_REG_Q5, + HV_SIMD_FP_REG_Q6, + HV_SIMD_FP_REG_Q7, + HV_SIMD_FP_REG_Q8, + HV_SIMD_FP_REG_Q9, + HV_SIMD_FP_REG_Q10, + HV_SIMD_FP_REG_Q11, + HV_SIMD_FP_REG_Q12, + HV_SIMD_FP_REG_Q13, + HV_SIMD_FP_REG_Q14, + HV_SIMD_FP_REG_Q15, + HV_SIMD_FP_REG_Q16, + HV_SIMD_FP_REG_Q17, + HV_SIMD_FP_REG_Q18, + HV_SIMD_FP_REG_Q19, + HV_SIMD_FP_REG_Q20, + HV_SIMD_FP_REG_Q21, + HV_SIMD_FP_REG_Q22, + HV_SIMD_FP_REG_Q23, + HV_SIMD_FP_REG_Q24, + HV_SIMD_FP_REG_Q25, + HV_SIMD_FP_REG_Q26, + HV_SIMD_FP_REG_Q27, + HV_SIMD_FP_REG_Q28, + HV_SIMD_FP_REG_Q29, + HV_SIMD_FP_REG_Q30, + HV_SIMD_FP_REG_Q31, + } + + enum hv_sys_reg_t : ushort + { + HV_SYS_REG_DBGBVR0_EL1 = 0x8004, + HV_SYS_REG_DBGBCR0_EL1 = 0x8005, + HV_SYS_REG_DBGWVR0_EL1 = 0x8006, + HV_SYS_REG_DBGWCR0_EL1 = 0x8007, + HV_SYS_REG_DBGBVR1_EL1 = 0x800c, + HV_SYS_REG_DBGBCR1_EL1 = 0x800d, + HV_SYS_REG_DBGWVR1_EL1 = 0x800e, + HV_SYS_REG_DBGWCR1_EL1 = 0x800f, + HV_SYS_REG_MDCCINT_EL1 = 0x8010, + HV_SYS_REG_MDSCR_EL1 = 0x8012, + HV_SYS_REG_DBGBVR2_EL1 = 0x8014, + HV_SYS_REG_DBGBCR2_EL1 = 0x8015, + HV_SYS_REG_DBGWVR2_EL1 = 0x8016, + HV_SYS_REG_DBGWCR2_EL1 = 0x8017, + HV_SYS_REG_DBGBVR3_EL1 = 0x801c, + HV_SYS_REG_DBGBCR3_EL1 = 0x801d, + HV_SYS_REG_DBGWVR3_EL1 = 0x801e, + HV_SYS_REG_DBGWCR3_EL1 = 0x801f, + HV_SYS_REG_DBGBVR4_EL1 = 0x8024, + HV_SYS_REG_DBGBCR4_EL1 = 0x8025, + HV_SYS_REG_DBGWVR4_EL1 = 0x8026, + HV_SYS_REG_DBGWCR4_EL1 = 0x8027, + HV_SYS_REG_DBGBVR5_EL1 = 0x802c, + HV_SYS_REG_DBGBCR5_EL1 = 0x802d, + HV_SYS_REG_DBGWVR5_EL1 = 0x802e, + HV_SYS_REG_DBGWCR5_EL1 = 0x802f, + HV_SYS_REG_DBGBVR6_EL1 = 0x8034, + HV_SYS_REG_DBGBCR6_EL1 = 0x8035, + HV_SYS_REG_DBGWVR6_EL1 = 0x8036, + HV_SYS_REG_DBGWCR6_EL1 = 0x8037, + HV_SYS_REG_DBGBVR7_EL1 = 0x803c, + HV_SYS_REG_DBGBCR7_EL1 = 0x803d, + HV_SYS_REG_DBGWVR7_EL1 = 0x803e, + HV_SYS_REG_DBGWCR7_EL1 = 0x803f, + HV_SYS_REG_DBGBVR8_EL1 = 0x8044, + HV_SYS_REG_DBGBCR8_EL1 = 0x8045, + HV_SYS_REG_DBGWVR8_EL1 = 0x8046, + HV_SYS_REG_DBGWCR8_EL1 = 0x8047, + HV_SYS_REG_DBGBVR9_EL1 = 0x804c, + HV_SYS_REG_DBGBCR9_EL1 = 0x804d, + HV_SYS_REG_DBGWVR9_EL1 = 0x804e, + HV_SYS_REG_DBGWCR9_EL1 = 0x804f, + HV_SYS_REG_DBGBVR10_EL1 = 0x8054, + HV_SYS_REG_DBGBCR10_EL1 = 0x8055, + HV_SYS_REG_DBGWVR10_EL1 = 0x8056, + HV_SYS_REG_DBGWCR10_EL1 = 0x8057, + HV_SYS_REG_DBGBVR11_EL1 = 0x805c, + HV_SYS_REG_DBGBCR11_EL1 = 0x805d, + HV_SYS_REG_DBGWVR11_EL1 = 0x805e, + HV_SYS_REG_DBGWCR11_EL1 = 0x805f, + HV_SYS_REG_DBGBVR12_EL1 = 0x8064, + HV_SYS_REG_DBGBCR12_EL1 = 0x8065, + HV_SYS_REG_DBGWVR12_EL1 = 0x8066, + HV_SYS_REG_DBGWCR12_EL1 = 0x8067, + HV_SYS_REG_DBGBVR13_EL1 = 0x806c, + HV_SYS_REG_DBGBCR13_EL1 = 0x806d, + HV_SYS_REG_DBGWVR13_EL1 = 0x806e, + HV_SYS_REG_DBGWCR13_EL1 = 0x806f, + HV_SYS_REG_DBGBVR14_EL1 = 0x8074, + HV_SYS_REG_DBGBCR14_EL1 = 0x8075, + HV_SYS_REG_DBGWVR14_EL1 = 0x8076, + HV_SYS_REG_DBGWCR14_EL1 = 0x8077, + HV_SYS_REG_DBGBVR15_EL1 = 0x807c, + HV_SYS_REG_DBGBCR15_EL1 = 0x807d, + HV_SYS_REG_DBGWVR15_EL1 = 0x807e, + HV_SYS_REG_DBGWCR15_EL1 = 0x807f, + HV_SYS_REG_MIDR_EL1 = 0xc000, + HV_SYS_REG_MPIDR_EL1 = 0xc005, + HV_SYS_REG_ID_AA64PFR0_EL1 = 0xc020, + HV_SYS_REG_ID_AA64PFR1_EL1 = 0xc021, + HV_SYS_REG_ID_AA64DFR0_EL1 = 0xc028, + HV_SYS_REG_ID_AA64DFR1_EL1 = 0xc029, + HV_SYS_REG_ID_AA64ISAR0_EL1 = 0xc030, + HV_SYS_REG_ID_AA64ISAR1_EL1 = 0xc031, + HV_SYS_REG_ID_AA64MMFR0_EL1 = 0xc038, + HV_SYS_REG_ID_AA64MMFR1_EL1 = 0xc039, + HV_SYS_REG_ID_AA64MMFR2_EL1 = 0xc03a, + HV_SYS_REG_SCTLR_EL1 = 0xc080, + HV_SYS_REG_CPACR_EL1 = 0xc082, + HV_SYS_REG_TTBR0_EL1 = 0xc100, + HV_SYS_REG_TTBR1_EL1 = 0xc101, + HV_SYS_REG_TCR_EL1 = 0xc102, + HV_SYS_REG_APIAKEYLO_EL1 = 0xc108, + HV_SYS_REG_APIAKEYHI_EL1 = 0xc109, + HV_SYS_REG_APIBKEYLO_EL1 = 0xc10a, + HV_SYS_REG_APIBKEYHI_EL1 = 0xc10b, + HV_SYS_REG_APDAKEYLO_EL1 = 0xc110, + HV_SYS_REG_APDAKEYHI_EL1 = 0xc111, + HV_SYS_REG_APDBKEYLO_EL1 = 0xc112, + HV_SYS_REG_APDBKEYHI_EL1 = 0xc113, + HV_SYS_REG_APGAKEYLO_EL1 = 0xc118, + HV_SYS_REG_APGAKEYHI_EL1 = 0xc119, + HV_SYS_REG_SPSR_EL1 = 0xc200, + HV_SYS_REG_ELR_EL1 = 0xc201, + HV_SYS_REG_SP_EL0 = 0xc208, + HV_SYS_REG_AFSR0_EL1 = 0xc288, + HV_SYS_REG_AFSR1_EL1 = 0xc289, + HV_SYS_REG_ESR_EL1 = 0xc290, + HV_SYS_REG_FAR_EL1 = 0xc300, + HV_SYS_REG_PAR_EL1 = 0xc3a0, + HV_SYS_REG_MAIR_EL1 = 0xc510, + HV_SYS_REG_AMAIR_EL1 = 0xc518, + HV_SYS_REG_VBAR_EL1 = 0xc600, + HV_SYS_REG_CONTEXTIDR_EL1 = 0xc681, + HV_SYS_REG_TPIDR_EL1 = 0xc684, + HV_SYS_REG_CNTKCTL_EL1 = 0xc708, + HV_SYS_REG_CSSELR_EL1 = 0xd000, + HV_SYS_REG_TPIDR_EL0 = 0xde82, + HV_SYS_REG_TPIDRRO_EL0 = 0xde83, + HV_SYS_REG_CNTV_CTL_EL0 = 0xdf19, + HV_SYS_REG_CNTV_CVAL_EL0 = 0xdf1a, + HV_SYS_REG_SP_EL1 = 0xe208, + } + + enum hv_memory_flags_t : ulong + { + HV_MEMORY_READ = 1UL << 0, + HV_MEMORY_WRITE = 1UL << 1, + HV_MEMORY_EXEC = 1UL << 2 + } + + enum hv_result_t : uint + { + HV_SUCCESS = 0, + HV_ERROR = 0xfae94001, + HV_BUSY = 0xfae94002, + HV_BAD_ARGUMENT = 0xfae94003, + HV_NO_RESOURCES = 0xfae94005, + HV_NO_DEVICE = 0xfae94006, + HV_DENIED = 0xfae94007, + HV_UNSUPPORTED = 0xfae9400f + } + + enum hv_interrupt_type_t : uint + { + HV_INTERRUPT_TYPE_IRQ, + HV_INTERRUPT_TYPE_FIQ + } + + struct hv_simd_fp_uchar16_t + { + public ulong Low; + public ulong High; + } + + static class HvResultExtensions + { + public static void ThrowOnError(this hv_result_t result) + { + if (result != hv_result_t.HV_SUCCESS) + { + throw new Exception($"Unexpected result \"{result}\"."); + } + } + } + + static partial class HvApi + { + public const string LibraryName = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor"; + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vm_get_max_vcpu_count(out uint max_vcpu_count); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vm_create(IntPtr config); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vm_destroy(); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vm_map(ulong addr, ulong ipa, ulong size, hv_memory_flags_t flags); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vm_unmap(ulong ipa, ulong size); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vm_protect(ulong ipa, ulong size, hv_memory_flags_t flags); + + [LibraryImport(LibraryName, SetLastError = true)] + public unsafe static partial hv_result_t hv_vcpu_create(out ulong vcpu, ref hv_vcpu_exit_t* exit, IntPtr config); + + [LibraryImport(LibraryName, SetLastError = true)] + public unsafe static partial hv_result_t hv_vcpu_destroy(ulong vcpu); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_run(ulong vcpu); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpus_exit(ref ulong vcpus, uint vcpu_count); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_set_vtimer_mask(ulong vcpu, [MarshalAs(UnmanagedType.Bool)] bool vtimer_is_masked); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_get_reg(ulong vcpu, hv_reg_t reg, out ulong value); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_set_reg(ulong vcpu, hv_reg_t reg, ulong value); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_get_simd_fp_reg(ulong vcpu, hv_simd_fp_reg_t reg, out hv_simd_fp_uchar16_t value); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_set_simd_fp_reg(ulong vcpu, hv_simd_fp_reg_t reg, hv_simd_fp_uchar16_t value); // DO NOT USE DIRECTLY! + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_get_sys_reg(ulong vcpu, hv_sys_reg_t reg, out ulong value); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_set_sys_reg(ulong vcpu, hv_sys_reg_t reg, ulong value); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_get_pending_interrupt(ulong vcpu, hv_interrupt_type_t type, [MarshalAs(UnmanagedType.Bool)] out bool pending); + + [LibraryImport(LibraryName, SetLastError = true)] + public static partial hv_result_t hv_vcpu_set_pending_interrupt(ulong vcpu, hv_interrupt_type_t type, [MarshalAs(UnmanagedType.Bool)] bool pending); + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvCpuContext.cs b/Ryujinx.Cpu/AppleHv/HvCpuContext.cs new file mode 100644 index 0000000000..de782d541b --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvCpuContext.cs @@ -0,0 +1,47 @@ +using ARMeilleure.Memory; +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvCpuContext : ICpuContext + { + private readonly ITickSource _tickSource; + private readonly HvMemoryManager _memoryManager; + + public HvCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit) + { + _tickSource = tickSource; + _memoryManager = (HvMemoryManager)memory; + } + + private void UnmapHandler(ulong address, ulong size) + { + } + + /// + public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks) + { + return new HvExecutionContext(_tickSource, exceptionCallbacks); + } + + /// + public void Execute(IExecutionContext context, ulong address) + { + ((HvExecutionContext)context).Execute(_memoryManager, address); + } + + /// + public void InvalidateCacheRegion(ulong address, ulong size) + { + } + + public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled) + { + return new DummyDiskCacheLoadState(); + } + + public void PrepareCodeRange(ulong address, ulong size) + { + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvEngine.cs b/Ryujinx.Cpu/AppleHv/HvEngine.cs new file mode 100644 index 0000000000..7ad99cb9c4 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvEngine.cs @@ -0,0 +1,20 @@ +using ARMeilleure.Memory; + +namespace Ryujinx.Cpu.AppleHv +{ + public class HvEngine : ICpuEngine + { + private readonly ITickSource _tickSource; + + public HvEngine(ITickSource tickSource) + { + _tickSource = tickSource; + } + + /// + public ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit) + { + return new HvCpuContext(_tickSource, memoryManager, for64Bit); + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvExecutionContext.cs b/Ryujinx.Cpu/AppleHv/HvExecutionContext.cs new file mode 100644 index 0000000000..dc1f6f6dad --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvExecutionContext.cs @@ -0,0 +1,284 @@ +using ARMeilleure.State; +using Ryujinx.Cpu.AppleHv.Arm; +using Ryujinx.Memory.Tracking; +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvExecutionContext : IExecutionContext + { + /// + public ulong Pc => _impl.ElrEl1; + + /// + public long TpidrEl0 + { + get => _impl.TpidrEl0; + set => _impl.TpidrEl0 = value; + } + + /// + public long TpidrroEl0 + { + get => _impl.TpidrroEl0; + set => _impl.TpidrroEl0 = value; + } + + /// + public uint Pstate + { + get => _impl.Pstate; + set => _impl.Pstate = value; + } + + /// + public uint Fpcr + { + get => _impl.Fpcr; + set => _impl.Fpcr = value; + } + + /// + public uint Fpsr + { + get => _impl.Fpsr; + set => _impl.Fpsr = value; + } + + /// + public bool IsAarch32 + { + get => false; + set + { + if (value) + { + throw new NotSupportedException(); + } + } + } + + /// + public bool Running { get; private set; } + + private readonly ICounter _counter; + private readonly IHvExecutionContext _shadowContext; + private IHvExecutionContext _impl; + + private readonly ExceptionCallbacks _exceptionCallbacks; + + public HvExecutionContext(ICounter counter, ExceptionCallbacks exceptionCallbacks) + { + _counter = counter; + _shadowContext = new HvExecutionContextShadow(); + _impl = _shadowContext; + _exceptionCallbacks = exceptionCallbacks; + Running = true; + } + + /// + public ulong GetX(int index) => _impl.GetX(index); + + /// + public void SetX(int index, ulong value) => _impl.SetX(index, value); + + /// + public V128 GetV(int index) => _impl.GetV(index); + + /// + public void SetV(int index, V128 value) => _impl.SetV(index, value); + + private void InterruptHandler() + { + _exceptionCallbacks.InterruptCallback?.Invoke(this); + } + + private void BreakHandler(ulong address, int imm) + { + _exceptionCallbacks.BreakCallback?.Invoke(this, address, imm); + } + + private void SupervisorCallHandler(ulong address, int imm) + { + _exceptionCallbacks.SupervisorCallback?.Invoke(this, address, imm); + } + + private void UndefinedHandler(ulong address, int opCode) + { + _exceptionCallbacks.UndefinedCallback?.Invoke(this, address, opCode); + } + + /// + public void RequestInterrupt() + { + _impl.RequestInterrupt(); + } + + /// + public void StopRunning() + { + Running = false; + RequestInterrupt(); + } + + public unsafe void Execute(HvMemoryManager memoryManager, ulong address) + { + HvVcpu vcpu = HvVcpuPool.Instance.Create(memoryManager.AddressSpace, _shadowContext, SwapContext); + + HvApi.hv_vcpu_set_reg(vcpu.Handle, hv_reg_t.HV_REG_PC, address).ThrowOnError(); + + while (Running) + { + HvApi.hv_vcpu_run(vcpu.Handle).ThrowOnError(); + + uint reason = vcpu.ExitInfo->reason; + + if (reason == 1) + { + uint hvEsr = (uint)vcpu.ExitInfo->exception.syndrome; + ExceptionClass hvEc = (ExceptionClass)(hvEsr >> 26); + + if (hvEc != ExceptionClass.HvcAarch64) + { + throw new Exception($"Unhandled exception from guest kernel with ESR 0x{hvEsr:X} ({hvEc})."); + } + + address = SynchronousException(memoryManager, ref vcpu); + HvApi.hv_vcpu_set_reg(vcpu.Handle, hv_reg_t.HV_REG_PC, address).ThrowOnError(); + } + else if (reason == 0) + { + if (_impl.GetAndClearInterruptRequested()) + { + ReturnToPool(vcpu); + InterruptHandler(); + vcpu = RentFromPool(memoryManager.AddressSpace, vcpu); + } + } + else + { + throw new Exception($"Unhandled exit reason {reason}."); + } + } + + HvVcpuPool.Instance.Destroy(vcpu, SwapContext); + } + + private ulong SynchronousException(HvMemoryManager memoryManager, ref HvVcpu vcpu) + { + ulong vcpuHandle = vcpu.Handle; + + HvApi.hv_vcpu_get_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, out ulong elr).ThrowOnError(); + HvApi.hv_vcpu_get_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_ESR_EL1, out ulong esr).ThrowOnError(); + + ExceptionClass ec = (ExceptionClass)((uint)esr >> 26); + + switch (ec) + { + case ExceptionClass.DataAbortLowerEl: + DataAbort(memoryManager.Tracking, vcpuHandle, (uint)esr); + break; + case ExceptionClass.TrappedMsrMrsSystem: + InstructionTrap((uint)esr); + HvApi.hv_vcpu_set_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, elr + 4UL).ThrowOnError(); + break; + case ExceptionClass.SvcAarch64: + ReturnToPool(vcpu); + ushort id = (ushort)esr; + SupervisorCallHandler(elr - 4UL, id); + vcpu = RentFromPool(memoryManager.AddressSpace, vcpu); + break; + default: + throw new Exception($"Unhandled guest exception {ec}."); + } + + // Make sure we will continue running at EL0. + if (memoryManager.AddressSpace.GetAndClearUserTlbInvalidationPending()) + { + // TODO: Invalidate only the range that was modified? + return HvAddressSpace.KernelRegionTlbiEretAddress; + } + else + { + return HvAddressSpace.KernelRegionEretAddress; + } + } + + private void DataAbort(MemoryTracking tracking, ulong vcpu, uint esr) + { + bool write = (esr & (1u << 6)) != 0; + bool farValid = (esr & (1u << 10)) == 0; + int accessSizeLog2 = (int)((esr >> 22) & 3); + + if (farValid) + { + HvApi.hv_vcpu_get_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_FAR_EL1, out ulong far).ThrowOnError(); + + ulong size = 1UL << accessSizeLog2; + + if (!tracking.VirtualMemoryEvent(far, size, write)) + { + string rw = write ? "write" : "read"; + throw new Exception($"Unhandled invalid memory access at VA 0x{far:X} with size 0x{size:X} ({rw})."); + } + } + else + { + throw new Exception($"Unhandled invalid memory access at unknown VA with ESR 0x{esr:X}."); + } + } + + private void InstructionTrap(uint esr) + { + bool read = (esr & 1) != 0; + uint rt = (esr >> 5) & 0x1f; + + if (read) + { + // Op0 Op2 Op1 CRn 00000 CRm + switch ((esr >> 1) & 0x1ffe0f) + { + case 0b11_000_011_1110_00000_0000: // CNTFRQ_EL0 + WriteRt(rt, _counter.Frequency); + break; + case 0b11_001_011_1110_00000_0000: // CNTPCT_EL0 + WriteRt(rt, _counter.Counter); + break; + default: + throw new Exception($"Unhandled system register read with ESR 0x{esr:X}"); + } + } + else + { + throw new Exception($"Unhandled system register write with ESR 0x{esr:X}"); + } + } + + private void WriteRt(uint rt, ulong value) + { + if (rt < 31) + { + SetX((int)rt, value); + } + } + + private void ReturnToPool(HvVcpu vcpu) + { + HvVcpuPool.Instance.Return(vcpu, SwapContext); + } + + private HvVcpu RentFromPool(HvAddressSpace addressSpace, HvVcpu vcpu) + { + return HvVcpuPool.Instance.Rent(addressSpace, _shadowContext, vcpu, SwapContext); + } + + private void SwapContext(IHvExecutionContext newContext) + { + _impl = newContext; + } + + public void Dispose() + { + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs b/Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs new file mode 100644 index 0000000000..c088ebdcf3 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs @@ -0,0 +1,59 @@ +using ARMeilleure.State; + +namespace Ryujinx.Cpu.AppleHv +{ + unsafe class HvExecutionContextShadow : IHvExecutionContext + { + public ulong Pc { get; set; } + public ulong ElrEl1 { get; set; } + public ulong EsrEl1 { get; set; } + + public long TpidrEl0 { get; set; } + public long TpidrroEl0 { get; set; } + + public uint Pstate { get; set; } + + public uint Fpcr { get; set; } + public uint Fpsr { get; set; } + + public bool IsAarch32 { get; set; } + + private readonly ulong[] _x; + private readonly V128[] _v; + + public HvExecutionContextShadow() + { + _x = new ulong[32]; + _v = new V128[32]; + } + + public ulong GetX(int index) + { + return _x[index]; + } + + public void SetX(int index, ulong value) + { + _x[index] = value; + } + + public V128 GetV(int index) + { + return _v[index]; + } + + public void SetV(int index, V128 value) + { + _v[index] = value; + } + + public void RequestInterrupt() + { + } + + public bool GetAndClearInterruptRequested() + { + return false; + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs b/Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs new file mode 100644 index 0000000000..4f6ebefa72 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs @@ -0,0 +1,196 @@ +using ARMeilleure.State; +using Ryujinx.Memory; +using System; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvExecutionContextVcpu : IHvExecutionContext + { + private static MemoryBlock _setSimdFpRegFuncMem; + private delegate hv_result_t SetSimdFpReg(ulong vcpu, hv_simd_fp_reg_t reg, in V128 value, IntPtr funcPtr); + private static SetSimdFpReg _setSimdFpReg; + private static IntPtr _setSimdFpRegNativePtr; + + static HvExecutionContextVcpu() + { + // .NET does not support passing vectors by value, so we need to pass a pointer and use a native + // function to load the value into a vector register. + _setSimdFpRegFuncMem = new MemoryBlock(MemoryBlock.GetPageSize()); + _setSimdFpRegFuncMem.Write(0, 0x3DC00040u); // LDR Q0, [X2] + _setSimdFpRegFuncMem.Write(4, 0xD61F0060u); // BR X3 + _setSimdFpRegFuncMem.Reprotect(0, _setSimdFpRegFuncMem.Size, MemoryPermission.ReadAndExecute); + + _setSimdFpReg = Marshal.GetDelegateForFunctionPointer(_setSimdFpRegFuncMem.Pointer); + + if (NativeLibrary.TryLoad(HvApi.LibraryName, out IntPtr hvLibHandle)) + { + _setSimdFpRegNativePtr = NativeLibrary.GetExport(hvLibHandle, nameof(HvApi.hv_vcpu_set_simd_fp_reg)); + } + } + + public ulong Pc + { + get + { + HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_PC, out ulong pc).ThrowOnError(); + return pc; + } + set + { + HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_PC, value).ThrowOnError(); + } + } + + public ulong ElrEl1 + { + get + { + HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, out ulong elr).ThrowOnError(); + return elr; + } + set + { + HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, value).ThrowOnError(); + } + } + + public ulong EsrEl1 + { + get + { + HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ESR_EL1, out ulong esr).ThrowOnError(); + return esr; + } + set + { + HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ESR_EL1, value).ThrowOnError(); + } + } + + public long TpidrEl0 + { + get + { + HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDR_EL0, out ulong tpidrEl0).ThrowOnError(); + return (long)tpidrEl0; + } + set + { + HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDR_EL0, (ulong)value).ThrowOnError(); + } + } + + public long TpidrroEl0 + { + get + { + HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDRRO_EL0, out ulong tpidrroEl0).ThrowOnError(); + return (long)tpidrroEl0; + } + set + { + HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDRRO_EL0, (ulong)value).ThrowOnError(); + } + } + + public uint Pstate + { + get + { + HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_CPSR, out ulong cpsr).ThrowOnError(); + return (uint)cpsr; + } + set + { + HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_CPSR, (ulong)value).ThrowOnError(); + } + } + + public uint Fpcr + { + get + { + HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_FPCR, out ulong fpcr).ThrowOnError(); + return (uint)fpcr; + } + set + { + HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_FPCR, (ulong)value).ThrowOnError(); + } + } + + public uint Fpsr + { + get + { + HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_FPSR, out ulong fpsr).ThrowOnError(); + return (uint)fpsr; + } + set + { + HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_FPSR, (ulong)value).ThrowOnError(); + } + } + + private ulong _vcpu; + private int _interruptRequested; + + public HvExecutionContextVcpu(ulong vcpu) + { + _vcpu = vcpu; + } + + public ulong GetX(int index) + { + if (index == 31) + { + HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_SP_EL0, out ulong value).ThrowOnError(); + return value; + } + else + { + HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_X0 + (uint)index, out ulong value).ThrowOnError(); + return value; + } + } + + public void SetX(int index, ulong value) + { + if (index == 31) + { + HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_SP_EL0, value).ThrowOnError(); + } + else + { + HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_X0 + (uint)index, value).ThrowOnError(); + } + } + + public V128 GetV(int index) + { + HvApi.hv_vcpu_get_simd_fp_reg(_vcpu, hv_simd_fp_reg_t.HV_SIMD_FP_REG_Q0 + (uint)index, out hv_simd_fp_uchar16_t value).ThrowOnError(); + return new V128(value.Low, value.High); + } + + public void SetV(int index, V128 value) + { + _setSimdFpReg(_vcpu, hv_simd_fp_reg_t.HV_SIMD_FP_REG_Q0 + (uint)index, value, _setSimdFpRegNativePtr).ThrowOnError(); + } + + public void RequestInterrupt() + { + if (Interlocked.Exchange(ref _interruptRequested, 1) == 0) + { + ulong vcpu = _vcpu; + HvApi.hv_vcpus_exit(ref vcpu, 1); + } + } + + public bool GetAndClearInterruptRequested() + { + return Interlocked.Exchange(ref _interruptRequested, 0) != 0; + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs b/Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs new file mode 100644 index 0000000000..7eefe130b9 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs @@ -0,0 +1,34 @@ +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvIpaAllocator + { + private const ulong AllocationGranule = 1UL << 14; + private const ulong IpaRegionSize = 1UL << 35; + + private readonly PrivateMemoryAllocator.Block _block; + + public HvIpaAllocator() + { + _block = new PrivateMemoryAllocator.Block(null, IpaRegionSize); + } + + public ulong Allocate(ulong size, ulong alignment = AllocationGranule) + { + ulong offset = _block.Allocate(size, alignment); + + if (offset == PrivateMemoryAllocator.InvalidOffset) + { + throw new InvalidOperationException($"No enough free IPA memory to allocate 0x{size:X} bytes with alignment 0x{alignment:X}."); + } + + return offset; + } + + public void Free(ulong offset, ulong size) + { + _block.Free(offset, size); + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocation.cs b/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocation.cs new file mode 100644 index 0000000000..94289d1c37 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocation.cs @@ -0,0 +1,34 @@ +using Ryujinx.Memory; +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + struct HvMemoryBlockAllocation : IDisposable + { + private readonly HvMemoryBlockAllocator _owner; + private readonly HvMemoryBlockAllocator.Block _block; + + public bool IsValid => _owner != null; + public MemoryBlock Memory => _block.Memory; + public ulong Ipa => _block.Ipa; + public ulong Offset { get; } + public ulong Size { get; } + + public HvMemoryBlockAllocation( + HvMemoryBlockAllocator owner, + HvMemoryBlockAllocator.Block block, + ulong offset, + ulong size) + { + _owner = owner; + _block = block; + Offset = offset; + Size = size; + } + + public void Dispose() + { + _owner.Free(_block, Offset, Size); + } + } +} diff --git a/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs b/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs new file mode 100644 index 0000000000..24c3a969cc --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs @@ -0,0 +1,59 @@ +using Ryujinx.Memory; +using System.Collections.Generic; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvMemoryBlockAllocator : PrivateMemoryAllocatorImpl + { + private const ulong InvalidOffset = ulong.MaxValue; + + public class Block : PrivateMemoryAllocator.Block + { + private readonly HvIpaAllocator _ipaAllocator; + public ulong Ipa { get; } + + public Block(HvIpaAllocator ipaAllocator, MemoryBlock memory, ulong size) : base(memory, size) + { + _ipaAllocator = ipaAllocator; + + lock (ipaAllocator) + { + Ipa = ipaAllocator.Allocate(size); + } + + HvApi.hv_vm_map((ulong)Memory.Pointer, Ipa, size, hv_memory_flags_t.HV_MEMORY_READ | hv_memory_flags_t.HV_MEMORY_WRITE).ThrowOnError(); + } + + public override void Destroy() + { + HvApi.hv_vm_unmap(Ipa, Size).ThrowOnError(); + + lock (_ipaAllocator) + { + _ipaAllocator.Free(Ipa, Size); + } + + base.Destroy(); + } + } + + private readonly HvIpaAllocator _ipaAllocator; + + public HvMemoryBlockAllocator(HvIpaAllocator ipaAllocator, int blockAlignment) : base(blockAlignment, MemoryAllocationFlags.None) + { + _ipaAllocator = ipaAllocator; + } + + public unsafe HvMemoryBlockAllocation Allocate(ulong size, ulong alignment) + { + var allocation = Allocate(size, alignment, CreateBlock); + + return new HvMemoryBlockAllocation(this, allocation.Block, allocation.Offset, allocation.Size); + } + + private Block CreateBlock(MemoryBlock memory, ulong size) + { + return new Block(_ipaAllocator, memory, size); + } + } +} diff --git a/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs new file mode 100644 index 0000000000..222dcae1b8 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs @@ -0,0 +1,947 @@ +using ARMeilleure.Memory; +using Ryujinx.Cpu.Tracking; +using Ryujinx.Memory; +using Ryujinx.Memory.Range; +using Ryujinx.Memory.Tracking; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Ryujinx.Cpu.AppleHv +{ + /// + /// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table. + /// + public class HvMemoryManager : MemoryManagerBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + { + public const int PageBits = 12; + public const int PageSize = 1 << PageBits; + public const int PageMask = PageSize - 1; + + public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. + public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. + + private enum HostMappedPtBits : ulong + { + Unmapped = 0, + Mapped, + WriteTracked, + ReadWriteTracked, + + MappedReplicated = 0x5555555555555555, + WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, + ReadWriteTrackedReplicated = ulong.MaxValue + } + + private readonly InvalidAccessHandler _invalidAccessHandler; + + private readonly ulong _addressSpaceSize; + + private readonly HvAddressSpace _addressSpace; + + internal HvAddressSpace AddressSpace => _addressSpace; + + private readonly MemoryBlock _backingMemory; + private readonly PageTable _pageTable; + + private readonly ulong[] _pageBitmap; + + public bool Supports4KBPages => true; + + public int AddressSpaceBits { get; } + + public IntPtr PageTablePointer => IntPtr.Zero; + + public MemoryManagerType Type => MemoryManagerType.SoftwarePageTable; + + public MemoryTracking Tracking { get; } + + public event Action UnmapEvent; + + /// + /// Creates a new instance of the Hypervisor memory manager. + /// + /// Physical backing memory where virtual memory will be mapped to + /// Size of the address space + /// Optional function to handle invalid memory accesses + public HvMemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null) + { + _backingMemory = backingMemory; + _pageTable = new PageTable(); + _invalidAccessHandler = invalidAccessHandler; + _addressSpaceSize = addressSpaceSize; + + ulong asSize = PageSize; + int asBits = PageBits; + + while (asSize < addressSpaceSize) + { + asSize <<= 1; + asBits++; + } + + _addressSpace = new HvAddressSpace(backingMemory, asSize); + + AddressSpaceBits = asBits; + + _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler); + } + + /// + /// Checks if the virtual address is part of the addressable space. + /// + /// Virtual address + /// True if the virtual address is part of the addressable space + private bool ValidateAddress(ulong va) + { + return va < _addressSpaceSize; + } + + /// + /// Checks if the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// True if the combination of virtual address and size is part of the addressable space + private bool ValidateAddressAndSize(ulong va, ulong size) + { + ulong endVa = va + size; + return endVa >= va && endVa >= size && endVa <= _addressSpaceSize; + } + + /// + /// Ensures the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// Throw when the memory region specified outside the addressable space + private void AssertValidAddressAndSize(ulong va, ulong size) + { + if (!ValidateAddressAndSize(va, size)) + { + throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}"); + } + } + + /// + /// Ensures the combination of virtual address and size is part of the addressable space and fully mapped. + /// + /// Virtual address of the range + /// Size of the range in bytes + private void AssertMapped(ulong va, ulong size) + { + if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size)) + { + throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + } + + /// + public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags) + { + AssertValidAddressAndSize(va, size); + + PtMap(va, pa, size); + _addressSpace.MapUser(va, pa, size, MemoryPermission.ReadWriteExecute); + AddMapping(va, size); + + Tracking.Map(va, size); + } + + private void PtMap(ulong va, ulong pa, ulong size) + { + while (size != 0) + { + _pageTable.Map(va, pa); + + va += PageSize; + pa += PageSize; + size -= PageSize; + } + } + + /// + public void MapForeign(ulong va, nuint hostPointer, ulong size) + { + throw new NotSupportedException(); + } + + /// + public void Unmap(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + UnmapEvent?.Invoke(va, size); + Tracking.Unmap(va, size); + + RemoveMapping(va, size); + _addressSpace.UnmapUser(va, size); + PtUnmap(va, size); + } + + private void PtUnmap(ulong va, ulong size) + { + while (size != 0) + { + _pageTable.Unmap(va); + + va += PageSize; + size -= PageSize; + } + } + + /// + public T Read(ulong va) where T : unmanaged + { + return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; + } + + /// + public T ReadTracked(ulong va) where T : unmanaged + { + try + { + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); + + return Read(va); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + + return default; + } + } + + /// + public void Read(ulong va, Span data) + { + ReadImpl(va, data); + } + + /// + public void Write(ulong va, T value) where T : unmanaged + { + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + } + + /// + public void Write(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + SignalMemoryTracking(va, (ulong)data.Length, true); + + WriteImpl(va, data); + } + + /// + public void WriteUntracked(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + WriteImpl(va, data); + } + + /// + public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return false; + } + + SignalMemoryTracking(va, (ulong)data.Length, false); + + if (IsContiguousAndMapped(va, data.Length)) + { + var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); + + bool changed = !data.SequenceEqual(target); + + if (changed) + { + data.CopyTo(target); + } + + return changed; + } + else + { + WriteImpl(va, data); + + return true; + } + } + + private void WriteImpl(ulong va, ReadOnlySpan data) + { + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + if (IsContiguousAndMapped(va, data.Length)) + { + data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = GetPhysicalAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); + } + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + /// + public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySpan.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); + } + else + { + Span data = new byte[size]; + + ReadImpl(va, data); + + return data; + } + } + + /// + public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return new WritableRegion(null, va, Memory.Empty); + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, true); + } + + if (IsContiguousAndMapped(va, size)) + { + return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); + } + else + { + Memory memory = new byte[size]; + + ReadImpl(va, memory.Span); + + return new WritableRegion(this, va, memory); + } + } + + /// + public ref T GetRef(ulong va) where T : unmanaged + { + if (!IsContiguous(va, Unsafe.SizeOf())) + { + ThrowMemoryNotContiguous(); + } + + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); + + return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsMapped(ulong va) + { + return ValidateAddress(va) && IsMappedImpl(va); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsMappedImpl(ulong va) + { + ulong page = va >> PageBits; + + int bit = (int)((page & 31) << 1); + + int pageIndex = (int)(page >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + + return ((pte >> bit) & 3) != 0; + } + + /// + public bool IsRangeMapped(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + return IsRangeMappedImpl(va, size); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) + { + startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); + endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); + + pageIndex = (int)(pageStart >> PageToPteShift); + pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); + } + + private bool IsRangeMappedImpl(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + + if (pages == 1) + { + return IsMappedImpl(va); + } + + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + // Check if either bit in each 2 bit page entry is set. + // OR the block with itself shifted down by 1, and check the first bit of each entry. + + ulong mask = BlockMappedMask & startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte = Volatile.Read(ref pageRef); + + pte |= pte >> 1; + if ((pte & mask) != mask) + { + return false; + } + + mask = BlockMappedMask; + } + + return true; + } + + private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsContiguous(ulong va, int size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) + { + return false; + } + + int pages = GetPagesCount(va, (uint)size, out va); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return false; + } + + if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + /// + public IEnumerable GetHostRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + var guestRegions = GetPhysicalRegionsImpl(va, size); + if (guestRegions == null) + { + return null; + } + + var regions = new HostMemoryRange[guestRegions.Count]; + + for (int i = 0; i < regions.Length; i++) + { + var guestRegion = guestRegions[i]; + IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size); + regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size); + } + + return regions; + } + + /// + public IEnumerable GetPhysicalRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + return GetPhysicalRegionsImpl(va, size); + } + + private List GetPhysicalRegionsImpl(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return null; + } + + int pages = GetPagesCount(va, (uint)size, out va); + + var regions = new List(); + + ulong regionStart = GetPhysicalAddressInternal(va); + ulong regionSize = PageSize; + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return null; + } + + ulong newPa = GetPhysicalAddressInternal(va + PageSize); + + if (GetPhysicalAddressInternal(va) + PageSize != newPa) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + regionStart = newPa; + regionSize = 0; + } + + va += PageSize; + regionSize += PageSize; + } + + regions.Add(new MemoryRange(regionStart, regionSize)); + + return regions; + } + + private void ReadImpl(ulong va, Span data) + { + if (data.Length == 0) + { + return; + } + + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = GetPhysicalAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size)); + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + /// + /// + /// This function also validates that the given range is both valid and mapped, and will throw if it is not. + /// + public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false) + { + AssertValidAddressAndSize(va, size); + + if (precise) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: true); + return; + } + + // Software table, used for managed memory tracking. + + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); + + int bit = (int)((pageStart & 31) << 1); + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + ulong state = ((pte >> bit) & 3); + + if (state >= tag) + { + Tracking.VirtualMemoryEvent(va, size, write); + return; + } + else if (state == 0) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte = Volatile.Read(ref pageRef); + ulong mappedMask = mask & BlockMappedMask; + + ulong mappedPte = pte | (pte >> 1); + if ((mappedPte & mappedMask) != mappedMask) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + + pte &= mask; + if ((pte & anyTrackingTag) != 0) // Search for any tracking. + { + // Writes trigger any tracking. + // Only trigger tracking from reads if both bits are set on any page. + if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) + { + Tracking.VirtualMemoryEvent(va, size, write); + break; + } + } + + mask = ulong.MaxValue; + } + } + } + + /// + /// Computes the number of pages in a virtual address range. + /// + /// Virtual address of the range + /// Size of the range + /// The virtual address of the beginning of the first page + /// This function does not differentiate between allocated and unallocated pages. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + /// + public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + { + // Protection is inverted on software pages, since the default value is 0. + protection = (~protection) & MemoryPermission.ReadAndWrite; + + int pages = GetPagesCount(va, size, out va); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, + MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, + _ => (ulong)HostMappedPtBits.ReadWriteTracked, + }; + + int bit = (int)((pageStart & 31) << 1); + + ulong tagMask = 3UL << bit; + ulong invTagMask = ~tagMask; + + ulong tag = protTag << bit; + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, + MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, + _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, + }; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Change the protection of all 2 bit entries that are mapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask &= mask; // Only update mapped pages within the given range. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); + + mask = ulong.MaxValue; + } + } + + protection = protection switch + { + MemoryPermission.None => MemoryPermission.ReadAndWrite, + MemoryPermission.Write => MemoryPermission.Read, + _ => MemoryPermission.None + }; + + _addressSpace.ReprotectUser(va, size, protection); + } + + /// + public CpuRegionHandle BeginTracking(ulong address, ulong size) + { + return new CpuRegionHandle(Tracking.BeginTracking(address, size)); + } + + /// + public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity) + { + return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity)); + } + + /// + public CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity) + { + return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity)); + } + + /// + /// Adds the given address mapping to the page table. + /// + /// Virtual memory address + /// Size to be mapped + private void AddMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Map all 2-bit entries that are unmapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); + + mask = ulong.MaxValue; + } + } + + /// + /// Removes the given address mapping from the page table. + /// + /// Virtual memory address + /// Size to be unmapped + private void RemoveMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + startMask = ~startMask; + endMask = ~endMask; + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask |= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); + + mask = 0; + } + } + + private ulong GetPhysicalAddressChecked(ulong va) + { + if (!IsMapped(va)) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); + } + + return GetPhysicalAddressInternal(va); + } + + private ulong GetPhysicalAddressInternal(ulong va) + { + return _pageTable.Read(va) + (va & PageMask); + } + + /// + /// Disposes of resources used by the memory manager. + /// + protected override void Destroy() + { + _addressSpace.Dispose(); + } + + private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvVcpu.cs b/Ryujinx.Cpu/AppleHv/HvVcpu.cs new file mode 100644 index 0000000000..484a9fe829 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvVcpu.cs @@ -0,0 +1,25 @@ +namespace Ryujinx.Cpu.AppleHv +{ + unsafe class HvVcpu + { + public readonly ulong Handle; + public readonly hv_vcpu_exit_t* ExitInfo; + public readonly IHvExecutionContext ShadowContext; + public readonly IHvExecutionContext NativeContext; + public readonly bool IsEphemeral; + + public HvVcpu( + ulong handle, + hv_vcpu_exit_t* exitInfo, + IHvExecutionContext shadowContext, + IHvExecutionContext nativeContext, + bool isEphemeral) + { + Handle = handle; + ExitInfo = exitInfo; + ShadowContext = shadowContext; + NativeContext = nativeContext; + IsEphemeral = isEphemeral; + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvVcpuPool.cs b/Ryujinx.Cpu/AppleHv/HvVcpuPool.cs new file mode 100644 index 0000000000..cb1944fe7e --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvVcpuPool.cs @@ -0,0 +1,103 @@ +using System; +using System.Threading; + +namespace Ryujinx.Cpu.AppleHv +{ + class HvVcpuPool + { + // Since there's a limit on the number of VCPUs we can create, + // and we assign one VCPU per guest thread, we need to ensure + // there are enough VCPUs available for at least the maximum number of active guest threads. + // To do that, we always destroy and re-create VCPUs that are above a given limit. + // Those VCPUs are called "ephemeral" here because they are not kept for long. + // + // In the future, we might want to consider a smarter approach that only makes + // VCPUs for threads that are not running frequently "ephemeral", but this is + // complicated because VCPUs can only be destroyed by the same thread that created them. + + private const int MaxActiveVcpus = 4; + + public static readonly HvVcpuPool Instance = new HvVcpuPool(); + + private int _totalVcpus; + private int _maxVcpus; + + public HvVcpuPool() + { + HvApi.hv_vm_get_max_vcpu_count(out uint maxVcpuCount).ThrowOnError(); + _maxVcpus = (int)maxVcpuCount; + } + + public HvVcpu Create(HvAddressSpace addressSpace, IHvExecutionContext shadowContext, Action swapContext) + { + HvVcpu vcpu = CreateNew(addressSpace, shadowContext); + vcpu.NativeContext.Load(shadowContext); + swapContext(vcpu.NativeContext); + return vcpu; + } + + public void Destroy(HvVcpu vcpu, Action swapContext) + { + vcpu.ShadowContext.Load(vcpu.NativeContext); + swapContext(vcpu.ShadowContext); + DestroyVcpu(vcpu); + } + + public void Return(HvVcpu vcpu, Action swapContext) + { + if (vcpu.IsEphemeral) + { + Destroy(vcpu, swapContext); + } + } + + public HvVcpu Rent(HvAddressSpace addressSpace, IHvExecutionContext shadowContext, HvVcpu vcpu, Action swapContext) + { + if (vcpu.IsEphemeral) + { + return Create(addressSpace, shadowContext, swapContext); + } + else + { + return vcpu; + } + } + + private unsafe HvVcpu CreateNew(HvAddressSpace addressSpace, IHvExecutionContext shadowContext) + { + int newCount = IncrementVcpuCount(); + bool isEphemeral = newCount > _maxVcpus - MaxActiveVcpus; + + // Create VCPU. + hv_vcpu_exit_t* exitInfo = null; + HvApi.hv_vcpu_create(out ulong vcpuHandle, ref exitInfo, IntPtr.Zero).ThrowOnError(); + + // Enable FP and SIMD instructions. + HvApi.hv_vcpu_set_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_CPACR_EL1, 0b11 << 20).ThrowOnError(); + + addressSpace.InitializeMmu(vcpuHandle); + + HvExecutionContextVcpu nativeContext = new HvExecutionContextVcpu(vcpuHandle); + + HvVcpu vcpu = new HvVcpu(vcpuHandle, exitInfo, shadowContext, nativeContext, isEphemeral); + + return vcpu; + } + + private void DestroyVcpu(HvVcpu vcpu) + { + HvApi.hv_vcpu_destroy(vcpu.Handle).ThrowOnError(); + DecrementVcpuCount(); + } + + private int IncrementVcpuCount() + { + return Interlocked.Increment(ref _totalVcpus); + } + + private void DecrementVcpuCount() + { + Interlocked.Decrement(ref _totalVcpus); + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/HvVm.cs b/Ryujinx.Cpu/AppleHv/HvVm.cs new file mode 100644 index 0000000000..d91abff9ae --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/HvVm.cs @@ -0,0 +1,68 @@ +using Ryujinx.Memory; +using System; + +namespace Ryujinx.Cpu.AppleHv +{ + static class HvVm + { + // This alignment allows us to use larger blocks on the page table. + private const ulong AsIpaAlignment = 1UL << 30; + + private static int _addressSpaces; + private static HvIpaAllocator _ipaAllocator; + private static object _lock = new object(); + + public static (ulong, HvIpaAllocator) CreateAddressSpace(MemoryBlock block) + { + HvIpaAllocator ipaAllocator; + + lock (_lock) + { + if (++_addressSpaces == 1) + { + HvApi.hv_vm_create(IntPtr.Zero).ThrowOnError(); + _ipaAllocator = ipaAllocator = new HvIpaAllocator(); + } + else + { + ipaAllocator = _ipaAllocator; + } + } + + ulong baseAddress; + + lock (ipaAllocator) + { + baseAddress = ipaAllocator.Allocate(block.Size, AsIpaAlignment); + } + + var rwx = hv_memory_flags_t.HV_MEMORY_READ | hv_memory_flags_t.HV_MEMORY_WRITE | hv_memory_flags_t.HV_MEMORY_EXEC; + + HvApi.hv_vm_map((ulong)block.Pointer, baseAddress, block.Size, rwx).ThrowOnError(); + + return (baseAddress, ipaAllocator); + } + + public static void DestroyAddressSpace(ulong address, ulong size) + { + HvApi.hv_vm_unmap(address, size); + + HvIpaAllocator ipaAllocator; + + lock (_lock) + { + if (--_addressSpaces == 0) + { + HvApi.hv_vm_destroy().ThrowOnError(); + } + + ipaAllocator = _ipaAllocator; + } + + lock (ipaAllocator) + { + ipaAllocator.Free(address, size); + } + } + } +} \ No newline at end of file diff --git a/Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs b/Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs new file mode 100644 index 0000000000..adf2dd9975 --- /dev/null +++ b/Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs @@ -0,0 +1,46 @@ +using ARMeilleure.State; + +namespace Ryujinx.Cpu.AppleHv +{ + public interface IHvExecutionContext + { + ulong Pc { get; set; } + ulong ElrEl1 { get; set; } + ulong EsrEl1 { get; set; } + + long TpidrEl0 { get; set; } + long TpidrroEl0 { get; set; } + + uint Pstate { get; set; } + + uint Fpcr { get; set; } + uint Fpsr { get; set; } + + ulong GetX(int index); + void SetX(int index, ulong value); + + V128 GetV(int index); + void SetV(int index, V128 value); + + public void Load(IHvExecutionContext context) + { + Pc = context.Pc; + ElrEl1 = context.ElrEl1; + EsrEl1 = context.EsrEl1; + TpidrEl0 = context.TpidrEl0; + TpidrroEl0 = context.TpidrroEl0; + Pstate = context.Pstate; + Fpcr = context.Fpcr; + Fpsr = context.Fpsr; + + for (int i = 0; i < 32; i++) + { + SetX(i, context.GetX(i)); + SetV(i, context.GetV(i)); + } + } + + void RequestInterrupt(); + bool GetAndClearInterruptRequested(); + } +} \ No newline at end of file diff --git a/Ryujinx.HLE/HLEConfiguration.cs b/Ryujinx.HLE/HLEConfiguration.cs index 8fd02a9629..e21157f9b6 100644 --- a/Ryujinx.HLE/HLEConfiguration.cs +++ b/Ryujinx.HLE/HLEConfiguration.cs @@ -148,6 +148,11 @@ namespace Ryujinx.HLE /// public float AudioVolume { get; set; } + /// + /// Use Hypervisor over JIT if available. + /// + internal readonly bool UseHypervisor; + /// /// An action called when HLE force a refresh of output after docked mode changed. /// @@ -175,7 +180,8 @@ namespace Ryujinx.HLE MemoryManagerMode memoryManagerMode, bool ignoreMissingServices, AspectRatio aspectRatio, - float audioVolume) + float audioVolume, + bool useHypervisor) { VirtualFileSystem = virtualFileSystem; LibHacHorizonManager = libHacHorizonManager; @@ -200,6 +206,7 @@ namespace Ryujinx.HLE IgnoreMissingServices = ignoreMissingServices; AspectRatio = aspectRatio; AudioVolume = audioVolume; + UseHypervisor = useHypervisor; } } } \ No newline at end of file diff --git a/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs index 5ecaf38e90..1b0d66ac62 100644 --- a/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs +++ b/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs @@ -1,17 +1,19 @@ using Ryujinx.Common.Configuration; using Ryujinx.Cpu; +using Ryujinx.Cpu.AppleHv; using Ryujinx.Cpu.Jit; using Ryujinx.Graphics.Gpu; using Ryujinx.HLE.HOS.Kernel; using Ryujinx.HLE.HOS.Kernel.Process; using Ryujinx.Memory; using System; +using System.Runtime.InteropServices; namespace Ryujinx.HLE.HOS { class ArmProcessContextFactory : IProcessContextFactory { - private readonly ICpuEngine _cpuEngine; + private readonly ITickSource _tickSource; private readonly GpuContext _gpu; private readonly string _titleIdText; private readonly string _displayVersion; @@ -22,7 +24,7 @@ namespace Ryujinx.HLE.HOS public IDiskCacheLoadState DiskCacheLoadState { get; private set; } public ArmProcessContextFactory( - ICpuEngine cpuEngine, + ITickSource tickSource, GpuContext gpu, string titleIdText, string displayVersion, @@ -30,7 +32,7 @@ namespace Ryujinx.HLE.HOS ulong codeAddress, ulong codeSize) { - _cpuEngine = cpuEngine; + _tickSource = tickSource; _gpu = gpu; _titleIdText = titleIdText; _displayVersion = displayVersion; @@ -41,31 +43,42 @@ namespace Ryujinx.HLE.HOS public IProcessContext Create(KernelContext context, ulong pid, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit) { - MemoryManagerMode mode = context.Device.Configuration.MemoryManagerMode; - - if (!MemoryBlock.SupportsFlags(MemoryAllocationFlags.ViewCompatible)) - { - mode = MemoryManagerMode.SoftwarePageTable; - } - IArmProcessContext processContext; - switch (mode) + if (OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64 && for64Bit && context.Device.Configuration.UseHypervisor) { - case MemoryManagerMode.SoftwarePageTable: - var memoryManager = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); - processContext = new ArmProcessContext(pid, _cpuEngine, _gpu, memoryManager, for64Bit); - break; + var cpuEngine = new HvEngine(_tickSource); + var memoryManager = new HvMemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, for64Bit); + } + else + { + MemoryManagerMode mode = context.Device.Configuration.MemoryManagerMode; - case MemoryManagerMode.HostMapped: - case MemoryManagerMode.HostMappedUnsafe: - bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe; - var memoryManagerHostMapped = new MemoryManagerHostMapped(context.Memory, addressSpaceSize, unsafeMode, invalidAccessHandler); - processContext = new ArmProcessContext(pid, _cpuEngine, _gpu, memoryManagerHostMapped, for64Bit); - break; + if (!MemoryBlock.SupportsFlags(MemoryAllocationFlags.ViewCompatible)) + { + mode = MemoryManagerMode.SoftwarePageTable; + } - default: - throw new ArgumentOutOfRangeException(); + var cpuEngine = new JitEngine(_tickSource); + + switch (mode) + { + case MemoryManagerMode.SoftwarePageTable: + var memoryManager = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, for64Bit); + break; + + case MemoryManagerMode.HostMapped: + case MemoryManagerMode.HostMappedUnsafe: + bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe; + var memoryManagerHostMapped = new MemoryManagerHostMapped(context.Memory, addressSpaceSize, unsafeMode, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManagerHostMapped, for64Bit); + break; + + default: + throw new ArgumentOutOfRangeException(); + } } DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize); diff --git a/Ryujinx.HLE/HOS/Horizon.cs b/Ryujinx.HLE/HOS/Horizon.cs index ca3f8103e8..9908cbba84 100644 --- a/Ryujinx.HLE/HOS/Horizon.cs +++ b/Ryujinx.HLE/HOS/Horizon.cs @@ -12,7 +12,6 @@ using Ryujinx.Audio.Renderer.Device; using Ryujinx.Audio.Renderer.Server; using Ryujinx.Common.Utilities; using Ryujinx.Cpu; -using Ryujinx.Cpu.Jit; using Ryujinx.HLE.FileSystem; using Ryujinx.HLE.HOS.Kernel; using Ryujinx.HLE.HOS.Kernel.Memory; @@ -61,7 +60,6 @@ namespace Ryujinx.HLE.HOS internal Switch Device { get; private set; } internal ITickSource TickSource { get; } - internal ICpuEngine CpuEngine { get; } internal SurfaceFlinger SurfaceFlinger { get; private set; } internal AudioManager AudioManager { get; private set; } @@ -130,7 +128,6 @@ namespace Ryujinx.HLE.HOS public Horizon(Switch device) { TickSource = new TickSource(KernelConstants.CounterFrequency); - CpuEngine = new JitEngine(TickSource); KernelContext = new KernelContext( TickSource, diff --git a/Ryujinx.HLE/HOS/ProgramLoader.cs b/Ryujinx.HLE/HOS/ProgramLoader.cs index 695f467214..1f6fd96d7e 100644 --- a/Ryujinx.HLE/HOS/ProgramLoader.cs +++ b/Ryujinx.HLE/HOS/ProgramLoader.cs @@ -129,7 +129,7 @@ namespace Ryujinx.HLE.HOS KProcess process = new KProcess(context); var processContextFactory = new ArmProcessContextFactory( - context.Device.System.CpuEngine, + context.Device.System.TickSource, context.Device.Gpu, string.Empty, string.Empty, @@ -308,7 +308,7 @@ namespace Ryujinx.HLE.HOS } var processContextFactory = new ArmProcessContextFactory( - context.Device.System.CpuEngine, + context.Device.System.TickSource, context.Device.Gpu, programInfo.TitleIdText, programInfo.DisplayVersion, diff --git a/Ryujinx.Headless.SDL2/Options.cs b/Ryujinx.Headless.SDL2/Options.cs index 49233bceac..5138a05351 100644 --- a/Ryujinx.Headless.SDL2/Options.cs +++ b/Ryujinx.Headless.SDL2/Options.cs @@ -129,6 +129,9 @@ namespace Ryujinx.Headless.SDL2 [Option("audio-volume", Required = false, Default = 1.0f, HelpText ="The audio level (0 to 1).")] public float AudioVolume { get; set; } + [Option("use-hypervisor", Required = false, Default = true, HelpText = "Uses Hypervisor over JIT if available.")] + public bool UseHypervisor { get; set; } + // Logging [Option("disable-file-logging", Required = false, Default = false, HelpText = "Disables logging to a file on disk.")] diff --git a/Ryujinx.Headless.SDL2/Program.cs b/Ryujinx.Headless.SDL2/Program.cs index 6ea3a98d73..f618e38d69 100644 --- a/Ryujinx.Headless.SDL2/Program.cs +++ b/Ryujinx.Headless.SDL2/Program.cs @@ -548,7 +548,8 @@ namespace Ryujinx.Headless.SDL2 options.MemoryManagerMode, options.IgnoreMissingServices, options.AspectRatio, - options.AudioVolume); + options.AudioVolume, + options.UseHypervisor); return new Switch(configuration); } diff --git a/Ryujinx.Ui.Common/Configuration/ConfigurationFileFormat.cs b/Ryujinx.Ui.Common/Configuration/ConfigurationFileFormat.cs index 2ebf65ac1c..226b5933b5 100644 --- a/Ryujinx.Ui.Common/Configuration/ConfigurationFileFormat.cs +++ b/Ryujinx.Ui.Common/Configuration/ConfigurationFileFormat.cs @@ -14,7 +14,7 @@ namespace Ryujinx.Ui.Common.Configuration /// /// The current version of the file format /// - public const int CurrentVersion = 42; + public const int CurrentVersion = 43; /// /// Version of the configuration file format @@ -330,6 +330,11 @@ namespace Ryujinx.Ui.Common.Configuration /// public string PreferredGpu { get; set; } + /// + /// Uses Hypervisor over JIT if available + /// + public bool UseHypervisor { get; set; } + /// /// Loads a configuration file from disk /// diff --git a/Ryujinx.Ui.Common/Configuration/ConfigurationState.cs b/Ryujinx.Ui.Common/Configuration/ConfigurationState.cs index e64c69ad65..f193b1570f 100644 --- a/Ryujinx.Ui.Common/Configuration/ConfigurationState.cs +++ b/Ryujinx.Ui.Common/Configuration/ConfigurationState.cs @@ -301,6 +301,11 @@ namespace Ryujinx.Ui.Common.Configuration /// public ReactiveObject IgnoreMissingServices { get; private set; } + /// + /// Uses Hypervisor over JIT if available + /// + public ReactiveObject UseHypervisor { get; private set; } + public SystemSection() { Language = new ReactiveObject(); @@ -327,6 +332,8 @@ namespace Ryujinx.Ui.Common.Configuration IgnoreMissingServices.Event += static (sender, e) => LogValueChange(sender, e, nameof(IgnoreMissingServices)); AudioVolume = new ReactiveObject(); AudioVolume.Event += static (sender, e) => LogValueChange(sender, e, nameof(AudioVolume)); + UseHypervisor = new ReactiveObject(); + UseHypervisor.Event += static (sender, e) => LogValueChange(sender, e, nameof(UseHypervisor)); } } @@ -566,6 +573,7 @@ namespace Ryujinx.Ui.Common.Configuration MemoryManagerMode = System.MemoryManagerMode, ExpandRam = System.ExpandRam, IgnoreMissingServices = System.IgnoreMissingServices, + UseHypervisor = System.UseHypervisor, GuiColumns = new GuiColumns { FavColumn = Ui.GuiColumns.FavColumn, @@ -652,6 +660,7 @@ namespace Ryujinx.Ui.Common.Configuration System.MemoryManagerMode.Value = MemoryManagerMode.HostMappedUnsafe; System.ExpandRam.Value = false; System.IgnoreMissingServices.Value = false; + System.UseHypervisor.Value = true; Ui.GuiColumns.FavColumn.Value = true; Ui.GuiColumns.IconColumn.Value = true; Ui.GuiColumns.AppColumn.Value = true; @@ -1192,6 +1201,13 @@ namespace Ryujinx.Ui.Common.Configuration configurationFileFormat.EnableMacroHLE = true; } + if (configurationFileFormat.Version < 43) + { + Ryujinx.Common.Logging.Logger.Warning?.Print(LogClass.Application, $"Outdated configuration version {configurationFileFormat.Version}, migrating to version 43."); + + configurationFileFormat.UseHypervisor = true; + } + Logger.EnableFileLog.Value = configurationFileFormat.EnableFileLog; Graphics.ResScale.Value = configurationFileFormat.ResScale; Graphics.ResScaleCustom.Value = configurationFileFormat.ResScaleCustom; @@ -1233,6 +1249,7 @@ namespace Ryujinx.Ui.Common.Configuration System.MemoryManagerMode.Value = configurationFileFormat.MemoryManagerMode; System.ExpandRam.Value = configurationFileFormat.ExpandRam; System.IgnoreMissingServices.Value = configurationFileFormat.IgnoreMissingServices; + System.UseHypervisor.Value = configurationFileFormat.UseHypervisor; Ui.GuiColumns.FavColumn.Value = configurationFileFormat.GuiColumns.FavColumn; Ui.GuiColumns.IconColumn.Value = configurationFileFormat.GuiColumns.IconColumn; Ui.GuiColumns.AppColumn.Value = configurationFileFormat.GuiColumns.AppColumn; @@ -1292,4 +1309,4 @@ namespace Ryujinx.Ui.Common.Configuration Instance = new ConfigurationState(); } } -} +} \ No newline at end of file diff --git a/Ryujinx/Ui/MainWindow.cs b/Ryujinx/Ui/MainWindow.cs index 53a97fb9f6..5051fb5f64 100644 --- a/Ryujinx/Ui/MainWindow.cs +++ b/Ryujinx/Ui/MainWindow.cs @@ -577,7 +577,8 @@ namespace Ryujinx.Ui ConfigurationState.Instance.System.MemoryManagerMode, ConfigurationState.Instance.System.IgnoreMissingServices, ConfigurationState.Instance.Graphics.AspectRatio, - ConfigurationState.Instance.System.AudioVolume); + ConfigurationState.Instance.System.AudioVolume, + ConfigurationState.Instance.System.UseHypervisor); _emulationContext = new HLE.Switch(configuration); }