17620d18db
* ARMeilleure: Add AVX512{F,VL,DQ,BW} detection
Add `UseAvx512Ortho` and `UseAvx512OrthoFloat` optimization flags as
short-hands for `F+VL` and `F+VL+DQ`.
* ARMeilleure: Add initial support for EVEX instruction encoding
Does not implement rounding, or exception controls.
* ARMeilleure: Add `X86Vpternlogd`
Accelerates the vector-`Not` instruction.
* ARMeilleure: Add check for `OSXSAVE` for AVX{2,512}
* ARMeilleure: Add check for `XCR0` flags
Add XCR0 register checks for AVX and AVX512F, following the guidelines
from section 14.3 and 15.2 from the Intel Architecture Software
Developer's Manual.
* ARMeilleure: Remove redundant `ReProtect` and `Dispose`, formatting
* ARMeilleure: Move XCR0 procedure to GetXcr0Eax
* ARMeilleure: Add `XCR0` to `FeatureInfo` structure
* ARMeilleure: Utilize `ReadOnlySpan` for Xcr0 assembly
Avoids an additional allocation
* ARMeilleure: Formatting fixes
* ARMeilleure: Fix EVEX encoding src2 register index
> Just like in VEX prefix, vvvv is provided in inverted form.
* ARMeilleure: Add `X86Vpternlogd` acceleration to `Vmvn_I`
Passes unit tests, verified instruction utilization
* ARMeilleure: Fix EVEX register operand designations
Operand 2 was being sourced improperly.
EVEX encoded instructions source their operands like so:
Operand 1: ModRM:reg
Operand 2: EVEX.vvvvv
Operand 3: ModRM:r/m
Operand 4: Imm
This fixes the improper register designations when emitting vpternlog.
Now "dest", "src1", "src2" arguments emit in the proper order in EVEX instructions.
* ARMeilleure: Add `X86Vpternlogd` acceleration to `Orn_V`
* ARMeilleure: PTC version bump
* ARMeilleure: Update EVEX encoding Debug.Assert to Debug.Fail
* ARMeilleure: Update EVEX encoding comment capitalization
1131 lines
39 KiB
C#
1131 lines
39 KiB
C#
using ARMeilleure.CodeGen;
|
|
using ARMeilleure.CodeGen.Linking;
|
|
using ARMeilleure.CodeGen.Unwinding;
|
|
using ARMeilleure.Common;
|
|
using ARMeilleure.Memory;
|
|
using Ryujinx.Common;
|
|
using Ryujinx.Common.Configuration;
|
|
using Ryujinx.Common.Logging;
|
|
using Ryujinx.Common.Memory;
|
|
using System;
|
|
using System.Buffers.Binary;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Runtime;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading;
|
|
|
|
using static ARMeilleure.Translation.PTC.PtcFormatter;
|
|
|
|
namespace ARMeilleure.Translation.PTC
|
|
{
|
|
using Arm64HardwareCapabilities = ARMeilleure.CodeGen.Arm64.HardwareCapabilities;
|
|
using X86HardwareCapabilities = ARMeilleure.CodeGen.X86.HardwareCapabilities;
|
|
|
|
class Ptc : IPtcLoadState
|
|
{
|
|
private const string OuterHeaderMagicString = "PTCohd\0\0";
|
|
private const string InnerHeaderMagicString = "PTCihd\0\0";
|
|
|
|
private const uint InternalVersion = 4485; //! To be incremented manually for each change to the ARMeilleure project.
|
|
|
|
private const string ActualDir = "0";
|
|
private const string BackupDir = "1";
|
|
|
|
private const string TitleIdTextDefault = "0000000000000000";
|
|
private const string DisplayVersionDefault = "0";
|
|
|
|
public static readonly Symbol PageTableSymbol = new(SymbolType.Special, 1);
|
|
public static readonly Symbol CountTableSymbol = new(SymbolType.Special, 2);
|
|
public static readonly Symbol DispatchStubSymbol = new(SymbolType.Special, 3);
|
|
|
|
private const byte FillingByte = 0x00;
|
|
private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
|
|
|
|
public PtcProfiler Profiler { get; }
|
|
|
|
// Carriers.
|
|
private MemoryStream _infosStream;
|
|
private List<byte[]> _codesList;
|
|
private MemoryStream _relocsStream;
|
|
private MemoryStream _unwindInfosStream;
|
|
|
|
private readonly ulong _outerHeaderMagic;
|
|
private readonly ulong _innerHeaderMagic;
|
|
|
|
private readonly ManualResetEvent _waitEvent;
|
|
|
|
private readonly object _lock;
|
|
|
|
private bool _disposed;
|
|
|
|
public string TitleIdText { get; private set; }
|
|
public string DisplayVersion { get; private set; }
|
|
|
|
private MemoryManagerType _memoryMode;
|
|
|
|
public string CachePathActual { get; private set; }
|
|
public string CachePathBackup { get; private set; }
|
|
|
|
public PtcState State { get; private set; }
|
|
|
|
// Progress reporting helpers.
|
|
private volatile int _translateCount;
|
|
private volatile int _translateTotalCount;
|
|
public event Action<PtcLoadingState, int, int> PtcStateChanged;
|
|
|
|
public Ptc()
|
|
{
|
|
Profiler = new PtcProfiler(this);
|
|
|
|
InitializeCarriers();
|
|
|
|
_outerHeaderMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(OuterHeaderMagicString).AsSpan());
|
|
_innerHeaderMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(InnerHeaderMagicString).AsSpan());
|
|
|
|
_waitEvent = new ManualResetEvent(true);
|
|
|
|
_lock = new object();
|
|
|
|
_disposed = false;
|
|
|
|
TitleIdText = TitleIdTextDefault;
|
|
DisplayVersion = DisplayVersionDefault;
|
|
|
|
CachePathActual = string.Empty;
|
|
CachePathBackup = string.Empty;
|
|
|
|
Disable();
|
|
}
|
|
|
|
public void Initialize(string titleIdText, string displayVersion, bool enabled, MemoryManagerType memoryMode)
|
|
{
|
|
Wait();
|
|
|
|
Profiler.Wait();
|
|
Profiler.ClearEntries();
|
|
|
|
Logger.Info?.Print(LogClass.Ptc, $"Initializing Profiled Persistent Translation Cache (enabled: {enabled}).");
|
|
|
|
if (!enabled || string.IsNullOrEmpty(titleIdText) || titleIdText == TitleIdTextDefault)
|
|
{
|
|
TitleIdText = TitleIdTextDefault;
|
|
DisplayVersion = DisplayVersionDefault;
|
|
|
|
CachePathActual = string.Empty;
|
|
CachePathBackup = string.Empty;
|
|
|
|
Disable();
|
|
|
|
return;
|
|
}
|
|
|
|
TitleIdText = titleIdText;
|
|
DisplayVersion = !string.IsNullOrEmpty(displayVersion) ? displayVersion : DisplayVersionDefault;
|
|
_memoryMode = memoryMode;
|
|
|
|
string workPathActual = Path.Combine(AppDataManager.GamesDirPath, TitleIdText, "cache", "cpu", ActualDir);
|
|
string workPathBackup = Path.Combine(AppDataManager.GamesDirPath, TitleIdText, "cache", "cpu", BackupDir);
|
|
|
|
if (!Directory.Exists(workPathActual))
|
|
{
|
|
Directory.CreateDirectory(workPathActual);
|
|
}
|
|
|
|
if (!Directory.Exists(workPathBackup))
|
|
{
|
|
Directory.CreateDirectory(workPathBackup);
|
|
}
|
|
|
|
CachePathActual = Path.Combine(workPathActual, DisplayVersion);
|
|
CachePathBackup = Path.Combine(workPathBackup, DisplayVersion);
|
|
|
|
PreLoad();
|
|
Profiler.PreLoad();
|
|
|
|
Enable();
|
|
}
|
|
|
|
private void InitializeCarriers()
|
|
{
|
|
_infosStream = MemoryStreamManager.Shared.GetStream();
|
|
_codesList = new List<byte[]>();
|
|
_relocsStream = MemoryStreamManager.Shared.GetStream();
|
|
_unwindInfosStream = MemoryStreamManager.Shared.GetStream();
|
|
}
|
|
|
|
private void DisposeCarriers()
|
|
{
|
|
_infosStream.Dispose();
|
|
_codesList.Clear();
|
|
_relocsStream.Dispose();
|
|
_unwindInfosStream.Dispose();
|
|
}
|
|
|
|
private bool AreCarriersEmpty()
|
|
{
|
|
return _infosStream.Length == 0L && _codesList.Count == 0 && _relocsStream.Length == 0L && _unwindInfosStream.Length == 0L;
|
|
}
|
|
|
|
private void ResetCarriersIfNeeded()
|
|
{
|
|
if (AreCarriersEmpty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
DisposeCarriers();
|
|
|
|
InitializeCarriers();
|
|
}
|
|
|
|
private void PreLoad()
|
|
{
|
|
string fileNameActual = $"{CachePathActual}.cache";
|
|
string fileNameBackup = $"{CachePathBackup}.cache";
|
|
|
|
FileInfo fileInfoActual = new FileInfo(fileNameActual);
|
|
FileInfo fileInfoBackup = new FileInfo(fileNameBackup);
|
|
|
|
if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
|
|
{
|
|
if (!Load(fileNameActual, false))
|
|
{
|
|
if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
|
|
{
|
|
Load(fileNameBackup, true);
|
|
}
|
|
}
|
|
}
|
|
else if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
|
|
{
|
|
Load(fileNameBackup, true);
|
|
}
|
|
}
|
|
|
|
private unsafe bool Load(string fileName, bool isBackup)
|
|
{
|
|
using (FileStream compressedStream = new(fileName, FileMode.Open))
|
|
using (DeflateStream deflateStream = new(compressedStream, CompressionMode.Decompress, true))
|
|
{
|
|
OuterHeader outerHeader = DeserializeStructure<OuterHeader>(compressedStream);
|
|
|
|
if (!outerHeader.IsHeaderValid())
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.Magic != _outerHeaderMagic)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.CacheFileVersion != InternalVersion)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.Endianness != GetEndianness())
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.FeatureInfo != GetFeatureInfo())
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.MemoryManagerMode != GetMemoryManagerMode())
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.OSPlatform != GetOSPlatform())
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (outerHeader.Architecture != (uint)RuntimeInformation.ProcessArchitecture)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
IntPtr intPtr = IntPtr.Zero;
|
|
|
|
try
|
|
{
|
|
intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize));
|
|
|
|
using (UnmanagedMemoryStream stream = new((byte*)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite))
|
|
{
|
|
try
|
|
{
|
|
deflateStream.CopyTo(stream);
|
|
}
|
|
catch
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
Debug.Assert(stream.Position == stream.Length);
|
|
|
|
stream.Seek(0L, SeekOrigin.Begin);
|
|
|
|
InnerHeader innerHeader = DeserializeStructure<InnerHeader>(stream);
|
|
|
|
if (!innerHeader.IsHeaderValid())
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
if (innerHeader.Magic != _innerHeaderMagic)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
ReadOnlySpan<byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength);
|
|
stream.Seek(innerHeader.InfosLength, SeekOrigin.Current);
|
|
|
|
Hash128 infosHash = XXHash128.ComputeHash(infosBytes);
|
|
|
|
if (innerHeader.InfosHash != infosHash)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
ReadOnlySpan<byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan<byte>.Empty;
|
|
stream.Seek(innerHeader.CodesLength, SeekOrigin.Current);
|
|
|
|
Hash128 codesHash = XXHash128.ComputeHash(codesBytes);
|
|
|
|
if (innerHeader.CodesHash != codesHash)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
ReadOnlySpan<byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength);
|
|
stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current);
|
|
|
|
Hash128 relocsHash = XXHash128.ComputeHash(relocsBytes);
|
|
|
|
if (innerHeader.RelocsHash != relocsHash)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
|
|
stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
|
|
|
|
Hash128 unwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
|
|
|
|
if (innerHeader.UnwindInfosHash != unwindInfosHash)
|
|
{
|
|
InvalidateCompressedStream(compressedStream);
|
|
|
|
return false;
|
|
}
|
|
|
|
Debug.Assert(stream.Position == stream.Length);
|
|
|
|
stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
|
|
|
|
_infosStream.Write(infosBytes);
|
|
stream.Seek(innerHeader.InfosLength, SeekOrigin.Current);
|
|
|
|
_codesList.ReadFrom(stream);
|
|
|
|
_relocsStream.Write(relocsBytes);
|
|
stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current);
|
|
|
|
_unwindInfosStream.Write(unwindInfosBytes);
|
|
stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
|
|
|
|
Debug.Assert(stream.Position == stream.Length);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (intPtr != IntPtr.Zero)
|
|
{
|
|
Marshal.FreeHGlobal(intPtr);
|
|
}
|
|
}
|
|
}
|
|
|
|
long fileSize = new FileInfo(fileName).Length;
|
|
|
|
Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Translation Cache" : "Loaded Translation Cache")} (size: {fileSize} bytes, translated functions: {GetEntriesCount()}).");
|
|
|
|
return true;
|
|
}
|
|
|
|
private void InvalidateCompressedStream(FileStream compressedStream)
|
|
{
|
|
compressedStream.SetLength(0L);
|
|
}
|
|
|
|
private void PreSave()
|
|
{
|
|
_waitEvent.Reset();
|
|
|
|
try
|
|
{
|
|
string fileNameActual = $"{CachePathActual}.cache";
|
|
string fileNameBackup = $"{CachePathBackup}.cache";
|
|
|
|
FileInfo fileInfoActual = new FileInfo(fileNameActual);
|
|
|
|
if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
|
|
{
|
|
File.Copy(fileNameActual, fileNameBackup, true);
|
|
}
|
|
|
|
Save(fileNameActual);
|
|
}
|
|
finally
|
|
{
|
|
ResetCarriersIfNeeded();
|
|
|
|
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
|
|
}
|
|
|
|
_waitEvent.Set();
|
|
}
|
|
|
|
private unsafe void Save(string fileName)
|
|
{
|
|
int translatedFuncsCount;
|
|
|
|
InnerHeader innerHeader = new InnerHeader();
|
|
|
|
innerHeader.Magic = _innerHeaderMagic;
|
|
|
|
innerHeader.InfosLength = (int)_infosStream.Length;
|
|
innerHeader.CodesLength = _codesList.Length();
|
|
innerHeader.RelocsLength = (int)_relocsStream.Length;
|
|
innerHeader.UnwindInfosLength = (int)_unwindInfosStream.Length;
|
|
|
|
OuterHeader outerHeader = new OuterHeader();
|
|
|
|
outerHeader.Magic = _outerHeaderMagic;
|
|
|
|
outerHeader.CacheFileVersion = InternalVersion;
|
|
outerHeader.Endianness = GetEndianness();
|
|
outerHeader.FeatureInfo = GetFeatureInfo();
|
|
outerHeader.MemoryManagerMode = GetMemoryManagerMode();
|
|
outerHeader.OSPlatform = GetOSPlatform();
|
|
outerHeader.Architecture = (uint)RuntimeInformation.ProcessArchitecture;
|
|
|
|
outerHeader.UncompressedStreamSize =
|
|
(long)Unsafe.SizeOf<InnerHeader>() +
|
|
innerHeader.InfosLength +
|
|
innerHeader.CodesLength +
|
|
innerHeader.RelocsLength +
|
|
innerHeader.UnwindInfosLength;
|
|
|
|
outerHeader.SetHeaderHash();
|
|
|
|
IntPtr intPtr = IntPtr.Zero;
|
|
|
|
try
|
|
{
|
|
intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize));
|
|
|
|
using (UnmanagedMemoryStream stream = new((byte*)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite))
|
|
{
|
|
stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
|
|
|
|
ReadOnlySpan<byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength);
|
|
_infosStream.WriteTo(stream);
|
|
|
|
ReadOnlySpan<byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan<byte>.Empty;
|
|
_codesList.WriteTo(stream);
|
|
|
|
ReadOnlySpan<byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength);
|
|
_relocsStream.WriteTo(stream);
|
|
|
|
ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
|
|
_unwindInfosStream.WriteTo(stream);
|
|
|
|
Debug.Assert(stream.Position == stream.Length);
|
|
|
|
innerHeader.InfosHash = XXHash128.ComputeHash(infosBytes);
|
|
innerHeader.CodesHash = XXHash128.ComputeHash(codesBytes);
|
|
innerHeader.RelocsHash = XXHash128.ComputeHash(relocsBytes);
|
|
innerHeader.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
|
|
|
|
innerHeader.SetHeaderHash();
|
|
|
|
stream.Seek(0L, SeekOrigin.Begin);
|
|
SerializeStructure(stream, innerHeader);
|
|
|
|
translatedFuncsCount = GetEntriesCount();
|
|
|
|
ResetCarriersIfNeeded();
|
|
|
|
using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate))
|
|
using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true))
|
|
{
|
|
try
|
|
{
|
|
SerializeStructure(compressedStream, outerHeader);
|
|
|
|
stream.Seek(0L, SeekOrigin.Begin);
|
|
stream.CopyTo(deflateStream);
|
|
}
|
|
catch
|
|
{
|
|
compressedStream.Position = 0L;
|
|
}
|
|
|
|
if (compressedStream.Position < compressedStream.Length)
|
|
{
|
|
compressedStream.SetLength(compressedStream.Position);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (intPtr != IntPtr.Zero)
|
|
{
|
|
Marshal.FreeHGlobal(intPtr);
|
|
}
|
|
}
|
|
|
|
long fileSize = new FileInfo(fileName).Length;
|
|
|
|
if (fileSize != 0L)
|
|
{
|
|
Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount}).");
|
|
}
|
|
}
|
|
|
|
public void LoadTranslations(Translator translator)
|
|
{
|
|
if (AreCarriersEmpty())
|
|
{
|
|
return;
|
|
}
|
|
|
|
long infosStreamLength = _infosStream.Length;
|
|
long relocsStreamLength = _relocsStream.Length;
|
|
long unwindInfosStreamLength = _unwindInfosStream.Length;
|
|
|
|
_infosStream.Seek(0L, SeekOrigin.Begin);
|
|
_relocsStream.Seek(0L, SeekOrigin.Begin);
|
|
_unwindInfosStream.Seek(0L, SeekOrigin.Begin);
|
|
|
|
using (BinaryReader relocsReader = new(_relocsStream, EncodingCache.UTF8NoBOM, true))
|
|
using (BinaryReader unwindInfosReader = new(_unwindInfosStream, EncodingCache.UTF8NoBOM, true))
|
|
{
|
|
for (int index = 0; index < GetEntriesCount(); index++)
|
|
{
|
|
InfoEntry infoEntry = DeserializeStructure<InfoEntry>(_infosStream);
|
|
|
|
if (infoEntry.Stubbed)
|
|
{
|
|
SkipCode(index, infoEntry.CodeLength);
|
|
SkipReloc(infoEntry.RelocEntriesCount);
|
|
SkipUnwindInfo(unwindInfosReader);
|
|
|
|
continue;
|
|
}
|
|
|
|
bool isEntryChanged = infoEntry.Hash != ComputeHash(translator.Memory, infoEntry.Address, infoEntry.GuestSize);
|
|
|
|
if (isEntryChanged || (!infoEntry.HighCq && Profiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) && value.HighCq))
|
|
{
|
|
infoEntry.Stubbed = true;
|
|
infoEntry.CodeLength = 0;
|
|
UpdateInfo(infoEntry);
|
|
|
|
StubCode(index);
|
|
StubReloc(infoEntry.RelocEntriesCount);
|
|
StubUnwindInfo(unwindInfosReader);
|
|
|
|
if (isEntryChanged)
|
|
{
|
|
Logger.Info?.Print(LogClass.Ptc, $"Invalidated translated function (address: 0x{infoEntry.Address:X16})");
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
byte[] code = ReadCode(index, infoEntry.CodeLength);
|
|
|
|
Counter<uint> callCounter = null;
|
|
|
|
if (infoEntry.RelocEntriesCount != 0)
|
|
{
|
|
RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
|
|
|
|
PatchCode(translator, code, relocEntries, out callCounter);
|
|
}
|
|
|
|
UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
|
|
|
|
TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
|
|
|
|
translator.RegisterFunction(infoEntry.Address, func);
|
|
|
|
bool isAddressUnique = translator.Functions.TryAdd(infoEntry.Address, infoEntry.GuestSize, func);
|
|
|
|
Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique.");
|
|
}
|
|
}
|
|
|
|
if (_infosStream.Length != infosStreamLength || _infosStream.Position != infosStreamLength ||
|
|
_relocsStream.Length != relocsStreamLength || _relocsStream.Position != relocsStreamLength ||
|
|
_unwindInfosStream.Length != unwindInfosStreamLength || _unwindInfosStream.Position != unwindInfosStreamLength)
|
|
{
|
|
throw new Exception("The length of a memory stream has changed, or its position has not reached or has exceeded its end.");
|
|
}
|
|
|
|
Logger.Info?.Print(LogClass.Ptc, $"{translator.Functions.Count} translated functions loaded");
|
|
}
|
|
|
|
private int GetEntriesCount()
|
|
{
|
|
return _codesList.Count;
|
|
}
|
|
|
|
[Conditional("DEBUG")]
|
|
private void SkipCode(int index, int codeLength)
|
|
{
|
|
Debug.Assert(_codesList[index].Length == 0);
|
|
Debug.Assert(codeLength == 0);
|
|
}
|
|
|
|
private void SkipReloc(int relocEntriesCount)
|
|
{
|
|
_relocsStream.Seek(relocEntriesCount * RelocEntry.Stride, SeekOrigin.Current);
|
|
}
|
|
|
|
private void SkipUnwindInfo(BinaryReader unwindInfosReader)
|
|
{
|
|
int pushEntriesLength = unwindInfosReader.ReadInt32();
|
|
|
|
_unwindInfosStream.Seek(pushEntriesLength * UnwindPushEntry.Stride + UnwindInfo.Stride, SeekOrigin.Current);
|
|
}
|
|
|
|
private byte[] ReadCode(int index, int codeLength)
|
|
{
|
|
Debug.Assert(_codesList[index].Length == codeLength);
|
|
|
|
return _codesList[index];
|
|
}
|
|
|
|
private RelocEntry[] GetRelocEntries(BinaryReader relocsReader, int relocEntriesCount)
|
|
{
|
|
RelocEntry[] relocEntries = new RelocEntry[relocEntriesCount];
|
|
|
|
for (int i = 0; i < relocEntriesCount; i++)
|
|
{
|
|
int position = relocsReader.ReadInt32();
|
|
SymbolType type = (SymbolType)relocsReader.ReadByte();
|
|
ulong value = relocsReader.ReadUInt64();
|
|
|
|
relocEntries[i] = new RelocEntry(position, new Symbol(type, value));
|
|
}
|
|
|
|
return relocEntries;
|
|
}
|
|
|
|
private void PatchCode(Translator translator, Span<byte> code, RelocEntry[] relocEntries, out Counter<uint> callCounter)
|
|
{
|
|
callCounter = null;
|
|
|
|
foreach (RelocEntry relocEntry in relocEntries)
|
|
{
|
|
IntPtr? imm = null;
|
|
Symbol symbol = relocEntry.Symbol;
|
|
|
|
if (symbol.Type == SymbolType.FunctionTable)
|
|
{
|
|
ulong guestAddress = symbol.Value;
|
|
|
|
if (translator.FunctionTable.IsValid(guestAddress))
|
|
{
|
|
unsafe { imm = (IntPtr)Unsafe.AsPointer(ref translator.FunctionTable.GetValue(guestAddress)); }
|
|
}
|
|
}
|
|
else if (symbol.Type == SymbolType.DelegateTable)
|
|
{
|
|
int index = (int)symbol.Value;
|
|
|
|
if (Delegates.TryGetDelegateFuncPtrByIndex(index, out IntPtr funcPtr))
|
|
{
|
|
imm = funcPtr;
|
|
}
|
|
}
|
|
else if (symbol == PageTableSymbol)
|
|
{
|
|
imm = translator.Memory.PageTablePointer;
|
|
}
|
|
else if (symbol == CountTableSymbol)
|
|
{
|
|
if (callCounter == null)
|
|
{
|
|
callCounter = new Counter<uint>(translator.CountTable);
|
|
}
|
|
|
|
unsafe { imm = (IntPtr)Unsafe.AsPointer(ref callCounter.Value); }
|
|
}
|
|
else if (symbol == DispatchStubSymbol)
|
|
{
|
|
imm = translator.Stubs.DispatchStub;
|
|
}
|
|
|
|
if (imm == null)
|
|
{
|
|
throw new Exception($"Unexpected reloc entry {relocEntry}.");
|
|
}
|
|
|
|
BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), (ulong)imm.Value);
|
|
}
|
|
}
|
|
|
|
private UnwindInfo ReadUnwindInfo(BinaryReader unwindInfosReader)
|
|
{
|
|
int pushEntriesLength = unwindInfosReader.ReadInt32();
|
|
|
|
UnwindPushEntry[] pushEntries = new UnwindPushEntry[pushEntriesLength];
|
|
|
|
for (int i = 0; i < pushEntriesLength; i++)
|
|
{
|
|
int pseudoOp = unwindInfosReader.ReadInt32();
|
|
int prologOffset = unwindInfosReader.ReadInt32();
|
|
int regIndex = unwindInfosReader.ReadInt32();
|
|
int stackOffsetOrAllocSize = unwindInfosReader.ReadInt32();
|
|
|
|
pushEntries[i] = new UnwindPushEntry((UnwindPseudoOp)pseudoOp, prologOffset, regIndex, stackOffsetOrAllocSize);
|
|
}
|
|
|
|
int prologueSize = unwindInfosReader.ReadInt32();
|
|
|
|
return new UnwindInfo(pushEntries, prologueSize);
|
|
}
|
|
|
|
private TranslatedFunction FastTranslate(
|
|
byte[] code,
|
|
Counter<uint> callCounter,
|
|
ulong guestSize,
|
|
UnwindInfo unwindInfo,
|
|
bool highCq)
|
|
{
|
|
var cFunc = new CompiledFunction(code, unwindInfo, RelocInfo.Empty);
|
|
var gFunc = cFunc.MapWithPointer<GuestFunction>(out IntPtr gFuncPointer);
|
|
|
|
return new TranslatedFunction(gFunc, gFuncPointer, callCounter, guestSize, highCq);
|
|
}
|
|
|
|
private void UpdateInfo(InfoEntry infoEntry)
|
|
{
|
|
_infosStream.Seek(-Unsafe.SizeOf<InfoEntry>(), SeekOrigin.Current);
|
|
|
|
SerializeStructure(_infosStream, infoEntry);
|
|
}
|
|
|
|
private void StubCode(int index)
|
|
{
|
|
_codesList[index] = Array.Empty<byte>();
|
|
}
|
|
|
|
private void StubReloc(int relocEntriesCount)
|
|
{
|
|
for (int i = 0; i < relocEntriesCount * RelocEntry.Stride; i++)
|
|
{
|
|
_relocsStream.WriteByte(FillingByte);
|
|
}
|
|
}
|
|
|
|
private void StubUnwindInfo(BinaryReader unwindInfosReader)
|
|
{
|
|
int pushEntriesLength = unwindInfosReader.ReadInt32();
|
|
|
|
for (int i = 0; i < pushEntriesLength * UnwindPushEntry.Stride + UnwindInfo.Stride; i++)
|
|
{
|
|
_unwindInfosStream.WriteByte(FillingByte);
|
|
}
|
|
}
|
|
|
|
public void MakeAndSaveTranslations(Translator translator)
|
|
{
|
|
var profiledFuncsToTranslate = Profiler.GetProfiledFuncsToTranslate(translator.Functions);
|
|
|
|
_translateCount = 0;
|
|
_translateTotalCount = profiledFuncsToTranslate.Count;
|
|
|
|
if (_translateTotalCount == 0)
|
|
{
|
|
ResetCarriersIfNeeded();
|
|
|
|
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
|
|
|
|
return;
|
|
}
|
|
|
|
int degreeOfParallelism = Environment.ProcessorCount;
|
|
|
|
// If there are enough cores lying around, we leave one alone for other tasks.
|
|
if (degreeOfParallelism > 4)
|
|
{
|
|
degreeOfParallelism--;
|
|
}
|
|
|
|
Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}");
|
|
|
|
PtcStateChanged?.Invoke(PtcLoadingState.Start, _translateCount, _translateTotalCount);
|
|
|
|
using AutoResetEvent progressReportEvent = new AutoResetEvent(false);
|
|
|
|
Thread progressReportThread = new Thread(ReportProgress)
|
|
{
|
|
Name = "Ptc.ProgressReporter",
|
|
Priority = ThreadPriority.Lowest,
|
|
IsBackground = true
|
|
};
|
|
|
|
progressReportThread.Start(progressReportEvent);
|
|
|
|
void TranslateFuncs()
|
|
{
|
|
while (profiledFuncsToTranslate.TryDequeue(out var item))
|
|
{
|
|
ulong address = item.address;
|
|
|
|
Debug.Assert(Profiler.IsAddressInStaticCodeRange(address));
|
|
|
|
TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq);
|
|
|
|
bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func);
|
|
|
|
Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique.");
|
|
|
|
Interlocked.Increment(ref _translateCount);
|
|
|
|
translator.RegisterFunction(address, func);
|
|
|
|
if (State != PtcState.Enabled)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
List<Thread> threads = new List<Thread>();
|
|
|
|
for (int i = 0; i < degreeOfParallelism; i++)
|
|
{
|
|
Thread thread = new Thread(TranslateFuncs);
|
|
thread.IsBackground = true;
|
|
|
|
threads.Add(thread);
|
|
}
|
|
|
|
Stopwatch sw = Stopwatch.StartNew();
|
|
|
|
threads.ForEach((thread) => thread.Start());
|
|
threads.ForEach((thread) => thread.Join());
|
|
|
|
threads.Clear();
|
|
|
|
progressReportEvent.Set();
|
|
progressReportThread.Join();
|
|
|
|
sw.Stop();
|
|
|
|
PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount);
|
|
|
|
Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
|
|
|
|
Thread preSaveThread = new Thread(PreSave);
|
|
preSaveThread.IsBackground = true;
|
|
preSaveThread.Start();
|
|
}
|
|
|
|
private void ReportProgress(object state)
|
|
{
|
|
const int refreshRate = 50; // ms.
|
|
|
|
AutoResetEvent endEvent = (AutoResetEvent)state;
|
|
|
|
int count = 0;
|
|
|
|
do
|
|
{
|
|
int newCount = _translateCount;
|
|
|
|
if (count != newCount)
|
|
{
|
|
PtcStateChanged?.Invoke(PtcLoadingState.Loading, newCount, _translateTotalCount);
|
|
count = newCount;
|
|
}
|
|
}
|
|
while (!endEvent.WaitOne(refreshRate));
|
|
}
|
|
|
|
public static Hash128 ComputeHash(IMemoryManager memory, ulong address, ulong guestSize)
|
|
{
|
|
return XXHash128.ComputeHash(memory.GetSpan(address, checked((int)(guestSize))));
|
|
}
|
|
|
|
public void WriteCompiledFunction(ulong address, ulong guestSize, Hash128 hash, bool highCq, CompiledFunction compiledFunc)
|
|
{
|
|
lock (_lock)
|
|
{
|
|
byte[] code = compiledFunc.Code;
|
|
RelocInfo relocInfo = compiledFunc.RelocInfo;
|
|
UnwindInfo unwindInfo = compiledFunc.UnwindInfo;
|
|
|
|
InfoEntry infoEntry = new InfoEntry();
|
|
|
|
infoEntry.Address = address;
|
|
infoEntry.GuestSize = guestSize;
|
|
infoEntry.Hash = hash;
|
|
infoEntry.HighCq = highCq;
|
|
infoEntry.Stubbed = false;
|
|
infoEntry.CodeLength = code.Length;
|
|
infoEntry.RelocEntriesCount = relocInfo.Entries.Length;
|
|
|
|
SerializeStructure(_infosStream, infoEntry);
|
|
|
|
WriteCode(code.AsSpan());
|
|
|
|
// WriteReloc.
|
|
using var relocInfoWriter = new BinaryWriter(_relocsStream, EncodingCache.UTF8NoBOM, true);
|
|
|
|
foreach (RelocEntry entry in relocInfo.Entries)
|
|
{
|
|
relocInfoWriter.Write(entry.Position);
|
|
relocInfoWriter.Write((byte)entry.Symbol.Type);
|
|
relocInfoWriter.Write(entry.Symbol.Value);
|
|
}
|
|
|
|
// WriteUnwindInfo.
|
|
using var unwindInfoWriter = new BinaryWriter(_unwindInfosStream, EncodingCache.UTF8NoBOM, true);
|
|
|
|
unwindInfoWriter.Write(unwindInfo.PushEntries.Length);
|
|
|
|
foreach (UnwindPushEntry unwindPushEntry in unwindInfo.PushEntries)
|
|
{
|
|
unwindInfoWriter.Write((int)unwindPushEntry.PseudoOp);
|
|
unwindInfoWriter.Write(unwindPushEntry.PrologOffset);
|
|
unwindInfoWriter.Write(unwindPushEntry.RegIndex);
|
|
unwindInfoWriter.Write(unwindPushEntry.StackOffsetOrAllocSize);
|
|
}
|
|
|
|
unwindInfoWriter.Write(unwindInfo.PrologSize);
|
|
}
|
|
}
|
|
|
|
private void WriteCode(ReadOnlySpan<byte> code)
|
|
{
|
|
_codesList.Add(code.ToArray());
|
|
}
|
|
|
|
public static bool GetEndianness()
|
|
{
|
|
return BitConverter.IsLittleEndian;
|
|
}
|
|
|
|
private static FeatureInfo GetFeatureInfo()
|
|
{
|
|
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
|
|
{
|
|
return new FeatureInfo(
|
|
(ulong)Arm64HardwareCapabilities.LinuxFeatureInfoHwCap,
|
|
(ulong)Arm64HardwareCapabilities.LinuxFeatureInfoHwCap2,
|
|
(ulong)Arm64HardwareCapabilities.MacOsFeatureInfo,
|
|
0,
|
|
0);
|
|
}
|
|
else if (RuntimeInformation.ProcessArchitecture == Architecture.X64)
|
|
{
|
|
return new FeatureInfo(
|
|
(ulong)X86HardwareCapabilities.FeatureInfo1Ecx,
|
|
(ulong)X86HardwareCapabilities.FeatureInfo1Edx,
|
|
(ulong)X86HardwareCapabilities.FeatureInfo7Ebx,
|
|
(ulong)X86HardwareCapabilities.FeatureInfo7Ecx,
|
|
(ulong)X86HardwareCapabilities.Xcr0InfoEax);
|
|
}
|
|
else
|
|
{
|
|
return new FeatureInfo(0, 0, 0, 0, 0);
|
|
}
|
|
}
|
|
|
|
private byte GetMemoryManagerMode()
|
|
{
|
|
return (byte)_memoryMode;
|
|
}
|
|
|
|
private static uint GetOSPlatform()
|
|
{
|
|
uint osPlatform = 0u;
|
|
|
|
osPlatform |= (OperatingSystem.IsFreeBSD() ? 1u : 0u) << 0;
|
|
osPlatform |= (OperatingSystem.IsLinux() ? 1u : 0u) << 1;
|
|
osPlatform |= (OperatingSystem.IsMacOS() ? 1u : 0u) << 2;
|
|
osPlatform |= (OperatingSystem.IsWindows() ? 1u : 0u) << 3;
|
|
|
|
return osPlatform;
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 86*/)]
|
|
private struct OuterHeader
|
|
{
|
|
public ulong Magic;
|
|
|
|
public uint CacheFileVersion;
|
|
|
|
public bool Endianness;
|
|
public FeatureInfo FeatureInfo;
|
|
public byte MemoryManagerMode;
|
|
public uint OSPlatform;
|
|
public uint Architecture;
|
|
|
|
public long UncompressedStreamSize;
|
|
|
|
public Hash128 HeaderHash;
|
|
|
|
public void SetHeaderHash()
|
|
{
|
|
Span<OuterHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
|
|
|
|
HeaderHash = XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<OuterHeader>() - Unsafe.SizeOf<Hash128>()));
|
|
}
|
|
|
|
public bool IsHeaderValid()
|
|
{
|
|
Span<OuterHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
|
|
|
|
return XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<OuterHeader>() - Unsafe.SizeOf<Hash128>())) == HeaderHash;
|
|
}
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 40*/)]
|
|
private record struct FeatureInfo(ulong FeatureInfo0, ulong FeatureInfo1, ulong FeatureInfo2, ulong FeatureInfo3, ulong FeatureInfo4);
|
|
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 128*/)]
|
|
private struct InnerHeader
|
|
{
|
|
public ulong Magic;
|
|
|
|
public int InfosLength;
|
|
public long CodesLength;
|
|
public int RelocsLength;
|
|
public int UnwindInfosLength;
|
|
|
|
public Hash128 InfosHash;
|
|
public Hash128 CodesHash;
|
|
public Hash128 RelocsHash;
|
|
public Hash128 UnwindInfosHash;
|
|
|
|
public Hash128 HeaderHash;
|
|
|
|
public void SetHeaderHash()
|
|
{
|
|
Span<InnerHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
|
|
|
|
HeaderHash = XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<InnerHeader>() - Unsafe.SizeOf<Hash128>()));
|
|
}
|
|
|
|
public bool IsHeaderValid()
|
|
{
|
|
Span<InnerHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
|
|
|
|
return XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<InnerHeader>() - Unsafe.SizeOf<Hash128>())) == HeaderHash;
|
|
}
|
|
}
|
|
|
|
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 42*/)]
|
|
private struct InfoEntry
|
|
{
|
|
public ulong Address;
|
|
public ulong GuestSize;
|
|
public Hash128 Hash;
|
|
public bool HighCq;
|
|
public bool Stubbed;
|
|
public int CodeLength;
|
|
public int RelocEntriesCount;
|
|
}
|
|
|
|
private void Enable()
|
|
{
|
|
State = PtcState.Enabled;
|
|
}
|
|
|
|
public void Continue()
|
|
{
|
|
if (State == PtcState.Enabled)
|
|
{
|
|
State = PtcState.Continuing;
|
|
}
|
|
}
|
|
|
|
public void Close()
|
|
{
|
|
if (State == PtcState.Enabled ||
|
|
State == PtcState.Continuing)
|
|
{
|
|
State = PtcState.Closing;
|
|
}
|
|
}
|
|
|
|
public void Disable()
|
|
{
|
|
State = PtcState.Disabled;
|
|
}
|
|
|
|
private void Wait()
|
|
{
|
|
_waitEvent.WaitOne();
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
if (!_disposed)
|
|
{
|
|
_disposed = true;
|
|
|
|
Wait();
|
|
_waitEvent.Dispose();
|
|
|
|
DisposeCarriers();
|
|
}
|
|
}
|
|
}
|
|
}
|