Ryujinx/ARMeilleure/Translation/PTC/Ptc.cs
Wunk 17620d18db
ARMeilleure: Add initial support for AVX512 (EVEX encoding) (cont) (#4147)
* ARMeilleure: Add AVX512{F,VL,DQ,BW} detection

Add `UseAvx512Ortho` and `UseAvx512OrthoFloat` optimization flags as
short-hands for `F+VL` and `F+VL+DQ`.

* ARMeilleure: Add initial support for EVEX instruction encoding

Does not implement rounding, or exception controls.

* ARMeilleure: Add `X86Vpternlogd`

Accelerates the vector-`Not` instruction.

* ARMeilleure: Add check for `OSXSAVE` for AVX{2,512}

* ARMeilleure: Add check for `XCR0` flags

Add XCR0 register checks for AVX and AVX512F, following the guidelines
from section 14.3 and 15.2 from the Intel Architecture Software
Developer's Manual.

* ARMeilleure: Remove redundant `ReProtect` and `Dispose`, formatting

* ARMeilleure: Move XCR0 procedure to GetXcr0Eax

* ARMeilleure: Add `XCR0` to `FeatureInfo` structure

* ARMeilleure: Utilize `ReadOnlySpan` for Xcr0 assembly

Avoids an additional allocation

* ARMeilleure: Formatting fixes

* ARMeilleure: Fix EVEX encoding src2 register index

> Just like in VEX prefix, vvvv is provided in inverted form.

* ARMeilleure: Add `X86Vpternlogd` acceleration to `Vmvn_I`

Passes unit tests, verified instruction utilization

* ARMeilleure: Fix EVEX register operand designations

Operand 2 was being sourced improperly.

EVEX encoded instructions source their operands like so:
Operand 1: ModRM:reg
Operand 2: EVEX.vvvvv
Operand 3: ModRM:r/m
Operand 4: Imm

This fixes the improper register designations when emitting vpternlog.
Now "dest", "src1", "src2" arguments emit in the proper order in EVEX instructions.

* ARMeilleure: Add `X86Vpternlogd` acceleration to `Orn_V`

* ARMeilleure: PTC version bump

* ARMeilleure: Update EVEX encoding Debug.Assert to Debug.Fail

* ARMeilleure: Update EVEX encoding comment capitalization
2023-03-20 16:09:24 -03:00

1131 lines
39 KiB
C#

using ARMeilleure.CodeGen;
using ARMeilleure.CodeGen.Linking;
using ARMeilleure.CodeGen.Unwinding;
using ARMeilleure.Common;
using ARMeilleure.Memory;
using Ryujinx.Common;
using Ryujinx.Common.Configuration;
using Ryujinx.Common.Logging;
using Ryujinx.Common.Memory;
using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Runtime;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using static ARMeilleure.Translation.PTC.PtcFormatter;
namespace ARMeilleure.Translation.PTC
{
using Arm64HardwareCapabilities = ARMeilleure.CodeGen.Arm64.HardwareCapabilities;
using X86HardwareCapabilities = ARMeilleure.CodeGen.X86.HardwareCapabilities;
class Ptc : IPtcLoadState
{
private const string OuterHeaderMagicString = "PTCohd\0\0";
private const string InnerHeaderMagicString = "PTCihd\0\0";
private const uint InternalVersion = 4485; //! To be incremented manually for each change to the ARMeilleure project.
private const string ActualDir = "0";
private const string BackupDir = "1";
private const string TitleIdTextDefault = "0000000000000000";
private const string DisplayVersionDefault = "0";
public static readonly Symbol PageTableSymbol = new(SymbolType.Special, 1);
public static readonly Symbol CountTableSymbol = new(SymbolType.Special, 2);
public static readonly Symbol DispatchStubSymbol = new(SymbolType.Special, 3);
private const byte FillingByte = 0x00;
private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
public PtcProfiler Profiler { get; }
// Carriers.
private MemoryStream _infosStream;
private List<byte[]> _codesList;
private MemoryStream _relocsStream;
private MemoryStream _unwindInfosStream;
private readonly ulong _outerHeaderMagic;
private readonly ulong _innerHeaderMagic;
private readonly ManualResetEvent _waitEvent;
private readonly object _lock;
private bool _disposed;
public string TitleIdText { get; private set; }
public string DisplayVersion { get; private set; }
private MemoryManagerType _memoryMode;
public string CachePathActual { get; private set; }
public string CachePathBackup { get; private set; }
public PtcState State { get; private set; }
// Progress reporting helpers.
private volatile int _translateCount;
private volatile int _translateTotalCount;
public event Action<PtcLoadingState, int, int> PtcStateChanged;
public Ptc()
{
Profiler = new PtcProfiler(this);
InitializeCarriers();
_outerHeaderMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(OuterHeaderMagicString).AsSpan());
_innerHeaderMagic = BinaryPrimitives.ReadUInt64LittleEndian(EncodingCache.UTF8NoBOM.GetBytes(InnerHeaderMagicString).AsSpan());
_waitEvent = new ManualResetEvent(true);
_lock = new object();
_disposed = false;
TitleIdText = TitleIdTextDefault;
DisplayVersion = DisplayVersionDefault;
CachePathActual = string.Empty;
CachePathBackup = string.Empty;
Disable();
}
public void Initialize(string titleIdText, string displayVersion, bool enabled, MemoryManagerType memoryMode)
{
Wait();
Profiler.Wait();
Profiler.ClearEntries();
Logger.Info?.Print(LogClass.Ptc, $"Initializing Profiled Persistent Translation Cache (enabled: {enabled}).");
if (!enabled || string.IsNullOrEmpty(titleIdText) || titleIdText == TitleIdTextDefault)
{
TitleIdText = TitleIdTextDefault;
DisplayVersion = DisplayVersionDefault;
CachePathActual = string.Empty;
CachePathBackup = string.Empty;
Disable();
return;
}
TitleIdText = titleIdText;
DisplayVersion = !string.IsNullOrEmpty(displayVersion) ? displayVersion : DisplayVersionDefault;
_memoryMode = memoryMode;
string workPathActual = Path.Combine(AppDataManager.GamesDirPath, TitleIdText, "cache", "cpu", ActualDir);
string workPathBackup = Path.Combine(AppDataManager.GamesDirPath, TitleIdText, "cache", "cpu", BackupDir);
if (!Directory.Exists(workPathActual))
{
Directory.CreateDirectory(workPathActual);
}
if (!Directory.Exists(workPathBackup))
{
Directory.CreateDirectory(workPathBackup);
}
CachePathActual = Path.Combine(workPathActual, DisplayVersion);
CachePathBackup = Path.Combine(workPathBackup, DisplayVersion);
PreLoad();
Profiler.PreLoad();
Enable();
}
private void InitializeCarriers()
{
_infosStream = MemoryStreamManager.Shared.GetStream();
_codesList = new List<byte[]>();
_relocsStream = MemoryStreamManager.Shared.GetStream();
_unwindInfosStream = MemoryStreamManager.Shared.GetStream();
}
private void DisposeCarriers()
{
_infosStream.Dispose();
_codesList.Clear();
_relocsStream.Dispose();
_unwindInfosStream.Dispose();
}
private bool AreCarriersEmpty()
{
return _infosStream.Length == 0L && _codesList.Count == 0 && _relocsStream.Length == 0L && _unwindInfosStream.Length == 0L;
}
private void ResetCarriersIfNeeded()
{
if (AreCarriersEmpty())
{
return;
}
DisposeCarriers();
InitializeCarriers();
}
private void PreLoad()
{
string fileNameActual = $"{CachePathActual}.cache";
string fileNameBackup = $"{CachePathBackup}.cache";
FileInfo fileInfoActual = new FileInfo(fileNameActual);
FileInfo fileInfoBackup = new FileInfo(fileNameBackup);
if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
{
if (!Load(fileNameActual, false))
{
if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
{
Load(fileNameBackup, true);
}
}
}
else if (fileInfoBackup.Exists && fileInfoBackup.Length != 0L)
{
Load(fileNameBackup, true);
}
}
private unsafe bool Load(string fileName, bool isBackup)
{
using (FileStream compressedStream = new(fileName, FileMode.Open))
using (DeflateStream deflateStream = new(compressedStream, CompressionMode.Decompress, true))
{
OuterHeader outerHeader = DeserializeStructure<OuterHeader>(compressedStream);
if (!outerHeader.IsHeaderValid())
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.Magic != _outerHeaderMagic)
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.CacheFileVersion != InternalVersion)
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.Endianness != GetEndianness())
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.FeatureInfo != GetFeatureInfo())
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.MemoryManagerMode != GetMemoryManagerMode())
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.OSPlatform != GetOSPlatform())
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (outerHeader.Architecture != (uint)RuntimeInformation.ProcessArchitecture)
{
InvalidateCompressedStream(compressedStream);
return false;
}
IntPtr intPtr = IntPtr.Zero;
try
{
intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize));
using (UnmanagedMemoryStream stream = new((byte*)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite))
{
try
{
deflateStream.CopyTo(stream);
}
catch
{
InvalidateCompressedStream(compressedStream);
return false;
}
Debug.Assert(stream.Position == stream.Length);
stream.Seek(0L, SeekOrigin.Begin);
InnerHeader innerHeader = DeserializeStructure<InnerHeader>(stream);
if (!innerHeader.IsHeaderValid())
{
InvalidateCompressedStream(compressedStream);
return false;
}
if (innerHeader.Magic != _innerHeaderMagic)
{
InvalidateCompressedStream(compressedStream);
return false;
}
ReadOnlySpan<byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength);
stream.Seek(innerHeader.InfosLength, SeekOrigin.Current);
Hash128 infosHash = XXHash128.ComputeHash(infosBytes);
if (innerHeader.InfosHash != infosHash)
{
InvalidateCompressedStream(compressedStream);
return false;
}
ReadOnlySpan<byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan<byte>.Empty;
stream.Seek(innerHeader.CodesLength, SeekOrigin.Current);
Hash128 codesHash = XXHash128.ComputeHash(codesBytes);
if (innerHeader.CodesHash != codesHash)
{
InvalidateCompressedStream(compressedStream);
return false;
}
ReadOnlySpan<byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength);
stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current);
Hash128 relocsHash = XXHash128.ComputeHash(relocsBytes);
if (innerHeader.RelocsHash != relocsHash)
{
InvalidateCompressedStream(compressedStream);
return false;
}
ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
Hash128 unwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
if (innerHeader.UnwindInfosHash != unwindInfosHash)
{
InvalidateCompressedStream(compressedStream);
return false;
}
Debug.Assert(stream.Position == stream.Length);
stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
_infosStream.Write(infosBytes);
stream.Seek(innerHeader.InfosLength, SeekOrigin.Current);
_codesList.ReadFrom(stream);
_relocsStream.Write(relocsBytes);
stream.Seek(innerHeader.RelocsLength, SeekOrigin.Current);
_unwindInfosStream.Write(unwindInfosBytes);
stream.Seek(innerHeader.UnwindInfosLength, SeekOrigin.Current);
Debug.Assert(stream.Position == stream.Length);
}
}
finally
{
if (intPtr != IntPtr.Zero)
{
Marshal.FreeHGlobal(intPtr);
}
}
}
long fileSize = new FileInfo(fileName).Length;
Logger.Info?.Print(LogClass.Ptc, $"{(isBackup ? "Loaded Backup Translation Cache" : "Loaded Translation Cache")} (size: {fileSize} bytes, translated functions: {GetEntriesCount()}).");
return true;
}
private void InvalidateCompressedStream(FileStream compressedStream)
{
compressedStream.SetLength(0L);
}
private void PreSave()
{
_waitEvent.Reset();
try
{
string fileNameActual = $"{CachePathActual}.cache";
string fileNameBackup = $"{CachePathBackup}.cache";
FileInfo fileInfoActual = new FileInfo(fileNameActual);
if (fileInfoActual.Exists && fileInfoActual.Length != 0L)
{
File.Copy(fileNameActual, fileNameBackup, true);
}
Save(fileNameActual);
}
finally
{
ResetCarriersIfNeeded();
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
}
_waitEvent.Set();
}
private unsafe void Save(string fileName)
{
int translatedFuncsCount;
InnerHeader innerHeader = new InnerHeader();
innerHeader.Magic = _innerHeaderMagic;
innerHeader.InfosLength = (int)_infosStream.Length;
innerHeader.CodesLength = _codesList.Length();
innerHeader.RelocsLength = (int)_relocsStream.Length;
innerHeader.UnwindInfosLength = (int)_unwindInfosStream.Length;
OuterHeader outerHeader = new OuterHeader();
outerHeader.Magic = _outerHeaderMagic;
outerHeader.CacheFileVersion = InternalVersion;
outerHeader.Endianness = GetEndianness();
outerHeader.FeatureInfo = GetFeatureInfo();
outerHeader.MemoryManagerMode = GetMemoryManagerMode();
outerHeader.OSPlatform = GetOSPlatform();
outerHeader.Architecture = (uint)RuntimeInformation.ProcessArchitecture;
outerHeader.UncompressedStreamSize =
(long)Unsafe.SizeOf<InnerHeader>() +
innerHeader.InfosLength +
innerHeader.CodesLength +
innerHeader.RelocsLength +
innerHeader.UnwindInfosLength;
outerHeader.SetHeaderHash();
IntPtr intPtr = IntPtr.Zero;
try
{
intPtr = Marshal.AllocHGlobal(new IntPtr(outerHeader.UncompressedStreamSize));
using (UnmanagedMemoryStream stream = new((byte*)intPtr.ToPointer(), outerHeader.UncompressedStreamSize, outerHeader.UncompressedStreamSize, FileAccess.ReadWrite))
{
stream.Seek((long)Unsafe.SizeOf<InnerHeader>(), SeekOrigin.Begin);
ReadOnlySpan<byte> infosBytes = new(stream.PositionPointer, innerHeader.InfosLength);
_infosStream.WriteTo(stream);
ReadOnlySpan<byte> codesBytes = (int)innerHeader.CodesLength > 0 ? new(stream.PositionPointer, (int)innerHeader.CodesLength) : ReadOnlySpan<byte>.Empty;
_codesList.WriteTo(stream);
ReadOnlySpan<byte> relocsBytes = new(stream.PositionPointer, innerHeader.RelocsLength);
_relocsStream.WriteTo(stream);
ReadOnlySpan<byte> unwindInfosBytes = new(stream.PositionPointer, innerHeader.UnwindInfosLength);
_unwindInfosStream.WriteTo(stream);
Debug.Assert(stream.Position == stream.Length);
innerHeader.InfosHash = XXHash128.ComputeHash(infosBytes);
innerHeader.CodesHash = XXHash128.ComputeHash(codesBytes);
innerHeader.RelocsHash = XXHash128.ComputeHash(relocsBytes);
innerHeader.UnwindInfosHash = XXHash128.ComputeHash(unwindInfosBytes);
innerHeader.SetHeaderHash();
stream.Seek(0L, SeekOrigin.Begin);
SerializeStructure(stream, innerHeader);
translatedFuncsCount = GetEntriesCount();
ResetCarriersIfNeeded();
using (FileStream compressedStream = new(fileName, FileMode.OpenOrCreate))
using (DeflateStream deflateStream = new(compressedStream, SaveCompressionLevel, true))
{
try
{
SerializeStructure(compressedStream, outerHeader);
stream.Seek(0L, SeekOrigin.Begin);
stream.CopyTo(deflateStream);
}
catch
{
compressedStream.Position = 0L;
}
if (compressedStream.Position < compressedStream.Length)
{
compressedStream.SetLength(compressedStream.Position);
}
}
}
}
finally
{
if (intPtr != IntPtr.Zero)
{
Marshal.FreeHGlobal(intPtr);
}
}
long fileSize = new FileInfo(fileName).Length;
if (fileSize != 0L)
{
Logger.Info?.Print(LogClass.Ptc, $"Saved Translation Cache (size: {fileSize} bytes, translated functions: {translatedFuncsCount}).");
}
}
public void LoadTranslations(Translator translator)
{
if (AreCarriersEmpty())
{
return;
}
long infosStreamLength = _infosStream.Length;
long relocsStreamLength = _relocsStream.Length;
long unwindInfosStreamLength = _unwindInfosStream.Length;
_infosStream.Seek(0L, SeekOrigin.Begin);
_relocsStream.Seek(0L, SeekOrigin.Begin);
_unwindInfosStream.Seek(0L, SeekOrigin.Begin);
using (BinaryReader relocsReader = new(_relocsStream, EncodingCache.UTF8NoBOM, true))
using (BinaryReader unwindInfosReader = new(_unwindInfosStream, EncodingCache.UTF8NoBOM, true))
{
for (int index = 0; index < GetEntriesCount(); index++)
{
InfoEntry infoEntry = DeserializeStructure<InfoEntry>(_infosStream);
if (infoEntry.Stubbed)
{
SkipCode(index, infoEntry.CodeLength);
SkipReloc(infoEntry.RelocEntriesCount);
SkipUnwindInfo(unwindInfosReader);
continue;
}
bool isEntryChanged = infoEntry.Hash != ComputeHash(translator.Memory, infoEntry.Address, infoEntry.GuestSize);
if (isEntryChanged || (!infoEntry.HighCq && Profiler.ProfiledFuncs.TryGetValue(infoEntry.Address, out var value) && value.HighCq))
{
infoEntry.Stubbed = true;
infoEntry.CodeLength = 0;
UpdateInfo(infoEntry);
StubCode(index);
StubReloc(infoEntry.RelocEntriesCount);
StubUnwindInfo(unwindInfosReader);
if (isEntryChanged)
{
Logger.Info?.Print(LogClass.Ptc, $"Invalidated translated function (address: 0x{infoEntry.Address:X16})");
}
continue;
}
byte[] code = ReadCode(index, infoEntry.CodeLength);
Counter<uint> callCounter = null;
if (infoEntry.RelocEntriesCount != 0)
{
RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
PatchCode(translator, code, relocEntries, out callCounter);
}
UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
translator.RegisterFunction(infoEntry.Address, func);
bool isAddressUnique = translator.Functions.TryAdd(infoEntry.Address, infoEntry.GuestSize, func);
Debug.Assert(isAddressUnique, $"The address 0x{infoEntry.Address:X16} is not unique.");
}
}
if (_infosStream.Length != infosStreamLength || _infosStream.Position != infosStreamLength ||
_relocsStream.Length != relocsStreamLength || _relocsStream.Position != relocsStreamLength ||
_unwindInfosStream.Length != unwindInfosStreamLength || _unwindInfosStream.Position != unwindInfosStreamLength)
{
throw new Exception("The length of a memory stream has changed, or its position has not reached or has exceeded its end.");
}
Logger.Info?.Print(LogClass.Ptc, $"{translator.Functions.Count} translated functions loaded");
}
private int GetEntriesCount()
{
return _codesList.Count;
}
[Conditional("DEBUG")]
private void SkipCode(int index, int codeLength)
{
Debug.Assert(_codesList[index].Length == 0);
Debug.Assert(codeLength == 0);
}
private void SkipReloc(int relocEntriesCount)
{
_relocsStream.Seek(relocEntriesCount * RelocEntry.Stride, SeekOrigin.Current);
}
private void SkipUnwindInfo(BinaryReader unwindInfosReader)
{
int pushEntriesLength = unwindInfosReader.ReadInt32();
_unwindInfosStream.Seek(pushEntriesLength * UnwindPushEntry.Stride + UnwindInfo.Stride, SeekOrigin.Current);
}
private byte[] ReadCode(int index, int codeLength)
{
Debug.Assert(_codesList[index].Length == codeLength);
return _codesList[index];
}
private RelocEntry[] GetRelocEntries(BinaryReader relocsReader, int relocEntriesCount)
{
RelocEntry[] relocEntries = new RelocEntry[relocEntriesCount];
for (int i = 0; i < relocEntriesCount; i++)
{
int position = relocsReader.ReadInt32();
SymbolType type = (SymbolType)relocsReader.ReadByte();
ulong value = relocsReader.ReadUInt64();
relocEntries[i] = new RelocEntry(position, new Symbol(type, value));
}
return relocEntries;
}
private void PatchCode(Translator translator, Span<byte> code, RelocEntry[] relocEntries, out Counter<uint> callCounter)
{
callCounter = null;
foreach (RelocEntry relocEntry in relocEntries)
{
IntPtr? imm = null;
Symbol symbol = relocEntry.Symbol;
if (symbol.Type == SymbolType.FunctionTable)
{
ulong guestAddress = symbol.Value;
if (translator.FunctionTable.IsValid(guestAddress))
{
unsafe { imm = (IntPtr)Unsafe.AsPointer(ref translator.FunctionTable.GetValue(guestAddress)); }
}
}
else if (symbol.Type == SymbolType.DelegateTable)
{
int index = (int)symbol.Value;
if (Delegates.TryGetDelegateFuncPtrByIndex(index, out IntPtr funcPtr))
{
imm = funcPtr;
}
}
else if (symbol == PageTableSymbol)
{
imm = translator.Memory.PageTablePointer;
}
else if (symbol == CountTableSymbol)
{
if (callCounter == null)
{
callCounter = new Counter<uint>(translator.CountTable);
}
unsafe { imm = (IntPtr)Unsafe.AsPointer(ref callCounter.Value); }
}
else if (symbol == DispatchStubSymbol)
{
imm = translator.Stubs.DispatchStub;
}
if (imm == null)
{
throw new Exception($"Unexpected reloc entry {relocEntry}.");
}
BinaryPrimitives.WriteUInt64LittleEndian(code.Slice(relocEntry.Position, 8), (ulong)imm.Value);
}
}
private UnwindInfo ReadUnwindInfo(BinaryReader unwindInfosReader)
{
int pushEntriesLength = unwindInfosReader.ReadInt32();
UnwindPushEntry[] pushEntries = new UnwindPushEntry[pushEntriesLength];
for (int i = 0; i < pushEntriesLength; i++)
{
int pseudoOp = unwindInfosReader.ReadInt32();
int prologOffset = unwindInfosReader.ReadInt32();
int regIndex = unwindInfosReader.ReadInt32();
int stackOffsetOrAllocSize = unwindInfosReader.ReadInt32();
pushEntries[i] = new UnwindPushEntry((UnwindPseudoOp)pseudoOp, prologOffset, regIndex, stackOffsetOrAllocSize);
}
int prologueSize = unwindInfosReader.ReadInt32();
return new UnwindInfo(pushEntries, prologueSize);
}
private TranslatedFunction FastTranslate(
byte[] code,
Counter<uint> callCounter,
ulong guestSize,
UnwindInfo unwindInfo,
bool highCq)
{
var cFunc = new CompiledFunction(code, unwindInfo, RelocInfo.Empty);
var gFunc = cFunc.MapWithPointer<GuestFunction>(out IntPtr gFuncPointer);
return new TranslatedFunction(gFunc, gFuncPointer, callCounter, guestSize, highCq);
}
private void UpdateInfo(InfoEntry infoEntry)
{
_infosStream.Seek(-Unsafe.SizeOf<InfoEntry>(), SeekOrigin.Current);
SerializeStructure(_infosStream, infoEntry);
}
private void StubCode(int index)
{
_codesList[index] = Array.Empty<byte>();
}
private void StubReloc(int relocEntriesCount)
{
for (int i = 0; i < relocEntriesCount * RelocEntry.Stride; i++)
{
_relocsStream.WriteByte(FillingByte);
}
}
private void StubUnwindInfo(BinaryReader unwindInfosReader)
{
int pushEntriesLength = unwindInfosReader.ReadInt32();
for (int i = 0; i < pushEntriesLength * UnwindPushEntry.Stride + UnwindInfo.Stride; i++)
{
_unwindInfosStream.WriteByte(FillingByte);
}
}
public void MakeAndSaveTranslations(Translator translator)
{
var profiledFuncsToTranslate = Profiler.GetProfiledFuncsToTranslate(translator.Functions);
_translateCount = 0;
_translateTotalCount = profiledFuncsToTranslate.Count;
if (_translateTotalCount == 0)
{
ResetCarriersIfNeeded();
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
return;
}
int degreeOfParallelism = Environment.ProcessorCount;
// If there are enough cores lying around, we leave one alone for other tasks.
if (degreeOfParallelism > 4)
{
degreeOfParallelism--;
}
Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}");
PtcStateChanged?.Invoke(PtcLoadingState.Start, _translateCount, _translateTotalCount);
using AutoResetEvent progressReportEvent = new AutoResetEvent(false);
Thread progressReportThread = new Thread(ReportProgress)
{
Name = "Ptc.ProgressReporter",
Priority = ThreadPriority.Lowest,
IsBackground = true
};
progressReportThread.Start(progressReportEvent);
void TranslateFuncs()
{
while (profiledFuncsToTranslate.TryDequeue(out var item))
{
ulong address = item.address;
Debug.Assert(Profiler.IsAddressInStaticCodeRange(address));
TranslatedFunction func = translator.Translate(address, item.funcProfile.Mode, item.funcProfile.HighCq);
bool isAddressUnique = translator.Functions.TryAdd(address, func.GuestSize, func);
Debug.Assert(isAddressUnique, $"The address 0x{address:X16} is not unique.");
Interlocked.Increment(ref _translateCount);
translator.RegisterFunction(address, func);
if (State != PtcState.Enabled)
{
break;
}
}
}
List<Thread> threads = new List<Thread>();
for (int i = 0; i < degreeOfParallelism; i++)
{
Thread thread = new Thread(TranslateFuncs);
thread.IsBackground = true;
threads.Add(thread);
}
Stopwatch sw = Stopwatch.StartNew();
threads.ForEach((thread) => thread.Start());
threads.ForEach((thread) => thread.Join());
threads.Clear();
progressReportEvent.Set();
progressReportThread.Join();
sw.Stop();
PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount);
Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s");
Thread preSaveThread = new Thread(PreSave);
preSaveThread.IsBackground = true;
preSaveThread.Start();
}
private void ReportProgress(object state)
{
const int refreshRate = 50; // ms.
AutoResetEvent endEvent = (AutoResetEvent)state;
int count = 0;
do
{
int newCount = _translateCount;
if (count != newCount)
{
PtcStateChanged?.Invoke(PtcLoadingState.Loading, newCount, _translateTotalCount);
count = newCount;
}
}
while (!endEvent.WaitOne(refreshRate));
}
public static Hash128 ComputeHash(IMemoryManager memory, ulong address, ulong guestSize)
{
return XXHash128.ComputeHash(memory.GetSpan(address, checked((int)(guestSize))));
}
public void WriteCompiledFunction(ulong address, ulong guestSize, Hash128 hash, bool highCq, CompiledFunction compiledFunc)
{
lock (_lock)
{
byte[] code = compiledFunc.Code;
RelocInfo relocInfo = compiledFunc.RelocInfo;
UnwindInfo unwindInfo = compiledFunc.UnwindInfo;
InfoEntry infoEntry = new InfoEntry();
infoEntry.Address = address;
infoEntry.GuestSize = guestSize;
infoEntry.Hash = hash;
infoEntry.HighCq = highCq;
infoEntry.Stubbed = false;
infoEntry.CodeLength = code.Length;
infoEntry.RelocEntriesCount = relocInfo.Entries.Length;
SerializeStructure(_infosStream, infoEntry);
WriteCode(code.AsSpan());
// WriteReloc.
using var relocInfoWriter = new BinaryWriter(_relocsStream, EncodingCache.UTF8NoBOM, true);
foreach (RelocEntry entry in relocInfo.Entries)
{
relocInfoWriter.Write(entry.Position);
relocInfoWriter.Write((byte)entry.Symbol.Type);
relocInfoWriter.Write(entry.Symbol.Value);
}
// WriteUnwindInfo.
using var unwindInfoWriter = new BinaryWriter(_unwindInfosStream, EncodingCache.UTF8NoBOM, true);
unwindInfoWriter.Write(unwindInfo.PushEntries.Length);
foreach (UnwindPushEntry unwindPushEntry in unwindInfo.PushEntries)
{
unwindInfoWriter.Write((int)unwindPushEntry.PseudoOp);
unwindInfoWriter.Write(unwindPushEntry.PrologOffset);
unwindInfoWriter.Write(unwindPushEntry.RegIndex);
unwindInfoWriter.Write(unwindPushEntry.StackOffsetOrAllocSize);
}
unwindInfoWriter.Write(unwindInfo.PrologSize);
}
}
private void WriteCode(ReadOnlySpan<byte> code)
{
_codesList.Add(code.ToArray());
}
public static bool GetEndianness()
{
return BitConverter.IsLittleEndian;
}
private static FeatureInfo GetFeatureInfo()
{
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
return new FeatureInfo(
(ulong)Arm64HardwareCapabilities.LinuxFeatureInfoHwCap,
(ulong)Arm64HardwareCapabilities.LinuxFeatureInfoHwCap2,
(ulong)Arm64HardwareCapabilities.MacOsFeatureInfo,
0,
0);
}
else if (RuntimeInformation.ProcessArchitecture == Architecture.X64)
{
return new FeatureInfo(
(ulong)X86HardwareCapabilities.FeatureInfo1Ecx,
(ulong)X86HardwareCapabilities.FeatureInfo1Edx,
(ulong)X86HardwareCapabilities.FeatureInfo7Ebx,
(ulong)X86HardwareCapabilities.FeatureInfo7Ecx,
(ulong)X86HardwareCapabilities.Xcr0InfoEax);
}
else
{
return new FeatureInfo(0, 0, 0, 0, 0);
}
}
private byte GetMemoryManagerMode()
{
return (byte)_memoryMode;
}
private static uint GetOSPlatform()
{
uint osPlatform = 0u;
osPlatform |= (OperatingSystem.IsFreeBSD() ? 1u : 0u) << 0;
osPlatform |= (OperatingSystem.IsLinux() ? 1u : 0u) << 1;
osPlatform |= (OperatingSystem.IsMacOS() ? 1u : 0u) << 2;
osPlatform |= (OperatingSystem.IsWindows() ? 1u : 0u) << 3;
return osPlatform;
}
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 86*/)]
private struct OuterHeader
{
public ulong Magic;
public uint CacheFileVersion;
public bool Endianness;
public FeatureInfo FeatureInfo;
public byte MemoryManagerMode;
public uint OSPlatform;
public uint Architecture;
public long UncompressedStreamSize;
public Hash128 HeaderHash;
public void SetHeaderHash()
{
Span<OuterHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
HeaderHash = XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<OuterHeader>() - Unsafe.SizeOf<Hash128>()));
}
public bool IsHeaderValid()
{
Span<OuterHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
return XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<OuterHeader>() - Unsafe.SizeOf<Hash128>())) == HeaderHash;
}
}
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 40*/)]
private record struct FeatureInfo(ulong FeatureInfo0, ulong FeatureInfo1, ulong FeatureInfo2, ulong FeatureInfo3, ulong FeatureInfo4);
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 128*/)]
private struct InnerHeader
{
public ulong Magic;
public int InfosLength;
public long CodesLength;
public int RelocsLength;
public int UnwindInfosLength;
public Hash128 InfosHash;
public Hash128 CodesHash;
public Hash128 RelocsHash;
public Hash128 UnwindInfosHash;
public Hash128 HeaderHash;
public void SetHeaderHash()
{
Span<InnerHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
HeaderHash = XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<InnerHeader>() - Unsafe.SizeOf<Hash128>()));
}
public bool IsHeaderValid()
{
Span<InnerHeader> spanHeader = MemoryMarshal.CreateSpan(ref this, 1);
return XXHash128.ComputeHash(MemoryMarshal.AsBytes(spanHeader).Slice(0, Unsafe.SizeOf<InnerHeader>() - Unsafe.SizeOf<Hash128>())) == HeaderHash;
}
}
[StructLayout(LayoutKind.Sequential, Pack = 1/*, Size = 42*/)]
private struct InfoEntry
{
public ulong Address;
public ulong GuestSize;
public Hash128 Hash;
public bool HighCq;
public bool Stubbed;
public int CodeLength;
public int RelocEntriesCount;
}
private void Enable()
{
State = PtcState.Enabled;
}
public void Continue()
{
if (State == PtcState.Enabled)
{
State = PtcState.Continuing;
}
}
public void Close()
{
if (State == PtcState.Enabled ||
State == PtcState.Continuing)
{
State = PtcState.Closing;
}
}
public void Disable()
{
State = PtcState.Disabled;
}
private void Wait()
{
_waitEvent.WaitOne();
}
public void Dispose()
{
if (!_disposed)
{
_disposed = true;
Wait();
_waitEvent.Dispose();
DisposeCarriers();
}
}
}
}