Ryujinx/ARMeilleure/Translation/EmitterContext.cs
FICTURE7 9d7627af64
Add multi-level function table (#2228)
* Add AddressTable<T>

* Use AddressTable<T> for dispatch

* Remove JumpTable & co.

* Add fallback for out of range addresses

* Add PPTC support

* Add documentation to `AddressTable<T>`

* Make AddressTable<T> configurable

* Fix table walk

* Fix IsMapped check

* Remove CountTableCapacity

* Add PPTC support for fast path

* Rename IsMapped to IsValid

* Remove stale comment

* Change format of address in exception message

* Add TranslatorStubs

* Split DispatchStub

Avoids recompilation of stubs during tests.

* Add hint for 64bit or 32bit

* Add documentation to `Symbol`

* Add documentation to `TranslatorStubs`

Make `TranslatorStubs` disposable as well.

* Add documentation to `SymbolType`

* Add `AddressTableEventSource` to monitor function table size

Add an EventSource which measures the amount of unmanaged bytes
allocated by AddressTable<T> instances.

 dotnet-counters monitor -n Ryujinx --counters ARMeilleure

* Add `AllowLcqInFunctionTable` optimization toggle

This is to reduce the impact this change has on the test duration.
Before everytime a test was ran, the FunctionTable would be initialized
and populated so that the newly compiled test would get registered to
it.

* Implement unmanaged dispatcher

Uses the DispatchStub to dispatch into the next translation, which
allows execution to stay in unmanaged for longer and skips a
ConcurrentDictionary look up when the target translation has been
registered to the FunctionTable.

* Remove redundant null check

* Tune levels of FunctionTable

Uses 5 levels instead of 4 and change unit of AddressTableEventSource
from KB to MB.

* Use 64-bit function table

Improves codegen for direct branches:

    mov qword [rax+0x408],0x10603560
 -  mov rcx,sub_10603560_OFFSET
 -  mov ecx,[rcx]
 -  mov ecx,ecx
 -  mov rdx,JIT_CACHE_BASE
 -  add rdx,rcx
 +  mov rcx,sub_10603560
 +  mov rdx,[rcx]
    mov rcx,rax

Improves codegen for dispatch stub:

    and rax,byte +0x1f
 -  mov eax,[rcx+rax*4]
 -  mov eax,eax
 -  mov rcx,JIT_CACHE_BASE
 -  lea rax,[rcx+rax]
 +  mov rax,[rcx+rax*8]
    mov rcx,rbx

* Remove `JitCacheSymbol` & `JitCache.Offset`

* Turn `Translator.Translate` into an instance method

We do not have to add more parameter to this method and related ones as
new structures are added & needed for translation.

* Add symbol only when PTC is enabled

Address LDj3SNuD's feedback

* Change `NativeContext.Running` to a 32-bit integer

* Fix PageTable symbol for host mapped
2021-05-29 18:06:28 -03:00

676 lines
20 KiB
C#

using ARMeilleure.Diagnostics;
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.State;
using ARMeilleure.Translation.PTC;
using System;
using System.Collections.Generic;
using System.Reflection;
using static ARMeilleure.IntermediateRepresentation.OperandHelper;
namespace ARMeilleure.Translation
{
class EmitterContext
{
private int _localsCount;
private readonly Dictionary<Operand, BasicBlock> _irLabels;
private readonly IntrusiveList<BasicBlock> _irBlocks;
private BasicBlock _irBlock;
private BasicBlock _ifBlock;
private bool _needsNewBlock;
private BasicBlockFrequency _nextBlockFreq;
public EmitterContext()
{
_localsCount = 0;
_irLabels = new Dictionary<Operand, BasicBlock>();
_irBlocks = new IntrusiveList<BasicBlock>();
_needsNewBlock = true;
_nextBlockFreq = BasicBlockFrequency.Default;
}
public Operand AllocateLocal(OperandType type)
{
Operand local = Local(type);
local.NumberLocal(++_localsCount);
return local;
}
public Operand Add(Operand op1, Operand op2)
{
return Add(Instruction.Add, Local(op1.Type), op1, op2);
}
public Operand BitwiseAnd(Operand op1, Operand op2)
{
return Add(Instruction.BitwiseAnd, Local(op1.Type), op1, op2);
}
public Operand BitwiseExclusiveOr(Operand op1, Operand op2)
{
return Add(Instruction.BitwiseExclusiveOr, Local(op1.Type), op1, op2);
}
public Operand BitwiseNot(Operand op1)
{
return Add(Instruction.BitwiseNot, Local(op1.Type), op1);
}
public Operand BitwiseOr(Operand op1, Operand op2)
{
return Add(Instruction.BitwiseOr, Local(op1.Type), op1, op2);
}
public void Branch(Operand label)
{
NewNextBlockIfNeeded();
BranchToLabel(label, uncond: true, BasicBlockFrequency.Default);
}
public void BranchIf(Operand label, Operand op1, Operand op2, Comparison comp, BasicBlockFrequency falseFreq = default)
{
Add(Instruction.BranchIf, null, op1, op2, Const((int)comp));
BranchToLabel(label, uncond: false, falseFreq);
}
public void BranchIfFalse(Operand label, Operand op1, BasicBlockFrequency falseFreq = default)
{
BranchIf(label, op1, Const(op1.Type, 0), Comparison.Equal, falseFreq);
}
public void BranchIfTrue(Operand label, Operand op1, BasicBlockFrequency falseFreq = default)
{
BranchIf(label, op1, Const(op1.Type, 0), Comparison.NotEqual, falseFreq);
}
public Operand ByteSwap(Operand op1)
{
return Add(Instruction.ByteSwap, Local(op1.Type), op1);
}
public virtual Operand Call(MethodInfo info, params Operand[] callArgs)
{
IntPtr funcPtr = Delegates.GetDelegateFuncPtr(info);
OperandType returnType = GetOperandType(info.ReturnType);
Symbols.Add((ulong)funcPtr.ToInt64(), info.Name);
return Call(Const(funcPtr.ToInt64()), returnType, callArgs);
}
protected static OperandType GetOperandType(Type type)
{
if (type == typeof(bool) || type == typeof(byte) ||
type == typeof(char) || type == typeof(short) ||
type == typeof(int) || type == typeof(sbyte) ||
type == typeof(ushort) || type == typeof(uint))
{
return OperandType.I32;
}
else if (type == typeof(long) || type == typeof(ulong))
{
return OperandType.I64;
}
else if (type == typeof(double))
{
return OperandType.FP64;
}
else if (type == typeof(float))
{
return OperandType.FP32;
}
else if (type == typeof(V128))
{
return OperandType.V128;
}
else if (type == typeof(void))
{
return OperandType.None;
}
else
{
throw new ArgumentException($"Invalid type \"{type.Name}\".");
}
}
public Operand Call(Operand address, OperandType returnType, params Operand[] callArgs)
{
Operand[] args = new Operand[callArgs.Length + 1];
args[0] = address;
Array.Copy(callArgs, 0, args, 1, callArgs.Length);
if (returnType != OperandType.None)
{
return Add(Instruction.Call, Local(returnType), args);
}
else
{
return Add(Instruction.Call, null, args);
}
}
public void Tailcall(Operand address, params Operand[] callArgs)
{
Operand[] args = new Operand[callArgs.Length + 1];
args[0] = address;
Array.Copy(callArgs, 0, args, 1, callArgs.Length);
Add(Instruction.Tailcall, null, args);
_needsNewBlock = true;
}
public Operand CompareAndSwap(Operand address, Operand expected, Operand desired)
{
return Add(Instruction.CompareAndSwap, Local(desired.Type), address, expected, desired);
}
public Operand CompareAndSwap16(Operand address, Operand expected, Operand desired)
{
return Add(Instruction.CompareAndSwap16, Local(OperandType.I32), address, expected, desired);
}
public Operand CompareAndSwap8(Operand address, Operand expected, Operand desired)
{
return Add(Instruction.CompareAndSwap8, Local(OperandType.I32), address, expected, desired);
}
public Operand ConditionalSelect(Operand op1, Operand op2, Operand op3)
{
return Add(Instruction.ConditionalSelect, Local(op2.Type), op1, op2, op3);
}
public Operand ConvertI64ToI32(Operand op1)
{
if (op1.Type != OperandType.I64)
{
throw new ArgumentException($"Invalid operand type \"{op1.Type}\".");
}
return Add(Instruction.ConvertI64ToI32, Local(OperandType.I32), op1);
}
public Operand ConvertToFP(OperandType type, Operand op1)
{
return Add(Instruction.ConvertToFP, Local(type), op1);
}
public Operand ConvertToFPUI(OperandType type, Operand op1)
{
return Add(Instruction.ConvertToFPUI, Local(type), op1);
}
public Operand Copy(Operand op1)
{
return Add(Instruction.Copy, Local(op1.Type), op1);
}
public Operand Copy(Operand dest, Operand op1)
{
if (dest.Kind != OperandKind.Register &&
(dest.Kind != OperandKind.LocalVariable || dest.GetLocalNumber() == 0))
{
throw new ArgumentException($"Destination operand must be a Register or a numbered LocalVariable.");
}
return Add(Instruction.Copy, dest, op1);
}
public Operand CountLeadingZeros(Operand op1)
{
return Add(Instruction.CountLeadingZeros, Local(op1.Type), op1);
}
public Operand Divide(Operand op1, Operand op2)
{
return Add(Instruction.Divide, Local(op1.Type), op1, op2);
}
public Operand DivideUI(Operand op1, Operand op2)
{
return Add(Instruction.DivideUI, Local(op1.Type), op1, op2);
}
public Operand ICompare(Operand op1, Operand op2, Comparison comp)
{
return Add(Instruction.Compare, Local(OperandType.I32), op1, op2, Const((int)comp));
}
public Operand ICompareEqual(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.Equal);
}
public Operand ICompareGreater(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.Greater);
}
public Operand ICompareGreaterOrEqual(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.GreaterOrEqual);
}
public Operand ICompareGreaterOrEqualUI(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.GreaterOrEqualUI);
}
public Operand ICompareGreaterUI(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.GreaterUI);
}
public Operand ICompareLess(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.Less);
}
public Operand ICompareLessOrEqual(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.LessOrEqual);
}
public Operand ICompareLessOrEqualUI(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.LessOrEqualUI);
}
public Operand ICompareLessUI(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.LessUI);
}
public Operand ICompareNotEqual(Operand op1, Operand op2)
{
return ICompare(op1, op2, Comparison.NotEqual);
}
public Operand Load(OperandType type, Operand address)
{
return Add(Instruction.Load, Local(type), address);
}
public Operand Load16(Operand address)
{
return Add(Instruction.Load16, Local(OperandType.I32), address);
}
public Operand Load8(Operand address)
{
return Add(Instruction.Load8, Local(OperandType.I32), address);
}
public Operand LoadArgument(OperandType type, int index)
{
return Add(Instruction.LoadArgument, Local(type), Const(index));
}
public void LoadFromContext()
{
_needsNewBlock = true;
Add(Instruction.LoadFromContext);
}
public Operand Multiply(Operand op1, Operand op2)
{
return Add(Instruction.Multiply, Local(op1.Type), op1, op2);
}
public Operand Multiply64HighSI(Operand op1, Operand op2)
{
return Add(Instruction.Multiply64HighSI, Local(OperandType.I64), op1, op2);
}
public Operand Multiply64HighUI(Operand op1, Operand op2)
{
return Add(Instruction.Multiply64HighUI, Local(OperandType.I64), op1, op2);
}
public Operand Negate(Operand op1)
{
return Add(Instruction.Negate, Local(op1.Type), op1);
}
public void Return()
{
Add(Instruction.Return);
_needsNewBlock = true;
}
public void Return(Operand op1)
{
Add(Instruction.Return, null, op1);
_needsNewBlock = true;
}
public Operand RotateRight(Operand op1, Operand op2)
{
return Add(Instruction.RotateRight, Local(op1.Type), op1, op2);
}
public Operand ShiftLeft(Operand op1, Operand op2)
{
return Add(Instruction.ShiftLeft, Local(op1.Type), op1, op2);
}
public Operand ShiftRightSI(Operand op1, Operand op2)
{
return Add(Instruction.ShiftRightSI, Local(op1.Type), op1, op2);
}
public Operand ShiftRightUI(Operand op1, Operand op2)
{
return Add(Instruction.ShiftRightUI, Local(op1.Type), op1, op2);
}
public Operand SignExtend16(OperandType type, Operand op1)
{
return Add(Instruction.SignExtend16, Local(type), op1);
}
public Operand SignExtend32(OperandType type, Operand op1)
{
return Add(Instruction.SignExtend32, Local(type), op1);
}
public Operand SignExtend8(OperandType type, Operand op1)
{
return Add(Instruction.SignExtend8, Local(type), op1);
}
public void Store(Operand address, Operand value)
{
Add(Instruction.Store, null, address, value);
}
public void Store16(Operand address, Operand value)
{
Add(Instruction.Store16, null, address, value);
}
public void Store8(Operand address, Operand value)
{
Add(Instruction.Store8, null, address, value);
}
public void StoreToContext()
{
Add(Instruction.StoreToContext);
_needsNewBlock = true;
}
public Operand Subtract(Operand op1, Operand op2)
{
return Add(Instruction.Subtract, Local(op1.Type), op1, op2);
}
public Operand VectorCreateScalar(Operand value)
{
return Add(Instruction.VectorCreateScalar, Local(OperandType.V128), value);
}
public Operand VectorExtract(OperandType type, Operand vector, int index)
{
return Add(Instruction.VectorExtract, Local(type), vector, Const(index));
}
public Operand VectorExtract16(Operand vector, int index)
{
return Add(Instruction.VectorExtract16, Local(OperandType.I32), vector, Const(index));
}
public Operand VectorExtract8(Operand vector, int index)
{
return Add(Instruction.VectorExtract8, Local(OperandType.I32), vector, Const(index));
}
public Operand VectorInsert(Operand vector, Operand value, int index)
{
return Add(Instruction.VectorInsert, Local(OperandType.V128), vector, value, Const(index));
}
public Operand VectorInsert16(Operand vector, Operand value, int index)
{
return Add(Instruction.VectorInsert16, Local(OperandType.V128), vector, value, Const(index));
}
public Operand VectorInsert8(Operand vector, Operand value, int index)
{
return Add(Instruction.VectorInsert8, Local(OperandType.V128), vector, value, Const(index));
}
public Operand VectorOne()
{
return Add(Instruction.VectorOne, Local(OperandType.V128));
}
public Operand VectorZero()
{
return Add(Instruction.VectorZero, Local(OperandType.V128));
}
public Operand VectorZeroUpper64(Operand vector)
{
return Add(Instruction.VectorZeroUpper64, Local(OperandType.V128), vector);
}
public Operand VectorZeroUpper96(Operand vector)
{
return Add(Instruction.VectorZeroUpper96, Local(OperandType.V128), vector);
}
public Operand ZeroExtend16(OperandType type, Operand op1)
{
return Add(Instruction.ZeroExtend16, Local(type), op1);
}
public Operand ZeroExtend32(OperandType type, Operand op1)
{
return Add(Instruction.ZeroExtend32, Local(type), op1);
}
public Operand ZeroExtend8(OperandType type, Operand op1)
{
return Add(Instruction.ZeroExtend8, Local(type), op1);
}
private void NewNextBlockIfNeeded()
{
if (_needsNewBlock)
{
NewNextBlock();
}
}
private Operand Add(Instruction inst, Operand dest = null)
{
NewNextBlockIfNeeded();
Operation operation = OperationHelper.Operation(inst, dest);
_irBlock.Operations.AddLast(operation);
return dest;
}
private Operand Add(Instruction inst, Operand dest, Operand[] sources)
{
NewNextBlockIfNeeded();
Operation operation = OperationHelper.Operation(inst, dest, sources);
_irBlock.Operations.AddLast(operation);
return dest;
}
private Operand Add(Instruction inst, Operand dest, Operand source0)
{
NewNextBlockIfNeeded();
Operation operation = OperationHelper.Operation(inst, dest, source0);
_irBlock.Operations.AddLast(operation);
return dest;
}
private Operand Add(Instruction inst, Operand dest, Operand source0, Operand source1)
{
NewNextBlockIfNeeded();
Operation operation = OperationHelper.Operation(inst, dest, source0, source1);
_irBlock.Operations.AddLast(operation);
return dest;
}
private Operand Add(Instruction inst, Operand dest, Operand source0, Operand source1, Operand source2)
{
NewNextBlockIfNeeded();
Operation operation = OperationHelper.Operation(inst, dest, source0, source1, source2);
_irBlock.Operations.AddLast(operation);
return dest;
}
public Operand AddIntrinsic(Intrinsic intrin, params Operand[] args)
{
return Add(intrin, Local(OperandType.V128), args);
}
public Operand AddIntrinsicInt(Intrinsic intrin, params Operand[] args)
{
return Add(intrin, Local(OperandType.I32), args);
}
public Operand AddIntrinsicLong(Intrinsic intrin, params Operand[] args)
{
return Add(intrin, Local(OperandType.I64), args);
}
public void AddIntrinsicNoRet(Intrinsic intrin, params Operand[] args)
{
Add(intrin, null, args);
}
private Operand Add(Intrinsic intrin, Operand dest, params Operand[] sources)
{
NewNextBlockIfNeeded();
IntrinsicOperation operation = new IntrinsicOperation(intrin, dest, sources);
_irBlock.Operations.AddLast(operation);
return dest;
}
private void BranchToLabel(Operand label, bool uncond, BasicBlockFrequency nextFreq)
{
if (!_irLabels.TryGetValue(label, out BasicBlock branchBlock))
{
branchBlock = new BasicBlock();
_irLabels.Add(label, branchBlock);
}
if (uncond)
{
_irBlock.AddSuccessor(branchBlock);
}
else
{
// Defer registration of successor to _irBlock so that the order of successors is correct.
_ifBlock = branchBlock;
}
_needsNewBlock = true;
_nextBlockFreq = nextFreq;
}
public void MarkLabel(Operand label, BasicBlockFrequency nextFreq = default)
{
_nextBlockFreq = nextFreq;
if (_irLabels.TryGetValue(label, out BasicBlock nextBlock))
{
nextBlock.Index = _irBlocks.Count;
_irBlocks.AddLast(nextBlock);
NextBlock(nextBlock);
}
else
{
NewNextBlock();
_irLabels.Add(label, _irBlock);
}
}
private void NewNextBlock()
{
BasicBlock block = new BasicBlock(_irBlocks.Count);
_irBlocks.AddLast(block);
NextBlock(block);
}
private void NextBlock(BasicBlock nextBlock)
{
if (_irBlock?.SuccessorCount == 0 && !EndsWithUnconditional(_irBlock))
{
_irBlock.AddSuccessor(nextBlock);
if (_ifBlock != null)
{
_irBlock.AddSuccessor(_ifBlock);
_ifBlock = null;
}
}
_irBlock = nextBlock;
_irBlock.Frequency = _nextBlockFreq;
_needsNewBlock = false;
_nextBlockFreq = BasicBlockFrequency.Default;
}
private static bool EndsWithUnconditional(BasicBlock block)
{
return block.Operations.Last is Operation lastOp &&
(lastOp.Instruction == Instruction.Return ||
lastOp.Instruction == Instruction.Tailcall);
}
public ControlFlowGraph GetControlFlowGraph()
{
return new ControlFlowGraph(_irBlocks.First, _irBlocks, _localsCount);
}
}
}