using ARMeilleure.CodeGen.Linking; using ARMeilleure.CodeGen.Optimizations; using ARMeilleure.CodeGen.RegisterAllocators; using ARMeilleure.CodeGen.Unwinding; using ARMeilleure.Common; using ARMeilleure.Diagnostics; using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using System; using System.Collections.Generic; using System.Diagnostics; using System.Numerics; using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.CodeGen.X86 { static class CodeGenerator { private const int RegistersCount = 16; private const int PageSize = 0x1000; private const int StackGuardSize = 0x2000; private static readonly Action[] _instTable; static CodeGenerator() { _instTable = new Action[EnumUtils.GetCount(typeof(Instruction))]; Add(Instruction.Add, GenerateAdd); Add(Instruction.BitwiseAnd, GenerateBitwiseAnd); Add(Instruction.BitwiseExclusiveOr, GenerateBitwiseExclusiveOr); Add(Instruction.BitwiseNot, GenerateBitwiseNot); Add(Instruction.BitwiseOr, GenerateBitwiseOr); Add(Instruction.BranchIf, GenerateBranchIf); Add(Instruction.ByteSwap, GenerateByteSwap); Add(Instruction.Call, GenerateCall); Add(Instruction.Clobber, GenerateClobber); Add(Instruction.Compare, GenerateCompare); Add(Instruction.CompareAndSwap, GenerateCompareAndSwap); Add(Instruction.CompareAndSwap16, GenerateCompareAndSwap16); Add(Instruction.CompareAndSwap8, GenerateCompareAndSwap8); Add(Instruction.ConditionalSelect, GenerateConditionalSelect); Add(Instruction.ConvertI64ToI32, GenerateConvertI64ToI32); Add(Instruction.ConvertToFP, GenerateConvertToFP); Add(Instruction.Copy, GenerateCopy); Add(Instruction.CountLeadingZeros, GenerateCountLeadingZeros); Add(Instruction.Divide, GenerateDivide); Add(Instruction.DivideUI, GenerateDivideUI); Add(Instruction.Fill, GenerateFill); Add(Instruction.Load, GenerateLoad); Add(Instruction.Load16, GenerateLoad16); Add(Instruction.Load8, GenerateLoad8); Add(Instruction.MemoryBarrier, GenerateMemoryBarrier); Add(Instruction.Multiply, GenerateMultiply); Add(Instruction.Multiply64HighSI, GenerateMultiply64HighSI); Add(Instruction.Multiply64HighUI, GenerateMultiply64HighUI); Add(Instruction.Negate, GenerateNegate); Add(Instruction.Return, GenerateReturn); Add(Instruction.RotateRight, GenerateRotateRight); Add(Instruction.ShiftLeft, GenerateShiftLeft); Add(Instruction.ShiftRightSI, GenerateShiftRightSI); Add(Instruction.ShiftRightUI, GenerateShiftRightUI); Add(Instruction.SignExtend16, GenerateSignExtend16); Add(Instruction.SignExtend32, GenerateSignExtend32); Add(Instruction.SignExtend8, GenerateSignExtend8); Add(Instruction.Spill, GenerateSpill); Add(Instruction.SpillArg, GenerateSpillArg); Add(Instruction.StackAlloc, GenerateStackAlloc); Add(Instruction.Store, GenerateStore); Add(Instruction.Store16, GenerateStore16); Add(Instruction.Store8, GenerateStore8); Add(Instruction.Subtract, GenerateSubtract); Add(Instruction.Tailcall, GenerateTailcall); Add(Instruction.VectorCreateScalar, GenerateVectorCreateScalar); Add(Instruction.VectorExtract, GenerateVectorExtract); Add(Instruction.VectorExtract16, GenerateVectorExtract16); Add(Instruction.VectorExtract8, GenerateVectorExtract8); Add(Instruction.VectorInsert, GenerateVectorInsert); Add(Instruction.VectorInsert16, GenerateVectorInsert16); Add(Instruction.VectorInsert8, GenerateVectorInsert8); Add(Instruction.VectorOne, GenerateVectorOne); Add(Instruction.VectorZero, GenerateVectorZero); Add(Instruction.VectorZeroUpper64, GenerateVectorZeroUpper64); Add(Instruction.VectorZeroUpper96, GenerateVectorZeroUpper96); Add(Instruction.ZeroExtend16, GenerateZeroExtend16); Add(Instruction.ZeroExtend32, GenerateZeroExtend32); Add(Instruction.ZeroExtend8, GenerateZeroExtend8); static void Add(Instruction inst, Action func) { _instTable[(int)inst] = func; } } public static CompiledFunction Generate(CompilerContext cctx) { ControlFlowGraph cfg = cctx.Cfg; Logger.StartPass(PassName.Optimization); if (cctx.Options.HasFlag(CompilerOptions.Optimize)) { if (cctx.Options.HasFlag(CompilerOptions.SsaForm)) { Optimizer.RunPass(cfg); } BlockPlacement.RunPass(cfg); } X86Optimizer.RunPass(cfg); Logger.EndPass(PassName.Optimization, cfg); Logger.StartPass(PassName.PreAllocation); StackAllocator stackAlloc = new(); PreAllocator.RunPass(cctx, stackAlloc, out int maxCallArgs); Logger.EndPass(PassName.PreAllocation, cfg); Logger.StartPass(PassName.RegisterAllocation); if (cctx.Options.HasFlag(CompilerOptions.SsaForm)) { Ssa.Deconstruct(cfg); } IRegisterAllocator regAlloc; if (cctx.Options.HasFlag(CompilerOptions.Lsra)) { regAlloc = new LinearScanAllocator(); } else { regAlloc = new HybridAllocator(); } RegisterMasks regMasks = new( CallingConvention.GetIntAvailableRegisters(), CallingConvention.GetVecAvailableRegisters(), CallingConvention.GetIntCallerSavedRegisters(), CallingConvention.GetVecCallerSavedRegisters(), CallingConvention.GetIntCalleeSavedRegisters(), CallingConvention.GetVecCalleeSavedRegisters(), RegistersCount); AllocationResult allocResult = regAlloc.RunPass(cfg, stackAlloc, regMasks); Logger.EndPass(PassName.RegisterAllocation, cfg); Logger.StartPass(PassName.CodeGeneration); bool relocatable = (cctx.Options & CompilerOptions.Relocatable) != 0; CodeGenContext context = new(allocResult, maxCallArgs, cfg.Blocks.Count, relocatable); UnwindInfo unwindInfo = WritePrologue(context); for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { context.EnterBlock(block); for (Operation node = block.Operations.First; node != default; node = node.ListNext) { GenerateOperation(context, node); } if (block.SuccessorsCount == 0) { // The only blocks which can have 0 successors are exit blocks. Operation last = block.Operations.Last; Debug.Assert(last.Instruction == Instruction.Tailcall || last.Instruction == Instruction.Return); } else { BasicBlock succ = block.GetSuccessor(0); if (succ != block.ListNext) { context.JumpTo(succ); } } } (byte[] code, RelocInfo relocInfo) = context.Assembler.GetCode(); Logger.EndPass(PassName.CodeGeneration); return new CompiledFunction(code, unwindInfo, relocInfo); } private static void GenerateOperation(CodeGenContext context, Operation operation) { if (operation.Instruction == Instruction.Extended) { IntrinsicInfo info = IntrinsicTable.GetInfo(operation.Intrinsic); switch (info.Type) { case IntrinsicType.Comis_: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); switch (operation.Intrinsic) { case Intrinsic.X86Comisdeq: context.Assembler.Comisd(src1, src2); context.Assembler.Setcc(dest, X86Condition.Equal); break; case Intrinsic.X86Comisdge: context.Assembler.Comisd(src1, src2); context.Assembler.Setcc(dest, X86Condition.AboveOrEqual); break; case Intrinsic.X86Comisdlt: context.Assembler.Comisd(src1, src2); context.Assembler.Setcc(dest, X86Condition.Below); break; case Intrinsic.X86Comisseq: context.Assembler.Comiss(src1, src2); context.Assembler.Setcc(dest, X86Condition.Equal); break; case Intrinsic.X86Comissge: context.Assembler.Comiss(src1, src2); context.Assembler.Setcc(dest, X86Condition.AboveOrEqual); break; case Intrinsic.X86Comisslt: context.Assembler.Comiss(src1, src2); context.Assembler.Setcc(dest, X86Condition.Below); break; } context.Assembler.Movzx8(dest, dest, OperandType.I32); break; } case IntrinsicType.Mxcsr: { Operand offset = operation.GetSource(0); Debug.Assert(offset.Kind == OperandKind.Constant); Debug.Assert(offset.Type == OperandType.I32); int offs = offset.AsInt32() + context.CallArgsRegionSize; Operand rsp = Register(X86Register.Rsp); Operand memOp = MemoryOp(OperandType.I32, rsp, default, Multiplier.x1, offs); Debug.Assert(HardwareCapabilities.SupportsSse || HardwareCapabilities.SupportsVexEncoding); if (operation.Intrinsic == Intrinsic.X86Ldmxcsr) { Operand bits = operation.GetSource(1); Debug.Assert(bits.Type == OperandType.I32); context.Assembler.Mov(memOp, bits, OperandType.I32); context.Assembler.Ldmxcsr(memOp); } else if (operation.Intrinsic == Intrinsic.X86Stmxcsr) { Operand dest = operation.Destination; Debug.Assert(dest.Type == OperandType.I32); context.Assembler.Stmxcsr(memOp); context.Assembler.Mov(dest, memOp, OperandType.I32); } break; } case IntrinsicType.PopCount: { Operand dest = operation.Destination; Operand source = operation.GetSource(0); EnsureSameType(dest, source); Debug.Assert(dest.Type.IsInteger()); context.Assembler.Popcnt(dest, source, dest.Type); break; } case IntrinsicType.Unary: { Operand dest = operation.Destination; Operand source = operation.GetSource(0); EnsureSameType(dest, source); Debug.Assert(!dest.Type.IsInteger()); context.Assembler.WriteInstruction(info.Inst, dest, source); break; } case IntrinsicType.UnaryToGpr: { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && !source.Type.IsInteger()); if (operation.Intrinsic == Intrinsic.X86Cvtsi2si) { if (dest.Type == OperandType.I32) { context.Assembler.Movd(dest, source); // int _mm_cvtsi128_si32(__m128i a) } else /* if (dest.Type == OperandType.I64) */ { context.Assembler.Movq(dest, source); // __int64 _mm_cvtsi128_si64(__m128i a) } } else { context.Assembler.WriteInstruction(info.Inst, dest, source, dest.Type); } break; } case IntrinsicType.Binary: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); EnsureSameType(dest, src1); if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(!dest.Type.IsInteger()); Debug.Assert(!src2.Type.IsInteger() || src2.Kind == OperandKind.Constant); context.Assembler.WriteInstruction(info.Inst, dest, src1, src2); break; } case IntrinsicType.BinaryGpr: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); EnsureSameType(dest, src1); if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(!dest.Type.IsInteger() && src2.Type.IsInteger()); context.Assembler.WriteInstruction(info.Inst, dest, src1, src2, src2.Type); break; } case IntrinsicType.Crc32: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); EnsureSameReg(dest, src1); Debug.Assert(dest.Type.IsInteger() && src1.Type.IsInteger() && src2.Type.IsInteger()); context.Assembler.WriteInstruction(info.Inst, dest, src2, dest.Type); break; } case IntrinsicType.BinaryImm: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); EnsureSameType(dest, src1); if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(!dest.Type.IsInteger() && src2.Kind == OperandKind.Constant); context.Assembler.WriteInstruction(info.Inst, dest, src1, src2.AsByte()); break; } case IntrinsicType.Ternary: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); EnsureSameType(dest, src1, src2, src3); Debug.Assert(!dest.Type.IsInteger()); if (info.Inst == X86Instruction.Blendvpd && HardwareCapabilities.SupportsVexEncoding) { context.Assembler.WriteInstruction(X86Instruction.Vblendvpd, dest, src1, src2, src3); } else if (info.Inst == X86Instruction.Blendvps && HardwareCapabilities.SupportsVexEncoding) { context.Assembler.WriteInstruction(X86Instruction.Vblendvps, dest, src1, src2, src3); } else if (info.Inst == X86Instruction.Pblendvb && HardwareCapabilities.SupportsVexEncoding) { context.Assembler.WriteInstruction(X86Instruction.Vpblendvb, dest, src1, src2, src3); } else { EnsureSameReg(dest, src1); Debug.Assert(src3.GetRegister().Index == 0); context.Assembler.WriteInstruction(info.Inst, dest, src1, src2); } break; } case IntrinsicType.TernaryImm: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); EnsureSameType(dest, src1, src2); if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(!dest.Type.IsInteger() && src3.Kind == OperandKind.Constant); context.Assembler.WriteInstruction(info.Inst, dest, src1, src2, src3.AsByte()); break; } case IntrinsicType.Fma: { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); Debug.Assert(HardwareCapabilities.SupportsVexEncoding); Debug.Assert(dest.Kind == OperandKind.Register && src1.Kind == OperandKind.Register && src2.Kind == OperandKind.Register); Debug.Assert(src3.Kind == OperandKind.Register || src3.Kind == OperandKind.Memory); EnsureSameType(dest, src1, src2, src3); Debug.Assert(dest.Type == OperandType.V128); Debug.Assert(dest.Value == src1.Value); context.Assembler.WriteInstruction(info.Inst, dest, src2, src3); break; } } } else { Action func = _instTable[(int)operation.Instruction]; if (func != null) { func(context, operation); } else { throw new ArgumentException($"Invalid instruction \"{operation.Instruction}\"."); } } } private static void GenerateAdd(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); if (dest.Type.IsInteger()) { // If Destination and Source 1 Operands are the same, perform a standard add as there are no benefits to using LEA. if (dest.Kind == src1.Kind && dest.Value == src1.Value) { ValidateBinOp(dest, src1, src2); context.Assembler.Add(dest, src2, dest.Type); } else { EnsureSameType(dest, src1, src2); int offset; Operand index; if (src2.Kind == OperandKind.Constant) { offset = src2.AsInt32(); index = default; } else { offset = 0; index = src2; } Operand memOp = MemoryOp(dest.Type, src1, index, Multiplier.x1, offset); context.Assembler.Lea(dest, memOp, dest.Type); } } else { ValidateBinOp(dest, src1, src2); if (dest.Type == OperandType.FP32) { context.Assembler.Addss(dest, src1, src2); } else /* if (dest.Type == OperandType.FP64) */ { context.Assembler.Addsd(dest, src1, src2); } } } private static void GenerateBitwiseAnd(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateBinOp(dest, src1, src2); Debug.Assert(dest.Type.IsInteger()); // Note: GenerateCompareCommon makes the assumption that BitwiseAnd will emit only a single `and` // instruction. context.Assembler.And(dest, src2, dest.Type); } private static void GenerateBitwiseExclusiveOr(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateBinOp(dest, src1, src2); if (dest.Type.IsInteger()) { context.Assembler.Xor(dest, src2, dest.Type); } else { context.Assembler.Xorps(dest, src1, src2); } } private static void GenerateBitwiseNot(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); ValidateUnOp(dest, source); Debug.Assert(dest.Type.IsInteger()); context.Assembler.Not(dest); } private static void GenerateBitwiseOr(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateBinOp(dest, src1, src2); Debug.Assert(dest.Type.IsInteger()); context.Assembler.Or(dest, src2, dest.Type); } private static void GenerateBranchIf(CodeGenContext context, Operation operation) { Operand comp = operation.GetSource(2); Debug.Assert(comp.Kind == OperandKind.Constant); var cond = ((Comparison)comp.AsInt32()).ToX86Condition(); GenerateCompareCommon(context, operation); context.JumpTo(cond, context.CurrBlock.GetSuccessor(1)); } private static void GenerateByteSwap(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); ValidateUnOp(dest, source); Debug.Assert(dest.Type.IsInteger()); context.Assembler.Bswap(dest); } private static void GenerateCall(CodeGenContext context, Operation operation) { context.Assembler.Call(operation.GetSource(0)); } private static void GenerateClobber(CodeGenContext context, Operation operation) { // This is only used to indicate that a register is clobbered to the // register allocator, we don't need to produce any code. } private static void GenerateCompare(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand comp = operation.GetSource(2); Debug.Assert(dest.Type == OperandType.I32); Debug.Assert(comp.Kind == OperandKind.Constant); var cond = ((Comparison)comp.AsInt32()).ToX86Condition(); GenerateCompareCommon(context, operation); context.Assembler.Setcc(dest, cond); context.Assembler.Movzx8(dest, dest, OperandType.I32); } private static void GenerateCompareCommon(CodeGenContext context, Operation operation) { Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); EnsureSameType(src1, src2); Debug.Assert(src1.Type.IsInteger()); if (src2.Kind == OperandKind.Constant && src2.Value == 0) { if (MatchOperation(operation.ListPrevious, Instruction.BitwiseAnd, src1.Type, src1.GetRegister())) { // Since the `test` and `and` instruction set the status flags in the same way, we can omit the // `test r,r` instruction when it is immediately preceded by an `and r,*` instruction. // // For example: // // and eax, 0x3 // test eax, eax // jz .L0 // // => // // and eax, 0x3 // jz .L0 } else { context.Assembler.Test(src1, src1, src1.Type); } } else { context.Assembler.Cmp(src1, src2, src1.Type); } } private static void GenerateCompareAndSwap(CodeGenContext context, Operation operation) { Operand src1 = operation.GetSource(0); if (operation.SourcesCount == 5) // CompareAndSwap128 has 5 sources, compared to CompareAndSwap64/32's 3. { Operand memOp = MemoryOp(OperandType.I64, src1); context.Assembler.Cmpxchg16b(memOp); } else { Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); EnsureSameType(src2, src3); Operand memOp = MemoryOp(src3.Type, src1); context.Assembler.Cmpxchg(memOp, src3); } } private static void GenerateCompareAndSwap16(CodeGenContext context, Operation operation) { Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); EnsureSameType(src2, src3); Operand memOp = MemoryOp(src3.Type, src1); context.Assembler.Cmpxchg16(memOp, src3); } private static void GenerateCompareAndSwap8(CodeGenContext context, Operation operation) { Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); EnsureSameType(src2, src3); Operand memOp = MemoryOp(src3.Type, src1); context.Assembler.Cmpxchg8(memOp, src3); } private static void GenerateConditionalSelect(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); Operand src3 = operation.GetSource(2); EnsureSameReg (dest, src3); EnsureSameType(dest, src2, src3); Debug.Assert(dest.Type.IsInteger()); Debug.Assert(src1.Type == OperandType.I32); context.Assembler.Test (src1, src1, src1.Type); context.Assembler.Cmovcc(dest, src2, dest.Type, X86Condition.NotEqual); } private static void GenerateConvertI64ToI32(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type == OperandType.I32 && source.Type == OperandType.I64); context.Assembler.Mov(dest, source, OperandType.I32); } private static void GenerateConvertToFP(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type == OperandType.FP32 || dest.Type == OperandType.FP64); if (dest.Type == OperandType.FP32) { Debug.Assert(source.Type.IsInteger() || source.Type == OperandType.FP64); if (source.Type.IsInteger()) { context.Assembler.Xorps (dest, dest, dest); context.Assembler.Cvtsi2ss(dest, dest, source, source.Type); } else /* if (source.Type == OperandType.FP64) */ { context.Assembler.Cvtsd2ss(dest, dest, source); GenerateZeroUpper96(context, dest, dest); } } else /* if (dest.Type == OperandType.FP64) */ { Debug.Assert(source.Type.IsInteger() || source.Type == OperandType.FP32); if (source.Type.IsInteger()) { context.Assembler.Xorps (dest, dest, dest); context.Assembler.Cvtsi2sd(dest, dest, source, source.Type); } else /* if (source.Type == OperandType.FP32) */ { context.Assembler.Cvtss2sd(dest, dest, source); GenerateZeroUpper64(context, dest, dest); } } } private static void GenerateCopy(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); EnsureSameType(dest, source); Debug.Assert(dest.Type.IsInteger() || source.Kind != OperandKind.Constant); // Moves to the same register are useless. if (dest.Kind == source.Kind && dest.Value == source.Value) { return; } if (dest.Kind == OperandKind.Register && source.Kind == OperandKind.Constant && source.Value == 0) { // Assemble "mov reg, 0" as "xor reg, reg" as the later is more efficient. context.Assembler.Xor(dest, dest, OperandType.I32); } else if (dest.Type.IsInteger()) { context.Assembler.Mov(dest, source, dest.Type); } else { context.Assembler.Movdqu(dest, source); } } private static void GenerateCountLeadingZeros(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); EnsureSameType(dest, source); Debug.Assert(dest.Type.IsInteger()); context.Assembler.Bsr(dest, source, dest.Type); int operandSize = dest.Type == OperandType.I32 ? 32 : 64; int operandMask = operandSize - 1; // When the input operand is 0, the result is undefined, however the // ZF flag is set. We are supposed to return the operand size on that // case. So, add an additional jump to handle that case, by moving the // operand size constant to the destination register. Operand neLabel = Label(); context.Assembler.Jcc(X86Condition.NotEqual, neLabel); context.Assembler.Mov(dest, Const(operandSize | operandMask), OperandType.I32); context.Assembler.MarkLabel(neLabel); // BSR returns the zero based index of the last bit set on the operand, // starting from the least significant bit. However we are supposed to // return the number of 0 bits on the high end. So, we invert the result // of the BSR using XOR to get the correct value. context.Assembler.Xor(dest, Const(operandMask), OperandType.I32); } private static void GenerateDivide(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand dividend = operation.GetSource(0); Operand divisor = operation.GetSource(1); if (!dest.Type.IsInteger()) { ValidateBinOp(dest, dividend, divisor); } if (dest.Type.IsInteger()) { divisor = operation.GetSource(2); EnsureSameType(dest, divisor); if (divisor.Type == OperandType.I32) { context.Assembler.Cdq(); } else { context.Assembler.Cqo(); } context.Assembler.Idiv(divisor); } else if (dest.Type == OperandType.FP32) { context.Assembler.Divss(dest, dividend, divisor); } else /* if (dest.Type == OperandType.FP64) */ { context.Assembler.Divsd(dest, dividend, divisor); } } private static void GenerateDivideUI(CodeGenContext context, Operation operation) { Operand divisor = operation.GetSource(2); Operand rdx = Register(X86Register.Rdx); Debug.Assert(divisor.Type.IsInteger()); context.Assembler.Xor(rdx, rdx, OperandType.I32); context.Assembler.Div(divisor); } private static void GenerateFill(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand offset = operation.GetSource(0); Debug.Assert(offset.Kind == OperandKind.Constant); int offs = offset.AsInt32() + context.CallArgsRegionSize; Operand rsp = Register(X86Register.Rsp); Operand memOp = MemoryOp(dest.Type, rsp, default, Multiplier.x1, offs); GenerateLoad(context, memOp, dest); } private static void GenerateLoad(CodeGenContext context, Operation operation) { Operand value = operation.Destination; Operand address = Memory(operation.GetSource(0), value.Type); GenerateLoad(context, address, value); } private static void GenerateLoad16(CodeGenContext context, Operation operation) { Operand value = operation.Destination; Operand address = Memory(operation.GetSource(0), value.Type); Debug.Assert(value.Type.IsInteger()); context.Assembler.Movzx16(value, address, value.Type); } private static void GenerateLoad8(CodeGenContext context, Operation operation) { Operand value = operation.Destination; Operand address = Memory(operation.GetSource(0), value.Type); Debug.Assert(value.Type.IsInteger()); context.Assembler.Movzx8(value, address, value.Type); } private static void GenerateMemoryBarrier(CodeGenContext context, Operation operation) { context.Assembler.LockOr(MemoryOp(OperandType.I64, Register(X86Register.Rsp)), Const(0), OperandType.I32); } private static void GenerateMultiply(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); if (src2.Kind != OperandKind.Constant) { EnsureSameReg(dest, src1); } EnsureSameType(dest, src1, src2); if (dest.Type.IsInteger()) { if (src2.Kind == OperandKind.Constant) { context.Assembler.Imul(dest, src1, src2, dest.Type); } else { context.Assembler.Imul(dest, src2, dest.Type); } } else if (dest.Type == OperandType.FP32) { context.Assembler.Mulss(dest, src1, src2); } else /* if (dest.Type == OperandType.FP64) */ { context.Assembler.Mulsd(dest, src1, src2); } } private static void GenerateMultiply64HighSI(CodeGenContext context, Operation operation) { Operand source = operation.GetSource(1); Debug.Assert(source.Type == OperandType.I64); context.Assembler.Imul(source); } private static void GenerateMultiply64HighUI(CodeGenContext context, Operation operation) { Operand source = operation.GetSource(1); Debug.Assert(source.Type == OperandType.I64); context.Assembler.Mul(source); } private static void GenerateNegate(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); ValidateUnOp(dest, source); Debug.Assert(dest.Type.IsInteger()); context.Assembler.Neg(dest); } private static void GenerateReturn(CodeGenContext context, Operation operation) { WriteEpilogue(context); context.Assembler.Return(); } private static void GenerateRotateRight(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateShift(dest, src1, src2); context.Assembler.Ror(dest, src2, dest.Type); } private static void GenerateShiftLeft(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateShift(dest, src1, src2); context.Assembler.Shl(dest, src2, dest.Type); } private static void GenerateShiftRightSI(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateShift(dest, src1, src2); context.Assembler.Sar(dest, src2, dest.Type); } private static void GenerateShiftRightUI(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateShift(dest, src1, src2); context.Assembler.Shr(dest, src2, dest.Type); } private static void GenerateSignExtend16(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); context.Assembler.Movsx16(dest, source, dest.Type); } private static void GenerateSignExtend32(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); context.Assembler.Movsx32(dest, source, dest.Type); } private static void GenerateSignExtend8(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); context.Assembler.Movsx8(dest, source, dest.Type); } private static void GenerateSpill(CodeGenContext context, Operation operation) { GenerateSpill(context, operation, context.CallArgsRegionSize); } private static void GenerateSpillArg(CodeGenContext context, Operation operation) { GenerateSpill(context, operation, 0); } private static void GenerateSpill(CodeGenContext context, Operation operation, int baseOffset) { Operand offset = operation.GetSource(0); Operand source = operation.GetSource(1); Debug.Assert(offset.Kind == OperandKind.Constant); int offs = offset.AsInt32() + baseOffset; Operand rsp = Register(X86Register.Rsp); Operand memOp = MemoryOp(source.Type, rsp, default, Multiplier.x1, offs); GenerateStore(context, memOp, source); } private static void GenerateStackAlloc(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand offset = operation.GetSource(0); Debug.Assert(offset.Kind == OperandKind.Constant); int offs = offset.AsInt32() + context.CallArgsRegionSize; Operand rsp = Register(X86Register.Rsp); Operand memOp = MemoryOp(OperandType.I64, rsp, default, Multiplier.x1, offs); context.Assembler.Lea(dest, memOp, OperandType.I64); } private static void GenerateStore(CodeGenContext context, Operation operation) { Operand value = operation.GetSource(1); Operand address = Memory(operation.GetSource(0), value.Type); GenerateStore(context, address, value); } private static void GenerateStore16(CodeGenContext context, Operation operation) { Operand value = operation.GetSource(1); Operand address = Memory(operation.GetSource(0), value.Type); Debug.Assert(value.Type.IsInteger()); context.Assembler.Mov16(address, value); } private static void GenerateStore8(CodeGenContext context, Operation operation) { Operand value = operation.GetSource(1); Operand address = Memory(operation.GetSource(0), value.Type); Debug.Assert(value.Type.IsInteger()); context.Assembler.Mov8(address, value); } private static void GenerateSubtract(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); ValidateBinOp(dest, src1, src2); if (dest.Type.IsInteger()) { context.Assembler.Sub(dest, src2, dest.Type); } else if (dest.Type == OperandType.FP32) { context.Assembler.Subss(dest, src1, src2); } else /* if (dest.Type == OperandType.FP64) */ { context.Assembler.Subsd(dest, src1, src2); } } private static void GenerateTailcall(CodeGenContext context, Operation operation) { WriteEpilogue(context); context.Assembler.Jmp(operation.GetSource(0)); } private static void GenerateVectorCreateScalar(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(!dest.Type.IsInteger() && source.Type.IsInteger()); if (source.Type == OperandType.I32) { context.Assembler.Movd(dest, source); // (__m128i _mm_cvtsi32_si128(int a)) } else /* if (source.Type == OperandType.I64) */ { context.Assembler.Movq(dest, source); // (__m128i _mm_cvtsi64_si128(__int64 a)) } } private static void GenerateVectorExtract(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; //Value Operand src1 = operation.GetSource(0); //Vector Operand src2 = operation.GetSource(1); //Index Debug.Assert(src1.Type == OperandType.V128); Debug.Assert(src2.Kind == OperandKind.Constant); byte index = src2.AsByte(); Debug.Assert(index < OperandType.V128.GetSizeInBytes() / dest.Type.GetSizeInBytes()); if (dest.Type == OperandType.I32) { if (index == 0) { context.Assembler.Movd(dest, src1); } else if (HardwareCapabilities.SupportsSse41) { context.Assembler.Pextrd(dest, src1, index); } else { int mask0 = 0b11_10_01_00; int mask1 = 0b11_10_01_00; mask0 = BitUtils.RotateRight(mask0, index * 2, 8); mask1 = BitUtils.RotateRight(mask1, 8 - index * 2, 8); context.Assembler.Pshufd(src1, src1, (byte)mask0); context.Assembler.Movd (dest, src1); context.Assembler.Pshufd(src1, src1, (byte)mask1); } } else if (dest.Type == OperandType.I64) { if (index == 0) { context.Assembler.Movq(dest, src1); } else if (HardwareCapabilities.SupportsSse41) { context.Assembler.Pextrq(dest, src1, index); } else { const byte mask = 0b01_00_11_10; context.Assembler.Pshufd(src1, src1, mask); context.Assembler.Movq (dest, src1); context.Assembler.Pshufd(src1, src1, mask); } } else { // Floating-point types. if ((index >= 2 && dest.Type == OperandType.FP32) || (index == 1 && dest.Type == OperandType.FP64)) { context.Assembler.Movhlps(dest, dest, src1); context.Assembler.Movq (dest, dest); } else { context.Assembler.Movq(dest, src1); } if (dest.Type == OperandType.FP32) { context.Assembler.Pshufd(dest, dest, (byte)(0xfc | (index & 1))); } } } private static void GenerateVectorExtract16(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; //Value Operand src1 = operation.GetSource(0); //Vector Operand src2 = operation.GetSource(1); //Index Debug.Assert(src1.Type == OperandType.V128); Debug.Assert(src2.Kind == OperandKind.Constant); byte index = src2.AsByte(); Debug.Assert(index < 8); context.Assembler.Pextrw(dest, src1, index); } private static void GenerateVectorExtract8(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; //Value Operand src1 = operation.GetSource(0); //Vector Operand src2 = operation.GetSource(1); //Index Debug.Assert(src1.Type == OperandType.V128); Debug.Assert(src2.Kind == OperandKind.Constant); byte index = src2.AsByte(); Debug.Assert(index < 16); if (HardwareCapabilities.SupportsSse41) { context.Assembler.Pextrb(dest, src1, index); } else { context.Assembler.Pextrw(dest, src1, (byte)(index >> 1)); if ((index & 1) != 0) { context.Assembler.Shr(dest, Const(8), OperandType.I32); } else { context.Assembler.Movzx8(dest, dest, OperandType.I32); } } } private static void GenerateVectorInsert(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); //Vector Operand src2 = operation.GetSource(1); //Value Operand src3 = operation.GetSource(2); //Index if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(src1.Type == OperandType.V128); Debug.Assert(src3.Kind == OperandKind.Constant); byte index = src3.AsByte(); void InsertIntSse2(int words) { if (dest.GetRegister() != src1.GetRegister()) { context.Assembler.Movdqu(dest, src1); } for (int word = 0; word < words; word++) { // Insert lower 16-bits. context.Assembler.Pinsrw(dest, dest, src2, (byte)(index * words + word)); // Move next word down. context.Assembler.Ror(src2, Const(16), src2.Type); } } if (src2.Type == OperandType.I32) { Debug.Assert(index < 4); if (HardwareCapabilities.SupportsSse41) { context.Assembler.Pinsrd(dest, src1, src2, index); } else { InsertIntSse2(2); } } else if (src2.Type == OperandType.I64) { Debug.Assert(index < 2); if (HardwareCapabilities.SupportsSse41) { context.Assembler.Pinsrq(dest, src1, src2, index); } else { InsertIntSse2(4); } } else if (src2.Type == OperandType.FP32) { Debug.Assert(index < 4); if (index != 0) { if (HardwareCapabilities.SupportsSse41) { context.Assembler.Insertps(dest, src1, src2, (byte)(index << 4)); } else { if (src1.GetRegister() == src2.GetRegister()) { int mask = 0b11_10_01_00; mask &= ~(0b11 << index * 2); context.Assembler.Pshufd(dest, src1, (byte)mask); } else { int mask0 = 0b11_10_01_00; int mask1 = 0b11_10_01_00; mask0 = BitUtils.RotateRight(mask0, index * 2, 8); mask1 = BitUtils.RotateRight(mask1, 8 - index * 2, 8); context.Assembler.Pshufd(src1, src1, (byte)mask0); // Lane to be inserted in position 0. context.Assembler.Movss (dest, src1, src2); // dest[127:0] = src1[127:32] | src2[31:0] context.Assembler.Pshufd(dest, dest, (byte)mask1); // Inserted lane in original position. if (dest.GetRegister() != src1.GetRegister()) { context.Assembler.Pshufd(src1, src1, (byte)mask1); // Restore src1. } } } } else { context.Assembler.Movss(dest, src1, src2); } } else /* if (src2.Type == OperandType.FP64) */ { Debug.Assert(index < 2); if (index != 0) { context.Assembler.Movlhps(dest, src1, src2); } else { context.Assembler.Movsd(dest, src1, src2); } } } private static void GenerateVectorInsert16(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); //Vector Operand src2 = operation.GetSource(1); //Value Operand src3 = operation.GetSource(2); //Index if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(src1.Type == OperandType.V128); Debug.Assert(src3.Kind == OperandKind.Constant); byte index = src3.AsByte(); context.Assembler.Pinsrw(dest, src1, src2, index); } private static void GenerateVectorInsert8(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand src1 = operation.GetSource(0); //Vector Operand src2 = operation.GetSource(1); //Value Operand src3 = operation.GetSource(2); //Index // It's not possible to emulate this instruction without // SSE 4.1 support without the use of a temporary register, // so we instead handle that case on the pre-allocator when // SSE 4.1 is not supported on the CPU. Debug.Assert(HardwareCapabilities.SupportsSse41); if (!HardwareCapabilities.SupportsVexEncoding) { EnsureSameReg(dest, src1); } Debug.Assert(src1.Type == OperandType.V128); Debug.Assert(src3.Kind == OperandKind.Constant); byte index = src3.AsByte(); context.Assembler.Pinsrb(dest, src1, src2, index); } private static void GenerateVectorOne(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Debug.Assert(!dest.Type.IsInteger()); context.Assembler.Pcmpeqw(dest, dest, dest); } private static void GenerateVectorZero(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Debug.Assert(!dest.Type.IsInteger()); context.Assembler.Xorps(dest, dest, dest); } private static void GenerateVectorZeroUpper64(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type == OperandType.V128 && source.Type == OperandType.V128); GenerateZeroUpper64(context, dest, source); } private static void GenerateVectorZeroUpper96(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type == OperandType.V128 && source.Type == OperandType.V128); GenerateZeroUpper96(context, dest, source); } private static void GenerateZeroExtend16(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); context.Assembler.Movzx16(dest, source, OperandType.I32); } private static void GenerateZeroExtend32(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); // We can eliminate the move if source is already 32-bit and the registers are the same. if (dest.Value == source.Value && source.Type == OperandType.I32) { return; } context.Assembler.Mov(dest, source, OperandType.I32); } private static void GenerateZeroExtend8(CodeGenContext context, Operation operation) { Operand dest = operation.Destination; Operand source = operation.GetSource(0); Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); context.Assembler.Movzx8(dest, source, OperandType.I32); } private static void GenerateLoad(CodeGenContext context, Operand address, Operand value) { switch (value.Type) { case OperandType.I32: context.Assembler.Mov (value, address, OperandType.I32); break; case OperandType.I64: context.Assembler.Mov (value, address, OperandType.I64); break; case OperandType.FP32: context.Assembler.Movd (value, address); break; case OperandType.FP64: context.Assembler.Movq (value, address); break; case OperandType.V128: context.Assembler.Movdqu(value, address); break; default: Debug.Assert(false); break; } } private static void GenerateStore(CodeGenContext context, Operand address, Operand value) { switch (value.Type) { case OperandType.I32: context.Assembler.Mov (address, value, OperandType.I32); break; case OperandType.I64: context.Assembler.Mov (address, value, OperandType.I64); break; case OperandType.FP32: context.Assembler.Movd (address, value); break; case OperandType.FP64: context.Assembler.Movq (address, value); break; case OperandType.V128: context.Assembler.Movdqu(address, value); break; default: Debug.Assert(false); break; } } private static void GenerateZeroUpper64(CodeGenContext context, Operand dest, Operand source) { context.Assembler.Movq(dest, source); } private static void GenerateZeroUpper96(CodeGenContext context, Operand dest, Operand source) { context.Assembler.Movq(dest, source); context.Assembler.Pshufd(dest, dest, 0xfc); } private static bool MatchOperation(Operation node, Instruction inst, OperandType destType, Register destReg) { if (node == default || node.DestinationsCount == 0) { return false; } if (node.Instruction != inst) { return false; } Operand dest = node.Destination; return dest.Kind == OperandKind.Register && dest.Type == destType && dest.GetRegister() == destReg; } [Conditional("DEBUG")] private static void ValidateUnOp(Operand dest, Operand source) { EnsureSameReg (dest, source); EnsureSameType(dest, source); } [Conditional("DEBUG")] private static void ValidateBinOp(Operand dest, Operand src1, Operand src2) { EnsureSameReg (dest, src1); EnsureSameType(dest, src1, src2); } [Conditional("DEBUG")] private static void ValidateShift(Operand dest, Operand src1, Operand src2) { EnsureSameReg (dest, src1); EnsureSameType(dest, src1); Debug.Assert(dest.Type.IsInteger() && src2.Type == OperandType.I32); } private static void EnsureSameReg(Operand op1, Operand op2) { if (!op1.Type.IsInteger() && HardwareCapabilities.SupportsVexEncoding) { return; } Debug.Assert(op1.Kind == OperandKind.Register || op1.Kind == OperandKind.Memory); Debug.Assert(op1.Kind == op2.Kind); Debug.Assert(op1.Value == op2.Value); } private static void EnsureSameType(Operand op1, Operand op2) { Debug.Assert(op1.Type == op2.Type); } private static void EnsureSameType(Operand op1, Operand op2, Operand op3) { Debug.Assert(op1.Type == op2.Type); Debug.Assert(op1.Type == op3.Type); } private static void EnsureSameType(Operand op1, Operand op2, Operand op3, Operand op4) { Debug.Assert(op1.Type == op2.Type); Debug.Assert(op1.Type == op3.Type); Debug.Assert(op1.Type == op4.Type); } private static UnwindInfo WritePrologue(CodeGenContext context) { List pushEntries = new List(); Operand rsp = Register(X86Register.Rsp); int mask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; while (mask != 0) { int bit = BitOperations.TrailingZeroCount(mask); context.Assembler.Push(Register((X86Register)bit)); pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: bit)); mask &= ~(1 << bit); } int reservedStackSize = context.CallArgsRegionSize + context.AllocResult.SpillRegionSize; reservedStackSize += context.XmmSaveRegionSize; if (reservedStackSize >= StackGuardSize) { GenerateInlineStackProbe(context, reservedStackSize); } if (reservedStackSize != 0) { context.Assembler.Sub(rsp, Const(reservedStackSize), OperandType.I64); pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.AllocStack, context.StreamOffset, stackOffsetOrAllocSize: reservedStackSize)); } int offset = reservedStackSize; mask = CallingConvention.GetVecCalleeSavedRegisters() & context.AllocResult.VecUsedRegisters; while (mask != 0) { int bit = BitOperations.TrailingZeroCount(mask); offset -= 16; Operand memOp = MemoryOp(OperandType.V128, rsp, default, Multiplier.x1, offset); context.Assembler.Movdqu(memOp, Xmm((X86Register)bit)); pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.SaveXmm128, context.StreamOffset, bit, offset)); mask &= ~(1 << bit); } return new UnwindInfo(pushEntries.ToArray(), context.StreamOffset); } private static void WriteEpilogue(CodeGenContext context) { Operand rsp = Register(X86Register.Rsp); int reservedStackSize = context.CallArgsRegionSize + context.AllocResult.SpillRegionSize; reservedStackSize += context.XmmSaveRegionSize; int offset = reservedStackSize; int mask = CallingConvention.GetVecCalleeSavedRegisters() & context.AllocResult.VecUsedRegisters; while (mask != 0) { int bit = BitOperations.TrailingZeroCount(mask); offset -= 16; Operand memOp = MemoryOp(OperandType.V128, rsp, default, Multiplier.x1, offset); context.Assembler.Movdqu(Xmm((X86Register)bit), memOp); mask &= ~(1 << bit); } if (reservedStackSize != 0) { context.Assembler.Add(rsp, Const(reservedStackSize), OperandType.I64); } mask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; while (mask != 0) { int bit = BitUtils.HighestBitSet(mask); context.Assembler.Pop(Register((X86Register)bit)); mask &= ~(1 << bit); } } private static void GenerateInlineStackProbe(CodeGenContext context, int size) { // Windows does lazy stack allocation, and there are just 2 // guard pages on the end of the stack. So, if the allocation // size we make is greater than this guard size, we must ensure // that the OS will map all pages that we'll use. We do that by // doing a dummy read on those pages, forcing a page fault and // the OS to map them. If they are already mapped, nothing happens. const int pageMask = PageSize - 1; size = (size + pageMask) & ~pageMask; Operand rsp = Register(X86Register.Rsp); Operand temp = Register(CallingConvention.GetIntReturnRegister()); for (int offset = PageSize; offset < size; offset += PageSize) { Operand memOp = MemoryOp(OperandType.I32, rsp, default, Multiplier.x1, -offset); context.Assembler.Mov(temp, memOp, OperandType.I32); } } private static Operand Memory(Operand operand, OperandType type) { if (operand.Kind == OperandKind.Memory) { return operand; } return MemoryOp(type, operand); } private static Operand Register(X86Register register, OperandType type = OperandType.I64) { return Operand.Factory.Register((int)register, RegisterType.Integer, type); } private static Operand Xmm(X86Register register) { return Operand.Factory.Register((int)register, RegisterType.Vector, OperandType.V128); } } }