Ryujinx/ARMeilleure/Instructions/InstEmitSimdArithmetic32.cs
sharmander 3332b29f01
CPU: Implement VFMA (Vector) (#1762)
* Implement VFMA.F64

* Simplify switch

* Simplify FMA Instructions into their own IntrinsicType.

* Remove whitespace

* Fix indentation

* Change tests for Vfnms -- disable inf / nan

* Move args up, not description ;)

* Implementation Complete.

All Tests Pass (Slow / Fast Path)

* Move location of function in assembler + test updates.

* Shift params upwards

* Remove unused function

* Update PTC version.

* Add comments / re-oreder opcode table.

* Remove whitespace

* Fix nit

* Fix nit.

* Fix whitespace

* Wrong opcode was used by a bad merge.

* Addressed rip's comments.
2020-12-15 00:01:52 -03:00

1396 lines
53 KiB
C#

using ARMeilleure.Decoders;
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.Translation;
using System;
using System.Diagnostics;
using static ARMeilleure.Instructions.InstEmitFlowHelper;
using static ARMeilleure.Instructions.InstEmitHelper;
using static ARMeilleure.Instructions.InstEmitSimdHelper;
using static ARMeilleure.Instructions.InstEmitSimdHelper32;
using static ARMeilleure.IntermediateRepresentation.OperandHelper;
namespace ARMeilleure.Instructions
{
static partial class InstEmit32
{
public static void Vabd_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
EmitVectorBinaryOpI32(context, (op1, op2) => EmitAbs(context, context.Subtract(op1, op2)), !op.U);
}
public static void Vabdl_I(ArmEmitterContext context)
{
OpCode32SimdRegLong op = (OpCode32SimdRegLong)context.CurrOp;
EmitVectorBinaryLongOpI32(context, (op1, op2) => EmitAbs(context, context.Subtract(op1, op2)), !op.U);
}
public static void Vabs_S(ArmEmitterContext context)
{
OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarUnaryOpSimd32(context, (m) =>
{
return EmitFloatAbs(context, m, (op.Size & 1) == 0, false);
});
}
else
{
EmitScalarUnaryOpF32(context, (op1) => EmitUnaryMathCall(context, nameof(Math.Abs), op1));
}
}
public static void Vabs_V(ArmEmitterContext context)
{
OpCode32SimdCmpZ op = (OpCode32SimdCmpZ)context.CurrOp;
if (op.F)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorUnaryOpSimd32(context, (m) =>
{
return EmitFloatAbs(context, m, (op.Size & 1) == 0, true);
});
}
else
{
EmitVectorUnaryOpF32(context, (op1) => EmitUnaryMathCall(context, nameof(Math.Abs), op1));
}
}
else
{
EmitVectorUnaryOpSx32(context, (op1) => EmitAbs(context, op1));
}
}
private static Operand EmitAbs(ArmEmitterContext context, Operand value)
{
Operand isPositive = context.ICompareGreaterOrEqual(value, Const(value.Type, 0));
return context.ConditionalSelect(isPositive, value, context.Negate(value));
}
public static void Vadd_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarBinaryOpF32(context, Intrinsic.X86Addss, Intrinsic.X86Addsd);
}
else if (Optimizations.FastFP)
{
EmitScalarBinaryOpF32(context, (op1, op2) => context.Add(op1, op2));
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) => EmitSoftFloatCall(context, nameof(SoftFloat32.FPAdd), op1, op2));
}
}
public static void Vadd_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorBinaryOpF32(context, Intrinsic.X86Addps, Intrinsic.X86Addpd);
}
else if (Optimizations.FastFP)
{
EmitVectorBinaryOpF32(context, (op1, op2) => context.Add(op1, op2));
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPAddFpscr), op1, op2));
}
}
public static void Vadd_I(ArmEmitterContext context)
{
if (Optimizations.UseSse2)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
EmitVectorBinaryOpSimd32(context, (op1, op2) => context.AddIntrinsic(X86PaddInstruction[op.Size], op1, op2));
}
else
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.Add(op1, op2));
}
}
public static void Vaddl_I(ArmEmitterContext context)
{
OpCode32SimdRegLong op = (OpCode32SimdRegLong)context.CurrOp;
EmitVectorBinaryLongOpI32(context, (op1, op2) => context.Add(op1, op2), !op.U);
}
public static void Vaddw_I(ArmEmitterContext context)
{
OpCode32SimdRegWide op = (OpCode32SimdRegWide)context.CurrOp;
EmitVectorBinaryWideOpI32(context, (op1, op2) => context.Add(op1, op2), !op.U);
}
public static void Vdup(ArmEmitterContext context)
{
OpCode32SimdDupGP op = (OpCode32SimdDupGP)context.CurrOp;
Operand insert = GetIntA32(context, op.Rt);
// Zero extend into an I64, then replicate. Saves the most time over elementwise inserts.
insert = op.Size switch
{
2 => context.Multiply(context.ZeroExtend32(OperandType.I64, insert), Const(0x0000000100000001u)),
1 => context.Multiply(context.ZeroExtend16(OperandType.I64, insert), Const(0x0001000100010001u)),
0 => context.Multiply(context.ZeroExtend8(OperandType.I64, insert), Const(0x0101010101010101u)),
_ => throw new InvalidOperationException($"Invalid Vdup size \"{op.Size}\".")
};
InsertScalar(context, op.Vd, insert);
if (op.Q)
{
InsertScalar(context, op.Vd + 1, insert);
}
}
public static void Vdup_1(ArmEmitterContext context)
{
OpCode32SimdDupElem op = (OpCode32SimdDupElem)context.CurrOp;
Operand insert = EmitVectorExtractZx32(context, op.Vm >> 1, ((op.Vm & 1) << (3 - op.Size)) + op.Index, op.Size);
// Zero extend into an I64, then replicate. Saves the most time over elementwise inserts.
insert = op.Size switch
{
2 => context.Multiply(context.ZeroExtend32(OperandType.I64, insert), Const(0x0000000100000001u)),
1 => context.Multiply(context.ZeroExtend16(OperandType.I64, insert), Const(0x0001000100010001u)),
0 => context.Multiply(context.ZeroExtend8(OperandType.I64, insert), Const(0x0101010101010101u)),
_ => throw new InvalidOperationException($"Invalid Vdup size \"{op.Size}\".")
};
InsertScalar(context, op.Vd, insert);
if (op.Q)
{
InsertScalar(context, op.Vd | 1, insert);
}
}
private static (long, long) MaskHelperByteSequence(int start, int length, int startByte)
{
int end = start + length;
int b = startByte;
long result = 0;
long result2 = 0;
for (int i = 0; i < 8; i++)
{
result |= (long)((i >= end || i < start) ? 0x80 : b++) << (i * 8);
}
for (int i = 8; i < 16; i++)
{
result2 |= (long)((i >= end || i < start) ? 0x80 : b++) << ((i - 8) * 8);
}
return (result2, result);
}
public static void Vext(ArmEmitterContext context)
{
OpCode32SimdExt op = (OpCode32SimdExt)context.CurrOp;
int elems = op.GetBytesCount();
int byteOff = op.Immediate;
if (Optimizations.UseSsse3)
{
EmitVectorBinaryOpSimd32(context, (n, m) =>
{
// Writing low to high of d: start <imm> into n, overlap into m.
// Then rotate n down by <imm>, m up by (elems)-imm.
// Then OR them together for the result.
(long nMaskHigh, long nMaskLow) = MaskHelperByteSequence(0, elems - byteOff, byteOff);
(long mMaskHigh, long mMaskLow) = MaskHelperByteSequence(elems - byteOff, byteOff, 0);
Operand nMask, mMask;
if (!op.Q)
{
// Do the same operation to the bytes in the top doubleword too, as our target could be in either.
nMaskHigh = nMaskLow + 0x0808080808080808L;
mMaskHigh = mMaskLow + 0x0808080808080808L;
}
nMask = X86GetElements(context, nMaskHigh, nMaskLow);
mMask = X86GetElements(context, mMaskHigh, mMaskLow);
Operand nPart = context.AddIntrinsic(Intrinsic.X86Pshufb, n, nMask);
Operand mPart = context.AddIntrinsic(Intrinsic.X86Pshufb, m, mMask);
return context.AddIntrinsic(Intrinsic.X86Por, nPart, mPart);
});
}
else
{
Operand res = GetVecA32(op.Qd);
for (int index = 0; index < elems; index++)
{
Operand extract;
if (byteOff >= elems)
{
extract = EmitVectorExtractZx32(context, op.Qm, op.Im + (byteOff - elems), op.Size);
}
else
{
extract = EmitVectorExtractZx32(context, op.Qn, op.In + byteOff, op.Size);
}
byteOff++;
res = EmitVectorInsert(context, res, extract, op.Id + index, op.Size);
}
context.Copy(GetVecA32(op.Qd), res);
}
}
public static void Vfma_V(ArmEmitterContext context) // Fused.
{
if (Optimizations.FastFP && Optimizations.UseFma)
{
// Vectors contain elements that are 32-bits in length always. The only thing that will change is the number of elements in a vector.
// The 64-bit variant will never be used.
EmitVectorTernaryOpF32(context, Intrinsic.X86Vfmadd231ps, Intrinsic.X86Vfmadd231pd);
}
else
{
EmitVectorTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulAdd), op1, op2, op3);
});
}
}
public static void Vfma_S(ArmEmitterContext context) // Fused.
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
// TODO: Use FMA instruction set.
EmitScalarTernaryOpF32(context, Intrinsic.X86Mulss, Intrinsic.X86Mulsd, Intrinsic.X86Addss, Intrinsic.X86Addsd);
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulAdd), op1, op2, op3);
});
}
}
public static void Vfms_S(ArmEmitterContext context) // Fused.
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
// TODO: Use FMA instruction set.
EmitScalarTernaryOpF32(context, Intrinsic.X86Mulss, Intrinsic.X86Mulsd, Intrinsic.X86Subss, Intrinsic.X86Subsd);
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulSub), op1, op2, op3);
});
}
}
public static void Vfnma_S(ArmEmitterContext context) // Fused.
{
if (Optimizations.FastFP && Optimizations.UseFma)
{
EmitScalarTernaryOpF32(context, Intrinsic.X86Vfnmsub231ss, Intrinsic.X86Vfnmsub231sd);
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulAdd), context.Negate(op1), context.Negate(op2), op3);
});
}
}
public static void Vfnms_S(ArmEmitterContext context) // Fused.
{
if (Optimizations.FastFP && Optimizations.UseFma)
{
EmitScalarTernaryOpF32(context, Intrinsic.X86Vfmsub231ss, Intrinsic.X86Vfmsub231sd);
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulAdd), context.Negate(op1), op2, op3);
});
}
}
public static void Vhadd(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (op.U)
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.ShiftRightUI(context.Add(op1, op2), Const(1)));
}
else
{
EmitVectorBinaryOpSx32(context, (op1, op2) => context.ShiftRightSI(context.Add(op1, op2), Const(1)));
}
}
public static void Vmov_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarUnaryOpF32(context, 0, 0);
}
else
{
EmitScalarUnaryOpF32(context, (op1) => op1);
}
}
public static void Vmovn(ArmEmitterContext context)
{
EmitVectorUnaryNarrowOp32(context, (op1) => op1);
}
public static void Vneg_S(ArmEmitterContext context)
{
OpCode32SimdS op = (OpCode32SimdS)context.CurrOp;
if (Optimizations.UseSse2)
{
EmitScalarUnaryOpSimd32(context, (m) =>
{
if ((op.Size & 1) == 0)
{
Operand mask = X86GetScalar(context, -0f);
return context.AddIntrinsic(Intrinsic.X86Xorps, mask, m);
}
else
{
Operand mask = X86GetScalar(context, -0d);
return context.AddIntrinsic(Intrinsic.X86Xorpd, mask, m);
}
});
}
else
{
EmitScalarUnaryOpF32(context, (op1) => context.Negate(op1));
}
}
public static void Vnmul_S(ArmEmitterContext context)
{
OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
if (Optimizations.UseSse2)
{
EmitScalarBinaryOpSimd32(context, (n, m) =>
{
if ((op.Size & 1) == 0)
{
Operand res = context.AddIntrinsic(Intrinsic.X86Mulss, n, m);
Operand mask = X86GetScalar(context, -0f);
return context.AddIntrinsic(Intrinsic.X86Xorps, mask, res);
}
else
{
Operand res = context.AddIntrinsic(Intrinsic.X86Mulsd, n, m);
Operand mask = X86GetScalar(context, -0d);
return context.AddIntrinsic(Intrinsic.X86Xorpd, mask, res);
}
});
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) => context.Negate(context.Multiply(op1, op2)));
}
}
public static void Vnmla_S(ArmEmitterContext context)
{
OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarTernaryOpSimd32(context, (d, n, m) =>
{
if ((op.Size & 1) == 0)
{
Operand res = context.AddIntrinsic(Intrinsic.X86Mulss, n, m);
res = context.AddIntrinsic(Intrinsic.X86Addss, d, res);
Operand mask = X86GetScalar(context, -0f);
return context.AddIntrinsic(Intrinsic.X86Xorps, mask, res);
}
else
{
Operand res = context.AddIntrinsic(Intrinsic.X86Mulsd, n, m);
res = context.AddIntrinsic(Intrinsic.X86Addsd, d, res);
Operand mask = X86GetScalar(context, -0d);
return context.AddIntrinsic(Intrinsic.X86Xorpd, mask, res);
}
});
}
else if (Optimizations.FastFP)
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return context.Negate(context.Add(op1, context.Multiply(op2, op3)));
});
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPNegMulAdd), op1, op2, op3);
});
}
}
public static void Vnmls_S(ArmEmitterContext context)
{
OpCode32SimdRegS op = (OpCode32SimdRegS)context.CurrOp;
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarTernaryOpSimd32(context, (d, n, m) =>
{
if ((op.Size & 1) == 0)
{
Operand res = context.AddIntrinsic(Intrinsic.X86Mulss, n, m);
Operand mask = X86GetScalar(context, -0f);
d = context.AddIntrinsic(Intrinsic.X86Xorps, mask, d);
return context.AddIntrinsic(Intrinsic.X86Addss, d, res);
}
else
{
Operand res = context.AddIntrinsic(Intrinsic.X86Mulsd, n, m);
Operand mask = X86GetScalar(context, -0d);
d = context.AddIntrinsic(Intrinsic.X86Xorpd, mask, res);
return context.AddIntrinsic(Intrinsic.X86Addsd, d, res);
}
});
}
else if (Optimizations.FastFP)
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return context.Add(context.Negate(op1), context.Multiply(op2, op3));
});
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPNegMulSub), op1, op2, op3);
});
}
}
public static void Vneg_V(ArmEmitterContext context)
{
OpCode32SimdCmpZ op = (OpCode32SimdCmpZ)context.CurrOp;
if (op.F)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorUnaryOpSimd32(context, (m) =>
{
if ((op.Size & 1) == 0)
{
Operand mask = X86GetAllElements(context, -0f);
return context.AddIntrinsic(Intrinsic.X86Xorps, mask, m);
}
else
{
Operand mask = X86GetAllElements(context, -0d);
return context.AddIntrinsic(Intrinsic.X86Xorpd, mask, m);
}
});
}
else
{
EmitVectorUnaryOpF32(context, (op1) => context.Negate(op1));
}
}
else
{
EmitVectorUnaryOpSx32(context, (op1) => context.Negate(op1));
}
}
public static void Vdiv_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarBinaryOpF32(context, Intrinsic.X86Divss, Intrinsic.X86Divsd);
}
else if (Optimizations.FastFP)
{
EmitScalarBinaryOpF32(context, (op1, op2) => context.Divide(op1, op2));
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPDiv), op1, op2);
});
}
}
public static void Vmaxnm_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse41)
{
EmitSse41MaxMinNumOpF32(context, true, true);
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) => EmitSoftFloatCall(context, nameof(SoftFloat32.FPMaxNum), op1, op2));
}
}
public static void Vmaxnm_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse41)
{
EmitSse41MaxMinNumOpF32(context, true, false);
}
else
{
EmitVectorBinaryOpSx32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMaxNumFpscr), op1, op2));
}
}
public static void Vminnm_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse41)
{
EmitSse41MaxMinNumOpF32(context, false, true);
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) => EmitSoftFloatCall(context, nameof(SoftFloat32.FPMinNum), op1, op2));
}
}
public static void Vminnm_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse41)
{
EmitSse41MaxMinNumOpF32(context, false, false);
}
else
{
EmitVectorBinaryOpSx32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMinNumFpscr), op1, op2));
}
}
public static void Vmax_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorBinaryOpF32(context, Intrinsic.X86Maxps, Intrinsic.X86Maxpd);
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMaxFpscr), op1, op2);
});
}
}
public static void Vmax_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (op.U)
{
if (Optimizations.UseSse2)
{
EmitVectorBinaryOpSimd32(context, (op1, op2) => context.AddIntrinsic(X86PmaxuInstruction[op.Size], op1, op2));
}
else
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.ConditionalSelect(context.ICompareGreaterUI(op1, op2), op1, op2));
}
}
else
{
if (Optimizations.UseSse2)
{
EmitVectorBinaryOpSimd32(context, (op1, op2) => context.AddIntrinsic(X86PmaxsInstruction[op.Size], op1, op2));
}
else
{
EmitVectorBinaryOpSx32(context, (op1, op2) => context.ConditionalSelect(context.ICompareGreater(op1, op2), op1, op2));
}
}
}
public static void Vmin_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorBinaryOpF32(context, Intrinsic.X86Minps, Intrinsic.X86Minpd);
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMinFpscr), op1, op2);
});
}
}
public static void Vmin_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (op.U)
{
if (Optimizations.UseSse2)
{
EmitVectorBinaryOpSimd32(context, (op1, op2) => context.AddIntrinsic(X86PminuInstruction[op.Size], op1, op2));
}
else
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.ConditionalSelect(context.ICompareLessUI(op1, op2), op1, op2));
}
}
else
{
if (Optimizations.UseSse2)
{
EmitVectorBinaryOpSimd32(context, (op1, op2) => context.AddIntrinsic(X86PminsInstruction[op.Size], op1, op2));
}
else
{
EmitVectorBinaryOpSx32(context, (op1, op2) => context.ConditionalSelect(context.ICompareLess(op1, op2), op1, op2));
}
}
}
public static void Vmla_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarTernaryOpF32(context, Intrinsic.X86Mulss, Intrinsic.X86Mulsd, Intrinsic.X86Addss, Intrinsic.X86Addsd);
}
else if (Optimizations.FastFP)
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return context.Add(op1, context.Multiply(op2, op3));
});
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
Operand res = EmitSoftFloatCall(context, nameof(SoftFloat32.FPMul), op2, op3);
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPAdd), op1, res);
});
}
}
public static void Vmla_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorTernaryOpF32(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd, Intrinsic.X86Addps, Intrinsic.X86Addpd);
}
else if (Optimizations.FastFP)
{
EmitVectorTernaryOpF32(context, (op1, op2, op3) => context.Add(op1, context.Multiply(op2, op3)));
}
else
{
EmitVectorTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMulAddFpscr), op1, op2, op3);
});
}
}
public static void Vmla_I(ArmEmitterContext context)
{
EmitVectorTernaryOpZx32(context, (op1, op2, op3) => context.Add(op1, context.Multiply(op2, op3)));
}
public static void Vmla_1(ArmEmitterContext context)
{
OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
if (op.F)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorsByScalarOpF32(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd, Intrinsic.X86Addps, Intrinsic.X86Addpd);
}
else if (Optimizations.FastFP)
{
EmitVectorsByScalarOpF32(context, (op1, op2, op3) => context.Add(op1, context.Multiply(op2, op3)));
}
else
{
EmitVectorsByScalarOpF32(context, (op1, op2, op3) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMulAddFpscr), op1, op2, op3));
}
}
else
{
EmitVectorsByScalarOpI32(context, (op1, op2, op3) => context.Add(op1, context.Multiply(op2, op3)), false);
}
}
public static void Vmls_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarTernaryOpF32(context, Intrinsic.X86Mulss, Intrinsic.X86Mulsd, Intrinsic.X86Subss, Intrinsic.X86Subsd);
}
else if (Optimizations.FastFP)
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
return context.Subtract(op1, context.Multiply(op2, op3));
});
}
else
{
EmitScalarTernaryOpF32(context, (op1, op2, op3) =>
{
Operand res = EmitSoftFloatCall(context, nameof(SoftFloat32.FPMul), op2, op3);
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPSub), op1, res);
});
}
}
public static void Vmls_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorTernaryOpF32(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd, Intrinsic.X86Subps, Intrinsic.X86Subpd);
}
else if (Optimizations.FastFP)
{
EmitVectorTernaryOpF32(context, (op1, op2, op3) => context.Subtract(op1, context.Multiply(op2, op3)));
}
else
{
EmitVectorTernaryOpF32(context, (op1, op2, op3) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMulSubFpscr), op1, op2, op3);
});
}
}
public static void Vmls_I(ArmEmitterContext context)
{
EmitVectorTernaryOpZx32(context, (op1, op2, op3) => context.Subtract(op1, context.Multiply(op2, op3)));
}
public static void Vmls_1(ArmEmitterContext context)
{
OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
if (op.F)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorsByScalarOpF32(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd, Intrinsic.X86Subps, Intrinsic.X86Subpd);
}
else if (Optimizations.FastFP)
{
EmitVectorsByScalarOpF32(context, (op1, op2, op3) => context.Subtract(op1, context.Multiply(op2, op3)));
}
else
{
EmitVectorsByScalarOpF32(context, (op1, op2, op3) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMulSubFpscr), op1, op2, op3));
}
}
else
{
EmitVectorsByScalarOpI32(context, (op1, op2, op3) => context.Subtract(op1, context.Multiply(op2, op3)), false);
}
}
public static void Vmlsl_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
EmitVectorTernaryLongOpI32(context, (opD, op1, op2) => context.Subtract(opD, context.Multiply(op1, op2)), !op.U);
}
public static void Vmul_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarBinaryOpF32(context, Intrinsic.X86Mulss, Intrinsic.X86Mulsd);
}
else if (Optimizations.FastFP)
{
EmitScalarBinaryOpF32(context, (op1, op2) => context.Multiply(op1, op2));
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMul), op1, op2);
});
}
}
public static void Vmul_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorBinaryOpF32(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd);
}
else if (Optimizations.FastFP)
{
EmitVectorBinaryOpF32(context, (op1, op2) => context.Multiply(op1, op2));
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMulFpscr), op1, op2);
});
}
}
public static void Vmul_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (op.U) // This instruction is always signed, U indicates polynomial mode.
{
EmitVectorBinaryOpZx32(context, (op1, op2) => EmitPolynomialMultiply(context, op1, op2, 8 << op.Size));
}
else
{
EmitVectorBinaryOpSx32(context, (op1, op2) => context.Multiply(op1, op2));
}
}
public static void Vmul_1(ArmEmitterContext context)
{
OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
if (op.F)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorByScalarOpF32(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd);
}
else if (Optimizations.FastFP)
{
EmitVectorByScalarOpF32(context, (op1, op2) => context.Multiply(op1, op2));
}
else
{
EmitVectorByScalarOpF32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMulFpscr), op1, op2));
}
}
else
{
EmitVectorByScalarOpI32(context, (op1, op2) => context.Multiply(op1, op2), false);
}
}
public static void Vmull_1(ArmEmitterContext context)
{
OpCode32SimdRegElem op = (OpCode32SimdRegElem)context.CurrOp;
EmitVectorByScalarLongOpI32(context, (op1, op2) => context.Multiply(op1, op2), !op.U);
}
public static void Vmull_I(ArmEmitterContext context)
{
OpCode32SimdRegLong op = (OpCode32SimdRegLong)context.CurrOp;
if (op.Polynomial)
{
EmitVectorBinaryLongOpI32(context, (op1, op2) => EmitPolynomialMultiply(context, op1, op2, 8 << op.Size), false);
}
else
{
EmitVectorBinaryLongOpI32(context, (op1, op2) => context.Multiply(op1, op2), !op.U);
}
}
public static void Vpadd_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitSse2VectorPairwiseOpF32(context, Intrinsic.X86Addps);
}
else
{
EmitVectorPairwiseOpF32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPAddFpscr), op1, op2));
}
}
public static void Vpadd_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (Optimizations.UseSsse3)
{
EmitSsse3VectorPairwiseOp32(context, X86PaddInstruction);
}
else
{
EmitVectorPairwiseOpI32(context, (op1, op2) => context.Add(op1, op2), !op.U);
}
}
public static void Vpmax_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitSse2VectorPairwiseOpF32(context, Intrinsic.X86Maxps);
}
else
{
EmitVectorPairwiseOpF32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat64.FPMaxFpscr), op1, op2));
}
}
public static void Vpmax_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (Optimizations.UseSsse3)
{
EmitSsse3VectorPairwiseOp32(context, op.U ? X86PmaxuInstruction : X86PmaxsInstruction);
}
else
{
EmitVectorPairwiseOpI32(context, (op1, op2) =>
{
Operand greater = op.U ? context.ICompareGreaterUI(op1, op2) : context.ICompareGreater(op1, op2);
return context.ConditionalSelect(greater, op1, op2);
}, !op.U);
}
}
public static void Vpmin_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitSse2VectorPairwiseOpF32(context, Intrinsic.X86Minps);
}
else
{
EmitVectorPairwiseOpF32(context, (op1, op2) => EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPMinFpscr), op1, op2));
}
}
public static void Vpmin_I(ArmEmitterContext context)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
if (Optimizations.UseSsse3)
{
EmitSsse3VectorPairwiseOp32(context, op.U ? X86PminuInstruction : X86PminsInstruction);
}
else
{
EmitVectorPairwiseOpI32(context, (op1, op2) =>
{
Operand greater = op.U ? context.ICompareLessUI(op1, op2) : context.ICompareLess(op1, op2);
return context.ConditionalSelect(greater, op1, op2);
}, !op.U);
}
}
public static void Vrev(ArmEmitterContext context)
{
OpCode32SimdRev op = (OpCode32SimdRev)context.CurrOp;
if (Optimizations.UseSsse3)
{
EmitVectorUnaryOpSimd32(context, (op1) =>
{
Operand mask;
switch (op.Size)
{
case 3:
// Rev64
switch (op.Opc)
{
case 0:
mask = X86GetElements(context, 0x08090a0b0c0d0e0fL, 0x0001020304050607L);
return context.AddIntrinsic(Intrinsic.X86Pshufb, op1, mask);
case 1:
mask = X86GetElements(context, 0x09080b0a0d0c0f0eL, 0x0100030205040706L);
return context.AddIntrinsic(Intrinsic.X86Pshufb, op1, mask);
case 2:
return context.AddIntrinsic(Intrinsic.X86Shufps, op1, op1, Const(1 | (0 << 2) | (3 << 4) | (2 << 6)));
}
break;
case 2:
// Rev32
switch (op.Opc)
{
case 0:
mask = X86GetElements(context, 0x0c0d0e0f_08090a0bL, 0x04050607_00010203L);
return context.AddIntrinsic(Intrinsic.X86Pshufb, op1, mask);
case 1:
mask = X86GetElements(context, 0x0d0c0f0e_09080b0aL, 0x05040706_01000302L);
return context.AddIntrinsic(Intrinsic.X86Pshufb, op1, mask);
}
break;
case 1:
// Rev16
mask = X86GetElements(context, 0x0e0f_0c0d_0a0b_0809L, 0x_0607_0405_0203_0001L);
return context.AddIntrinsic(Intrinsic.X86Pshufb, op1, mask);
}
throw new InvalidOperationException("Invalid VREV Opcode + Size combo."); // Should be unreachable.
});
}
else
{
EmitVectorUnaryOpZx32(context, (op1) =>
{
switch (op.Opc)
{
case 0:
switch (op.Size) // Swap bytes.
{
case 1:
return InstEmitAluHelper.EmitReverseBytes16_32Op(context, op1);
case 2:
case 3:
return context.ByteSwap(op1);
}
break;
case 1:
switch (op.Size)
{
case 2:
return context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op1, Const(0xffff0000)), Const(16)),
context.ShiftLeft(context.BitwiseAnd(op1, Const(0x0000ffff)), Const(16)));
case 3:
return context.BitwiseOr(
context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op1, Const(0xffff000000000000ul)), Const(48)),
context.ShiftLeft(context.BitwiseAnd(op1, Const(0x000000000000fffful)), Const(48))),
context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op1, Const(0x0000ffff00000000ul)), Const(16)),
context.ShiftLeft(context.BitwiseAnd(op1, Const(0x00000000ffff0000ul)), Const(16))));
}
break;
case 2:
// Swap upper and lower halves.
return context.BitwiseOr(context.ShiftRightUI(context.BitwiseAnd(op1, Const(0xffffffff00000000ul)), Const(32)),
context.ShiftLeft(context.BitwiseAnd(op1, Const(0x00000000fffffffful)), Const(32)));
}
throw new InvalidOperationException("Invalid VREV Opcode + Size combo."); // Should be unreachable.
});
}
}
public static void Vrecpe(ArmEmitterContext context)
{
OpCode32SimdSqrte op = (OpCode32SimdSqrte)context.CurrOp;
if (op.F)
{
int sizeF = op.Size & 1;
if (Optimizations.FastFP && Optimizations.UseSse2 && sizeF == 0)
{
EmitVectorUnaryOpF32(context, Intrinsic.X86Rcpps, 0);
}
else
{
EmitVectorUnaryOpF32(context, (op1) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPRecipEstimateFpscr), op1);
});
}
}
else
{
throw new NotImplementedException("Integer Vrecpe not currently implemented.");
}
}
public static void Vrecps(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
bool single = (op.Size & 1) == 0;
// (2 - (n*m))
EmitVectorBinaryOpSimd32(context, (n, m) =>
{
if (single)
{
Operand maskTwo = X86GetAllElements(context, 2f);
Operand res = context.AddIntrinsic(Intrinsic.X86Mulps, n, m);
return context.AddIntrinsic(Intrinsic.X86Subps, maskTwo, res);
}
else
{
Operand maskTwo = X86GetAllElements(context, 2d);
Operand res = context.AddIntrinsic(Intrinsic.X86Mulpd, n, m);
return context.AddIntrinsic(Intrinsic.X86Subpd, maskTwo, res);
}
});
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPRecipStep), op1, op2);
});
}
}
public static void Vrsqrte(ArmEmitterContext context)
{
OpCode32SimdSqrte op = (OpCode32SimdSqrte)context.CurrOp;
if (op.F)
{
int sizeF = op.Size & 1;
if (Optimizations.FastFP && Optimizations.UseSse2 && sizeF == 0)
{
EmitVectorUnaryOpF32(context, Intrinsic.X86Rsqrtps, 0);
}
else
{
EmitVectorUnaryOpF32(context, (op1) =>
{
return EmitSoftFloatCallDefaultFpscr(context, nameof(SoftFloat32.FPRSqrtEstimateFpscr), op1);
});
}
}
else
{
throw new NotImplementedException("Integer Vrsqrte not currently implemented.");
}
}
public static void Vrsqrts(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
bool single = (op.Size & 1) == 0;
// (3 - (n*m)) / 2
EmitVectorBinaryOpSimd32(context, (n, m) =>
{
if (single)
{
Operand maskHalf = X86GetAllElements(context, 0.5f);
Operand maskThree = X86GetAllElements(context, 3f);
Operand res = context.AddIntrinsic(Intrinsic.X86Mulps, n, m);
res = context.AddIntrinsic(Intrinsic.X86Subps, maskThree, res);
return context.AddIntrinsic(Intrinsic.X86Mulps, maskHalf, res);
}
else
{
Operand maskHalf = X86GetAllElements(context, 0.5d);
Operand maskThree = X86GetAllElements(context, 3d);
Operand res = context.AddIntrinsic(Intrinsic.X86Mulpd, n, m);
res = context.AddIntrinsic(Intrinsic.X86Subpd, maskThree, res);
return context.AddIntrinsic(Intrinsic.X86Mulpd, maskHalf, res);
}
});
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPRSqrtStep), op1, op2);
});
}
}
public static void Vsel(ArmEmitterContext context)
{
OpCode32SimdSel op = (OpCode32SimdSel)context.CurrOp;
Operand condition = null;
switch (op.Cc)
{
case OpCode32SimdSelMode.Eq:
condition = GetCondTrue(context, Condition.Eq);
break;
case OpCode32SimdSelMode.Ge:
condition = GetCondTrue(context, Condition.Ge);
break;
case OpCode32SimdSelMode.Gt:
condition = GetCondTrue(context, Condition.Gt);
break;
case OpCode32SimdSelMode.Vs:
condition = GetCondTrue(context, Condition.Vs);
break;
}
EmitScalarBinaryOpI32(context, (op1, op2) =>
{
return context.ConditionalSelect(condition, op1, op2);
});
}
public static void Vsqrt_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarUnaryOpF32(context, Intrinsic.X86Sqrtss, Intrinsic.X86Sqrtsd);
}
else
{
EmitScalarUnaryOpF32(context, (op1) =>
{
return EmitSoftFloatCall(context, nameof(SoftFloat32.FPSqrt), op1);
});
}
}
public static void Vsub_S(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitScalarBinaryOpF32(context, Intrinsic.X86Subss, Intrinsic.X86Subsd);
}
else
{
EmitScalarBinaryOpF32(context, (op1, op2) => context.Subtract(op1, op2));
}
}
public static void Vsub_V(ArmEmitterContext context)
{
if (Optimizations.FastFP && Optimizations.UseSse2)
{
EmitVectorBinaryOpF32(context, Intrinsic.X86Subps, Intrinsic.X86Subpd);
}
else
{
EmitVectorBinaryOpF32(context, (op1, op2) => context.Subtract(op1, op2));
}
}
public static void Vsub_I(ArmEmitterContext context)
{
if (Optimizations.UseSse2)
{
OpCode32SimdReg op = (OpCode32SimdReg)context.CurrOp;
EmitVectorBinaryOpSimd32(context, (op1, op2) => context.AddIntrinsic(X86PsubInstruction[op.Size], op1, op2));
}
else
{
EmitVectorBinaryOpZx32(context, (op1, op2) => context.Subtract(op1, op2));
}
}
public static void Vsubw_I(ArmEmitterContext context)
{
OpCode32SimdRegWide op = (OpCode32SimdRegWide)context.CurrOp;
EmitVectorBinaryWideOpI32(context, (op1, op2) => context.Subtract(op1, op2), !op.U);
}
private static void EmitSse41MaxMinNumOpF32(ArmEmitterContext context, bool isMaxNum, bool scalar)
{
IOpCode32Simd op = (IOpCode32Simd)context.CurrOp;
Func<Operand, Operand, Operand> genericEmit = (n, m) =>
{
Operand nNum = context.Copy(n);
Operand mNum = context.Copy(m);
InstEmit.EmitSse2VectorIsNaNOpF(context, nNum, out Operand nQNaNMask, out _, isQNaN: true);
InstEmit.EmitSse2VectorIsNaNOpF(context, mNum, out Operand mQNaNMask, out _, isQNaN: true);
int sizeF = op.Size & 1;
if (sizeF == 0)
{
Operand negInfMask = X86GetAllElements(context, isMaxNum ? float.NegativeInfinity : float.PositiveInfinity);
Operand nMask = context.AddIntrinsic(Intrinsic.X86Andnps, mQNaNMask, nQNaNMask);
Operand mMask = context.AddIntrinsic(Intrinsic.X86Andnps, nQNaNMask, mQNaNMask);
nNum = context.AddIntrinsic(Intrinsic.X86Blendvps, nNum, negInfMask, nMask);
mNum = context.AddIntrinsic(Intrinsic.X86Blendvps, mNum, negInfMask, mMask);
return context.AddIntrinsic(isMaxNum ? Intrinsic.X86Maxps : Intrinsic.X86Minps, nNum, mNum);
}
else /* if (sizeF == 1) */
{
Operand negInfMask = X86GetAllElements(context, isMaxNum ? double.NegativeInfinity : double.PositiveInfinity);
Operand nMask = context.AddIntrinsic(Intrinsic.X86Andnpd, mQNaNMask, nQNaNMask);
Operand mMask = context.AddIntrinsic(Intrinsic.X86Andnpd, nQNaNMask, mQNaNMask);
nNum = context.AddIntrinsic(Intrinsic.X86Blendvpd, nNum, negInfMask, nMask);
mNum = context.AddIntrinsic(Intrinsic.X86Blendvpd, mNum, negInfMask, mMask);
return context.AddIntrinsic(isMaxNum ? Intrinsic.X86Maxpd : Intrinsic.X86Minpd, nNum, mNum);
}
};
if (scalar)
{
EmitScalarBinaryOpSimd32(context, genericEmit);
}
else
{
EmitVectorBinaryOpSimd32(context, genericEmit);
}
}
private static Operand EmitPolynomialMultiply(ArmEmitterContext context, Operand op1, Operand op2, int eSize)
{
Debug.Assert(eSize <= 32);
Operand result = eSize == 32 ? Const(0L) : Const(0);
if (eSize == 32)
{
op1 = context.ZeroExtend32(OperandType.I64, op1);
op2 = context.ZeroExtend32(OperandType.I64, op2);
}
for (int i = 0; i < eSize; i++)
{
Operand mask = context.BitwiseAnd(op1, Const(op1.Type, 1L << i));
result = context.BitwiseExclusiveOr(result, context.Multiply(op2, mask));
}
return result;
}
}
}