Ryujinx/Ryujinx.Graphics.Nvdec.Vp9/Dsp/Convolve.cs

950 lines
30 KiB
C#
Raw Normal View History

using Ryujinx.Common.Memory;
using Ryujinx.Graphics.Nvdec.Vp9.Common;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using static Ryujinx.Graphics.Nvdec.Vp9.Dsp.Filter;
namespace Ryujinx.Graphics.Nvdec.Vp9.Dsp
{
internal static class Convolve
{
private const bool UseIntrinsics = true;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<int> MultiplyAddAdjacent(
Vector128<short> vsrc0,
Vector128<short> vsrc1,
Vector128<short> vsrc2,
Vector128<short> vsrc3,
Vector128<short> vfilter,
Vector128<int> zero)
{
// < sumN, sumN, sumN, sumN >
Vector128<int> sum0 = Sse2.MultiplyAddAdjacent(vsrc0, vfilter);
Vector128<int> sum1 = Sse2.MultiplyAddAdjacent(vsrc1, vfilter);
Vector128<int> sum2 = Sse2.MultiplyAddAdjacent(vsrc2, vfilter);
Vector128<int> sum3 = Sse2.MultiplyAddAdjacent(vsrc3, vfilter);
// < 0, 0, sumN, sumN >
sum0 = Ssse3.HorizontalAdd(sum0, zero);
sum1 = Ssse3.HorizontalAdd(sum1, zero);
sum2 = Ssse3.HorizontalAdd(sum2, zero);
sum3 = Ssse3.HorizontalAdd(sum3, zero);
// < 0, 0, 0, sumN >
sum0 = Ssse3.HorizontalAdd(sum0, zero);
sum1 = Ssse3.HorizontalAdd(sum1, zero);
sum2 = Ssse3.HorizontalAdd(sum2, zero);
sum3 = Ssse3.HorizontalAdd(sum3, zero);
// < 0, 0, sum1, sum0 >
Vector128<int> sum01 = Sse2.UnpackLow(sum0, sum1);
// < 0, 0, sum3, sum2 >
Vector128<int> sum23 = Sse2.UnpackLow(sum2, sum3);
// < sum3, sum2, sum1, sum0 >
return Sse.MoveLowToHigh(sum01.AsSingle(), sum23.AsSingle()).AsInt32();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<int> RoundShift(Vector128<int> value, Vector128<int> const64)
{
return Sse2.ShiftRightArithmetic(Sse2.Add(value, const64), FilterBits);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<byte> PackUnsignedSaturate(Vector128<int> value, Vector128<int> zero)
{
return Sse2.PackUnsignedSaturate(Sse41.PackUnsignedSaturate(value, zero).AsInt16(), zero.AsInt16());
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void ConvolveHorizSse41(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] xFilters,
int x0Q4,
int w,
int h)
{
Vector128<int> zero = Vector128<int>.Zero;
Vector128<int> const64 = Vector128.Create(64);
ulong x, y;
src -= SubpelTaps / 2 - 1;
fixed (Array8<short>* xFilter = xFilters)
{
Vector128<short> vfilter = Sse2.LoadVector128((short*)xFilter + (uint)(x0Q4 & SubpelMask) * 8);
for (y = 0; y < (uint)h; ++y)
{
ulong srcOffset = (uint)x0Q4 >> SubpelBits;
for (x = 0; x < (uint)w; x += 4)
{
Vector128<short> vsrc0 = Sse41.ConvertToVector128Int16(&src[srcOffset + x]);
Vector128<short> vsrc1 = Sse41.ConvertToVector128Int16(&src[srcOffset + x + 1]);
Vector128<short> vsrc2 = Sse41.ConvertToVector128Int16(&src[srcOffset + x + 2]);
Vector128<short> vsrc3 = Sse41.ConvertToVector128Int16(&src[srcOffset + x + 3]);
Vector128<int> sum0123 = MultiplyAddAdjacent(vsrc0, vsrc1, vsrc2, vsrc3, vfilter, zero);
Sse.StoreScalar((float*)&dst[x], PackUnsignedSaturate(RoundShift(sum0123, const64), zero).AsSingle());
}
src += srcStride;
dst += dstStride;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void ConvolveHoriz(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] xFilters,
int x0Q4,
int xStepQ4,
int w,
int h)
{
if (Sse41.IsSupported && UseIntrinsics && xStepQ4 == 1 << SubpelBits)
{
ConvolveHorizSse41(src, srcStride, dst, dstStride, xFilters, x0Q4, w, h);
return;
}
int x, y;
src -= SubpelTaps / 2 - 1;
for (y = 0; y < h; ++y)
{
int xQ4 = x0Q4;
for (x = 0; x < w; ++x)
{
byte* srcX = &src[xQ4 >> SubpelBits];
ref Array8<short> xFilter = ref xFilters[xQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcX[k] * xFilter[k];
}
dst[x] = BitUtils.ClipPixel(BitUtils.RoundPowerOfTwo(sum, FilterBits));
xQ4 += xStepQ4;
}
src += srcStride;
dst += dstStride;
}
}
private static unsafe void ConvolveAvgHoriz(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] xFilters,
int x0Q4,
int xStepQ4,
int w,
int h)
{
int x, y;
src -= SubpelTaps / 2 - 1;
for (y = 0; y < h; ++y)
{
int xQ4 = x0Q4;
for (x = 0; x < w; ++x)
{
byte* srcX = &src[xQ4 >> SubpelBits];
ref Array8<short> xFilter = ref xFilters[xQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcX[k] * xFilter[k];
}
dst[x] = (byte)BitUtils.RoundPowerOfTwo(dst[x] + BitUtils.ClipPixel(BitUtils.RoundPowerOfTwo(sum, FilterBits)), 1);
xQ4 += xStepQ4;
}
src += srcStride;
dst += dstStride;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void ConvolveVertAvx2(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] yFilters,
int y0Q4,
int w,
int h)
{
Vector128<int> zero = Vector128<int>.Zero;
Vector128<int> const64 = Vector128.Create(64);
Vector256<int> indices = Vector256.Create(
0,
srcStride,
srcStride * 2,
srcStride * 3,
srcStride * 4,
srcStride * 5,
srcStride * 6,
srcStride * 7);
ulong x, y;
src -= srcStride * (SubpelTaps / 2 - 1);
fixed (Array8<short>* yFilter = yFilters)
{
Vector128<short> vfilter = Sse2.LoadVector128((short*)yFilter + (uint)(y0Q4 & SubpelMask) * 8);
ulong srcBaseY = (uint)y0Q4 >> SubpelBits;
for (y = 0; y < (uint)h; ++y)
{
ulong srcOffset = (srcBaseY + y) * (uint)srcStride;
for (x = 0; x < (uint)w; x += 4)
{
Vector256<int> vsrc = Avx2.GatherVector256((uint*)&src[srcOffset + x], indices, 1).AsInt32();
Vector128<int> vsrcL = vsrc.GetLower();
Vector128<int> vsrcH = vsrc.GetUpper();
Vector128<byte> vsrcUnpck11 = Sse2.UnpackLow(vsrcL.AsByte(), vsrcH.AsByte());
Vector128<byte> vsrcUnpck12 = Sse2.UnpackHigh(vsrcL.AsByte(), vsrcH.AsByte());
Vector128<byte> vsrcUnpck21 = Sse2.UnpackLow(vsrcUnpck11, vsrcUnpck12);
Vector128<byte> vsrcUnpck22 = Sse2.UnpackHigh(vsrcUnpck11, vsrcUnpck12);
Vector128<byte> vsrc01 = Sse2.UnpackLow(vsrcUnpck21, vsrcUnpck22);
Vector128<byte> vsrc23 = Sse2.UnpackHigh(vsrcUnpck21, vsrcUnpck22);
Vector128<byte> vsrc11 = Sse.MoveHighToLow(vsrc01.AsSingle(), vsrc01.AsSingle()).AsByte();
Vector128<byte> vsrc33 = Sse.MoveHighToLow(vsrc23.AsSingle(), vsrc23.AsSingle()).AsByte();
Vector128<short> vsrc0 = Sse41.ConvertToVector128Int16(vsrc01);
Vector128<short> vsrc1 = Sse41.ConvertToVector128Int16(vsrc11);
Vector128<short> vsrc2 = Sse41.ConvertToVector128Int16(vsrc23);
Vector128<short> vsrc3 = Sse41.ConvertToVector128Int16(vsrc33);
Vector128<int> sum0123 = MultiplyAddAdjacent(vsrc0, vsrc1, vsrc2, vsrc3, vfilter, zero);
Sse.StoreScalar((float*)&dst[x], PackUnsignedSaturate(RoundShift(sum0123, const64), zero).AsSingle());
}
dst += dstStride;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe void ConvolveVert(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] yFilters,
int y0Q4,
int yStepQ4,
int w,
int h)
{
if (Avx2.IsSupported && UseIntrinsics && yStepQ4 == 1 << SubpelBits)
{
ConvolveVertAvx2(src, srcStride, dst, dstStride, yFilters, y0Q4, w, h);
return;
}
int x, y;
src -= srcStride * (SubpelTaps / 2 - 1);
for (x = 0; x < w; ++x)
{
int yQ4 = y0Q4;
for (y = 0; y < h; ++y)
{
byte* srcY = &src[(yQ4 >> SubpelBits) * srcStride];
ref Array8<short> yFilter = ref yFilters[yQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcY[k * srcStride] * yFilter[k];
}
dst[y * dstStride] = BitUtils.ClipPixel(BitUtils.RoundPowerOfTwo(sum, FilterBits));
yQ4 += yStepQ4;
}
++src;
++dst;
}
}
private static unsafe void ConvolveAvgVert(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] yFilters,
int y0Q4,
int yStepQ4,
int w,
int h)
{
int x, y;
src -= srcStride * (SubpelTaps / 2 - 1);
for (x = 0; x < w; ++x)
{
int yQ4 = y0Q4;
for (y = 0; y < h; ++y)
{
byte* srcY = &src[(yQ4 >> SubpelBits) * srcStride];
ref Array8<short> yFilter = ref yFilters[yQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcY[k * srcStride] * yFilter[k];
}
dst[y * dstStride] = (byte)BitUtils.RoundPowerOfTwo(
dst[y * dstStride] + BitUtils.ClipPixel(BitUtils.RoundPowerOfTwo(sum, FilterBits)), 1);
yQ4 += yStepQ4;
}
++src;
++dst;
}
}
public static unsafe void Convolve8Horiz(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
ConvolveHoriz(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, w, h);
}
public static unsafe void Convolve8AvgHoriz(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
ConvolveAvgHoriz(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, w, h);
}
public static unsafe void Convolve8Vert(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
ConvolveVert(src, srcStride, dst, dstStride, filter, y0Q4, yStepQ4, w, h);
}
public static unsafe void Convolve8AvgVert(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
ConvolveAvgVert(src, srcStride, dst, dstStride, filter, y0Q4, yStepQ4, w, h);
}
[StructLayout(LayoutKind.Sequential, Size = 64 * 135)]
struct Temp
{
}
public static unsafe void Convolve8(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
// Note: Fixed size intermediate buffer, temp, places limits on parameters.
// 2d filtering proceeds in 2 steps:
// (1) Interpolate horizontally into an intermediate buffer, temp.
// (2) Interpolate temp vertically to derive the sub-pixel result.
// Deriving the maximum number of rows in the temp buffer (135):
// --Smallest scaling factor is x1/2 ==> yStepQ4 = 32 (Normative).
// --Largest block size is 64x64 pixels.
// --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
// original frame (in 1/16th pixel units).
// --Must round-up because block may be located at sub-pixel position.
// --Require an additional SubpelTaps rows for the 8-tap filter tails.
// --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
// When calling in frame scaling function, the smallest scaling factor is x1/4
// ==> yStepQ4 = 64. Since w and h are at most 16, the temp buffer is still
// big enough.
Temp tempStruct;
byte* temp = (byte*)Unsafe.AsPointer(ref tempStruct); // Avoid zero initialization.
int intermediateHeight = (((h - 1) * yStepQ4 + y0Q4) >> SubpelBits) + SubpelTaps;
Debug.Assert(w <= 64);
Debug.Assert(h <= 64);
Debug.Assert(yStepQ4 <= 32 || (yStepQ4 <= 64 && h <= 32));
Debug.Assert(xStepQ4 <= 64);
ConvolveHoriz(src - srcStride * (SubpelTaps / 2 - 1), srcStride, temp, 64, filter, x0Q4, xStepQ4, w, intermediateHeight);
ConvolveVert(temp + 64 * (SubpelTaps / 2 - 1), 64, dst, dstStride, filter, y0Q4, yStepQ4, w, h);
}
public static unsafe void Convolve8Avg(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
// Fixed size intermediate buffer places limits on parameters.
byte* temp = stackalloc byte[64 * 64];
Debug.Assert(w <= 64);
Debug.Assert(h <= 64);
Convolve8(src, srcStride, temp, 64, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
ConvolveAvg(temp, 64, dst, dstStride, null, 0, 0, 0, 0, w, h);
}
public static unsafe void ConvolveCopy(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
int r;
for (r = h; r > 0; --r)
{
MemoryUtil.Copy(dst, src, w);
src += srcStride;
dst += dstStride;
}
}
public static unsafe void ConvolveAvg(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
int x, y;
for (y = 0; y < h; ++y)
{
for (x = 0; x < w; ++x)
{
dst[x] = (byte)BitUtils.RoundPowerOfTwo(dst[x] + src[x], 1);
}
src += srcStride;
dst += dstStride;
}
}
public static unsafe void ScaledHoriz(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
Convolve8Horiz(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
}
public static unsafe void ScaledVert(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
Convolve8Vert(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
}
public static unsafe void Scaled2D(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
Convolve8(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
}
public static unsafe void ScaledAvgHoriz(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
Convolve8AvgHoriz(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
}
public static unsafe void ScaledAvgVert(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
Convolve8AvgVert(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
}
public static unsafe void ScaledAvg2D(
byte* src,
int srcStride,
byte* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h)
{
Convolve8Avg(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h);
}
private static unsafe void HighbdConvolveHoriz(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] xFilters,
int x0Q4,
int xStepQ4,
int w,
int h,
int bd)
{
int x, y;
src -= SubpelTaps / 2 - 1;
for (y = 0; y < h; ++y)
{
int xQ4 = x0Q4;
for (x = 0; x < w; ++x)
{
ushort* srcX = &src[xQ4 >> SubpelBits];
ref Array8<short> xFilter = ref xFilters[xQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcX[k] * xFilter[k];
}
dst[x] = BitUtils.ClipPixelHighbd(BitUtils.RoundPowerOfTwo(sum, FilterBits), bd);
xQ4 += xStepQ4;
}
src += srcStride;
dst += dstStride;
}
}
private static unsafe void HighbdConvolveAvgHoriz(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] xFilters,
int x0Q4,
int xStepQ4,
int w,
int h,
int bd)
{
int x, y;
src -= SubpelTaps / 2 - 1;
for (y = 0; y < h; ++y)
{
int xQ4 = x0Q4;
for (x = 0; x < w; ++x)
{
ushort* srcX = &src[xQ4 >> SubpelBits];
ref Array8<short> xFilter = ref xFilters[xQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcX[k] * xFilter[k];
}
dst[x] = (ushort)BitUtils.RoundPowerOfTwo(dst[x] + BitUtils.ClipPixelHighbd(BitUtils.RoundPowerOfTwo(sum, FilterBits), bd), 1);
xQ4 += xStepQ4;
}
src += srcStride;
dst += dstStride;
}
}
private static unsafe void HighbdConvolveVert(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] yFilters,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
int x, y;
src -= srcStride * (SubpelTaps / 2 - 1);
for (x = 0; x < w; ++x)
{
int yQ4 = y0Q4;
for (y = 0; y < h; ++y)
{
ushort* srcY = &src[(yQ4 >> SubpelBits) * srcStride];
ref Array8<short> yFilter = ref yFilters[yQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcY[k * srcStride] * yFilter[k];
}
dst[y * dstStride] = BitUtils.ClipPixelHighbd(BitUtils.RoundPowerOfTwo(sum, FilterBits), bd);
yQ4 += yStepQ4;
}
++src;
++dst;
}
}
private static unsafe void HighConvolveAvgVert(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] yFilters,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
int x, y;
src -= srcStride * (SubpelTaps / 2 - 1);
for (x = 0; x < w; ++x)
{
int yQ4 = y0Q4;
for (y = 0; y < h; ++y)
{
ushort* srcY = &src[(yQ4 >> SubpelBits) * srcStride];
ref Array8<short> yFilter = ref yFilters[yQ4 & SubpelMask];
int k, sum = 0;
for (k = 0; k < SubpelTaps; ++k)
{
sum += srcY[k * srcStride] * yFilter[k];
}
dst[y * dstStride] = (ushort)BitUtils.RoundPowerOfTwo(
dst[y * dstStride] + BitUtils.ClipPixelHighbd(BitUtils.RoundPowerOfTwo(sum, FilterBits), bd), 1);
yQ4 += yStepQ4;
}
++src;
++dst;
}
}
private static unsafe void HighbdConvolve(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
// Note: Fixed size intermediate buffer, temp, places limits on parameters.
// 2d filtering proceeds in 2 steps:
// (1) Interpolate horizontally into an intermediate buffer, temp.
// (2) Interpolate temp vertically to derive the sub-pixel result.
// Deriving the maximum number of rows in the temp buffer (135):
// --Smallest scaling factor is x1/2 ==> yStepQ4 = 32 (Normative).
// --Largest block size is 64x64 pixels.
// --64 rows in the downscaled frame span a distance of (64 - 1) * 32 in the
// original frame (in 1/16th pixel units).
// --Must round-up because block may be located at sub-pixel position.
// --Require an additional SubpelTaps rows for the 8-tap filter tails.
// --((64 - 1) * 32 + 15) >> 4 + 8 = 135.
ushort* temp = stackalloc ushort[64 * 135];
int intermediateHeight = (((h - 1) * yStepQ4 + y0Q4) >> SubpelBits) + SubpelTaps;
Debug.Assert(w <= 64);
Debug.Assert(h <= 64);
Debug.Assert(yStepQ4 <= 32);
Debug.Assert(xStepQ4 <= 32);
HighbdConvolveHoriz(src - srcStride * (SubpelTaps / 2 - 1), srcStride, temp, 64, filter, x0Q4, xStepQ4, w, intermediateHeight, bd);
HighbdConvolveVert(temp + 64 * (SubpelTaps / 2 - 1), 64, dst, dstStride, filter, y0Q4, yStepQ4, w, h, bd);
}
public static unsafe void HighbdConvolve8Horiz(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
HighbdConvolveHoriz(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, w, h, bd);
}
public static unsafe void HighbdConvolve8AvgHoriz(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
HighbdConvolveAvgHoriz(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, w, h, bd);
}
public static unsafe void HighbdConvolve8Vert(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
HighbdConvolveVert(src, srcStride, dst, dstStride, filter, y0Q4, yStepQ4, w, h, bd);
}
public static unsafe void HighbdConvolve8AvgVert(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
HighConvolveAvgVert(src, srcStride, dst, dstStride, filter, y0Q4, yStepQ4, w, h, bd);
}
public static unsafe void HighbdConvolve8(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
HighbdConvolve(src, srcStride, dst, dstStride, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h, bd);
}
public static unsafe void HighbdConvolve8Avg(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
// Fixed size intermediate buffer places limits on parameters.
ushort* temp = stackalloc ushort[64 * 64];
Debug.Assert(w <= 64);
Debug.Assert(h <= 64);
HighbdConvolve8(src, srcStride, temp, 64, filter, x0Q4, xStepQ4, y0Q4, yStepQ4, w, h, bd);
HighbdConvolveAvg(temp, 64, dst, dstStride, null, 0, 0, 0, 0, w, h, bd);
}
public static unsafe void HighbdConvolveCopy(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
int r;
for (r = h; r > 0; --r)
{
MemoryUtil.Copy(dst, src, w);
src += srcStride;
dst += dstStride;
}
}
public static unsafe void HighbdConvolveAvg(
ushort* src,
int srcStride,
ushort* dst,
int dstStride,
Array8<short>[] filter,
int x0Q4,
int xStepQ4,
int y0Q4,
int yStepQ4,
int w,
int h,
int bd)
{
int x, y;
for (y = 0; y < h; ++y)
{
for (x = 0; x < w; ++x)
{
dst[x] = (ushort)BitUtils.RoundPowerOfTwo(dst[x] + src[x], 1);
}
src += srcStride;
dst += dstStride;
}
}
}
}