Return mapped buffer pointer directly for flush, WriteableRegion for textures (#2494)

* Return mapped buffer pointer directly for flush, WriteableRegion for textures

A few changes here to generally improve performance, even for platforms not using the persistent buffer flush.

- Texture and buffer flush now return a ReadOnlySpan<byte>. It's guaranteed that this span is pinned in memory, but it will be overwritten on the next flush from that thread, so it is expected that the data is used before calling again.
- As a result, persistent mappings no longer copy to a new array - rather the persistent map is returned directly as a Span<>. A similar host array is used for the glGet flushes instead of allocating new arrays each time.
- Texture flushes now do their layout conversion into a WriteableRegion when the texture is not MultiRange, which allows the flush to happen directly into guest memory rather than into a temporary span, then copied over. This avoids another copy when doing layout conversion.

Overall, this saves 1 data copy for buffer flush, 1 copy for linear textures with matching source/target stride, and 2 copies for block textures or linear textures with mismatching strides.

* Fix tests

* Fix array pointer for Mesa/Intel path

* Address some feedback

* Update method for getting array pointer.
This commit is contained in:
riperiperi 2021-07-19 23:10:54 +01:00 committed by GitHub
parent 10e17ab423
commit 4b60371e64
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 143 additions and 66 deletions

View file

@ -237,7 +237,7 @@ namespace Ryujinx.Cpu
} }
/// <inheritdoc/> /// <inheritdoc/>
public unsafe WritableRegion GetWritableRegion(ulong va, int size) public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{ {
if (size == 0) if (size == 0)
{ {
@ -246,6 +246,11 @@ namespace Ryujinx.Cpu
if (IsContiguousAndMapped(va, size)) if (IsContiguousAndMapped(va, size))
{ {
if (tracked)
{
SignalMemoryTracking(va, (ulong)size, true);
}
return new WritableRegion(null, va, new NativeMemoryManager<byte>((byte*)GetHostAddress(va), size).Memory); return new WritableRegion(null, va, new NativeMemoryManager<byte>((byte*)GetHostAddress(va), size).Memory);
} }
else else
@ -254,7 +259,7 @@ namespace Ryujinx.Cpu
GetSpan(va, size).CopyTo(memory.Span); GetSpan(va, size).CopyTo(memory.Span);
return new WritableRegion(this, va, memory); return new WritableRegion(this, va, memory, tracked);
} }
} }

View file

@ -285,9 +285,16 @@ namespace Ryujinx.Cpu
} }
/// <inheritdoc/> /// <inheritdoc/>
public WritableRegion GetWritableRegion(ulong va, int size) public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{
if (tracked)
{
SignalMemoryTracking(va, (ulong)size, true);
}
else
{ {
AssertMapped(va, (ulong)size); AssertMapped(va, (ulong)size);
}
return _addressSpaceMirror.GetWritableRegion(va, size); return _addressSpaceMirror.GetWritableRegion(va, size);
} }

View file

@ -27,7 +27,7 @@ namespace Ryujinx.Graphics.GAL
void DeleteBuffer(BufferHandle buffer); void DeleteBuffer(BufferHandle buffer);
byte[] GetBufferData(BufferHandle buffer, int offset, int size); ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size);
Capabilities GetCapabilities(); Capabilities GetCapabilities();

View file

@ -14,7 +14,7 @@ namespace Ryujinx.Graphics.GAL
ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel); ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel);
byte[] GetData(); ReadOnlySpan<byte> GetData();
void SetData(ReadOnlySpan<byte> data); void SetData(ReadOnlySpan<byte> data);
void SetData(ReadOnlySpan<byte> data, int layer, int level); void SetData(ReadOnlySpan<byte> data, int layer, int level);

View file

@ -4,6 +4,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Gpu.Memory;
using Ryujinx.Graphics.Texture; using Ryujinx.Graphics.Texture;
using Ryujinx.Graphics.Texture.Astc; using Ryujinx.Graphics.Texture.Astc;
using Ryujinx.Memory;
using Ryujinx.Memory.Range; using Ryujinx.Memory.Range;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
@ -821,14 +822,7 @@ namespace Ryujinx.Graphics.Gpu.Image
return; // Flushing this format is not supported, as it may have been converted to another host format. return; // Flushing this format is not supported, as it may have been converted to another host format.
} }
if (tracked) FlushTextureDataToGuest(tracked);
{
_physicalMemory.Write(Range, GetTextureDataFromGpu(tracked));
}
else
{
_physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(tracked));
}
} }
/// <summary> /// <summary>
@ -864,10 +858,44 @@ namespace Ryujinx.Graphics.Gpu.Image
texture = _flushHostTexture = GetScaledHostTexture(1f, _flushHostTexture); texture = _flushHostTexture = GetScaledHostTexture(1f, _flushHostTexture);
} }
_physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(false, texture)); FlushTextureDataToGuest(false, texture);
}); });
} }
/// <summary>
/// Gets data from the host GPU, and flushes it to guest memory.
/// </summary>
/// <remarks>
/// This method should be used to retrieve data that was modified by the host GPU.
/// This is not cheap, avoid doing that unless strictly needed.
/// When possible, the data is written directly into guest memory, rather than copied.
/// </remarks>
/// <param name="tracked">True if writing the texture data is tracked, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
private void FlushTextureDataToGuest(bool tracked, ITexture texture = null)
{
if (Range.Count == 1)
{
MemoryRange subrange = Range.GetSubRange(0);
using (WritableRegion region = _physicalMemory.GetWritableRegion(subrange.Address, (int)subrange.Size, tracked))
{
GetTextureDataFromGpu(region.Memory.Span, tracked, texture);
}
}
else
{
if (tracked)
{
_physicalMemory.Write(Range, GetTextureDataFromGpu(Span<byte>.Empty, true, texture));
}
else
{
_physicalMemory.WriteUntracked(Range, GetTextureDataFromGpu(Span<byte>.Empty, false, texture));
}
}
}
/// <summary> /// <summary>
/// Gets data from the host GPU. /// Gets data from the host GPU.
/// </summary> /// </summary>
@ -875,8 +903,11 @@ namespace Ryujinx.Graphics.Gpu.Image
/// This method should be used to retrieve data that was modified by the host GPU. /// This method should be used to retrieve data that was modified by the host GPU.
/// This is not cheap, avoid doing that unless strictly needed. /// This is not cheap, avoid doing that unless strictly needed.
/// </remarks> /// </remarks>
/// <returns>Host texture data</returns> /// <param name="output">An output span to place the texture data into. If empty, one is generated</param>
private ReadOnlySpan<byte> GetTextureDataFromGpu(bool blacklist, ITexture texture = null) /// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param>
/// <param name="texture">The specific host texture to flush. Defaults to this texture</param>
/// <returns>The span containing the texture data</returns>
private ReadOnlySpan<byte> GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null)
{ {
ReadOnlySpan<byte> data; ReadOnlySpan<byte> data;
@ -909,6 +940,7 @@ namespace Ryujinx.Graphics.Gpu.Image
if (Info.IsLinear) if (Info.IsLinear)
{ {
data = LayoutConverter.ConvertLinearToLinearStrided( data = LayoutConverter.ConvertLinearToLinearStrided(
output,
Info.Width, Info.Width,
Info.Height, Info.Height,
Info.FormatInfo.BlockWidth, Info.FormatInfo.BlockWidth,
@ -920,6 +952,7 @@ namespace Ryujinx.Graphics.Gpu.Image
else else
{ {
data = LayoutConverter.ConvertLinearToBlockLinear( data = LayoutConverter.ConvertLinearToBlockLinear(
output,
Info.Width, Info.Width,
Info.Height, Info.Height,
_depth, _depth,

View file

@ -412,7 +412,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
{ {
int offset = (int)(address - Address); int offset = (int)(address - Address);
byte[] data = _context.Renderer.GetBufferData(Handle, offset, (int)size); ReadOnlySpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
// TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
_physicalMemory.WriteUntracked(address, data); _physicalMemory.WriteUntracked(address, data);

View file

@ -128,10 +128,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary> /// </summary>
/// <param name="address">Start address of the range</param> /// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes to be range</param> /// <param name="size">Size in bytes to be range</param>
/// <param name="tracked">True if write tracking is triggered on the span</param>
/// <returns>A writable region with the data at the specified memory location</returns> /// <returns>A writable region with the data at the specified memory location</returns>
public WritableRegion GetWritableRegion(ulong address, int size) public WritableRegion GetWritableRegion(ulong address, int size, bool tracked = false)
{ {
return _cpuMemory.GetWritableRegion(address, size); return _cpuMemory.GetWritableRegion(address, size, tracked);
} }
/// <summary> /// <summary>

View file

@ -55,15 +55,22 @@ namespace Ryujinx.Graphics.OpenGL
(IntPtr)size); (IntPtr)size);
} }
public static byte[] GetData(BufferHandle buffer, int offset, int size) public static unsafe ReadOnlySpan<byte> GetData(Renderer renderer, BufferHandle buffer, int offset, int size)
{ {
if (HwCapabilities.UsePersistentBufferForFlush)
{
return renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size);
}
else
{
IntPtr target = renderer.PersistentBuffers.Default.GetHostArray(size);
GL.BindBuffer(BufferTarget.CopyReadBuffer, buffer.ToInt32()); GL.BindBuffer(BufferTarget.CopyReadBuffer, buffer.ToInt32());
byte[] data = new byte[size]; GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target);
GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, data); return new ReadOnlySpan<byte>(target.ToPointer(), size);
}
return data;
} }
public static void Resize(BufferHandle handle, int size) public static void Resize(BufferHandle handle, int size)

View file

@ -38,9 +38,9 @@ namespace Ryujinx.Graphics.OpenGL.Image
throw new NotSupportedException(); throw new NotSupportedException();
} }
public byte[] GetData() public ReadOnlySpan<byte> GetData()
{ {
return Buffer.GetData(_buffer, _bufferOffset, _bufferSize); return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize);
} }
public void SetData(ReadOnlySpan<byte> data) public void SetData(ReadOnlySpan<byte> data)

View file

@ -119,7 +119,7 @@ namespace Ryujinx.Graphics.OpenGL.Image
_renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter); _renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter);
} }
public byte[] GetData() public unsafe ReadOnlySpan<byte> GetData()
{ {
int size = 0; int size = 0;
@ -134,17 +134,11 @@ namespace Ryujinx.Graphics.OpenGL.Image
} }
else else
{ {
byte[] data = new byte[size]; IntPtr target = _renderer.PersistentBuffers.Default.GetHostArray(size);
unsafe WriteTo(target);
{
fixed (byte* ptr = data)
{
WriteTo((IntPtr)ptr);
}
}
return data; return new ReadOnlySpan<byte>(target.ToPointer(), size);
} }
} }

View file

@ -1,4 +1,5 @@
using System; using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using OpenTK.Graphics.OpenGL; using OpenTK.Graphics.OpenGL;
using Ryujinx.Common.Logging; using Ryujinx.Common.Logging;
@ -27,6 +28,9 @@ namespace Ryujinx.Graphics.OpenGL
private int _copyBufferHandle; private int _copyBufferHandle;
private int _copyBufferSize; private int _copyBufferSize;
private byte[] _data;
private IntPtr _dataMap;
private void EnsureBuffer(int requiredSize) private void EnsureBuffer(int requiredSize)
{ {
if (_copyBufferSize < requiredSize && _copyBufferHandle != 0) if (_copyBufferSize < requiredSize && _copyBufferHandle != 0)
@ -48,6 +52,18 @@ namespace Ryujinx.Graphics.OpenGL
} }
} }
public unsafe IntPtr GetHostArray(int requiredSize)
{
if (_data == null || _data.Length < requiredSize)
{
_data = GC.AllocateUninitializedArray<byte>(requiredSize, true);
_dataMap = (IntPtr)Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(_data));
}
return _dataMap;
}
private void Sync() private void Sync()
{ {
GL.MemoryBarrier(MemoryBarrierFlags.ClientMappedBufferBarrierBit); GL.MemoryBarrier(MemoryBarrierFlags.ClientMappedBufferBarrierBit);
@ -63,7 +79,7 @@ namespace Ryujinx.Graphics.OpenGL
GL.DeleteSync(sync); GL.DeleteSync(sync);
} }
public byte[] GetTextureData(TextureView view, int size) public unsafe ReadOnlySpan<byte> GetTextureData(TextureView view, int size)
{ {
EnsureBuffer(size); EnsureBuffer(size);
@ -73,16 +89,12 @@ namespace Ryujinx.Graphics.OpenGL
GL.BindBuffer(BufferTarget.PixelPackBuffer, 0); GL.BindBuffer(BufferTarget.PixelPackBuffer, 0);
byte[] data = new byte[size];
Sync(); Sync();
Marshal.Copy(_bufferMap, data, 0, size); return new ReadOnlySpan<byte>(_bufferMap.ToPointer(), size);
return data;
} }
public byte[] GetBufferData(BufferHandle buffer, int offset, int size) public unsafe ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{ {
EnsureBuffer(size); EnsureBuffer(size);
@ -93,13 +105,9 @@ namespace Ryujinx.Graphics.OpenGL
GL.BindBuffer(BufferTarget.CopyWriteBuffer, 0); GL.BindBuffer(BufferTarget.CopyWriteBuffer, 0);
byte[] data = new byte[size];
Sync(); Sync();
Marshal.Copy(_bufferMap, data, 0, size); return new ReadOnlySpan<byte>(_bufferMap.ToPointer(), size);
return data;
} }
public void Dispose() public void Dispose()

View file

@ -91,16 +91,9 @@ namespace Ryujinx.Graphics.OpenGL
Buffer.Delete(buffer); Buffer.Delete(buffer);
} }
public byte[] GetBufferData(BufferHandle buffer, int offset, int size) public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size)
{ {
if (HwCapabilities.UsePersistentBufferForFlush) return Buffer.GetData(this, buffer, offset, size);
{
return PersistentBuffers.Default.GetBufferData(buffer, offset, size);
}
else
{
return Buffer.GetData(buffer, offset, size);
}
} }
public Capabilities GetCapabilities() public Capabilities GetCapabilities()

View file

@ -359,6 +359,7 @@ namespace Ryujinx.Graphics.Texture
} }
public static ReadOnlySpan<byte> ConvertLinearToBlockLinear( public static ReadOnlySpan<byte> ConvertLinearToBlockLinear(
Span<byte> output,
int width, int width,
int height, int height,
int depth, int depth,
@ -373,7 +374,10 @@ namespace Ryujinx.Graphics.Texture
SizeInfo sizeInfo, SizeInfo sizeInfo,
ReadOnlySpan<byte> data) ReadOnlySpan<byte> data)
{ {
Span<byte> output = new byte[sizeInfo.TotalSize]; if (output.Length == 0)
{
output = new byte[sizeInfo.TotalSize];
}
int inOffs = 0; int inOffs = 0;
@ -500,6 +504,7 @@ namespace Ryujinx.Graphics.Texture
} }
public static ReadOnlySpan<byte> ConvertLinearToLinearStrided( public static ReadOnlySpan<byte> ConvertLinearToLinearStrided(
Span<byte> output,
int width, int width,
int height, int height,
int blockWidth, int blockWidth,
@ -515,11 +520,22 @@ namespace Ryujinx.Graphics.Texture
int lineSize = width * bytesPerPixel; int lineSize = width * bytesPerPixel;
if (inStride == stride) if (inStride == stride)
{
if (output.Length != 0)
{
data.CopyTo(output);
return output;
}
else
{ {
return data; return data;
} }
}
Span<byte> output = new byte[h * stride]; if (output.Length == 0)
{
output = new byte[h * stride];
}
int inOffs = 0; int inOffs = 0;
int outOffs = 0; int outOffs = 0;

View file

@ -49,7 +49,7 @@ namespace Ryujinx.Memory.Tests
throw new NotImplementedException(); throw new NotImplementedException();
} }
public WritableRegion GetWritableRegion(ulong va, int size) public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{ {
throw new NotImplementedException(); throw new NotImplementedException();
} }

View file

@ -207,9 +207,10 @@ namespace Ryujinx.Memory
/// </remarks> /// </remarks>
/// <param name="va">Virtual address of the data</param> /// <param name="va">Virtual address of the data</param>
/// <param name="size">Size of the data</param> /// <param name="size">Size of the data</param>
/// <param name="tracked">True if write tracking is triggered on the span</param>
/// <returns>A writable region of memory containing the data</returns> /// <returns>A writable region of memory containing the data</returns>
/// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception> /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
public unsafe WritableRegion GetWritableRegion(ulong va, int size) public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{ {
if (size == 0) if (size == 0)
{ {

View file

@ -87,9 +87,10 @@ namespace Ryujinx.Memory
/// </summary> /// </summary>
/// <param name="va">Virtual address of the data</param> /// <param name="va">Virtual address of the data</param>
/// <param name="size">Size of the data</param> /// <param name="size">Size of the data</param>
/// <param name="tracked">True if write tracking is triggered on the span</param>
/// <returns>A writable region of memory containing the data</returns> /// <returns>A writable region of memory containing the data</returns>
/// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception> /// <exception cref="InvalidMemoryRegionException">Throw for unhandled invalid or unmapped memory accesses</exception>
WritableRegion GetWritableRegion(ulong va, int size); WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false);
/// <summary> /// <summary>
/// Gets a reference for the given type at the specified virtual memory address. /// Gets a reference for the given type at the specified virtual memory address.

View file

@ -5,5 +5,7 @@ namespace Ryujinx.Memory
public interface IWritableBlock public interface IWritableBlock
{ {
void Write(ulong va, ReadOnlySpan<byte> data); void Write(ulong va, ReadOnlySpan<byte> data);
void WriteUntracked(ulong va, ReadOnlySpan<byte> data) => Write(va, data);
} }
} }

View file

@ -6,24 +6,33 @@ namespace Ryujinx.Memory
{ {
private readonly IWritableBlock _block; private readonly IWritableBlock _block;
private readonly ulong _va; private readonly ulong _va;
private readonly bool _tracked;
private bool NeedsWriteback => _block != null; private bool NeedsWriteback => _block != null;
public Memory<byte> Memory { get; } public Memory<byte> Memory { get; }
public WritableRegion(IWritableBlock block, ulong va, Memory<byte> memory) public WritableRegion(IWritableBlock block, ulong va, Memory<byte> memory, bool tracked = false)
{ {
_block = block; _block = block;
_va = va; _va = va;
_tracked = tracked;
Memory = memory; Memory = memory;
} }
public void Dispose() public void Dispose()
{ {
if (NeedsWriteback) if (NeedsWriteback)
{
if (_tracked)
{ {
_block.Write(_va, Memory.Span); _block.Write(_va, Memory.Span);
} }
else
{
_block.WriteUntracked(_va, Memory.Span);
}
}
} }
} }
} }