diff --git a/src/Ryujinx.Graphics.GAL/BufferAccess.cs b/src/Ryujinx.Graphics.GAL/BufferAccess.cs new file mode 100644 index 000000000..3a2d6c9b6 --- /dev/null +++ b/src/Ryujinx.Graphics.GAL/BufferAccess.cs @@ -0,0 +1,8 @@ +namespace Ryujinx.Graphics.GAL +{ + public enum BufferAccess + { + Default, + FlushPersistent + } +} diff --git a/src/Ryujinx.Graphics.GAL/IRenderer.cs b/src/Ryujinx.Graphics.GAL/IRenderer.cs index 2af7b5db7..d36dd26b6 100644 --- a/src/Ryujinx.Graphics.GAL/IRenderer.cs +++ b/src/Ryujinx.Graphics.GAL/IRenderer.cs @@ -21,11 +21,14 @@ namespace Ryujinx.Graphics.GAL { return CreateBuffer(size, BufferHandle.Null); } + BufferHandle CreateBuffer(nint pointer, int size); + BufferHandle CreateBuffer(int size, BufferAccess access); IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info); ISampler CreateSampler(SamplerCreateInfo info); ITexture CreateTexture(TextureCreateInfo info, float scale); + bool PrepareHostMapping(nint address, ulong size); void CreateSync(ulong id, bool strict); diff --git a/src/Ryujinx.Graphics.GAL/ITexture.cs b/src/Ryujinx.Graphics.GAL/ITexture.cs index 792c863cb..2f9f5fbff 100644 --- a/src/Ryujinx.Graphics.GAL/ITexture.cs +++ b/src/Ryujinx.Graphics.GAL/ITexture.cs @@ -12,6 +12,7 @@ namespace Ryujinx.Graphics.GAL void CopyTo(ITexture destination, int firstLayer, int firstLevel); void CopyTo(ITexture destination, int srcLayer, int dstLayer, int srcLevel, int dstLevel); void CopyTo(ITexture destination, Extents2D srcRegion, Extents2D dstRegion, bool linearFilter); + void CopyTo(BufferRange range, int layer, int level, int stride); ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel); diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs b/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs index 063b7edf9..9f6e483cd 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs @@ -43,6 +43,8 @@ namespace Ryujinx.Graphics.GAL.Multithreading Register(CommandType.Action); Register(CommandType.CreateBuffer); + Register(CommandType.CreateBufferAccess); + Register(CommandType.CreateHostBuffer); Register(CommandType.CreateProgram); Register(CommandType.CreateSampler); Register(CommandType.CreateSync); @@ -69,6 +71,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading Register(CommandType.TextureCopyTo); Register(CommandType.TextureCopyToScaled); Register(CommandType.TextureCopyToSlice); + Register(CommandType.TextureCopyToBuffer); Register(CommandType.TextureCreateView); Register(CommandType.TextureGetData); Register(CommandType.TextureGetDataSlice); diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs b/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs index 61e729b44..8d9c1ec8d 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs @@ -4,6 +4,8 @@ { Action, CreateBuffer, + CreateBufferAccess, + CreateHostBuffer, CreateProgram, CreateSampler, CreateSync, @@ -29,6 +31,7 @@ SamplerDispose, TextureCopyTo, + TextureCopyToBuffer, TextureCopyToScaled, TextureCopyToSlice, TextureCreateView, diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferAccessCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferAccessCommand.cs new file mode 100644 index 000000000..ece98b70f --- /dev/null +++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferAccessCommand.cs @@ -0,0 +1,22 @@ +namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer +{ + struct CreateBufferAccessCommand : IGALCommand, IGALCommand + { + public CommandType CommandType => CommandType.CreateBufferAccess; + private BufferHandle _threadedHandle; + private int _size; + private BufferAccess _access; + + public void Set(BufferHandle threadedHandle, int size, BufferAccess access) + { + _threadedHandle = threadedHandle; + _size = size; + _access = access; + } + + public static void Run(ref CreateBufferAccessCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, command._access)); + } + } +} diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateHostBufferCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateHostBufferCommand.cs new file mode 100644 index 000000000..e25f84687 --- /dev/null +++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateHostBufferCommand.cs @@ -0,0 +1,22 @@ +namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer +{ + struct CreateHostBufferCommand : IGALCommand, IGALCommand + { + public CommandType CommandType => CommandType.CreateHostBuffer; + private BufferHandle _threadedHandle; + private nint _pointer; + private int _size; + + public void Set(BufferHandle threadedHandle, nint pointer, int size) + { + _threadedHandle = threadedHandle; + _pointer = pointer; + _size = size; + } + + public static void Run(ref CreateHostBufferCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._pointer, command._size)); + } + } +} diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureCopyToBufferCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureCopyToBufferCommand.cs new file mode 100644 index 000000000..ac0e07d67 --- /dev/null +++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureCopyToBufferCommand.cs @@ -0,0 +1,29 @@ +using Ryujinx.Graphics.GAL.Multithreading.Model; +using Ryujinx.Graphics.GAL.Multithreading.Resources; + +namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture +{ + struct TextureCopyToBufferCommand : IGALCommand, IGALCommand + { + public CommandType CommandType => CommandType.TextureCopyToBuffer; + private TableRef _texture; + private BufferRange _range; + private int _layer; + private int _level; + private int _stride; + + public void Set(TableRef texture, BufferRange range, int layer, int level, int stride) + { + _texture = texture; + _range = range; + _layer = layer; + _level = level; + _stride = stride; + } + + public static void Run(ref TextureCopyToBufferCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + command._texture.Get(threaded).Base.CopyTo(threaded.Buffers.MapBufferRange(command._range), command._layer, command._level, command._stride); + } + } +} diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs index ee1cfa29b..bb0b05fb8 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs @@ -108,6 +108,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources } } + public void CopyTo(BufferRange range, int layer, int level, int stride) + { + _renderer.New().Set(Ref(this), range, layer, level, stride); + _renderer.QueueCommand(); + } + public void SetData(SpanOrArray data) { _renderer.New().Set(Ref(this), Ref(data.ToArray())); diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs index 2148f43f9..3e179621e 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs @@ -57,6 +57,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading private int _refConsumerPtr; private Action _interruptAction; + private object _interruptLock = new(); public event EventHandler ScreenCaptured; @@ -274,6 +275,24 @@ namespace Ryujinx.Graphics.GAL.Multithreading return handle; } + public BufferHandle CreateBuffer(nint pointer, int size) + { + BufferHandle handle = Buffers.CreateBufferHandle(); + New().Set(handle, pointer, size); + QueueCommand(); + + return handle; + } + + public BufferHandle CreateBuffer(int size, BufferAccess access) + { + BufferHandle handle = Buffers.CreateBufferHandle(); + New().Set(handle, size, access); + QueueCommand(); + + return handle; + } + public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info) { var program = new ThreadedProgram(this); @@ -448,11 +467,14 @@ namespace Ryujinx.Graphics.GAL.Multithreading } else { - while (Interlocked.CompareExchange(ref _interruptAction, action, null) != null) { } + lock (_interruptLock) + { + while (Interlocked.CompareExchange(ref _interruptAction, action, null) != null) { } - _galWorkAvailable.Set(); + _galWorkAvailable.Set(); - _interruptRun.WaitOne(); + _interruptRun.WaitOne(); + } } } @@ -461,6 +483,11 @@ namespace Ryujinx.Graphics.GAL.Multithreading // Threaded renderer ignores given interrupt action, as it provides its own to the child renderer. } + public bool PrepareHostMapping(nint address, ulong size) + { + return _baseRenderer.PrepareHostMapping(address, size); + } + public void Dispose() { // Dispose must happen from the render thread, after all commands have completed. diff --git a/src/Ryujinx.Graphics.Gpu/Engine/GPFifo/GPFifoClass.cs b/src/Ryujinx.Graphics.Gpu/Engine/GPFifo/GPFifoClass.cs index e80d98a15..7a11c649c 100644 --- a/src/Ryujinx.Graphics.Gpu/Engine/GPFifo/GPFifoClass.cs +++ b/src/Ryujinx.Graphics.Gpu/Engine/GPFifo/GPFifoClass.cs @@ -1,5 +1,6 @@ using Ryujinx.Graphics.Device; using Ryujinx.Graphics.Gpu.Engine.MME; +using Ryujinx.Graphics.Gpu.Synchronization; using System; using System.Collections.Generic; using System.Threading; @@ -59,7 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo if (_createSyncPending) { _createSyncPending = false; - _context.CreateHostSyncIfNeeded(false, false); + _context.CreateHostSyncIfNeeded(HostSyncFlags.None); } } @@ -157,7 +158,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo } else if (operation == SyncpointbOperation.Incr) { - _context.CreateHostSyncIfNeeded(true, true); + _context.CreateHostSyncIfNeeded(HostSyncFlags.StrictSyncpoint); _context.Synchronization.IncrementSyncpoint(syncpointId); } @@ -184,7 +185,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.GPFifo { _context.Renderer.Pipeline.CommandBufferBarrier(); - _context.CreateHostSyncIfNeeded(false, true); + _context.CreateHostSyncIfNeeded(HostSyncFlags.Strict); } /// diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs index caeee18e5..1386071cf 100644 --- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs +++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs @@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Engine.GPFifo; using Ryujinx.Graphics.Gpu.Engine.InlineToMemory; using Ryujinx.Graphics.Gpu.Engine.Threed.Blender; +using Ryujinx.Graphics.Gpu.Synchronization; using System; using System.Collections.Generic; using System.Runtime.CompilerServices; @@ -254,7 +255,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed uint syncpointId = (uint)argument & 0xFFFF; _context.AdvanceSequence(); - _context.CreateHostSyncIfNeeded(true, true); + _context.CreateHostSyncIfNeeded(HostSyncFlags.StrictSyncpoint); _context.Renderer.UpdateCounters(); // Poll the query counters, the game may want an updated result. _context.Synchronization.IncrementSyncpoint(syncpointId); } diff --git a/src/Ryujinx.Graphics.Gpu/GpuContext.cs b/src/Ryujinx.Graphics.Gpu/GpuContext.cs index 917588632..bab62b952 100644 --- a/src/Ryujinx.Graphics.Gpu/GpuContext.cs +++ b/src/Ryujinx.Graphics.Gpu/GpuContext.cs @@ -60,14 +60,14 @@ namespace Ryujinx.Graphics.Gpu /// If there are more than 0 items when this happens, a host sync object will be generated for the given , /// and the SyncNumber will be incremented. /// - internal List SyncActions { get; } + internal List SyncActions { get; } /// /// Actions to be performed when a CPU waiting syncpoint is triggered. /// If there are more than 0 items when this happens, a host sync object will be generated for the given , /// and the SyncNumber will be incremented. /// - internal List SyncpointActions { get; } + internal List SyncpointActions { get; } /// /// Buffer migrations that are currently in-flight. These are checked whenever sync is created to determine if buffer migration @@ -114,8 +114,8 @@ namespace Ryujinx.Graphics.Gpu HostInitalized = new ManualResetEvent(false); - SyncActions = new List(); - SyncpointActions = new List(); + SyncActions = new List(); + SyncpointActions = new List(); BufferMigrations = new List(); DeferredActions = new Queue(); @@ -296,9 +296,9 @@ namespace Ryujinx.Graphics.Gpu /// Registers an action to be performed the next time a syncpoint is incremented. /// This will also ensure a host sync object is created, and is incremented. /// - /// The action to be performed on sync object creation + /// The resource with action to be performed on sync object creation /// True if the sync action should only run when syncpoints are incremented - public void RegisterSyncAction(Action action, bool syncpointOnly = false) + internal void RegisterSyncAction(ISyncActionHandler action, bool syncpointOnly = false) { if (syncpointOnly) { @@ -315,10 +315,13 @@ namespace Ryujinx.Graphics.Gpu /// Creates a host sync object if there are any pending sync actions. The actions will then be called. /// If no actions are present, a host sync object is not created. /// - /// True if host sync is being created by a syncpoint - /// True if the sync should signal as soon as possible - public void CreateHostSyncIfNeeded(bool syncpoint, bool strict) + /// Modifiers for how host sync should be created + internal void CreateHostSyncIfNeeded(HostSyncFlags flags) { + bool syncpoint = flags.HasFlag(HostSyncFlags.Syncpoint); + bool strict = flags.HasFlag(HostSyncFlags.Strict); + bool force = flags.HasFlag(HostSyncFlags.Force); + if (BufferMigrations.Count > 0) { ulong currentSyncNumber = Renderer.GetCurrentSync(); @@ -336,24 +339,17 @@ namespace Ryujinx.Graphics.Gpu } } - if (_pendingSync || (syncpoint && SyncpointActions.Count > 0)) + if (force || _pendingSync || (syncpoint && SyncpointActions.Count > 0)) { Renderer.CreateSync(SyncNumber, strict); + SyncActions.ForEach(action => action.SyncPreAction(syncpoint)); + SyncpointActions.ForEach(action => action.SyncPreAction(syncpoint)); + SyncNumber++; - foreach (Action action in SyncActions) - { - action(); - } - - foreach (Action action in SyncpointActions) - { - action(); - } - - SyncActions.Clear(); - SyncpointActions.Clear(); + SyncActions.RemoveAll(action => action.SyncAction(syncpoint)); + SyncpointActions.RemoveAll(action => action.SyncAction(syncpoint)); } _pendingSync = false; diff --git a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs index f0df55e68..3b2579889 100644 --- a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs +++ b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs @@ -1423,7 +1423,7 @@ namespace Ryujinx.Graphics.Gpu.Image _scaledSetScore = Math.Max(0, _scaledSetScore - 1); } - if (_modifiedStale || Group.HasCopyDependencies) + if (_modifiedStale || Group.HasCopyDependencies || Group.HasFlushBuffer) { _modifiedStale = false; Group.SignalModifying(this, bound); @@ -1685,6 +1685,8 @@ namespace Ryujinx.Graphics.Gpu.Image if (Group.Storage == this) { + Group.Unmapped(); + Group.ClearModified(unmapRange); } } diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs index 234e7e8c7..14ab5d1e4 100644 --- a/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs +++ b/src/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs @@ -57,6 +57,12 @@ namespace Ryujinx.Graphics.Gpu.Image /// public bool HasCopyDependencies { get; set; } + /// + /// Indicates if the texture group has a pre-emptive flush buffer. + /// When one is present, the group must always be notified on unbind. + /// + public bool HasFlushBuffer => _flushBuffer != BufferHandle.Null; + /// /// Indicates if this texture has any incompatible overlaps alive. /// @@ -89,6 +95,10 @@ namespace Ryujinx.Graphics.Gpu.Image private bool _incompatibleOverlapsDirty = true; private bool _flushIncompatibleOverlaps; + private BufferHandle _flushBuffer; + private bool _flushBufferImported; + private bool _flushBufferInvalid; + /// /// Create a new texture group. /// @@ -464,8 +474,9 @@ namespace Ryujinx.Graphics.Gpu.Image /// /// True if writing the texture data is tracked, false otherwise /// The index of the slice to flush + /// Whether the flushed texture data is up to date in the flush buffer /// The specific host texture to flush. Defaults to the storage texture - private void FlushTextureDataSliceToGuest(bool tracked, int sliceIndex, ITexture texture = null) + private void FlushTextureDataSliceToGuest(bool tracked, int sliceIndex, bool inBuffer, ITexture texture = null) { (int layer, int level) = GetLayerLevelForView(sliceIndex); @@ -475,7 +486,16 @@ namespace Ryujinx.Graphics.Gpu.Image using WritableRegion region = _physicalMemory.GetWritableRegion(Storage.Range.Slice((ulong)offset, (ulong)size), tracked); - Storage.GetTextureDataSliceFromGpu(region.Memory.Span, layer, level, tracked, texture); + if (inBuffer) + { + using PinnedSpan data = _context.Renderer.GetBufferData(_flushBuffer, offset, size); + + Storage.ConvertFromHostCompatibleFormat(region.Memory.Span, data.Get(), level, true); + } + else + { + Storage.GetTextureDataSliceFromGpu(region.Memory.Span, layer, level, tracked, texture); + } } /// @@ -484,12 +504,13 @@ namespace Ryujinx.Graphics.Gpu.Image /// True if writing the texture data is tracked, false otherwise /// The first slice to flush /// The slice to finish flushing on (exclusive) + /// Whether the flushed texture data is up to date in the flush buffer /// The specific host texture to flush. Defaults to the storage texture - private void FlushSliceRange(bool tracked, int sliceStart, int sliceEnd, ITexture texture = null) + private void FlushSliceRange(bool tracked, int sliceStart, int sliceEnd, bool inBuffer, ITexture texture = null) { for (int i = sliceStart; i < sliceEnd; i++) { - FlushTextureDataSliceToGuest(tracked, i, texture); + FlushTextureDataSliceToGuest(tracked, i, inBuffer, texture); } } @@ -520,7 +541,7 @@ namespace Ryujinx.Graphics.Gpu.Image { if (endSlice > startSlice) { - FlushSliceRange(tracked, startSlice, endSlice); + FlushSliceRange(tracked, startSlice, endSlice, false); flushed = true; } @@ -553,7 +574,7 @@ namespace Ryujinx.Graphics.Gpu.Image } else { - FlushSliceRange(tracked, startSlice, endSlice); + FlushSliceRange(tracked, startSlice, endSlice, false); } flushed = true; @@ -565,6 +586,58 @@ namespace Ryujinx.Graphics.Gpu.Image return flushed; } + /// + /// Flush the texture data into a persistently mapped buffer. + /// If the buffer does not exist, this method will create it. + /// + /// Handle of the texture group to flush slices of + public void FlushIntoBuffer(TextureGroupHandle handle) + { + // Ensure that the buffer exists. + + if (_flushBufferInvalid && _flushBuffer != BufferHandle.Null) + { + _flushBufferInvalid = false; + _context.Renderer.DeleteBuffer(_flushBuffer); + _flushBuffer = BufferHandle.Null; + } + + if (_flushBuffer == BufferHandle.Null) + { + if (!TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities)) + { + return; + } + + bool canImport = Storage.Info.IsLinear && Storage.Info.Stride >= Storage.Info.Width * Storage.Info.FormatInfo.BytesPerPixel; + + var hostPointer = canImport ? _physicalMemory.GetHostPointer(Storage.Range) : 0; + + if (hostPointer != 0 && _context.Renderer.PrepareHostMapping(hostPointer, Storage.Size)) + { + _flushBuffer = _context.Renderer.CreateBuffer(hostPointer, (int)Storage.Size); + _flushBufferImported = true; + } + else + { + _flushBuffer = _context.Renderer.CreateBuffer((int)Storage.Size, BufferAccess.FlushPersistent); + _flushBufferImported = false; + } + + Storage.BlacklistScale(); + } + + int sliceStart = handle.BaseSlice; + int sliceEnd = sliceStart + handle.SliceCount; + + for (int i = sliceStart; i < sliceEnd; i++) + { + (int layer, int level) = GetLayerLevelForView(i); + + Storage.GetFlushTexture().CopyTo(new BufferRange(_flushBuffer, _allOffsets[i], _sliceSizes[level]), layer, level, _flushBufferImported ? Storage.Info.Stride : 0); + } + } + /// /// Clears competing modified flags for all incompatible ranges, if they have possibly been modified. /// @@ -1570,10 +1643,7 @@ namespace Ryujinx.Graphics.Gpu.Image _context.Renderer.BackgroundContextAction(() => { - if (!isGpuThread) - { - handle.Sync(_context); - } + bool inBuffer = !isGpuThread && handle.Sync(_context); Storage.SignalModifiedDirty(); @@ -1585,13 +1655,24 @@ namespace Ryujinx.Graphics.Gpu.Image } } - if (TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities)) + if (TextureCompatibility.CanTextureFlush(Storage.Info, _context.Capabilities) && !(inBuffer && _flushBufferImported)) { - FlushSliceRange(false, handle.BaseSlice, handle.BaseSlice + handle.SliceCount, Storage.GetFlushTexture()); + FlushSliceRange(false, handle.BaseSlice, handle.BaseSlice + handle.SliceCount, inBuffer, Storage.GetFlushTexture()); } }); } + /// + /// Called if any part of the storage texture is unmapped. + /// + public void Unmapped() + { + if (_flushBufferImported) + { + _flushBufferInvalid = true; + } + } + /// /// Dispose this texture group, disposing all related memory tracking handles. /// @@ -1606,6 +1687,11 @@ namespace Ryujinx.Graphics.Gpu.Image { incompatible.Group._incompatibleOverlaps.RemoveAll(overlap => overlap.Group == this); } + + if (_flushBuffer != BufferHandle.Null) + { + _context.Renderer.DeleteBuffer(_flushBuffer); + } } } } diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs index ebb4e9aeb..9f66744be 100644 --- a/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs +++ b/src/Ryujinx.Graphics.Gpu/Image/TextureGroupHandle.cs @@ -1,4 +1,5 @@ using Ryujinx.Cpu.Tracking; +using Ryujinx.Graphics.Gpu.Synchronization; using System; using System.Collections.Generic; using System.Linq; @@ -13,8 +14,14 @@ namespace Ryujinx.Graphics.Gpu.Image /// Also tracks copy dependencies for the handle - references to other handles that must be kept /// in sync with this one before use. /// - class TextureGroupHandle : IDisposable + class TextureGroupHandle : ISyncActionHandler, IDisposable { + private const int FlushBalanceIncrement = 6; + private const int FlushBalanceWriteCost = 1; + private const int FlushBalanceThreshold = 7; + private const int FlushBalanceMax = 60; + private const int FlushBalanceMin = -10; + private TextureGroup _group; private int _bindCount; private int _firstLevel; @@ -26,6 +33,8 @@ namespace Ryujinx.Graphics.Gpu.Image /// The sync number last registered. /// private ulong _registeredSync; + private ulong _registeredBufferSync = ulong.MaxValue; + private ulong _registeredBufferGuestSync = ulong.MaxValue; /// /// The sync number when the texture was last modified by GPU. @@ -42,6 +51,12 @@ namespace Ryujinx.Graphics.Gpu.Image /// private bool _syncActionRegistered; + /// + /// Determines the balance of synced writes to flushes. + /// Used to determine if the texture should always write data to a persistent buffer for flush. + /// + private int _flushBalance; + /// /// The byte offset from the start of the storage of this handle. /// @@ -132,6 +147,12 @@ namespace Ryujinx.Graphics.Gpu.Image } Handles = handles; + + if (group.Storage.Info.IsLinear) + { + // Linear textures are presumed to be used for readback initially. + _flushBalance = FlushBalanceThreshold + FlushBalanceIncrement; + } } /// @@ -159,6 +180,35 @@ namespace Ryujinx.Graphics.Gpu.Image } } + /// + /// Determine if the next sync will copy into the flush buffer. + /// + /// True if it will copy, false otherwise + private bool NextSyncCopies() + { + return _flushBalance - FlushBalanceWriteCost > FlushBalanceThreshold; + } + + /// + /// Alters the flush balance by the given value. Should increase significantly with each sync, decrease with each write. + /// A flush balance higher than the threshold will cause a texture to repeatedly copy to a flush buffer on each use. + /// + /// Value to add to the existing flush balance + /// True if the new balance is over the threshold, false otherwise + private bool ModifyFlushBalance(int modifier) + { + int result; + int existingBalance; + do + { + existingBalance = _flushBalance; + result = Math.Max(FlushBalanceMin, Math.Min(FlushBalanceMax, existingBalance + modifier)); + } + while (Interlocked.CompareExchange(ref _flushBalance, result, existingBalance) != existingBalance); + + return result > FlushBalanceThreshold; + } + /// /// Adds a single texture view as an overlap if its range overlaps. /// @@ -204,7 +254,7 @@ namespace Ryujinx.Graphics.Gpu.Image if (!_syncActionRegistered) { _modifiedSync = context.SyncNumber; - context.RegisterSyncAction(SyncAction, true); + context.RegisterSyncAction(this, true); _syncActionRegistered = true; } @@ -241,6 +291,13 @@ namespace Ryujinx.Graphics.Gpu.Image { SignalModified(context); + if (!bound && _syncActionRegistered && NextSyncCopies()) + { + // On unbind, textures that flush often should immediately create sync so their result can be obtained as soon as possible. + + context.CreateHostSyncIfNeeded(HostSyncFlags.Force); + } + // Note: Bind count currently resets to 0 on inherit for safety, as the handle <-> view relationship can change. _bindCount = Math.Max(0, _bindCount + (bound ? 1 : -1)); } @@ -266,25 +323,35 @@ namespace Ryujinx.Graphics.Gpu.Image /// removing the modified flag if it was reached, or leaving it set if it has not yet been created. /// /// The GPU context used to wait for sync - public void Sync(GpuContext context) + /// True if the texture data can be read from the flush buffer + public bool Sync(GpuContext context) { - ulong registeredSync = _registeredSync; - long diff = (long)(context.SyncNumber - registeredSync); + // Currently assumes the calling thread is a guest thread. + + bool inBuffer = _registeredBufferGuestSync != ulong.MaxValue; + ulong sync = inBuffer ? _registeredBufferGuestSync : _registeredSync; + + long diff = (long)(context.SyncNumber - sync); + + ModifyFlushBalance(FlushBalanceIncrement); if (diff > 0) { - context.Renderer.WaitSync(registeredSync); + context.Renderer.WaitSync(sync); - if ((long)(_modifiedSync - registeredSync) > 0) + if ((long)(_modifiedSync - sync) > 0) { // Flush the data in a previous state. Do not remove the modified flag - it will be removed to ignore following writes. - return; + return inBuffer; } Modified = false; + + return inBuffer; } // If the difference is <= 0, no data is not ready yet. Flush any data we can without waiting or removing modified flag. + return false; } /// @@ -296,15 +363,41 @@ namespace Ryujinx.Graphics.Gpu.Image Interlocked.Exchange(ref _actionRegistered, 0); } + /// + /// Action to perform before a sync number is registered after modification. + /// This action will copy the texture data to the flush buffer if this texture + /// flushes often enough, which is determined by the flush balance. + /// + /// + public void SyncPreAction(bool syncpoint) + { + if (syncpoint || NextSyncCopies()) + { + if (ModifyFlushBalance(0) && _registeredBufferSync != _modifiedSync) + { + _group.FlushIntoBuffer(this); + _registeredBufferSync = _modifiedSync; + } + } + } + /// /// Action to perform when a sync number is registered after modification. /// This action will register a read tracking action on the memory tracking handle so that a flush from CPU can happen. /// - private void SyncAction() + /// + public bool SyncAction(bool syncpoint) { // The storage will need to signal modified again to update the sync number in future. _group.Storage.SignalModifiedDirty(); + bool lastInBuffer = _registeredBufferSync == _modifiedSync; + + if (!lastInBuffer) + { + _registeredBufferSync = ulong.MaxValue; + } + lock (Overlaps) { foreach (Texture texture in Overlaps) @@ -314,6 +407,7 @@ namespace Ryujinx.Graphics.Gpu.Image } // Register region tracking for CPU? (again) + _registeredSync = _modifiedSync; _syncActionRegistered = false; @@ -321,6 +415,14 @@ namespace Ryujinx.Graphics.Gpu.Image { _group.RegisterAction(this); } + + if (syncpoint) + { + _registeredBufferGuestSync = _registeredBufferSync; + } + + // If the last modification is in the buffer, keep this sync action alive until it sees a syncpoint. + return syncpoint || !lastInBuffer; } /// diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index f267dfda7..ef8c80746 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -1,5 +1,6 @@ using Ryujinx.Cpu.Tracking; using Ryujinx.Graphics.GAL; +using Ryujinx.Graphics.Gpu.Synchronization; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; @@ -11,7 +12,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// /// Buffer, used to store vertex and index data, uniform and storage buffers, and others. /// - class Buffer : IRange, IDisposable + class Buffer : IRange, ISyncActionHandler, IDisposable { private const ulong GranularBufferThreshold = 4096; @@ -248,7 +249,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (!_syncActionRegistered) { - _context.RegisterSyncAction(SyncAction); + _context.RegisterSyncAction(this); _syncActionRegistered = true; } } @@ -267,7 +268,8 @@ namespace Ryujinx.Graphics.Gpu.Memory /// Action to be performed when a syncpoint is reached after modification. /// This will register read/write tracking to flush the buffer from GPU when its memory is used. /// - private void SyncAction() + /// + public bool SyncAction(bool syncpoint) { _syncActionRegistered = false; @@ -284,6 +286,8 @@ namespace Ryujinx.Graphics.Gpu.Memory _memoryTracking.RegisterAction(_externalFlushDelegate); SynchronizeMemory(Address, Size); } + + return true; } /// @@ -296,7 +300,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (from._syncActionRegistered && !_syncActionRegistered) { - _context.RegisterSyncAction(SyncAction); + _context.RegisterSyncAction(this); _syncActionRegistered = true; } diff --git a/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs b/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs index bd33383e5..b976667c3 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/PhysicalMemory.cs @@ -8,6 +8,7 @@ using Ryujinx.Memory.Tracking; using System; using System.Collections.Generic; using System.Runtime.InteropServices; +using System.Linq; using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory @@ -82,6 +83,34 @@ namespace Ryujinx.Graphics.Gpu.Memory } } + /// + /// Gets a host pointer for a given range of application memory. + /// If the memory region is not a single contiguous block, this method returns 0. + /// + /// + /// Getting a host pointer is unsafe. It should be considered invalid immediately if the GPU memory is unmapped. + /// + /// Ranges of physical memory where the target data is located + /// Pointer to the range of memory + public nint GetHostPointer(MultiRange range) + { + if (range.Count == 1) + { + var singleRange = range.GetSubRange(0); + if (singleRange.Address != MemoryManager.PteUnmapped) + { + var regions = _cpuMemory.GetHostRegions(singleRange.Address, singleRange.Size); + + if (regions != null && regions.Count() == 1) + { + return (nint)regions.First().Address; + } + } + } + + return 0; + } + /// /// Gets a span of data from the application process. /// diff --git a/src/Ryujinx.Graphics.Gpu/Synchronization/HostSyncFlags.cs b/src/Ryujinx.Graphics.Gpu/Synchronization/HostSyncFlags.cs new file mode 100644 index 000000000..426669914 --- /dev/null +++ b/src/Ryujinx.Graphics.Gpu/Synchronization/HostSyncFlags.cs @@ -0,0 +1,30 @@ +using System; + +namespace Ryujinx.Graphics.Gpu.Synchronization +{ + /// + /// Modifier flags for creating host sync. + /// + [Flags] + internal enum HostSyncFlags + { + None = 0, + + /// + /// Present if host sync is being created by a syncpoint. + /// + Syncpoint = 1 << 0, + + /// + /// Present if the sync should signal as soon as possible. + /// + Strict = 1 << 1, + + /// + /// Present will force the sync to be created, even if no actions are eligible. + /// + Force = 1 << 2, + + StrictSyncpoint = Strict | Syncpoint + } +} diff --git a/src/Ryujinx.Graphics.Gpu/Synchronization/ISyncActionHandler.cs b/src/Ryujinx.Graphics.Gpu/Synchronization/ISyncActionHandler.cs new file mode 100644 index 000000000..d470d2f07 --- /dev/null +++ b/src/Ryujinx.Graphics.Gpu/Synchronization/ISyncActionHandler.cs @@ -0,0 +1,22 @@ +namespace Ryujinx.Graphics.Gpu.Synchronization +{ + /// + /// This interface indicates that a class can be registered for a sync action. + /// + interface ISyncActionHandler + { + /// + /// Action to be performed when some synchronizing action is reached after modification. + /// Generally used to register read/write tracking to flush resources from GPU when their memory is used. + /// + /// True if the action is a guest syncpoint + /// True if the action is to be removed, false otherwise + bool SyncAction(bool syncpoint); + + /// + /// Action to be performed immediately before sync is created. + /// + /// True if the action is a guest syncpoint + void SyncPreAction(bool syncpoint) { } + } +} diff --git a/src/Ryujinx.Graphics.OpenGL/Buffer.cs b/src/Ryujinx.Graphics.OpenGL/Buffer.cs index af7d191a6..2a5143101 100644 --- a/src/Ryujinx.Graphics.OpenGL/Buffer.cs +++ b/src/Ryujinx.Graphics.OpenGL/Buffer.cs @@ -42,6 +42,20 @@ namespace Ryujinx.Graphics.OpenGL return Handle.FromInt32(handle); } + public static BufferHandle CreatePersistent(int size) + { + int handle = GL.GenBuffer(); + + GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle); + GL.BufferStorage(BufferTarget.CopyWriteBuffer, size, IntPtr.Zero, + BufferStorageFlags.MapPersistentBit | + BufferStorageFlags.MapCoherentBit | + BufferStorageFlags.ClientStorageBit | + BufferStorageFlags.MapReadBit); + + return Handle.FromInt32(handle); + } + public static void Copy(BufferHandle source, BufferHandle destination, int srcOffset, int dstOffset, int size) { GL.BindBuffer(BufferTarget.CopyReadBuffer, source.ToInt32()); @@ -60,7 +74,11 @@ namespace Ryujinx.Graphics.OpenGL // Data in the persistent buffer and host array is guaranteed to be available // until the next time the host thread requests data. - if (HwCapabilities.UsePersistentBufferForFlush) + if (renderer.PersistentBuffers.TryGet(buffer, out IntPtr ptr)) + { + return new PinnedSpan(IntPtr.Add(ptr, offset).ToPointer(), size); + } + else if (HwCapabilities.UsePersistentBufferForFlush) { return PinnedSpan.UnsafeFromSpan(renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size)); } diff --git a/src/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs b/src/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs index 1e9e4d6b2..116f70cc7 100644 --- a/src/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs +++ b/src/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs @@ -49,6 +49,11 @@ namespace Ryujinx.Graphics.OpenGL.Image return GetData(); } + public void CopyTo(BufferRange range, int layer, int level, int stride) + { + throw new NotImplementedException(); + } + public void SetData(SpanOrArray data) { var dataSpan = data.AsSpan(); diff --git a/src/Ryujinx.Graphics.OpenGL/Image/TextureView.cs b/src/Ryujinx.Graphics.OpenGL/Image/TextureView.cs index 804b3b03e..f24a58fc3 100644 --- a/src/Ryujinx.Graphics.OpenGL/Image/TextureView.cs +++ b/src/Ryujinx.Graphics.OpenGL/Image/TextureView.cs @@ -3,6 +3,7 @@ using Ryujinx.Common; using Ryujinx.Common.Memory; using Ryujinx.Graphics.GAL; using System; +using System.Diagnostics; namespace Ryujinx.Graphics.OpenGL.Image { @@ -287,6 +288,26 @@ namespace Ryujinx.Graphics.OpenGL.Image } } + public void CopyTo(BufferRange range, int layer, int level, int stride) + { + if (stride != 0 && stride != BitUtils.AlignUp(Info.Width * Info.BytesPerPixel, 4)) + { + throw new NotSupportedException("Stride conversion for texture copy to buffer not supported."); + } + + GL.BindBuffer(BufferTarget.PixelPackBuffer, range.Handle.ToInt32()); + + FormatInfo format = FormatTable.GetFormatInfo(Info.Format); + if (format.PixelFormat == PixelFormat.DepthStencil) + { + throw new InvalidOperationException("DepthStencil copy to buffer is not supported for layer/level > 0."); + } + + int offset = WriteToPbo2D(range.Offset, layer, level); + + Debug.Assert(offset == 0); + } + public void WriteToPbo(int offset, bool forceBgra) { WriteTo(IntPtr.Zero + offset, forceBgra); diff --git a/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs b/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs index 3903b4d4e..7d5fe8931 100644 --- a/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs +++ b/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs @@ -58,10 +58,31 @@ namespace Ryujinx.Graphics.OpenGL } public BufferHandle CreateBuffer(int size, BufferHandle storageHint) + { + return CreateBuffer(size, GAL.BufferAccess.Default); + } + + public BufferHandle CreateBuffer(int size, GAL.BufferAccess access) { BufferCount++; - return Buffer.Create(size); + if (access == GAL.BufferAccess.FlushPersistent) + { + BufferHandle handle = Buffer.CreatePersistent(size); + + PersistentBuffers.Map(handle, size); + + return handle; + } + else + { + return Buffer.Create(size); + } + } + + public BufferHandle CreateBuffer(nint pointer, int size) + { + throw new NotSupportedException(); } public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info) @@ -88,6 +109,8 @@ namespace Ryujinx.Graphics.OpenGL public void DeleteBuffer(BufferHandle buffer) { + PersistentBuffers.Unmap(buffer); + Buffer.Delete(buffer); } @@ -272,5 +295,10 @@ namespace Ryujinx.Graphics.OpenGL { ScreenCaptured?.Invoke(this, bitmap); } + + public bool PrepareHostMapping(nint address, ulong size) + { + return false; + } } } diff --git a/src/Ryujinx.Graphics.OpenGL/PersistentBuffers.cs b/src/Ryujinx.Graphics.OpenGL/PersistentBuffers.cs index 654e25b9d..aac288b7e 100644 --- a/src/Ryujinx.Graphics.OpenGL/PersistentBuffers.cs +++ b/src/Ryujinx.Graphics.OpenGL/PersistentBuffers.cs @@ -1,8 +1,9 @@ -using OpenTK.Graphics.OpenGL; +using OpenTK.Graphics.OpenGL; using Ryujinx.Common.Logging; using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.OpenGL.Image; using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -13,6 +14,8 @@ namespace Ryujinx.Graphics.OpenGL private PersistentBuffer _main = new PersistentBuffer(); private PersistentBuffer _background = new PersistentBuffer(); + private Dictionary _maps = new Dictionary(); + public PersistentBuffer Default => BackgroundContextWorker.InBackground ? _background : _main; public void Dispose() @@ -20,6 +23,30 @@ namespace Ryujinx.Graphics.OpenGL _main?.Dispose(); _background?.Dispose(); } + + public void Map(BufferHandle handle, int size) + { + GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle.ToInt32()); + IntPtr ptr = GL.MapBufferRange(BufferTarget.CopyWriteBuffer, IntPtr.Zero, size, BufferAccessMask.MapReadBit | BufferAccessMask.MapPersistentBit); + + _maps[handle] = ptr; + } + + public void Unmap(BufferHandle handle) + { + if (_maps.ContainsKey(handle)) + { + GL.BindBuffer(BufferTarget.CopyWriteBuffer, handle.ToInt32()); + GL.UnmapBuffer(BufferTarget.CopyWriteBuffer); + + _maps.Remove(handle); + } + } + + public bool TryGet(BufferHandle handle, out IntPtr ptr) + { + return _maps.TryGetValue(handle, out ptr); + } } class PersistentBuffer : IDisposable diff --git a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs index 21b81bdd3..a1ea6836f 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs @@ -33,6 +33,7 @@ namespace Ryujinx.Graphics.Vulkan private MemoryAllocation _allocation; private Auto _buffer; private Auto _allocationAuto; + private bool _allocationImported; private ulong _bufferHandle; private CacheByRange _cachedConvertedBuffers; @@ -81,6 +82,26 @@ namespace Ryujinx.Graphics.Vulkan _flushLock = new ReaderWriterLock(); } + public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto allocation, int size, BufferAllocationType type, BufferAllocationType currentType, int offset) + { + _gd = gd; + _device = device; + _allocation = allocation.GetUnsafe(); + _allocationAuto = allocation; + _allocationImported = true; + _waitable = new MultiFenceHolder(size); + _buffer = new Auto(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _bufferHandle = buffer.Handle; + Size = size; + _map = _allocation.HostPointer + offset; + + _baseType = type; + _currentType = currentType; + DesiredType = currentType; + + _flushLock = new ReaderWriterLock(); + } + public bool TryBackingSwap(ref CommandBufferScoped? cbs) { if (_swapQueued && DesiredType != _currentType) @@ -775,8 +796,15 @@ namespace Ryujinx.Graphics.Vulkan _gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_buffer, (ulong)Size); _buffer.Dispose(); - _allocationAuto.Dispose(); _cachedConvertedBuffers.Dispose(); + if (_allocationImported) + { + _allocationAuto.DecrementReferenceCount(); + } + else + { + _allocationAuto.Dispose(); + } _flushLock.AcquireWriterLock(Timeout.Infinite); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs index f8f41e5b2..27678ed5e 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs @@ -10,7 +10,7 @@ namespace Ryujinx.Graphics.Vulkan { class BufferManager : IDisposable { - private const MemoryPropertyFlags DefaultBufferMemoryFlags = + public const MemoryPropertyFlags DefaultBufferMemoryFlags = MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit | MemoryPropertyFlags.HostCachedBit; @@ -40,6 +40,10 @@ namespace Ryujinx.Graphics.Vulkan BufferUsageFlags.VertexBufferBit | BufferUsageFlags.TransformFeedbackBufferBitExt; + private const BufferUsageFlags HostImportedBufferUsageFlags = + BufferUsageFlags.TransferSrcBit | + BufferUsageFlags.TransferDstBit; + private readonly Device _device; private readonly IdList _buffers; @@ -48,11 +52,47 @@ namespace Ryujinx.Graphics.Vulkan public StagingBuffer StagingBuffer { get; } + public MemoryRequirements HostImportedBufferMemoryRequirements { get; } + public BufferManager(VulkanRenderer gd, Device device) { _device = device; _buffers = new IdList(); StagingBuffer = new StagingBuffer(gd, this); + + HostImportedBufferMemoryRequirements = GetHostImportedUsageRequirements(gd); + } + + public unsafe BufferHandle CreateHostImported(VulkanRenderer gd, nint pointer, int size) + { + var usage = HostImportedBufferUsageFlags; + + if (gd.Capabilities.SupportsIndirectParameters) + { + usage |= BufferUsageFlags.IndirectBufferBit; + } + + var bufferCreateInfo = new BufferCreateInfo() + { + SType = StructureType.BufferCreateInfo, + Size = (ulong)size, + Usage = usage, + SharingMode = SharingMode.Exclusive + }; + + gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); + + (Auto allocation, ulong offset) = gd.HostMemoryAllocator.GetExistingAllocation(pointer, (ulong)size); + + gd.Api.BindBufferMemory(_device, buffer, allocation.GetUnsafe().Memory, allocation.GetUnsafe().Offset + offset); + + var holder = new BufferHolder(gd, _device, buffer, allocation, size, BufferAllocationType.HostMapped, BufferAllocationType.HostMapped, (int)offset); + + BufferCount++; + + ulong handle64 = (uint)_buffers.Add(holder); + + return Unsafe.As(ref handle64); } public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default) @@ -75,6 +115,32 @@ namespace Ryujinx.Graphics.Vulkan return Unsafe.As(ref handle64); } + public unsafe MemoryRequirements GetHostImportedUsageRequirements(VulkanRenderer gd) + { + var usage = HostImportedBufferUsageFlags; + + if (gd.Capabilities.SupportsIndirectParameters) + { + usage |= BufferUsageFlags.IndirectBufferBit; + } + + var bufferCreateInfo = new BufferCreateInfo() + { + SType = StructureType.BufferCreateInfo, + Size = (ulong)Environment.SystemPageSize, + Usage = usage, + SharingMode = SharingMode.Exclusive + }; + + gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); + + gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); + + gd.Api.DestroyBuffer(_device, buffer, null); + + return requirements; + } + public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking( VulkanRenderer gd, int size, diff --git a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs index b69c64aa8..1f03b68c2 100644 --- a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs +++ b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs @@ -364,6 +364,15 @@ namespace Ryujinx.Graphics.Vulkan }; } + public static BufferAllocationType Convert(this BufferAccess access) + { + return access switch + { + BufferAccess.FlushPersistent => BufferAllocationType.HostMapped, + _ => BufferAllocationType.Auto + }; + } + private static T2 LogInvalidAndReturn(T1 value, string name, T2 defaultValue = default) { Logger.Debug?.Print(LogClass.Gpu, $"Invalid {name} enum value: {value}."); diff --git a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs index ab82d7b4a..5cab4113a 100644 --- a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs +++ b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs @@ -41,6 +41,7 @@ namespace Ryujinx.Graphics.Vulkan public readonly bool SupportsPipelineStatisticsQuery; public readonly bool SupportsGeometryShader; public readonly bool SupportsViewportArray2; + public readonly bool SupportsHostImportedMemory; public readonly uint MinSubgroupSize; public readonly uint MaxSubgroupSize; public readonly ShaderStageFlags RequiredSubgroupSizeStages; @@ -75,6 +76,7 @@ namespace Ryujinx.Graphics.Vulkan bool supportsPipelineStatisticsQuery, bool supportsGeometryShader, bool supportsViewportArray2, + bool supportsHostImportedMemory, uint minSubgroupSize, uint maxSubgroupSize, ShaderStageFlags requiredSubgroupSizeStages, @@ -108,6 +110,7 @@ namespace Ryujinx.Graphics.Vulkan SupportsPipelineStatisticsQuery = supportsPipelineStatisticsQuery; SupportsGeometryShader = supportsGeometryShader; SupportsViewportArray2 = supportsViewportArray2; + SupportsHostImportedMemory = supportsHostImportedMemory; MinSubgroupSize = minSubgroupSize; MaxSubgroupSize = maxSubgroupSize; RequiredSubgroupSizeStages = requiredSubgroupSizeStages; diff --git a/src/Ryujinx.Graphics.Vulkan/HelperShader.cs b/src/Ryujinx.Graphics.Vulkan/HelperShader.cs index c57edaf7c..155c7f6fe 100644 --- a/src/Ryujinx.Graphics.Vulkan/HelperShader.cs +++ b/src/Ryujinx.Graphics.Vulkan/HelperShader.cs @@ -31,6 +31,7 @@ namespace Ryujinx.Graphics.Vulkan private readonly IProgram _programColorClearSI; private readonly IProgram _programColorClearUI; private readonly IProgram _programStrideChange; + private readonly IProgram _programConvertD32S8ToD24S8; private readonly IProgram _programConvertIndexBuffer; private readonly IProgram _programConvertIndirectData; private readonly IProgram _programColorCopyShortening; @@ -158,6 +159,17 @@ namespace Ryujinx.Graphics.Vulkan new ShaderSource(ShaderBinaries.ColorDrawToMsFragmentShaderSource, colorDrawToMsFragmentBindings, ShaderStage.Fragment, TargetLanguage.Spirv), }); + var convertD32S8ToD24S8Bindings = new ShaderBindings( + new[] { 0 }, + new[] { 1, 2 }, + Array.Empty(), + Array.Empty()); + + _programConvertD32S8ToD24S8 = gd.CreateProgramWithMinimalLayout(new[] + { + new ShaderSource(ShaderBinaries.ConvertD32S8ToD24S8ShaderSource, convertD32S8ToD24S8Bindings, ShaderStage.Compute, TargetLanguage.Spirv), + }); + var convertIndexBufferBindings = new ShaderBindings( new[] { 0 }, new[] { 1, 2 }, @@ -1644,6 +1656,82 @@ namespace Ryujinx.Graphics.Vulkan _pipeline.Finish(gd, cbs); } + public unsafe void ConvertD32S8ToD24S8(VulkanRenderer gd, CommandBufferScoped cbs, BufferHolder src, Auto dstBufferAuto, int pixelCount, int dstOffset) + { + int inSize = pixelCount * 2 * sizeof(int); + int outSize = pixelCount * sizeof(int); + + var srcBufferAuto = src.GetBuffer(); + + var srcBuffer = srcBufferAuto.Get(cbs, 0, inSize).Value; + var dstBuffer = dstBufferAuto.Get(cbs, dstOffset, outSize).Value; + + var access = AccessFlags.ShaderWriteBit; + var stage = PipelineStageFlags.ComputeShaderBit; + + BufferHolder.InsertBufferBarrier( + gd, + cbs.CommandBuffer, + srcBuffer, + BufferHolder.DefaultAccessFlags, + AccessFlags.ShaderReadBit, + PipelineStageFlags.AllCommandsBit, + stage, + 0, + outSize); + + BufferHolder.InsertBufferBarrier( + gd, + cbs.CommandBuffer, + dstBuffer, + BufferHolder.DefaultAccessFlags, + access, + PipelineStageFlags.AllCommandsBit, + stage, + 0, + outSize); + + const int ParamsBufferSize = sizeof(int) * 2; + + Span shaderParams = stackalloc int[2]; + + shaderParams[0] = pixelCount; + shaderParams[1] = dstOffset; + + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize); + + gd.BufferManager.SetData(bufferHandle, 0, shaderParams); + + _pipeline.SetCommandBuffer(cbs); + + _pipeline.SetUniformBuffers(stackalloc[] { new BufferAssignment(0, new BufferRange(bufferHandle, 0, ParamsBufferSize)) }); + + Span> sbRanges = new Auto[2]; + + sbRanges[0] = srcBufferAuto; + sbRanges[1] = dstBufferAuto; + + _pipeline.SetStorageBuffers(1, sbRanges); + + _pipeline.SetProgram(_programConvertD32S8ToD24S8); + _pipeline.DispatchCompute(1, 1, 1); + + gd.BufferManager.Delete(bufferHandle); + + _pipeline.Finish(gd, cbs); + + BufferHolder.InsertBufferBarrier( + gd, + cbs.CommandBuffer, + dstBuffer, + access, + BufferHolder.DefaultAccessFlags, + stage, + PipelineStageFlags.AllCommandsBit, + 0, + outSize); + } + protected virtual void Dispose(bool disposing) { if (disposing) diff --git a/src/Ryujinx.Graphics.Vulkan/HostMemoryAllocator.cs b/src/Ryujinx.Graphics.Vulkan/HostMemoryAllocator.cs new file mode 100644 index 000000000..e62b2dbba --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/HostMemoryAllocator.cs @@ -0,0 +1,188 @@ +using Ryujinx.Common; +using Ryujinx.Common.Collections; +using Ryujinx.Common.Logging; +using Silk.NET.Vulkan; +using Silk.NET.Vulkan.Extensions.EXT; +using System; +using System.Collections.Generic; + +namespace Ryujinx.Graphics.Vulkan +{ + internal class HostMemoryAllocator + { + private struct HostMemoryAllocation + { + public readonly Auto Allocation; + public readonly IntPtr Pointer; + public readonly ulong Size; + + public ulong Start => (ulong)Pointer; + public ulong End => (ulong)Pointer + Size; + + public HostMemoryAllocation(Auto allocation, IntPtr pointer, ulong size) + { + Allocation = allocation; + Pointer = pointer; + Size = size; + } + } + + private readonly MemoryAllocator _allocator; + private readonly Vk _api; + private readonly ExtExternalMemoryHost _hostMemoryApi; + private readonly Device _device; + private readonly object _lock = new(); + + private List _allocations; + private IntervalTree _allocationTree; + + public HostMemoryAllocator(MemoryAllocator allocator, Vk api, ExtExternalMemoryHost hostMemoryApi, Device device) + { + _allocator = allocator; + _api = api; + _hostMemoryApi = hostMemoryApi; + _device = device; + + _allocations = new List(); + _allocationTree = new IntervalTree(); + } + + public unsafe bool TryImport( + MemoryRequirements requirements, + MemoryPropertyFlags flags, + IntPtr pointer, + ulong size) + { + lock (_lock) + { + // Does a compatible allocation exist in the tree? + var allocations = new HostMemoryAllocation[10]; + + ulong start = (ulong)pointer; + ulong end = start + size; + + int count = _allocationTree.Get(start, end, ref allocations); + + // A compatible range is one that where the start and end completely cover the requested range. + for (int i = 0; i < count; i++) + { + HostMemoryAllocation existing = allocations[i]; + + if (start >= existing.Start && end <= existing.End) + { + try + { + existing.Allocation.IncrementReferenceCount(); + + return true; + } + catch (InvalidOperationException) + { + // Can throw if the allocation has been disposed. + // Just continue the search if this happens. + } + } + } + + nint pageAlignedPointer = BitUtils.AlignDown(pointer, Environment.SystemPageSize); + nint pageAlignedEnd = BitUtils.AlignUp((nint)((ulong)pointer + size), Environment.SystemPageSize); + ulong pageAlignedSize = (ulong)(pageAlignedEnd - pageAlignedPointer); + + Result getResult = _hostMemoryApi.GetMemoryHostPointerProperties(_device, ExternalMemoryHandleTypeFlags.HostAllocationBitExt, (void*)pageAlignedPointer, out MemoryHostPointerPropertiesEXT properties); + if (getResult < Result.Success) + { + return false; + } + + int memoryTypeIndex = _allocator.FindSuitableMemoryTypeIndex(properties.MemoryTypeBits & requirements.MemoryTypeBits, flags); + if (memoryTypeIndex < 0) + { + return false; + } + + ImportMemoryHostPointerInfoEXT importInfo = new ImportMemoryHostPointerInfoEXT() + { + SType = StructureType.ImportMemoryHostPointerInfoExt, + HandleType = ExternalMemoryHandleTypeFlags.HostAllocationBitExt, + PHostPointer = (void*)pageAlignedPointer + }; + + var memoryAllocateInfo = new MemoryAllocateInfo() + { + SType = StructureType.MemoryAllocateInfo, + AllocationSize = pageAlignedSize, + MemoryTypeIndex = (uint)memoryTypeIndex, + PNext = &importInfo + }; + + Result result = _api.AllocateMemory(_device, memoryAllocateInfo, null, out var deviceMemory); + + if (result < Result.Success) + { + Logger.Debug?.PrintMsg(LogClass.Gpu, $"Host mapping import 0x{pageAlignedPointer:x16} 0x{pageAlignedSize:x8} failed."); + return false; + } + + var allocation = new MemoryAllocation(this, deviceMemory, pageAlignedPointer, 0, pageAlignedSize); + var allocAuto = new Auto(allocation); + var hostAlloc = new HostMemoryAllocation(allocAuto, pageAlignedPointer, pageAlignedSize); + + allocAuto.IncrementReferenceCount(); + allocAuto.Dispose(); // Kept alive by ref count only. + + // Register this mapping for future use. + + _allocationTree.Add(hostAlloc.Start, hostAlloc.End, hostAlloc); + _allocations.Add(hostAlloc); + } + + return true; + } + + public (Auto, ulong) GetExistingAllocation(IntPtr pointer, ulong size) + { + lock (_lock) + { + // Does a compatible allocation exist in the tree? + var allocations = new HostMemoryAllocation[10]; + + ulong start = (ulong)pointer; + ulong end = start + size; + + int count = _allocationTree.Get(start, end, ref allocations); + + // A compatible range is one that where the start and end completely cover the requested range. + for (int i = 0; i < count; i++) + { + HostMemoryAllocation existing = allocations[i]; + + if (start >= existing.Start && end <= existing.End) + { + return (existing.Allocation, start - existing.Start); + } + } + + throw new InvalidOperationException($"No host allocation was prepared for requested range 0x{pointer:x16}:0x{size:x16}."); + } + } + + public void Free(DeviceMemory memory, ulong offset, ulong size) + { + lock (_lock) + { + _allocations.RemoveAll(allocation => + { + if (allocation.Allocation.GetUnsafe().Memory.Handle == memory.Handle) + { + _allocationTree.Remove(allocation.Start, allocation); + return true; + } + + return false; + }); + } + + _api.FreeMemory(_device, memory, ReadOnlySpan.Empty); + } + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/MemoryAllocation.cs b/src/Ryujinx.Graphics.Vulkan/MemoryAllocation.cs index 76de12967..3c98c417c 100644 --- a/src/Ryujinx.Graphics.Vulkan/MemoryAllocation.cs +++ b/src/Ryujinx.Graphics.Vulkan/MemoryAllocation.cs @@ -7,6 +7,7 @@ namespace Ryujinx.Graphics.Vulkan { private readonly MemoryAllocatorBlockList _owner; private readonly MemoryAllocatorBlockList.Block _block; + private readonly HostMemoryAllocator _hostMemory; public DeviceMemory Memory { get; } public IntPtr HostPointer { get;} @@ -29,9 +30,30 @@ namespace Ryujinx.Graphics.Vulkan Size = size; } + public MemoryAllocation( + HostMemoryAllocator hostMemory, + DeviceMemory memory, + IntPtr hostPointer, + ulong offset, + ulong size) + { + _hostMemory = hostMemory; + Memory = memory; + HostPointer = hostPointer; + Offset = offset; + Size = size; + } + public void Dispose() { - _owner.Free(_block, Offset, Size); + if (_hostMemory != null) + { + _hostMemory.Free(Memory, Offset, Size); + } + else + { + _owner.Free(_block, Offset, Size); + } } } } diff --git a/src/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs b/src/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs index 3139e2091..462d8f052 100644 --- a/src/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs +++ b/src/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs @@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.Vulkan return newBl.Allocate(size, alignment, map); } - private int FindSuitableMemoryTypeIndex( + internal int FindSuitableMemoryTypeIndex( uint memoryTypeBits, MemoryPropertyFlags flags) { diff --git a/src/Ryujinx.Graphics.Vulkan/Shaders/ConvertD32S8ToD24S8ShaderSource.comp b/src/Ryujinx.Graphics.Vulkan/Shaders/ConvertD32S8ToD24S8ShaderSource.comp new file mode 100644 index 000000000..d3a74b1c8 --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/Shaders/ConvertD32S8ToD24S8ShaderSource.comp @@ -0,0 +1,58 @@ +#version 450 core + +#extension GL_EXT_scalar_block_layout : require + +layout (local_size_x = 64, local_size_y = 1, local_size_z = 1) in; + +layout (std430, set = 0, binding = 0) uniform stride_arguments +{ + int pixelCount; + int dstStartOffset; +}; + +layout (std430, set = 1, binding = 1) buffer in_s +{ + uint[] in_data; +}; + +layout (std430, set = 1, binding = 2) buffer out_s +{ + uint[] out_data; +}; + +void main() +{ + // Determine what slice of the stride copies this invocation will perform. + int invocations = int(gl_WorkGroupSize.x); + + int copiesRequired = pixelCount; + + // Find the copies that this invocation should perform. + + // - Copies that all invocations perform. + int allInvocationCopies = copiesRequired / invocations; + + // - Extra remainder copy that this invocation performs. + int index = int(gl_LocalInvocationID.x); + int extra = (index < (copiesRequired % invocations)) ? 1 : 0; + + int copyCount = allInvocationCopies + extra; + + // Finally, get the starting offset. Make sure to count extra copies. + + int startCopy = allInvocationCopies * index + min(copiesRequired % invocations, index); + + int srcOffset = startCopy * 2; + int dstOffset = dstStartOffset + startCopy; + + // Perform the conversion for this region. + for (int i = 0; i < copyCount; i++) + { + float depth = uintBitsToFloat(in_data[srcOffset++]); + uint stencil = in_data[srcOffset++]; + + uint rescaledDepth = uint(clamp(depth, 0.0, 1.0) * 16777215.0); + + out_data[dstOffset++] = (rescaledDepth << 8) | (stencil & 0xff); + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/Shaders/ShaderBinaries.cs b/src/Ryujinx.Graphics.Vulkan/Shaders/ShaderBinaries.cs index c9dde7b63..69d1fa3ce 100644 --- a/src/Ryujinx.Graphics.Vulkan/Shaders/ShaderBinaries.cs +++ b/src/Ryujinx.Graphics.Vulkan/Shaders/ShaderBinaries.cs @@ -1236,6 +1236,213 @@ namespace Ryujinx.Graphics.Vulkan.Shaders 0x38, 0x00, 0x01, 0x00, }; + public static readonly byte[] ConvertD32S8ToD24S8ShaderSource = new byte[] + { + 0x03, 0x02, 0x23, 0x07, 0x00, 0x05, 0x01, 0x00, 0x0A, 0x00, 0x0D, 0x00, 0x77, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x06, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x47, 0x4C, 0x53, 0x4C, 0x2E, 0x73, 0x74, 0x64, 0x2E, 0x34, 0x35, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0F, 0x00, 0x09, 0x00, 0x05, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6D, 0x61, 0x69, 0x6E, + 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x4F, 0x00, 0x00, 0x00, + 0x67, 0x00, 0x00, 0x00, 0x10, 0x00, 0x06, 0x00, 0x04, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00, 0x03, 0x00, + 0x02, 0x00, 0x00, 0x00, 0xC2, 0x01, 0x00, 0x00, 0x04, 0x00, 0x08, 0x00, 0x47, 0x4C, 0x5F, 0x45, + 0x58, 0x54, 0x5F, 0x73, 0x63, 0x61, 0x6C, 0x61, 0x72, 0x5F, 0x62, 0x6C, 0x6F, 0x63, 0x6B, 0x5F, + 0x6C, 0x61, 0x79, 0x6F, 0x75, 0x74, 0x00, 0x00, 0x04, 0x00, 0x0A, 0x00, 0x47, 0x4C, 0x5F, 0x47, + 0x4F, 0x4F, 0x47, 0x4C, 0x45, 0x5F, 0x63, 0x70, 0x70, 0x5F, 0x73, 0x74, 0x79, 0x6C, 0x65, 0x5F, + 0x6C, 0x69, 0x6E, 0x65, 0x5F, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, 0x00, + 0x04, 0x00, 0x08, 0x00, 0x47, 0x4C, 0x5F, 0x47, 0x4F, 0x4F, 0x47, 0x4C, 0x45, 0x5F, 0x69, 0x6E, + 0x63, 0x6C, 0x75, 0x64, 0x65, 0x5F, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x00, + 0x05, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x6D, 0x61, 0x69, 0x6E, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x05, 0x00, 0x08, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x76, 0x6F, 0x63, 0x61, 0x74, 0x69, + 0x6F, 0x6E, 0x73, 0x00, 0x05, 0x00, 0x06, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x63, 0x6F, 0x70, 0x69, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x00, 0x00, 0x05, 0x00, 0x07, 0x00, + 0x0B, 0x00, 0x00, 0x00, 0x73, 0x74, 0x72, 0x69, 0x64, 0x65, 0x5F, 0x61, 0x72, 0x67, 0x75, 0x6D, + 0x65, 0x6E, 0x74, 0x73, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0x0B, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x69, 0x78, 0x65, 0x6C, 0x43, 0x6F, 0x75, 0x6E, 0x74, 0x00, 0x00, + 0x06, 0x00, 0x07, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x73, 0x74, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x4F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, + 0x0D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x07, 0x00, 0x12, 0x00, 0x00, 0x00, + 0x61, 0x6C, 0x6C, 0x49, 0x6E, 0x76, 0x6F, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x43, 0x6F, 0x70, + 0x69, 0x65, 0x73, 0x00, 0x05, 0x00, 0x04, 0x00, 0x16, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x64, 0x65, + 0x78, 0x00, 0x00, 0x00, 0x05, 0x00, 0x08, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x67, 0x6C, 0x5F, 0x4C, + 0x6F, 0x63, 0x61, 0x6C, 0x49, 0x6E, 0x76, 0x6F, 0x63, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x49, 0x44, + 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x20, 0x00, 0x00, 0x00, 0x65, 0x78, 0x74, 0x72, + 0x61, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x29, 0x00, 0x00, 0x00, 0x63, 0x6F, 0x70, 0x79, + 0x43, 0x6F, 0x75, 0x6E, 0x74, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, 0x2D, 0x00, 0x00, 0x00, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6F, 0x70, 0x79, 0x00, 0x00, 0x00, 0x05, 0x00, 0x05, 0x00, + 0x37, 0x00, 0x00, 0x00, 0x73, 0x72, 0x63, 0x4F, 0x66, 0x66, 0x73, 0x65, 0x74, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x05, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x64, 0x73, 0x74, 0x4F, 0x66, 0x66, 0x73, 0x65, + 0x74, 0x00, 0x00, 0x00, 0x05, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x04, 0x00, 0x4B, 0x00, 0x00, 0x00, 0x64, 0x65, 0x70, 0x74, 0x68, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x04, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x5F, 0x73, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x05, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x6E, 0x5F, 0x64, + 0x61, 0x74, 0x61, 0x00, 0x05, 0x00, 0x03, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x04, 0x00, 0x57, 0x00, 0x00, 0x00, 0x73, 0x74, 0x65, 0x6E, 0x63, 0x69, 0x6C, 0x00, + 0x05, 0x00, 0x06, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x72, 0x65, 0x73, 0x63, 0x61, 0x6C, 0x65, 0x64, + 0x44, 0x65, 0x70, 0x74, 0x68, 0x00, 0x00, 0x00, 0x05, 0x00, 0x04, 0x00, 0x65, 0x00, 0x00, 0x00, + 0x6F, 0x75, 0x74, 0x5F, 0x73, 0x00, 0x00, 0x00, 0x06, 0x00, 0x06, 0x00, 0x65, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x6F, 0x75, 0x74, 0x5F, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x03, 0x00, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x05, 0x00, + 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x05, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x4C, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x05, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x64, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x05, 0x00, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x47, 0x00, 0x03, 0x00, 0x65, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x67, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x67, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x04, 0x00, 0x76, 0x00, 0x00, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x19, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x02, 0x00, 0x02, 0x00, 0x00, 0x00, 0x21, 0x00, 0x03, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x15, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x04, 0x00, 0x0B, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x0B, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x0F, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x15, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x04, 0x00, 0x18, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x19, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x18, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, 0x19, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x1C, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x14, 0x00, 0x02, 0x00, 0x25, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x16, 0x00, 0x03, 0x00, + 0x49, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x4A, 0x00, 0x00, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x49, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x03, 0x00, 0x4C, 0x00, 0x00, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x03, 0x00, 0x4D, 0x00, 0x00, 0x00, 0x4C, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x04, 0x00, 0x4E, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x4D, 0x00, 0x00, 0x00, + 0x3B, 0x00, 0x04, 0x00, 0x4E, 0x00, 0x00, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x04, 0x00, 0x52, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x04, 0x00, 0x56, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x2B, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2B, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3F, + 0x2B, 0x00, 0x04, 0x00, 0x49, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x7F, 0x4B, + 0x1D, 0x00, 0x03, 0x00, 0x64, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x03, 0x00, + 0x65, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x20, 0x00, 0x04, 0x00, 0x66, 0x00, 0x00, 0x00, + 0x0C, 0x00, 0x00, 0x00, 0x65, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, 0x66, 0x00, 0x00, 0x00, + 0x67, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x6B, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x6E, 0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x74, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, + 0x75, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x06, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x76, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, 0x75, 0x00, 0x00, 0x00, + 0x36, 0x00, 0x05, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, 0x05, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x2D, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x07, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x4A, 0x00, 0x00, 0x00, 0x4B, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x56, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x04, 0x00, + 0x56, 0x00, 0x00, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x0F, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x0A, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x13, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x87, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x15, 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x1C, 0x00, 0x00, 0x00, + 0x1D, 0x00, 0x00, 0x00, 0x1A, 0x00, 0x00, 0x00, 0x1B, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x1D, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x1E, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x16, 0x00, 0x00, 0x00, 0x1F, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x21, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x22, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x23, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x8B, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x24, 0x00, 0x00, 0x00, 0x22, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0xB1, 0x00, 0x05, 0x00, + 0x25, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, + 0xA9, 0x00, 0x06, 0x00, 0x06, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x26, 0x00, 0x00, 0x00, + 0x27, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, 0x20, 0x00, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2A, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x00, 0x00, + 0x2A, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, 0x29, 0x00, 0x00, 0x00, + 0x2C, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2E, 0x00, 0x00, 0x00, + 0x12, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x2F, 0x00, 0x00, 0x00, + 0x16, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, + 0x2E, 0x00, 0x00, 0x00, 0x2F, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x31, 0x00, 0x00, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x32, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x8B, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x07, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, + 0x33, 0x00, 0x00, 0x00, 0x34, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x36, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x35, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x2D, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x38, 0x00, 0x00, 0x00, 0x2D, 0x00, 0x00, 0x00, 0x84, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x3A, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00, 0x39, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x37, 0x00, 0x00, 0x00, 0x3A, 0x00, 0x00, 0x00, 0x41, 0x00, 0x05, 0x00, 0x0F, 0x00, 0x00, 0x00, + 0x3C, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x00, 0x00, 0x2D, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x00, 0x00, + 0x3E, 0x00, 0x03, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00, 0x41, 0x00, 0x00, 0x00, + 0xF8, 0x00, 0x02, 0x00, 0x41, 0x00, 0x00, 0x00, 0xF6, 0x00, 0x04, 0x00, 0x43, 0x00, 0x00, 0x00, + 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00, 0x45, 0x00, 0x00, 0x00, + 0xF8, 0x00, 0x02, 0x00, 0x45, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x46, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x47, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0xB1, 0x00, 0x05, 0x00, 0x25, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x47, 0x00, 0x00, 0x00, 0xFA, 0x00, 0x04, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, + 0x42, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, + 0x37, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, 0x51, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, 0x37, 0x00, 0x00, 0x00, + 0x51, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x52, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, + 0x4F, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x04, 0x00, + 0x49, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x4B, 0x00, 0x00, 0x00, 0x55, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x58, 0x00, 0x00, 0x00, 0x37, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x59, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, 0x3E, 0x00, 0x03, 0x00, + 0x37, 0x00, 0x00, 0x00, 0x59, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x52, 0x00, 0x00, 0x00, + 0x5A, 0x00, 0x00, 0x00, 0x4F, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, + 0x3D, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x5B, 0x00, 0x00, 0x00, 0x5A, 0x00, 0x00, 0x00, + 0x3E, 0x00, 0x03, 0x00, 0x57, 0x00, 0x00, 0x00, 0x5B, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x49, 0x00, 0x00, 0x00, 0x5D, 0x00, 0x00, 0x00, 0x4B, 0x00, 0x00, 0x00, 0x0C, 0x00, 0x08, 0x00, + 0x49, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, + 0x5D, 0x00, 0x00, 0x00, 0x5E, 0x00, 0x00, 0x00, 0x5F, 0x00, 0x00, 0x00, 0x85, 0x00, 0x05, 0x00, + 0x49, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, 0x60, 0x00, 0x00, 0x00, 0x61, 0x00, 0x00, 0x00, + 0x6D, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x62, 0x00, 0x00, 0x00, + 0x3E, 0x00, 0x03, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x63, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, + 0x3E, 0x00, 0x03, 0x00, 0x3B, 0x00, 0x00, 0x00, 0x69, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x6A, 0x00, 0x00, 0x00, 0x5C, 0x00, 0x00, 0x00, 0xC4, 0x00, 0x05, 0x00, + 0x17, 0x00, 0x00, 0x00, 0x6C, 0x00, 0x00, 0x00, 0x6A, 0x00, 0x00, 0x00, 0x6B, 0x00, 0x00, 0x00, + 0x3D, 0x00, 0x04, 0x00, 0x17, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, + 0xC7, 0x00, 0x05, 0x00, 0x17, 0x00, 0x00, 0x00, 0x6F, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00, + 0x6E, 0x00, 0x00, 0x00, 0xC5, 0x00, 0x05, 0x00, 0x17, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, + 0x6C, 0x00, 0x00, 0x00, 0x6F, 0x00, 0x00, 0x00, 0x41, 0x00, 0x06, 0x00, 0x52, 0x00, 0x00, 0x00, + 0x71, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x00, 0x0E, 0x00, 0x00, 0x00, 0x68, 0x00, 0x00, 0x00, + 0x3E, 0x00, 0x03, 0x00, 0x71, 0x00, 0x00, 0x00, 0x70, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00, + 0x44, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, 0x44, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x04, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x00, 0x05, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0x72, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00, + 0x3E, 0x00, 0x03, 0x00, 0x40, 0x00, 0x00, 0x00, 0x73, 0x00, 0x00, 0x00, 0xF9, 0x00, 0x02, 0x00, + 0x41, 0x00, 0x00, 0x00, 0xF8, 0x00, 0x02, 0x00, 0x43, 0x00, 0x00, 0x00, 0xFD, 0x00, 0x01, 0x00, + 0x38, 0x00, 0x01, 0x00 + }; + public static readonly byte[] ConvertIndexBufferShaderSource = new byte[] { 0x03, 0x02, 0x23, 0x07, 0x00, 0x00, 0x01, 0x00, 0x0A, 0x00, 0x08, 0x00, 0x91, 0x00, 0x00, 0x00, diff --git a/src/Ryujinx.Graphics.Vulkan/SyncManager.cs b/src/Ryujinx.Graphics.Vulkan/SyncManager.cs index 432d224f0..b3f6e8e5a 100644 --- a/src/Ryujinx.Graphics.Vulkan/SyncManager.cs +++ b/src/Ryujinx.Graphics.Vulkan/SyncManager.cs @@ -125,6 +125,24 @@ namespace Ryujinx.Graphics.Vulkan if (result != null) { + if (result.Waitable == null) + { + return; + } + + long beforeTicks = Stopwatch.GetTimestamp(); + + if (result.NeedsFlush(FlushId)) + { + _gd.InterruptAction(() => + { + if (result.NeedsFlush(FlushId)) + { + _gd.FlushAllCommands(); + } + }); + } + lock (result) { if (result.Waitable == null) @@ -132,19 +150,6 @@ namespace Ryujinx.Graphics.Vulkan return; } - long beforeTicks = Stopwatch.GetTimestamp(); - - if (result.NeedsFlush(FlushId)) - { - _gd.InterruptAction(() => - { - if (result.NeedsFlush(FlushId)) - { - _gd.FlushAllCommands(); - } - }); - } - bool signaled = result.Signalled || result.Waitable.WaitForFences(_gd.Api, _device, 1000000000); if (!signaled) diff --git a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs index 738bf57d1..66951153d 100644 --- a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs @@ -67,6 +67,11 @@ namespace Ryujinx.Graphics.Vulkan return GetData(); } + public void CopyTo(BufferRange range, int layer, int level, int stride) + { + throw new NotImplementedException(); + } + public void Release() { if (_gd.Textures.Remove(this)) diff --git a/src/Ryujinx.Graphics.Vulkan/TextureView.cs b/src/Ryujinx.Graphics.Vulkan/TextureView.cs index cd280d5f4..62d481eb9 100644 --- a/src/Ryujinx.Graphics.Vulkan/TextureView.cs +++ b/src/Ryujinx.Graphics.Vulkan/TextureView.cs @@ -563,6 +563,34 @@ namespace Ryujinx.Graphics.Vulkan } } + public void CopyTo(BufferRange range, int layer, int level, int stride) + { + _gd.PipelineInternal.EndRenderPass(); + var cbs = _gd.PipelineInternal.CurrentCommandBuffer; + + int outSize = Info.GetMipSize(level); + int hostSize = GetBufferDataLength(outSize); + + var image = GetImage().Get(cbs).Value; + int offset = range.Offset; + + Auto autoBuffer = _gd.BufferManager.GetBuffer(cbs.CommandBuffer, range.Handle, true); + VkBuffer buffer = autoBuffer.Get(cbs, range.Offset, outSize).Value; + + if (PrepareOutputBuffer(cbs, hostSize, buffer, out VkBuffer copyToBuffer, out BufferHolder tempCopyHolder)) + { + offset = 0; + } + + CopyFromOrToBuffer(cbs.CommandBuffer, copyToBuffer, image, hostSize, true, layer, level, 1, 1, singleSlice: true, offset, stride); + + if (tempCopyHolder != null) + { + CopyDataToOutputBuffer(cbs, tempCopyHolder, autoBuffer, hostSize, range.Offset); + tempCopyHolder.Dispose(); + } + } + private ReadOnlySpan GetData(CommandBufferPool cbp, PersistentFlushBuffer flushBuffer) { int size = 0; @@ -693,6 +721,30 @@ namespace Ryujinx.Graphics.Vulkan return storage; } + private bool PrepareOutputBuffer(CommandBufferScoped cbs, int hostSize, VkBuffer target, out VkBuffer copyTarget, out BufferHolder copyTargetHolder) + { + if (NeedsD24S8Conversion()) + { + copyTargetHolder = _gd.BufferManager.Create(_gd, hostSize); + copyTarget = copyTargetHolder.GetBuffer().Get(cbs, 0, hostSize).Value; + + return true; + } + + copyTarget = target; + copyTargetHolder = null; + + return false; + } + + private void CopyDataToOutputBuffer(CommandBufferScoped cbs, BufferHolder hostData, Auto copyTarget, int hostSize, int dstOffset) + { + if (NeedsD24S8Conversion()) + { + _gd.HelperShader.ConvertD32S8ToD24S8(_gd, cbs, hostData, copyTarget, hostSize / (2 * sizeof(int)), dstOffset); + } + } + private bool NeedsD24S8Conversion() { return FormatCapabilities.IsD24S8(Info.Format) && VkFormat == VkFormat.D32SfloatS8Uint; @@ -708,7 +760,9 @@ namespace Ryujinx.Graphics.Vulkan int dstLevel, int dstLayers, int dstLevels, - bool singleSlice) + bool singleSlice, + int offset = 0, + int stride = 0) { bool is3D = Info.Target == Target.Texture3D; int width = Math.Max(1, Info.Width >> dstLevel); @@ -718,8 +772,6 @@ namespace Ryujinx.Graphics.Vulkan int layers = dstLayers; int levels = dstLevels; - int offset = 0; - for (int level = 0; level < levels; level++) { int mipSize = GetBufferDataLength(Info.GetMipSize2D(dstLevel + level) * dstLayers); @@ -731,7 +783,7 @@ namespace Ryujinx.Graphics.Vulkan break; } - int rowLength = (Info.GetMipStride(dstLevel + level) / Info.BytesPerPixel) * Info.BlockWidth; + int rowLength = ((stride == 0 ? Info.GetMipStride(dstLevel + level) : stride) / Info.BytesPerPixel) * Info.BlockWidth; var aspectFlags = Info.Format.ConvertAspectFlags(); diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs b/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs index 50a6fcb90..bad6641e3 100644 --- a/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs +++ b/src/Ryujinx.Graphics.Vulkan/VulkanInitialization.cs @@ -27,6 +27,7 @@ namespace Ryujinx.Graphics.Vulkan ExtTransformFeedback.ExtensionName, KhrDrawIndirectCount.ExtensionName, KhrPushDescriptor.ExtensionName, + ExtExternalMemoryHost.ExtensionName, "VK_EXT_blend_operation_advanced", "VK_EXT_custom_border_color", "VK_EXT_descriptor_indexing", // Enabling this works around an issue with disposed buffer bindings on RADV. diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs index e7475b6b3..06fec4f1b 100644 --- a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs +++ b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs @@ -44,6 +44,7 @@ namespace Ryujinx.Graphics.Vulkan internal object QueueLock { get; private set; } internal MemoryAllocator MemoryAllocator { get; private set; } + internal HostMemoryAllocator HostMemoryAllocator { get; private set; } internal CommandBufferPool CommandBufferPool { get; private set; } internal DescriptorSetManager DescriptorSetManager { get; private set; } internal PipelineLayoutCache PipelineLayoutCache { get; private set; } @@ -307,6 +308,7 @@ namespace Ryujinx.Graphics.Vulkan _physicalDevice.PhysicalDeviceFeatures.PipelineStatisticsQuery, _physicalDevice.PhysicalDeviceFeatures.GeometryShader, _physicalDevice.IsDeviceExtensionPresent("VK_NV_viewport_array2"), + _physicalDevice.IsDeviceExtensionPresent(ExtExternalMemoryHost.ExtensionName), propertiesSubgroupSizeControl.MinSubgroupSize, propertiesSubgroupSizeControl.MaxSubgroupSize, propertiesSubgroupSizeControl.RequiredSubgroupSizeStages, @@ -319,6 +321,9 @@ namespace Ryujinx.Graphics.Vulkan MemoryAllocator = new MemoryAllocator(Api, _physicalDevice, _device); + Api.TryGetDeviceExtension(_instance.Instance, _device, out ExtExternalMemoryHost hostMemoryApi); + HostMemoryAllocator = new HostMemoryAllocator(MemoryAllocator, Api, hostMemoryApi, _device); + CommandBufferPool = new CommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex); DescriptorSetManager = new DescriptorSetManager(_device); @@ -375,11 +380,21 @@ namespace Ryujinx.Graphics.Vulkan _initialized = true; } + public BufferHandle CreateBuffer(int size, BufferAccess access) + { + return BufferManager.CreateWithHandle(this, size, access.Convert()); + } + public BufferHandle CreateBuffer(int size, BufferHandle storageHint) { return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint); } + public BufferHandle CreateBuffer(nint pointer, int size) + { + return BufferManager.CreateHostImported(this, pointer, size); + } + public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info) { bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute; @@ -816,5 +831,11 @@ namespace Ryujinx.Graphics.Vulkan // Last step destroy the instance _instance.Dispose(); } + + public bool PrepareHostMapping(nint address, ulong size) + { + return Capabilities.SupportsHostImportedMemory && + HostMemoryAllocator.TryImport(BufferManager.HostImportedBufferMemoryRequirements, BufferManager.DefaultBufferMemoryFlags, address, size); + } } } \ No newline at end of file