using Ryujinx.Graphics.Gpu.Synchronization;
using Ryujinx.Memory.Tracking;
using System;
using System.Collections.Generic;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Image
{
///
/// A tracking handle for a texture group, which represents a range of views in a storage texture.
/// Retains a list of overlapping texture views, a modified flag, and tracking for each
/// CPU VA range that the views cover.
/// Also tracks copy dependencies for the handle - references to other handles that must be kept
/// in sync with this one before use.
///
class TextureGroupHandle : ISyncActionHandler, IDisposable
{
private const int FlushBalanceIncrement = 6;
private const int FlushBalanceWriteCost = 1;
private const int FlushBalanceThreshold = 7;
private const int FlushBalanceMax = 60;
private const int FlushBalanceMin = -10;
private readonly TextureGroup _group;
private int _bindCount;
private readonly int _firstLevel;
private readonly int _firstLayer;
// Sync state for texture flush.
///
/// The sync number last registered.
///
private ulong _registeredSync;
private ulong _registeredBufferSync = ulong.MaxValue;
private ulong _registeredBufferGuestSync = ulong.MaxValue;
///
/// The sync number when the texture was last modified by GPU.
///
private ulong _modifiedSync;
///
/// Whether a tracking action is currently registered or not. (0/1)
///
private int _actionRegistered;
///
/// Whether a sync action is currently registered or not.
///
private bool _syncActionRegistered;
///
/// Determines the balance of synced writes to flushes.
/// Used to determine if the texture should always write data to a persistent buffer for flush.
///
private int _flushBalance;
///
/// The byte offset from the start of the storage of this handle.
///
public int Offset { get; }
///
/// The size in bytes covered by this handle.
///
public int Size { get; }
///
/// The base slice index for this handle.
///
public int BaseSlice { get; }
///
/// The number of slices covered by this handle.
///
public int SliceCount { get; }
///
/// The textures which this handle overlaps with.
///
public List Overlaps { get; }
///
/// The CPU memory tracking handles that cover this handle.
///
public RegionHandle[] Handles { get; }
///
/// True if a texture overlapping this handle has been modified. Is set false when the flush action is called.
///
public bool Modified { get; set; }
///
/// Dependencies to handles from other texture groups.
///
public List Dependencies { get; }
///
/// A flag indicating that a copy is required from one of the dependencies.
///
public bool NeedsCopy => DeferredCopy != null;
///
/// A data copy that must be acknowledged the next time this handle is used.
///
public TextureGroupHandle DeferredCopy { get; set; }
///
/// Create a new texture group handle, representing a range of views in a storage texture.
///
/// The TextureGroup that the handle belongs to
/// The byte offset from the start of the storage of the handle
/// The size in bytes covered by the handle
/// All views of the storage texture, used to calculate overlaps
/// The first layer of this handle in the storage texture
/// The first level of this handle in the storage texture
/// The base slice index of this handle
/// The number of slices this handle covers
/// The memory tracking handles that cover this handle
public TextureGroupHandle(TextureGroup group,
int offset,
ulong size,
List views,
int firstLayer,
int firstLevel,
int baseSlice,
int sliceCount,
RegionHandle[] handles)
{
_group = group;
_firstLayer = firstLayer;
_firstLevel = firstLevel;
Offset = offset;
Size = (int)size;
Overlaps = new List();
Dependencies = new List();
BaseSlice = baseSlice;
SliceCount = sliceCount;
if (views != null)
{
RecalculateOverlaps(group, views);
}
Handles = handles;
if (group.Storage.Info.IsLinear)
{
// Linear textures are presumed to be used for readback initially.
_flushBalance = FlushBalanceThreshold + FlushBalanceIncrement;
}
foreach (RegionHandle handle in handles)
{
handle.RegisterDirtyEvent(DirtyAction);
}
}
///
/// The action to perform when a memory tracking handle is flipped to dirty.
/// This notifies overlapping textures that the memory needs to be synchronized.
///
private void DirtyAction()
{
// Notify all textures that belong to this handle.
_group.Storage.SignalGroupDirty();
lock (Overlaps)
{
foreach (Texture overlap in Overlaps)
{
overlap.SignalGroupDirty();
}
}
DeferredCopy = null;
}
///
/// Discards all data for this handle.
/// This clears all dirty flags, modified flags, and pending copies from other handles.
///
public void DiscardData()
{
Modified = false;
DeferredCopy = null;
foreach (RegionHandle handle in Handles)
{
if (handle.Dirty)
{
handle.Reprotect();
}
}
}
///
/// Calculate a list of which views overlap this handle.
///
/// The parent texture group, used to find a view's base CPU VA offset
/// The list of views to search for overlaps
public void RecalculateOverlaps(TextureGroup group, List views)
{
// Overlaps can be accessed from the memory tracking signal handler, so access must be atomic.
lock (Overlaps)
{
int endOffset = Offset + Size;
Overlaps.Clear();
foreach (Texture view in views)
{
int viewOffset = group.FindOffset(view);
if (viewOffset < endOffset && Offset < viewOffset + (int)view.Size)
{
Overlaps.Add(view);
}
}
}
}
///
/// Determine if the next sync will copy into the flush buffer.
///
/// True if it will copy, false otherwise
private bool NextSyncCopies()
{
return _flushBalance - FlushBalanceWriteCost > FlushBalanceThreshold;
}
///
/// Alters the flush balance by the given value. Should increase significantly with each sync, decrease with each write.
/// A flush balance higher than the threshold will cause a texture to repeatedly copy to a flush buffer on each use.
///
/// Value to add to the existing flush balance
/// True if the new balance is over the threshold, false otherwise
private bool ModifyFlushBalance(int modifier)
{
int result;
int existingBalance;
do
{
existingBalance = _flushBalance;
result = Math.Max(FlushBalanceMin, Math.Min(FlushBalanceMax, existingBalance + modifier));
}
while (Interlocked.CompareExchange(ref _flushBalance, result, existingBalance) != existingBalance);
return result > FlushBalanceThreshold;
}
///
/// Adds a single texture view as an overlap if its range overlaps.
///
/// The offset of the view in the group
/// The texture to add as an overlap
public void AddOverlap(int offset, Texture view)
{
// Overlaps can be accessed from the memory tracking signal handler, so access must be atomic.
if (OverlapsWith(offset, (int)view.Size))
{
lock (Overlaps)
{
Overlaps.Add(view);
}
}
}
///
/// Removes a single texture view as an overlap if its range overlaps.
///
/// The offset of the view in the group
/// The texture to add as an overlap
public void RemoveOverlap(int offset, Texture view)
{
// Overlaps can be accessed from the memory tracking signal handler, so access must be atomic.
if (OverlapsWith(offset, (int)view.Size))
{
lock (Overlaps)
{
Overlaps.Remove(view);
}
}
}
///
/// Registers a sync action to happen for this handle, and an interim flush action on the tracking handle.
///
/// The GPU context to register a sync action on
private void RegisterSync(GpuContext context)
{
if (!_syncActionRegistered)
{
_modifiedSync = context.SyncNumber;
context.RegisterSyncAction(this, true);
_syncActionRegistered = true;
}
if (Interlocked.Exchange(ref _actionRegistered, 1) == 0)
{
_group.RegisterAction(this);
}
}
///
/// Signal that this handle has been modified to any existing dependencies, and set the modified flag.
///
/// The GPU context to register a sync action on
public void SignalModified(GpuContext context)
{
Modified = true;
// If this handle has any copy dependencies, notify the other handle that a copy needs to be performed.
foreach (TextureDependency dependency in Dependencies)
{
dependency.SignalModified();
}
RegisterSync(context);
}
///
/// Signal that this handle has either started or ended being modified.
///
/// True if this handle is being bound, false if unbound
/// The GPU context to register a sync action on
public void SignalModifying(bool bound, GpuContext context)
{
SignalModified(context);
if (!bound && _syncActionRegistered && NextSyncCopies())
{
// On unbind, textures that flush often should immediately create sync so their result can be obtained as soon as possible.
context.CreateHostSyncIfNeeded(HostSyncFlags.Force);
}
// Note: Bind count currently resets to 0 on inherit for safety, as the handle <-> view relationship can change.
_bindCount = Math.Max(0, _bindCount + (bound ? 1 : -1));
}
///
/// Synchronize dependent textures, if any of them have deferred a copy from this texture.
///
public void SynchronizeDependents()
{
foreach (TextureDependency dependency in Dependencies)
{
TextureGroupHandle otherHandle = dependency.Other.Handle;
if (otherHandle.DeferredCopy == this)
{
otherHandle._group.Storage.SynchronizeMemory();
}
}
}
///
/// Wait for the latest sync number that the texture handle was written to,
/// removing the modified flag if it was reached, or leaving it set if it has not yet been created.
///
/// The GPU context used to wait for sync
/// True if the texture data can be read from the flush buffer
public bool Sync(GpuContext context)
{
// Currently assumes the calling thread is a guest thread.
bool inBuffer = _registeredBufferGuestSync != ulong.MaxValue;
ulong sync = inBuffer ? _registeredBufferGuestSync : _registeredSync;
long diff = (long)(context.SyncNumber - sync);
ModifyFlushBalance(FlushBalanceIncrement);
if (diff > 0)
{
context.Renderer.WaitSync(sync);
if ((long)(_modifiedSync - sync) > 0)
{
// Flush the data in a previous state. Do not remove the modified flag - it will be removed to ignore following writes.
return inBuffer;
}
Modified = false;
return inBuffer;
}
// If the difference is <= 0, no data is not ready yet. Flush any data we can without waiting or removing modified flag.
return false;
}
///
/// Clears the action registered variable, indicating that the tracking action should be
/// re-registered on the next modification.
///
public void ClearActionRegistered()
{
Interlocked.Exchange(ref _actionRegistered, 0);
}
///
/// Action to perform before a sync number is registered after modification.
/// This action will copy the texture data to the flush buffer if this texture
/// flushes often enough, which is determined by the flush balance.
///
///
public void SyncPreAction(bool syncpoint)
{
if (syncpoint || NextSyncCopies())
{
if (ModifyFlushBalance(0) && _registeredBufferSync != _modifiedSync)
{
_group.FlushIntoBuffer(this);
_registeredBufferSync = _modifiedSync;
}
}
}
///
/// Action to perform when a sync number is registered after modification.
/// This action will register a read tracking action on the memory tracking handle so that a flush from CPU can happen.
///
///
public bool SyncAction(bool syncpoint)
{
// The storage will need to signal modified again to update the sync number in future.
_group.Storage.SignalModifiedDirty();
bool lastInBuffer = _registeredBufferSync == _modifiedSync;
if (!lastInBuffer)
{
_registeredBufferSync = ulong.MaxValue;
}
lock (Overlaps)
{
foreach (Texture texture in Overlaps)
{
texture.SignalModifiedDirty();
}
}
// Register region tracking for CPU? (again)
_registeredSync = _modifiedSync;
_syncActionRegistered = false;
if (Interlocked.Exchange(ref _actionRegistered, 1) == 0)
{
_group.RegisterAction(this);
}
if (syncpoint)
{
_registeredBufferGuestSync = _registeredBufferSync;
}
// If the last modification is in the buffer, keep this sync action alive until it sees a syncpoint.
return syncpoint || !lastInBuffer;
}
///
/// Signal that a copy dependent texture has been modified, and must have its data copied to this one.
///
/// The texture handle that must defer a copy to this one
public void DeferCopy(TextureGroupHandle copyFrom)
{
Modified = false;
DeferredCopy = copyFrom;
_group.Storage.SignalGroupDirty();
foreach (Texture overlap in Overlaps)
{
overlap.SignalGroupDirty();
}
}
///
/// Create a copy dependency between this handle, and another.
///
/// The handle to create a copy dependency to
/// True if a copy should be deferred to all of the other handle's dependencies
public void CreateCopyDependency(TextureGroupHandle other, bool copyToOther = false)
{
// Does this dependency already exist?
foreach (TextureDependency existing in Dependencies)
{
if (existing.Other.Handle == other)
{
// Do not need to create it again. May need to set the dirty flag.
return;
}
}
_group.HasCopyDependencies = true;
other._group.HasCopyDependencies = true;
TextureDependency dependency = new(this);
TextureDependency otherDependency = new(other);
dependency.Other = otherDependency;
otherDependency.Other = dependency;
Dependencies.Add(dependency);
other.Dependencies.Add(otherDependency);
// Recursively create dependency:
// All of this handle's dependencies must depend on the other.
foreach (TextureDependency existing in Dependencies.ToArray())
{
if (existing != dependency && existing.Other.Handle != other)
{
existing.Other.Handle.CreateCopyDependency(other);
}
}
// All of the other handle's dependencies must depend on this.
foreach (TextureDependency existing in other.Dependencies.ToArray())
{
if (existing != otherDependency && existing.Other.Handle != this)
{
existing.Other.Handle.CreateCopyDependency(this);
if (copyToOther && Modified)
{
existing.Other.Handle.DeferCopy(this);
}
}
}
}
///
/// Remove a dependency from this handle's dependency list.
///
/// The dependency to remove
public void RemoveDependency(TextureDependency dependency)
{
Dependencies.Remove(dependency);
}
///
/// Check if any of this handle's memory tracking handles are dirty.
///
/// True if at least one of the handles is dirty
private bool CheckDirty()
{
return Array.Exists(Handles, handle => handle.Dirty);
}
///
/// Perform a copy from the provided handle to this one, or perform a deferred copy if none is provided.
///
/// GPU context to register sync for modified handles
/// The handle to copy from. If not provided, this method will copy from and clear the deferred copy instead
/// True if the copy was performed, false otherwise
public bool Copy(GpuContext context, TextureGroupHandle fromHandle = null)
{
bool result = false;
bool shouldCopy = false;
if (fromHandle == null)
{
fromHandle = DeferredCopy;
if (fromHandle != null)
{
// Only copy if the copy texture is still modified.
// DeferredCopy will be set to null if new data is written from CPU (see the DirtyAction method).
// It will also set as unmodified if a copy is deferred to it.
shouldCopy = true;
if (fromHandle._bindCount == 0)
{
// Repeat the copy in future if the bind count is greater than 0.
DeferredCopy = null;
}
}
}
else
{
// Copies happen directly when initializing a copy dependency.
// If dirty, do not copy. Its data no longer matters, and this handle should also be dirty.
// Also, only direct copy if the data in this handle is not already modified (can be set by copies from modified handles).
shouldCopy = !fromHandle.CheckDirty() && (fromHandle.Modified || !Modified);
}
if (shouldCopy)
{
Texture from = fromHandle._group.Storage;
Texture to = _group.Storage;
if (from.ScaleFactor != to.ScaleFactor)
{
to.PropagateScale(from);
}
from.HostTexture.CopyTo(
to.HostTexture,
fromHandle._firstLayer,
_firstLayer,
fromHandle._firstLevel,
_firstLevel);
if (fromHandle.Modified)
{
Modified = true;
RegisterSync(context);
}
result = true;
}
return result;
}
///
/// Check if this handle has a dependency to a given texture group.
///
/// The texture group to check for
/// True if there is a dependency, false otherwise
public bool HasDependencyTo(TextureGroup group)
{
foreach (TextureDependency dep in Dependencies)
{
if (dep.Other.Handle._group == group)
{
return true;
}
}
return false;
}
///
/// Inherit modified flags and dependencies from another texture handle.
///
/// The texture handle to inherit from
/// Whether the handle should inherit copy dependencies or not
public void Inherit(TextureGroupHandle old, bool withCopies)
{
Modified |= old.Modified;
if (withCopies)
{
foreach (TextureDependency dependency in old.Dependencies.ToArray())
{
CreateCopyDependency(dependency.Other.Handle);
if (dependency.Other.Handle.DeferredCopy == old)
{
dependency.Other.Handle.DeferredCopy = this;
}
}
DeferredCopy = old.DeferredCopy;
}
}
///
/// Check if this region overlaps with another.
///
/// Base address
/// Size of the region
/// True if overlapping, false otherwise
public bool OverlapsWith(int offset, int size)
{
return Offset < offset + size && offset < Offset + Size;
}
///
/// Dispose this texture group handle, removing all its dependencies and disposing its memory tracking handles.
///
public void Dispose()
{
foreach (RegionHandle handle in Handles)
{
handle.Dispose();
}
foreach (TextureDependency dependency in Dependencies.ToArray())
{
dependency.Other.Handle.RemoveDependency(dependency.Other);
}
}
}
}