using Ryujinx.Common.Memory; using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Gpu.Shader.DiskCache; using Ryujinx.Graphics.Shader; using System; using System.Collections.Generic; using System.Linq; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Shader { class ShaderSpecializationState { private const uint ComsMagic = (byte)'C' | ((byte)'O' << 8) | ((byte)'M' << 16) | ((byte)'S' << 24); private const uint GfxsMagic = (byte)'G' | ((byte)'F' << 8) | ((byte)'X' << 16) | ((byte)'S' << 24); private const uint TfbdMagic = (byte)'T' | ((byte)'F' << 8) | ((byte)'B' << 16) | ((byte)'D' << 24); private const uint TexkMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'K' << 24); private const uint TexsMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'S' << 24); /// /// Flags indicating GPU state that is used by the shader. /// [Flags] private enum QueriedStateFlags { EarlyZForce = 1 << 0, PrimitiveTopology = 1 << 1, TessellationMode = 1 << 2, TransformFeedback = 1 << 3 } private QueriedStateFlags _queriedState; private bool _compute; private byte _constantBufferUsePerStage; /// /// Compute engine state. /// public GpuChannelComputeState ComputeState; /// /// 3D engine state. /// public GpuChannelGraphicsState GraphicsState; /// /// Contant buffers bound at the time the shader was compiled, per stage. /// public Array5 ConstantBufferUse; /// /// Transform feedback buffers active at the time the shader was compiled. /// public TransformFeedbackDescriptor[] TransformFeedbackDescriptors; /// /// Flags indicating texture state that is used by the shader. /// [Flags] private enum QueriedTextureStateFlags { TextureFormat = 1 << 0, SamplerType = 1 << 1, CoordNormalized = 1 << 2 } /// /// Reference type wrapping a value. /// private class Box { /// /// Wrapped value. /// public T Value; } /// /// State of a texture or image that is accessed by the shader. /// private struct TextureSpecializationState { // New fields should be added to the end of the struct to keep disk shader cache compatibility. /// /// Flags indicating which state of the texture the shader depends on. /// public QueriedTextureStateFlags QueriedFlags; /// /// Encoded texture format value. /// public uint Format; /// /// True if the texture format is sRGB, false otherwise. /// public bool FormatSrgb; /// /// Texture target. /// public Image.TextureTarget TextureTarget; /// /// Indicates if the coordinates used to sample the texture are normalized or not (0.0..1.0 or 0..Width/Height). /// public bool CoordNormalized; } /// /// Texture binding information, used to identify each texture accessed by the shader. /// private struct TextureKey : IEquatable { // New fields should be added to the end of the struct to keep disk shader cache compatibility. /// /// Shader stage where the texture is used. /// public readonly int StageIndex; /// /// Texture handle offset in words on the texture buffer. /// public readonly int Handle; /// /// Constant buffer slot of the texture buffer (-1 to use the texture buffer index GPU register). /// public readonly int CbufSlot; /// /// Creates a new texture key. /// /// Shader stage where the texture is used /// Texture handle offset in words on the texture buffer /// Constant buffer slot of the texture buffer (-1 to use the texture buffer index GPU register) public TextureKey(int stageIndex, int handle, int cbufSlot) { StageIndex = stageIndex; Handle = handle; CbufSlot = cbufSlot; } public override bool Equals(object obj) { return obj is TextureKey textureKey && Equals(textureKey); } public bool Equals(TextureKey other) { return StageIndex == other.StageIndex && Handle == other.Handle && CbufSlot == other.CbufSlot; } public override int GetHashCode() { return HashCode.Combine(StageIndex, Handle, CbufSlot); } } private readonly Dictionary> _textureSpecialization; private KeyValuePair>[] _allTextures; private Box[][] _textureByBinding; private Box[][] _imageByBinding; /// /// Creates a new instance of the shader specialization state. /// private ShaderSpecializationState() { _textureSpecialization = new Dictionary>(); } /// /// Creates a new instance of the shader specialization state. /// /// Current compute engine state public ShaderSpecializationState(GpuChannelComputeState state) : this() { ComputeState = state; _compute = true; } /// /// Creates a new instance of the shader specialization state. /// /// Current 3D engine state /// Optional transform feedback buffers in use, if any public ShaderSpecializationState(GpuChannelGraphicsState state, TransformFeedbackDescriptor[] descriptors) : this() { GraphicsState = state; _compute = false; if (descriptors != null) { TransformFeedbackDescriptors = descriptors; _queriedState |= QueriedStateFlags.TransformFeedback; } } /// /// Prepare the shader specialization state for quick binding lookups. /// /// The shader stages public void Prepare(CachedShaderStage[] stages) { _allTextures = _textureSpecialization.ToArray(); _textureByBinding = new Box[stages.Length][]; _imageByBinding = new Box[stages.Length][]; for (int i = 0; i < stages.Length; i++) { CachedShaderStage stage = stages[i]; if (stage?.Info != null) { var textures = stage.Info.Textures; var images = stage.Info.Images; var texBindings = new Box[textures.Count]; var imageBindings = new Box[images.Count]; int stageIndex = Math.Max(i - 1, 0); // Don't count VertexA for looking up spec state. No-Op for compute. for (int j = 0; j < textures.Count; j++) { var texture = textures[j]; texBindings[j] = GetTextureSpecState(stageIndex, texture.HandleIndex, texture.CbufSlot); } for (int j = 0; j < images.Count; j++) { var image = images[j]; imageBindings[j] = GetTextureSpecState(stageIndex, image.HandleIndex, image.CbufSlot); } _textureByBinding[i] = texBindings; _imageByBinding[i] = imageBindings; } } } /// /// Indicates that the shader accesses the early Z force state. /// public void RecordEarlyZForce() { _queriedState |= QueriedStateFlags.EarlyZForce; } /// /// Indicates that the shader accesses the primitive topology state. /// public void RecordPrimitiveTopology() { _queriedState |= QueriedStateFlags.PrimitiveTopology; } /// /// Indicates that the shader accesses the tessellation mode state. /// public void RecordTessellationMode() { _queriedState |= QueriedStateFlags.TessellationMode; } /// /// Indicates that the shader accesses the constant buffer use state. /// /// Shader stage index /// Mask indicating the constant buffers bound at the time of the shader compilation public void RecordConstantBufferUse(int stageIndex, uint useMask) { ConstantBufferUse[stageIndex] = useMask; _constantBufferUsePerStage |= (byte)(1 << stageIndex); } /// /// Indicates that a given texture is accessed by the shader. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Descriptor of the texture public void RegisterTexture(int stageIndex, int handle, int cbufSlot, Image.TextureDescriptor descriptor) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.Format = descriptor.UnpackFormat(); state.Value.FormatSrgb = descriptor.UnpackSrgb(); state.Value.TextureTarget = descriptor.UnpackTextureTarget(); state.Value.CoordNormalized = descriptor.UnpackTextureCoordNormalized(); } /// /// Indicates that a given texture is accessed by the shader. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Maxwell texture format value /// Whenever the texture format is a sRGB format /// Texture target type /// Whenever the texture coordinates used on the shader are considered normalized public void RegisterTexture( int stageIndex, int handle, int cbufSlot, uint format, bool formatSrgb, Image.TextureTarget target, bool coordNormalized) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.Format = format; state.Value.FormatSrgb = formatSrgb; state.Value.TextureTarget = target; state.Value.CoordNormalized = coordNormalized; } /// /// Indicates that the format of a given texture was used during the shader translation process. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public void RecordTextureFormat(int stageIndex, int handle, int cbufSlot) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.QueriedFlags |= QueriedTextureStateFlags.TextureFormat; } /// /// Indicates that the target of a given texture was used during the shader translation process. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public void RecordTextureSamplerType(int stageIndex, int handle, int cbufSlot) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.QueriedFlags |= QueriedTextureStateFlags.SamplerType; } /// /// Indicates that the coordinate normalization state of a given texture was used during the shader translation process. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public void RecordTextureCoordNormalized(int stageIndex, int handle, int cbufSlot) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.QueriedFlags |= QueriedTextureStateFlags.CoordNormalized; } /// /// Checks if a given texture was registerd on this specialization state. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public bool TextureRegistered(int stageIndex, int handle, int cbufSlot) { return GetTextureSpecState(stageIndex, handle, cbufSlot) != null; } /// /// Gets the recorded format of a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public (uint, bool) GetFormat(int stageIndex, int handle, int cbufSlot) { TextureSpecializationState state = GetTextureSpecState(stageIndex, handle, cbufSlot).Value; return (state.Format, state.FormatSrgb); } /// /// Gets the recorded target of a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public Image.TextureTarget GetTextureTarget(int stageIndex, int handle, int cbufSlot) { return GetTextureSpecState(stageIndex, handle, cbufSlot).Value.TextureTarget; } /// /// Gets the recorded coordinate normalization state of a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public bool GetCoordNormalized(int stageIndex, int handle, int cbufSlot) { return GetTextureSpecState(stageIndex, handle, cbufSlot).Value.CoordNormalized; } /// /// Gets texture specialization state for a given texture, or create a new one if not present. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Texture specialization state private Box GetOrCreateTextureSpecState(int stageIndex, int handle, int cbufSlot) { TextureKey key = new TextureKey(stageIndex, handle, cbufSlot); if (!_textureSpecialization.TryGetValue(key, out Box state)) { _textureSpecialization.Add(key, state = new Box()); } return state; } /// /// Gets texture specialization state for a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Texture specialization state private Box GetTextureSpecState(int stageIndex, int handle, int cbufSlot) { TextureKey key = new TextureKey(stageIndex, handle, cbufSlot); if (_textureSpecialization.TryGetValue(key, out Box state)) { return state; } return null; } /// /// Checks if the recorded state matches the current GPU 3D engine state. /// /// GPU channel /// Texture pool state /// Graphics state /// Indicates whether texture descriptors should be checked /// True if the state matches, false otherwise public bool MatchesGraphics(GpuChannel channel, GpuChannelPoolState poolState, GpuChannelGraphicsState graphicsState, bool checkTextures) { if (graphicsState.ViewportTransformDisable != GraphicsState.ViewportTransformDisable) { return false; } bool thisA2cDitherEnable = GraphicsState.AlphaToCoverageEnable && GraphicsState.AlphaToCoverageDitherEnable; bool otherA2cDitherEnable = graphicsState.AlphaToCoverageEnable && graphicsState.AlphaToCoverageDitherEnable; if (otherA2cDitherEnable != thisA2cDitherEnable) { return false; } return Matches(channel, poolState, checkTextures, isCompute: false); } /// /// Checks if the recorded state matches the current GPU compute engine state. /// /// GPU channel /// Texture pool state /// Indicates whether texture descriptors should be checked /// True if the state matches, false otherwise public bool MatchesCompute(GpuChannel channel, GpuChannelPoolState poolState, bool checkTextures) { return Matches(channel, poolState, checkTextures, isCompute: true); } /// /// Fetch the constant buffers used for a texture to cache. /// /// GPU channel /// Indicates whenever the check is requested by the 3D or compute engine /// The currently cached texture buffer index /// The currently cached sampler buffer index /// The currently cached texture buffer data /// The currently cached sampler buffer data /// The currently cached stage /// The new texture buffer index /// The new sampler buffer index /// Stage index of the constant buffer [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void UpdateCachedBuffer( GpuChannel channel, bool isCompute, ref int cachedTextureBufferIndex, ref int cachedSamplerBufferIndex, ref ReadOnlySpan cachedTextureBuffer, ref ReadOnlySpan cachedSamplerBuffer, ref int cachedStageIndex, int textureBufferIndex, int samplerBufferIndex, int stageIndex) { bool stageChange = stageIndex != cachedStageIndex; if (stageChange || textureBufferIndex != cachedTextureBufferIndex) { ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex); cachedTextureBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedTextureBufferIndex = textureBufferIndex; if (samplerBufferIndex == textureBufferIndex) { cachedSamplerBuffer = cachedTextureBuffer; cachedSamplerBufferIndex = samplerBufferIndex; } } if (stageChange || samplerBufferIndex != cachedSamplerBufferIndex) { ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex); cachedSamplerBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedSamplerBufferIndex = samplerBufferIndex; } cachedStageIndex = stageIndex; } /// /// Checks if the recorded state matches the current GPU state. /// /// GPU channel /// Texture pool state /// Indicates whether texture descriptors should be checked /// Indicates whenever the check is requested by the 3D or compute engine /// True if the state matches, false otherwise private bool Matches(GpuChannel channel, GpuChannelPoolState poolState, bool checkTextures, bool isCompute) { int constantBufferUsePerStageMask = _constantBufferUsePerStage; while (constantBufferUsePerStageMask != 0) { int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask); uint useMask = isCompute ? channel.BufferManager.GetComputeUniformBufferUseMask() : channel.BufferManager.GetGraphicsUniformBufferUseMask(index); if (ConstantBufferUse[index] != useMask) { return false; } constantBufferUsePerStageMask &= ~(1 << index); } if (checkTextures) { TexturePool pool = channel.TextureManager.GetTexturePool(poolState.TexturePoolGpuVa, poolState.TexturePoolMaximumId); int cachedTextureBufferIndex = -1; int cachedSamplerBufferIndex = -1; int cachedStageIndex = -1; ReadOnlySpan cachedTextureBuffer = Span.Empty; ReadOnlySpan cachedSamplerBuffer = Span.Empty; foreach (var kv in _allTextures) { TextureKey textureKey = kv.Key; (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(textureKey.CbufSlot, poolState.TextureBufferIndex); UpdateCachedBuffer(channel, isCompute, ref cachedTextureBufferIndex, ref cachedSamplerBufferIndex, ref cachedTextureBuffer, ref cachedSamplerBuffer, ref cachedStageIndex, textureBufferIndex, samplerBufferIndex, textureKey.StageIndex); int packedId = TextureHandle.ReadPackedId(textureKey.Handle, cachedTextureBuffer, cachedSamplerBuffer); int textureId = TextureHandle.UnpackTextureId(packedId); if (pool.IsValidId(textureId)) { ref readonly Image.TextureDescriptor descriptor = ref pool.GetDescriptorRef(textureId); if (!MatchesTexture(kv.Value, descriptor)) { return false; } } } } return true; } /// /// Checks if the recorded texture state matches the given texture descriptor. /// /// Texture specialization state /// Texture descriptor /// True if the state matches, false otherwise [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool MatchesTexture(Box specializationState, in Image.TextureDescriptor descriptor) { if (specializationState != null) { if (specializationState.Value.QueriedFlags.HasFlag(QueriedTextureStateFlags.CoordNormalized) && specializationState.Value.CoordNormalized != descriptor.UnpackTextureCoordNormalized()) { return false; } } return true; } /// /// Checks if the recorded texture state for a given texture binding matches a texture descriptor. /// /// The shader stage /// The texture index /// Texture descriptor /// True if the state matches, false otherwise public bool MatchesTexture(ShaderStage stage, int index, in Image.TextureDescriptor descriptor) { Box specializationState = _textureByBinding[(int)stage][index]; return MatchesTexture(specializationState, descriptor); } /// /// Checks if the recorded texture state for a given image binding matches a texture descriptor. /// /// The shader stage /// The texture index /// Texture descriptor /// True if the state matches, false otherwise public bool MatchesImage(ShaderStage stage, int index, in Image.TextureDescriptor descriptor) { Box specializationState = _imageByBinding[(int)stage][index]; return MatchesTexture(specializationState, descriptor); } /// /// Reads shader specialization state that has been serialized. /// /// Data reader /// Shader specialization state public static ShaderSpecializationState Read(ref BinarySerializer dataReader) { ShaderSpecializationState specState = new ShaderSpecializationState(); dataReader.Read(ref specState._queriedState); dataReader.Read(ref specState._compute); if (specState._compute) { dataReader.ReadWithMagicAndSize(ref specState.ComputeState, ComsMagic); } else { dataReader.ReadWithMagicAndSize(ref specState.GraphicsState, GfxsMagic); } dataReader.Read(ref specState._constantBufferUsePerStage); int constantBufferUsePerStageMask = specState._constantBufferUsePerStage; while (constantBufferUsePerStageMask != 0) { int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask); dataReader.Read(ref specState.ConstantBufferUse[index]); constantBufferUsePerStageMask &= ~(1 << index); } if (specState._queriedState.HasFlag(QueriedStateFlags.TransformFeedback)) { ushort tfCount = 0; dataReader.Read(ref tfCount); specState.TransformFeedbackDescriptors = new TransformFeedbackDescriptor[tfCount]; for (int index = 0; index < tfCount; index++) { dataReader.ReadWithMagicAndSize(ref specState.TransformFeedbackDescriptors[index], TfbdMagic); } } ushort count = 0; dataReader.Read(ref count); for (int index = 0; index < count; index++) { TextureKey textureKey = default; Box textureState = new Box(); dataReader.ReadWithMagicAndSize(ref textureKey, TexkMagic); dataReader.ReadWithMagicAndSize(ref textureState.Value, TexsMagic); specState._textureSpecialization[textureKey] = textureState; } return specState; } /// /// Serializes the shader specialization state. /// /// Data writer public void Write(ref BinarySerializer dataWriter) { dataWriter.Write(ref _queriedState); dataWriter.Write(ref _compute); if (_compute) { dataWriter.WriteWithMagicAndSize(ref ComputeState, ComsMagic); } else { dataWriter.WriteWithMagicAndSize(ref GraphicsState, GfxsMagic); } dataWriter.Write(ref _constantBufferUsePerStage); int constantBufferUsePerStageMask = _constantBufferUsePerStage; while (constantBufferUsePerStageMask != 0) { int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask); dataWriter.Write(ref ConstantBufferUse[index]); constantBufferUsePerStageMask &= ~(1 << index); } if (_queriedState.HasFlag(QueriedStateFlags.TransformFeedback)) { ushort tfCount = (ushort)TransformFeedbackDescriptors.Length; dataWriter.Write(ref tfCount); for (int index = 0; index < TransformFeedbackDescriptors.Length; index++) { dataWriter.WriteWithMagicAndSize(ref TransformFeedbackDescriptors[index], TfbdMagic); } } ushort count = (ushort)_textureSpecialization.Count; dataWriter.Write(ref count); foreach (var kv in _textureSpecialization) { var textureKey = kv.Key; var textureState = kv.Value; dataWriter.WriteWithMagicAndSize(ref textureKey, TexkMagic); dataWriter.WriteWithMagicAndSize(ref textureState.Value, TexsMagic); } } } }