using Ryujinx.Common.Memory; using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.Graphics.Gpu.Shader.DiskCache; using Ryujinx.Graphics.Shader; using System; using System.Collections.Generic; using System.Linq; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Gpu.Shader { class ShaderSpecializationState { private const uint ComsMagic = (byte)'C' | ((byte)'O' << 8) | ((byte)'M' << 16) | ((byte)'S' << 24); private const uint GfxsMagic = (byte)'G' | ((byte)'F' << 8) | ((byte)'X' << 16) | ((byte)'S' << 24); private const uint TfbdMagic = (byte)'T' | ((byte)'F' << 8) | ((byte)'B' << 16) | ((byte)'D' << 24); private const uint TexkMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'K' << 24); private const uint TexsMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'S' << 24); private const uint PgpsMagic = (byte)'P' | ((byte)'G' << 8) | ((byte)'P' << 16) | ((byte)'S' << 24); /// /// Flags indicating GPU state that is used by the shader. /// [Flags] private enum QueriedStateFlags { EarlyZForce = 1 << 0, PrimitiveTopology = 1 << 1, TessellationMode = 1 << 2, TransformFeedback = 1 << 3 } private QueriedStateFlags _queriedState; private bool _compute; private byte _constantBufferUsePerStage; /// /// Compute engine state. /// public GpuChannelComputeState ComputeState; /// /// 3D engine state. /// public GpuChannelGraphicsState GraphicsState; /// /// Contant buffers bound at the time the shader was compiled, per stage. /// public Array5 ConstantBufferUse; /// /// Pipeline state captured at the time of shader use. /// public ProgramPipelineState? PipelineState; /// /// Transform feedback buffers active at the time the shader was compiled. /// public TransformFeedbackDescriptor[] TransformFeedbackDescriptors; /// /// Flags indicating texture state that is used by the shader. /// [Flags] private enum QueriedTextureStateFlags { TextureFormat = 1 << 0, SamplerType = 1 << 1, CoordNormalized = 1 << 2 } /// /// Reference type wrapping a value. /// private class Box { /// /// Wrapped value. /// public T Value; } /// /// State of a texture or image that is accessed by the shader. /// private struct TextureSpecializationState { // New fields should be added to the end of the struct to keep disk shader cache compatibility. /// /// Flags indicating which state of the texture the shader depends on. /// public QueriedTextureStateFlags QueriedFlags; /// /// Encoded texture format value. /// public uint Format; /// /// True if the texture format is sRGB, false otherwise. /// public bool FormatSrgb; /// /// Texture target. /// public TextureTarget TextureTarget; /// /// Indicates if the coordinates used to sample the texture are normalized or not (0.0..1.0 or 0..Width/Height). /// public bool CoordNormalized; } /// /// Texture binding information, used to identify each texture accessed by the shader. /// private readonly record struct TextureKey { // New fields should be added to the end of the struct to keep disk shader cache compatibility. /// /// Shader stage where the texture is used. /// public readonly int StageIndex; /// /// Texture handle offset in words on the texture buffer. /// public readonly int Handle; /// /// Constant buffer slot of the texture buffer (-1 to use the texture buffer index GPU register). /// public readonly int CbufSlot; /// /// Creates a new texture key. /// /// Shader stage where the texture is used /// Texture handle offset in words on the texture buffer /// Constant buffer slot of the texture buffer (-1 to use the texture buffer index GPU register) public TextureKey(int stageIndex, int handle, int cbufSlot) { StageIndex = stageIndex; Handle = handle; CbufSlot = cbufSlot; } } private readonly Dictionary> _textureSpecialization; private KeyValuePair>[] _allTextures; private Box[][] _textureByBinding; private Box[][] _imageByBinding; /// /// Creates a new instance of the shader specialization state. /// private ShaderSpecializationState() { _textureSpecialization = new Dictionary>(); } /// /// Creates a new instance of the shader specialization state. /// /// Current compute engine state public ShaderSpecializationState(ref GpuChannelComputeState state) : this() { ComputeState = state; _compute = true; } /// /// Creates a new instance of the shader specialization state. /// /// Current 3D engine state /// Optional transform feedback buffers in use, if any private ShaderSpecializationState(ref GpuChannelGraphicsState state, TransformFeedbackDescriptor[] descriptors) : this() { GraphicsState = state; _compute = false; if (descriptors != null) { TransformFeedbackDescriptors = descriptors; _queriedState |= QueriedStateFlags.TransformFeedback; } } /// /// Prepare the shader specialization state for quick binding lookups. /// /// The shader stages public void Prepare(CachedShaderStage[] stages) { _allTextures = _textureSpecialization.ToArray(); _textureByBinding = new Box[stages.Length][]; _imageByBinding = new Box[stages.Length][]; for (int i = 0; i < stages.Length; i++) { CachedShaderStage stage = stages[i]; if (stage?.Info != null) { var textures = stage.Info.Textures; var images = stage.Info.Images; var texBindings = new Box[textures.Count]; var imageBindings = new Box[images.Count]; int stageIndex = Math.Max(i - 1, 0); // Don't count VertexA for looking up spec state. No-Op for compute. for (int j = 0; j < textures.Count; j++) { var texture = textures[j]; texBindings[j] = GetTextureSpecState(stageIndex, texture.HandleIndex, texture.CbufSlot); } for (int j = 0; j < images.Count; j++) { var image = images[j]; imageBindings[j] = GetTextureSpecState(stageIndex, image.HandleIndex, image.CbufSlot); } _textureByBinding[i] = texBindings; _imageByBinding[i] = imageBindings; } } } /// /// Creates a new instance of the shader specialization state. /// /// Current 3D engine state /// Current program pipeline state /// Optional transform feedback buffers in use, if any public ShaderSpecializationState( ref GpuChannelGraphicsState state, ref ProgramPipelineState pipelineState, TransformFeedbackDescriptor[] descriptors) : this(ref state, descriptors) { PipelineState = pipelineState; } /// /// Creates a new instance of the shader specialization state. /// /// Current 3D engine state /// Current program pipeline state /// Optional transform feedback buffers in use, if any public ShaderSpecializationState( ref GpuChannelGraphicsState state, ProgramPipelineState? pipelineState, TransformFeedbackDescriptor[] descriptors) : this(ref state, descriptors) { PipelineState = pipelineState; } /// /// Indicates that the shader accesses the early Z force state. /// public void RecordEarlyZForce() { _queriedState |= QueriedStateFlags.EarlyZForce; } /// /// Indicates that the shader accesses the primitive topology state. /// public void RecordPrimitiveTopology() { _queriedState |= QueriedStateFlags.PrimitiveTopology; } /// /// Indicates that the shader accesses the tessellation mode state. /// public void RecordTessellationMode() { _queriedState |= QueriedStateFlags.TessellationMode; } /// /// Indicates that the shader accesses the constant buffer use state. /// /// Shader stage index /// Mask indicating the constant buffers bound at the time of the shader compilation public void RecordConstantBufferUse(int stageIndex, uint useMask) { ConstantBufferUse[stageIndex] = useMask; _constantBufferUsePerStage |= (byte)(1 << stageIndex); } /// /// Indicates that a given texture is accessed by the shader. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Descriptor of the texture public void RegisterTexture(int stageIndex, int handle, int cbufSlot, Image.TextureDescriptor descriptor) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.Format = descriptor.UnpackFormat(); state.Value.FormatSrgb = descriptor.UnpackSrgb(); state.Value.TextureTarget = descriptor.UnpackTextureTarget(); state.Value.CoordNormalized = descriptor.UnpackTextureCoordNormalized(); } /// /// Indicates that a given texture is accessed by the shader. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Maxwell texture format value /// Whenever the texture format is a sRGB format /// Texture target type /// Whenever the texture coordinates used on the shader are considered normalized public void RegisterTexture( int stageIndex, int handle, int cbufSlot, uint format, bool formatSrgb, TextureTarget target, bool coordNormalized) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.Format = format; state.Value.FormatSrgb = formatSrgb; state.Value.TextureTarget = target; state.Value.CoordNormalized = coordNormalized; } /// /// Indicates that the format of a given texture was used during the shader translation process. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public void RecordTextureFormat(int stageIndex, int handle, int cbufSlot) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.QueriedFlags |= QueriedTextureStateFlags.TextureFormat; } /// /// Indicates that the target of a given texture was used during the shader translation process. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public void RecordTextureSamplerType(int stageIndex, int handle, int cbufSlot) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.QueriedFlags |= QueriedTextureStateFlags.SamplerType; } /// /// Indicates that the coordinate normalization state of a given texture was used during the shader translation process. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public void RecordTextureCoordNormalized(int stageIndex, int handle, int cbufSlot) { Box state = GetOrCreateTextureSpecState(stageIndex, handle, cbufSlot); state.Value.QueriedFlags |= QueriedTextureStateFlags.CoordNormalized; } /// /// Checks if primitive topology was queried by the shader. /// /// True if queried, false otherwise public bool IsPrimitiveTopologyQueried() { return _queriedState.HasFlag(QueriedStateFlags.PrimitiveTopology); } /// /// Checks if a given texture was registerd on this specialization state. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public bool TextureRegistered(int stageIndex, int handle, int cbufSlot) { return GetTextureSpecState(stageIndex, handle, cbufSlot) != null; } /// /// Gets the recorded format of a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public (uint, bool) GetFormat(int stageIndex, int handle, int cbufSlot) { TextureSpecializationState state = GetTextureSpecState(stageIndex, handle, cbufSlot).Value; return (state.Format, state.FormatSrgb); } /// /// Gets the recorded target of a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public TextureTarget GetTextureTarget(int stageIndex, int handle, int cbufSlot) { return GetTextureSpecState(stageIndex, handle, cbufSlot).Value.TextureTarget; } /// /// Gets the recorded coordinate normalization state of a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer public bool GetCoordNormalized(int stageIndex, int handle, int cbufSlot) { return GetTextureSpecState(stageIndex, handle, cbufSlot).Value.CoordNormalized; } /// /// Gets texture specialization state for a given texture, or create a new one if not present. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Texture specialization state private Box GetOrCreateTextureSpecState(int stageIndex, int handle, int cbufSlot) { TextureKey key = new TextureKey(stageIndex, handle, cbufSlot); if (!_textureSpecialization.TryGetValue(key, out Box state)) { _textureSpecialization.Add(key, state = new Box()); } return state; } /// /// Gets texture specialization state for a given texture. /// /// Shader stage where the texture is used /// Offset in words of the texture handle on the texture buffer /// Slot of the texture buffer constant buffer /// Texture specialization state private Box GetTextureSpecState(int stageIndex, int handle, int cbufSlot) { TextureKey key = new TextureKey(stageIndex, handle, cbufSlot); if (_textureSpecialization.TryGetValue(key, out Box state)) { return state; } return null; } /// /// Checks if the recorded state matches the current GPU 3D engine state. /// /// GPU channel /// Texture pool state /// Graphics state /// Indicates whether the vertex shader accesses draw parameters /// Indicates whether texture descriptors should be checked /// True if the state matches, false otherwise public bool MatchesGraphics( GpuChannel channel, ref GpuChannelPoolState poolState, ref GpuChannelGraphicsState graphicsState, bool usesDrawParameters, bool checkTextures) { if (graphicsState.ViewportTransformDisable != GraphicsState.ViewportTransformDisable) { return false; } bool thisA2cDitherEnable = GraphicsState.AlphaToCoverageEnable && GraphicsState.AlphaToCoverageDitherEnable; bool otherA2cDitherEnable = graphicsState.AlphaToCoverageEnable && graphicsState.AlphaToCoverageDitherEnable; if (otherA2cDitherEnable != thisA2cDitherEnable) { return false; } if (graphicsState.DepthMode != GraphicsState.DepthMode) { return false; } if (graphicsState.AlphaTestEnable != GraphicsState.AlphaTestEnable) { return false; } if (graphicsState.AlphaTestEnable && (graphicsState.AlphaTestCompare != GraphicsState.AlphaTestCompare || graphicsState.AlphaTestReference != GraphicsState.AlphaTestReference)) { return false; } if (!graphicsState.AttributeTypes.AsSpan().SequenceEqual(GraphicsState.AttributeTypes.AsSpan())) { return false; } if (usesDrawParameters && graphicsState.HasConstantBufferDrawParameters != GraphicsState.HasConstantBufferDrawParameters) { return false; } if (graphicsState.HasUnalignedStorageBuffer != GraphicsState.HasUnalignedStorageBuffer) { return false; } if (channel.Capabilities.NeedsFragmentOutputSpecialization && !graphicsState.FragmentOutputTypes.AsSpan().SequenceEqual(GraphicsState.FragmentOutputTypes.AsSpan())) { return false; } if (graphicsState.DualSourceBlendEnable != GraphicsState.DualSourceBlendEnable) { return false; } return Matches(channel, ref poolState, checkTextures, isCompute: false); } /// /// Checks if the recorded state matches the current GPU compute engine state. /// /// GPU channel /// Texture pool state /// Compute state /// Indicates whether texture descriptors should be checked /// True if the state matches, false otherwise public bool MatchesCompute(GpuChannel channel, ref GpuChannelPoolState poolState, GpuChannelComputeState computeState, bool checkTextures) { if (computeState.HasUnalignedStorageBuffer != ComputeState.HasUnalignedStorageBuffer) { return false; } return Matches(channel, ref poolState, checkTextures, isCompute: true); } /// /// Fetch the constant buffers used for a texture to cache. /// /// GPU channel /// Indicates whenever the check is requested by the 3D or compute engine /// The currently cached texture buffer index /// The currently cached sampler buffer index /// The currently cached texture buffer data /// The currently cached sampler buffer data /// The currently cached stage /// The new texture buffer index /// The new sampler buffer index /// Stage index of the constant buffer [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void UpdateCachedBuffer( GpuChannel channel, bool isCompute, scoped ref int cachedTextureBufferIndex, scoped ref int cachedSamplerBufferIndex, scoped ref ReadOnlySpan cachedTextureBuffer, scoped ref ReadOnlySpan cachedSamplerBuffer, scoped ref int cachedStageIndex, int textureBufferIndex, int samplerBufferIndex, int stageIndex) { bool stageChange = stageIndex != cachedStageIndex; if (stageChange || textureBufferIndex != cachedTextureBufferIndex) { ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex); cachedTextureBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedTextureBufferIndex = textureBufferIndex; if (samplerBufferIndex == textureBufferIndex) { cachedSamplerBuffer = cachedTextureBuffer; cachedSamplerBufferIndex = samplerBufferIndex; } } if (stageChange || samplerBufferIndex != cachedSamplerBufferIndex) { ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex); cachedSamplerBuffer = MemoryMarshal.Cast(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size)); cachedSamplerBufferIndex = samplerBufferIndex; } cachedStageIndex = stageIndex; } /// /// Checks if the recorded state matches the current GPU state. /// /// GPU channel /// Texture pool state /// Indicates whether texture descriptors should be checked /// Indicates whenever the check is requested by the 3D or compute engine /// True if the state matches, false otherwise private bool Matches(GpuChannel channel, ref GpuChannelPoolState poolState, bool checkTextures, bool isCompute) { int constantBufferUsePerStageMask = _constantBufferUsePerStage; while (constantBufferUsePerStageMask != 0) { int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask); uint useMask = isCompute ? channel.BufferManager.GetComputeUniformBufferUseMask() : channel.BufferManager.GetGraphicsUniformBufferUseMask(index); if (ConstantBufferUse[index] != useMask) { return false; } constantBufferUsePerStageMask &= ~(1 << index); } if (checkTextures) { TexturePool pool = channel.TextureManager.GetTexturePool(poolState.TexturePoolGpuVa, poolState.TexturePoolMaximumId); int cachedTextureBufferIndex = -1; int cachedSamplerBufferIndex = -1; int cachedStageIndex = -1; ReadOnlySpan cachedTextureBuffer = Span.Empty; ReadOnlySpan cachedSamplerBuffer = Span.Empty; foreach (var kv in _allTextures) { TextureKey textureKey = kv.Key; (int textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(textureKey.CbufSlot, poolState.TextureBufferIndex); UpdateCachedBuffer(channel, isCompute, ref cachedTextureBufferIndex, ref cachedSamplerBufferIndex, ref cachedTextureBuffer, ref cachedSamplerBuffer, ref cachedStageIndex, textureBufferIndex, samplerBufferIndex, textureKey.StageIndex); int packedId = TextureHandle.ReadPackedId(textureKey.Handle, cachedTextureBuffer, cachedSamplerBuffer); int textureId = TextureHandle.UnpackTextureId(packedId); if (pool.IsValidId(textureId)) { ref readonly Image.TextureDescriptor descriptor = ref pool.GetDescriptorRef(textureId); if (!MatchesTexture(kv.Value, descriptor)) { return false; } } } } return true; } /// /// Checks if the recorded texture state matches the given texture descriptor. /// /// Texture specialization state /// Texture descriptor /// True if the state matches, false otherwise [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool MatchesTexture(Box specializationState, in Image.TextureDescriptor descriptor) { if (specializationState != null) { if (specializationState.Value.QueriedFlags.HasFlag(QueriedTextureStateFlags.CoordNormalized) && specializationState.Value.CoordNormalized != descriptor.UnpackTextureCoordNormalized()) { return false; } } return true; } /// /// Checks if the recorded texture state for a given texture binding matches a texture descriptor. /// /// The shader stage /// The texture index /// Texture descriptor /// True if the state matches, false otherwise public bool MatchesTexture(ShaderStage stage, int index, in Image.TextureDescriptor descriptor) { Box specializationState = _textureByBinding[(int)stage][index]; return MatchesTexture(specializationState, descriptor); } /// /// Checks if the recorded texture state for a given image binding matches a texture descriptor. /// /// The shader stage /// The texture index /// Texture descriptor /// True if the state matches, false otherwise public bool MatchesImage(ShaderStage stage, int index, in Image.TextureDescriptor descriptor) { Box specializationState = _imageByBinding[(int)stage][index]; return MatchesTexture(specializationState, descriptor); } /// /// Reads shader specialization state that has been serialized. /// /// Data reader /// Shader specialization state public static ShaderSpecializationState Read(ref BinarySerializer dataReader) { ShaderSpecializationState specState = new ShaderSpecializationState(); dataReader.Read(ref specState._queriedState); dataReader.Read(ref specState._compute); if (specState._compute) { dataReader.ReadWithMagicAndSize(ref specState.ComputeState, ComsMagic); } else { dataReader.ReadWithMagicAndSize(ref specState.GraphicsState, GfxsMagic); } dataReader.Read(ref specState._constantBufferUsePerStage); int constantBufferUsePerStageMask = specState._constantBufferUsePerStage; while (constantBufferUsePerStageMask != 0) { int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask); dataReader.Read(ref specState.ConstantBufferUse[index]); constantBufferUsePerStageMask &= ~(1 << index); } bool hasPipelineState = false; dataReader.Read(ref hasPipelineState); if (hasPipelineState) { ProgramPipelineState pipelineState = default; dataReader.ReadWithMagicAndSize(ref pipelineState, PgpsMagic); specState.PipelineState = pipelineState; } if (specState._queriedState.HasFlag(QueriedStateFlags.TransformFeedback)) { ushort tfCount = 0; dataReader.Read(ref tfCount); specState.TransformFeedbackDescriptors = new TransformFeedbackDescriptor[tfCount]; for (int index = 0; index < tfCount; index++) { dataReader.ReadWithMagicAndSize(ref specState.TransformFeedbackDescriptors[index], TfbdMagic); } } ushort count = 0; dataReader.Read(ref count); for (int index = 0; index < count; index++) { TextureKey textureKey = default; Box textureState = new Box(); dataReader.ReadWithMagicAndSize(ref textureKey, TexkMagic); dataReader.ReadWithMagicAndSize(ref textureState.Value, TexsMagic); specState._textureSpecialization[textureKey] = textureState; } return specState; } /// /// Serializes the shader specialization state. /// /// Data writer public void Write(ref BinarySerializer dataWriter) { dataWriter.Write(ref _queriedState); dataWriter.Write(ref _compute); if (_compute) { dataWriter.WriteWithMagicAndSize(ref ComputeState, ComsMagic); } else { dataWriter.WriteWithMagicAndSize(ref GraphicsState, GfxsMagic); } dataWriter.Write(ref _constantBufferUsePerStage); int constantBufferUsePerStageMask = _constantBufferUsePerStage; while (constantBufferUsePerStageMask != 0) { int index = BitOperations.TrailingZeroCount(constantBufferUsePerStageMask); dataWriter.Write(ref ConstantBufferUse[index]); constantBufferUsePerStageMask &= ~(1 << index); } bool hasPipelineState = PipelineState.HasValue; dataWriter.Write(ref hasPipelineState); if (hasPipelineState) { ProgramPipelineState pipelineState = PipelineState.Value; dataWriter.WriteWithMagicAndSize(ref pipelineState, PgpsMagic); } if (_queriedState.HasFlag(QueriedStateFlags.TransformFeedback)) { ushort tfCount = (ushort)TransformFeedbackDescriptors.Length; dataWriter.Write(ref tfCount); for (int index = 0; index < TransformFeedbackDescriptors.Length; index++) { dataWriter.WriteWithMagicAndSize(ref TransformFeedbackDescriptors[index], TfbdMagic); } } ushort count = (ushort)_textureSpecialization.Count; dataWriter.Write(ref count); foreach (var kv in _textureSpecialization) { var textureKey = kv.Key; var textureState = kv.Value; dataWriter.WriteWithMagicAndSize(ref textureKey, TexkMagic); dataWriter.WriteWithMagicAndSize(ref textureState.Value, TexsMagic); } } } }