using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Shader;
using Ryujinx.Graphics.Shader.Translation;
using System;
using System.IO;
using System.Numerics;
using System.Runtime.CompilerServices;
namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
{
///
/// On-disk shader cache storage for host code.
///
class DiskCacheHostStorage
{
private const uint TocsMagic = (byte)'T' | ((byte)'O' << 8) | ((byte)'C' << 16) | ((byte)'S' << 24);
private const uint TochMagic = (byte)'T' | ((byte)'O' << 8) | ((byte)'C' << 16) | ((byte)'H' << 24);
private const uint ShdiMagic = (byte)'S' | ((byte)'H' << 8) | ((byte)'D' << 16) | ((byte)'I' << 24);
private const uint BufdMagic = (byte)'B' | ((byte)'U' << 8) | ((byte)'F' << 16) | ((byte)'D' << 24);
private const uint TexdMagic = (byte)'T' | ((byte)'E' << 8) | ((byte)'X' << 16) | ((byte)'D' << 24);
private const ushort FileFormatVersionMajor = 1;
private const ushort FileFormatVersionMinor = 2;
private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor;
private const uint CodeGenVersion = 14;
private const string SharedTocFileName = "shared.toc";
private const string SharedDataFileName = "shared.data";
private readonly string _basePath;
public bool CacheEnabled => !string.IsNullOrEmpty(_basePath);
///
/// TOC (Table of contents) file header.
///
private struct TocHeader
{
///
/// Magic value, for validation and identification.
///
public uint Magic;
///
/// File format version.
///
public uint FormatVersion;
///
/// Generated shader code version.
///
public uint CodeGenVersion;
///
/// Header padding.
///
public uint Padding;
///
/// Timestamp of when the file was first created.
///
public ulong Timestamp;
///
/// Reserved space, to be used in the future. Write as zero.
///
public ulong Reserved;
}
///
/// Offset and size pair.
///
private struct OffsetAndSize
{
///
/// Offset.
///
public ulong Offset;
///
/// Size of uncompressed data.
///
public uint UncompressedSize;
///
/// Size of compressed data.
///
public uint CompressedSize;
}
///
/// Per-stage data entry.
///
private struct DataEntryPerStage
{
///
/// Index of the guest code on the guest code cache TOC file.
///
public int GuestCodeIndex;
}
///
/// Per-program data entry.
///
private struct DataEntry
{
///
/// Bit mask where each bit set is a used shader stage. Should be zero for compute shaders.
///
public uint StagesBitMask;
}
///
/// Per-stage shader information, returned by the translator.
///
private struct DataShaderInfo
{
///
/// Total constant buffers used.
///
public ushort CBuffersCount;
///
/// Total storage buffers used.
///
public ushort SBuffersCount;
///
/// Total textures used.
///
public ushort TexturesCount;
///
/// Total images used.
///
public ushort ImagesCount;
///
/// Shader stage.
///
public ShaderStage Stage;
///
/// Indicates if the shader accesses the Instance ID built-in variable.
///
public bool UsesInstanceId;
///
/// Indicates if the shader modifies the Layer built-in variable.
///
public bool UsesRtLayer;
///
/// Bit mask with the clip distances written on the vertex stage.
///
public byte ClipDistancesWritten;
///
/// Bit mask of the render target components written by the fragment stage.
///
public int FragmentOutputMap;
}
private readonly DiskCacheGuestStorage _guestStorage;
///
/// Creates a disk cache host storage.
///
/// Base path of the shader cache
public DiskCacheHostStorage(string basePath)
{
_basePath = basePath;
_guestStorage = new DiskCacheGuestStorage(basePath);
if (CacheEnabled)
{
Directory.CreateDirectory(basePath);
}
}
///
/// Gets the total of host programs on the cache.
///
/// Host programs count
public int GetProgramCount()
{
string tocFilePath = Path.Combine(_basePath, SharedTocFileName);
if (!File.Exists(tocFilePath))
{
return 0;
}
return Math.Max((int)((new FileInfo(tocFilePath).Length - Unsafe.SizeOf()) / sizeof(ulong)), 0);
}
///
/// Guest the name of the host program cache file, with extension.
///
/// GPU context
/// Name of the file, without extension
private static string GetHostFileName(GpuContext context)
{
string apiName = context.Capabilities.Api.ToString().ToLowerInvariant();
string vendorName = RemoveInvalidCharacters(context.Capabilities.VendorName.ToLowerInvariant());
return $"{apiName}_{vendorName}";
}
///
/// Removes invalid path characters and spaces from a file name.
///
/// File name
/// Filtered file name
private static string RemoveInvalidCharacters(string fileName)
{
int indexOfSpace = fileName.IndexOf(' ');
if (indexOfSpace >= 0)
{
fileName = fileName.Substring(0, indexOfSpace);
}
return string.Concat(fileName.Split(Path.GetInvalidFileNameChars(), StringSplitOptions.RemoveEmptyEntries));
}
///
/// Gets the name of the TOC host file.
///
/// GPU context
/// File name
private static string GetHostTocFileName(GpuContext context)
{
return GetHostFileName(context) + ".toc";
}
///
/// Gets the name of the data host file.
///
/// GPU context
/// File name
private static string GetHostDataFileName(GpuContext context)
{
return GetHostFileName(context) + ".data";
}
///
/// Checks if a disk cache exists for the current application.
///
/// True if a disk cache exists, false otherwise
public bool CacheExists()
{
string tocFilePath = Path.Combine(_basePath, SharedTocFileName);
string dataFilePath = Path.Combine(_basePath, SharedDataFileName);
if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath) || !_guestStorage.TocFileExists() || !_guestStorage.DataFileExists())
{
return false;
}
return true;
}
///
/// Loads all shaders from the cache.
///
/// GPU context
/// Parallel disk cache loader
public void LoadShaders(GpuContext context, ParallelDiskCacheLoader loader)
{
if (!CacheExists())
{
return;
}
Stream hostTocFileStream = null;
Stream hostDataFileStream = null;
try
{
using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: false);
using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: false);
using var guestTocFileStream = _guestStorage.OpenTocFileStream();
using var guestDataFileStream = _guestStorage.OpenDataFileStream();
BinarySerializer tocReader = new BinarySerializer(tocFileStream);
BinarySerializer dataReader = new BinarySerializer(dataFileStream);
TocHeader header = new TocHeader();
if (!tocReader.TryRead(ref header) || header.Magic != TocsMagic)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
}
if (header.FormatVersion != FileFormatVersionPacked)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.IncompatibleVersion);
}
bool loadHostCache = header.CodeGenVersion == CodeGenVersion;
int programIndex = 0;
DataEntry entry = new DataEntry();
while (tocFileStream.Position < tocFileStream.Length && loader.Active)
{
ulong dataOffset = 0;
tocReader.Read(ref dataOffset);
if ((ulong)dataOffset >= (ulong)dataFileStream.Length)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
}
dataFileStream.Seek((long)dataOffset, SeekOrigin.Begin);
dataReader.BeginCompression();
dataReader.Read(ref entry);
uint stagesBitMask = entry.StagesBitMask;
if ((stagesBitMask & ~0x3fu) != 0)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
}
bool isCompute = stagesBitMask == 0;
if (isCompute)
{
stagesBitMask = 1;
}
GuestCodeAndCbData?[] guestShaders = new GuestCodeAndCbData?[isCompute ? 1 : Constants.ShaderStages + 1];
DataEntryPerStage stageEntry = new DataEntryPerStage();
while (stagesBitMask != 0)
{
int stageIndex = BitOperations.TrailingZeroCount(stagesBitMask);
dataReader.Read(ref stageEntry);
guestShaders[stageIndex] = _guestStorage.LoadShader(
guestTocFileStream,
guestDataFileStream,
stageEntry.GuestCodeIndex);
stagesBitMask &= ~(1u << stageIndex);
}
ShaderSpecializationState specState = ShaderSpecializationState.Read(ref dataReader);
dataReader.EndCompression();
if (loadHostCache)
{
(byte[] hostCode, CachedShaderStage[] shaders) = ReadHostCode(
context,
ref hostTocFileStream,
ref hostDataFileStream,
guestShaders,
programIndex,
header.Timestamp);
if (hostCode != null)
{
bool hasFragmentShader = shaders.Length > 5 && shaders[5] != null;
int fragmentOutputMap = hasFragmentShader ? shaders[5].Info.FragmentOutputMap : -1;
ShaderInfo shaderInfo = specState.PipelineState.HasValue
? new ShaderInfo(fragmentOutputMap, specState.PipelineState.Value, fromCache: true)
: new ShaderInfo(fragmentOutputMap, fromCache: true);
IProgram hostProgram;
if (context.Capabilities.Api == TargetApi.Vulkan)
{
ShaderSource[] shaderSources = ShaderBinarySerializer.Unpack(shaders, hostCode, isCompute);
hostProgram = context.Renderer.CreateProgram(shaderSources, shaderInfo);
}
else
{
hostProgram = context.Renderer.LoadProgramBinary(hostCode, hasFragmentShader, shaderInfo);
}
CachedShaderProgram program = new CachedShaderProgram(hostProgram, specState, shaders);
loader.QueueHostProgram(program, hostCode, programIndex, isCompute);
}
else
{
loadHostCache = false;
}
}
if (!loadHostCache)
{
loader.QueueGuestProgram(guestShaders, specState, programIndex, isCompute);
}
loader.CheckCompilation();
programIndex++;
}
}
finally
{
_guestStorage.ClearMemoryCache();
hostTocFileStream?.Dispose();
hostDataFileStream?.Dispose();
}
}
///
/// Reads the host code for a given shader, if existent.
///
/// GPU context
/// Host TOC file stream, intialized if needed
/// Host data file stream, initialized if needed
/// Guest shader code for each active stage
/// Index of the program on the cache
/// Timestamp of the shared cache file. The host file must be newer than it
/// Host binary code, or null if not found
private (byte[], CachedShaderStage[]) ReadHostCode(
GpuContext context,
ref Stream tocFileStream,
ref Stream dataFileStream,
GuestCodeAndCbData?[] guestShaders,
int programIndex,
ulong expectedTimestamp)
{
if (tocFileStream == null && dataFileStream == null)
{
string tocFilePath = Path.Combine(_basePath, GetHostTocFileName(context));
string dataFilePath = Path.Combine(_basePath, GetHostDataFileName(context));
if (!File.Exists(tocFilePath) || !File.Exists(dataFilePath))
{
return (null, null);
}
tocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: false);
dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: false);
BinarySerializer tempTocReader = new BinarySerializer(tocFileStream);
TocHeader header = new TocHeader();
tempTocReader.Read(ref header);
if (header.Timestamp < expectedTimestamp)
{
return (null, null);
}
}
int offset = Unsafe.SizeOf() + programIndex * Unsafe.SizeOf();
if (offset + Unsafe.SizeOf() > tocFileStream.Length)
{
return (null, null);
}
if ((ulong)offset >= (ulong)dataFileStream.Length)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
}
tocFileStream.Seek(offset, SeekOrigin.Begin);
BinarySerializer tocReader = new BinarySerializer(tocFileStream);
OffsetAndSize offsetAndSize = new OffsetAndSize();
tocReader.Read(ref offsetAndSize);
if (offsetAndSize.Offset >= (ulong)dataFileStream.Length)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedGeneric);
}
dataFileStream.Seek((long)offsetAndSize.Offset, SeekOrigin.Begin);
byte[] hostCode = new byte[offsetAndSize.UncompressedSize];
BinarySerializer.ReadCompressed(dataFileStream, hostCode);
CachedShaderStage[] shaders = new CachedShaderStage[guestShaders.Length];
BinarySerializer dataReader = new BinarySerializer(dataFileStream);
dataFileStream.Seek((long)(offsetAndSize.Offset + offsetAndSize.CompressedSize), SeekOrigin.Begin);
dataReader.BeginCompression();
for (int index = 0; index < guestShaders.Length; index++)
{
if (!guestShaders[index].HasValue)
{
continue;
}
GuestCodeAndCbData guestShader = guestShaders[index].Value;
ShaderProgramInfo info = index != 0 || guestShaders.Length == 1 ? ReadShaderProgramInfo(ref dataReader) : null;
shaders[index] = new CachedShaderStage(info, guestShader.Code, guestShader.Cb1Data);
}
dataReader.EndCompression();
return (hostCode, shaders);
}
///
/// Gets output streams for the disk cache, for faster batch writing.
///
/// The GPU context, used to determine the host disk cache
/// A collection of disk cache output streams
public DiskCacheOutputStreams GetOutputStreams(GpuContext context)
{
var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true);
var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true);
var hostTocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true);
var hostDataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true);
return new DiskCacheOutputStreams(tocFileStream, dataFileStream, hostTocFileStream, hostDataFileStream);
}
///
/// Adds a shader to the cache.
///
/// GPU context
/// Cached program
/// Optional host binary code
/// Output streams to use
public void AddShader(GpuContext context, CachedShaderProgram program, ReadOnlySpan hostCode, DiskCacheOutputStreams streams = null)
{
uint stagesBitMask = 0;
for (int index = 0; index < program.Shaders.Length; index++)
{
var shader = program.Shaders[index];
if (shader == null || (shader.Info != null && shader.Info.Stage == ShaderStage.Compute))
{
continue;
}
stagesBitMask |= 1u << index;
}
var tocFileStream = streams != null ? streams.TocFileStream : DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true);
var dataFileStream = streams != null ? streams.DataFileStream : DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true);
ulong timestamp = (ulong)DateTime.UtcNow.Subtract(DateTime.UnixEpoch).TotalSeconds;
if (tocFileStream.Length == 0)
{
TocHeader header = new TocHeader();
CreateToc(tocFileStream, ref header, TocsMagic, CodeGenVersion, timestamp);
}
tocFileStream.Seek(0, SeekOrigin.End);
dataFileStream.Seek(0, SeekOrigin.End);
BinarySerializer tocWriter = new BinarySerializer(tocFileStream);
BinarySerializer dataWriter = new BinarySerializer(dataFileStream);
ulong dataOffset = (ulong)dataFileStream.Position;
tocWriter.Write(ref dataOffset);
DataEntry entry = new DataEntry();
entry.StagesBitMask = stagesBitMask;
dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm());
dataWriter.Write(ref entry);
DataEntryPerStage stageEntry = new DataEntryPerStage();
for (int index = 0; index < program.Shaders.Length; index++)
{
var shader = program.Shaders[index];
if (shader == null)
{
continue;
}
stageEntry.GuestCodeIndex = _guestStorage.AddShader(shader.Code, shader.Cb1Data);
dataWriter.Write(ref stageEntry);
}
program.SpecializationState.Write(ref dataWriter);
dataWriter.EndCompression();
if (streams == null)
{
tocFileStream.Dispose();
dataFileStream.Dispose();
}
if (hostCode.IsEmpty)
{
return;
}
WriteHostCode(context, hostCode, program.Shaders, streams, timestamp);
}
///
/// Clears all content from the guest cache files.
///
public void ClearGuestCache()
{
_guestStorage.ClearCache();
}
///
/// Clears all content from the shared cache files.
///
/// GPU context
public void ClearSharedCache()
{
using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, SharedTocFileName, writable: true);
using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, SharedDataFileName, writable: true);
tocFileStream.SetLength(0);
dataFileStream.SetLength(0);
}
///
/// Deletes all content from the host cache files.
///
/// GPU context
public void ClearHostCache(GpuContext context)
{
using var tocFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true);
using var dataFileStream = DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true);
tocFileStream.SetLength(0);
dataFileStream.SetLength(0);
}
///
/// Writes the host binary code on the host cache.
///
/// GPU context
/// Host binary code
/// Shader stages to be added to the host cache
/// Output streams to use
/// File creation timestamp
private void WriteHostCode(
GpuContext context,
ReadOnlySpan hostCode,
CachedShaderStage[] shaders,
DiskCacheOutputStreams streams,
ulong timestamp)
{
var tocFileStream = streams != null ? streams.HostTocFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostTocFileName(context), writable: true);
var dataFileStream = streams != null ? streams.HostDataFileStream : DiskCacheCommon.OpenFile(_basePath, GetHostDataFileName(context), writable: true);
if (tocFileStream.Length == 0)
{
TocHeader header = new TocHeader();
CreateToc(tocFileStream, ref header, TochMagic, 0, timestamp);
}
tocFileStream.Seek(0, SeekOrigin.End);
dataFileStream.Seek(0, SeekOrigin.End);
BinarySerializer tocWriter = new BinarySerializer(tocFileStream);
BinarySerializer dataWriter = new BinarySerializer(dataFileStream);
OffsetAndSize offsetAndSize = new OffsetAndSize();
offsetAndSize.Offset = (ulong)dataFileStream.Position;
offsetAndSize.UncompressedSize = (uint)hostCode.Length;
long dataStartPosition = dataFileStream.Position;
BinarySerializer.WriteCompressed(dataFileStream, hostCode, DiskCacheCommon.GetCompressionAlgorithm());
offsetAndSize.CompressedSize = (uint)(dataFileStream.Position - dataStartPosition);
tocWriter.Write(ref offsetAndSize);
dataWriter.BeginCompression(DiskCacheCommon.GetCompressionAlgorithm());
for (int index = 0; index < shaders.Length; index++)
{
if (shaders[index] != null)
{
WriteShaderProgramInfo(ref dataWriter, shaders[index].Info);
}
}
dataWriter.EndCompression();
if (streams == null)
{
tocFileStream.Dispose();
dataFileStream.Dispose();
}
}
///
/// Creates a TOC file for the host or shared cache.
///
/// TOC file stream
/// Set to the TOC file header
/// Magic value to be written
/// Shader codegen version, only valid for the host file
/// File creation timestamp
private void CreateToc(Stream tocFileStream, ref TocHeader header, uint magic, uint codegenVersion, ulong timestamp)
{
BinarySerializer writer = new BinarySerializer(tocFileStream);
header.Magic = magic;
header.FormatVersion = FileFormatVersionPacked;
header.CodeGenVersion = codegenVersion;
header.Padding = 0;
header.Reserved = 0;
header.Timestamp = timestamp;
if (tocFileStream.Length > 0)
{
tocFileStream.Seek(0, SeekOrigin.Begin);
tocFileStream.SetLength(0);
}
writer.Write(ref header);
}
///
/// Reads the shader program info from the cache.
///
/// Cache data reader
/// Shader program info
private static ShaderProgramInfo ReadShaderProgramInfo(ref BinarySerializer dataReader)
{
DataShaderInfo dataInfo = new DataShaderInfo();
dataReader.ReadWithMagicAndSize(ref dataInfo, ShdiMagic);
BufferDescriptor[] cBuffers = new BufferDescriptor[dataInfo.CBuffersCount];
BufferDescriptor[] sBuffers = new BufferDescriptor[dataInfo.SBuffersCount];
TextureDescriptor[] textures = new TextureDescriptor[dataInfo.TexturesCount];
TextureDescriptor[] images = new TextureDescriptor[dataInfo.ImagesCount];
for (int index = 0; index < dataInfo.CBuffersCount; index++)
{
dataReader.ReadWithMagicAndSize(ref cBuffers[index], BufdMagic);
}
for (int index = 0; index < dataInfo.SBuffersCount; index++)
{
dataReader.ReadWithMagicAndSize(ref sBuffers[index], BufdMagic);
}
for (int index = 0; index < dataInfo.TexturesCount; index++)
{
dataReader.ReadWithMagicAndSize(ref textures[index], TexdMagic);
}
for (int index = 0; index < dataInfo.ImagesCount; index++)
{
dataReader.ReadWithMagicAndSize(ref images[index], TexdMagic);
}
return new ShaderProgramInfo(
cBuffers,
sBuffers,
textures,
images,
dataInfo.Stage,
dataInfo.UsesInstanceId,
dataInfo.UsesRtLayer,
dataInfo.ClipDistancesWritten,
dataInfo.FragmentOutputMap);
}
///
/// Writes the shader program info into the cache.
///
/// Cache data writer
/// Program info
private static void WriteShaderProgramInfo(ref BinarySerializer dataWriter, ShaderProgramInfo info)
{
if (info == null)
{
return;
}
DataShaderInfo dataInfo = new DataShaderInfo();
dataInfo.CBuffersCount = (ushort)info.CBuffers.Count;
dataInfo.SBuffersCount = (ushort)info.SBuffers.Count;
dataInfo.TexturesCount = (ushort)info.Textures.Count;
dataInfo.ImagesCount = (ushort)info.Images.Count;
dataInfo.Stage = info.Stage;
dataInfo.UsesInstanceId = info.UsesInstanceId;
dataInfo.UsesRtLayer = info.UsesRtLayer;
dataInfo.ClipDistancesWritten = info.ClipDistancesWritten;
dataInfo.FragmentOutputMap = info.FragmentOutputMap;
dataWriter.WriteWithMagicAndSize(ref dataInfo, ShdiMagic);
for (int index = 0; index < info.CBuffers.Count; index++)
{
var entry = info.CBuffers[index];
dataWriter.WriteWithMagicAndSize(ref entry, BufdMagic);
}
for (int index = 0; index < info.SBuffers.Count; index++)
{
var entry = info.SBuffers[index];
dataWriter.WriteWithMagicAndSize(ref entry, BufdMagic);
}
for (int index = 0; index < info.Textures.Count; index++)
{
var entry = info.Textures[index];
dataWriter.WriteWithMagicAndSize(ref entry, TexdMagic);
}
for (int index = 0; index < info.Images.Count; index++)
{
var entry = info.Images[index];
dataWriter.WriteWithMagicAndSize(ref entry, TexdMagic);
}
}
}
}