Ryujinx/Ryujinx.Graphics.Gpu/Shader/DiskCache/BinarySerializer.cs
gdkchan 43ebd7a9bb
New shader cache implementation (#3194)
* New shader cache implementation

* Remove some debug code

* Take transform feedback varying count into account

* Create shader cache directory if it does not exist + fragment output map related fixes

* Remove debug code

* Only check texture descriptors if the constant buffer is bound

* Also check CPU VA on GetSpanMapped

* Remove more unused code and move cache related code

* XML docs + remove more unused methods

* Better codegen for TransformFeedbackDescriptor.AsSpan

* Support migration from old cache format, remove more unused code

Shader cache rebuild now also rewrites the shared toc and data files

* Fix migration error with BRX shaders

* Add a limit to the async translation queue

 Avoid async translation threads not being able to keep up and the queue growing very large

* Re-create specialization state on recompile

This might be required if a new version of the shader translator requires more or less state, or if there is a bug related to the GPU state access

* Make shader cache more error resilient

* Add some missing XML docs and move GpuAccessor docs to the interface/use inheritdoc

* Address early PR feedback

* Fix rebase

* Remove IRenderer.CompileShader and IShader interface, replace with new ShaderSource struct passed to CreateProgram directly

* Handle some missing exceptions

* Make shader cache purge delete both old and new shader caches

* Register textures on new specialization state

* Translate and compile shaders in forward order (eliminates diffs due to different binding numbers)

* Limit in-flight shader compilation to the maximum number of compilation threads

* Replace ParallelDiskCacheLoader state changed event with a callback function

* Better handling for invalid constant buffer 1 data length

* Do not create the old cache directory structure if the old cache does not exist

* Constant buffer use should be per-stage. This change will invalidate existing new caches (file format version was incremented)

* Replace rectangle texture with just coordinate normalization

* Skip incompatible shaders that are missing texture information, instead of crashing

This is required if we, for example, support new texture instruction to the shader translator, and then they allow access to textures that were not accessed before. In this scenario, the old cache entry is no longer usable

* Fix coordinates normalization on cubemap textures

* Check if title ID is null before combining shader cache path

* More robust constant buffer address validation on spec state

* More robust constant buffer address validation on spec state (2)

* Regenerate shader cache with one stream, rather than one per shader.

* Only create shader cache directory during initialization

* Logging improvements

* Proper shader program disposal

* PR feedback, and add a comment on serialized structs

* XML docs for RegisterTexture

Co-authored-by: riperiperi <rhy3756547@hotmail.com>
2022-04-10 10:49:44 -03:00

216 lines
7.8 KiB
C#

using System;
using System.IO;
using System.IO.Compression;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.Gpu.Shader.DiskCache
{
/// <summary>
/// Binary data serializer.
/// </summary>
struct BinarySerializer
{
private readonly Stream _stream;
private Stream _activeStream;
/// <summary>
/// Creates a new binary serializer.
/// </summary>
/// <param name="stream">Stream to read from or write into</param>
public BinarySerializer(Stream stream)
{
_stream = stream;
_activeStream = stream;
}
/// <summary>
/// Reads data from the stream.
/// </summary>
/// <typeparam name="T">Type of the data</typeparam>
/// <param name="data">Data read</param>
public void Read<T>(ref T data) where T : unmanaged
{
Span<byte> buffer = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref data, 1));
for (int offset = 0; offset < buffer.Length;)
{
offset += _activeStream.Read(buffer.Slice(offset));
}
}
/// <summary>
/// Tries to read data from the stream.
/// </summary>
/// <typeparam name="T">Type of the data</typeparam>
/// <param name="data">Data read</param>
/// <returns>True if the read was successful, false otherwise</returns>
public bool TryRead<T>(ref T data) where T : unmanaged
{
// Length is unknown on compressed streams.
if (_activeStream == _stream)
{
int size = Unsafe.SizeOf<T>();
if (_activeStream.Length - _activeStream.Position < size)
{
return false;
}
}
Read(ref data);
return true;
}
/// <summary>
/// Reads data prefixed with a magic and size from the stream.
/// </summary>
/// <typeparam name="T">Type of the data</typeparam>
/// <param name="data">Data read</param>
/// <param name="magic">Expected magic value, for validation</param>
public void ReadWithMagicAndSize<T>(ref T data, uint magic) where T : unmanaged
{
uint actualMagic = 0;
int size = 0;
Read(ref actualMagic);
Read(ref size);
if (actualMagic != magic)
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedInvalidMagic);
}
// Structs are expected to expand but not shrink between versions.
if (size > Unsafe.SizeOf<T>())
{
throw new DiskCacheLoadException(DiskCacheLoadResult.FileCorruptedInvalidLength);
}
Span<byte> buffer = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref data, 1)).Slice(0, size);
for (int offset = 0; offset < buffer.Length;)
{
offset += _activeStream.Read(buffer.Slice(offset));
}
}
/// <summary>
/// Writes data into the stream.
/// </summary>
/// <typeparam name="T">Type of the data</typeparam>
/// <param name="data">Data to be written</param>
public void Write<T>(ref T data) where T : unmanaged
{
Span<byte> buffer = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref data, 1));
_activeStream.Write(buffer);
}
/// <summary>
/// Writes data prefixed with a magic and size into the stream.
/// </summary>
/// <typeparam name="T">Type of the data</typeparam>
/// <param name="data">Data to write</param>
/// <param name="magic">Magic value to write</param>
public void WriteWithMagicAndSize<T>(ref T data, uint magic) where T : unmanaged
{
int size = Unsafe.SizeOf<T>();
Write(ref magic);
Write(ref size);
Span<byte> buffer = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref data, 1));
_activeStream.Write(buffer);
}
/// <summary>
/// Indicates that all data that will be read from the stream has been compressed.
/// </summary>
public void BeginCompression()
{
CompressionAlgorithm algorithm = CompressionAlgorithm.None;
Read(ref algorithm);
if (algorithm == CompressionAlgorithm.Deflate)
{
_activeStream = new DeflateStream(_stream, CompressionMode.Decompress, true);
}
}
/// <summary>
/// Indicates that all data that will be written into the stream should be compressed.
/// </summary>
/// <param name="algorithm">Compression algorithm that should be used</param>
public void BeginCompression(CompressionAlgorithm algorithm)
{
Write(ref algorithm);
if (algorithm == CompressionAlgorithm.Deflate)
{
_activeStream = new DeflateStream(_stream, CompressionLevel.SmallestSize, true);
}
}
/// <summary>
/// Indicates the end of a compressed chunck.
/// </summary>
/// <remarks>
/// Any data written after this will not be compressed unless <see cref="BeginCompression(CompressionAlgorithm)"/> is called again.
/// Any data read after this will be assumed to be uncompressed unless <see cref="BeginCompression"/> is called again.
/// </remarks>
public void EndCompression()
{
if (_activeStream != _stream)
{
_activeStream.Dispose();
_activeStream = _stream;
}
}
/// <summary>
/// Reads compressed data from the stream.
/// </summary>
/// <remarks>
/// <paramref name="data"/> must have the exact length of the uncompressed data,
/// otherwise decompression will fail.
/// </remarks>
/// <param name="stream">Stream to read from</param>
/// <param name="data">Buffer to write the uncompressed data into</param>
public static void ReadCompressed(Stream stream, Span<byte> data)
{
CompressionAlgorithm algorithm = (CompressionAlgorithm)stream.ReadByte();
switch (algorithm)
{
case CompressionAlgorithm.None:
stream.Read(data);
break;
case CompressionAlgorithm.Deflate:
stream = new DeflateStream(stream, CompressionMode.Decompress, true);
for (int offset = 0; offset < data.Length;)
{
offset += stream.Read(data.Slice(offset));
}
stream.Dispose();
break;
}
}
/// <summary>
/// Compresses and writes the compressed data into the stream.
/// </summary>
/// <param name="stream">Stream to write into</param>
/// <param name="data">Data to compress</param>
/// <param name="algorithm">Compression algorithm to be used</param>
public static void WriteCompressed(Stream stream, ReadOnlySpan<byte> data, CompressionAlgorithm algorithm)
{
stream.WriteByte((byte)algorithm);
switch (algorithm)
{
case CompressionAlgorithm.None:
stream.Write(data);
break;
case CompressionAlgorithm.Deflate:
stream = new DeflateStream(stream, CompressionLevel.SmallestSize, true);
stream.Write(data);
stream.Dispose();
break;
}
}
}
}