salieri: Support read-only mode if archive is already opened (#1807)

This improves shader cache resilience when people opens another program that touch the cache.zip.
This commit is contained in:
Mary 2020-12-13 08:46:07 +01:00 committed by GitHub
parent 19d18662ea
commit 6bc2733c17
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 124 additions and 25 deletions

View file

@ -116,6 +116,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
/// </summary> /// </summary>
private ZipArchive _cacheArchive; private ZipArchive _cacheArchive;
public bool IsReadOnly { get; }
/// <summary> /// <summary>
/// Immutable copy of the hash table. /// Immutable copy of the hash table.
/// </summary> /// </summary>
@ -167,6 +169,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
_hashType = hashType; _hashType = hashType;
_version = version; _version = version;
_hashTable = new HashSet<Hash128>(); _hashTable = new HashSet<Hash128>();
IsReadOnly = CacheHelper.IsArchiveReadOnly(GetArchivePath());
Load(); Load();
@ -230,6 +233,13 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
/// <param name="entries">Entries to remove from the manifest</param> /// <param name="entries">Entries to remove from the manifest</param>
public void RemoveManifestEntriesAsync(HashSet<Hash128> entries) public void RemoveManifestEntriesAsync(HashSet<Hash128> entries)
{ {
if (IsReadOnly)
{
Logger.Warning?.Print(LogClass.Gpu, "Trying to remove manifest entries on a read-only cache, ignoring.");
return;
}
_fileWriterWorkerQueue.Add(new CacheFileOperationTask _fileWriterWorkerQueue.Add(new CacheFileOperationTask
{ {
Type = CacheFileOperation.RemoveManifestEntries, Type = CacheFileOperation.RemoveManifestEntries,
@ -308,6 +318,20 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
string archivePath = GetArchivePath(); string archivePath = GetArchivePath();
if (IsReadOnly)
{
Logger.Warning?.Print(LogClass.Gpu, $"Cache collection archive in read-only, archiving task skipped.");
return;
}
if (CacheHelper.IsArchiveReadOnly(archivePath))
{
Logger.Warning?.Print(LogClass.Gpu, $"Cache collection archive in use, archiving task skipped.");
return;
}
// Open the zip in read/write. // Open the zip in read/write.
_cacheArchive = ZipFile.Open(archivePath, ZipArchiveMode.Update); _cacheArchive = ZipFile.Open(archivePath, ZipArchiveMode.Update);
@ -446,6 +470,13 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
/// <param name="value">The value to cache</param> /// <param name="value">The value to cache</param>
public void AddValue(ref Hash128 keyHash, byte[] value) public void AddValue(ref Hash128 keyHash, byte[] value)
{ {
if (IsReadOnly)
{
Logger.Warning?.Print(LogClass.Gpu, "Trying to add {keyHash} on a read-only cache, ignoring.");
return;
}
Debug.Assert(value != null); Debug.Assert(value != null);
bool isAlreadyPresent; bool isAlreadyPresent;
@ -488,6 +519,13 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
/// <param name="value">The value to cache</param> /// <param name="value">The value to cache</param>
public void ReplaceValue(ref Hash128 keyHash, byte[] value) public void ReplaceValue(ref Hash128 keyHash, byte[] value)
{ {
if (IsReadOnly)
{
Logger.Warning?.Print(LogClass.Gpu, "Trying to replace {keyHash} on a read-only cache, ignoring.");
return;
}
Debug.Assert(value != null); Debug.Assert(value != null);
// Only queue file change operations // Only queue file change operations

View file

@ -496,5 +496,27 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
} }
} }
} }
public static bool IsArchiveReadOnly(string archivePath)
{
FileInfo info = new FileInfo(archivePath);
if (!info.Exists)
{
return false;
}
try
{
using (FileStream stream = info.Open(FileMode.Open, FileAccess.Read, FileShare.None))
{
return false;
}
}
catch (IOException)
{
return true;
}
}
} }
} }

View file

@ -1,9 +1,7 @@
using Ryujinx.Common; using Ryujinx.Common;
using Ryujinx.Common.Configuration;
using Ryujinx.Graphics.Gpu.Shader.Cache.Definition; using Ryujinx.Graphics.Gpu.Shader.Cache.Definition;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
namespace Ryujinx.Graphics.Gpu.Shader.Cache namespace Ryujinx.Graphics.Gpu.Shader.Cache
{ {
@ -31,6 +29,8 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
/// </summary> /// </summary>
private const ulong GuestCacheVersion = 1759; private const ulong GuestCacheVersion = 1759;
public bool IsReadOnly => _guestProgramCache.IsReadOnly || _hostProgramCache.IsReadOnly;
/// <summary> /// <summary>
/// Create a new cache manager instance /// Create a new cache manager instance
/// </summary> /// </summary>

View file

@ -146,7 +146,12 @@ namespace Ryujinx.Graphics.Gpu.Shader.Cache
string guestBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, CacheGraphicsApi.Guest, "", "program"); string guestBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, CacheGraphicsApi.Guest, "", "program");
string hostBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, "host"); string hostBaseCacheDirectory = CacheHelper.GenerateCachePath(baseCacheDirectory, graphicsApi, shaderProvider, "host");
if (CacheHelper.TryReadManifestHeader(CacheHelper.GetManifestPath(guestBaseCacheDirectory), out CacheManifestHeader header)) string guestArchivePath = CacheHelper.GetArchivePath(guestBaseCacheDirectory);
string hostArchivePath = CacheHelper.GetArchivePath(hostBaseCacheDirectory);
bool isReadOnly = CacheHelper.IsArchiveReadOnly(guestArchivePath) || CacheHelper.IsArchiveReadOnly(hostArchivePath);
if (!isReadOnly && CacheHelper.TryReadManifestHeader(CacheHelper.GetManifestPath(guestBaseCacheDirectory), out CacheManifestHeader header))
{ {
if (NeedHashRecompute(header.Version, out ulong newVersion)) if (NeedHashRecompute(header.Version, out ulong newVersion))
{ {

View file

@ -61,7 +61,18 @@ namespace Ryujinx.Graphics.Gpu.Shader
{ {
_cacheManager = new CacheManager(CacheGraphicsApi.OpenGL, CacheHashType.XxHash128, "glsl", GraphicsConfig.TitleId, ShaderCodeGenVersion); _cacheManager = new CacheManager(CacheGraphicsApi.OpenGL, CacheHashType.XxHash128, "glsl", GraphicsConfig.TitleId, ShaderCodeGenVersion);
HashSet<Hash128> invalidEntries = new HashSet<Hash128>(); bool isReadOnly = _cacheManager.IsReadOnly;
HashSet<Hash128> invalidEntries = null;
if (isReadOnly)
{
Logger.Warning?.Print(LogClass.Gpu, "Loading shader cache in read-only mode (cache in use by another program!)");
}
else
{
invalidEntries = new HashSet<Hash128>();
}
ReadOnlySpan<Hash128> guestProgramList = _cacheManager.GetGuestProgramList(); ReadOnlySpan<Hash128> guestProgramList = _cacheManager.GetGuestProgramList();
@ -84,7 +95,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
Logger.Error?.Print(LogClass.Gpu, $"Ignoring orphan shader hash {key} in cache (is the cache incomplete?)"); Logger.Error?.Print(LogClass.Gpu, $"Ignoring orphan shader hash {key} in cache (is the cache incomplete?)");
// Should not happen, but if someone messed with the cache it's better to catch it. // Should not happen, but if someone messed with the cache it's better to catch it.
invalidEntries.Add(key); invalidEntries?.Add(key);
continue; continue;
} }
@ -141,6 +152,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
// As the host program was invalidated, save the new entry in the cache. // As the host program was invalidated, save the new entry in the cache.
hostProgramBinary = HostShaderCacheEntry.Create(hostProgram.GetBinary(), new ShaderCodeHolder[] { shader }); hostProgramBinary = HostShaderCacheEntry.Create(hostProgram.GetBinary(), new ShaderCodeHolder[] { shader });
if (!isReadOnly)
{
if (hasHostCache) if (hasHostCache)
{ {
_cacheManager.ReplaceHostProgram(ref key, hostProgramBinary); _cacheManager.ReplaceHostProgram(ref key, hostProgramBinary);
@ -152,6 +165,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
_cacheManager.AddHostProgram(ref key, hostProgramBinary); _cacheManager.AddHostProgram(ref key, hostProgramBinary);
} }
} }
}
_cpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shader)); _cpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shader));
} }
@ -270,6 +284,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
// As the host program was invalidated, save the new entry in the cache. // As the host program was invalidated, save the new entry in the cache.
hostProgramBinary = HostShaderCacheEntry.Create(hostProgram.GetBinary(), shaders); hostProgramBinary = HostShaderCacheEntry.Create(hostProgram.GetBinary(), shaders);
if (!isReadOnly)
{
if (hasHostCache) if (hasHostCache)
{ {
_cacheManager.ReplaceHostProgram(ref key, hostProgramBinary); _cacheManager.ReplaceHostProgram(ref key, hostProgramBinary);
@ -281,15 +297,19 @@ namespace Ryujinx.Graphics.Gpu.Shader
_cacheManager.AddHostProgram(ref key, hostProgramBinary); _cacheManager.AddHostProgram(ref key, hostProgramBinary);
} }
} }
}
_gpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shaders)); _gpProgramsDiskCache.Add(key, new ShaderBundle(hostProgram, shaders));
} }
} }
if (!isReadOnly)
{
// Remove entries that are broken in the cache // Remove entries that are broken in the cache
_cacheManager.RemoveManifestEntries(invalidEntries); _cacheManager.RemoveManifestEntries(invalidEntries);
_cacheManager.FlushToArchive(); _cacheManager.FlushToArchive();
_cacheManager.Synchronize(); _cacheManager.Synchronize();
}
Logger.Info?.Print(LogClass.Gpu, "Shader cache loaded."); Logger.Info?.Print(LogClass.Gpu, "Shader cache loaded.");
} }
@ -343,12 +363,15 @@ namespace Ryujinx.Graphics.Gpu.Shader
sharedMemorySize); sharedMemorySize);
bool isShaderCacheEnabled = _cacheManager != null; bool isShaderCacheEnabled = _cacheManager != null;
bool isShaderCacheReadOnly = false;
Hash128 programCodeHash = default; Hash128 programCodeHash = default;
GuestShaderCacheEntry[] shaderCacheEntries = null; GuestShaderCacheEntry[] shaderCacheEntries = null;
if (isShaderCacheEnabled) if (isShaderCacheEnabled)
{ {
isShaderCacheReadOnly = _cacheManager.IsReadOnly;
// Compute hash and prepare data for shader disk cache comparison. // Compute hash and prepare data for shader disk cache comparison.
shaderCacheEntries = CacheHelper.CreateShaderCacheEntries(_context.MemoryManager, shaderContexts); shaderCacheEntries = CacheHelper.CreateShaderCacheEntries(_context.MemoryManager, shaderContexts);
programCodeHash = CacheHelper.ComputeGuestHashFromCache(shaderCacheEntries); programCodeHash = CacheHelper.ComputeGuestHashFromCache(shaderCacheEntries);
@ -378,9 +401,13 @@ namespace Ryujinx.Graphics.Gpu.Shader
if (isShaderCacheEnabled) if (isShaderCacheEnabled)
{ {
_cpProgramsDiskCache.Add(programCodeHash, cpShader); _cpProgramsDiskCache.Add(programCodeHash, cpShader);
if (!isShaderCacheReadOnly)
{
_cacheManager.SaveProgram(ref programCodeHash, CacheHelper.CreateGuestProgramDump(shaderCacheEntries), hostProgramBinary); _cacheManager.SaveProgram(ref programCodeHash, CacheHelper.CreateGuestProgramDump(shaderCacheEntries), hostProgramBinary);
} }
} }
}
if (!isCached) if (!isCached)
{ {
@ -447,12 +474,15 @@ namespace Ryujinx.Graphics.Gpu.Shader
shaderContexts[4] = DecodeGraphicsShader(state, counts, flags, ShaderStage.Fragment, addresses.Fragment); shaderContexts[4] = DecodeGraphicsShader(state, counts, flags, ShaderStage.Fragment, addresses.Fragment);
bool isShaderCacheEnabled = _cacheManager != null; bool isShaderCacheEnabled = _cacheManager != null;
bool isShaderCacheReadOnly = false;
Hash128 programCodeHash = default; Hash128 programCodeHash = default;
GuestShaderCacheEntry[] shaderCacheEntries = null; GuestShaderCacheEntry[] shaderCacheEntries = null;
if (isShaderCacheEnabled) if (isShaderCacheEnabled)
{ {
isShaderCacheReadOnly = _cacheManager.IsReadOnly;
// Compute hash and prepare data for shader disk cache comparison. // Compute hash and prepare data for shader disk cache comparison.
shaderCacheEntries = CacheHelper.CreateShaderCacheEntries(_context.MemoryManager, shaderContexts); shaderCacheEntries = CacheHelper.CreateShaderCacheEntries(_context.MemoryManager, shaderContexts);
programCodeHash = CacheHelper.ComputeGuestHashFromCache(shaderCacheEntries, tfd); programCodeHash = CacheHelper.ComputeGuestHashFromCache(shaderCacheEntries, tfd);
@ -504,9 +534,13 @@ namespace Ryujinx.Graphics.Gpu.Shader
if (isShaderCacheEnabled) if (isShaderCacheEnabled)
{ {
_gpProgramsDiskCache.Add(programCodeHash, gpShaders); _gpProgramsDiskCache.Add(programCodeHash, gpShaders);
if (!isShaderCacheReadOnly)
{
_cacheManager.SaveProgram(ref programCodeHash, CacheHelper.CreateGuestProgramDump(shaderCacheEntries, tfd), hostProgramBinary); _cacheManager.SaveProgram(ref programCodeHash, CacheHelper.CreateGuestProgramDump(shaderCacheEntries, tfd), hostProgramBinary);
} }
} }
}
if (!isCached) if (!isCached)
{ {