From 1377be9902b467b89be21a49abc60ab8d3a5b1d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Lam?= Date: Sun, 24 Nov 2019 20:32:10 +0100 Subject: [PATCH 1/3] file_sys: Move IPS patching code into separate source file In anticipation of a new BPS patcher. --- src/core/CMakeLists.txt | 2 + src/core/file_sys/ncch_container.cpp | 52 ++------------------------ src/core/file_sys/ncch_container.h | 4 +- src/core/file_sys/patch.cpp | 55 ++++++++++++++++++++++++++++ src/core/file_sys/patch.h | 15 ++++++++ src/core/loader/ncch.cpp | 2 +- 6 files changed, 78 insertions(+), 52 deletions(-) create mode 100644 src/core/file_sys/patch.cpp create mode 100644 src/core/file_sys/patch.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 67167fe5d..064e44f94 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -74,6 +74,8 @@ add_library(core STATIC file_sys/ivfc_archive.h file_sys/ncch_container.cpp file_sys/ncch_container.h + file_sys/patch.cpp + file_sys/patch.h file_sys/path_parser.cpp file_sys/path_parser.h file_sys/romfs_reader.cpp diff --git a/src/core/file_sys/ncch_container.cpp b/src/core/file_sys/ncch_container.cpp index 5e0286494..0412fa288 100644 --- a/src/core/file_sys/ncch_container.cpp +++ b/src/core/file_sys/ncch_container.cpp @@ -12,6 +12,7 @@ #include "common/logging/log.h" #include "core/core.h" #include "core/file_sys/ncch_container.h" +#include "core/file_sys/patch.h" #include "core/file_sys/seed_db.h" #include "core/hw/aes/key.h" #include "core/loader/loader.h" @@ -24,53 +25,6 @@ namespace FileSys { static const int kMaxSections = 8; ///< Maximum number of sections (files) in an ExeFs static const int kBlockSize = 0x200; ///< Size of ExeFS blocks (in bytes) -/** - * Attempts to patch a buffer using an IPS - * @param ips Vector of the patches to apply - * @param buffer Vector to patch data into - */ -static void ApplyIPS(std::vector& ips, std::vector& buffer) { - u32 cursor = 5; - u32 patch_length = ips.size() - 3; - std::string ips_header(ips.begin(), ips.begin() + 5); - - if (ips_header != "PATCH") { - LOG_INFO(Service_FS, "Attempted to load invalid IPS"); - return; - } - - while (cursor < patch_length) { - std::string eof_check(ips.begin() + cursor, ips.begin() + cursor + 3); - - if (eof_check == "EOF") - return; - - u32 offset = ips[cursor] << 16 | ips[cursor + 1] << 8 | ips[cursor + 2]; - std::size_t length = ips[cursor + 3] << 8 | ips[cursor + 4]; - - // check for an rle record - if (length == 0) { - length = ips[cursor + 5] << 8 | ips[cursor + 6]; - - if (buffer.size() < offset + length) - return; - - for (u32 i = 0; i < length; ++i) - buffer[offset + i] = ips[cursor + 7]; - - cursor += 8; - - continue; - } - - if (buffer.size() < offset + length) - return; - - std::memcpy(&buffer[offset], &ips[cursor + 5], length); - cursor += length + 5; - } -} - /** * Get the decompressed size of an LZSS compressed ExeFS file * @param buffer Buffer of compressed file @@ -553,7 +507,7 @@ Loader::ResultStatus NCCHContainer::LoadSectionExeFS(const char* name, std::vect return Loader::ResultStatus::ErrorNotUsed; } -bool NCCHContainer::ApplyIPSPatch(std::vector& code) const { +bool NCCHContainer::ApplyCodePatch(std::vector& code) const { const std::string override_ips = filepath + ".exefsdir/code.ips"; FileUtil::IOFile ips_file{override_ips, "rb"}; @@ -565,7 +519,7 @@ bool NCCHContainer::ApplyIPSPatch(std::vector& code) const { return false; LOG_INFO(Service_FS, "File {} patching code.bin", override_ips); - ApplyIPS(ips, code); + Patch::ApplyIpsPatch(ips, code); return true; } diff --git a/src/core/file_sys/ncch_container.h b/src/core/file_sys/ncch_container.h index 00d44c818..8752ffe39 100644 --- a/src/core/file_sys/ncch_container.h +++ b/src/core/file_sys/ncch_container.h @@ -272,11 +272,11 @@ public: Loader::ResultStatus ReadExtdataId(u64& extdata_id); /** - * Apply an IPS patch for .code (if it exists). + * Apply a patch for .code (if it exists). * This should only be called after allocating .bss. * @return bool true if a patch was applied, false otherwise */ - bool ApplyIPSPatch(std::vector& code) const; + bool ApplyCodePatch(std::vector& code) const; /** * Checks whether the NCCH container contains an ExeFS diff --git a/src/core/file_sys/patch.cpp b/src/core/file_sys/patch.cpp new file mode 100644 index 000000000..4bd7d7246 --- /dev/null +++ b/src/core/file_sys/patch.cpp @@ -0,0 +1,55 @@ +// Copyright 2019 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include "common/logging/log.h" +#include "core/file_sys/patch.h" + +namespace FileSys::Patch { + +bool ApplyIpsPatch(const std::vector& ips, std::vector& buffer) { + u32 cursor = 5; + u32 patch_length = ips.size() - 3; + std::string ips_header(ips.begin(), ips.begin() + 5); + + if (ips_header != "PATCH") { + LOG_INFO(Service_FS, "Attempted to load invalid IPS"); + return false; + } + + while (cursor < patch_length) { + std::string eof_check(ips.begin() + cursor, ips.begin() + cursor + 3); + + if (eof_check == "EOF") + return false; + + u32 offset = ips[cursor] << 16 | ips[cursor + 1] << 8 | ips[cursor + 2]; + std::size_t length = ips[cursor + 3] << 8 | ips[cursor + 4]; + + // check for an rle record + if (length == 0) { + length = ips[cursor + 5] << 8 | ips[cursor + 6]; + + if (buffer.size() < offset + length) + return false; + + for (u32 i = 0; i < length; ++i) + buffer[offset + i] = ips[cursor + 7]; + + cursor += 8; + + continue; + } + + if (buffer.size() < offset + length) + return false; + + std::memcpy(&buffer[offset], &ips[cursor + 5], length); + cursor += length + 5; + } + return true; +} + +} // namespace FileSys::Patch diff --git a/src/core/file_sys/patch.h b/src/core/file_sys/patch.h new file mode 100644 index 000000000..b77ca2bcc --- /dev/null +++ b/src/core/file_sys/patch.h @@ -0,0 +1,15 @@ +// Copyright 2019 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include + +#include "common/common_types.h" + +namespace FileSys::Patch { + +bool ApplyIpsPatch(const std::vector& patch, std::vector& buffer); + +} // namespace FileSys::Patch diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp index 016a11ef2..07edad796 100644 --- a/src/core/loader/ncch.cpp +++ b/src/core/loader/ncch.cpp @@ -101,7 +101,7 @@ ResultStatus AppLoader_NCCH::LoadExec(std::shared_ptr& process) bss_page_size; // Apply any IPS patch now that the entire codeset (including .bss) has been allocated - overlay_ncch->ApplyIPSPatch(code); + overlay_ncch->ApplyCodePatch(code); codeset->entrypoint = codeset->CodeSegment().addr; codeset->memory = std::move(code); From 3140086c60118834fb673bc0b07bcda007520f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Lam?= Date: Sat, 21 Dec 2019 13:31:34 +0100 Subject: [PATCH 2/3] file_sys: Handle patch applying failures This changes ApplyCodePatch to return a ResultStatus, which makes it possible to determine whether patch applying has failed. Previously, only a boolean was returned, and false was returned when no patch was found OR when a patch was found but applying it failed. This also changes AppLoader_NCCH to return an error if patching fails because the executable is likely to be left in an inconsistent state and we should not proceed booting in that case. --- src/core/file_sys/ncch_container.cpp | 12 +++++++----- src/core/file_sys/ncch_container.h | 4 ++-- src/core/loader/ncch.cpp | 6 ++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/core/file_sys/ncch_container.cpp b/src/core/file_sys/ncch_container.cpp index 0412fa288..ec6bb3827 100644 --- a/src/core/file_sys/ncch_container.cpp +++ b/src/core/file_sys/ncch_container.cpp @@ -507,20 +507,22 @@ Loader::ResultStatus NCCHContainer::LoadSectionExeFS(const char* name, std::vect return Loader::ResultStatus::ErrorNotUsed; } -bool NCCHContainer::ApplyCodePatch(std::vector& code) const { +Loader::ResultStatus NCCHContainer::ApplyCodePatch(std::vector& code) const { const std::string override_ips = filepath + ".exefsdir/code.ips"; FileUtil::IOFile ips_file{override_ips, "rb"}; if (!ips_file) - return false; + return Loader::ResultStatus::ErrorNotUsed; std::vector ips(ips_file.GetSize()); if (ips_file.ReadBytes(ips.data(), ips.size()) != ips.size()) - return false; + return Loader::ResultStatus::Error; LOG_INFO(Service_FS, "File {} patching code.bin", override_ips); - Patch::ApplyIpsPatch(ips, code); - return true; + if (!Patch::ApplyIpsPatch(ips, code)) + return Loader::ResultStatus::Error; + + return Loader::ResultStatus::Success; } Loader::ResultStatus NCCHContainer::LoadOverrideExeFSSection(const char* name, diff --git a/src/core/file_sys/ncch_container.h b/src/core/file_sys/ncch_container.h index 8752ffe39..f06ee8ef6 100644 --- a/src/core/file_sys/ncch_container.h +++ b/src/core/file_sys/ncch_container.h @@ -274,9 +274,9 @@ public: /** * Apply a patch for .code (if it exists). * This should only be called after allocating .bss. - * @return bool true if a patch was applied, false otherwise + * @return ResultStatus success if a patch was applied, ErrorNotUsed if no patch was found */ - bool ApplyCodePatch(std::vector& code) const; + Loader::ResultStatus ApplyCodePatch(std::vector& code) const; /** * Checks whether the NCCH container contains an ExeFS diff --git a/src/core/loader/ncch.cpp b/src/core/loader/ncch.cpp index 07edad796..2e688e011 100644 --- a/src/core/loader/ncch.cpp +++ b/src/core/loader/ncch.cpp @@ -100,8 +100,10 @@ ResultStatus AppLoader_NCCH::LoadExec(std::shared_ptr& process) overlay_ncch->exheader_header.codeset_info.data.num_max_pages * Memory::PAGE_SIZE + bss_page_size; - // Apply any IPS patch now that the entire codeset (including .bss) has been allocated - overlay_ncch->ApplyCodePatch(code); + // Apply patches now that the entire codeset (including .bss) has been allocated + const ResultStatus patch_result = overlay_ncch->ApplyCodePatch(code); + if (patch_result != ResultStatus::Success && patch_result != ResultStatus::ErrorNotUsed) + return patch_result; codeset->entrypoint = codeset->CodeSegment().addr; codeset->memory = std::move(code); From 756d231ff90a63b216385fcd9dd013cb8aee5b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Lam?= Date: Mon, 25 Nov 2019 20:49:48 +0100 Subject: [PATCH 3/3] file_sys: Add support for BPS patches The BPS format allows distributing patches that are smaller and that do not contain copyrighted content if data is relocated (unlike non-trivial IPS patches). This is essential for games such as MM3D that have three barely different code revisions. Supporting all three versions would demand an unreasonable amount of work; with BPS patches only one version has to be supported. --- src/core/file_sys/ncch_container.cpp | 32 ++-- src/core/file_sys/patch.cpp | 213 +++++++++++++++++++++++++++ src/core/file_sys/patch.h | 2 + 3 files changed, 236 insertions(+), 11 deletions(-) diff --git a/src/core/file_sys/ncch_container.cpp b/src/core/file_sys/ncch_container.cpp index ec6bb3827..f0687fa9e 100644 --- a/src/core/file_sys/ncch_container.cpp +++ b/src/core/file_sys/ncch_container.cpp @@ -508,21 +508,31 @@ Loader::ResultStatus NCCHContainer::LoadSectionExeFS(const char* name, std::vect } Loader::ResultStatus NCCHContainer::ApplyCodePatch(std::vector& code) const { - const std::string override_ips = filepath + ".exefsdir/code.ips"; + struct PatchLocation { + std::string path; + bool (*patch_fn)(const std::vector& patch, std::vector& code); + }; + const std::array patch_paths{{ + {filepath + ".exefsdir/code.ips", Patch::ApplyIpsPatch}, + {filepath + ".exefsdir/code.bps", Patch::ApplyBpsPatch}, + }}; - FileUtil::IOFile ips_file{override_ips, "rb"}; - if (!ips_file) - return Loader::ResultStatus::ErrorNotUsed; + for (const PatchLocation& info : patch_paths) { + FileUtil::IOFile file{info.path, "rb"}; + if (!file) + continue; - std::vector ips(ips_file.GetSize()); - if (ips_file.ReadBytes(ips.data(), ips.size()) != ips.size()) - return Loader::ResultStatus::Error; + std::vector patch(file.GetSize()); + if (file.ReadBytes(patch.data(), patch.size()) != patch.size()) + return Loader::ResultStatus::Error; - LOG_INFO(Service_FS, "File {} patching code.bin", override_ips); - if (!Patch::ApplyIpsPatch(ips, code)) - return Loader::ResultStatus::Error; + LOG_INFO(Service_FS, "File {} patching code.bin", info.path); + if (!info.patch_fn(patch, code)) + return Loader::ResultStatus::Error; - return Loader::ResultStatus::Success; + return Loader::ResultStatus::Success; + } + return Loader::ResultStatus::ErrorNotUsed; } Loader::ResultStatus NCCHContainer::LoadOverrideExeFSSection(const char* name, diff --git a/src/core/file_sys/patch.cpp b/src/core/file_sys/patch.cpp index 4bd7d7246..c93767165 100644 --- a/src/core/file_sys/patch.cpp +++ b/src/core/file_sys/patch.cpp @@ -2,8 +2,13 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include +#include #include +#include +#include +#include #include "common/logging/log.h" #include "core/file_sys/patch.h" @@ -52,4 +57,212 @@ bool ApplyIpsPatch(const std::vector& ips, std::vector& buffer) { return true; } +namespace Bps { + +// The BPS format uses variable length encoding for all integers. +// Realistically uint32s are more than enough for code patching. +using Number = u32; + +constexpr std::size_t FooterSize = 12; + +// The BPS format uses CRC32 checksums. +static u32 crc32(const u8* data, std::size_t size) { + boost::crc_32_type result; + result.process_bytes(data, size); + return result.checksum(); +} + +// Utility class to make keeping track of offsets and bound checks less error prone. +template +class Stream { +public: + Stream(T* ptr, std::size_t size) : m_ptr{ptr}, m_size{size} {} + + bool Read(void* buffer, std::size_t length) { + if (m_offset + length > m_size) + return false; + std::memcpy(buffer, m_ptr + m_offset, length); + m_offset += length; + return true; + } + + template + bool CopyFrom(Stream& other, std::size_t length) { + if (m_offset + length > m_size) + return false; + if (!other.Read(m_ptr + m_offset, length)) + return false; + m_offset += length; + return true; + } + + template + std::optional Read() { + static_assert(std::is_pod_v); + ValueType val{}; + if (!Read(&val, sizeof(val))) + return std::nullopt; + return val; + } + + Number ReadNumber() { + Number data = 0, shift = 1; + std::optional x; + while ((x = Read())) { + data += (*x & 0x7f) * shift; + if (*x & 0x80) + break; + shift <<= 7; + data += shift; + } + return data; + } + + auto data() const { + return m_ptr; + } + + std::size_t size() const { + return m_size; + } + + std::size_t Tell() const { + return m_offset; + } + + bool Seek(size_t offset) { + if (offset > m_size) + return false; + m_offset = offset; + return true; + } + +private: + T* m_ptr = nullptr; + std::size_t m_size = 0; + std::size_t m_offset = 0; +}; + +class PatchApplier { +public: + PatchApplier(Stream source, Stream target, Stream patch) + : m_source{source}, m_target{target}, m_patch{patch} {} + + bool Apply() { + const auto magic = *m_patch.Read>(); + if (std::string_view(magic.data(), magic.size()) != "BPS1") { + LOG_ERROR(Service_FS, "Invalid BPS magic"); + return false; + } + + const Bps::Number source_size = m_patch.ReadNumber(); + const Bps::Number target_size = m_patch.ReadNumber(); + const Bps::Number metadata_size = m_patch.ReadNumber(); + if (source_size > m_source.size() || target_size > m_target.size() || metadata_size != 0) { + LOG_ERROR(Service_FS, "Invalid sizes"); + return false; + } + + const std::size_t command_start_offset = m_patch.Tell(); + const std::size_t command_end_offset = m_patch.size() - FooterSize; + m_patch.Seek(command_end_offset); + const u32 source_crc32 = *m_patch.Read(); + const u32 target_crc32 = *m_patch.Read(); + m_patch.Seek(command_start_offset); + + if (crc32(m_source.data(), source_size) != source_crc32) { + LOG_ERROR(Service_FS, "Unexpected source hash"); + return false; + } + + // Process all patch commands. + std::memset(m_target.data(), 0, m_target.size()); + while (m_patch.Tell() < command_end_offset) { + if (!HandleCommand()) + return false; + } + + if (crc32(m_target.data(), target_size) != target_crc32) { + LOG_ERROR(Service_FS, "Unexpected target hash"); + return false; + } + + return true; + } + +private: + bool HandleCommand() { + const std::size_t offset = m_patch.Tell(); + const Number data = m_patch.ReadNumber(); + const Number command = data & 3; + const Number length = (data >> 2) + 1; + + const bool ok = [&] { + switch (command) { + case 0: + return SourceRead(length); + case 1: + return TargetRead(length); + case 2: + return SourceCopy(length); + case 3: + return TargetCopy(length); + default: + return false; + } + }(); + if (!ok) + LOG_ERROR(Service_FS, "Failed to process command {} at 0x{:x}", command, offset); + return ok; + } + + bool SourceRead(Number length) { + return m_source.Seek(m_target.Tell()) && m_target.CopyFrom(m_source, length); + } + + bool TargetRead(Number length) { + return m_target.CopyFrom(m_patch, length); + } + + bool SourceCopy(Number length) { + const Number data = m_patch.ReadNumber(); + m_source_relative_offset += (data & 1 ? -1 : +1) * int(data >> 1); + if (!m_source.Seek(m_source_relative_offset) || !m_target.CopyFrom(m_source, length)) + return false; + m_source_relative_offset += length; + return true; + } + + bool TargetCopy(Number length) { + const Number data = m_patch.ReadNumber(); + m_target_relative_offset += (data & 1 ? -1 : +1) * int(data >> 1); + if (m_target.Tell() + length > m_target.size()) + return false; + if (m_target_relative_offset + length > m_target.size()) + return false; + // Byte by byte copy. + for (size_t i = 0; i < length; ++i) + m_target.data()[m_target.Tell() + i] = m_target.data()[m_target_relative_offset++]; + m_target.Seek(m_target.Tell() + length); + return true; + } + + std::size_t m_source_relative_offset = 0; + std::size_t m_target_relative_offset = 0; + Stream m_source; + Stream m_target; + Stream m_patch; +}; + +} // namespace Bps + +bool ApplyBpsPatch(const std::vector& patch, std::vector& buffer) { + const std::vector source = buffer; + Bps::Stream source_stream{source.data(), source.size()}; + Bps::Stream target_stream{buffer.data(), buffer.size()}; + Bps::Stream patch_stream{patch.data(), patch.size()}; + Bps::PatchApplier applier{source_stream, target_stream, patch_stream}; + return applier.Apply(); +} + } // namespace FileSys::Patch diff --git a/src/core/file_sys/patch.h b/src/core/file_sys/patch.h index b77ca2bcc..9a8118475 100644 --- a/src/core/file_sys/patch.h +++ b/src/core/file_sys/patch.h @@ -12,4 +12,6 @@ namespace FileSys::Patch { bool ApplyIpsPatch(const std::vector& patch, std::vector& buffer); +bool ApplyBpsPatch(const std::vector& patch, std::vector& buffer); + } // namespace FileSys::Patch