2024-03-05 21:49:07 +01:00
|
|
|
// SPDX-FileCopyrightText: Copyright 2019 suyu Emulator Project
|
2022-04-23 10:59:50 +02:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2019-03-02 21:20:28 +01:00
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
#include <atomic>
|
2020-11-18 01:58:41 +01:00
|
|
|
|
2019-03-02 21:20:28 +01:00
|
|
|
#include "common/common_types.h"
|
2023-10-23 03:16:38 +02:00
|
|
|
#include "common/typed_address.h"
|
2020-04-09 04:49:51 +02:00
|
|
|
#include "common/virtual_buffer.h"
|
2019-03-02 21:20:28 +01:00
|
|
|
|
|
|
|
namespace Common {
|
|
|
|
|
|
|
|
enum class PageType : u8 {
|
|
|
|
/// Page is unmapped and should cause an access error.
|
|
|
|
Unmapped,
|
|
|
|
/// Page is mapped to regular memory. This is the only type you can get pointers to.
|
|
|
|
Memory,
|
2022-06-06 18:56:01 +02:00
|
|
|
/// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use
|
|
|
|
/// the callbacks.
|
|
|
|
DebugMemory,
|
2019-03-02 21:20:28 +01:00
|
|
|
/// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and
|
|
|
|
/// invalidation
|
|
|
|
RasterizerCachedMemory,
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A (reasonably) fast way of allowing switchable and remappable process address spaces. It loosely
|
|
|
|
* mimics the way a real CPU page table works.
|
|
|
|
*/
|
|
|
|
struct PageTable {
|
2022-02-19 08:42:27 +01:00
|
|
|
struct TraversalEntry {
|
|
|
|
u64 phys_addr{};
|
|
|
|
std::size_t block_size{};
|
|
|
|
};
|
|
|
|
|
|
|
|
struct TraversalContext {
|
|
|
|
u64 next_page{};
|
|
|
|
u64 next_offset{};
|
|
|
|
};
|
|
|
|
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
/// Number of bits reserved for attribute tagging.
|
|
|
|
/// This can be at most the guaranteed alignment of the pointers in the page table.
|
|
|
|
static constexpr int ATTRIBUTE_BITS = 2;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Pair of host pointer and page type attribute.
|
|
|
|
* This uses the lower bits of a given pointer to store the attribute tag.
|
|
|
|
* Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method
|
|
|
|
* call. In other words, they are guaranteed to be synchronized at all times.
|
|
|
|
*/
|
|
|
|
class PageInfo {
|
|
|
|
public:
|
|
|
|
/// Returns the page pointer
|
Fixes and workarounds to make UBSan happier on macOS
There are still some other issues not addressed here, but it's a start.
Workarounds for false-positive reports:
- `RasterizerAccelerated`: Put a gigantic array behind a `unique_ptr`,
because UBSan has a [hardcoded limit](https://stackoverflow.com/questions/64531383/c-runtime-error-using-fsanitize-undefined-object-has-a-possibly-invalid-vp)
of how big it thinks objects can be, specifically when dealing with
offset-to-top values used with multiple inheritance. Hopefully this
doesn't have a performance impact.
- `QueryCacheBase::QueryCacheBase`: Avoid an operation that UBSan thinks
is UB even though it at least arguably isn't. See the link in the
comment for more information.
Fixes for correct reports:
- `PageTable`, `Memory`: Use `uintptr_t` values instead of pointers to
avoid UB from pointer overflow (when pointer arithmetic wraps around
the address space).
- `KScheduler::Reload`: `thread->GetOwnerProcess()` can be `nullptr`;
avoid calling methods on it in this case. (The existing code returns
a garbage reference to a field, which is then passed into
`LoadWatchpointArray`, and apparently it's never used, so it's
harmless in practice but still triggers UBSan.)
- `KAutoObject::Close`: This function calls `this->Destroy()`, which
overwrites the beginning of the object with junk (specifically a free
list pointer). Then it calls `this->UnregisterWithKernel()`. UBSan
complains about a type mismatch because the vtable has been
overwritten, and I believe this is indeed UB. `UnregisterWithKernel`
also loads `m_kernel` from the 'freed' object, which seems to be
technically safe (the overwriting doesn't extend as far as that
field), but seems dubious. Switch to a `static` method and load
`m_kernel` in advance.
2023-07-02 00:00:39 +02:00
|
|
|
[[nodiscard]] uintptr_t Pointer() const noexcept {
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
return ExtractPointer(raw.load(std::memory_order_relaxed));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns the page type attribute
|
|
|
|
[[nodiscard]] PageType Type() const noexcept {
|
|
|
|
return ExtractType(raw.load(std::memory_order_relaxed));
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns the page pointer and attribute pair, extracted from the same atomic read
|
Fixes and workarounds to make UBSan happier on macOS
There are still some other issues not addressed here, but it's a start.
Workarounds for false-positive reports:
- `RasterizerAccelerated`: Put a gigantic array behind a `unique_ptr`,
because UBSan has a [hardcoded limit](https://stackoverflow.com/questions/64531383/c-runtime-error-using-fsanitize-undefined-object-has-a-possibly-invalid-vp)
of how big it thinks objects can be, specifically when dealing with
offset-to-top values used with multiple inheritance. Hopefully this
doesn't have a performance impact.
- `QueryCacheBase::QueryCacheBase`: Avoid an operation that UBSan thinks
is UB even though it at least arguably isn't. See the link in the
comment for more information.
Fixes for correct reports:
- `PageTable`, `Memory`: Use `uintptr_t` values instead of pointers to
avoid UB from pointer overflow (when pointer arithmetic wraps around
the address space).
- `KScheduler::Reload`: `thread->GetOwnerProcess()` can be `nullptr`;
avoid calling methods on it in this case. (The existing code returns
a garbage reference to a field, which is then passed into
`LoadWatchpointArray`, and apparently it's never used, so it's
harmless in practice but still triggers UBSan.)
- `KAutoObject::Close`: This function calls `this->Destroy()`, which
overwrites the beginning of the object with junk (specifically a free
list pointer). Then it calls `this->UnregisterWithKernel()`. UBSan
complains about a type mismatch because the vtable has been
overwritten, and I believe this is indeed UB. `UnregisterWithKernel`
also loads `m_kernel` from the 'freed' object, which seems to be
technically safe (the overwriting doesn't extend as far as that
field), but seems dubious. Switch to a `static` method and load
`m_kernel` in advance.
2023-07-02 00:00:39 +02:00
|
|
|
[[nodiscard]] std::pair<uintptr_t, PageType> PointerType() const noexcept {
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed);
|
|
|
|
return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)};
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns the raw representation of the page information.
|
|
|
|
/// Use ExtractPointer and ExtractType to unpack the value.
|
|
|
|
[[nodiscard]] uintptr_t Raw() const noexcept {
|
|
|
|
return raw.load(std::memory_order_relaxed);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Write a page pointer and type pair atomically
|
Fixes and workarounds to make UBSan happier on macOS
There are still some other issues not addressed here, but it's a start.
Workarounds for false-positive reports:
- `RasterizerAccelerated`: Put a gigantic array behind a `unique_ptr`,
because UBSan has a [hardcoded limit](https://stackoverflow.com/questions/64531383/c-runtime-error-using-fsanitize-undefined-object-has-a-possibly-invalid-vp)
of how big it thinks objects can be, specifically when dealing with
offset-to-top values used with multiple inheritance. Hopefully this
doesn't have a performance impact.
- `QueryCacheBase::QueryCacheBase`: Avoid an operation that UBSan thinks
is UB even though it at least arguably isn't. See the link in the
comment for more information.
Fixes for correct reports:
- `PageTable`, `Memory`: Use `uintptr_t` values instead of pointers to
avoid UB from pointer overflow (when pointer arithmetic wraps around
the address space).
- `KScheduler::Reload`: `thread->GetOwnerProcess()` can be `nullptr`;
avoid calling methods on it in this case. (The existing code returns
a garbage reference to a field, which is then passed into
`LoadWatchpointArray`, and apparently it's never used, so it's
harmless in practice but still triggers UBSan.)
- `KAutoObject::Close`: This function calls `this->Destroy()`, which
overwrites the beginning of the object with junk (specifically a free
list pointer). Then it calls `this->UnregisterWithKernel()`. UBSan
complains about a type mismatch because the vtable has been
overwritten, and I believe this is indeed UB. `UnregisterWithKernel`
also loads `m_kernel` from the 'freed' object, which seems to be
technically safe (the overwriting doesn't extend as far as that
field), but seems dubious. Switch to a `static` method and load
`m_kernel` in advance.
2023-07-02 00:00:39 +02:00
|
|
|
void Store(uintptr_t pointer, PageType type) noexcept {
|
|
|
|
raw.store(pointer | static_cast<uintptr_t>(type));
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Unpack a pointer from a page info raw representation
|
Fixes and workarounds to make UBSan happier on macOS
There are still some other issues not addressed here, but it's a start.
Workarounds for false-positive reports:
- `RasterizerAccelerated`: Put a gigantic array behind a `unique_ptr`,
because UBSan has a [hardcoded limit](https://stackoverflow.com/questions/64531383/c-runtime-error-using-fsanitize-undefined-object-has-a-possibly-invalid-vp)
of how big it thinks objects can be, specifically when dealing with
offset-to-top values used with multiple inheritance. Hopefully this
doesn't have a performance impact.
- `QueryCacheBase::QueryCacheBase`: Avoid an operation that UBSan thinks
is UB even though it at least arguably isn't. See the link in the
comment for more information.
Fixes for correct reports:
- `PageTable`, `Memory`: Use `uintptr_t` values instead of pointers to
avoid UB from pointer overflow (when pointer arithmetic wraps around
the address space).
- `KScheduler::Reload`: `thread->GetOwnerProcess()` can be `nullptr`;
avoid calling methods on it in this case. (The existing code returns
a garbage reference to a field, which is then passed into
`LoadWatchpointArray`, and apparently it's never used, so it's
harmless in practice but still triggers UBSan.)
- `KAutoObject::Close`: This function calls `this->Destroy()`, which
overwrites the beginning of the object with junk (specifically a free
list pointer). Then it calls `this->UnregisterWithKernel()`. UBSan
complains about a type mismatch because the vtable has been
overwritten, and I believe this is indeed UB. `UnregisterWithKernel`
also loads `m_kernel` from the 'freed' object, which seems to be
technically safe (the overwriting doesn't extend as far as that
field), but seems dubious. Switch to a `static` method and load
`m_kernel` in advance.
2023-07-02 00:00:39 +02:00
|
|
|
[[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept {
|
|
|
|
return raw & (~uintptr_t{0} << ATTRIBUTE_BITS);
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Unpack a page type from a page info raw representation
|
|
|
|
[[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept {
|
|
|
|
return static_cast<PageType>(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1));
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
std::atomic<uintptr_t> raw;
|
|
|
|
};
|
|
|
|
|
2020-04-09 04:49:51 +02:00
|
|
|
PageTable();
|
2020-11-18 01:58:41 +01:00
|
|
|
~PageTable() noexcept;
|
|
|
|
|
|
|
|
PageTable(const PageTable&) = delete;
|
|
|
|
PageTable& operator=(const PageTable&) = delete;
|
|
|
|
|
|
|
|
PageTable(PageTable&&) noexcept = default;
|
|
|
|
PageTable& operator=(PageTable&&) noexcept = default;
|
2019-03-02 21:20:28 +01:00
|
|
|
|
2023-10-23 03:16:38 +02:00
|
|
|
bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context,
|
|
|
|
Common::ProcessAddress address) const;
|
|
|
|
bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const;
|
2022-02-19 08:42:27 +01:00
|
|
|
|
2019-03-02 21:20:28 +01:00
|
|
|
/**
|
2021-01-02 15:00:05 +01:00
|
|
|
* Resizes the page table to be able to accommodate enough pages within
|
2019-03-02 21:20:28 +01:00
|
|
|
* a given address space.
|
|
|
|
*
|
|
|
|
* @param address_space_width_in_bits The address size width in bits.
|
2020-11-18 01:45:17 +01:00
|
|
|
* @param page_size_in_bits The page size in bits.
|
2019-03-02 21:20:28 +01:00
|
|
|
*/
|
2022-02-19 08:42:27 +01:00
|
|
|
void Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits);
|
2019-03-02 21:20:28 +01:00
|
|
|
|
2022-02-19 08:42:27 +01:00
|
|
|
std::size_t GetAddressSpaceBits() const {
|
2021-05-29 09:24:09 +02:00
|
|
|
return current_address_space_width_in_bits;
|
|
|
|
}
|
|
|
|
|
2023-10-23 03:16:38 +02:00
|
|
|
bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr,
|
|
|
|
Common::ProcessAddress virt_addr) const {
|
|
|
|
if (virt_addr > (1ULL << this->GetAddressSpaceBits())) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
*out_phys_addr = backing_addr[virt_addr / page_size] + GetInteger(virt_addr);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-03-02 21:20:28 +01:00
|
|
|
/**
|
|
|
|
* Vector of memory pointers backing each page. An entry can only be non-null if the
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
* corresponding attribute element is of type `Memory`.
|
2019-03-02 21:20:28 +01:00
|
|
|
*/
|
core/memory: Read and write page table atomically
Squash attributes into the pointer's integer, making them an uintptr_t
pair containing 2 bits at the bottom and then the pointer. These bits
are currently unused thanks to alignment requirements.
Configure Dynarmic to mask out these bits on pointer reads.
While we are at it, remove some unused attributes carried over from
Citra.
Read/Write and other hot functions use a two step unpacking process that
is less readable to stop MSVC from emitting an extra AND instruction in
the hot path:
mov rdi,rcx
shr rdx,0Ch
mov r8,qword ptr [rax+8]
mov rax,qword ptr [r8+rdx*8]
mov rdx,rax
-and al,3
and rdx,0FFFFFFFFFFFFFFFCh
je Core::Memory::Memory::Impl::Read<unsigned char>
mov rax,qword ptr [vaddr]
movzx eax,byte ptr [rdx+rax]
2020-12-30 01:16:57 +01:00
|
|
|
VirtualBuffer<PageInfo> pointers;
|
2023-05-29 01:35:51 +02:00
|
|
|
VirtualBuffer<u64> blocks;
|
2020-03-13 21:33:47 +01:00
|
|
|
|
2020-04-09 04:49:51 +02:00
|
|
|
VirtualBuffer<u64> backing_addr;
|
2021-05-29 09:24:09 +02:00
|
|
|
|
2022-02-19 08:42:27 +01:00
|
|
|
std::size_t current_address_space_width_in_bits{};
|
|
|
|
|
|
|
|
u8* fastmem_arena{};
|
2020-01-19 01:49:30 +01:00
|
|
|
|
2022-02-19 08:42:27 +01:00
|
|
|
std::size_t page_size{};
|
2020-03-13 21:33:47 +01:00
|
|
|
};
|
|
|
|
|
2019-03-02 21:20:28 +01:00
|
|
|
} // namespace Common
|