2022-04-23 10:59:50 +02:00
|
|
|
// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project
|
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2021-02-13 02:02:51 +01:00
|
|
|
#include "core/hle/kernel/k_memory_block_manager.h"
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2021-02-13 02:02:51 +01:00
|
|
|
namespace Kernel {
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
KMemoryBlockManager::KMemoryBlockManager() = default;
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
Result KMemoryBlockManager::Initialize(VAddr st, VAddr nd, KMemoryBlockSlabManager* slab_manager) {
|
|
|
|
// Allocate a block to encapsulate the address space, insert it into the tree.
|
|
|
|
KMemoryBlock* start_block = slab_manager->Allocate();
|
|
|
|
R_UNLESS(start_block != nullptr, ResultOutOfResource);
|
|
|
|
|
|
|
|
// Set our start and end.
|
|
|
|
m_start_address = st;
|
|
|
|
m_end_address = nd;
|
|
|
|
ASSERT(Common::IsAligned(m_start_address, PageSize));
|
|
|
|
ASSERT(Common::IsAligned(m_end_address, PageSize));
|
|
|
|
|
|
|
|
// Initialize and insert the block.
|
|
|
|
start_block->Initialize(m_start_address, (m_end_address - m_start_address) / PageSize,
|
|
|
|
KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None);
|
|
|
|
m_memory_block_tree.insert(*start_block);
|
|
|
|
|
2022-10-15 07:55:51 +02:00
|
|
|
R_SUCCEED();
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
void KMemoryBlockManager::Finalize(KMemoryBlockSlabManager* slab_manager,
|
|
|
|
HostUnmapCallback&& host_unmap_callback) {
|
|
|
|
// Erase every block until we have none left.
|
|
|
|
auto it = m_memory_block_tree.begin();
|
|
|
|
while (it != m_memory_block_tree.end()) {
|
|
|
|
KMemoryBlock* block = std::addressof(*it);
|
|
|
|
it = m_memory_block_tree.erase(it);
|
|
|
|
slab_manager->Free(block);
|
|
|
|
host_unmap_callback(block->GetAddress(), block->GetSize());
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
ASSERT(m_memory_block_tree.empty());
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
VAddr KMemoryBlockManager::FindFreeArea(VAddr region_start, size_t region_num_pages,
|
|
|
|
size_t num_pages, size_t alignment, size_t offset,
|
|
|
|
size_t guard_pages) const {
|
|
|
|
if (num_pages > 0) {
|
|
|
|
const VAddr region_end = region_start + region_num_pages * PageSize;
|
|
|
|
const VAddr region_last = region_end - 1;
|
|
|
|
for (const_iterator it = this->FindIterator(region_start); it != m_memory_block_tree.cend();
|
|
|
|
it++) {
|
|
|
|
const KMemoryInfo info = it->GetMemoryInfo();
|
|
|
|
if (region_last < info.GetAddress()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (info.m_state != KMemoryState::Free) {
|
|
|
|
continue;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
VAddr area = (info.GetAddress() <= region_start) ? region_start : info.GetAddress();
|
|
|
|
area += guard_pages * PageSize;
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
const VAddr offset_area = Common::AlignDown(area, alignment) + offset;
|
|
|
|
area = (area <= offset_area) ? offset_area : offset_area + alignment;
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
const VAddr area_end = area + num_pages * PageSize + guard_pages * PageSize;
|
|
|
|
const VAddr area_last = area_end - 1;
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
if (info.GetAddress() <= area && area < area_last && area_last <= region_last &&
|
|
|
|
area_last <= info.GetLastAddress()) {
|
|
|
|
return area;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return {};
|
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
void KMemoryBlockManager::CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator* allocator,
|
|
|
|
VAddr address, size_t num_pages) {
|
|
|
|
// Find the iterator now that we've updated.
|
|
|
|
iterator it = this->FindIterator(address);
|
|
|
|
if (address != m_start_address) {
|
|
|
|
it--;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Coalesce blocks that we can.
|
|
|
|
while (true) {
|
|
|
|
iterator prev = it++;
|
|
|
|
if (it == m_memory_block_tree.end()) {
|
|
|
|
break;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
if (prev->CanMergeWith(*it)) {
|
|
|
|
KMemoryBlock* block = std::addressof(*it);
|
|
|
|
m_memory_block_tree.erase(it);
|
|
|
|
prev->Add(*block);
|
|
|
|
allocator->Free(block);
|
|
|
|
it = prev;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
if (address + num_pages * PageSize < it->GetMemoryInfo().GetEndAddress()) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address,
|
|
|
|
size_t num_pages, KMemoryState state, KMemoryPermission perm,
|
|
|
|
KMemoryAttribute attr,
|
|
|
|
KMemoryBlockDisableMergeAttribute set_disable_attr,
|
|
|
|
KMemoryBlockDisableMergeAttribute clear_disable_attr) {
|
|
|
|
// Ensure for auditing that we never end up with an invalid tree.
|
|
|
|
KScopedMemoryBlockManagerAuditor auditor(this);
|
|
|
|
ASSERT(Common::IsAligned(address, PageSize));
|
|
|
|
ASSERT((attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) ==
|
|
|
|
KMemoryAttribute::None);
|
|
|
|
|
|
|
|
VAddr cur_address = address;
|
|
|
|
size_t remaining_pages = num_pages;
|
|
|
|
iterator it = this->FindIterator(address);
|
|
|
|
|
|
|
|
while (remaining_pages > 0) {
|
|
|
|
const size_t remaining_size = remaining_pages * PageSize;
|
|
|
|
KMemoryInfo cur_info = it->GetMemoryInfo();
|
|
|
|
if (it->HasProperties(state, perm, attr)) {
|
|
|
|
// If we already have the right properties, just advance.
|
|
|
|
if (cur_address + remaining_size < cur_info.GetEndAddress()) {
|
|
|
|
remaining_pages = 0;
|
|
|
|
cur_address += remaining_size;
|
|
|
|
} else {
|
|
|
|
remaining_pages =
|
|
|
|
(cur_address + remaining_size - cur_info.GetEndAddress()) / PageSize;
|
|
|
|
cur_address = cur_info.GetEndAddress();
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
2022-09-10 06:12:37 +02:00
|
|
|
} else {
|
|
|
|
// If we need to, create a new block before and insert it.
|
|
|
|
if (cur_info.GetAddress() != cur_address) {
|
|
|
|
KMemoryBlock* new_block = allocator->Allocate();
|
|
|
|
|
|
|
|
it->Split(new_block, cur_address);
|
|
|
|
it = m_memory_block_tree.insert(*new_block);
|
|
|
|
it++;
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
cur_info = it->GetMemoryInfo();
|
|
|
|
cur_address = cur_info.GetAddress();
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// If we need to, create a new block after and insert it.
|
|
|
|
if (cur_info.GetSize() > remaining_size) {
|
|
|
|
KMemoryBlock* new_block = allocator->Allocate();
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
it->Split(new_block, cur_address + remaining_size);
|
|
|
|
it = m_memory_block_tree.insert(*new_block);
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
cur_info = it->GetMemoryInfo();
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Update block state.
|
|
|
|
it->Update(state, perm, attr, cur_address == address, static_cast<u8>(set_disable_attr),
|
|
|
|
static_cast<u8>(clear_disable_attr));
|
|
|
|
cur_address += cur_info.GetSize();
|
|
|
|
remaining_pages -= cur_info.GetNumPages();
|
|
|
|
}
|
|
|
|
it++;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
2022-09-10 06:12:37 +02:00
|
|
|
|
|
|
|
this->CoalesceForUpdate(allocator, address, num_pages);
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
void KMemoryBlockManager::UpdateIfMatch(KMemoryBlockManagerUpdateAllocator* allocator,
|
|
|
|
VAddr address, size_t num_pages, KMemoryState test_state,
|
|
|
|
KMemoryPermission test_perm, KMemoryAttribute test_attr,
|
|
|
|
KMemoryState state, KMemoryPermission perm,
|
|
|
|
KMemoryAttribute attr) {
|
|
|
|
// Ensure for auditing that we never end up with an invalid tree.
|
|
|
|
KScopedMemoryBlockManagerAuditor auditor(this);
|
|
|
|
ASSERT(Common::IsAligned(address, PageSize));
|
|
|
|
ASSERT((attr & (KMemoryAttribute::IpcLocked | KMemoryAttribute::DeviceShared)) ==
|
|
|
|
KMemoryAttribute::None);
|
|
|
|
|
|
|
|
VAddr cur_address = address;
|
|
|
|
size_t remaining_pages = num_pages;
|
|
|
|
iterator it = this->FindIterator(address);
|
|
|
|
|
|
|
|
while (remaining_pages > 0) {
|
|
|
|
const size_t remaining_size = remaining_pages * PageSize;
|
|
|
|
KMemoryInfo cur_info = it->GetMemoryInfo();
|
|
|
|
if (it->HasProperties(test_state, test_perm, test_attr) &&
|
|
|
|
!it->HasProperties(state, perm, attr)) {
|
|
|
|
// If we need to, create a new block before and insert it.
|
|
|
|
if (cur_info.GetAddress() != cur_address) {
|
|
|
|
KMemoryBlock* new_block = allocator->Allocate();
|
|
|
|
|
|
|
|
it->Split(new_block, cur_address);
|
|
|
|
it = m_memory_block_tree.insert(*new_block);
|
|
|
|
it++;
|
|
|
|
|
|
|
|
cur_info = it->GetMemoryInfo();
|
|
|
|
cur_address = cur_info.GetAddress();
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// If we need to, create a new block after and insert it.
|
|
|
|
if (cur_info.GetSize() > remaining_size) {
|
|
|
|
KMemoryBlock* new_block = allocator->Allocate();
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
it->Split(new_block, cur_address + remaining_size);
|
|
|
|
it = m_memory_block_tree.insert(*new_block);
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
cur_info = it->GetMemoryInfo();
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Update block state.
|
|
|
|
it->Update(state, perm, attr, false, 0, 0);
|
|
|
|
cur_address += cur_info.GetSize();
|
|
|
|
remaining_pages -= cur_info.GetNumPages();
|
|
|
|
} else {
|
|
|
|
// If we already have the right properties, just advance.
|
|
|
|
if (cur_address + remaining_size < cur_info.GetEndAddress()) {
|
|
|
|
remaining_pages = 0;
|
|
|
|
cur_address += remaining_size;
|
|
|
|
} else {
|
|
|
|
remaining_pages =
|
|
|
|
(cur_address + remaining_size - cur_info.GetEndAddress()) / PageSize;
|
|
|
|
cur_address = cur_info.GetEndAddress();
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
}
|
2022-09-10 06:12:37 +02:00
|
|
|
it++;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
2022-09-10 06:12:37 +02:00
|
|
|
|
|
|
|
this->CoalesceForUpdate(allocator, address, num_pages);
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator* allocator, VAddr address,
|
|
|
|
size_t num_pages, MemoryBlockLockFunction lock_func,
|
2021-02-13 02:02:51 +01:00
|
|
|
KMemoryPermission perm) {
|
2022-09-10 06:12:37 +02:00
|
|
|
// Ensure for auditing that we never end up with an invalid tree.
|
|
|
|
KScopedMemoryBlockManagerAuditor auditor(this);
|
|
|
|
ASSERT(Common::IsAligned(address, PageSize));
|
2020-04-23 17:37:12 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
VAddr cur_address = address;
|
|
|
|
size_t remaining_pages = num_pages;
|
|
|
|
iterator it = this->FindIterator(address);
|
2020-04-23 17:37:12 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
const VAddr end_address = address + (num_pages * PageSize);
|
2020-04-23 17:37:12 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
while (remaining_pages > 0) {
|
|
|
|
const size_t remaining_size = remaining_pages * PageSize;
|
|
|
|
KMemoryInfo cur_info = it->GetMemoryInfo();
|
2020-04-23 17:37:12 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// If we need to, create a new block before and insert it.
|
|
|
|
if (cur_info.m_address != cur_address) {
|
|
|
|
KMemoryBlock* new_block = allocator->Allocate();
|
2020-04-23 17:37:12 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
it->Split(new_block, cur_address);
|
|
|
|
it = m_memory_block_tree.insert(*new_block);
|
|
|
|
it++;
|
2020-04-23 17:37:12 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
cur_info = it->GetMemoryInfo();
|
|
|
|
cur_address = cur_info.GetAddress();
|
2020-04-23 17:37:12 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
if (cur_info.GetSize() > remaining_size) {
|
|
|
|
// If we need to, create a new block after and insert it.
|
|
|
|
KMemoryBlock* new_block = allocator->Allocate();
|
|
|
|
|
|
|
|
it->Split(new_block, cur_address + remaining_size);
|
|
|
|
it = m_memory_block_tree.insert(*new_block);
|
|
|
|
|
|
|
|
cur_info = it->GetMemoryInfo();
|
2020-04-23 17:37:12 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Call the locked update function.
|
|
|
|
(std::addressof(*it)->*lock_func)(perm, cur_info.GetAddress() == address,
|
|
|
|
cur_info.GetEndAddress() == end_address);
|
|
|
|
cur_address += cur_info.GetSize();
|
|
|
|
remaining_pages -= cur_info.GetNumPages();
|
|
|
|
it++;
|
2020-04-23 17:37:12 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
this->CoalesceForUpdate(allocator, address, num_pages);
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Debug.
|
|
|
|
bool KMemoryBlockManager::CheckState() const {
|
|
|
|
// Loop over every block, ensuring that we are sorted and coalesced.
|
|
|
|
auto it = m_memory_block_tree.cbegin();
|
|
|
|
auto prev = it++;
|
|
|
|
while (it != m_memory_block_tree.cend()) {
|
|
|
|
const KMemoryInfo prev_info = prev->GetMemoryInfo();
|
|
|
|
const KMemoryInfo cur_info = it->GetMemoryInfo();
|
|
|
|
|
|
|
|
// Sequential blocks which can be merged should be merged.
|
|
|
|
if (prev->CanMergeWith(*it)) {
|
|
|
|
return false;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Sequential blocks should be sequential.
|
|
|
|
if (prev_info.GetEndAddress() != cur_info.GetAddress()) {
|
|
|
|
return false;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// If the block is ipc locked, it must have a count.
|
|
|
|
if ((cur_info.m_attribute & KMemoryAttribute::IpcLocked) != KMemoryAttribute::None &&
|
|
|
|
cur_info.m_ipc_lock_count == 0) {
|
|
|
|
return false;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// If the block is device shared, it must have a count.
|
|
|
|
if ((cur_info.m_attribute & KMemoryAttribute::DeviceShared) != KMemoryAttribute::None &&
|
|
|
|
cur_info.m_device_use_count == 0) {
|
|
|
|
return false;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
2022-09-10 06:12:37 +02:00
|
|
|
|
|
|
|
// Advance the iterator.
|
|
|
|
prev = it++;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// Our loop will miss checking the last block, potentially, so check it.
|
|
|
|
if (prev != m_memory_block_tree.cend()) {
|
|
|
|
const KMemoryInfo prev_info = prev->GetMemoryInfo();
|
|
|
|
// If the block is ipc locked, it must have a count.
|
|
|
|
if ((prev_info.m_attribute & KMemoryAttribute::IpcLocked) != KMemoryAttribute::None &&
|
|
|
|
prev_info.m_ipc_lock_count == 0) {
|
|
|
|
return false;
|
|
|
|
}
|
2020-04-05 21:26:25 +02:00
|
|
|
|
2022-09-10 06:12:37 +02:00
|
|
|
// If the block is device shared, it must have a count.
|
|
|
|
if ((prev_info.m_attribute & KMemoryAttribute::DeviceShared) != KMemoryAttribute::None &&
|
|
|
|
prev_info.m_device_use_count == 0) {
|
|
|
|
return false;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
}
|
2022-09-10 06:12:37 +02:00
|
|
|
|
|
|
|
return true;
|
2020-04-05 21:26:25 +02:00
|
|
|
}
|
|
|
|
|
2021-02-13 02:02:51 +01:00
|
|
|
} // namespace Kernel
|