/* * Copyright (c) Atmosphère-NX * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include namespace ams::kern { namespace { constexpr const std::pair MemoryStateNames[] = { {KMemoryState_Free , "----- Free -----"}, {KMemoryState_IoMemory , "IoMemory "}, {KMemoryState_IoRegister , "IoRegister "}, {KMemoryState_Static , "Static "}, {KMemoryState_Code , "Code "}, {KMemoryState_CodeData , "CodeData "}, {KMemoryState_Normal , "Normal "}, {KMemoryState_Shared , "Shared "}, {KMemoryState_AliasCode , "AliasCode "}, {KMemoryState_AliasCodeData , "AliasCodeData "}, {KMemoryState_Ipc , "Ipc "}, {KMemoryState_Stack , "Stack "}, {KMemoryState_ThreadLocal , "ThreadLocal "}, {KMemoryState_Transfered , "Transfered "}, {KMemoryState_SharedTransfered , "SharedTransfered"}, {KMemoryState_SharedCode , "SharedCode "}, {KMemoryState_Inaccessible , "Inaccessible "}, {KMemoryState_NonSecureIpc , "NonSecureIpc "}, {KMemoryState_NonDeviceIpc , "NonDeviceIpc "}, {KMemoryState_Kernel , "Kernel "}, {KMemoryState_GeneratedCode , "GeneratedCode "}, {KMemoryState_CodeOut , "CodeOut "}, {KMemoryState_Coverage , "Coverage "}, }; constexpr const char *GetMemoryStateName(KMemoryState state) { for (size_t i = 0; i < util::size(MemoryStateNames); i++) { if (std::get<0>(MemoryStateNames[i]) == state) { return std::get<1>(MemoryStateNames[i]); } } return "Unknown "; } constexpr const char *GetMemoryPermissionString(const KMemoryBlock &block) { if (block.GetState() == KMemoryState_Free) { return " "; } else { switch (block.GetPermission()) { case KMemoryPermission_UserReadExecute: return "r-x"; case KMemoryPermission_UserRead: return "r--"; case KMemoryPermission_UserReadWrite: return "rw-"; default: return "---"; } } } void DumpMemoryBlock(const KMemoryBlock &block) { const char *state = GetMemoryStateName(block.GetState()); const char *perm = GetMemoryPermissionString(block); const uintptr_t start = GetInteger(block.GetAddress()); const uintptr_t end = GetInteger(block.GetLastAddress()); const size_t kb = block.GetSize() / 1_KB; const char l = (block.GetAttribute() & KMemoryAttribute_Locked) ? 'L' : '-'; const char i = (block.GetAttribute() & KMemoryAttribute_IpcLocked) ? 'I' : '-'; const char d = (block.GetAttribute() & KMemoryAttribute_DeviceShared) ? 'D' : '-'; const char u = (block.GetAttribute() & KMemoryAttribute_Uncached) ? 'U' : '-'; MESOSPHERE_LOG("0x%10lx - 0x%10lx (%9zu KB) %s %s %c%c%c%c [%d, %d]\n", start, end, kb, perm, state, l, i, d, u, block.GetIpcLockCount(), block.GetDeviceUseCount()); } } Result KMemoryBlockManager::Initialize(KProcessAddress st, KProcessAddress nd, KMemoryBlockSlabManager *slab_manager) { /* Allocate a block to encapsulate the address space, insert it into the tree. */ KMemoryBlock *start_block = slab_manager->Allocate(); R_UNLESS(start_block != nullptr, svc::ResultOutOfResource()); /* Set our start and end. */ m_start_address = st; m_end_address = nd; MESOSPHERE_ASSERT(util::IsAligned(GetInteger(m_start_address), PageSize)); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(m_end_address), PageSize)); /* Initialize and insert the block. */ start_block->Initialize(m_start_address, (m_end_address - m_start_address) / PageSize, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None); m_memory_block_tree.insert(*start_block); R_SUCCEED(); } void KMemoryBlockManager::Finalize(KMemoryBlockSlabManager *slab_manager) { /* Erase every block until we have none left. */ auto it = m_memory_block_tree.begin(); while (it != m_memory_block_tree.end()) { KMemoryBlock *block = std::addressof(*it); it = m_memory_block_tree.erase(it); slab_manager->Free(block); } MESOSPHERE_ASSERT(m_memory_block_tree.empty()); } KProcessAddress KMemoryBlockManager::FindFreeArea(KProcessAddress region_start, size_t region_num_pages, size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const { if (num_pages > 0) { const KProcessAddress region_end = region_start + region_num_pages * PageSize; const KProcessAddress region_last = region_end - 1; for (const_iterator it = this->FindIterator(region_start); it != m_memory_block_tree.cend(); it++) { if (region_last < it->GetAddress()) { break; } if (it->GetState() != KMemoryState_Free) { continue; } KProcessAddress area = (it->GetAddress() <= GetInteger(region_start)) ? region_start : it->GetAddress(); area += guard_pages * PageSize; const KProcessAddress offset_area = util::AlignDown(GetInteger(area), alignment) + offset; area = (area <= offset_area) ? offset_area : offset_area + alignment; const KProcessAddress area_end = area + num_pages * PageSize + guard_pages * PageSize; const KProcessAddress area_last = area_end - 1; if (GetInteger(it->GetAddress()) <= GetInteger(area) && area < area_last && area_last <= region_last && GetInteger(area_last) <= GetInteger(it->GetLastAddress())) { return area; } } } return Null; } void KMemoryBlockManager::CoalesceForUpdate(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages) { /* Find the iterator now that we've updated. */ iterator it = this->FindIterator(address); if (address != m_start_address) { it--; } /* Coalesce blocks that we can. */ while (true) { iterator prev = it++; if (it == m_memory_block_tree.end()) { break; } if (prev->CanMergeWith(*it)) { KMemoryBlock *block = std::addressof(*it); m_memory_block_tree.erase(it); prev->Add(*block); allocator->Free(block); it = prev; } if (address + num_pages * PageSize < it->GetEndAddress()) { break; } } } void KMemoryBlockManager::Update(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize)); MESOSPHERE_ASSERT((attr & (KMemoryAttribute_IpcLocked | KMemoryAttribute_DeviceShared)) == 0); KProcessAddress cur_address = address; size_t remaining_pages = num_pages; iterator it = this->FindIterator(address); while (remaining_pages > 0) { const size_t remaining_size = remaining_pages * PageSize; if (it->HasProperties(state, perm, attr)) { /* If we already have the right properties, just advance. */ if (cur_address + remaining_size < it->GetEndAddress()) { remaining_pages = 0; cur_address += remaining_size; } else { remaining_pages = (cur_address + remaining_size - it->GetEndAddress()) / PageSize; cur_address = it->GetEndAddress(); } } else { /* If we need to, create a new block before and insert it. */ if (it->GetAddress() != GetInteger(cur_address)) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address); it = m_memory_block_tree.insert(*new_block); it++; cur_address = it->GetAddress(); } /* If we need to, create a new block after and insert it. */ if (it->GetSize() > remaining_size) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address + remaining_size); it = m_memory_block_tree.insert(*new_block); } /* Update block state. */ it->Update(state, perm, attr, it->GetAddress() == address, set_disable_attr, clear_disable_attr); cur_address += it->GetSize(); remaining_pages -= it->GetNumPages(); } it++; } this->CoalesceForUpdate(allocator, address, num_pages); } void KMemoryBlockManager::UpdateIfMatch(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, KMemoryState test_state, KMemoryPermission test_perm, KMemoryAttribute test_attr, KMemoryState state, KMemoryPermission perm, KMemoryAttribute attr, KMemoryBlockDisableMergeAttribute set_disable_attr, KMemoryBlockDisableMergeAttribute clear_disable_attr) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize)); MESOSPHERE_ASSERT((attr & (KMemoryAttribute_IpcLocked | KMemoryAttribute_DeviceShared)) == 0); KProcessAddress cur_address = address; size_t remaining_pages = num_pages; iterator it = this->FindIterator(address); while (remaining_pages > 0) { const size_t remaining_size = remaining_pages * PageSize; if (it->HasProperties(test_state, test_perm, test_attr) && !it->HasProperties(state, perm, attr)) { /* If we need to, create a new block before and insert it. */ if (it->GetAddress() != GetInteger(cur_address)) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address); it = m_memory_block_tree.insert(*new_block); it++; cur_address = it->GetAddress(); } /* If we need to, create a new block after and insert it. */ if (it->GetSize() > remaining_size) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address + remaining_size); it = m_memory_block_tree.insert(*new_block); } /* Update block state. */ it->Update(state, perm, attr, it->GetAddress() == address, set_disable_attr, clear_disable_attr); cur_address += it->GetSize(); remaining_pages -= it->GetNumPages(); } else { /* If we already have the right properties, just advance. */ if (cur_address + remaining_size < it->GetEndAddress()) { remaining_pages = 0; cur_address += remaining_size; } else { remaining_pages = (cur_address + remaining_size - it->GetEndAddress()) / PageSize; cur_address = it->GetEndAddress(); } } it++; } this->CoalesceForUpdate(allocator, address, num_pages); } void KMemoryBlockManager::UpdateLock(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, MemoryBlockLockFunction lock_func, KMemoryPermission perm) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize)); KProcessAddress cur_address = address; size_t remaining_pages = num_pages; iterator it = this->FindIterator(address); const KProcessAddress end_address = address + (num_pages * PageSize); while (remaining_pages > 0) { const size_t remaining_size = remaining_pages * PageSize; /* If we need to, create a new block before and insert it. */ if (it->GetAddress() != cur_address) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address); it = m_memory_block_tree.insert(*new_block); it++; cur_address = it->GetAddress(); } if (it->GetSize() > remaining_size) { /* If we need to, create a new block after and insert it. */ KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address + remaining_size); it = m_memory_block_tree.insert(*new_block); } /* Call the locked update function. */ (std::addressof(*it)->*lock_func)(perm, it->GetAddress() == address, it->GetEndAddress() == end_address); cur_address += it->GetSize(); remaining_pages -= it->GetNumPages(); it++; } this->CoalesceForUpdate(allocator, address, num_pages); } void KMemoryBlockManager::UpdateAttribute(KMemoryBlockManagerUpdateAllocator *allocator, KProcessAddress address, size_t num_pages, u32 mask, u32 attr) { /* Ensure for auditing that we never end up with an invalid tree. */ KScopedMemoryBlockManagerAuditor auditor(this); MESOSPHERE_ASSERT(util::IsAligned(GetInteger(address), PageSize)); KProcessAddress cur_address = address; size_t remaining_pages = num_pages; iterator it = this->FindIterator(address); while (remaining_pages > 0) { const size_t remaining_size = remaining_pages * PageSize; if ((it->GetAttribute() & mask) != attr) { /* If we need to, create a new block before and insert it. */ if (it->GetAddress() != GetInteger(cur_address)) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address); it = m_memory_block_tree.insert(*new_block); it++; cur_address = it->GetAddress(); } /* If we need to, create a new block after and insert it. */ if (it->GetSize() > remaining_size) { KMemoryBlock *new_block = allocator->Allocate(); it->Split(new_block, cur_address + remaining_size); it = m_memory_block_tree.insert(*new_block); } /* Update block state. */ it->UpdateAttribute(mask, attr); cur_address += it->GetSize(); remaining_pages -= it->GetNumPages(); } else { /* If we already have the right attributes, just advance. */ if (cur_address + remaining_size < it->GetEndAddress()) { remaining_pages = 0; cur_address += remaining_size; } else { remaining_pages = (cur_address + remaining_size - it->GetEndAddress()) / PageSize; cur_address = it->GetEndAddress(); } } it++; } this->CoalesceForUpdate(allocator, address, num_pages); } /* Debug. */ bool KMemoryBlockManager::CheckState() const { /* If we fail, we should dump blocks. */ auto dump_guard = SCOPE_GUARD { this->DumpBlocks(); }; /* Loop over every block, ensuring that we are sorted and coalesced. */ auto it = m_memory_block_tree.cbegin(); auto prev = it++; while (it != m_memory_block_tree.cend()) { /* Sequential blocks which can be merged should be merged. */ if (prev->CanMergeWith(*it)) { return false; } /* Sequential blocks should be sequential. */ if (prev->GetEndAddress() != it->GetAddress()) { return false; } /* If the block is ipc locked, it must have a count. */ if ((it->GetAttribute() & KMemoryAttribute_IpcLocked) != 0 && it->GetIpcLockCount() == 0) { return false; } /* If the block is device shared, it must have a count. */ if ((it->GetAttribute() & KMemoryAttribute_DeviceShared) != 0 && it->GetDeviceUseCount() == 0) { return false; } /* Advance the iterator. */ prev = it++; } /* Our loop will miss checking the last block, potentially, so check it. */ if (prev != m_memory_block_tree.cend()) { /* If the block is ipc locked, it must have a count. */ if ((prev->GetAttribute() & KMemoryAttribute_IpcLocked) != 0 && prev->GetIpcLockCount() == 0) { return false; } /* If the block is device shared, it must have a count. */ if ((prev->GetAttribute() & KMemoryAttribute_DeviceShared) != 0 && prev->GetDeviceUseCount() == 0) { return false; } } /* We're valid, so no need to print. */ dump_guard.Cancel(); return true; } void KMemoryBlockManager::DumpBlocks() const { /* Dump each block. */ for (const auto &block : m_memory_block_tree) { DumpMemoryBlock(block); } } }