diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index d9d27e02e..cd979eb25 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -38,6 +38,8 @@ /* Core functionality. */ #include "mesosphere/kern_select_interrupts.hpp" +#include "mesosphere/kern_k_page_heap.hpp" +#include "mesosphere/kern_k_memory_manager.hpp" /* Supervisor Calls. */ #include "mesosphere/kern_svc.hpp" diff --git a/libraries/libmesosphere/include/mesosphere/board/nintendo/switch/kern_k_system_control.hpp b/libraries/libmesosphere/include/mesosphere/board/nintendo/switch/kern_k_system_control.hpp index 631cc471a..f568ac270 100644 --- a/libraries/libmesosphere/include/mesosphere/board/nintendo/switch/kern_k_system_control.hpp +++ b/libraries/libmesosphere/include/mesosphere/board/nintendo/switch/kern_k_system_control.hpp @@ -27,6 +27,9 @@ namespace ams::kern { static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address); static bool ShouldIncreaseThreadResourceLimit(); static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg); + static size_t GetApplicationPoolSize(); + static size_t GetAppletPoolSize(); + static size_t GetMinimumNonSecureSystemPoolSize(); /* Randomness. */ static void GenerateRandomBytes(void *dst, size_t size); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index 4aa49dee4..d1bd1d8f3 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -50,6 +50,12 @@ namespace ams::kern { KMemoryRegionType_VirtualKernelTraceBuffer = 0x4A, KMemoryRegionType_VirtualKernelInitPt = 0x19A, + KMemoryRegionType_VirtualDramMetadataPool = 0x29A, + KMemoryRegionType_VirtualDramApplicationPool = 0x271A, + KMemoryRegionType_VirtualDramAppletPool = 0x1B1A, + KMemoryRegionType_VirtualDramSystemNonSecurePool = 0x331A, + KMemoryRegionType_VirtualDramSystemPool = 0x2B1A, + KMemoryRegionType_Uart = 0x1D, KMemoryRegionType_InterruptDistributor = 0x4D, KMemoryRegionType_InterruptController = 0x2D, @@ -76,8 +82,13 @@ namespace ams::kern { KMemoryRegionType_DramLinearMapped = KMemoryRegionType_Dram | KMemoryRegionAttr_LinearMapped, - KMemoryRegionType_DramReservedEarly = 0x16 | KMemoryRegionAttr_NoUserMap, - KMemoryRegionType_DramPoolPartition = 0x26 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped, + KMemoryRegionType_DramReservedEarly = 0x16 | KMemoryRegionAttr_NoUserMap, + KMemoryRegionType_DramPoolPartition = 0x26 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped, + KMemoryRegionType_DramMetadataPool = 0x166 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected, + KMemoryRegionType_DramApplicationPool = 0x7A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped, + KMemoryRegionType_DramAppletPool = 0xBA6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped, + KMemoryRegionType_DramSystemNonSecurePool = 0xDA6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped, + KMemoryRegionType_DramSystemPool = 0x13A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected, KMemoryRegionType_DramKernel = 0xE | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected, KMemoryRegionType_DramKernelCode = 0xCE | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected, @@ -228,6 +239,16 @@ namespace ams::kern { MESOSPHERE_INIT_ABORT(); } + iterator FindFirstDerivedBlock(u32 type_id) { + for (auto it = this->begin(); it != this->end(); it++) { + if (it->IsDerivedFrom(type_id)) { + return it; + } + } + MESOSPHERE_INIT_ABORT(); + } + + DerivedRegionExtents GetDerivedRegionExtents(u32 type_id) { DerivedRegionExtents extents = { .first_block = nullptr, .last_block = nullptr }; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp new file mode 100644 index 000000000..f958d4e34 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::kern { + + class KMemoryManager { + public: + static constexpr size_t PageSize = 0x1000; /* TODO: Elsewhere? */ + private: + class Impl { + public: + static size_t CalculateMetadataOverheadSize(size_t region_size); + }; + public: + static size_t CalculateMetadataOverheadSize(size_t region_size) { + return Impl::CalculateMetadataOverheadSize(region_size); + } + }; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp new file mode 100644 index 000000000..2a74efdc3 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::kern { + + class KPageHeap { + private: + class Block { + private: + class Bitmap { + /* TODO: This is a four-level bitmap tracking page usage. */ + private: + static constexpr s32 GetRequiredDepth(size_t region_size) { + s32 depth = 0; + while (true) { + region_size /= BITSIZEOF(u64); + depth++; + if (region_size == 0) { + return depth; + } + } + } + public: + static constexpr size_t CalculateMetadataOverheadSize(size_t region_size) { + size_t overhead_bits = 0; + for (s32 depth = GetRequiredDepth(region_size) - 1; depth >= 0; depth--) { + region_size = util::AlignUp(region_size, BITSIZEOF(u64)) / BITSIZEOF(u64); + overhead_bits += region_size; + } + return overhead_bits * sizeof(u64); + } + }; + public: + static constexpr size_t CalculateMetadataOverheadSize(size_t region_size, size_t cur_block_shift, size_t next_block_shift) { + const size_t cur_block_size = (1ul << cur_block_shift); + const size_t next_block_size = (1ul << next_block_shift); + const size_t align = (next_block_shift != 0) ? next_block_size : cur_block_size; + return Bitmap::CalculateMetadataOverheadSize((align * 2 + util::AlignUp(region_size, align)) / cur_block_size); + } + }; + public: + static size_t CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts); + }; + +} diff --git a/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp b/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp index 881f834eb..6a73bad56 100644 --- a/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp +++ b/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp @@ -34,6 +34,30 @@ namespace ams::kern { return util::BitPack32{static_cast(value)}; } + ALWAYS_INLINE u32 GetMemoryModeForInit() { + u64 value = 0; + smc::init::GetConfig(&value, 1, smc::ConfigItem::MemoryMode); + return static_cast(value); + } + + ALWAYS_INLINE smc::MemoryArrangement GetMemoryArrangeForInit() { + switch(GetMemoryModeForInit() & 0x3F) { + case 0x01: + default: + return smc::MemoryArrangement_4GB; + case 0x02: + return smc::MemoryArrangement_4GBForAppletDev; + case 0x03: + return smc::MemoryArrangement_4GBForSystemDev; + case 0x11: + return smc::MemoryArrangement_6GB; + case 0x12: + return smc::MemoryArrangement_6GBForAppletDev; + case 0x21: + return smc::MemoryArrangement_8GB; + } + } + ALWAYS_INLINE u64 GenerateRandomU64ForInit() { u64 value; smc::init::GenerateRandomBytes(&value, sizeof(value)); @@ -69,6 +93,47 @@ namespace ams::kern { return GetKernelConfigurationForInit().Get(); } + size_t KSystemControl::Init::GetApplicationPoolSize() { + switch (GetMemoryArrangeForInit()) { + case smc::MemoryArrangement_4GB: + default: + return 3285_MB; + case smc::MemoryArrangement_4GBForAppletDev: + return 2048_MB; + case smc::MemoryArrangement_4GBForSystemDev: + return 3285_MB; + case smc::MemoryArrangement_6GB: + return 4916_MB; + case smc::MemoryArrangement_6GBForAppletDev: + return 3285_MB; + case smc::MemoryArrangement_8GB: + return 4916_MB; + } + } + + size_t KSystemControl::Init::GetAppletPoolSize() { + switch (GetMemoryArrangeForInit()) { + case smc::MemoryArrangement_4GB: + default: + return 507_MB; + case smc::MemoryArrangement_4GBForAppletDev: + return 1554_MB; + case smc::MemoryArrangement_4GBForSystemDev: + return 448_MB; + case smc::MemoryArrangement_6GB: + return 562_MB; + case smc::MemoryArrangement_6GBForAppletDev: + return 2193_MB; + case smc::MemoryArrangement_8GB: + return 2193_MB; + } + } + + size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() { + /* TODO: Where does this constant actually come from? */ + return 0x29C8000; + } + void KSystemControl::Init::CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) { smc::init::CpuOn(core_id, entrypoint, arg); } diff --git a/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.hpp b/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.hpp index 1417c29e8..4d0a43315 100644 --- a/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.hpp +++ b/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.hpp @@ -25,6 +25,15 @@ namespace ams::kern::smc { MemorySize_8GB = 2, }; + enum MemoryArrangement { + MemoryArrangement_4GB = 0, + MemoryArrangement_4GBForAppletDev = 1, + MemoryArrangement_4GBForSystemDev = 2, + MemoryArrangement_6GB = 3, + MemoryArrangement_6GBForAppletDev = 4, + MemoryArrangement_8GB = 5, + }; + enum class ConfigItem : u32 { /* Standard config items. */ DisableProgramVerification = 1, diff --git a/libraries/libmesosphere/source/kern_k_memory_layout.cpp b/libraries/libmesosphere/source/kern_k_memory_layout.cpp index 8cb084bda..3f185e6fc 100644 --- a/libraries/libmesosphere/source/kern_k_memory_layout.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_layout.cpp @@ -138,6 +138,9 @@ namespace ams::kern { constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable); + constexpr size_t CarveoutAlignment = 0x20000; + constexpr size_t CarveoutSizeMax = 512_MB - CarveoutAlignment; + constexpr size_t CoreLocalRegionAlign = PageSize; constexpr size_t CoreLocalRegionSize = PageSize * (1 + cpu::NumCores); constexpr size_t CoreLocalRegionSizeWithGuards = CoreLocalRegionSize + 2 * PageSize; @@ -177,6 +180,12 @@ namespace ams::kern { } + void InsertPoolPartitionBlockIntoBothTrees(size_t start, size_t size, KMemoryRegionType phys_type, KMemoryRegionType virt_type, u32 &cur_attr) { + const u32 attr = cur_attr++; + MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryBlockTree().Insert(start, size, phys_type, attr)); + MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryBlockTree().Insert(KMemoryLayout::GetPhysicalMemoryBlockTree().FindFirstBlockByTypeAttr(phys_type, attr)->GetPairAddress(), size, virt_type, attr)); + } + } void SetupCoreLocalRegionMemoryBlocks(KInitialPageTable &page_table, KInitialPageAllocator &page_allocator) { @@ -214,7 +223,62 @@ namespace ams::kern { } void SetupPoolPartitionMemoryBlocks() { - /* TODO */ + /* Start by identifying the extents of the DRAM memory region. */ + const auto dram_extents = KMemoryLayout::GetPhysicalMemoryBlockTree().GetDerivedRegionExtents(KMemoryRegionType_Dram); + + /* Get Application and Applet pool sizes. */ + const size_t application_pool_size = KSystemControl::Init::GetApplicationPoolSize(); + const size_t applet_pool_size = KSystemControl::Init::GetAppletPoolSize(); + const size_t unsafe_system_pool_min_size = KSystemControl::Init::GetMinimumNonSecureSystemPoolSize(); + + /* Find the start of the kernel DRAM region. */ + const uintptr_t kernel_dram_start = KMemoryLayout::GetPhysicalMemoryBlockTree().FindFirstDerivedBlock(KMemoryRegionType_DramKernel)->GetAddress(); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(kernel_dram_start, CarveoutAlignment)); + + /* Find the start of the pool partitions region. */ + const uintptr_t pool_partitions_start = KMemoryLayout::GetPhysicalMemoryBlockTree().FindFirstBlockByTypeAttr(KMemoryRegionType_DramPoolPartition)->GetAddress(); + + /* Decide on starting addresses for our pools. */ + const uintptr_t application_pool_start = dram_extents.last_block->GetEndAddress() - application_pool_size; + const uintptr_t applet_pool_start = application_pool_start - applet_pool_size; + const uintptr_t unsafe_system_pool_start = std::min(kernel_dram_start + CarveoutSizeMax, util::AlignDown(applet_pool_start - unsafe_system_pool_min_size, CarveoutAlignment)); + const size_t unsafe_system_pool_size = applet_pool_start - unsafe_system_pool_start; + + /* We want to arrange application pool depending on where the middle of dram is. */ + const uintptr_t dram_midpoint = (dram_extents.first_block->GetAddress() + dram_extents.last_block->GetEndAddress()) / 2; + u32 cur_pool_attr = 0; + size_t total_overhead_size = 0; + if (dram_extents.last_block->GetEndAddress() <= dram_midpoint || dram_midpoint <= application_pool_start) { + InsertPoolPartitionBlockIntoBothTrees(application_pool_start, application_pool_size, KMemoryRegionType_DramApplicationPool, KMemoryRegionType_VirtualDramApplicationPool, cur_pool_attr); + total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(application_pool_size); + } else { + const size_t first_application_pool_size = dram_midpoint - application_pool_start; + const size_t second_application_pool_size = application_pool_start + application_pool_size - dram_midpoint; + InsertPoolPartitionBlockIntoBothTrees(application_pool_start, first_application_pool_size, KMemoryRegionType_DramApplicationPool, KMemoryRegionType_VirtualDramApplicationPool, cur_pool_attr); + InsertPoolPartitionBlockIntoBothTrees(dram_midpoint, second_application_pool_size, KMemoryRegionType_DramApplicationPool, KMemoryRegionType_VirtualDramApplicationPool, cur_pool_attr); + total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(first_application_pool_size); + total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(second_application_pool_size); + } + + /* Insert the applet pool. */ + InsertPoolPartitionBlockIntoBothTrees(applet_pool_start, applet_pool_size, KMemoryRegionType_DramAppletPool, KMemoryRegionType_VirtualDramAppletPool, cur_pool_attr); + total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(applet_pool_size); + + /* Insert the nonsecure system pool. */ + InsertPoolPartitionBlockIntoBothTrees(unsafe_system_pool_start, unsafe_system_pool_size, KMemoryRegionType_DramSystemNonSecurePool, KMemoryRegionType_VirtualDramSystemNonSecurePool, cur_pool_attr); + total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(unsafe_system_pool_size); + + /* Insert the metadata pool. */ + total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize((unsafe_system_pool_start - pool_partitions_start) - total_overhead_size); + const uintptr_t metadata_pool_start = unsafe_system_pool_start - total_overhead_size; + const size_t metadata_pool_size = total_overhead_size; + u32 metadata_pool_attr = 0; + InsertPoolPartitionBlockIntoBothTrees(metadata_pool_start, metadata_pool_size, KMemoryRegionType_DramMetadataPool, KMemoryRegionType_VirtualDramMetadataPool, metadata_pool_attr); + + /* Insert the system pool. */ + const uintptr_t system_pool_size = metadata_pool_start - pool_partitions_start; + InsertPoolPartitionBlockIntoBothTrees(pool_partitions_start, system_pool_size, KMemoryRegionType_DramSystemPool, KMemoryRegionType_VirtualDramSystemPool, cur_pool_attr); + } } diff --git a/libraries/libmesosphere/source/kern_k_memory_manager.cpp b/libraries/libmesosphere/source/kern_k_memory_manager.cpp new file mode 100644 index 000000000..57483fa69 --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_memory_manager.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + namespace { + + constexpr size_t g_memory_block_page_shifts[] = { 0xC, 0x10, 0x15, 0x16, 0x19, 0x1D, 0x1E }; + constexpr size_t NumMemoryBlockPageShifts = util::size(g_memory_block_page_shifts); + + } + + size_t KMemoryManager::Impl::CalculateMetadataOverheadSize(size_t region_size) { + const size_t ref_count_size = (region_size / PageSize) * sizeof(u16); + const size_t bitmap_size = (util::AlignUp((region_size / PageSize), BITSIZEOF(u64)) / BITSIZEOF(u64)) * sizeof(u64); + const size_t page_heap_size = KPageHeap::CalculateMetadataOverheadSize(region_size, g_memory_block_page_shifts, NumMemoryBlockPageShifts); + return util::AlignUp(page_heap_size + bitmap_size + ref_count_size, PageSize); + } + +} diff --git a/libraries/libmesosphere/source/kern_k_page_heap.cpp b/libraries/libmesosphere/source/kern_k_page_heap.cpp new file mode 100644 index 000000000..6dce64507 --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_page_heap.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + size_t KPageHeap::CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts) { + size_t overhead_size = 0; + for (size_t i = 0; i < num_block_shifts; i++) { + overhead_size += KPageHeap::Block::CalculateMetadataOverheadSize(region_size, block_shifts[i], (i != num_block_shifts - 1) ? block_shifts[i + 1] : 0); + } + return util::AlignUp(overhead_size, KMemoryManager::PageSize); + } + +} diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp index 33b020a87..bdae68d1c 100644 --- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -245,6 +245,7 @@ namespace ams::kern::init { ttbr1_table.Map(block_virt_addr, block.GetSize(), block.GetAddress(), KernelRwDataAttribute, g_initial_page_allocator); MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryBlockTree().Insert(block_virt_addr, block.GetSize(), GetTypeForVirtualLinearMapping(block.GetType()))); + block.SetPairAddress(block_virt_addr); KMemoryLayout::GetVirtualMemoryBlockTree().FindContainingBlock(block_virt_addr)->SetPairAddress(block.GetAddress()); }