kern: implement pool partition memblock setup

libraries/libmesosphere/include/mesosphere.hpp

@@ -38,6 +38,8 @@
 
 /* Core functionality. */
 #include "mesosphere/kern_select_interrupts.hpp"
+#include "mesosphere/kern_k_page_heap.hpp"
+#include "mesosphere/kern_k_memory_manager.hpp"
 
 /* Supervisor Calls. */
 #include "mesosphere/kern_svc.hpp"

libraries/libmesosphere/include/mesosphere/board/nintendo/switch/kern_k_system_control.hpp

@@ -27,6 +27,9 @@ namespace ams::kern {
                     static KPhysicalAddress GetKernelPhysicalBaseAddress(uintptr_t base_address);
                     static bool ShouldIncreaseThreadResourceLimit();
                     static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg);
+                    static size_t GetApplicationPoolSize();
+                    static size_t GetAppletPoolSize();
+                    static size_t GetMinimumNonSecureSystemPoolSize();
 
                     /* Randomness. */
                     static void GenerateRandomBytes(void *dst, size_t size);

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -50,6 +50,12 @@ namespace ams::kern {
         KMemoryRegionType_VirtualKernelTraceBuffer  = 0x4A,
         KMemoryRegionType_VirtualKernelInitPt       = 0x19A,
 
+        KMemoryRegionType_VirtualDramMetadataPool        = 0x29A,
+        KMemoryRegionType_VirtualDramApplicationPool     = 0x271A,
+        KMemoryRegionType_VirtualDramAppletPool          = 0x1B1A,
+        KMemoryRegionType_VirtualDramSystemNonSecurePool = 0x331A,
+        KMemoryRegionType_VirtualDramSystemPool          = 0x2B1A,
+
         KMemoryRegionType_Uart                      = 0x1D,
         KMemoryRegionType_InterruptDistributor      = 0x4D,
         KMemoryRegionType_InterruptController       = 0x2D,
@@ -76,8 +82,13 @@ namespace ams::kern {
 
         KMemoryRegionType_DramLinearMapped  = KMemoryRegionType_Dram  | KMemoryRegionAttr_LinearMapped,
 
-        KMemoryRegionType_DramReservedEarly = 0x16  | KMemoryRegionAttr_NoUserMap,
+        KMemoryRegionType_DramReservedEarly       = 0x16  | KMemoryRegionAttr_NoUserMap,
-        KMemoryRegionType_DramPoolPartition = 0x26  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
+        KMemoryRegionType_DramPoolPartition       = 0x26  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
+        KMemoryRegionType_DramMetadataPool        = 0x166  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected,
+        KMemoryRegionType_DramApplicationPool     = 0x7A6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
+        KMemoryRegionType_DramAppletPool          = 0xBA6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
+        KMemoryRegionType_DramSystemNonSecurePool = 0xDA6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
+        KMemoryRegionType_DramSystemPool          = 0x13A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected,
 
         KMemoryRegionType_DramKernel        = 0xE   | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
         KMemoryRegionType_DramKernelCode    = 0xCE  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
@@ -228,6 +239,16 @@ namespace ams::kern {
                 MESOSPHERE_INIT_ABORT();
             }
 
+            iterator FindFirstDerivedBlock(u32 type_id) {
+                for (auto it = this->begin(); it != this->end(); it++) {
+                    if (it->IsDerivedFrom(type_id)) {
+                        return it;
+                    }
+                }
+                MESOSPHERE_INIT_ABORT();
+            }
+
+
             DerivedRegionExtents GetDerivedRegionExtents(u32 type_id) {
                 DerivedRegionExtents extents = { .first_block = nullptr, .last_block = nullptr };
 

libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::kern {
+
+    class KMemoryManager {
+        public:
+            static constexpr size_t PageSize = 0x1000; /* TODO: Elsewhere? */
+        private:
+            class Impl {
+                public:
+                    static size_t CalculateMetadataOverheadSize(size_t region_size);
+            };
+        public:
+            static size_t CalculateMetadataOverheadSize(size_t region_size) {
+                return Impl::CalculateMetadataOverheadSize(region_size);
+            }
+    };
+
+}

libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <vapours.hpp>
+
+namespace ams::kern {
+
+    class KPageHeap {
+        private:
+            class Block {
+                private:
+                    class Bitmap {
+                        /* TODO: This is a four-level bitmap tracking page usage. */
+                        private:
+                            static constexpr s32 GetRequiredDepth(size_t region_size) {
+                                s32 depth = 0;
+                                while (true) {
+                                    region_size /= BITSIZEOF(u64);
+                                    depth++;
+                                    if (region_size == 0) {
+                                        return depth;
+                                    }
+                                }
+                            }
+                        public:
+                            static constexpr size_t CalculateMetadataOverheadSize(size_t region_size) {
+                                size_t overhead_bits = 0;
+                                for (s32 depth = GetRequiredDepth(region_size) - 1; depth >= 0; depth--) {
+                                    region_size = util::AlignUp(region_size, BITSIZEOF(u64)) / BITSIZEOF(u64);
+                                    overhead_bits += region_size;
+                                }
+                                return overhead_bits * sizeof(u64);
+                            }
+                    };
+                public:
+                    static constexpr size_t CalculateMetadataOverheadSize(size_t region_size, size_t cur_block_shift, size_t next_block_shift) {
+                        const size_t cur_block_size  = (1ul << cur_block_shift);
+                        const size_t next_block_size = (1ul << next_block_shift);
+                        const size_t align = (next_block_shift != 0) ? next_block_size : cur_block_size;
+                        return Bitmap::CalculateMetadataOverheadSize((align * 2 + util::AlignUp(region_size, align)) / cur_block_size);
+                    }
+            };
+        public:
+            static size_t CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts);
+    };
+
+}

libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp

@@ -34,6 +34,30 @@ namespace ams::kern {
             return util::BitPack32{static_cast<u32>(value)};
         }
 
+        ALWAYS_INLINE u32 GetMemoryModeForInit() {
+            u64 value = 0;
+            smc::init::GetConfig(&value, 1, smc::ConfigItem::MemoryMode);
+            return static_cast<u32>(value);
+        }
+
+        ALWAYS_INLINE smc::MemoryArrangement GetMemoryArrangeForInit() {
+            switch(GetMemoryModeForInit() & 0x3F) {
+                case 0x01:
+                default:
+                    return smc::MemoryArrangement_4GB;
+                case 0x02:
+                    return smc::MemoryArrangement_4GBForAppletDev;
+                case 0x03:
+                    return smc::MemoryArrangement_4GBForSystemDev;
+                case 0x11:
+                    return smc::MemoryArrangement_6GB;
+                case 0x12:
+                    return smc::MemoryArrangement_6GBForAppletDev;
+                case 0x21:
+                    return smc::MemoryArrangement_8GB;
+            }
+        }
+
         ALWAYS_INLINE u64 GenerateRandomU64ForInit() {
             u64 value;
             smc::init::GenerateRandomBytes(&value, sizeof(value));
@@ -69,6 +93,47 @@ namespace ams::kern {
         return GetKernelConfigurationForInit().Get<smc::KernelConfiguration::IncreaseThreadResourceLimit>();
     }
 
+    size_t KSystemControl::Init::GetApplicationPoolSize() {
+        switch (GetMemoryArrangeForInit()) {
+            case smc::MemoryArrangement_4GB:
+            default:
+                return 3285_MB;
+            case smc::MemoryArrangement_4GBForAppletDev:
+                return 2048_MB;
+            case smc::MemoryArrangement_4GBForSystemDev:
+                return 3285_MB;
+            case smc::MemoryArrangement_6GB:
+                return 4916_MB;
+            case smc::MemoryArrangement_6GBForAppletDev:
+                return 3285_MB;
+            case smc::MemoryArrangement_8GB:
+                return 4916_MB;
+        }
+    }
+
+    size_t KSystemControl::Init::GetAppletPoolSize() {
+        switch (GetMemoryArrangeForInit()) {
+            case smc::MemoryArrangement_4GB:
+            default:
+                return 507_MB;
+            case smc::MemoryArrangement_4GBForAppletDev:
+                return 1554_MB;
+            case smc::MemoryArrangement_4GBForSystemDev:
+                return 448_MB;
+            case smc::MemoryArrangement_6GB:
+                return 562_MB;
+            case smc::MemoryArrangement_6GBForAppletDev:
+                return 2193_MB;
+            case smc::MemoryArrangement_8GB:
+                return 2193_MB;
+        }
+    }
+
+    size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() {
+        /* TODO: Where does this constant actually come from? */
+        return 0x29C8000;
+    }
+
     void KSystemControl::Init::CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) {
         smc::init::CpuOn(core_id, entrypoint, arg);
     }

libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.hpp

@@ -25,6 +25,15 @@ namespace ams::kern::smc {
         MemorySize_8GB = 2,
     };
 
+    enum MemoryArrangement {
+        MemoryArrangement_4GB             = 0,
+        MemoryArrangement_4GBForAppletDev = 1,
+        MemoryArrangement_4GBForSystemDev = 2,
+        MemoryArrangement_6GB             = 3,
+        MemoryArrangement_6GBForAppletDev = 4,
+        MemoryArrangement_8GB             = 5,
+    };
+
     enum class ConfigItem : u32 {
         /* Standard config items. */
         DisableProgramVerification  = 1,

libraries/libmesosphere/source/kern_k_memory_layout.cpp

@@ -138,6 +138,9 @@ namespace ams::kern {
 
             constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable);
 
+            constexpr size_t CarveoutAlignment             = 0x20000;
+            constexpr size_t CarveoutSizeMax               = 512_MB - CarveoutAlignment;
+
             constexpr size_t CoreLocalRegionAlign          = PageSize;
             constexpr size_t CoreLocalRegionSize           = PageSize * (1 + cpu::NumCores);
             constexpr size_t CoreLocalRegionSizeWithGuards = CoreLocalRegionSize + 2 * PageSize;
@@ -177,6 +180,12 @@ namespace ams::kern {
 
             }
 
+            void InsertPoolPartitionBlockIntoBothTrees(size_t start, size_t size, KMemoryRegionType phys_type, KMemoryRegionType virt_type, u32 &cur_attr) {
+                const u32 attr = cur_attr++;
+                MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryBlockTree().Insert(start, size, phys_type, attr));
+                MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryBlockTree().Insert(KMemoryLayout::GetPhysicalMemoryBlockTree().FindFirstBlockByTypeAttr(phys_type, attr)->GetPairAddress(), size, virt_type, attr));
+            }
+
         }
 
         void SetupCoreLocalRegionMemoryBlocks(KInitialPageTable &page_table, KInitialPageAllocator &page_allocator) {
@@ -214,7 +223,62 @@ namespace ams::kern {
         }
 
         void SetupPoolPartitionMemoryBlocks() {
-            /* TODO */
+            /* Start by identifying the extents of the DRAM memory region. */
+            const auto dram_extents = KMemoryLayout::GetPhysicalMemoryBlockTree().GetDerivedRegionExtents(KMemoryRegionType_Dram);
+
+            /* Get Application and Applet pool sizes. */
+            const size_t application_pool_size       = KSystemControl::Init::GetApplicationPoolSize();
+            const size_t applet_pool_size            = KSystemControl::Init::GetAppletPoolSize();
+            const size_t unsafe_system_pool_min_size = KSystemControl::Init::GetMinimumNonSecureSystemPoolSize();
+
+            /* Find the start of the kernel DRAM region. */
+            const uintptr_t kernel_dram_start = KMemoryLayout::GetPhysicalMemoryBlockTree().FindFirstDerivedBlock(KMemoryRegionType_DramKernel)->GetAddress();
+            MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(kernel_dram_start, CarveoutAlignment));
+
+            /* Find the start of the pool partitions region. */
+            const uintptr_t pool_partitions_start = KMemoryLayout::GetPhysicalMemoryBlockTree().FindFirstBlockByTypeAttr(KMemoryRegionType_DramPoolPartition)->GetAddress();
+
+            /* Decide on starting addresses for our pools. */
+            const uintptr_t application_pool_start   = dram_extents.last_block->GetEndAddress() - application_pool_size;
+            const uintptr_t applet_pool_start        = application_pool_start - applet_pool_size;
+            const uintptr_t unsafe_system_pool_start = std::min(kernel_dram_start + CarveoutSizeMax, util::AlignDown(applet_pool_start - unsafe_system_pool_min_size, CarveoutAlignment));
+            const size_t    unsafe_system_pool_size  = applet_pool_start - unsafe_system_pool_start;
+
+            /* We want to arrange application pool depending on where the middle of dram is. */
+            const uintptr_t dram_midpoint = (dram_extents.first_block->GetAddress() + dram_extents.last_block->GetEndAddress()) / 2;
+            u32 cur_pool_attr = 0;
+            size_t total_overhead_size = 0;
+            if (dram_extents.last_block->GetEndAddress() <= dram_midpoint || dram_midpoint <= application_pool_start) {
+                InsertPoolPartitionBlockIntoBothTrees(application_pool_start, application_pool_size, KMemoryRegionType_DramApplicationPool, KMemoryRegionType_VirtualDramApplicationPool, cur_pool_attr);
+                total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(application_pool_size);
+            } else {
+                const size_t first_application_pool_size  = dram_midpoint - application_pool_start;
+                const size_t second_application_pool_size = application_pool_start + application_pool_size - dram_midpoint;
+                InsertPoolPartitionBlockIntoBothTrees(application_pool_start, first_application_pool_size, KMemoryRegionType_DramApplicationPool, KMemoryRegionType_VirtualDramApplicationPool, cur_pool_attr);
+                InsertPoolPartitionBlockIntoBothTrees(dram_midpoint, second_application_pool_size, KMemoryRegionType_DramApplicationPool, KMemoryRegionType_VirtualDramApplicationPool, cur_pool_attr);
+                total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(first_application_pool_size);
+                total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(second_application_pool_size);
+            }
+
+            /* Insert the applet pool. */
+            InsertPoolPartitionBlockIntoBothTrees(applet_pool_start, applet_pool_size, KMemoryRegionType_DramAppletPool, KMemoryRegionType_VirtualDramAppletPool, cur_pool_attr);
+            total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(applet_pool_size);
+
+            /* Insert the nonsecure system pool. */
+            InsertPoolPartitionBlockIntoBothTrees(unsafe_system_pool_start, unsafe_system_pool_size, KMemoryRegionType_DramSystemNonSecurePool, KMemoryRegionType_VirtualDramSystemNonSecurePool, cur_pool_attr);
+            total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize(unsafe_system_pool_size);
+
+            /* Insert the metadata pool. */
+            total_overhead_size += KMemoryManager::CalculateMetadataOverheadSize((unsafe_system_pool_start - pool_partitions_start) - total_overhead_size);
+            const uintptr_t metadata_pool_start = unsafe_system_pool_start - total_overhead_size;
+            const size_t    metadata_pool_size  = total_overhead_size;
+            u32 metadata_pool_attr = 0;
+            InsertPoolPartitionBlockIntoBothTrees(metadata_pool_start, metadata_pool_size, KMemoryRegionType_DramMetadataPool, KMemoryRegionType_VirtualDramMetadataPool, metadata_pool_attr);
+
+            /* Insert the system pool. */
+            const uintptr_t system_pool_size = metadata_pool_start - pool_partitions_start;
+            InsertPoolPartitionBlockIntoBothTrees(pool_partitions_start, system_pool_size, KMemoryRegionType_DramSystemPool, KMemoryRegionType_VirtualDramSystemPool, cur_pool_attr);
+
         }
 
     }

libraries/libmesosphere/source/kern_k_memory_manager.cpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+namespace ams::kern {
+
+    namespace {
+
+        constexpr size_t g_memory_block_page_shifts[] = { 0xC, 0x10, 0x15, 0x16, 0x19, 0x1D, 0x1E };
+        constexpr size_t NumMemoryBlockPageShifts = util::size(g_memory_block_page_shifts);
+
+    }
+
+    size_t KMemoryManager::Impl::CalculateMetadataOverheadSize(size_t region_size) {
+        const size_t ref_count_size = (region_size / PageSize) * sizeof(u16);
+        const size_t bitmap_size    = (util::AlignUp((region_size / PageSize), BITSIZEOF(u64)) / BITSIZEOF(u64)) * sizeof(u64);
+        const size_t page_heap_size = KPageHeap::CalculateMetadataOverheadSize(region_size, g_memory_block_page_shifts, NumMemoryBlockPageShifts);
+        return util::AlignUp(page_heap_size + bitmap_size + ref_count_size, PageSize);
+    }
+
+}

libraries/libmesosphere/source/kern_k_page_heap.cpp

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+namespace ams::kern {
+
+    size_t KPageHeap::CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts) {
+        size_t overhead_size = 0;
+        for (size_t i = 0; i < num_block_shifts; i++) {
+            overhead_size += KPageHeap::Block::CalculateMetadataOverheadSize(region_size, block_shifts[i], (i != num_block_shifts - 1) ? block_shifts[i + 1] : 0);
+        }
+        return util::AlignUp(overhead_size, KMemoryManager::PageSize);
+    }
+
+}

mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp

@@ -245,6 +245,7 @@ namespace ams::kern::init {
             ttbr1_table.Map(block_virt_addr, block.GetSize(), block.GetAddress(), KernelRwDataAttribute, g_initial_page_allocator);
 
             MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryBlockTree().Insert(block_virt_addr, block.GetSize(), GetTypeForVirtualLinearMapping(block.GetType())));
+            block.SetPairAddress(block_virt_addr);
             KMemoryLayout::GetVirtualMemoryBlockTree().FindContainingBlock(block_virt_addr)->SetPairAddress(block.GetAddress());
         }