diff --git a/libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp b/libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp index cec7a44ab..8cfdd9f07 100644 --- a/libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp +++ b/libraries/libmesosphere/include/mesosphere/init/kern_init_slab_setup.hpp @@ -40,6 +40,7 @@ namespace ams::kern::init { const KSlabResourceCounts &GetSlabResourceCounts(); size_t CalculateTotalSlabHeapSize(); + NOINLINE void InitializeKPageBufferSlabHeap(); NOINLINE void InitializeSlabHeaps(); } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index 0b1b14835..36461fe16 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -467,6 +467,10 @@ namespace ams::kern { return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualDramMetadataPool); } + static NOINLINE KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { + return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address)); + } + static NOINLINE auto GetCarveoutRegionExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected); } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp index 66ec5c2f8..0c728dbdc 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp @@ -59,8 +59,28 @@ namespace ams::kern { size_t Initialize(const KMemoryRegion *region, Pool pool, KVirtualAddress metadata_region, KVirtualAddress metadata_region_end); - constexpr ALWAYS_INLINE void SetNext(Impl *n) { this->next = n; } - constexpr ALWAYS_INLINE void SetPrev(Impl *n) { this->prev = n; } + KVirtualAddress AllocateBlock(s32 index) { return this->heap.AllocateBlock(index); } + void Free(KVirtualAddress addr, size_t num_pages) { this->heap.Free(addr, num_pages); } + + void TrackAllocationForOptimizedProcess(KVirtualAddress block, size_t num_pages); + + constexpr KVirtualAddress GetEndAddress() const { return this->heap.GetEndAddress(); } + + constexpr void SetNext(Impl *n) { this->next = n; } + constexpr void SetPrev(Impl *n) { this->prev = n; } + constexpr Impl *GetNext() const { return this->next; } + constexpr Impl *GetPrev() const { return this->prev; } + + void Open(KLightLock *pool_locks, KVirtualAddress address, size_t num_pages) { + KScopedLightLock lk(pool_locks[this->pool]); + + size_t index = this->heap.GetPageOffset(address); + const size_t end = index + num_pages; + while (index < end) { + const RefCount ref_count = (++this->page_reference_counts[index++]); + MESOSPHERE_ABORT_UNLESS(ref_count > 0); + } + } public: static size_t CalculateMetadataOverheadSize(size_t region_size); }; @@ -72,6 +92,10 @@ namespace ams::kern { size_t num_managers; u64 optimized_process_ids[Pool_Count]; bool has_optimized_process[Pool_Count]; + private: + Impl &GetManager(KVirtualAddress address) { + return this->managers[KMemoryLayout::GetVirtualLinearRegion(address).GetAttributes()]; + } public: constexpr KMemoryManager() : pool_locks(), pool_managers_head(), pool_managers_tail(), managers(), num_managers(), optimized_process_ids(), has_optimized_process() @@ -80,6 +104,19 @@ namespace ams::kern { } void Initialize(KVirtualAddress metadata_region, size_t metadata_region_size); + + KVirtualAddress AllocateContinuous(size_t num_pages, size_t align_pages, u32 option); + + void Open(KVirtualAddress address, size_t num_pages) { + /* Repeatedly open references until we've done so for all pages. */ + while (num_pages) { + auto &manager = this->GetManager(address); + const size_t cur_pages = std::min(num_pages, (manager.GetEndAddress() - address) / PageSize); + manager.Open(this->pool_locks, address, cur_pages); + num_pages -= cur_pages; + address += cur_pages * PageSize; + } + } public: static size_t CalculateMetadataOverheadSize(size_t region_size) { return Impl::CalculateMetadataOverheadSize(region_size); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp index ffc822faa..fd8fc81db 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_heap.hpp @@ -22,6 +22,33 @@ namespace ams::kern { private: static constexpr inline size_t MemoryBlockPageShifts[] = { 0xC, 0x10, 0x15, 0x16, 0x19, 0x1D, 0x1E }; static constexpr size_t NumMemoryBlockPageShifts = util::size(MemoryBlockPageShifts); + public: + static constexpr s32 GetAlignedBlockIndex(size_t num_pages, size_t align_pages) { + const size_t target_pages = std::max(num_pages, align_pages); + for (size_t i = 0; i < NumMemoryBlockPageShifts; i++) { + if (target_pages <= (size_t(1) << MemoryBlockPageShifts[i]) / PageSize) { + return static_cast(i); + } + } + return -1; + } + + static constexpr s32 GetBlockIndex(size_t num_pages) { + for (size_t i = 0; i < NumMemoryBlockPageShifts; i++) { + if (num_pages >= (size_t(1) << MemoryBlockPageShifts[i]) / PageSize) { + return static_cast(i); + } + } + return -1; + } + + static constexpr size_t GetBlockSize(size_t index) { + return size_t(1) << MemoryBlockPageShifts[index]; + } + + static constexpr size_t GetBlockNumPages(size_t index) { + return GetBlockSize(index) / PageSize; + } private: class Block { private: @@ -31,27 +58,143 @@ namespace ams::kern { private: u64 *bit_storages[MaxDepth]; size_t num_bits; - size_t depth; + size_t used_depths; public: - constexpr Bitmap() : bit_storages(), num_bits(), depth() { /* ... */ } + constexpr Bitmap() : bit_storages(), num_bits(), used_depths() { /* ... */ } + + constexpr size_t GetNumBits() const { return this->num_bits; } + constexpr s32 GetHighestDepthIndex() const { return static_cast(this->used_depths) - 1; } u64 *Initialize(u64 *storage, size_t size) { /* Initially, everything is un-set. */ this->num_bits = 0; /* Calculate the needed bitmap depth. */ - this->depth = static_cast(GetRequiredDepth(size)); - MESOSPHERE_ASSERT(this->depth <= MaxDepth); + this->used_depths = static_cast(GetRequiredDepth(size)); + MESOSPHERE_ASSERT(this->used_depths <= MaxDepth); /* Set the bitmap pointers. */ - for (s32 d = static_cast(this->depth) - 1; d >= 0; d--) { - this->bit_storages[d] = storage; + for (s32 depth = this->GetHighestDepthIndex(); depth >= 0; depth--) { + this->bit_storages[depth] = storage; size = util::AlignUp(size, BITSIZEOF(u64)) / BITSIZEOF(u64); storage += size; } return storage; } + + ssize_t FindFreeBlock() const { + uintptr_t offset = 0; + s32 depth = 0; + + do { + const u64 v = this->bit_storages[depth][offset]; + if (v == 0) { + /* If depth is bigger than zero, then a previous level indicated a block was free. */ + MESOSPHERE_ASSERT(depth == 0); + return -1; + } + offset = offset * BITSIZEOF(u64) + __builtin_ctzll(v); + ++depth; + } while (depth < static_cast(this->used_depths)); + + return static_cast(offset); + } + + void SetBit(size_t offset) { + this->SetBit(this->GetHighestDepthIndex(), offset); + this->num_bits++; + } + + void ClearBit(size_t offset) { + this->ClearBit(this->GetHighestDepthIndex(), offset); + this->num_bits--; + } + + bool ClearRange(size_t offset, size_t count) { + s32 depth = this->GetHighestDepthIndex(); + u64 *bits = this->bit_storages[depth]; + size_t bit_ind = offset / BITSIZEOF(u64); + if (AMS_LIKELY(count < BITSIZEOF(u64))) { + const size_t shift = offset % BITSIZEOF(u64); + MESOSPHERE_ASSERT(shift + count <= BITSIZEOF(u64)); + /* Check that all the bits are set. */ + const u64 mask = ((u64(1) << count) - 1) << shift; + u64 v = bits[bit_ind]; + if ((v & mask) != mask) { + return false; + } + + /* Clear the bits. */ + v &= ~mask; + bits[bit_ind] = v; + if (v == 0) { + this->ClearBit(depth - 1, bit_ind); + } + } else { + MESOSPHERE_ASSERT(offset % BITSIZEOF(u64) == 0); + MESOSPHERE_ASSERT(count % BITSIZEOF(u64) == 0); + /* Check that all the bits are set. */ + size_t remaining = count; + size_t i = 0; + do { + if (bits[bit_ind + i++] != ~u64(0)) { + return false; + } + remaining -= BITSIZEOF(u64); + } while (remaining > 0); + + /* Clear the bits. */ + remaining = count; + i = 0; + do { + bits[bit_ind + i] = 0; + this->ClearBit(depth - 1, bit_ind + i); + i++; + remaining -= BITSIZEOF(u64); + } while (remaining > 0); + } + + this->num_bits -= count; + return true; + } + private: + void SetBit(s32 depth, size_t offset) { + while (depth >= 0) { + size_t ind = offset / BITSIZEOF(u64); + size_t which = offset % BITSIZEOF(u64); + const u64 mask = u64(1) << which; + + u64 *bit = std::addressof(this->bit_storages[depth][ind]); + u64 v = *bit; + MESOSPHERE_ASSERT((v & mask) == 0); + *bit = v | mask; + if (v) { + break; + } + offset = ind; + depth--; + } + } + + void ClearBit(s32 depth, size_t offset) { + while (depth >= 0) { + size_t ind = offset / BITSIZEOF(u64); + size_t which = offset % BITSIZEOF(u64); + const u64 mask = u64(1) << which; + + u64 *bit = std::addressof(this->bit_storages[depth][ind]); + u64 v = *bit; + MESOSPHERE_ASSERT((v & mask) != 0); + v &= ~mask; + *bit = v; + if (v) { + break; + } + offset = ind; + depth--; + } + } private: static constexpr s32 GetRequiredDepth(size_t region_size) { s32 depth = 0; @@ -82,6 +225,13 @@ namespace ams::kern { public: constexpr Block() : bitmap(), heap_address(), end_offset(), block_shift(), next_block_shift() { /* ... */ } + constexpr size_t GetShift() const { return this->block_shift; } + constexpr size_t GetNextShift() const { return this->next_block_shift; } + constexpr size_t GetSize() const { return u64(1) << this->GetShift(); } + constexpr size_t GetNumPages() const { return this->GetSize() / PageSize; } + constexpr size_t GetNumFreeBlocks() const { return this->bitmap.GetNumBits(); } + constexpr size_t GetNumFreePages() const { return this->GetNumFreeBlocks() * this->GetNumPages(); } + u64 *Initialize(KVirtualAddress addr, size_t size, size_t bs, size_t nbs, u64 *bit_storage) { /* Set shifts. */ this->block_shift = bs; @@ -89,18 +239,49 @@ namespace ams::kern { /* Align up the address. */ KVirtualAddress end = addr + size; - const size_t align = (this->next_block_shift != 0) ? (1ul << this->next_block_shift) : (this->block_shift); + const size_t align = (this->next_block_shift != 0) ? (u64(1) << this->next_block_shift) : (this->block_shift); addr = util::AlignDown(GetInteger(addr), align); end = util::AlignUp(GetInteger(end), align); this->heap_address = addr; - this->end_offset = (end - addr) / (1ul << this->block_shift); + this->end_offset = (end - addr) / (u64(1) << this->block_shift); return this->bitmap.Initialize(bit_storage, this->end_offset); } + + KVirtualAddress PushBlock(KVirtualAddress address) { + /* Set the bit for the free block. */ + size_t offset = (address - this->heap_address) >> this->GetShift(); + this->bitmap.SetBit(offset); + + /* If we have a next shift, try to clear the blocks below this one and return the new address. */ + if (this->GetNextShift()) { + const size_t diff = u64(1) << (this->GetNextShift() - this->GetShift()); + offset = util::AlignDown(offset, diff); + if (this->bitmap.ClearRange(offset, diff)) { + return this->heap_address + (offset << this->GetShift()); + } + } + + /* We couldn't coalesce, or we're already as big as possible. */ + return Null; + } + + KVirtualAddress PopBlock() { + /* Find a free block. */ + ssize_t soffset = this->bitmap.FindFreeBlock(); + if (soffset < 0) { + return Null; + } + const size_t offset = static_cast(soffset); + + /* Update our tracking and return it. */ + this->bitmap.ClearBit(offset); + return this->heap_address + (offset << this->GetShift()); + } public: static constexpr size_t CalculateMetadataOverheadSize(size_t region_size, size_t cur_block_shift, size_t next_block_shift) { - const size_t cur_block_size = (1ul << cur_block_shift); - const size_t next_block_size = (1ul << next_block_shift); + const size_t cur_block_size = (u64(1) << cur_block_shift); + const size_t next_block_size = (u64(1) << next_block_shift); const size_t align = (next_block_shift != 0) ? next_block_size : cur_block_size; return Bitmap::CalculateMetadataOverheadSize((align * 2 + util::AlignUp(region_size, align)) / cur_block_size); } @@ -113,12 +294,25 @@ namespace ams::kern { Block blocks[NumMemoryBlockPageShifts]; private: void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress metadata_address, size_t metadata_size, const size_t *block_shifts, size_t num_block_shifts); + size_t GetNumFreePages() const; + + void FreeBlock(KVirtualAddress block, s32 index); public: constexpr KPageHeap() : heap_address(), heap_size(), used_size(), num_blocks(), blocks() { /* ... */ } + constexpr KVirtualAddress GetEndAddress() const { return this->heap_address + this->heap_size; } + constexpr size_t GetPageOffset(KVirtualAddress block) const { return (block - this->heap_address) / PageSize; } + void Initialize(KVirtualAddress heap_address, size_t heap_size, KVirtualAddress metadata_address, size_t metadata_size) { return Initialize(heap_address, heap_size, metadata_address, metadata_size, MemoryBlockPageShifts, NumMemoryBlockPageShifts); } + + void UpdateUsedSize() { + this->used_size = this->heap_size - (this->GetNumFreePages() * PageSize); + } + + KVirtualAddress AllocateBlock(s32 index); + void Free(KVirtualAddress addr, size_t num_pages); private: static size_t CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts); public: diff --git a/libraries/libmesosphere/source/init/kern_init_slab_setup.cpp b/libraries/libmesosphere/source/init/kern_init_slab_setup.cpp index e6680971f..108ab9205 100644 --- a/libraries/libmesosphere/source/init/kern_init_slab_setup.cpp +++ b/libraries/libmesosphere/source/init/kern_init_slab_setup.cpp @@ -135,6 +135,26 @@ namespace ams::kern::init { return size; } + void InitializeKPageBufferSlabHeap() { + const auto &counts = GetSlabResourceCounts(); + const size_t num_pages = counts.num_KProcess + counts.num_KThread + (counts.num_KProcess + counts.num_KThread) / 8; + const size_t slab_size = num_pages * PageSize; + + /* Reserve memory from the system resource limit. */ + MESOSPHERE_ABORT_UNLESS(Kernel::GetSystemResourceLimit().Reserve(ams::svc::LimitableResource_PhysicalMemoryMax, slab_size)); + + /* Allocate memory for the slab. */ + constexpr auto AllocateOption = KMemoryManager::EncodeOption(KMemoryManager::Pool_System, KMemoryManager::Direction_FromFront); + const KVirtualAddress slab_address = Kernel::GetMemoryManager().AllocateContinuous(num_pages, 1, AllocateOption); + MESOSPHERE_ABORT_UNLESS(slab_address != Null); + + /* Open references to the slab. */ + Kernel::GetMemoryManager().Open(slab_address, num_pages); + + /* Initialize the slabheap. */ + KPageBuffer::InitializeSlabHeap(GetVoidPointer(slab_address), slab_size); + } + void InitializeSlabHeaps() { /* Get the start of the slab region, since that's where we'll be working. */ KVirtualAddress address = KMemoryLayout::GetSlabRegionAddress(); diff --git a/libraries/libmesosphere/source/kern_k_memory_manager.cpp b/libraries/libmesosphere/source/kern_k_memory_manager.cpp index aff610d26..ac51f0570 100644 --- a/libraries/libmesosphere/source/kern_k_memory_manager.cpp +++ b/libraries/libmesosphere/source/kern_k_memory_manager.cpp @@ -87,6 +87,57 @@ namespace ams::kern { } } + KVirtualAddress KMemoryManager::AllocateContinuous(size_t num_pages, size_t align_pages, u32 option) { + /* Early return if we're allocating no pages. */ + if (num_pages == 0) { + return Null; + } + + /* Lock the pool that we're allocating from. */ + const auto [pool, dir] = DecodeOption(option); + KScopedLightLock lk(this->pool_locks[pool]); + + /* Choose a heap based on our page size request. */ + const s32 heap_index = KPageHeap::GetAlignedBlockIndex(num_pages, align_pages); + + /* Loop, trying to iterate from each block. */ + Impl *chosen_manager = nullptr; + KVirtualAddress allocated_block = Null; + if (dir == Direction_FromBack) { + for (chosen_manager = this->pool_managers_tail[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetPrev()) { + allocated_block = chosen_manager->AllocateBlock(heap_index); + if (allocated_block != Null) { + break; + } + } + } else { + for (chosen_manager = this->pool_managers_head[pool]; chosen_manager != nullptr; chosen_manager = chosen_manager->GetNext()) { + allocated_block = chosen_manager->AllocateBlock(heap_index); + if (allocated_block != Null) { + break; + } + } + } + + /* If we failed to allocate, quit now. */ + if (allocated_block == Null) { + return Null; + } + + /* If we allocated more than we need, free some. */ + const size_t allocated_pages = KPageHeap::GetBlockNumPages(heap_index); + if (allocated_pages > num_pages) { + chosen_manager->Free(allocated_block + num_pages * PageSize, allocated_pages - num_pages); + } + + /* Maintain the optimized memory bitmap, if we should. */ + if (this->has_optimized_process[pool]) { + chosen_manager->TrackAllocationForOptimizedProcess(allocated_block, num_pages); + } + + return allocated_block; + } + size_t KMemoryManager::Impl::Initialize(const KMemoryRegion *region, Pool p, KVirtualAddress metadata, KVirtualAddress metadata_end) { /* Calculate metadata sizes. */ const size_t ref_count_size = (region->GetSize() / PageSize) * sizeof(u16); @@ -107,9 +158,25 @@ namespace ams::kern { /* Initialize the manager's KPageHeap. */ this->heap.Initialize(region->GetAddress(), region->GetSize(), metadata + manager_size, page_heap_size); + /* Free the memory to the heap. */ + this->heap.Free(region->GetAddress(), region->GetSize() / PageSize); + + /* Update the heap's used size. */ + this->heap.UpdateUsedSize(); + return total_metadata_size; } + void KMemoryManager::Impl::TrackAllocationForOptimizedProcess(KVirtualAddress block, size_t num_pages) { + size_t offset = this->heap.GetPageOffset(block); + const size_t last = offset + num_pages - 1; + u64 *optimize_map = GetPointer(this->metadata_region); + while (offset <= last) { + optimize_map[offset / BITSIZEOF(u64)] &= ~(u64(1) << (offset % BITSIZEOF(u64))); + offset++; + } + } + size_t KMemoryManager::Impl::CalculateMetadataOverheadSize(size_t region_size) { const size_t ref_count_size = (region_size / PageSize) * sizeof(u16); const size_t optimize_map_size = (util::AlignUp((region_size / PageSize), BITSIZEOF(u64)) / BITSIZEOF(u64)) * sizeof(u64); diff --git a/libraries/libmesosphere/source/kern_k_page_heap.cpp b/libraries/libmesosphere/source/kern_k_page_heap.cpp index 298ca8572..fe0f5010c 100644 --- a/libraries/libmesosphere/source/kern_k_page_heap.cpp +++ b/libraries/libmesosphere/source/kern_k_page_heap.cpp @@ -41,6 +41,87 @@ namespace ams::kern { MESOSPHERE_ABORT_UNLESS(KVirtualAddress(cur_bitmap_storage) <= metadata_end); } + size_t KPageHeap::GetNumFreePages() const { + size_t num_free = 0; + + for (size_t i = 0; i < this->num_blocks; i++) { + num_free += this->blocks[i].GetNumFreePages(); + } + + return num_free; + } + + KVirtualAddress KPageHeap::AllocateBlock(s32 index) { + const size_t needed_size = this->blocks[index].GetSize(); + + for (s32 i = index; i < static_cast(this->num_blocks); i++) { + if (const KVirtualAddress addr = this->blocks[index].PopBlock(); addr != Null) { + if (const size_t allocated_size = this->blocks[index].GetSize(); allocated_size > needed_size) { + this->Free(addr + needed_size, (allocated_size - needed_size) / PageSize); + } + return addr; + } + } + + return Null; + } + + void KPageHeap::FreeBlock(KVirtualAddress block, s32 index) { + do { + block = this->blocks[index++].PushBlock(block); + } while (block != Null); + } + + void KPageHeap::Free(KVirtualAddress addr, size_t num_pages) { + /* Freeing no pages is a no-op. */ + if (num_pages == 0) { + return; + } + + /* Find the largest block size that we can free, and free as many as possible. */ + s32 big_index = static_cast(this->num_blocks) - 1; + const KVirtualAddress start = addr; + const KVirtualAddress end = addr + num_pages * PageSize; + KVirtualAddress before_start = start; + KVirtualAddress before_end = start; + KVirtualAddress after_start = end; + KVirtualAddress after_end = end; + while (big_index >= 0) { + const size_t block_size = this->blocks[big_index].GetSize(); + const KVirtualAddress big_start = util::AlignUp(GetInteger(start), block_size); + const KVirtualAddress big_end = util::AlignDown(GetInteger(end), block_size); + if (big_start < big_end) { + /* Free as many big blocks as we can. */ + for (auto block = big_start; block < big_end; block += block_size) { + this->FreeBlock(block, big_index); + } + before_end = big_start; + after_start = big_end; + break; + } + big_index--; + } + MESOSPHERE_ASSERT(big_index >= 0); + + /* Free space before the big blocks. */ + for (s32 i = big_index; i >= 0; i--) { + const size_t block_size = this->blocks[i].GetSize(); + while (before_start + block_size <= before_end) { + before_end -= block_size; + this->FreeBlock(before_end, i); + } + } + + /* Free space after the big blocks. */ + for (s32 i = big_index; i >= 0; i--) { + const size_t block_size = this->blocks[i].GetSize(); + while (after_start + block_size <= after_end) { + after_start += block_size; + this->FreeBlock(after_start, i); + } + } + } + size_t KPageHeap::CalculateMetadataOverheadSize(size_t region_size, const size_t *block_shifts, size_t num_block_shifts) { size_t overhead_size = 0; for (size_t i = 0; i < num_block_shifts; i++) { diff --git a/libraries/libmesosphere/source/kern_main.cpp b/libraries/libmesosphere/source/kern_main.cpp index b7363d32f..abbe3355d 100644 --- a/libraries/libmesosphere/source/kern_main.cpp +++ b/libraries/libmesosphere/source/kern_main.cpp @@ -38,10 +38,11 @@ namespace ams::kern { /* Initialize KSystemControl. */ KSystemControl::Initialize(); - /* Initialize the memory manager. */ + /* Initialize the memory manager and the KPageBuffer slabheap. */ { const auto &metadata_region = KMemoryLayout::GetMetadataPoolRegion(); Kernel::GetMemoryManager().Initialize(metadata_region.GetAddress(), metadata_region.GetSize()); + init::InitializeKPageBufferSlabHeap(); } /* Note: this is not actually done here, it's done later in main after more stuff is setup. */