From 50a91b1d6e2c9c616413c84242beb158c3ad9710 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Mon, 6 Apr 2020 01:39:27 -0700 Subject: [PATCH] fs: implement system heap --- .../include/stratosphere/fssystem.hpp | 1 + .../fssystem_file_system_buddy_heap.hpp | 189 +++++++++ .../fssystem_file_system_buddy_heap.cpp | 360 ++++++++++++++++++ .../include/vapours/results/fs_results.hpp | 2 +- 4 files changed, 551 insertions(+), 1 deletion(-) create mode 100644 libraries/libstratosphere/include/stratosphere/fssystem/buffers/fssystem_file_system_buddy_heap.hpp create mode 100644 libraries/libstratosphere/source/fssystem/buffers/fssystem_file_system_buddy_heap.cpp diff --git a/libraries/libstratosphere/include/stratosphere/fssystem.hpp b/libraries/libstratosphere/include/stratosphere/fssystem.hpp index 09097e8c5..c7802c151 100644 --- a/libraries/libstratosphere/include/stratosphere/fssystem.hpp +++ b/libraries/libstratosphere/include/stratosphere/fssystem.hpp @@ -25,3 +25,4 @@ #include #include #include +#include diff --git a/libraries/libstratosphere/include/stratosphere/fssystem/buffers/fssystem_file_system_buddy_heap.hpp b/libraries/libstratosphere/include/stratosphere/fssystem/buffers/fssystem_file_system_buddy_heap.hpp new file mode 100644 index 000000000..0da255696 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/fssystem/buffers/fssystem_file_system_buddy_heap.hpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::fssystem { + + class FileSystemBuddyHeap { + NON_COPYABLE(FileSystemBuddyHeap); + NON_MOVEABLE(FileSystemBuddyHeap); + public: + static constexpr size_t BufferAlignment = sizeof(void *); + static constexpr size_t BlockSizeMin = 2 * sizeof(void *); + static constexpr s32 OrderUpperLimit = BITSIZEOF(s32) - 1; + private: + class PageList; + + struct PageEntry { PageEntry *next; }; + static_assert(std::is_pod::value); + static_assert(sizeof(PageEntry) <= BlockSizeMin); + + class PageList : public ::ams::fs::impl::Newable { + NON_COPYABLE(PageList); + NON_MOVEABLE(PageList); + private: + PageEntry *first_page_entry; + PageEntry *last_page_entry; + s32 entry_count; + public: + constexpr PageList() : first_page_entry(), last_page_entry(), entry_count() { /* ... */ } + + constexpr bool IsEmpty() const { return this->entry_count == 0; } + constexpr s32 GetSize() const { return this->entry_count; } + + constexpr const PageEntry *GetFront() const { return this->first_page_entry; } + public: + PageEntry *PopFront(); + void PushBack(PageEntry *page_entry); + bool Remove(PageEntry *page_entry); + }; + private: + size_t block_size; + s32 order_max; + uintptr_t heap_start; + size_t heap_size; + + PageList *free_lists; + size_t total_free_size; + PageList *external_free_lists; + std::unique_ptr internal_free_lists; + public: + static constexpr s32 GetBlockCountFromOrder(s32 order) { + AMS_ASSERT(0 <= order); + AMS_ASSERT(order < OrderUpperLimit); + return (1 << order); + } + + static constexpr size_t QueryWorkBufferSize(s32 order_max) { + AMS_ASSERT(0 < order_max && order_max < OrderUpperLimit); + return util::AlignUp(sizeof(PageList) * (order_max + 1) + alignof(PageList), alignof(u64)); + } + + static constexpr s32 QueryOrderMax(size_t size, size_t block_size) { + AMS_ASSERT(size >= block_size); + AMS_ASSERT(block_size >= BlockSizeMin); + AMS_ASSERT(util::IsPowerOfTwo(block_size)); + const auto block_count = static_cast(util::AlignUp(size, block_size) / block_size); + for (auto order = 1; true; order++) { + if (block_count <= GetBlockCountFromOrder(order)) { + return order; + } + } + } + + public: + constexpr FileSystemBuddyHeap() : block_size(), order_max(), heap_start(), heap_size(), free_lists(), total_free_size(), external_free_lists(), internal_free_lists() { /* ... */ } + + Result Initialize(uintptr_t address, size_t size, size_t block_size, s32 order_max); + + Result Initialize(uintptr_t address, size_t size, size_t block_size) { + return this->Initialize(address, size, block_size, QueryOrderMax(size, block_size)); + } + + Result Initialize(uintptr_t address, size_t size, size_t block_size, s32 order_max, void *work, size_t work_size) { + AMS_ASSERT(work_size >= QueryWorkBufferSize(order_max)); + const auto aligned_work = util::AlignUp(reinterpret_cast(work), alignof(PageList)); + this->external_free_lists = reinterpret_cast(aligned_work); + return this->Initialize(address, size, block_size, order_max); + } + + Result Initialize(uintptr_t address, size_t size, size_t block_size, void *work, size_t work_size) { + return this->Initialize(address, size, block_size, QueryOrderMax(size, block_size), work, work_size); + } + + void Finalize(); + + void *AllocateByOrder(s32 order); + void Free(void *ptr, s32 order); + + size_t GetTotalFreeSize() const; + size_t GetAllocatableSizeMax() const; + void Dump() const; + + s32 GetOrderFromBytes(size_t size) const { + AMS_ASSERT(this->free_lists != nullptr); + return this->GetOrderFromBlockCount(this->GetBlockCountFromSize(size)); + } + + size_t GetBytesFromOrder(s32 order) const { + AMS_ASSERT(this->free_lists != nullptr); + AMS_ASSERT(0 <= order); + AMS_ASSERT(order < this->GetOrderMax()); + return (this->GetBlockSize() << order); + } + + s32 GetOrderMax() const { + AMS_ASSERT(this->free_lists != nullptr); + return this->order_max; + } + + size_t GetBlockSize() const { + AMS_ASSERT(this->free_lists != nullptr); + return this->block_size; + } + + s32 GetPageBlockCountMax() const { + AMS_ASSERT(this->free_lists != nullptr); + return 1 << this->GetOrderMax(); + } + + size_t GetPageSizeMax() const { + AMS_ASSERT(this->free_lists != nullptr); + return this->GetPageBlockCountMax() * this->GetBlockSize(); + } + private: + void DivideBuddies(PageEntry *page_entry, s32 required_order, s32 chosen_order); + void JoinBuddies(PageEntry *page_entry, s32 order); + PageEntry *GetBuddy(PageEntry *page_entry, s32 order); + PageEntry *GetFreePageEntry(s32 order); + + s32 GetOrderFromBlockCount(s32 block_count) const; + + s32 GetBlockCountFromSize(size_t size) const { + const size_t bsize = this->GetBlockSize(); + return static_cast(util::AlignUp(size, bsize) / bsize); + } + + uintptr_t GetAddressFromPageEntry(const PageEntry &page_entry) const { + const uintptr_t address = reinterpret_cast(std::addressof(page_entry)); + AMS_ASSERT(this->heap_start <= address); + AMS_ASSERT(address < this->heap_start + this->heap_size); + AMS_ASSERT(util::IsAligned(address - this->heap_start, this->GetBlockSize())); + return address; + } + + PageEntry *GetPageEntryFromAddress(uintptr_t address) const { + AMS_ASSERT(this->heap_start <= address); + AMS_ASSERT(address < this->heap_start + this->heap_size); + return reinterpret_cast(this->heap_start + util::AlignDown(address - this->heap_start, this->GetBlockSize())); + } + + s32 GetIndexFromPageEntry(const PageEntry &page_entry) const { + const uintptr_t address = reinterpret_cast(std::addressof(page_entry)); + AMS_ASSERT(this->heap_start <= address); + AMS_ASSERT(address < this->heap_start + this->heap_size); + AMS_ASSERT(util::IsAligned(address - this->heap_start, this->GetBlockSize())); + return static_cast((address - this->heap_start) / this->GetBlockSize()); + } + + bool IsAlignedToOrder(const PageEntry *page_entry, s32 order) const { + return util::IsAligned(GetIndexFromPageEntry(*page_entry), GetBlockCountFromOrder(order)); + } + }; + +} diff --git a/libraries/libstratosphere/source/fssystem/buffers/fssystem_file_system_buddy_heap.cpp b/libraries/libstratosphere/source/fssystem/buffers/fssystem_file_system_buddy_heap.cpp new file mode 100644 index 000000000..7e84fe6bb --- /dev/null +++ b/libraries/libstratosphere/source/fssystem/buffers/fssystem_file_system_buddy_heap.cpp @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::fssystem { + + FileSystemBuddyHeap::PageEntry *FileSystemBuddyHeap::PageList::PopFront() { + AMS_ASSERT(this->entry_count > 0); + + /* Get the first entry. */ + auto page_entry = this->first_page_entry; + + /* Advance our list. */ + this->first_page_entry = page_entry->next; + page_entry->next = nullptr; + + /* Decrement our count. */ + --this->entry_count; + AMS_ASSERT(this->entry_count >= 0); + + /* If this was our last page, clear our last entry. */ + if (this->entry_count == 0) { + this->last_page_entry = nullptr; + } + + return page_entry; + } + + void FileSystemBuddyHeap::PageList::PushBack(PageEntry *page_entry) { + AMS_ASSERT(page_entry != nullptr); + + /* If we're empty, we want to set the first page entry. */ + if (this->IsEmpty()) { + this->first_page_entry = page_entry; + } else { + /* We're not empty, so push the page to the back. */ + AMS_ASSERT(this->last_page_entry != page_entry); + this->last_page_entry->next = page_entry; + } + + /* Set our last page entry to be this one, and link it to the list. */ + this->last_page_entry = page_entry; + this->last_page_entry->next = nullptr; + + /* Increment our entry count. */ + ++this->entry_count; + AMS_ASSERT(this->entry_count > 0); + } + + bool FileSystemBuddyHeap::PageList::Remove(PageEntry *page_entry) { + AMS_ASSERT(page_entry != nullptr); + + /* If we're empty, we can't remove the page list. */ + if (this->IsEmpty()) { + return false; + } + + /* We're going to loop over all pages to find this one, then unlink it. */ + PageEntry *prev_entry = nullptr; + PageEntry *cur_entry = this->first_page_entry; + + while (true) { + /* Check if we found the page. */ + if (cur_entry == page_entry) { + if (cur_entry == this->first_page_entry) { + /* If it's the first page, we just set our first. */ + this->first_page_entry = cur_entry->next; + } else if (cur_entry == this->last_page_entry) { + /* If it's the last page, we set our last. */ + this->last_page_entry = prev_entry; + this->last_page_entry->next = nullptr; + } else { + /* If it's in the middle, we just unlink. */ + prev_entry->next = cur_entry->next; + } + + /* Unlink this entry's next. */ + cur_entry->next = nullptr; + + /* Update our entry count. */ + --this->entry_count; + AMS_ASSERT(this->entry_count >= 0); + + return true; + } + + /* If we have no next page, we can't remove. */ + if (cur_entry->next == nullptr) { + return false; + } + + /* Advance to the next item in the list. */ + prev_entry = cur_entry; + cur_entry = cur_entry->next; + } + } + + Result FileSystemBuddyHeap::Initialize(uintptr_t address, size_t size, size_t block_size, s32 order_max) { + /* Ensure our preconditions. */ + AMS_ASSERT(this->free_lists == nullptr); + AMS_ASSERT(address != 0); + AMS_ASSERT(util::IsAligned(address, BufferAlignment)); + AMS_ASSERT(block_size >= BlockSizeMin); + AMS_ASSERT(util::IsPowerOfTwo(block_size)); + AMS_ASSERT(size >= block_size); + AMS_ASSERT(order_max > 0); + AMS_ASSERT(order_max < OrderUpperLimit); + + /* Set up our basic member variables */ + this->block_size = block_size; + this->order_max = order_max; + this->heap_start = address; + this->heap_size = (size / this->block_size) * this->block_size; + + this->total_free_size = 0; + + /* Determine page sizes. */ + const auto max_page_size = this->block_size << this->order_max; + const auto max_page_count = util::AlignUp(this->heap_size, max_page_size) / max_page_size; + AMS_ASSERT(max_page_count > 0); + + /* Setup the free lists. */ + if (this->external_free_lists != nullptr) { + AMS_ASSERT(this->internal_free_lists == nullptr); + this->free_lists = this->external_free_lists; + } else { + this->internal_free_lists.reset(new PageList[this->order_max + 1]); + this->free_lists = this->internal_free_lists.get(); + R_UNLESS(this->free_lists != nullptr, fs::ResultAllocationFailureInFileSystemBuddyHeapA()); + } + + /* All but the last page region should go to the max order. */ + for (size_t i = 0; i < max_page_count - 1; i++) { + auto page_entry = this->GetPageEntryFromAddress(this->heap_start + i * max_page_size); + this->free_lists[this->order_max].PushBack(page_entry); + } + this->total_free_size += this->free_lists[this->order_max].GetSize() * this->GetBytesFromOrder(this->order_max); + + /* Allocate remaining space to smaller orders as possible. */ + { + auto remaining = this->heap_size - (max_page_count - 1) * max_page_size; + auto cur_address = this->heap_start + (max_page_count - 1) * max_page_size; + AMS_ASSERT(util::IsAligned(remaining, this->block_size)); + + do { + /* Determine what order we can use. */ + auto order = GetOrderFromBytes(remaining + 1); + if (order < 0) { + AMS_ASSERT(GetOrderFromBytes(remaining) == this->order_max); + order = this->order_max + 1; + } + AMS_ASSERT(0 < order); + AMS_ASSERT(order <= this->order_max + 1); + + /* Add to the correct free list. */ + this->free_lists[order - 1].PushBack(GetPageEntryFromAddress(cur_address)); + this->total_free_size += GetBytesFromOrder(order - 1); + + /* Move on to the next order. */ + const auto page_size = GetBytesFromOrder(order - 1); + cur_address += page_size; + remaining -= page_size; + } while (this->block_size <= remaining); + } + + return ResultSuccess(); + } + + void FileSystemBuddyHeap::Finalize() { + AMS_ASSERT(this->free_lists != nullptr); + this->free_lists = nullptr; + this->external_free_lists = nullptr; + this->internal_free_lists.reset(); + } + + void *FileSystemBuddyHeap::AllocateByOrder(s32 order) { + AMS_ASSERT(this->free_lists != nullptr); + AMS_ASSERT(order >= 0); + AMS_ASSERT(order <= this->GetOrderMax()); + + /* Get the page entry. */ + if (const auto page_entry = this->GetFreePageEntry(order); page_entry != nullptr) { + /* Ensure we're allocating an unlinked page. */ + AMS_ASSERT(page_entry->next == nullptr); + + /* Return the address for this entry. */ + return reinterpret_cast(this->GetAddressFromPageEntry(*page_entry)); + } else { + return nullptr; + } + } + + void FileSystemBuddyHeap::Free(void *ptr, s32 order) { + AMS_ASSERT(this->free_lists != nullptr); + AMS_ASSERT(order >= 0); + AMS_ASSERT(order <= this->GetOrderMax()); + + /* Allow free(nullptr) */ + if (ptr == nullptr) { + return; + } + + /* Ensure the pointer is block aligned. */ + AMS_ASSERT(util::IsAligned(reinterpret_cast(ptr) - this->heap_start, this->GetBlockSize())); + + /* Get the page entry. */ + auto page_entry = this->GetPageEntryFromAddress(reinterpret_cast(ptr)); + AMS_ASSERT(this->IsAlignedToOrder(page_entry, order)); + + /* Reinsert into the free lists. */ + this->JoinBuddies(page_entry, order); + } + + size_t FileSystemBuddyHeap::GetTotalFreeSize() const { + AMS_ASSERT(this->free_lists != nullptr); + return this->total_free_size; + } + + size_t FileSystemBuddyHeap::GetAllocatableSizeMax() const { + AMS_ASSERT(this->free_lists != nullptr); + + /* The maximum allocatable size is a chunk from the biggest non-empty order. */ + for (s32 order = this->GetOrderMax(); order >= 0; --order) { + if (!this->free_lists[order].IsEmpty()) { + return this->GetBytesFromOrder(order); + } + } + + /* If all orders are empty, then we can't allocate anything. */ + return 0; + } + + void FileSystemBuddyHeap::Dump() const { + AMS_ASSERT(this->free_lists != nullptr); + /* TODO: Support logging metrics. */ + } + + void FileSystemBuddyHeap::DivideBuddies(PageEntry *page_entry, s32 required_order, s32 chosen_order) { + AMS_ASSERT(page_entry != nullptr); + AMS_ASSERT(required_order >= 0); + AMS_ASSERT(chosen_order >= required_order); + AMS_ASSERT(chosen_order <= this->GetOrderMax()); + + /* Start at the end of the entry. */ + auto address = this->GetAddressFromPageEntry(*page_entry) + this->GetBytesFromOrder(chosen_order); + + for (auto order = chosen_order; order > required_order; --order) { + /* For each order, subtract that order's size from the address to get the start of a new block. */ + address -= this->GetBytesFromOrder(order - 1); + auto divided_entry = this->GetPageEntryFromAddress(address); + + /* Push back to the list. */ + this->free_lists[order - 1].PushBack(divided_entry); + this->total_free_size += this->GetBytesFromOrder(order - 1); + } + } + + void FileSystemBuddyHeap::JoinBuddies(PageEntry *page_entry, s32 order) { + AMS_ASSERT(page_entry != nullptr); + AMS_ASSERT(order >= 0); + AMS_ASSERT(order <= this->GetOrderMax()); + + auto cur_entry = page_entry; + auto cur_order = order; + + while (cur_order < this->GetOrderMax()) { + /* Get the buddy page. */ + const auto buddy_entry = this->GetBuddy(cur_entry, cur_order); + + /* Check whether the buddy is in the relevant free list. */ + if (buddy_entry != nullptr && this->free_lists[cur_order].Remove(buddy_entry)) { + this->total_free_size -= GetBytesFromOrder(cur_order); + + /* Ensure we coalesce with the correct buddy when page is aligned */ + if (!this->IsAlignedToOrder(cur_entry, cur_order + 1)) { + cur_entry = buddy_entry; + } + + ++cur_order; + } else { + /* Buddy isn't in the free list, so we can't coalesce. */ + break; + } + } + + /* Insert the coalesced entry into the free list. */ + this->free_lists[cur_order].PushBack(cur_entry); + this->total_free_size += this->GetBytesFromOrder(cur_order); + } + + FileSystemBuddyHeap::PageEntry *FileSystemBuddyHeap::GetBuddy(PageEntry *page_entry, s32 order) { + AMS_ASSERT(page_entry != nullptr); + AMS_ASSERT(order >= 0); + AMS_ASSERT(order <= this->GetOrderMax()); + + const auto address = this->GetAddressFromPageEntry(*page_entry); + const auto offset = this->GetBlockCountFromOrder(order) * this->GetBlockSize(); + + if (this->IsAlignedToOrder(page_entry, order + 1)) { + /* If the page entry is aligned to the next order, return the buddy block to the right of the current entry. */ + return (address + offset < this->heap_start + this->heap_size) ? GetPageEntryFromAddress(address + offset) : nullptr; + } else { + /* If the page entry isn't aligned, return the buddy block to the left of the current entry. */ + return (this->heap_start <= address - offset) ? GetPageEntryFromAddress(address - offset) : nullptr; + } + } + + FileSystemBuddyHeap::PageEntry *FileSystemBuddyHeap::GetFreePageEntry(s32 order) { + AMS_ASSERT(order >= 0); + AMS_ASSERT(order <= this->GetOrderMax()); + + /* Try orders from low to high until we find a free page entry. */ + for (auto cur_order = order; cur_order <= this->GetOrderMax(); cur_order++) { + if (auto &free_list = this->free_lists[cur_order]; !free_list.IsEmpty()) { + /* The current list isn't empty, so grab an entry from it. */ + PageEntry *page_entry = free_list.PopFront(); + AMS_ASSERT(page_entry != nullptr); + + /* Update size bookkeeping. */ + this->total_free_size -= GetBytesFromOrder(cur_order); + + /* If we allocated more memory than needed, free the unneeded portion. */ + this->DivideBuddies(page_entry, order, cur_order); + AMS_ASSERT(page_entry->next == nullptr); + + /* Return the newly-divided entry. */ + return page_entry; + } + } + + /* We failed to find a free page. */ + return nullptr; + } + + s32 FileSystemBuddyHeap::GetOrderFromBlockCount(s32 block_count) const { + AMS_ASSERT(block_count >= 0); + + /* Return the first order with a big enough block count. */ + for (s32 order = 0; order <= this->GetOrderMax(); ++order) { + if (block_count <= this->GetBlockCountFromOrder(order)) { + return order; + } + } + return -1; + } + +} diff --git a/libraries/libvapours/include/vapours/results/fs_results.hpp b/libraries/libvapours/include/vapours/results/fs_results.hpp index fed6b2453..89dbbdaa7 100644 --- a/libraries/libvapours/include/vapours/results/fs_results.hpp +++ b/libraries/libvapours/include/vapours/results/fs_results.hpp @@ -76,8 +76,8 @@ namespace ams::fs { R_DEFINE_ERROR_RESULT(AllocationFailureInRomFsFileSystemB, 3248); R_DEFINE_ERROR_RESULT(AllocationFailureInRomFsFileSystemC, 3249); R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemCreatorA, 3280); + R_DEFINE_ERROR_RESULT(AllocationFailureInFileSystemBuddyHeapA, 3294); R_DEFINE_ERROR_RESULT(AllocationFailureInDirectorySaveDataFileSystem, 3321); - R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemA, 3347); R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemB, 3348); R_DEFINE_ERROR_RESULT(AllocationFailureInPartitionFileSystemC, 3349);