From 059c706f192b43e714ecef02707d9fd27f42a01b Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Thu, 30 Jan 2020 16:51:35 -0800 Subject: [PATCH] kern: implement KThreadLocalPage --- .../libmesosphere/include/mesosphere.hpp | 1 + .../arm64/init/kern_k_init_page_table.hpp | 7 +- .../include/mesosphere/kern_common.hpp | 6 ++ .../mesosphere/kern_k_core_local_region.hpp | 8 +- .../mesosphere/kern_k_memory_layout.hpp | 8 ++ .../mesosphere/kern_k_memory_manager.hpp | 2 - .../include/mesosphere/kern_k_page_buffer.hpp | 42 +++++++++ .../include/mesosphere/kern_k_process.hpp | 31 +++++++ .../mesosphere/kern_k_thread_local_page.hpp | 93 +++++++++++++++++++ .../mesosphere/kern_k_typed_address.hpp | 4 + .../include/mesosphere/kern_panic.hpp | 8 ++ .../libmesosphere/source/kern_k_page_heap.cpp | 2 +- .../source/kern_k_thread_local_page.cpp | 73 +++++++++++++++ .../vapours/results/results_common.hpp | 38 ++++---- .../include/vapours/svc/svc_types_common.hpp | 2 + 15 files changed, 295 insertions(+), 30 deletions(-) create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_process.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_thread_local_page.hpp create mode 100644 libraries/libmesosphere/source/kern_k_thread_local_page.cpp diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index 6f829e2c1..1131b930d 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -52,6 +52,7 @@ /* Auto Objects. */ #include #include +#include /* Supervisor Calls. */ #include diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index 7d5a0249c..194950f3e 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -21,11 +21,10 @@ namespace ams::kern::init { - constexpr size_t PageSize = 0x1000; - constexpr size_t L1BlockSize = 0x40000000; - constexpr size_t L2BlockSize = 0x200000; + constexpr size_t L1BlockSize = 1_GB; + constexpr size_t L2BlockSize = 2_MB; constexpr size_t L2ContiguousBlockSize = 0x10 * L2BlockSize; - constexpr size_t L3BlockSize = 0x1000; + constexpr size_t L3BlockSize = PageSize; constexpr size_t L3ContiguousBlockSize = 0x10 * L3BlockSize; class PageTableEntry { diff --git a/libraries/libmesosphere/include/mesosphere/kern_common.hpp b/libraries/libmesosphere/include/mesosphere/kern_common.hpp index 7fa55dbdd..8f12b78f7 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_common.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_common.hpp @@ -17,3 +17,9 @@ #include #include #include + +namespace ams::kern { + + constexpr size_t PageSize = 4_KB; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp index 1d37dd612..8b4b74a15 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp @@ -44,18 +44,18 @@ namespace ams::kern { std::atomic num_specific_svc[0x80]; u32 perf_counters[6]; }; - static_assert(sizeof(KCoreLocalContext) < KMemoryManager::PageSize); + static_assert(sizeof(KCoreLocalContext) < PageSize); struct KCoreLocalPage { KCoreLocalContext context; - u8 padding[KMemoryManager::PageSize - sizeof(KCoreLocalContext)]; + u8 padding[PageSize - sizeof(KCoreLocalContext)]; }; - static_assert(sizeof(KCoreLocalPage) == KMemoryManager::PageSize); + static_assert(sizeof(KCoreLocalPage) == PageSize); struct KCoreLocalRegion { KCoreLocalPage current; KCoreLocalPage absolute[cpu::NumCores]; }; - static_assert(sizeof(KCoreLocalRegion) == KMemoryManager::PageSize * (1 + cpu::NumCores)); + static_assert(sizeof(KCoreLocalRegion) == PageSize * (1 + cpu::NumCores)); } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index aee5f47af..5eadbcacc 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -397,6 +397,14 @@ namespace ams::kern { static ALWAYS_INLINE KMemoryBlockTree &GetVirtualLinearMemoryBlockTree() { return s_virtual_linear_tree; } static ALWAYS_INLINE KMemoryBlockTree &GetPhysicalLinearMemoryBlockTree() { return s_physical_linear_tree; } + static ALWAYS_INLINE KVirtualAddress GetLinearVirtualAddress(KPhysicalAddress address) { + return GetInteger(address) + s_linear_phys_to_virt_diff; + } + + static ALWAYS_INLINE KPhysicalAddress GetLinearPhysicalAddress(KVirtualAddress address) { + return GetInteger(address) + s_linear_virt_to_phys_diff; + } + static NOINLINE KVirtualAddress GetMainStackTopAddress(s32 core_id) { return GetVirtualMemoryBlockTree().FindFirstBlockByTypeAttr(KMemoryRegionType_KernelMiscMainStack, static_cast(core_id))->GetEndAddress(); } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp index 00f296ec7..ce9d4443f 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_manager.hpp @@ -19,8 +19,6 @@ namespace ams::kern { class KMemoryManager { - public: - static constexpr size_t PageSize = 0x1000; /* TODO: Elsewhere? */ private: class Impl { public: diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp new file mode 100644 index 000000000..b9a99c2c2 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_buffer.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::kern { + + class KPageBuffer : public KSlabAllocated { + private: + alignas(PageSize) u8 buffer[PageSize]; + public: + KPageBuffer() { + std::memset(buffer, 0, sizeof(buffer)); + } + + static ALWAYS_INLINE KPageBuffer *FromPhysicalAddress(KPhysicalAddress phys_addr) { + const KVirtualAddress virt_addr = KMemoryLayout::GetLinearVirtualAddress(phys_addr); + + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(phys_addr), PageSize)); + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(virt_addr), PageSize)); + + return GetPointer(virt_addr); + } + }; + static_assert(sizeof(KPageBuffer) == PageSize); + static_assert(alignof(KPageBuffer) == PageSize); + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp new file mode 100644 index 000000000..2d3e2b21c --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include +#include +#include + +namespace ams::kern { + + class KProcess final : public KAutoObjectWithSlabHeapAndContainer { + MESOSPHERE_AUTOOBJECT_TRAITS(KProcess, KSynchronizationObject); + /* TODO: This is a placeholder definition. */ + }; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread_local_page.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread_local_page.hpp new file mode 100644 index 000000000..9e8c3fe6e --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread_local_page.hpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include + +namespace ams::kern { + + class KThread; + class KProcess; + + class KThreadLocalPage : public util::IntrusiveRedBlackTreeBaseNode, public KSlabAllocated { + public: + static constexpr size_t RegionsPerPage = PageSize / ams::svc::ThreadLocalRegionSize; + static_assert(RegionsPerPage > 0); + private: + KProcessAddress virt_addr; + KProcess *owner; + bool is_region_free[RegionsPerPage]; + public: + constexpr explicit KThreadLocalPage(KProcessAddress addr) : virt_addr(addr), owner(nullptr), is_region_free() { + for (size_t i = 0; i < RegionsPerPage; i++) { + this->is_region_free[i] = true; + } + } + + constexpr explicit KThreadLocalPage() : KThreadLocalPage(Null) { /* ... */ } + + constexpr ALWAYS_INLINE KProcessAddress GetAddress() const { return this->virt_addr; } + private: + constexpr ALWAYS_INLINE KProcessAddress GetRegionAddress(size_t i) { + return this->GetAddress() + i * ams::svc::ThreadLocalRegionSize; + } + + constexpr ALWAYS_INLINE bool Contains(KProcessAddress addr) { + return this->GetAddress() <= addr && addr < this->GetAddress() + PageSize; + } + + constexpr ALWAYS_INLINE size_t GetRegionIndex(KProcessAddress addr) { + MESOSPHERE_ASSERT(util::IsAligned(GetInteger(addr), ams::svc::ThreadLocalRegionSize)); + MESOSPHERE_ASSERT(this->Contains(addr)); + return (addr - this->GetAddress()) / ams::svc::ThreadLocalRegionSize; + } + public: + Result Initialize(KProcess *process); + Result Finalize(); + + KProcessAddress Reserve(); + void Release(KProcessAddress addr); + + bool IsAllUsed() const { + for (size_t i = 0; i < RegionsPerPage; i++) { + if (this->is_region_free[i]) { + return false; + } + } + return true; + } + + bool IsAllFree() const { + for (size_t i = 0; i < RegionsPerPage; i++) { + if (!this->is_region_free[i]) { + return false; + } + } + return true; + } + + bool IsAnyUsed() const { + return !this->IsAllFree(); + } + + bool IsAnyFree() const { + return !this->IsAllUsed(); + } + }; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_typed_address.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_typed_address.hpp index 3e967624d..fc177de9b 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_typed_address.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_typed_address.hpp @@ -50,6 +50,10 @@ namespace ams::kern { return this->address - rhs; } + constexpr ALWAYS_INLINE ptrdiff_t operator-(KTypedAddress rhs) const { + return this->address - rhs.address; + } + template constexpr ALWAYS_INLINE KTypedAddress operator+=(I rhs) { static_assert(std::is_integral::value); diff --git a/libraries/libmesosphere/include/mesosphere/kern_panic.hpp b/libraries/libmesosphere/include/mesosphere/kern_panic.hpp index 8a41955c6..63bef9b34 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_panic.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_panic.hpp @@ -65,3 +65,11 @@ namespace ams::kern { MESOSPHERE_INIT_ABORT(); \ } \ }) + +#define MESOSPHERE_R_ABORT_UNLESS(expr) \ + ({ \ + const ::ams::Result _tmp_meso_r_abort_res = static_cast<::ams::Result>(expr); \ + if (AMS_UNLIKELY((R_FAILED(_tmp_meso_r_abort_res))) { \ + MESOSPHERE_PANIC("Result Abort(): %s 2%03d-%04d", #expr, _tmp_meso_r_abort_res.GetModule(), _tmp_meso_r_abort_res.GetDescription()); \ + } \ + }) diff --git a/libraries/libmesosphere/source/kern_k_page_heap.cpp b/libraries/libmesosphere/source/kern_k_page_heap.cpp index 6dce64507..3e61d6a86 100644 --- a/libraries/libmesosphere/source/kern_k_page_heap.cpp +++ b/libraries/libmesosphere/source/kern_k_page_heap.cpp @@ -22,7 +22,7 @@ namespace ams::kern { for (size_t i = 0; i < num_block_shifts; i++) { overhead_size += KPageHeap::Block::CalculateMetadataOverheadSize(region_size, block_shifts[i], (i != num_block_shifts - 1) ? block_shifts[i + 1] : 0); } - return util::AlignUp(overhead_size, KMemoryManager::PageSize); + return util::AlignUp(overhead_size, PageSize); } } diff --git a/libraries/libmesosphere/source/kern_k_thread_local_page.cpp b/libraries/libmesosphere/source/kern_k_thread_local_page.cpp new file mode 100644 index 000000000..efc5a3099 --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_thread_local_page.cpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + Result KThreadLocalPage::Initialize(KProcess *process) { + MESOSPHERE_ASSERT_THIS(); + + /* Set that this process owns us. */ + this->owner = process; + + /* Allocate a new page. */ + KPageBuffer *page_buf = KPageBuffer::Allocate(); + R_UNLESS(page_buf != nullptr, svc::ResultOutOfMemory()); + auto page_buf_guard = SCOPE_GUARD { KPageBuffer::Free(page_buf); }; + + /* Map the address in. */ + /* TODO: R_TRY(this->owner->GetPageTable().Map(...)); */ + + /* We succeeded. */ + page_buf_guard.Cancel(); + return ResultSuccess(); + } + + Result KThreadLocalPage::Finalize() { + MESOSPHERE_ASSERT_THIS(); + + /* Get the physical address of the page. */ + KPhysicalAddress phys_addr = Null; + /* TODO: MESOSPHERE_ABORT_UNLESS(this->owner->GetPageTable().GetPhysicalAddress(&phys_addr, this->GetAddress())); */ + + /* Unmap the page. */ + /* TODO: R_TRY(this->owner->GetPageTable().Unmap(...); */ + + /* Free the page. */ + KPageBuffer::Free(KPageBuffer::FromPhysicalAddress(phys_addr)); + return ResultSuccess(); + } + + KProcessAddress KThreadLocalPage::Reserve() { + MESOSPHERE_ASSERT_THIS(); + + for (size_t i = 0; i < util::size(this->is_region_free); i++) { + if (this->is_region_free[i]) { + this->is_region_free[i] = false; + return this->GetRegionAddress(i); + } + } + + return Null; + } + + void KThreadLocalPage::Release(KProcessAddress addr) { + MESOSPHERE_ASSERT_THIS(); + + this->is_region_free[this->GetRegionIndex(addr)] = true; + } + +} diff --git a/libraries/libvapours/include/vapours/results/results_common.hpp b/libraries/libvapours/include/vapours/results/results_common.hpp index ed34f13d1..cb977ab99 100644 --- a/libraries/libvapours/include/vapours/results/results_common.hpp +++ b/libraries/libvapours/include/vapours/results/results_common.hpp @@ -57,8 +57,8 @@ namespace ams { using BaseType = typename ResultTraits::BaseType; static constexpr BaseType SuccessValue = ResultTraits::SuccessValue; public: - constexpr inline BaseType GetModule() const { return ResultTraits::GetModuleFromValue(static_cast(this)->GetValue()); } - constexpr inline BaseType GetDescription() const { return ResultTraits::GetDescriptionFromValue(static_cast(this)->GetValue()); } + constexpr ALWAYS_INLINE BaseType GetModule() const { return ResultTraits::GetModuleFromValue(static_cast(this)->GetValue()); } + constexpr ALWAYS_INLINE BaseType GetDescription() const { return ResultTraits::GetDescriptionFromValue(static_cast(this)->GetValue()); } }; class ResultConstructor; @@ -81,15 +81,15 @@ namespace ams { /* TODO: It sure would be nice to make this private. */ constexpr Result(typename Base::BaseType v) : value(v) { static_assert(std::is_same::value); } - constexpr inline operator ResultSuccess() const; + constexpr ALWAYS_INLINE operator ResultSuccess() const; NX_CONSTEXPR bool CanAccept(Result result) { return true; } - constexpr inline bool IsSuccess() const { return this->GetValue() == Base::SuccessValue; } - constexpr inline bool IsFailure() const { return !this->IsSuccess(); } - constexpr inline typename Base::BaseType GetModule() const { return Base::GetModule(); } - constexpr inline typename Base::BaseType GetDescription() const { return Base::GetDescription(); } + constexpr ALWAYS_INLINE bool IsSuccess() const { return this->GetValue() == Base::SuccessValue; } + constexpr ALWAYS_INLINE bool IsFailure() const { return !this->IsSuccess(); } + constexpr ALWAYS_INLINE typename Base::BaseType GetModule() const { return Base::GetModule(); } + constexpr ALWAYS_INLINE typename Base::BaseType GetDescription() const { return Base::GetDescription(); } - constexpr inline typename Base::BaseType GetValue() const { return this->value; } + constexpr ALWAYS_INLINE typename Base::BaseType GetValue() const { return this->value; } }; static_assert(sizeof(Result) == sizeof(Result::Base::BaseType), "sizeof(Result) == sizeof(Result::Base::BaseType)"); static_assert(std::is_trivially_destructible::value, "std::is_trivially_destructible::value"); @@ -98,12 +98,12 @@ namespace ams { class ResultConstructor { public: - static constexpr inline Result MakeResult(ResultTraits::BaseType value) { + static constexpr ALWAYS_INLINE Result MakeResult(ResultTraits::BaseType value) { return Result(value); } }; - constexpr inline Result MakeResult(ResultTraits::BaseType value) { + constexpr ALWAYS_INLINE Result MakeResult(ResultTraits::BaseType value) { return ResultConstructor::MakeResult(value); } @@ -116,12 +116,12 @@ namespace ams { constexpr operator Result() const { return result::impl::MakeResult(Base::SuccessValue); } NX_CONSTEXPR bool CanAccept(Result result) { return result.IsSuccess(); } - constexpr inline bool IsSuccess() const { return true; } - constexpr inline bool IsFailure() const { return !this->IsSuccess(); } - constexpr inline typename Base::BaseType GetModule() const { return Base::GetModule(); } - constexpr inline typename Base::BaseType GetDescription() const { return Base::GetDescription(); } + constexpr ALWAYS_INLINE bool IsSuccess() const { return true; } + constexpr ALWAYS_INLINE bool IsFailure() const { return !this->IsSuccess(); } + constexpr ALWAYS_INLINE typename Base::BaseType GetModule() const { return Base::GetModule(); } + constexpr ALWAYS_INLINE typename Base::BaseType GetDescription() const { return Base::GetDescription(); } - constexpr inline typename Base::BaseType GetValue() const { return Base::SuccessValue; } + constexpr ALWAYS_INLINE typename Base::BaseType GetValue() const { return Base::SuccessValue; } }; namespace result::impl { @@ -130,7 +130,7 @@ namespace ams { } - constexpr inline Result::operator ResultSuccess() const { + constexpr ALWAYS_INLINE Result::operator ResultSuccess() const { if (!ResultSuccess::CanAccept(*this)) { result::impl::OnResultAssertion(*this); } @@ -151,10 +151,10 @@ namespace ams { constexpr operator Result() const { return MakeResult(Value); } constexpr operator ResultSuccess() const { OnResultAssertion(Value); } - constexpr inline bool IsSuccess() const { return false; } - constexpr inline bool IsFailure() const { return !this->IsSuccess(); } + constexpr ALWAYS_INLINE bool IsSuccess() const { return false; } + constexpr ALWAYS_INLINE bool IsFailure() const { return !this->IsSuccess(); } - constexpr inline typename Base::BaseType GetValue() const { return Value; } + constexpr ALWAYS_INLINE typename Base::BaseType GetValue() const { return Value; } }; template diff --git a/libraries/libvapours/include/vapours/svc/svc_types_common.hpp b/libraries/libvapours/include/vapours/svc/svc_types_common.hpp index 34dfc3b3d..3bd20e6c8 100644 --- a/libraries/libvapours/include/vapours/svc/svc_types_common.hpp +++ b/libraries/libvapours/include/vapours/svc/svc_types_common.hpp @@ -275,6 +275,8 @@ namespace ams::svc { ThreadActivity_Paused = 1, }; + constexpr size_t ThreadLocalRegionSize = 0x200; + /* Process types. */ enum ProcessInfoType : u32 { ProcessInfoType_ProcessState = 0,