From 2faf3d33b52e93fac375619a865675f34a12bbc5 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Wed, 29 Jan 2020 22:06:25 -0800 Subject: [PATCH] kern: Implement KAutoObject, KSlabHeap, KLightLock --- .../libmesosphere/include/mesosphere.hpp | 5 + .../include/mesosphere/kern_k_auto_object.hpp | 217 ++++++++++++++++++ .../kern_k_auto_object_container.hpp | 65 ++++++ .../include/mesosphere/kern_k_class_token.hpp | 127 ++++++++++ .../include/mesosphere/kern_k_light_lock.hpp | 73 ++++++ .../include/mesosphere/kern_k_slab_heap.hpp | 183 +++++++++++++++ .../include/mesosphere/kern_k_thread.hpp | 15 +- .../include/mesosphere/kern_kernel.hpp | 3 +- .../include/mesosphere/kern_slab_helpers.hpp | 116 ++++++++++ .../source/kern_k_auto_object.cpp | 25 ++ .../source/kern_k_auto_object_container.cpp | 51 ++++ .../source/kern_k_light_lock.cpp | 28 +++ .../libmesosphere/source/kern_kernel.cpp | 7 +- libraries/libmesosphere/source/kern_main.cpp | 9 + .../util/util_intrusive_red_black_tree.hpp | 2 +- .../source/arch/arm64/init/kern_init_core.cpp | 2 +- 16 files changed, 923 insertions(+), 5 deletions(-) create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp create mode 100644 libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp create mode 100644 libraries/libmesosphere/source/kern_k_auto_object.cpp create mode 100644 libraries/libmesosphere/source/kern_k_auto_object_container.cpp create mode 100644 libraries/libmesosphere/source/kern_k_light_lock.cpp diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index ea3442e7e..88e755c83 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -43,8 +43,13 @@ #include "mesosphere/kern_k_memory_manager.hpp" #include "mesosphere/kern_k_interrupt_task_manager.hpp" #include "mesosphere/kern_k_core_local_region.hpp" +#include "mesosphere/kern_k_slab_heap.hpp" +#include "mesosphere/kern_k_light_lock.hpp" #include "mesosphere/kern_kernel.hpp" +/* Auto Objects. */ +#include "mesosphere/kern_k_auto_object.hpp" + /* Supervisor Calls. */ #include "mesosphere/kern_svc.hpp" diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp new file mode 100644 index 000000000..19bcf949d --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include + +namespace ams::kern { + + class KProcess; + + #define MESOSPHERE_AUTOOBJECT_TRAITS(CLASS) \ + private: \ + friend class KClassTokenGenerator; \ + static constexpr inline auto ObjectType = KClassTokenGenerator::ObjectType::CLASS; \ + static constexpr inline const char * const TypeName = #CLASS; \ + static constexpr inline ClassTokenType ClassToken = ClassToken; \ + public: \ + static constexpr ALWAYS_INLINE TypeObj GetStaticTypeObj() { return TypeObj(TypeName, ClassToken); } \ + static constexpr ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; } \ + virtual TypeObj GetTypeObj() const { return TypeObj(TypeName, ClassToken); } \ + virtual const char *GetTypeName() { return TypeName; } \ + private: + + + + class KAutoObject { + NON_COPYABLE(KAutoObject); + NON_MOVEABLE(KAutoObject); + protected: + class TypeObj { + private: + const char *name; + ClassTokenType class_token; + public: + constexpr explicit TypeObj(const char *n, ClassTokenType tok) : name(n), class_token(tok) { /* ... */ } + + constexpr ALWAYS_INLINE const char *GetName() const { return this->name; } + constexpr ALWAYS_INLINE ClassTokenType GetClassToken() const { return this->class_token; } + + constexpr ALWAYS_INLINE bool operator==(const TypeObj &rhs) { + return this->GetClassToken() == rhs.GetClassToken(); + } + + constexpr ALWAYS_INLINE bool operator!=(const TypeObj &rhs) { + return this->GetClassToken() != rhs.GetClassToken(); + } + + constexpr ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) { + return (this->GetClassToken() | rhs.GetClassToken()) == this->GetClassToken(); + } + }; + private: + std::atomic ref_count; + public: + static KAutoObject *Create(KAutoObject *ptr); + public: + constexpr ALWAYS_INLINE explicit KAutoObject() : ref_count(0) { /* ... */ } + virtual ~KAutoObject() { /* ... */ } + + /* Destroy is responsible for destroying the auto object's resources when ref_count hits zero. */ + virtual void Destroy() { /* ... */ } + + /* Finalize is responsible for cleaning up resource, but does not destroy the object. */ + virtual void Finalize() { /* ... */ } + + virtual KProcess *GetOwner() const { return nullptr; } + + u32 GetReferenceCount() const { + return this->ref_count; + } + + ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) const { + return this->GetTypeObj().IsDerivedFrom(rhs); + } + + ALWAYS_INLINE bool IsDerivedFrom(const KAutoObject &rhs) const { + return this->IsDerivedFrom(rhs.GetTypeObj()); + } + + template + ALWAYS_INLINE Derived DynamicCast() { + static_assert(std::is_pointer::value); + using DerivedType = typename std::remove_pointer::type; + + if (AMS_LIKELY(this->IsDerivedFrom(DerivedType::GetStaticTypeObj()))) { + return static_cast(this); + } else { + return nullptr; + } + } + + template + ALWAYS_INLINE const Derived DynamicCast() const { + static_assert(std::is_pointer::value); + using DerivedType = typename std::remove_pointer::type; + + if (AMS_LIKELY(this->IsDerivedFrom(DerivedType::GetStaticTypeObj()))) { + return static_cast(this); + } else { + return nullptr; + } + } + + ALWAYS_INLINE bool Open() { + /* Atomically increment the reference count, only if it's positive. */ + u32 cur_ref_count = this->ref_count.load(std::memory_order_acquire); + do { + if (AMS_UNLIKELY(cur_ref_count == 0)) { + return false; + } + MESOSPHERE_ABORT_UNLESS(cur_ref_count < cur_ref_count + 1); + } while (!this->ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count + 1, std::memory_order_relaxed)); + + return true; + } + + ALWAYS_INLINE void Close() { + /* Atomically decrement the reference count, not allowing it to become negative. */ + u32 cur_ref_count = this->ref_count.load(std::memory_order_acquire); + do { + MESOSPHERE_ABORT_UNLESS(cur_ref_count > 0); + } while (!this->ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, std::memory_order_relaxed)); + + /* If ref count hits zero, destroy the object. */ + if (cur_ref_count - 1 == 0) { + this->Destroy(); + } + } + + /* Ensure that we have functional type object getters. */ + MESOSPHERE_AUTOOBJECT_TRAITS(KAutoObject); + }; + + class KAutoObjectWithListContainer; + + class KAutoObjectWithList : public KAutoObject { + private: + friend class KAutoObjectWithListContainer; + private: + util::IntrusiveRedBlackTreeNode list_node; + public: + static ALWAYS_INLINE int Compare(const KAutoObjectWithList &lhs, const KAutoObjectWithList &rhs) { + const u64 lid = lhs.GetId(); + const u64 rid = rhs.GetId(); + + if (lid < rid) { + return -1; + } else if (lid > rid) { + return 1; + } else { + return 0; + } + } + public: + virtual u64 GetId() const { + return reinterpret_cast(this); + } + }; + + template + class KScopedAutoObject { + static_assert(std::is_base_of::value); + NON_COPYABLE(KScopedAutoObject); + private: + T *obj; + private: + constexpr ALWAYS_INLINE void Swap(KScopedAutoObject &rhs) { + /* TODO: C++20 constexpr std::swap */ + T *tmp = rhs.obj; + rhs.obj = this->obj; + this->obj = tmp; + } + public: + constexpr ALWAYS_INLINE KScopedAutoObject() : obj(nullptr) { /* ... */ } + constexpr ALWAYS_INLINE KScopedAutoObject(T *o) : obj(o) { /* ... */ } + ALWAYS_INLINE ~KScopedAutoObject() { + if (this->obj != nullptr) { + this->obj->Close(); + } + this->obj = nullptr; + } + + constexpr ALWAYS_INLINE KScopedAutoObject(KScopedAutoObject &&rhs) { + this->obj = rhs.obj; + rhs.obj = nullptr; + } + + constexpr ALWAYS_INLINE KScopedAutoObject &operator=(KScopedAutoObject &&rhs) { + rhs.Swap(*this); + return *this; + } + + constexpr ALWAYS_INLINE T *operator->() { return this->obj; } + constexpr ALWAYS_INLINE T &operator*() { return *this->obj; } + + constexpr ALWAYS_INLINE void Reset(T *o) { + KScopedAutoObject(o).Swap(*this); + } + }; + + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp new file mode 100644 index 000000000..52504f667 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::kern { + + class KAutoObjectWithListContainer { + NON_COPYABLE(KAutoObjectWithListContainer); + NON_MOVEABLE(KAutoObjectWithListContainer); + private: + using ListType = util::IntrusiveRedBlackTreeMemberTraits<&KAutoObjectWithList::list_node>::TreeType; + public: + class ListAccessor : public KScopedLightLock { + private: + ListType &list; + public: + explicit ListAccessor(KAutoObjectWithListContainer *container) : KScopedLightLock(container->lock), list(container->object_list) { /* ... */ } + explicit ListAccessor(KAutoObjectWithListContainer &container) : KScopedLightLock(container.lock), list(container.object_list) { /* ... */ } + + typename ListType::iterator begin() const { + return this->list.begin(); + } + + typename ListType::iterator end() const { + return this->list.end(); + } + + typename ListType::iterator find(typename ListType::const_reference ref) const { + return this->list.find(ref); + } + }; + + friend class ListAccessor; + private: + KLightLock lock; + ListType object_list; + public: + constexpr KAutoObjectWithListContainer() : lock(), object_list() { /* ... */ } + + void Initialize() { /* Nothing to do. */ } + void Finalize() { /* Nothing to do. */ } + + Result Register(KAutoObjectWithList *obj); + Result Unregister(KAutoObjectWithList *obj); + size_t GetOwnedCount(KProcess *owner); + }; + + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp new file mode 100644 index 000000000..b1640f6a8 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_class_token.hpp @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::kern { + + class KAutoObject; + + class KClassTokenGenerator { + public: + using TokenBaseType = u16; + public: + static constexpr size_t BaseClassBits = 8; + static constexpr size_t FinalClassBits = (sizeof(TokenBaseType) * CHAR_BIT) - BaseClassBits; + /* One bit per base class. */ + static constexpr size_t NumBaseClasses = BaseClassBits; + /* Final classes are permutations of three bits. */ + static constexpr size_t NumFinalClasses = [] { + TokenBaseType index = 0; + for (size_t i = 0; i < FinalClassBits; i++) { + for (size_t j = i + 1; j < FinalClassBits; j++) { + for (size_t k = j + 1; k < FinalClassBits; k++) { + index++; + } + } + } + return index; + }(); + private: + template + static constexpr inline TokenBaseType BaseClassToken = BIT(Index); + + template + static constexpr inline TokenBaseType FinalClassToken = [] { + TokenBaseType index = 0; + for (size_t i = 0; i < FinalClassBits; i++) { + for (size_t j = i + 1; j < FinalClassBits; j++) { + for (size_t k = j + 1; k < FinalClassBits; k++) { + if ((index++) == Index) { + return ((1ul << i) | (1ul << j) | (1ul << k)) << BaseClassBits; + } + } + } + } + __builtin_unreachable(); + }(); + + template + static constexpr inline TokenBaseType GetClassToken() { + static_assert(std::is_base_of::value); + if constexpr (std::is_same::value) { + static_assert(T::ObjectType == ObjectType::BaseClassesStart); + return BaseClassToken<0>; + } else if constexpr (!std::is_final::value) { + static_assert(ObjectType::BaseClassesStart < T::ObjectType && T::ObjectType < ObjectType::BaseClassesEnd); + constexpr auto ClassIndex = static_cast(T::ObjectType) - static_cast(ObjectType::BaseClassesStart); + return BaseClassToken | GetClassToken(); + } else if constexpr (ObjectType::FinalClassesStart <= T::ObjectType && T::ObjectType < ObjectType::FinalClassesEnd) { + constexpr auto ClassIndex = static_cast(T::ObjectType) - static_cast(ObjectType::FinalClassesStart); + return FinalClassToken | GetClassToken(); + } else { + static_assert(!std::is_same::value, "GetClassToken: Invalid Type"); + } + }; + public: + enum class ObjectType { + BaseClassesStart = 0, + + KAutoObject = BaseClassesStart, + KSynchronizationObject, + KReadableEvent, + + BaseClassesEnd, + + FinalClassesStart = BaseClassesEnd, + + KInterruptEvent = FinalClassesStart, + KDebug, + KThread, + KServerPort, + KServerSession, + KClientPort, + KClientSession, + KProcess, + KResourceLimit, + KLightSession, + KPort, + KSession, + KSharedMemory, + KEvent, + KWritableEvent, + KLightClientSession, + KLightServerSession, + KTransferMemory, + KDeviceAddressSpace, + KSessionRequest, + KCodeMemory, + + FinalClassesEnd = FinalClassesStart + NumFinalClasses, + }; + + template + static constexpr inline TokenBaseType ClassToken = GetClassToken(); + }; + + using ClassTokenType = KClassTokenGenerator::TokenBaseType; + + template + static constexpr inline ClassTokenType ClassToken = KClassTokenGenerator::ClassToken; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp new file mode 100644 index 000000000..eb0cab5e1 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::kern { + + class KLightLock { + private: + std::atomic tag; + public: + constexpr KLightLock() : tag(0) { /* ... */ } + + void Lock() { + const uintptr_t cur_thread = reinterpret_cast(GetCurrentThreadPointer()); + + while (true) { + uintptr_t old_tag = this->tag.load(std::memory_order_relaxed); + + while (!this->tag.compare_exchange_weak(old_tag, (old_tag == 0) ? cur_thread : old_tag | 1, std::memory_order_acquire)) { + /* ... */ + } + + if ((old_tag == 0) || ((old_tag | 1) == (cur_thread | 1))) { + break; + } + + this->LockSlowPath(old_tag | 1, cur_thread); + } + } + + void Unlock() { + const uintptr_t cur_thread = reinterpret_cast(GetCurrentThreadPointer()); + uintptr_t expected = cur_thread; + if (!this->tag.compare_exchange_weak(expected, 0, std::memory_order_release)) { + this->UnlockSlowPath(cur_thread); + } + } + + void LockSlowPath(uintptr_t owner, uintptr_t cur_thread); + void UnlockSlowPath(uintptr_t cur_thread); + }; + + class KScopedLightLock { + private: + KLightLock *lock; + public: + explicit ALWAYS_INLINE KScopedLightLock(KLightLock *l) : lock(l) { + this->lock->Lock(); + } + ALWAYS_INLINE ~KScopedLightLock() { + this->lock->Unlock(); + } + + explicit ALWAYS_INLINE KScopedLightLock(KLightLock &l) : KScopedLightLock(std::addressof(l)) { /* ... */ } + }; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp new file mode 100644 index 000000000..e504ef539 --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::kern { + + namespace impl { + + class KSlabHeapImpl { + NON_COPYABLE(KSlabHeapImpl); + NON_MOVEABLE(KSlabHeapImpl); + public: + struct Node { + Node *next; + }; + private: + std::atomic head; + size_t obj_size; + public: + constexpr KSlabHeapImpl() : head(nullptr), obj_size(0) { /* ... */ } + + void Initialize(size_t size) { + MESOSPHERE_INIT_ABORT_UNLESS(this->head == nullptr); + this->obj_size = size; + } + + Node *GetHead() const { + return this->head; + } + + size_t GetObjectSize() const { + return this->obj_size; + } + + void *Allocate() { + Node *ret = this->head.load(); + + do { + if (AMS_UNLIKELY(ret == nullptr)) { + break; + } + } while (!this->head.compare_exchange_weak(ret, ret->next)); + + return ret; + } + + void Free(void *obj) { + Node *node = reinterpret_cast(obj); + + Node *cur_head = this->head.load(); + do { + node->next = cur_head; + } while (!this->head.compare_exchange_weak(cur_head, node)); + } + }; + + } + + class KSlabHeapBase { + NON_COPYABLE(KSlabHeapBase); + NON_MOVEABLE(KSlabHeapBase); + private: + using Impl = impl::KSlabHeapImpl; + private: + Impl impl; + uintptr_t peak; + uintptr_t start; + uintptr_t end; + private: + ALWAYS_INLINE Impl *GetImpl() { + return std::addressof(this->impl); + } + ALWAYS_INLINE const Impl *GetImpl() const { + return std::addressof(this->impl); + } + public: + constexpr KSlabHeapBase() : impl(), peak(0), start(0), end(0) { /* ... */ } + + ALWAYS_INLINE bool Contains(uintptr_t address) const { + return this->start <= address && address < this->end; + } + + void InitializeImpl(size_t obj_size, void *memory, size_t memory_size) { + /* Ensure we don't initialize a slab using null memory. */ + MESOSPHERE_ABORT_UNLESS(memory != nullptr); + + /* Initialize the base allocator. */ + this->GetImpl()->Initialize(obj_size); + + /* Set our tracking variables. */ + const size_t num_obj = (memory_size / obj_size); + this->start = reinterpret_cast(memory); + this->end = this->start + num_obj * obj_size; + this->peak = this->start; + + /* Free the objects. */ + u8 *cur = reinterpret_cast(this->end); + + for (size_t i = 0; i < num_obj; i++) { + cur -= obj_size; + this->GetImpl()->Free(cur); + } + } + + size_t GetSlabHeapSize() const { + return (this->end - this->start) / this->GetObjectSize(); + } + + size_t GetObjectSize() const { + return this->GetImpl()->GetObjectSize(); + } + + void *AllocateImpl() { + void *obj = this->GetImpl()->Allocate(); + + /* TODO: under some debug define, track the peak for statistics, as N does? */ + + return obj; + } + + void FreeImpl(void *obj) { + /* Don't allow freeing an object that wasn't allocated from this heap. */ + MESOSPHERE_ABORT_UNLESS(this->Contains(reinterpret_cast(obj))); + + this->GetImpl()->Free(obj); + } + + size_t GetObjectIndexImpl(const void *obj) const { + return (reinterpret_cast(obj) - this->start) / this->GetObjectSize(); + } + + size_t GetPeakIndex() const { + return this->GetObjectIndexImpl(reinterpret_cast(this->peak)); + } + + uintptr_t GetSlabHeapAddress() const { + return this->start; + } + }; + + template + class KSlabHeap : public KSlabHeapBase { + public: + constexpr KSlabHeap() : KSlabHeapBase() { /* ... */ } + + void Initialize(void *memory, size_t memory_size) { + this->InitializeImpl(sizeof(T), memory, memory_size); + } + + T *Allocate() { + T *obj = reinterpret_cast(this->AllocateImpl()); + if (AMS_LIKELY(obj != nullptr)) { + new (obj) T(); + } + return obj; + } + + void Free(T *obj) { + this->FreeImpl(obj); + } + + size_t GetObjectIndex(const T *obj) const { + return this->GetObjectIndexImpl(obj); + } + }; + +} diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index 8959a381e..b6f35639a 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -14,11 +14,24 @@ * along with this program. If not, see . */ #pragma once +#include namespace ams::kern { - class KThread { + class KThread : KAutoObjectWithSlabHeapAndContainer { + public: + struct StackParameters { + alignas(0x10) u8 svc_permission[0x10]; + std::atomic dpc_flags; + u8 current_svc_id; + bool is_calling_svc; + bool is_in_exception_handler; + bool has_exception_svc_perms; + s32 disable_count; + void *context; /* TODO: KThreadContext * */ + }; + static_assert(alignof(StackParameters) == 0x10); /* TODO: This should be a KAutoObject, and this is a placeholder definition. */ }; diff --git a/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp b/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp index 4eb1e97d5..c1b95e15a 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp @@ -31,7 +31,8 @@ namespace ams::kern { private: static inline State s_state = State::Invalid; public: - static void Initialize(s32 core_id); + static NOINLINE void Initialize(s32 core_id); + static NOINLINE void InitializeCoreThreads(s32 core_id); static ALWAYS_INLINE State GetState() { return s_state; } static ALWAYS_INLINE void SetState(State state) { s_state = state; } diff --git a/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp b/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp new file mode 100644 index 000000000..31cf1613a --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/kern_slab_helpers.hpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include +#include +#include + +namespace ams::kern { + + template + class KSlabAllocated { + private: + static inline KSlabHeap s_slab_heap; + public: + constexpr KSlabAllocated() { /* ... */ } + + size_t GetSlabIndex() const { + return s_slab_heap.GetIndex(static_cast(this)); + } + public: + static void InitializeSlabHeap(void *memory, size_t memory_size) { + s_slab_heap.Initialize(memory, memory_size); + } + + static ALWAYS_INLINE Derived *Allocate() { + return s_slab_heap.Allocate(); + } + + static ALWAYS_INLINE void Free(Derived *obj) { + s_slab_heap.Free(obj); + } + + static size_t GetObjectSize() { return s_slab_heap.GetObjectSize(); } + static size_t GetSlabHeapSize() { return s_slab_heap.GetSlabHeapSize(); } + static size_t GetPeakIndex() { return s_slab_heap.GetPeakIndex(); } + static uintptr_t GetSlabHeapAddress() { return s_slab_heap.GetSlabHeapAddress(); } + }; + + template + class KAutoObjectWithSlabHeapAndContainer : public Base { + static_assert(std::is_base_of::value); + private: + static inline KSlabHeap s_slab_heap; + static inline KAutoObjectWithListContainer s_container; + private: + static ALWAYS_INLINE Derived *Allocate() { + return s_slab_heap.Allocate(); + } + + static ALWAYS_INLINE void Free(Derived *obj) { + s_slab_heap.Free(obj); + } + public: + constexpr KAutoObjectWithSlabHeapAndContainer() : Base() { /* ... */ } + virtual ~KAutoObjectWithSlabHeapAndContainer() { /* ... */ } + + virtual void Destroy() override { + const bool is_initialized = this->IsInitialized(); + uintptr_t arg = 0; + if (is_initialized) { + s_container.Unregister(this); + arg = this->GetPostDestroyArgument(); + this->Finalize(); + } + Free(static_cast(this)); + if (is_initialized) { + Derived::PostDestroy(arg); + } + } + + virtual bool IsInitialized() const { return true; } + virtual uintptr_t GetPostDestroyArgument() const { return 0; } + + size_t GetSlabIndex() const { + return s_slab_heap.GetIndex(static_cast(this)); + } + public: + static void InitializeSlabHeap(void *memory, size_t memory_size) { + s_slab_heap.Initialize(memory, memory_size); + s_container.Initialize(); + } + + static Derived *Create() { + Derived *obj = Allocate(); + if (AMS_LIKELY(obj != nullptr)) { + KAutoObject::Create(obj); + } + return obj; + } + + static Result Register(Derived *obj) { + return s_container.Register(obj); + } + + static size_t GetObjectSize() { return s_slab_heap.GetObjectSize(); } + static size_t GetSlabHeapSize() { return s_slab_heap.GetSlabHeapSize(); } + static size_t GetPeakIndex() { return s_slab_heap.GetPeakIndex(); } + static uintptr_t GetSlabHeapAddress() { return s_slab_heap.GetSlabHeapAddress(); } + }; + +} diff --git a/libraries/libmesosphere/source/kern_k_auto_object.cpp b/libraries/libmesosphere/source/kern_k_auto_object.cpp new file mode 100644 index 000000000..5a023d5bb --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_auto_object.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + KAutoObject *KAutoObject::Create(KAutoObject *obj) { + obj->ref_count = 1; + return obj; + } + +} diff --git a/libraries/libmesosphere/source/kern_k_auto_object_container.cpp b/libraries/libmesosphere/source/kern_k_auto_object_container.cpp new file mode 100644 index 000000000..f6cea8023 --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_auto_object_container.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + + Result KAutoObjectWithListContainer::Register(KAutoObjectWithList *obj) { + KScopedLightLock lk(this->lock); + + this->object_list.insert(*obj); + + return ResultSuccess(); + } + + Result KAutoObjectWithListContainer::Unregister(KAutoObjectWithList *obj) { + KScopedLightLock lk(this->lock); + + this->object_list.erase(this->object_list.iterator_to(*obj)); + + return ams::svc::ResultNotFound(); + } + + size_t KAutoObjectWithListContainer::GetOwnedCount(KProcess *owner) { + KScopedLightLock lk(this->lock); + + size_t count = 0; + + for (auto &obj : this->object_list) { + if (obj.GetOwner() == owner) { + count++; + } + } + + return count; + } + +} diff --git a/libraries/libmesosphere/source/kern_k_light_lock.cpp b/libraries/libmesosphere/source/kern_k_light_lock.cpp new file mode 100644 index 000000000..b77bcf888 --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_light_lock.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + void KLightLock::LockSlowPath(uintptr_t owner, uintptr_t cur_thread) { + /* TODO: Implement (requires KThread, KScheduler) */ + } + + void KLightLock::UnlockSlowPath(uintptr_t cur_thread) { + /* TODO: Implement (requires KThread, KScheduler) */ + } + +} diff --git a/libraries/libmesosphere/source/kern_kernel.cpp b/libraries/libmesosphere/source/kern_kernel.cpp index e00cdb563..ea5b5a4f5 100644 --- a/libraries/libmesosphere/source/kern_kernel.cpp +++ b/libraries/libmesosphere/source/kern_kernel.cpp @@ -17,7 +17,7 @@ namespace ams::kern { - NOINLINE void Kernel::Initialize(s32 core_id) { + void Kernel::Initialize(s32 core_id) { /* Construct the core local region object in place. */ KCoreLocalContext *clc = GetPointer(KMemoryLayout::GetCoreLocalRegionAddress()); new (clc) KCoreLocalContext; @@ -46,4 +46,9 @@ namespace ams::kern { } } + void Kernel::InitializeCoreThreads(s32 core_id) { + /* TODO: This function wants to setup the main thread and the idle thread. */ + /* It also wants to initialize the scheduler/interrupt manager/hardware timer. */ + } + } diff --git a/libraries/libmesosphere/source/kern_main.cpp b/libraries/libmesosphere/source/kern_main.cpp index 0906fe3fc..26a891bf5 100644 --- a/libraries/libmesosphere/source/kern_main.cpp +++ b/libraries/libmesosphere/source/kern_main.cpp @@ -25,6 +25,15 @@ namespace ams::kern { /* Ensure that all cores get to this point before proceeding. */ cpu::SynchronizeAllCores(); + /* Initialize the main and idle thread for each core. */ + /* Synchronize after each init to ensure the cores go in order. */ + for (size_t i = 0; i < cpu::NumCores; i++) { + if (static_cast(i) == core_id) { + Kernel::InitializeCoreThreads(core_id); + } + cpu::SynchronizeAllCores(); + } + /* TODO: Implement more of Main() */ while (true) { /* ... */ } } diff --git a/libraries/libvapours/include/vapours/util/util_intrusive_red_black_tree.hpp b/libraries/libvapours/include/vapours/util/util_intrusive_red_black_tree.hpp index ad2212076..93d244cb5 100644 --- a/libraries/libvapours/include/vapours/util/util_intrusive_red_black_tree.hpp +++ b/libraries/libvapours/include/vapours/util/util_intrusive_red_black_tree.hpp @@ -274,7 +274,7 @@ namespace ams::util { } private: static constexpr TYPED_STORAGE(Derived) DerivedStorage = {}; - static_assert(std::addressof(GetParent(GetNode(GetPointer(DerivedStorage)))) == GetPointer(DerivedStorage)); + static_assert(GetParent(GetNode(GetPointer(DerivedStorage))) == GetPointer(DerivedStorage)); }; template diff --git a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp index 513727ab7..8c555c259 100644 --- a/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp +++ b/mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp @@ -303,7 +303,7 @@ namespace ams::kern::init { init_args->cpuactlr = cpu::GetCpuActlrEl1(); init_args->cpuectlr = cpu::GetCpuEctlrEl1(); init_args->sctlr = cpu::GetSctlrEl1(); - init_args->sp = GetInteger(KMemoryLayout::GetMainStackTopAddress(core_id)); + init_args->sp = GetInteger(KMemoryLayout::GetMainStackTopAddress(core_id)) - sizeof(KThread::StackParameters); init_args->entrypoint = reinterpret_cast(::ams::kern::HorizonKernelMain); init_args->argument = static_cast(core_id); init_args->setup_function = reinterpret_cast(::ams::kern::init::StartOtherCore);