From 72671d39abc6bd779862508a21d6f20c91d45091 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 1 Dec 2020 15:19:29 -0800 Subject: [PATCH] kern: cleanup KThread, optimize/normalize KThreadQueue/KWaitObject --- .../include/mesosphere/kern_k_event_info.hpp | 1 - .../include/mesosphere/kern_k_thread.hpp | 45 +++----- .../mesosphere/kern_k_thread_queue.hpp | 104 +++++------------- .../include/mesosphere/kern_k_wait_object.hpp | 36 +----- .../arch/arm64/kern_k_thread_context.cpp | 2 +- .../source/kern_k_debug_base.cpp | 4 +- .../source/kern_k_light_server_session.cpp | 8 +- .../libmesosphere/source/kern_k_thread.cpp | 5 +- .../source/kern_k_wait_object.cpp | 19 +--- .../include/vapours/svc/svc_types_dmnt.hpp | 2 +- 10 files changed, 58 insertions(+), 168 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_event_info.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_event_info.hpp index 3c1e846a2..315869578 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_event_info.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_event_info.hpp @@ -24,7 +24,6 @@ namespace ams::kern { struct InfoCreateThread { u32 thread_id; uintptr_t tls_address; - uintptr_t entrypoint; }; struct InfoExitProcess { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index b94f575d2..db75c317b 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -31,7 +31,7 @@ namespace ams::kern { using KThreadFunction = void (*)(uintptr_t); - class KThread final : public KAutoObjectWithSlabHeapAndContainer, public KTimerTask, public KWorkerTask { + class KThread final : public KAutoObjectWithSlabHeapAndContainer, public util::IntrusiveListBaseNode, public KTimerTask, public KWorkerTask { MESOSPHERE_AUTOOBJECT_TRAITS(KThread, KSynchronizationObject); private: friend class KProcess; @@ -111,6 +111,8 @@ namespace ams::kern { constexpr void SetPrev(KThread *t) { this->prev = t; } constexpr void SetNext(KThread *t) { this->next = t; } }; + + using WaiterList = util::IntrusiveListBaseTraits::ListType; private: static constexpr size_t PriorityInheritanceCountMax = 10; union SyncObjectBuffer { @@ -141,13 +143,19 @@ namespace ams::kern { static inline std::atomic s_next_thread_id = 0; private: alignas(16) KThreadContext thread_context{}; + util::IntrusiveListNode process_list_node{}; + util::IntrusiveRedBlackTreeNode condvar_arbiter_tree_node{}; + s32 priority{}; + + using ConditionVariableThreadTreeTraits = util::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&KThread::condvar_arbiter_tree_node>; + using ConditionVariableThreadTree = ConditionVariableThreadTreeTraits::TreeType; + + ConditionVariableThreadTree *condvar_tree{}; + uintptr_t condvar_key{}; KAffinityMask affinity_mask{}; u64 thread_id{}; std::atomic cpu_time{}; KSynchronizationObject *synced_object{}; - KLightLock *waiting_lock{}; - uintptr_t condvar_key{}; - uintptr_t entrypoint{}; KProcessAddress address_key{}; KProcess *parent{}; void *kernel_stack_top{}; @@ -159,43 +167,31 @@ namespace ams::kern { s64 schedule_count{}; s64 last_scheduled_tick{}; QueueEntry per_core_priority_queue_entry[cpu::NumCores]{}; - QueueEntry sleeping_queue_entry{}; + KLightLock *waiting_lock{}; + KThreadQueue *sleeping_queue{}; - util::IntrusiveListNode waiter_list_node{}; - util::IntrusiveRedBlackTreeNode condvar_arbiter_tree_node{}; - util::IntrusiveListNode process_list_node{}; - - using WaiterListTraits = util::IntrusiveListMemberTraitsDeferredAssert<&KThread::waiter_list_node>; - using WaiterList = WaiterListTraits::ListType; - - using ConditionVariableThreadTreeTraits = util::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&KThread::condvar_arbiter_tree_node>; - using ConditionVariableThreadTree = ConditionVariableThreadTreeTraits::TreeType; WaiterList waiter_list{}; WaiterList pinned_waiter_list{}; KThread *lock_owner{}; - ConditionVariableThreadTree *condvar_tree{}; uintptr_t debug_params[3]{}; u32 address_key_value{}; u32 suspend_request_flags{}; u32 suspend_allowed_flags{}; Result wait_result; Result debug_exception_result; - s32 priority{}; - s32 current_core_id{}; - s32 core_id{}; s32 base_priority{}; s32 ideal_core_id{}; s32 num_kernel_waiters{}; + s32 current_core_id{}; + s32 core_id{}; KAffinityMask original_affinity_mask{}; s32 original_ideal_core_id{}; s32 num_core_migration_disables{}; ThreadState thread_state{}; std::atomic termination_requested{}; - bool ipc_cancelled{}; bool wait_cancelled{}; bool cancellable{}; - bool registered{}; bool signaled{}; bool initialized{}; bool debug_attached{}; @@ -393,8 +389,6 @@ namespace ams::kern { constexpr QueueEntry &GetPriorityQueueEntry(s32 core) { return this->per_core_priority_queue_entry[core]; } constexpr const QueueEntry &GetPriorityQueueEntry(s32 core) const { return this->per_core_priority_queue_entry[core]; } - constexpr QueueEntry &GetSleepingQueueEntry() { return this->sleeping_queue_entry; } - constexpr const QueueEntry &GetSleepingQueueEntry() const { return this->sleeping_queue_entry; } constexpr void SetSleepingQueue(KThreadQueue *q) { this->sleeping_queue = q; } constexpr ConditionVariableThreadTree *GetConditionVariableTree() const { return this->condvar_tree; } @@ -459,8 +453,6 @@ namespace ams::kern { constexpr KProcess *GetOwnerProcess() const { return this->parent; } constexpr bool IsUserThread() const { return this->parent != nullptr; } - constexpr uintptr_t GetEntrypoint() const { return this->entrypoint; } - constexpr KProcessAddress GetThreadLocalRegionAddress() const { return this->tls_address; } constexpr void *GetThreadLocalRegionHeapAddress() const { return this->tls_heap_address; } @@ -541,10 +533,6 @@ namespace ams::kern { virtual void OnTimer() override; virtual void DoWorkerTask() override; public: - static constexpr bool IsWaiterListValid() { - return WaiterListTraits::IsValid(); - } - static constexpr bool IsConditionVariableThreadTreeValid() { return ConditionVariableThreadTreeTraits::IsValid(); } @@ -555,7 +543,6 @@ namespace ams::kern { using ConditionVariableThreadTreeType = ConditionVariableThreadTree; }; static_assert(alignof(KThread) == 0x10); - static_assert(KThread::IsWaiterListValid()); static_assert(KThread::IsConditionVariableThreadTreeValid()); class KScopedDisableDispatch { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp index e56042750..fc18b564b 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread_queue.hpp @@ -21,92 +21,38 @@ namespace ams::kern { class KThreadQueue { private: - using Entry = KThread::QueueEntry; - private: - Entry root; + KThread::WaiterList wait_list; public: - constexpr ALWAYS_INLINE KThreadQueue() : root() { /* ... */ } + constexpr ALWAYS_INLINE KThreadQueue() : wait_list() { /* ... */ } - constexpr ALWAYS_INLINE bool IsEmpty() const { return this->root.GetNext() == nullptr; } + bool IsEmpty() const { return this->wait_list.empty(); } - constexpr ALWAYS_INLINE KThread *GetFront() const { return this->root.GetNext(); } - constexpr ALWAYS_INLINE KThread *GetNext(KThread *t) const { return t->GetSleepingQueueEntry().GetNext(); } - private: - constexpr ALWAYS_INLINE KThread *GetBack() const { return this->root.GetPrev(); } - - constexpr ALWAYS_INLINE void Enqueue(KThread *add) { - /* Get the entry associated with the added thread. */ - Entry &add_entry = add->GetSleepingQueueEntry(); - - /* Get the entry associated with the end of the queue. */ - KThread *tail = this->GetBack(); - Entry &tail_entry = (tail != nullptr) ? tail->GetSleepingQueueEntry() : this->root; - - /* Link the entries. */ - add_entry.SetPrev(tail); - add_entry.SetNext(nullptr); - tail_entry.SetNext(add); - this->root.SetPrev(add); - } - - constexpr ALWAYS_INLINE void Remove(KThread *remove) { - /* Get the entry associated with the thread. */ - Entry &remove_entry = remove->GetSleepingQueueEntry(); - - /* Get the entries associated with next and prev. */ - KThread *prev = remove_entry.GetPrev(); - KThread *next = remove_entry.GetNext(); - Entry &prev_entry = (prev != nullptr) ? prev->GetSleepingQueueEntry() : this->root; - Entry &next_entry = (next != nullptr) ? next->GetSleepingQueueEntry() : this->root; - - /* Unlink. */ - prev_entry.SetNext(next); - next_entry.SetPrev(prev); - } - public: - constexpr ALWAYS_INLINE void Dequeue() { - /* Get the front of the queue. */ - KThread *head = this->GetFront(); - if (head == nullptr) { - return; - } - - MESOSPHERE_ASSERT(head->GetState() == KThread::ThreadState_Waiting); - - /* Get the entry for the next head. */ - KThread *next = GetNext(head); - Entry &next_entry = (next != nullptr) ? next->GetSleepingQueueEntry() : this->root; - - /* Link the entries. */ - this->root.SetNext(next); - next_entry.SetPrev(nullptr); - - /* Clear the head's queue. */ - head->SetSleepingQueue(nullptr); - } + KThread::WaiterList::iterator begin() { return this->wait_list.begin(); } + KThread::WaiterList::iterator end() { return this->wait_list.end(); } bool SleepThread(KThread *t) { + KScopedSchedulerLock sl; + + /* If the thread needs terminating, don't enqueue it. */ + if (t->IsTerminationRequested()) { + return false; + } + /* Set the thread's queue and mark it as waiting. */ t->SetSleepingQueue(this); t->SetState(KThread::ThreadState_Waiting); /* Add the thread to the queue. */ - this->Enqueue(t); - - /* If the thread needs terminating, undo our work. */ - if (t->IsTerminationRequested()) { - this->WakeupThread(t); - return false; - } + this->wait_list.push_back(*t); return true; } void WakeupThread(KThread *t) { - MESOSPHERE_ASSERT(t->GetState() == KThread::ThreadState_Waiting); + KScopedSchedulerLock sl; /* Remove the thread from the queue. */ - this->Remove(t); + this->wait_list.erase(this->wait_list.iterator_to(*t)); /* Mark the thread as no longer sleeping. */ t->SetState(KThread::ThreadState_Runnable); @@ -114,18 +60,24 @@ namespace ams::kern { } KThread *WakeupFrontThread() { - KThread *front = this->GetFront(); - if (front != nullptr) { - MESOSPHERE_ASSERT(front->GetState() == KThread::ThreadState_Waiting); + KScopedSchedulerLock sl; + if (this->wait_list.empty()) { + return nullptr; + } else { /* Remove the thread from the queue. */ - this->Dequeue(); + auto it = this->wait_list.begin(); + KThread *thread = std::addressof(*it); + this->wait_list.erase(it); + + MESOSPHERE_ASSERT(thread->GetState() == KThread::ThreadState_Waiting); /* Mark the thread as no longer sleeping. */ - front->SetState(KThread::ThreadState_Runnable); - front->SetSleepingQueue(nullptr); + thread->SetState(KThread::ThreadState_Runnable); + thread->SetSleepingQueue(nullptr); + + return thread; } - return front; } }; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp index 80a0ed78c..16041e411 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_wait_object.hpp @@ -22,45 +22,13 @@ namespace ams::kern { class KWaitObject : public KTimerTask { private: - using Entry = KThread::QueueEntry; - private: - Entry root; + KThread::WaiterList wait_list; bool timer_used; public: - constexpr KWaitObject() : root(), timer_used() { /* ... */ } + constexpr KWaitObject() : wait_list(), timer_used() { /* ... */ } virtual void OnTimer() override; Result Synchronize(s64 timeout); - private: - constexpr ALWAYS_INLINE void Enqueue(KThread *add) { - /* Get the entry associated with the added thread. */ - Entry &add_entry = add->GetSleepingQueueEntry(); - - /* Get the entry associated with the end of the queue. */ - KThread *tail = this->root.GetPrev(); - Entry &tail_entry = (tail != nullptr) ? tail->GetSleepingQueueEntry() : this->root; - - /* Link the entries. */ - add_entry.SetPrev(tail); - add_entry.SetNext(nullptr); - tail_entry.SetNext(add); - this->root.SetPrev(add); - } - - constexpr ALWAYS_INLINE void Remove(KThread *remove) { - /* Get the entry associated with the thread. */ - Entry &remove_entry = remove->GetSleepingQueueEntry(); - - /* Get the entries associated with next and prev. */ - KThread *prev = remove_entry.GetPrev(); - KThread *next = remove_entry.GetNext(); - Entry &prev_entry = (prev != nullptr) ? prev->GetSleepingQueueEntry() : this->root; - Entry &next_entry = (next != nullptr) ? next->GetSleepingQueueEntry() : this->root; - - /* Unlink. */ - prev_entry.SetNext(next); - next_entry.SetPrev(prev); - } }; } diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp index 4009dce74..f31d95972 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp @@ -26,7 +26,7 @@ namespace ams::kern::arch::arm64 { /* Send KDebug event for this thread's creation. */ { KScopedInterruptEnable ei; - KDebug::OnDebugEvent(ams::svc::DebugEvent_CreateThread, GetCurrentThread().GetId(), GetInteger(GetCurrentThread().GetThreadLocalRegionAddress()), GetCurrentThread().GetEntrypoint()); + KDebug::OnDebugEvent(ams::svc::DebugEvent_CreateThread, GetCurrentThread().GetId(), GetInteger(GetCurrentThread().GetThreadLocalRegionAddress())); } /* Handle any pending dpc. */ diff --git a/libraries/libmesosphere/source/kern_k_debug_base.cpp b/libraries/libmesosphere/source/kern_k_debug_base.cpp index 601c1bb70..8f96f9b10 100644 --- a/libraries/libmesosphere/source/kern_k_debug_base.cpp +++ b/libraries/libmesosphere/source/kern_k_debug_base.cpp @@ -327,7 +327,7 @@ namespace ams::kern { it->SetDebugAttached(); /* Send the event. */ - this->PushDebugEvent(ams::svc::DebugEvent_CreateThread, it->GetId(), GetInteger(it->GetThreadLocalRegionAddress()), it->GetEntrypoint()); + this->PushDebugEvent(ams::svc::DebugEvent_CreateThread, it->GetId(), GetInteger(it->GetThreadLocalRegionAddress())); } } } @@ -682,7 +682,6 @@ namespace ams::kern { /* Set the thread creation info. */ info->info.create_thread.thread_id = param0; info->info.create_thread.tls_address = param1; - info->info.create_thread.entrypoint = param2; } break; case ams::svc::DebugEvent_ExitProcess: @@ -842,7 +841,6 @@ namespace ams::kern { { out->info.create_thread.thread_id = info->info.create_thread.thread_id; out->info.create_thread.tls_address = info->info.create_thread.tls_address; - out->info.create_thread.entrypoint = info->info.create_thread.entrypoint; } break; case ams::svc::DebugEvent_ExitProcess: diff --git a/libraries/libmesosphere/source/kern_k_light_server_session.cpp b/libraries/libmesosphere/source/kern_k_light_server_session.cpp index d049fee48..766b35f90 100644 --- a/libraries/libmesosphere/source/kern_k_light_server_session.cpp +++ b/libraries/libmesosphere/source/kern_k_light_server_session.cpp @@ -74,7 +74,7 @@ namespace ams::kern { /* If we can reply, do so. */ if (!this->current_request->IsTerminationRequested()) { MESOSPHERE_ASSERT(this->current_request->GetState() == KThread::ThreadState_Waiting); - MESOSPHERE_ASSERT(this->current_request == this->request_queue.GetFront()); + MESOSPHERE_ASSERT(this->request_queue.begin() != this->request_queue.end() && this->current_request == std::addressof(*this->request_queue.begin())); std::memcpy(this->current_request->GetLightSessionData(), server_thread->GetLightSessionData(), KLightSession::DataSize); this->request_queue.WakeupThread(this->current_request); } @@ -110,8 +110,8 @@ namespace ams::kern { R_UNLESS(!this->parent->IsServerClosed(), svc::ResultSessionClosed()); /* If we have a request available, use it. */ - if (this->current_request == nullptr && this->request_queue.IsEmpty()) { - this->current_request = this->request_queue.GetFront(); + if (this->current_request == nullptr && !this->request_queue.IsEmpty()) { + this->current_request = std::addressof(*this->request_queue.begin()); this->current_request->Open(); this->server_thread = server_thread; break; @@ -148,7 +148,7 @@ namespace ams::kern { /* Reply to the current request. */ if (!this->current_request->IsTerminationRequested()) { MESOSPHERE_ASSERT(this->current_request->GetState() == KThread::ThreadState_Waiting); - MESOSPHERE_ASSERT(this->current_request == this->request_queue.GetFront()); + MESOSPHERE_ASSERT(this->request_queue.begin() != this->request_queue.end() && this->current_request == std::addressof(*this->request_queue.begin())); this->request_queue.WakeupThread(this->current_request); this->current_request->SetSyncedObject(nullptr, svc::ResultSessionClosed()); } diff --git a/libraries/libmesosphere/source/kern_k_thread.cpp b/libraries/libmesosphere/source/kern_k_thread.cpp index 25c81bc68..2a484c7a2 100644 --- a/libraries/libmesosphere/source/kern_k_thread.cpp +++ b/libraries/libmesosphere/source/kern_k_thread.cpp @@ -98,7 +98,6 @@ namespace ams::kern { /* Set sync booleans. */ this->signaled = false; - this->ipc_cancelled = false; this->termination_requested = false; this->wait_cancelled = false; this->cancellable = false; @@ -119,7 +118,6 @@ namespace ams::kern { this->waiting_lock = nullptr; /* Initialize sleeping queue. */ - this->sleeping_queue_entry.Initialize(); this->sleeping_queue = nullptr; /* Set suspend flags. */ @@ -141,7 +139,6 @@ namespace ams::kern { /* We have no waiters, but we do have an entrypoint. */ this->num_kernel_waiters = 0; - this->entrypoint = reinterpret_cast(func); /* Set our current core id. */ this->current_core_id = core; @@ -172,7 +169,7 @@ namespace ams::kern { const bool is_64_bit = this->parent ? this->parent->Is64Bit() : IsDefault64Bit; const bool is_user = (type == ThreadType_User); const bool is_main = (type == ThreadType_Main); - this->thread_context.Initialize(this->entrypoint, reinterpret_cast(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main); + this->thread_context.Initialize(reinterpret_cast(func), reinterpret_cast(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main); /* Setup the stack parameters. */ StackParameters &sp = this->GetStackParameters(); diff --git a/libraries/libmesosphere/source/kern_k_wait_object.cpp b/libraries/libmesosphere/source/kern_k_wait_object.cpp index 6055cf17e..36f21c0b2 100644 --- a/libraries/libmesosphere/source/kern_k_wait_object.cpp +++ b/libraries/libmesosphere/source/kern_k_wait_object.cpp @@ -21,19 +21,8 @@ namespace ams::kern { MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); /* Wake up all the waiting threads. */ - Entry *entry = std::addressof(this->root); - while (true) { - /* Get the next thread. */ - KThread *thread = entry->GetNext(); - if (thread == nullptr) { - break; - } - - /* Wake it up. */ - thread->Wakeup(); - - /* Advance. */ - entry = std::addressof(thread->GetSleepingQueueEntry()); + for (KThread &thread : this->wait_list) { + thread.Wakeup(); } } @@ -73,7 +62,7 @@ namespace ams::kern { this->OnTimer(); } else { /* Otherwise, sleep until the timeout occurs. */ - this->Enqueue(cur_thread); + this->wait_list.push_back(GetCurrentThread()); cur_thread->SetState(KThread::ThreadState_Waiting); cur_thread->SetSyncedObject(nullptr, svc::ResultTimedOut()); } @@ -93,7 +82,7 @@ namespace ams::kern { /* Remove the thread from our queue. */ if (timeout != 0) { - this->Remove(cur_thread); + this->wait_list.erase(this->wait_list.iterator_to(GetCurrentThread())); } } diff --git a/libraries/libvapours/include/vapours/svc/svc_types_dmnt.hpp b/libraries/libvapours/include/vapours/svc/svc_types_dmnt.hpp index e65ab27c8..a0788d428 100644 --- a/libraries/libvapours/include/vapours/svc/svc_types_dmnt.hpp +++ b/libraries/libvapours/include/vapours/svc/svc_types_dmnt.hpp @@ -31,7 +31,7 @@ namespace ams::svc { struct DebugInfoCreateThread { u64 thread_id; u64 tls_address; - u64 entrypoint; + /* Removed in 11.0.0 u64 entrypoint; */ }; struct DebugInfoExitProcess {