diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp index 48e6bfe46..5378b8012 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -123,6 +123,20 @@ namespace ams::kern { void StartTermination(); void FinishTermination(); + + void PinThread(s32 core_id, KThread *thread) { + MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast(cpu::NumCores)); + MESOSPHERE_ASSERT(thread != nullptr); + MESOSPHERE_ASSERT(this->pinned_threads[core_id] == nullptr); + this->pinned_threads[core_id] = thread; + } + + void UnpinThread(s32 core_id, KThread *thread) { + MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast(cpu::NumCores)); + MESOSPHERE_ASSERT(thread != nullptr); + MESOSPHERE_ASSERT(this->pinned_threads[core_id] == thread); + this->pinned_threads[core_id] = nullptr; + } public: KProcess() { /* ... */ } virtual ~KProcess() { /* ... */ } @@ -207,20 +221,6 @@ namespace ams::kern { return this->pinned_threads[core_id]; } - void PinThread(s32 core_id, KThread *thread) { - MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast(cpu::NumCores)); - MESOSPHERE_ASSERT(thread != nullptr); - MESOSPHERE_ASSERT(this->pinned_threads[core_id] == nullptr); - this->pinned_threads[core_id] = thread; - } - - void UnpinThread(s32 core_id, KThread *thread) { - MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast(cpu::NumCores)); - MESOSPHERE_ASSERT(thread != nullptr); - MESOSPHERE_ASSERT(this->pinned_threads[core_id] == thread); - this->pinned_threads[core_id] = nullptr; - } - void CopySvcPermissionsTo(KThread::StackParameters &sp) { this->capabilities.CopySvcPermissionsTo(sp); } @@ -327,6 +327,7 @@ namespace ams::kern { Result SetActivity(ams::svc::ProcessActivity activity); void PinCurrentThread(); + void UnpinCurrentThread(); Result SignalToAddress(KProcessAddress address) { return this->cond_var.SignalToAddress(address); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp index 27ea0a1ef..b38956fb2 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp @@ -76,7 +76,7 @@ namespace ams::kern { } ALWAYS_INLINE void RequestScheduleOnInterrupt() { - SetSchedulerUpdateNeeded(); + this->state.needs_scheduling = true; if (CanSchedule()) { this->ScheduleOnInterrupt(); @@ -100,11 +100,7 @@ namespace ams::kern { } private: /* Static private API. */ - static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; } - static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; } - static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; } static ALWAYS_INLINE KSchedulerPriorityQueue &GetPriorityQueue() { return s_priority_queue; } - static NOINLINE u64 UpdateHighestPriorityThreadsImpl(); static NOINLINE void InterruptTaskThreadToRunnable(); @@ -113,6 +109,10 @@ namespace ams::kern { static ALWAYS_INLINE bool CanSchedule() { return GetCurrentThread().GetDisableDispatchCount() == 0; } static ALWAYS_INLINE bool IsSchedulerLockedByCurrentThread() { return s_scheduler_lock.IsLockedByCurrentThread(); } + static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; } + static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; } + static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; } + static ALWAYS_INLINE void DisableScheduling() { MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 0); GetCurrentThread().DisableDispatch(); @@ -139,9 +139,6 @@ namespace ams::kern { static NOINLINE void ClearPreviousThread(KThread *thread); - static NOINLINE void PinCurrentThread(KProcess *cur_process); - static NOINLINE void UnpinCurrentThread(KProcess *cur_process); - static NOINLINE void OnThreadStateChanged(KThread *thread, KThread::ThreadState old_state); static NOINLINE void OnThreadPriorityChanged(KThread *thread, s32 old_priority); static NOINLINE void OnThreadAffinityMaskChanged(KThread *thread, const KAffinityMask &old_affinity, s32 old_core); diff --git a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp index b728ca612..263a304ea 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp @@ -511,7 +511,7 @@ namespace ams::kern::arch::arm64 { KScopedSchedulerLock lk; /* Pin the current thread. */ - KScheduler::PinCurrentThread(GetCurrentProcessPointer()); + GetCurrentProcess().PinCurrentThread(); /* Set the interrupt flag for the thread. */ GetCurrentThread().SetInterruptFlag(); diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp index f1995e103..cda867fe5 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp @@ -180,7 +180,7 @@ namespace ams::kern::arch::arm64 { KScopedSchedulerLock sl; /* Pin the current thread. */ - KScheduler::PinCurrentThread(GetCurrentProcessPointer()); + GetCurrentProcess().PinCurrentThread(); /* Set the interrupt flag for the thread. */ GetCurrentThread().SetInterruptFlag(); diff --git a/libraries/libmesosphere/source/kern_k_light_lock.cpp b/libraries/libmesosphere/source/kern_k_light_lock.cpp index e0a317539..a085e1ce7 100644 --- a/libraries/libmesosphere/source/kern_k_light_lock.cpp +++ b/libraries/libmesosphere/source/kern_k_light_lock.cpp @@ -37,10 +37,13 @@ namespace ams::kern { /* Set thread states. */ if (AMS_LIKELY(cur_thread->GetState() == KThread::ThreadState_Runnable)) { cur_thread->SetState(KThread::ThreadState_Waiting); + } else { + KScheduler::SetSchedulerUpdateNeeded(); } if (owner_thread->IsSuspended()) { owner_thread->ContinueIfHasKernelWaiters(); + KScheduler::SetSchedulerUpdateNeeded(); } } @@ -75,6 +78,8 @@ namespace ams::kern { if (AMS_LIKELY(next_owner->GetState() == KThread::ThreadState_Waiting)) { next_owner->SetState(KThread::ThreadState_Runnable); + } else { + KScheduler::SetSchedulerUpdateNeeded(); } if (next_owner->IsSuspended()) { diff --git a/libraries/libmesosphere/source/kern_k_process.cpp b/libraries/libmesosphere/source/kern_k_process.cpp index e5bf37f62..499dd6392 100644 --- a/libraries/libmesosphere/source/kern_k_process.cpp +++ b/libraries/libmesosphere/source/kern_k_process.cpp @@ -737,6 +737,8 @@ namespace ams::kern { this->exception_thread->AddWaiter(cur_thread); if (cur_thread->GetState() == KThread::ThreadState_Runnable) { cur_thread->SetState(KThread::ThreadState_Waiting); + } else { + KScheduler::SetSchedulerUpdateNeeded(); } } /* Remove the thread as a waiter from the lock owner. */ @@ -745,6 +747,7 @@ namespace ams::kern { KThread *owner_thread = cur_thread->GetLockOwner(); if (owner_thread != nullptr) { owner_thread->RemoveWaiter(cur_thread); + KScheduler::SetSchedulerUpdateNeeded(); } } } @@ -769,6 +772,8 @@ namespace ams::kern { if (next != nullptr) { if (next->GetState() == KThread::ThreadState_Waiting) { next->SetState(KThread::ThreadState_Runnable); + } else { + KScheduler::SetSchedulerUpdateNeeded(); } } @@ -980,6 +985,36 @@ namespace ams::kern { return ResultSuccess(); } + void KProcess::PinCurrentThread() { + MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); + + /* Get the current thread. */ + const s32 core_id = GetCurrentCoreId(); + KThread *cur_thread = GetCurrentThreadPointer(); + + /* Pin it. */ + this->PinThread(core_id, cur_thread); + cur_thread->Pin(); + + /* An update is needed. */ + KScheduler::SetSchedulerUpdateNeeded(); + } + + void KProcess::UnpinCurrentThread() { + MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); + + /* Get the current thread. */ + const s32 core_id = GetCurrentCoreId(); + KThread *cur_thread = GetCurrentThreadPointer(); + + /* Unpin it. */ + cur_thread->Unpin(); + this->UnpinThread(core_id, cur_thread); + + /* An update is needed. */ + KScheduler::SetSchedulerUpdateNeeded(); + } + Result KProcess::GetThreadList(s32 *out_num_threads, ams::kern::svc::KUserPointer out_thread_ids, s32 max_out_count) { /* Lock the list. */ KScopedLightLock lk(this->list_lock); diff --git a/libraries/libmesosphere/source/kern_k_scheduler.cpp b/libraries/libmesosphere/source/kern_k_scheduler.cpp index 05a25e1fa..233515868 100644 --- a/libraries/libmesosphere/source/kern_k_scheduler.cpp +++ b/libraries/libmesosphere/source/kern_k_scheduler.cpp @@ -277,36 +277,6 @@ namespace ams::kern { } } - void KScheduler::PinCurrentThread(KProcess *cur_process) { - MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread()); - - /* Get the current thread. */ - const s32 core_id = GetCurrentCoreId(); - KThread *cur_thread = GetCurrentThreadPointer(); - - /* Pin it. */ - cur_process->PinThread(core_id, cur_thread); - cur_thread->Pin(); - - /* An update is needed. */ - SetSchedulerUpdateNeeded(); - } - - void KScheduler::UnpinCurrentThread(KProcess *cur_process) { - MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread()); - - /* Get the current thread. */ - const s32 core_id = GetCurrentCoreId(); - KThread *cur_thread = GetCurrentThreadPointer(); - - /* Unpin it. */ - cur_thread->Unpin(); - cur_process->UnpinThread(core_id, cur_thread); - - /* An update is needed. */ - SetSchedulerUpdateNeeded(); - } - void KScheduler::OnThreadStateChanged(KThread *thread, KThread::ThreadState old_state) { MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread()); diff --git a/libraries/libmesosphere/source/kern_k_thread.cpp b/libraries/libmesosphere/source/kern_k_thread.cpp index ae84126fa..a1cbbbe8a 100644 --- a/libraries/libmesosphere/source/kern_k_thread.cpp +++ b/libraries/libmesosphere/source/kern_k_thread.cpp @@ -328,7 +328,7 @@ namespace ams::kern { if (this->parent != nullptr) { this->parent->ReleaseUserException(this); if (this->parent->GetPinnedThread(GetCurrentCoreId()) == this) { - KScheduler::UnpinCurrentThread(this->parent); + this->parent->UnpinCurrentThread(); } } diff --git a/libraries/libmesosphere/source/svc/kern_svc_synchronization.cpp b/libraries/libmesosphere/source/svc/kern_svc_synchronization.cpp index 1db753eea..7c4b8c8e3 100644 --- a/libraries/libmesosphere/source/svc/kern_svc_synchronization.cpp +++ b/libraries/libmesosphere/source/svc/kern_svc_synchronization.cpp @@ -123,7 +123,7 @@ namespace ams::kern::svc { GetCurrentThread().ClearInterruptFlag(); /* Unpin the current thread. */ - KScheduler::UnpinCurrentThread(cur_process); + cur_process->UnpinCurrentThread(); } }