kern: fix missing scheduler updates in KLightLock/Exception, fix RequestScheduleOnInterrupt

libraries/libmesosphere/include/mesosphere/kern_k_process.hpp

@@ -123,6 +123,20 @@ namespace ams::kern {
 
             void StartTermination();
             void FinishTermination();
+
+            void PinThread(s32 core_id, KThread *thread) {
+                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
+                MESOSPHERE_ASSERT(thread != nullptr);
+                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == nullptr);
+                this->pinned_threads[core_id] = thread;
+            }
+
+            void UnpinThread(s32 core_id, KThread *thread) {
+                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
+                MESOSPHERE_ASSERT(thread != nullptr);
+                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == thread);
+                this->pinned_threads[core_id] = nullptr;
+            }
         public:
             KProcess() { /* ... */ }
             virtual ~KProcess() { /* ... */ }
@@ -207,20 +221,6 @@ namespace ams::kern {
                 return this->pinned_threads[core_id];
             }
 
-            void PinThread(s32 core_id, KThread *thread) {
-                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
-                MESOSPHERE_ASSERT(thread != nullptr);
-                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == nullptr);
-                this->pinned_threads[core_id] = thread;
-            }
-
-            void UnpinThread(s32 core_id, KThread *thread) {
-                MESOSPHERE_ASSERT(0 <= core_id && core_id < static_cast<s32>(cpu::NumCores));
-                MESOSPHERE_ASSERT(thread != nullptr);
-                MESOSPHERE_ASSERT(this->pinned_threads[core_id] == thread);
-                this->pinned_threads[core_id] = nullptr;
-            }
-
             void CopySvcPermissionsTo(KThread::StackParameters &sp) {
                 this->capabilities.CopySvcPermissionsTo(sp);
             }
@@ -327,6 +327,7 @@ namespace ams::kern {
             Result SetActivity(ams::svc::ProcessActivity activity);
 
             void PinCurrentThread();
+            void UnpinCurrentThread();
 
             Result SignalToAddress(KProcessAddress address) {
                 return this->cond_var.SignalToAddress(address);

libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp

@@ -76,7 +76,7 @@ namespace ams::kern {
             }
 
             ALWAYS_INLINE void RequestScheduleOnInterrupt() {
-                SetSchedulerUpdateNeeded();
+                this->state.needs_scheduling = true;
 
                 if (CanSchedule()) {
                     this->ScheduleOnInterrupt();
@@ -100,11 +100,7 @@ namespace ams::kern {
             }
         private:
             /* Static private API. */
-            static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; }
-            static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; }
-            static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; }
             static ALWAYS_INLINE KSchedulerPriorityQueue &GetPriorityQueue() { return s_priority_queue; }
-
             static NOINLINE u64 UpdateHighestPriorityThreadsImpl();
 
             static NOINLINE void InterruptTaskThreadToRunnable();
@@ -113,6 +109,10 @@ namespace ams::kern {
             static ALWAYS_INLINE bool CanSchedule() { return GetCurrentThread().GetDisableDispatchCount() == 0; }
             static ALWAYS_INLINE bool IsSchedulerLockedByCurrentThread() { return s_scheduler_lock.IsLockedByCurrentThread(); }
 
+            static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; }
+            static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; }
+            static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; }
+
             static ALWAYS_INLINE void DisableScheduling() {
                 MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 0);
                 GetCurrentThread().DisableDispatch();
@@ -139,9 +139,6 @@ namespace ams::kern {
 
             static NOINLINE void ClearPreviousThread(KThread *thread);
 
-            static NOINLINE void PinCurrentThread(KProcess *cur_process);
-            static NOINLINE void UnpinCurrentThread(KProcess *cur_process);
-
             static NOINLINE void OnThreadStateChanged(KThread *thread, KThread::ThreadState old_state);
             static NOINLINE void OnThreadPriorityChanged(KThread *thread, s32 old_priority);
             static NOINLINE void OnThreadAffinityMaskChanged(KThread *thread, const KAffinityMask &old_affinity, s32 old_core);

libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp

@@ -511,7 +511,7 @@ namespace ams::kern::arch::arm64 {
                     KScopedSchedulerLock lk;
 
                     /* Pin the current thread. */
-                    KScheduler::PinCurrentThread(GetCurrentProcessPointer());
+                    GetCurrentProcess().PinCurrentThread();
 
                     /* Set the interrupt flag for the thread. */
                     GetCurrentThread().SetInterruptFlag();

libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp

@@ -180,7 +180,7 @@ namespace ams::kern::arch::arm64 {
                 KScopedSchedulerLock sl;
 
                 /* Pin the current thread. */
-                KScheduler::PinCurrentThread(GetCurrentProcessPointer());
+                GetCurrentProcess().PinCurrentThread();
 
                 /* Set the interrupt flag for the thread. */
                 GetCurrentThread().SetInterruptFlag();

libraries/libmesosphere/source/kern_k_light_lock.cpp

@@ -37,10 +37,13 @@ namespace ams::kern {
             /* Set thread states. */
             if (AMS_LIKELY(cur_thread->GetState() == KThread::ThreadState_Runnable)) {
                 cur_thread->SetState(KThread::ThreadState_Waiting);
+            } else {
+                KScheduler::SetSchedulerUpdateNeeded();
             }
 
             if (owner_thread->IsSuspended()) {
                 owner_thread->ContinueIfHasKernelWaiters();
+                KScheduler::SetSchedulerUpdateNeeded();
             }
         }
 
@@ -75,6 +78,8 @@ namespace ams::kern {
 
                 if (AMS_LIKELY(next_owner->GetState() == KThread::ThreadState_Waiting)) {
                     next_owner->SetState(KThread::ThreadState_Runnable);
+                } else {
+                    KScheduler::SetSchedulerUpdateNeeded();
                 }
 
                 if (next_owner->IsSuspended()) {

libraries/libmesosphere/source/kern_k_process.cpp

@@ -737,6 +737,8 @@ namespace ams::kern {
                     this->exception_thread->AddWaiter(cur_thread);
                     if (cur_thread->GetState() == KThread::ThreadState_Runnable) {
                         cur_thread->SetState(KThread::ThreadState_Waiting);
+                    } else {
+                        KScheduler::SetSchedulerUpdateNeeded();
                     }
                 }
                 /* Remove the thread as a waiter from the lock owner. */
@@ -745,6 +747,7 @@ namespace ams::kern {
                     KThread *owner_thread = cur_thread->GetLockOwner();
                     if (owner_thread != nullptr) {
                         owner_thread->RemoveWaiter(cur_thread);
+                        KScheduler::SetSchedulerUpdateNeeded();
                     }
                 }
             }
@@ -769,6 +772,8 @@ namespace ams::kern {
             if (next != nullptr) {
                 if (next->GetState() == KThread::ThreadState_Waiting) {
                     next->SetState(KThread::ThreadState_Runnable);
+                } else {
+                    KScheduler::SetSchedulerUpdateNeeded();
                 }
             }
 
@@ -980,6 +985,36 @@ namespace ams::kern {
         return ResultSuccess();
     }
 
+    void KProcess::PinCurrentThread() {
+        MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
+
+        /* Get the current thread. */
+        const s32 core_id   = GetCurrentCoreId();
+        KThread *cur_thread = GetCurrentThreadPointer();
+
+        /* Pin it. */
+        this->PinThread(core_id, cur_thread);
+        cur_thread->Pin();
+
+        /* An update is needed. */
+        KScheduler::SetSchedulerUpdateNeeded();
+    }
+
+    void KProcess::UnpinCurrentThread() {
+        MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread());
+
+        /* Get the current thread. */
+        const s32 core_id   = GetCurrentCoreId();
+        KThread *cur_thread = GetCurrentThreadPointer();
+
+        /* Unpin it. */
+        cur_thread->Unpin();
+        this->UnpinThread(core_id, cur_thread);
+
+        /* An update is needed. */
+        KScheduler::SetSchedulerUpdateNeeded();
+    }
+
     Result KProcess::GetThreadList(s32 *out_num_threads, ams::kern::svc::KUserPointer<u64 *> out_thread_ids, s32 max_out_count) {
         /* Lock the list. */
         KScopedLightLock lk(this->list_lock);

libraries/libmesosphere/source/kern_k_scheduler.cpp

@@ -277,36 +277,6 @@ namespace ams::kern {
         }
     }
 
-    void KScheduler::PinCurrentThread(KProcess *cur_process) {
-        MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread());
-
-        /* Get the current thread. */
-        const s32 core_id   = GetCurrentCoreId();
-        KThread *cur_thread = GetCurrentThreadPointer();
-
-        /* Pin it. */
-        cur_process->PinThread(core_id, cur_thread);
-        cur_thread->Pin();
-
-        /* An update is needed. */
-        SetSchedulerUpdateNeeded();
-    }
-
-    void KScheduler::UnpinCurrentThread(KProcess *cur_process) {
-        MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread());
-
-        /* Get the current thread. */
-        const s32 core_id   = GetCurrentCoreId();
-        KThread *cur_thread = GetCurrentThreadPointer();
-
-        /* Unpin it. */
-        cur_thread->Unpin();
-        cur_process->UnpinThread(core_id, cur_thread);
-
-        /* An update is needed. */
-        SetSchedulerUpdateNeeded();
-    }
-
     void KScheduler::OnThreadStateChanged(KThread *thread, KThread::ThreadState old_state) {
         MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread());
 

libraries/libmesosphere/source/kern_k_thread.cpp

@@ -328,7 +328,7 @@ namespace ams::kern {
         if (this->parent != nullptr) {
             this->parent->ReleaseUserException(this);
             if (this->parent->GetPinnedThread(GetCurrentCoreId()) == this) {
-                KScheduler::UnpinCurrentThread(this->parent);
+                this->parent->UnpinCurrentThread();
             }
         }
 

libraries/libmesosphere/source/svc/kern_svc_synchronization.cpp

@@ -123,7 +123,7 @@ namespace ams::kern::svc {
                 GetCurrentThread().ClearInterruptFlag();
 
                 /* Unpin the current thread. */
-                KScheduler::UnpinCurrentThread(cur_process);
+                cur_process->UnpinCurrentThread();
             }
         }