/* * Copyright (c) Atmosphère-NX * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #pragma once #include #include #include #include #include namespace ams::kern { using KSchedulerPriorityQueue = KPriorityQueue; static_assert(std::is_same::value); static_assert(KSchedulerPriorityQueue::NumCores == cpu::NumCores); static_assert(KSchedulerPriorityQueue::NumPriority == BITSIZEOF(u64)); class KScopedSchedulerLock; class KScopedSchedulerLockAndSleep; class KScheduler { NON_COPYABLE(KScheduler); NON_MOVEABLE(KScheduler); public: static constexpr s32 HighestCoreMigrationAllowedPriority = 2; static_assert(ams::svc::LowestThreadPriority >= HighestCoreMigrationAllowedPriority); static_assert(ams::svc::HighestThreadPriority <= HighestCoreMigrationAllowedPriority); struct SchedulingState { util::Atomic needs_scheduling{false}; bool interrupt_task_runnable{false}; bool should_count_idle{false}; u64 idle_count{0}; KThread *highest_priority_thread{nullptr}; void *idle_thread_stack{nullptr}; util::Atomic prev_thread{nullptr}; KInterruptTaskManager *interrupt_task_manager{nullptr}; constexpr SchedulingState() = default; }; private: friend class KScopedSchedulerLock; friend class KScopedSchedulerLockAndSleep; friend class KScopedDisableDispatch; private: SchedulingState m_state; bool m_is_active; s32 m_core_id; s64 m_last_context_switch_time; KThread *m_idle_thread; util::Atomic m_current_thread; public: constexpr KScheduler() : m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr) { m_state.needs_scheduling = true; m_state.interrupt_task_runnable = false; m_state.should_count_idle = false; m_state.idle_count = 0; m_state.idle_thread_stack = nullptr; m_state.highest_priority_thread = nullptr; m_state.prev_thread = nullptr; m_state.interrupt_task_manager = nullptr; } NOINLINE void Initialize(KThread *idle_thread); NOINLINE void Activate(); ALWAYS_INLINE void SetInterruptTaskRunnable() { m_state.interrupt_task_runnable = true; m_state.needs_scheduling = true; } ALWAYS_INLINE void RequestScheduleOnInterrupt() { m_state.needs_scheduling = true; if (CanSchedule()) { this->ScheduleOnInterrupt(); } } ALWAYS_INLINE u64 GetIdleCount() const { return m_state.idle_count; } ALWAYS_INLINE KThread *GetIdleThread() const { return m_idle_thread; } ALWAYS_INLINE KThread *GetPreviousThread() const { return m_state.prev_thread.Load(); } ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const { return m_current_thread.Load(); } ALWAYS_INLINE s64 GetLastContextSwitchTime() const { return m_last_context_switch_time; } private: /* Static private API. */ static ALWAYS_INLINE KSchedulerPriorityQueue &GetPriorityQueue() { return s_priority_queue; } static NOINLINE u64 UpdateHighestPriorityThreadsImpl(); public: /* Static public API. */ static ALWAYS_INLINE bool CanSchedule() { return GetCurrentThread().GetDisableDispatchCount() == 0; } static ALWAYS_INLINE bool IsSchedulerLockedByCurrentThread() { return s_scheduler_lock.IsLockedByCurrentThread(); } static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; } static ALWAYS_INLINE void SetSchedulerUpdateNeeded() { s_scheduler_update_needed = true; } static ALWAYS_INLINE void ClearSchedulerUpdateNeeded() { s_scheduler_update_needed = false; } static ALWAYS_INLINE void DisableScheduling() { MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 0); GetCurrentThread().DisableDispatch(); } static ALWAYS_INLINE void EnableScheduling(u64 cores_needing_scheduling) { MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 1); GetCurrentScheduler().RescheduleOtherCores(cores_needing_scheduling); if (GetCurrentThread().GetDisableDispatchCount() > 1) { GetCurrentThread().EnableDispatch(); } else { GetCurrentScheduler().RescheduleCurrentCore(); } } static ALWAYS_INLINE u64 UpdateHighestPriorityThreads() { if (IsSchedulerUpdateNeeded()) { return UpdateHighestPriorityThreadsImpl(); } else { return 0; } } static NOINLINE void ClearPreviousThread(KThread *thread); static NOINLINE void OnThreadStateChanged(KThread *thread, KThread::ThreadState old_state); static NOINLINE void OnThreadPriorityChanged(KThread *thread, s32 old_priority); static NOINLINE void OnThreadAffinityMaskChanged(KThread *thread, const KAffinityMask &old_affinity, s32 old_core); static NOINLINE void RotateScheduledQueue(s32 priority, s32 core_id); static NOINLINE void YieldWithoutCoreMigration(); static NOINLINE void YieldWithCoreMigration(); static NOINLINE void YieldToAnyThread(); private: /* Instanced private API. */ void ScheduleImpl(); void SwitchThread(KThread *next_thread); ALWAYS_INLINE void Schedule() { MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() == 1); MESOSPHERE_ASSERT(m_core_id == GetCurrentCoreId()); this->ScheduleImpl(); } ALWAYS_INLINE void ScheduleOnInterrupt() { GetCurrentThread().DisableDispatch(); this->Schedule(); GetCurrentThread().EnableDispatch(); } void RescheduleOtherCores(u64 cores_needing_scheduling); ALWAYS_INLINE void RescheduleCurrentCore() { MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() == 1); GetCurrentThread().EnableDispatch(); if (m_state.needs_scheduling.Load()) { /* Disable interrupts, and then check again if rescheduling is needed. */ KScopedInterruptDisable intr_disable; GetCurrentScheduler().RescheduleCurrentCoreImpl(); } } ALWAYS_INLINE void RescheduleCurrentCoreImpl() { /* Check that scheduling is needed. */ if (AMS_LIKELY(m_state.needs_scheduling.Load())) { GetCurrentThread().DisableDispatch(); this->Schedule(); GetCurrentThread().EnableDispatch(); } } NOINLINE u64 UpdateHighestPriorityThread(KThread *thread); public: using LockType = KAbstractSchedulerLock; private: static bool s_scheduler_update_needed; static KSchedulerPriorityQueue s_priority_queue; static LockType s_scheduler_lock; public: static consteval bool ValidateAssemblyOffsets(); }; consteval bool KScheduler::ValidateAssemblyOffsets() { static_assert(AMS_OFFSETOF(KScheduler, m_state.needs_scheduling) == KSCHEDULER_NEEDS_SCHEDULING); static_assert(AMS_OFFSETOF(KScheduler, m_state.interrupt_task_runnable) == KSCHEDULER_INTERRUPT_TASK_RUNNABLE); static_assert(AMS_OFFSETOF(KScheduler, m_state.highest_priority_thread) == KSCHEDULER_HIGHEST_PRIORITY_THREAD); static_assert(AMS_OFFSETOF(KScheduler, m_state.idle_thread_stack) == KSCHEDULER_IDLE_THREAD_STACK); static_assert(AMS_OFFSETOF(KScheduler, m_state.prev_thread) == KSCHEDULER_PREVIOUS_THREAD); static_assert(AMS_OFFSETOF(KScheduler, m_state.interrupt_task_manager) == KSCHEDULER_INTERRUPT_TASK_MANAGER); return true; } static_assert(KScheduler::ValidateAssemblyOffsets()); class KScopedSchedulerLock : KScopedLock { public: explicit ALWAYS_INLINE KScopedSchedulerLock() : KScopedLock(KScheduler::s_scheduler_lock) { /* ... */ } ALWAYS_INLINE ~KScopedSchedulerLock() { /* ... */ } }; }