diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp index e84b7a228..50a25ad63 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp @@ -47,23 +47,23 @@ namespace ams::kern::arch::arm64 { constexpr KGlobalInterruptEntry() : handler(nullptr), manually_cleared(false), needs_clear(false) { /* ... */ } }; private: - static KSpinLock s_lock; - static std::array s_global_interrupts; - static KInterruptController::GlobalState s_global_state; - static bool s_global_state_saved; + KCoreLocalInterruptEntry core_local_interrupts[cpu::NumCores][KInterruptController::NumLocalInterrupts]{}; + KInterruptController interrupt_controller{}; + KInterruptController::LocalState local_states[cpu::NumCores]{}; + bool local_state_saved[cpu::NumCores]{}; + mutable KSpinLock global_interrupt_lock{}; + KGlobalInterruptEntry global_interrupts[KInterruptController::NumGlobalInterrupts]{}; + KInterruptController::GlobalState global_state{}; + bool global_state_saved{}; private: - KCoreLocalInterruptEntry core_local_interrupts[KInterruptController::NumLocalInterrupts]; - KInterruptController interrupt_controller; - KInterruptController::LocalState local_state; - bool local_state_saved; - private: - static ALWAYS_INLINE KSpinLock &GetLock() { return s_lock; } - static ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return s_global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; } - ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[KInterruptController::GetLocalInterruptIndex(irq)]; } + ALWAYS_INLINE KSpinLock &GetGlobalInterruptLock() const { return this->global_interrupt_lock; } + ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return this->global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; } + ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[GetCurrentCoreId()][KInterruptController::GetLocalInterruptIndex(irq)]; } bool OnHandleInterrupt(); public: - constexpr KInterruptManager() : core_local_interrupts(), interrupt_controller(), local_state(), local_state_saved(false) { /* ... */ } + constexpr KInterruptManager() = default; + NOINLINE void Initialize(s32 core_id); NOINLINE void Finalize(s32 core_id); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp index 8b4b74a15..d9d4fd32d 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp @@ -29,7 +29,6 @@ namespace ams::kern { KCurrentContext current; KScheduler scheduler; KInterruptTaskManager interrupt_task_manager; - KInterruptManager interrupt_manager; KHardwareTimer hardware_timer; /* Everything after this point is for debugging. */ /* Retail kernel doesn't even consistently update these fields. */ diff --git a/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp b/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp index eb98eaa9c..67e781721 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp @@ -37,8 +37,6 @@ namespace ams::kern { class KSynchronization; class KUnsafeMemory; - - #if defined(ATMOSPHERE_ARCH_ARM64) namespace arch::arm64 { @@ -75,6 +73,7 @@ namespace ams::kern { static KSynchronization s_synchronization; static KUnsafeMemory s_unsafe_memory; static KWorkerTaskManager s_worker_task_managers[KWorkerTaskManager::WorkerType_Count]; + static KInterruptManager s_interrupt_manager; private: static ALWAYS_INLINE KCoreLocalContext &GetCoreLocalContext() { return reinterpret_cast(cpu::GetCoreLocalRegionAddress())->current.context; @@ -111,7 +110,7 @@ namespace ams::kern { } static ALWAYS_INLINE KInterruptManager &GetInterruptManager() { - return GetCoreLocalContext().interrupt_manager; + return s_interrupt_manager; } static ALWAYS_INLINE KHardwareTimer &GetHardwareTimer() { diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp index 51b2a4b36..a81aba58e 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp @@ -17,12 +17,6 @@ namespace ams::kern::arch::arm64 { - /* Instantiate static members in specific translation unit. */ - KSpinLock KInterruptManager::s_lock; - std::array KInterruptManager::s_global_interrupts; - KInterruptController::GlobalState KInterruptManager::s_global_state; - bool KInterruptManager::s_global_state_saved; - void KInterruptManager::Initialize(s32 core_id) { this->interrupt_controller.Initialize(core_id); } @@ -32,23 +26,26 @@ namespace ams::kern::arch::arm64 { } void KInterruptManager::Save(s32 core_id) { + /* Verify core id. */ + MESOSPHERE_ASSERT(core_id == GetCurrentCoreId()); + /* Ensure all cores get to this point before continuing. */ cpu::SynchronizeAllCores(); /* If on core 0, save the global interrupts. */ if (core_id == 0) { - MESOSPHERE_ABORT_UNLESS(!s_global_state_saved); - this->interrupt_controller.SaveGlobal(std::addressof(s_global_state)); - s_global_state_saved = true; + MESOSPHERE_ABORT_UNLESS(!this->global_state_saved); + this->interrupt_controller.SaveGlobal(std::addressof(this->global_state)); + this->global_state_saved = true; } /* Ensure all cores get to this point before continuing. */ cpu::SynchronizeAllCores(); /* Save all local interrupts. */ - MESOSPHERE_ABORT_UNLESS(!this->local_state_saved); - this->interrupt_controller.SaveCoreLocal(std::addressof(this->local_state)); - this->local_state_saved = true; + MESOSPHERE_ABORT_UNLESS(!this->local_state_saved[core_id]); + this->interrupt_controller.SaveCoreLocal(std::addressof(this->local_states[core_id])); + this->local_state_saved[core_id] = true; /* Ensure all cores get to this point before continuing. */ cpu::SynchronizeAllCores(); @@ -68,6 +65,9 @@ namespace ams::kern::arch::arm64 { } void KInterruptManager::Restore(s32 core_id) { + /* Verify core id. */ + MESOSPHERE_ASSERT(core_id == GetCurrentCoreId()); + /* Ensure all cores get to this point before continuing. */ cpu::SynchronizeAllCores(); @@ -88,18 +88,18 @@ namespace ams::kern::arch::arm64 { cpu::SynchronizeAllCores(); /* Restore all local interrupts. */ - MESOSPHERE_ASSERT(this->local_state_saved); - this->interrupt_controller.RestoreCoreLocal(std::addressof(this->local_state)); - this->local_state_saved = false; + MESOSPHERE_ASSERT(this->local_state_saved[core_id]); + this->interrupt_controller.RestoreCoreLocal(std::addressof(this->local_states[core_id])); + this->local_state_saved[core_id] = false; /* Ensure all cores get to this point before continuing. */ cpu::SynchronizeAllCores(); /* If on core 0, restore the global interrupts. */ if (core_id == 0) { - MESOSPHERE_ASSERT(s_global_state_saved); - this->interrupt_controller.RestoreGlobal(std::addressof(s_global_state)); - s_global_state_saved = false; + MESOSPHERE_ASSERT(this->global_state_saved); + this->interrupt_controller.RestoreGlobal(std::addressof(this->global_state)); + this->global_state_saved = false; } /* Ensure all cores get to this point before continuing. */ @@ -136,7 +136,7 @@ namespace ams::kern::arch::arm64 { MESOSPHERE_LOG("Core%d: Unhandled local interrupt %d\n", GetCurrentCoreId(), irq); } } else if (KInterruptController::IsGlobal(irq)) { - KScopedSpinLock lk(GetLock()); + KScopedSpinLock lk(this->GetGlobalInterruptLock()); /* Get global interrupt entry. */ auto &entry = GetGlobalInterruptEntry(irq); @@ -213,7 +213,7 @@ namespace ams::kern::arch::arm64 { KScopedInterruptDisable di; if (KInterruptController::IsGlobal(irq)) { - KScopedSpinLock lk(GetLock()); + KScopedSpinLock lk(this->GetGlobalInterruptLock()); return this->BindGlobal(handler, irq, core_id, priority, manual_clear, level); } else { MESOSPHERE_ASSERT(core_id == GetCurrentCoreId()); @@ -227,7 +227,7 @@ namespace ams::kern::arch::arm64 { KScopedInterruptDisable di; if (KInterruptController::IsGlobal(irq)) { - KScopedSpinLock lk(GetLock()); + KScopedSpinLock lk(this->GetGlobalInterruptLock()); return this->UnbindGlobal(irq); } else { MESOSPHERE_ASSERT(core_id == GetCurrentCoreId()); @@ -239,7 +239,7 @@ namespace ams::kern::arch::arm64 { R_UNLESS(KInterruptController::IsGlobal(irq), svc::ResultOutOfRange()); KScopedInterruptDisable di; - KScopedSpinLock lk(GetLock()); + KScopedSpinLock lk(this->GetGlobalInterruptLock()); return this->ClearGlobal(irq); } @@ -249,7 +249,7 @@ namespace ams::kern::arch::arm64 { KScopedInterruptDisable di; if (KInterruptController::IsGlobal(irq)) { - KScopedSpinLock lk(GetLock()); + KScopedSpinLock lk(this->GetGlobalInterruptLock()); return this->ClearGlobal(irq); } else { MESOSPHERE_ASSERT(core_id == GetCurrentCoreId()); diff --git a/mesosphere/kernel/source/kern_kernel_instantiations.cpp b/mesosphere/kernel/source/kern_kernel_instantiations.cpp index 5bed676ec..818dc4254 100644 --- a/mesosphere/kernel/source/kern_kernel_instantiations.cpp +++ b/mesosphere/kernel/source/kern_kernel_instantiations.cpp @@ -29,6 +29,7 @@ namespace ams::kern { constinit KSynchronization Kernel::s_synchronization; constinit KUnsafeMemory Kernel::s_unsafe_memory; constinit KWorkerTaskManager Kernel::s_worker_task_managers[KWorkerTaskManager::WorkerType_Count]; + constinit KInterruptManager Kernel::s_interrupt_manager; namespace {