From 919b8124dc340402c2220d38ed7bdab6d00a46fd Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Sat, 8 Feb 2020 02:49:32 -0800 Subject: [PATCH] kern: Implement exception vector ASM --- .../libmesosphere/include/mesosphere.hpp | 4 +- .../arch/arm64/kern_cpu_system_registers.hpp | 23 + .../arm64/kern_k_interrupt_controller.hpp | 8 + .../arch/arm64/kern_k_interrupt_manager.hpp | 4 + .../arch/arm64/kern_k_thread_context.hpp | 2 + .../arm64/kern_userspace_memory_access.hpp | 25 + .../include/mesosphere/kern_common.hpp | 4 + .../mesosphere/kern_k_current_context.hpp | 2 +- .../kern_k_interrupt_task_manager.hpp | 6 +- .../mesosphere/kern_k_memory_layout.hpp | 4 +- .../include/mesosphere/kern_k_process.hpp | 4 +- .../include/mesosphere/kern_k_scheduler.hpp | 13 + .../include/mesosphere/kern_k_thread.hpp | 51 +- .../include/mesosphere/kern_panic.hpp | 4 +- .../mesosphere/svc/kern_svc_tables.hpp | 2 +- .../arch/arm64/kern_exception_handlers.cpp | 166 +++++ .../arch/arm64/kern_exception_handlers_asm.s | 610 ++++++++++++++++++ .../arch/arm64/kern_k_interrupt_manager.cpp | 95 +++ .../arch/arm64/kern_k_thread_context.cpp | 26 + .../arm64/kern_userspace_memory_access_asm.s | 35 + .../arch/arm64/svc/kern_svc_handlers_asm.s | 343 ++++++++++ .../source/arch/arm64/svc/kern_svc_tables.cpp | 33 +- .../libmesosphere/source/kern_k_process.cpp | 24 + .../libmesosphere/source/kern_k_scheduler.cpp | 6 +- .../libmesosphere/source/kern_kernel.cpp | 2 +- .../source/arch/arm64/exception_vectors.s | 61 +- 26 files changed, 1497 insertions(+), 60 deletions(-) create mode 100644 libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp create mode 100644 libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp create mode 100644 libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s create mode 100644 libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s create mode 100644 libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s create mode 100644 libraries/libmesosphere/source/kern_k_process.cpp diff --git a/libraries/libmesosphere/include/mesosphere.hpp b/libraries/libmesosphere/include/mesosphere.hpp index f5019be44..2950f09e0 100644 --- a/libraries/libmesosphere/include/mesosphere.hpp +++ b/libraries/libmesosphere/include/mesosphere.hpp @@ -15,14 +15,12 @@ */ #pragma once -#define MESOSPHERE_BUILD_FOR_AUDITING - /* All kernel code should have access to libvapours. */ #include /* First, pull in core macros (panic, etc). */ -#include #include +#include /* Primitive types. */ #include diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp index 8cca7ec86..f2c39edf4 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp @@ -43,6 +43,7 @@ namespace ams::kern::arm64::cpu { MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(VbarEl1, vbar_el1) + MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(FarEl1, far_el1) MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(ParEl1, par_el1) MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(SctlrEl1, sctlr_el1) @@ -56,6 +57,10 @@ namespace ams::kern::arm64::cpu { MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(TpidrRoEl0, tpidrro_el0) + MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(EsrEl1, esr_el1) + MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr0El1, afsr0_el1) + MESOSPHERE_CPU_DEFINE_SYSREG_ACCESSORS(Afsr1El1, afsr1_el1) + #define FOR_I_IN_0_TO_15(HANDLER, ...) \ HANDLER(0, ## __VA_ARGS__) HANDLER(1, ## __VA_ARGS__) HANDLER(2, ## __VA_ARGS__) HANDLER(3, ## __VA_ARGS__) \ HANDLER(4, ## __VA_ARGS__) HANDLER(5, ## __VA_ARGS__) HANDLER(6, ## __VA_ARGS__) HANDLER(7, ## __VA_ARGS__) \ @@ -139,6 +144,24 @@ namespace ams::kern::arm64::cpu { } }; + MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(ArchitecturalFeatureAccessControl) { + public: + MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(ArchitecturalFeatureAccessControl, cpacr_el1) + + constexpr ALWAYS_INLINE decltype(auto) SetFpEnabled(bool en) { + if (en) { + this->SetBits(20, 2, 0x3); + } else { + this->SetBits(20, 2, 0x0); + } + return *this; + } + + constexpr ALWAYS_INLINE bool IsFpEnabled() { + return this->GetBits(20, 2) != 0; + } + }; + MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(DebugFeature) { public: MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS_FUNCTIONS(DebugFeature, id_aa64dfr0_el1) diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp index 63a70244f..b2780cc85 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_controller.hpp @@ -143,6 +143,14 @@ namespace ams::kern::arm64 { void Initialize(s32 core_id); void Finalize(s32 core_id); public: + u32 GetIrq() const { + return this->gicc->iar; + } + + static constexpr s32 ConvertRawIrq(u32 irq) { + return (irq == 0x3FF) ? -1 : (irq & 0x3FF); + } + void Enable(s32 irq) const { this->gicd->isenabler[irq / BITSIZEOF(u32)] = (1u << (irq % BITSIZEOF(u32))); } diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp index c885c81b3..26e5cc705 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp @@ -60,6 +60,8 @@ namespace ams::kern::arm64 { static ALWAYS_INLINE KSpinLock &GetLock() { return s_lock; } static ALWAYS_INLINE KGlobalInterruptEntry &GetGlobalInterruptEntry(s32 irq) { return s_global_interrupts[KInterruptController::GetGlobalInterruptIndex(irq)]; } ALWAYS_INLINE KCoreLocalInterruptEntry &GetLocalInterruptEntry(s32 irq) { return this->core_local_interrupts[KInterruptController::GetLocalInterruptIndex(irq)]; } + + bool OnHandleInterrupt(); public: constexpr KInterruptManager() : core_local_interrupts(), interrupt_controller(), local_state(), local_state_saved(false) { /* ... */ } NOINLINE void Initialize(s32 core_id); @@ -79,6 +81,8 @@ namespace ams::kern::arm64 { this->interrupt_controller.SendInterProcessorInterrupt(irq); } + static void HandleInterrupt(bool user_mode); + /* Implement more KInterruptManager functionality. */ private: Result BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level); diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp index 9fb2b8ab5..c257879d8 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp @@ -62,6 +62,8 @@ namespace ams::kern::arm64 { Result Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main); Result Finalize(); + static void FpuContextSwitchHandler(KThread *thread); + /* TODO: More methods (especially FPU management) */ }; diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp new file mode 100644 index 000000000..5bd1d8d3f --- /dev/null +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::kern::arm64 { + + void UserspaceMemoryAccessFunctionAreaBegin(); + + void UserspaceMemoryAccessFunctionAreaEnd(); + +} \ No newline at end of file diff --git a/libraries/libmesosphere/include/mesosphere/kern_common.hpp b/libraries/libmesosphere/include/mesosphere/kern_common.hpp index 57ef9f117..90402fd74 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_common.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_common.hpp @@ -24,6 +24,10 @@ namespace ams::kern { } +#if 1 +#define MESOSPHERE_BUILD_FOR_AUDITING +#endif + #ifdef MESOSPHERE_BUILD_FOR_AUDITING #define MESOSPHERE_BUILD_FOR_DEBUGGING #endif diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_current_context.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_current_context.hpp index 5bac774a2..ae8a7947d 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_current_context.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_current_context.hpp @@ -29,7 +29,7 @@ namespace ams::kern { KScheduler *scheduler; KInterruptTaskManager *interrupt_task_manager; s32 core_id; - void *exception_stack_bottom; + void *exception_stack_top; }; static_assert(std::is_pod::value); static_assert(sizeof(KCurrentContext) <= cpu::DataCacheLineSize); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_interrupt_task_manager.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_interrupt_task_manager.hpp index 1ad5628a6..8ccd01cc0 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_interrupt_task_manager.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_interrupt_task_manager.hpp @@ -29,9 +29,9 @@ namespace ams::kern { public: constexpr TaskQueue() : head(nullptr), tail(nullptr) { /* ... */ } - ALWAYS_INLINE KInterruptTask *GetHead() { return this->head; } - ALWAYS_INLINE bool IsEmpty() const { return this->head == nullptr; } - ALWAYS_INLINE void Clear() { this->head = nullptr; this->tail = nullptr; } + constexpr KInterruptTask *GetHead() { return this->head; } + constexpr bool IsEmpty() const { return this->head == nullptr; } + constexpr void Clear() { this->head = nullptr; this->tail = nullptr; } void Enqueue(KInterruptTask *task); void Dequeue(); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index 9a83d988c..b137ff65e 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -439,8 +439,8 @@ namespace ams::kern { return GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelMiscIdleStack, static_cast(core_id))->GetEndAddress(); } - static NOINLINE KVirtualAddress GetExceptionStackBottomAddress(s32 core_id) { - return GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelMiscExceptionStack, static_cast(core_id))->GetAddress(); + static NOINLINE KVirtualAddress GetExceptionStackTopAddress(s32 core_id) { + return GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelMiscExceptionStack, static_cast(core_id))->GetEndAddress(); } static NOINLINE KVirtualAddress GetSlabRegionAddress() { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp index 3a381839a..967ae19b5 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -32,7 +32,9 @@ namespace ams::kern { constexpr ALWAYS_INLINE bool Is64Bit() const { /* TODO */ return true; } - ALWAYS_INLINE KThread *GetSuggestedTopThread(s32 core_id) { /* TODO */ return nullptr; } + ALWAYS_INLINE KThread *GetPreemptionStatePinnedThread(s32 core_id) { /* TODO */ return nullptr; } + + void SetPreemptionState(); }; } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp index 2acdd667d..1f5f42ead 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp @@ -74,6 +74,14 @@ namespace ams::kern { NOINLINE void Initialize(KThread *idle_thread); NOINLINE void Activate(); + + ALWAYS_INLINE void RequestScheduleOnInterrupt() { + SetSchedulerUpdateNeeded(); + + if (CanSchedule()) { + this->ScheduleOnInterrupt(); + } + } private: /* Static private API. */ static ALWAYS_INLINE bool IsSchedulerUpdateNeeded() { return s_scheduler_update_needed; } @@ -130,6 +138,11 @@ namespace ams::kern { this->ScheduleImpl(); } + ALWAYS_INLINE void ScheduleOnInterrupt() { + KScopedDisableDispatch dd; + this->Schedule(); + } + void RescheduleOtherCores(u64 cores_needing_scheduling); ALWAYS_INLINE void RescheduleCurrentCore() { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index e04b4abc8..c720d7ff2 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -202,23 +202,57 @@ namespace ams::kern { public: ALWAYS_INLINE s32 GetDisableDispatchCount() const { MESOSPHERE_ASSERT_THIS(); - return GetStackParameters().disable_count; + return this->GetStackParameters().disable_count; } ALWAYS_INLINE void DisableDispatch() { MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 0); - GetStackParameters().disable_count++; + this->GetStackParameters().disable_count++; } ALWAYS_INLINE void EnableDispatch() { MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() > 0); - GetStackParameters().disable_count--; + this->GetStackParameters().disable_count--; + } + + ALWAYS_INLINE void SetInExceptionHandler() { + MESOSPHERE_ASSERT_THIS(); + this->GetStackParameters().is_in_exception_handler = true; + } + + ALWAYS_INLINE void ClearInExceptionHandler() { + MESOSPHERE_ASSERT_THIS(); + this->GetStackParameters().is_in_exception_handler = false; + } + + ALWAYS_INLINE bool IsInExceptionHandler() const { + MESOSPHERE_ASSERT_THIS(); + return this->GetStackParameters().is_in_exception_handler; + } + + ALWAYS_INLINE void RegisterDpc(DpcFlag flag) { + this->GetStackParameters().dpc_flags |= flag; + } + + ALWAYS_INLINE void ClearDpc(DpcFlag flag) { + this->GetStackParameters().dpc_flags &= ~flag; + } + + ALWAYS_INLINE u8 GetDpc() const { + return this->GetStackParameters().dpc_flags; + } + + ALWAYS_INLINE bool HasDpc() const { + MESOSPHERE_ASSERT_THIS(); + return this->GetDpc() != 0;; } private: void Suspend(); public: + constexpr KThreadContext *GetContext() { return std::addressof(this->thread_context); } + constexpr const KThreadContext *GetContext() const { return std::addressof(this->thread_context); } constexpr const KAffinityMask &GetAffinityMask() const { return this->affinity_mask; } constexpr ThreadState GetState() const { return static_cast(this->thread_state & ThreadState_Mask); } constexpr ThreadState GetRawState() const { return this->thread_state; } @@ -248,6 +282,9 @@ namespace ams::kern { constexpr KProcessAddress GetThreadLocalRegionAddress() const { return this->tls_address; } constexpr void *GetThreadLocalRegionHeapAddress() const { return this->tls_heap_address; } + constexpr u16 GetUserPreemptionState() const { return *GetPointer(this->tls_address + 0x100); } + constexpr void SetKernelPreemptionState(u16 state) const { *GetPointer(this->tls_address + 0x100 + sizeof(u16)) = state; } + void AddCpuTime(s64 amount) { this->cpu_time += amount; } @@ -267,14 +304,6 @@ namespace ams::kern { /* TODO: This is kind of a placeholder definition. */ - ALWAYS_INLINE bool IsInExceptionHandler() const { - return GetStackParameters().is_in_exception_handler; - } - - ALWAYS_INLINE void SetInExceptionHandler() { - GetStackParameters().is_in_exception_handler = true; - } - ALWAYS_INLINE bool IsTerminationRequested() const { return this->termination_requested || this->GetRawState() == ThreadState_Terminated; } diff --git a/libraries/libmesosphere/include/mesosphere/kern_panic.hpp b/libraries/libmesosphere/include/mesosphere/kern_panic.hpp index 8f5937cae..02f607500 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_panic.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_panic.hpp @@ -25,9 +25,9 @@ namespace ams::kern { } #ifdef MESOSPHERE_ENABLE_DEBUG_PRINT -#define MESOSPHERE_PANIC(...) ams::kern::Panic(__FILE__, __LINE__, __VA_ARGS__) +#define MESOSPHERE_PANIC(...) ::ams::kern::Panic(__FILE__, __LINE__, __VA_ARGS__) #else -#define MESOSPHERE_PANIC(...) ams::kern::Panic() +#define MESOSPHERE_PANIC(...) ::ams::kern::Panic() #endif #ifdef MESOSPHERE_ENABLE_ASSERTIONS diff --git a/libraries/libmesosphere/include/mesosphere/svc/kern_svc_tables.hpp b/libraries/libmesosphere/include/mesosphere/svc/kern_svc_tables.hpp index c1ff646e2..39975c0ac 100644 --- a/libraries/libmesosphere/include/mesosphere/svc/kern_svc_tables.hpp +++ b/libraries/libmesosphere/include/mesosphere/svc/kern_svc_tables.hpp @@ -15,7 +15,7 @@ */ #pragma once #include -#include +#include namespace ams::kern::svc { diff --git a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp new file mode 100644 index 000000000..1ab3d0d6e --- /dev/null +++ b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern::arm64 { + + namespace { + + constexpr u32 GetInstructionData(const KExceptionContext *context, u64 esr) { + /* Check for THUMB usermode */ + if ((context->psr & 0x3F) == 0x30) { + u32 insn = *reinterpret_cast(context->pc & ~0x1); + /* Check if the instruction was 32-bit. */ + if ((esr >> 25) & 1) { + insn = (insn << 16) | *reinterpret_cast((context->pc & ~0x1) + sizeof(u16)); + } + return insn; + } else { + /* Not thumb, so just get the instruction. */ + return *reinterpret_cast(context->pc); + } + } + + void HandleUserException(KExceptionContext *context, u64 esr, u64 far, u64 afsr0, u64 afsr1, u32 data) { + KProcess *cur_process = GetCurrentProcessPointer(); + bool should_process_user_exception = KTargetSystem::IsUserExceptionHandlersEnabled(); + + const u64 ec = (esr >> 26) & 0x3F; + switch (ec) { + case 0x0: /* Unknown */ + case 0xE: /* Illegal Execution State */ + case 0x11: /* SVC instruction from Aarch32 */ + case 0x15: /* SVC instruction from Aarch64 */ + case 0x22: /* PC Misalignment */ + case 0x26: /* SP Misalignment */ + case 0x2F: /* SError */ + case 0x30: /* Breakpoint from lower EL */ + case 0x32: /* SoftwareStep from lower EL */ + case 0x34: /* Watchpoint from lower EL */ + case 0x38: /* BKPT instruction */ + case 0x3C: /* BRK instruction */ + break; + default: + { + /* TODO: Get memory state. */ + /* If state is KMemoryState_Code and the user can't read it, set should_process_user_exception = true; */ + } + break; + } + + if (should_process_user_exception) { + /* TODO: Process the user exception. */ + } + + { + /* TODO: Process for KDebug. */ + + MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", 0ul /* TODO: cur_process->GetProgramId() */); + + /* TODO: if (!svc::ResultNotHandled::Includes(res)) { debug process } */ + } + + /* TODO: cur_process->Exit(); */ + (void)cur_process; + } + + } + + /* NOTE: This function is called from ASM. */ + void FpuContextSwitchHandler() { + KThreadContext::FpuContextSwitchHandler(GetCurrentThreadPointer()); + } + + /* NOTE: This function is called from ASM. */ + void HandleException(KExceptionContext *context) { + MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled()); + + /* Retrieve information about the exception. */ + const u64 esr = cpu::GetEsrEl1(); + const u64 afsr0 = cpu::GetAfsr0El1(); + const u64 afsr1 = cpu::GetAfsr1El1(); + u64 far = 0; + u32 data = 0; + + /* Collect far and data based on the ec. */ + switch ((esr >> 26) & 0x3F) { + case 0x0: /* Unknown */ + case 0xE: /* Illegal Execution State */ + case 0x38: /* BKPT instruction */ + case 0x3C: /* BRK instruction */ + far = context->pc; + data = GetInstructionData(context, esr); + break; + case 0x11: /* SVC instruction from Aarch32 */ + if (context->psr & 0x20) { + /* Thumb mode. */ + context->pc -= 2; + } else { + /* ARM mode. */ + context->pc -= 4; + } + far = context->pc; + break; + case 0x15: /* SVC instruction from Aarch64 */ + context->pc -= 4; + far = context->pc; + break; + case 0x30: /* Breakpoint from lower EL */ + far = context->pc; + break; + default: + far = cpu::GetFarEl1(); + break; + } + + /* Note that we're in an exception handler. */ + GetCurrentThread().SetInExceptionHandler(); + { + const bool is_user_mode = (context->psr & 0xF) == 0; + if (is_user_mode) { + /* Handle any changes needed to the user preemption state. */ + if (GetCurrentThread().GetUserPreemptionState() != 0 && GetCurrentProcess().GetPreemptionStatePinnedThread(GetCurrentCoreId()) == nullptr) { + KScopedSchedulerLock lk; + + /* Note the preemption state in process. */ + GetCurrentProcess().SetPreemptionState(); + + /* Set the kernel preemption state flag. */ + GetCurrentThread().SetKernelPreemptionState(1); + } + + /* Enable interrupts while we process the usermode exception. */ + { + KScopedInterruptEnable ei; + + HandleUserException(context, esr, far, afsr0, afsr1, data); + } + } else { + MESOSPHERE_PANIC("Unhandled Exception in Supervisor Mode\n"); + } + + MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled()); + + /* Handle any DPC requests. */ + while (GetCurrentThread().HasDpc()) { + KDpcManager::HandleDpc(); + } + } + /* Note that we're no longer in an exception handler. */ + GetCurrentThread().ClearInExceptionHandler(); + } + +} diff --git a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s new file mode 100644 index 000000000..68983e504 --- /dev/null +++ b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s @@ -0,0 +1,610 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* ams::kern::arm64::EL1IrqExceptionHandler() */ +.section .text._ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv +.type _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, %function +_ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv: + /* Save registers that need saving. */ + sub sp, sp, #(8 * 24) + + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x30, [sp, #(8 * 22)] + + mrs x19, sp_el0 + mrs x20, elr_el1 + mrs x21, spsr_el1 + mov w21, w21 + + /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ + mrs x18, tpidr_el1 + mov x0, #0 + bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb + + /* Restore registers that we saved. */ + msr sp_el0, x19 + msr elr_el1, x20 + msr spsr_el1, x21 + + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x30, [sp, #(8 * 22)] + + add sp, sp, #(8 * 24) + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL0IrqExceptionHandler() */ +.section .text._ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv +.type _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, %function +_ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv: + /* Save registers that need saving. */ + sub sp, sp, #(8 * 36) + + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ + mrs x18, tpidr_el1 + mov x0, #1 + bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb + + /* Restore state from the context. */ + ldp x30, x20, [sp, #(8 * 30)] + ldp x21, x22, [sp, #(8 * 32)] + ldr x23, [sp, #(8 * 34)] + msr sp_el0, x20 + msr elr_el1, x21 + msr spsr_el1, x22 + msr tpidr_el0, x23 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + add sp, sp, #0x120 + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL0SynchronousExceptionHandler() */ +.section .text._ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv +.type _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, %function +_ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv: + /* Save x16 and x17, so that we can use them as scratch. */ + stp x16, x17, [sp, #-16]! + + /* Get and parse the exception syndrome register. */ + mrs x16, esr_el1 + lsr x17, x16, #0x1a + + /* Is this an aarch32 SVC? */ + cmp x17, #0x11 + b.eq 2f + + /* Is this an aarch64 SVC? */ + cmp x17, #0x15 + b.eq 3f + + /* Is this an FPU error? */ + cmp x17, #0x7 + b.eq 4f + + /* Is this an instruction abort? */ + cmp x17, #0x21 + b.eq 5f + + /* Is this a data abort? */ + cmp x17, #0x25 + b.eq 5f + +1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */ + /* It is also not an SVC or an FPU exception. Handle it generically! */ + + /* Restore x16 and x17. */ + ldp x16, x17, [sp], 16 + + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + + /* Restore state from the context. */ + ldp x30, x20, [sp, #(8 * 30)] + ldp x21, x22, [sp, #(8 * 32)] + ldr x23, [sp, #(8 * 34)] + msr sp_el0, x20 + msr elr_el1, x21 + msr spsr_el1, x22 + msr tpidr_el0, x23 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + add sp, sp, #0x120 + + /* Return from the exception. */ + eret + +2: /* SVC from aarch32. */ + ldp x16, x17, [sp], 16 + b _ZN3ams4kern5arm6412SvcHandler32Ev + +3: /* SVC from aarch64. */ + ldp x16, x17, [sp], 16 + b _ZN3ams4kern5arm6412SvcHandler64Ev + +4: /* FPU exception. */ + ldp x16, x17, [sp], 16 + b _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv + +5: /* Check if there's a TLB conflict that caused the abort. */ + and x17, x16, #0x3F + cmp x17, #0x30 + b.ne 1b + + /* Get the ASID in x17. */ + mrs x17, ttbr0_el1 + and x17, x17, #(0xFFFF << 48) + + /* Check if FAR is valid by examining the FnV bit. */ + tbnz x16, #10, 6f + + /* FAR is valid, so we can invalidate the address it holds. */ + mrs x16, far_el1 + lsr x16, x16, #12 + orr x17, x16, x17 + tlbi vaae1, x17 + b 7f + +6: /* There's a TLB conflict and FAR isn't valid. */ + /* Invalidate the entire TLB. */ + tlbi vmalle1 + +7: /* Return from a TLB conflict. */ + /* Ensure instruction consistency. */ + dsb ish + isb + + /* Restore x16 and x17. */ + ldp x16, x17, [sp], 16 + + /* Return from the exception. */ + eret + + +/* ams::kern::arm64::EL1SynchronousExceptionHandler() */ +.section .text._ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv +.type _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, %function +_ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv: + /* Nintendo uses the "unused" virtual timer compare value as a scratch register. */ + msr cntv_cval_el0, x0 + + /* Get and parse the exception syndrome register. */ + mrs x0, esr_el1 + lsr x0, x0, #0x1a + + /* Is this an instruction abort? */ + cmp x0, #0x21 + b.eq 5f + + /* Is this a data abort? */ + cmp x0, #0x25 + b.eq 5f + +1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */ + /* Load the CoreLocalContext into x0. */ + mrs x0, tpidr_el1 + cbz x0, 2f + + /* Load the exception stack top from the context. */ + ldr x0, [x0, #0x28] + + /* Setup the stack for a generic exception handle */ + sub x0, x0, #0x20 + str x1, [x0, #16] + mov x1, sp + str x1, [x0] + mov sp, x0 + ldr x1, [x0, #16] + mrs x0, cntv_cval_el0 + str x0, [sp, #8] + + /* Check again if this is a data abort from EL1. */ + mrs x0, esr_el1 + lsr x1, x0, #0x1a + cmp x1, #0x25 + b.ne 3f + + /* Data abort. Check if it was from trying to access userspace memory. */ + mrs x1, elr_el1 + adr x0, _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv + cmp x1, x0 + b.lo 3f + adr x0, _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv + cmp x1, x0 + b.hs 3f + + /* We aborted trying to access userspace memory. */ + /* All functions that access user memory return a boolean for whether they succeeded. */ + /* With that in mind, we can simply restore the stack pointer and return false directly. */ + ldr x0, [sp] + mov sp, x0 + + /* Return false. */ + mov x0, #0x0 + msr elr_el1, x30 + eret + +2: /* The CoreLocalContext is nullptr. */ + /* Setup the stack for a generic exception handle. */ + /* NOTE: Nintendo does not restore X0 here, and thus saves nullptr. */ + /* This is probably not their intention, so we'll fix it. */ + /* NOTE: Nintendo also does not really save SP correctly, and so we */ + /* will also fix that. */ + mov x0, sp + sub x0, x0, #0x20 + str x1, [x0, #16] + mov x1, sp + str x1, [x0] + mov sp, x0 + mrs x0, cntv_cval_el0 + str x0, [sp, #8] + +3: /* The exception wasn't an triggered by copying memory from userspace. */ + ldr x0, [sp, #8] + ldr x1, [sp, #16] + + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + +4: /* HandleException should never return. The best we can do is infinite loop. */ + b 4b + +5: /* Check if there's a TLB conflict that caused the abort. */ + /* NOTE: There is a Nintendo bug in this code that we correct. */ + /* Nintendo compares the low 6 bits of x0 without restoring the value. */ + /* They intend to check the DFSC/IFSC bits of esr_el1, but because they */ + /* shifted esr earlier, the check is invalid and always fails. */ + mrs x0, esr_el1 + and x0, x0, #0x3F + cmp x0, #0x30 + b.ne 1b + + /* Check if FAR is valid by examining the FnV bit. */ + /* NOTE: Nintendo again has a bug here, the same as above. */ + /* They do not refresh the value of x0, and again compare with */ + /* the relevant bit already masked out of x0. */ + mrs x0, esr_el1 + tbnz x0, #10, 6f + + /* FAR is valid, so we can invalidate the address it holds. */ + mrs x0, far_el1 + lsr x0, x0, #12 + tlbi vaae1, x0 + b 7f + +6: /* There's a TLB conflict and FAR isn't valid. */ + /* Invalidate the entire TLB. */ + tlbi vmalle1 + +7: /* Return from a TLB conflict. */ + /* Ensure instruction consistency. */ + dsb ish + isb + + /* Restore x0 from scratch. */ + mrs x0, cntv_cval_el0 + + /* Return from the exception. */ + eret + + +/* ams::kern::arm64::FpuAccessExceptionHandler() */ +.section .text._ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv +.type _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, %function +_ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv: + /* Save registers that need saving. */ + sub sp, sp, #(8 * 24) + + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x30, [sp, #(8 * 22)] + + mrs x18, tpidr_el1 + mrs x19, sp_el0 + mrs x20, elr_el1 + mrs x21, spsr_el1 + + /* Invoke the FPU context switch handler. */ + bl _ZN3ams4kern5arm6423FpuContextSwitchHandlerEv + + /* Restore registers that we saved. */ + msr sp_el0, x19 + msr elr_el1, x20 + msr spsr_el1, x21 + + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x30, [sp, #(8 * 22)] + + add sp, sp, #(8 * 24) + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL1SystemErrorHandler() */ +.section .text._ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv +.type _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, %function +_ZN3ams4kern5arm6421EL1SystemErrorHandlerEv: + /* Nintendo uses the "unused" virtual timer compare value as a scratch register. */ + msr cntv_cval_el0, x0 + + /* Load the exception stack top from the context. */ + ldr x0, [x0, #0x28] + + /* Setup the stack for a generic exception handle */ + sub x0, x0, #0x20 + str x1, [x0, #16] + mov x1, sp + str x1, [x0] + mov sp, x0 + ldr x1, [x0, #16] + mrs x0, cntv_cval_el0 + str x0, [sp, #8] + + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + +1: /* HandleException should never return. The best we can do is infinite loop. */ + b 1b + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL0SystemErrorHandler() */ +.section .text._ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv +.type _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, %function +_ZN3ams4kern5arm6421EL0SystemErrorHandlerEv: + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + + /* Restore state from the context. */ + ldp x30, x20, [sp, #(8 * 30)] + ldp x21, x22, [sp, #(8 * 32)] + ldr x23, [sp, #(8 * 34)] + msr sp_el0, x20 + msr elr_el1, x21 + msr spsr_el1, x22 + msr tpidr_el0, x23 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + add sp, sp, #0x120 + + /* Return from the exception. */ + eret + diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp index f00409371..c1fec3e51 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp @@ -31,6 +31,101 @@ namespace ams::kern::arm64 { this->interrupt_controller.Finalize(core_id); } + bool KInterruptManager::OnHandleInterrupt() { + /* Get the interrupt id. */ + const u32 raw_irq = this->interrupt_controller.GetIrq(); + const s32 irq = KInterruptController::ConvertRawIrq(raw_irq); + + /* If the IRQ is spurious, we don't need to reschedule. */ + if (irq < 0) { + return false; + } + + KInterruptTask *task = nullptr; + if (KInterruptController::IsLocal(irq)) { + /* Get local interrupt entry. */ + auto &entry = GetLocalInterruptEntry(irq); + if (entry.handler != nullptr) { + /* Set manual clear needed if relevant. */ + if (entry.manually_cleared) { + this->interrupt_controller.Disable(irq); + entry.needs_clear = true; + } + + /* Set the handler. */ + task = entry.handler->OnInterrupt(irq); + } else { + MESOSPHERE_LOG("Core%d: Unhandled local interrupt %d\n", GetCurrentCoreId(), irq); + } + } else if (KInterruptController::IsGlobal(irq)) { + KScopedSpinLock lk(GetLock()); + + /* Get global interrupt entry. */ + auto &entry = GetGlobalInterruptEntry(irq); + if (entry.handler != nullptr) { + /* Set manual clear needed if relevant. */ + if (entry.manually_cleared) { + this->interrupt_controller.Disable(irq); + entry.needs_clear = true; + } + + /* Set the handler. */ + task = entry.handler->OnInterrupt(irq); + } else { + MESOSPHERE_LOG("Core%d: Unhandled global interrupt %d\n", GetCurrentCoreId(), irq); + } + } else { + MESOSPHERE_LOG("Invalid interrupt %d\n", irq); + } + + /* If we found no task, then we don't need to reschedule. */ + if (task == nullptr) { + return false; + } + + /* If the task isn't the dummy task, we should add it to the queue. */ + if (task != GetDummyInterruptTask()) { + /* TODO: Kernel::GetInterruptTaskManager().Enqueue(task); */ + } + + return true; + } + + void KInterruptManager::HandleInterrupt(bool user_mode) { + /* On interrupt, call OnHandleInterrupt() to determine if we need rescheduling and handle. */ + const bool needs_scheduling = Kernel::GetInterruptManager().OnHandleInterrupt(); + + /* If we need scheduling, */ + if (needs_scheduling) { + /* Handle any changes needed to the user preemption state. */ + if (user_mode && GetCurrentThread().GetUserPreemptionState() != 0 && GetCurrentProcess().GetPreemptionStatePinnedThread(GetCurrentCoreId()) == nullptr) { + KScopedSchedulerLock sl; + + /* Note the preemption state in process. */ + GetCurrentProcess().SetPreemptionState(); + + /* Set the kernel preemption state flag. */ + GetCurrentThread().SetKernelPreemptionState(1);; + + /* Request interrupt scheduling. */ + Kernel::GetScheduler().RequestScheduleOnInterrupt(); + } else { + /* Request interrupt scheduling. */ + Kernel::GetScheduler().RequestScheduleOnInterrupt(); + } + } + + /* If user mode, check if the thread needs termination. */ + /* If it does, we can take advantage of this to terminate it. */ + if (user_mode) { + KThread *cur_thread = GetCurrentThreadPointer(); + if (cur_thread->IsTerminationRequested()) { + KScopedInterruptEnable ei; + cur_thread->Exit(); + } + } + } + Result KInterruptManager::BindHandler(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level) { R_UNLESS(KInterruptController::IsGlobal(irq) || KInterruptController::IsLocal(irq), svc::ResultOutOfRange()); diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp index d9950f1e8..4cc2c828f 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp @@ -27,6 +27,15 @@ namespace ams::kern::arm64 { namespace { + ALWAYS_INLINE bool IsFpuEnabled() { + return cpu::ArchitecturalFeatureAccessControlRegisterAccessor().IsFpEnabled(); + } + + ALWAYS_INLINE void EnableFpu() { + cpu::ArchitecturalFeatureAccessControlRegisterAccessor().SetFpEnabled(true).Store(); + cpu::InstructionMemoryBarrier(); + } + uintptr_t SetupStackForUserModeThreadStarter(KVirtualAddress pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_64_bit) { /* NOTE: Stack layout on entry looks like following: */ /* SP */ @@ -128,4 +137,21 @@ namespace ams::kern::arm64 { return ResultSuccess(); } + void KThreadContext::FpuContextSwitchHandler(KThread *thread) { + MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled()); + MESOSPHERE_ASSERT(!IsFpuEnabled()); + + /* Enable the FPU. */ + EnableFpu(); + + /* Restore the FPU registers. */ + KProcess *process = thread->GetOwnerProcess(); + MESOSPHERE_ASSERT(process != nullptr); + if (process->Is64Bit()) { + RestoreFpuRegisters64(*thread->GetContext()); + } else { + RestoreFpuRegisters32(*thread->GetContext()); + } + } + } diff --git a/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s b/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s new file mode 100644 index 000000000..3f7deae0b --- /dev/null +++ b/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaBegin() */ +.section .text._ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, "ax", %progbits +.global _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv +.type _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, %function +_ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv: +/* NOTE: This is not a real function, and only exists as a label for safety. */ + +/* ================ All Userspace Memory Functions after this line. ================ */ + +/* TODO */ + +/* ================ All Userspace Memory Functions before this line. ================ */ + +/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaEnd() */ +.section .text._ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, "ax", %progbits +.global _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv +.type _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, %function +_ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv: +/* NOTE: This is not a real function, and only exists as a label for safety. */ \ No newline at end of file diff --git a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s new file mode 100644 index 000000000..dc037f365 --- /dev/null +++ b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* ams::kern::arm64::SvcHandler64() */ +.section .text._ZN3ams4kern5arm6412SvcHandler64Ev, "ax", %progbits +.global _ZN3ams4kern5arm6412SvcHandler64Ev +.type _ZN3ams4kern5arm6412SvcHandler64Ev, %function +_ZN3ams4kern5arm6412SvcHandler64Ev: + /* Create a KExceptionContext for the exception. */ + sub sp, sp, #0x120 + + /* Save registers needed for ReturnFromException */ + stp x9, x10, [sp, #(8 * 9)] + str x11, [sp, #(8 * 11)] + str x18, [sp, #(8 * 18)] + + mrs x8, sp_el0 + mrs x9, elr_el1 + mrs x10, spsr_el1 + mrs x11, tpidr_el0 + + /* Save callee-saved registers. */ + stp x19, x20, [sp, #(8 * 19)] + stp x21, x22, [sp, #(8 * 21)] + stp x23, x24, [sp, #(8 * 23)] + stp x25, x26, [sp, #(8 * 25)] + stp x27, x28, [sp, #(8 * 27)] + + /* Save miscellaneous registers. */ + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x29, x30, [sp, #(8 * 29)] + stp x8, x9, [sp, #(8 * 31)] + stp x10, x11, [sp, #(8 * 33)] + + /* Check if the SVC index is out of range. */ + mrs x8, esr_el1 + and x8, x8, #0xFF + cmp x8, #0x80 + b.ge 3f + + /* Check the specific SVC permission bit for allowal. */ + mov x9, sp + add x9, x9, x8, lsr#3 + ldrb w9, [x9, #0x120] + and x10, x8, #0x7 + lsr x10, x9, x10 + tst x10, #1 + b.eq 3f + + /* Check if our preemption state allows us to call SVCs. */ + mrs x10, tpidrro_el0 + ldrh w10, [x10, #0x100] + cbz w10, 1f + + /* It might not, so check the stack params to see if we must not allow the SVC. */ + ldrb w10, [sp, #(0x120 + 0x14)] + cbz w10, 3f + +1: /* We can call the SVC. */ + adr x10, _ZN3ams4kern3svc10SvcTable64E + ldr x11, [x10, x8, lsl#3] + cbz x11, 3f + + /* Note that we're calling the SVC. */ + mov w10, #1 + strb w10, [sp, #(0x120 + 0x12)] + strb w8, [sp, #(0x120 + 0x11)] + + /* Invoke the SVC handler. */ + mrs x18, tpidr_el1 + msr daifclr, #2 + blr x11 + msr daifset, #2 + +2: /* We completed the SVC, and we should handle DPC. */ + /* Check the dpc flags. */ + ldrb w8, [sp, #(0x120 + 0x10)] + cbz w8, 4f + + /* We have DPC to do! */ + /* Save registers and call ams::kern::KDpcManager::HandleDpc(). */ + sub sp, sp, #0x40 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + bl _ZN3ams4kern11KDpcManager9HandleDpcEv + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + add sp, sp, #0x40 + b 2b + +3: /* Invalid SVC. */ + /* Setup the context to call into HandleException. */ + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp xzr, xzr, [sp, #(8 * 8)] + stp xzr, xzr, [sp, #(8 * 10)] + stp xzr, xzr, [sp, #(8 * 12)] + stp xzr, xzr, [sp, #(8 * 14)] + stp xzr, xzr, [sp, #(8 * 16)] + stp xzr, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + + /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + + /* Restore registers. */ + ldp x30, x8, [sp, #(8 * 30)] + ldp x9, x10, [sp, #(8 * 32)] + ldr x11, [sp, #(8 * 34)] + msr sp_el0, x8 + msr elr_el1, x9 + msr spsr_el1, x10 + msr tpidr_el0, x11 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + + /* Return. */ + add sp, sp, #0x120 + eret + +4: /* Return from SVC. */ + /* Clear our in-SVC note. */ + strb wzr, [sp, #(0x120 + 0x12)] + + /* Restore registers. */ + ldp x30, x8, [sp, #(8 * 30)] + ldp x9, x10, [sp, #(8 * 32)] + ldr x11, [sp, #(8 * 34)] + msr sp_el0, x8 + msr elr_el1, x9 + msr spsr_el1, x10 + msr tpidr_el0, x11 + + /* Clear registers. */ + mov x8, xzr + mov x9, xzr + mov x10, xzr + mov x11, xzr + mov x12, xzr + mov x13, xzr + mov x14, xzr + mov x15, xzr + mov x16, xzr + mov x17, xzr + mov x18, xzr + + /* Return. */ + add sp, sp, #0x120 + eret + +/* ams::kern::arm64::SvcHandler32() */ +.section .text._ZN3ams4kern5arm6412SvcHandler32Ev, "ax", %progbits +.global _ZN3ams4kern5arm6412SvcHandler32Ev +.type _ZN3ams4kern5arm6412SvcHandler32Ev, %function +_ZN3ams4kern5arm6412SvcHandler32Ev: + /* Ensure that our registers are 32-bit. */ + mov w0, w0 + mov w1, w1 + mov w2, w2 + mov w3, w3 + mov w4, w4 + mov w5, w5 + mov w6, w6 + mov w7, w7 + + /* Create a KExceptionContext for the exception. */ + sub sp, sp, #0x120 + + /* Save system registers */ + mrs x17, elr_el1 + mrs x20, spsr_el1 + mrs x19, tpidr_el0 + stp x17, x20, [sp, #(8 * 32)] + str x19, [sp, #(8 * 34)] + + /* Save registers. */ + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, xzr, [sp, #(8 * 14)] + + /* Check if the SVC index is out of range. */ + mrs x16, esr_el1 + and x16, x16, #0xFF + cmp x16, #0x80 + b.ge 3f + + /* Check the specific SVC permission bit for allowal. */ + mov x20, sp + add x20, x20, x16, lsr#3 + ldrb w20, [x20, #0x120] + and x17, x16, #0x7 + lsr x17, x20, x17 + tst x17, #1 + b.eq 3f + + /* Check if our preemption state allows us to call SVCs. */ + mrs x15, tpidrro_el0 + ldrh w15, [x15, #0x100] + cbz w15, 1f + + /* It might not, so check the stack params to see if we must not allow the SVC. */ + ldrb w15, [sp, #(0x120 + 0x14)] + cbz w15, 3f + +1: /* We can call the SVC. */ + adr x15, _ZN3ams4kern3svc16SvcTable64From32E + ldr x19, [x15, x16, lsl#3] + cbz x19, 3f + + /* Note that we're calling the SVC. */ + mov w15, #1 + strb w15, [sp, #(0x120 + 0x12)] + strb w16, [sp, #(0x120 + 0x11)] + + /* Invoke the SVC handler. */ + mrs x18, tpidr_el1 + msr daifclr, #2 + blr x19 + msr daifset, #2 + +2: /* We completed the SVC, and we should handle DPC. */ + /* Check the dpc flags. */ + ldrb w16, [sp, #(0x120 + 0x10)] + cbz w16, 4f + + /* We have DPC to do! */ + /* Save registers and call ams::kern::KDpcManager::HandleDpc(). */ + sub sp, sp, #0x20 + stp w0, w1, [sp, #(4 * 0)] + stp w2, w3, [sp, #(4 * 2)] + stp w4, w5, [sp, #(4 * 4)] + stp w6, w7, [sp, #(4 * 6)] + bl _ZN3ams4kern11KDpcManager9HandleDpcEv + ldp w0, w1, [sp, #(4 * 0)] + ldp w2, w3, [sp, #(4 * 2)] + ldp w4, w5, [sp, #(4 * 4)] + ldp w6, w7, [sp, #(4 * 6)] + add sp, sp, #0x20 + b 2b + +3: /* Invalid SVC. */ + /* Setup the context to call into HandleException. */ + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp xzr, xzr, [sp, #(8 * 16)] + stp xzr, xzr, [sp, #(8 * 18)] + stp xzr, xzr, [sp, #(8 * 20)] + stp xzr, xzr, [sp, #(8 * 22)] + stp xzr, xzr, [sp, #(8 * 24)] + stp xzr, xzr, [sp, #(8 * 26)] + stp xzr, xzr, [sp, #(8 * 28)] + stp xzr, xzr, [sp, #(8 * 30)] + + /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + + /* Restore registers. */ + ldp x17, x20, [sp, #(8 * 32)] + ldr x19, [sp, #(8 * 34)] + msr elr_el1, x17 + msr spsr_el1, x20 + msr tpidr_el0, x19 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + + /* Return. */ + add sp, sp, #0x120 + eret + +4: /* Return from SVC. */ + /* Clear our in-SVC note. */ + strb wzr, [sp, #(0x120 + 0x12)] + + /* Restore registers. */ + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, xzr, [sp, #(8 * 14)] + ldp x17, x20, [sp, #(8 * 32)] + ldr x19, [sp, #(8 * 34)] + msr elr_el1, x17 + msr spsr_el1, x20 + msr tpidr_el0, x19 + + /* Return. */ + add sp, sp, #0x120 + eret diff --git a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp index ed4a1b15a..0e68435ca 100644 --- a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp +++ b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_tables.cpp @@ -13,24 +13,32 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -#include +#include +#include #include -/* TODO: Enable compilation of this file when the kernel supports supervisor calls. */ -#if 0 namespace ams::kern::svc { namespace { - #define DECLARE_SVC_STRUCT(ID, RETURN_TYPE, NAME, ...) \ - class NAME { \ - private: \ - using Impl = ::ams::svc::codegen::KernelSvcWrapper<::ams::kern::svc::NAME##64, ::ams::kern::svc::NAME##64From32>; \ - public: \ - static NOINLINE void Call64() { return Impl::Call64(); } \ - static NOINLINE void Call64From32() { return Impl::Call64From32(); } \ - }; - + /* TODO: Enable compilation of this file when the kernel supports supervisor calls. */ + #if 0 + #define DECLARE_SVC_STRUCT(ID, RETURN_TYPE, NAME, ...) \ + class NAME { \ + private: \ + using Impl = ::ams::svc::codegen::KernelSvcWrapper<::ams::kern::svc::NAME##64, ::ams::kern::svc::NAME##64From32>; \ + public: \ + static NOINLINE void Call64() { return Impl::Call64(); } \ + static NOINLINE void Call64From32() { return Impl::Call64From32(); } \ + }; + #else + #define DECLARE_SVC_STRUCT(ID, RETURN_TYPE, NAME, ...) \ + class NAME { \ + public: \ + static NOINLINE void Call64() { MESOSPHERE_LOG("Stubbed Svc"#NAME"64 was called\n"); } \ + static NOINLINE void Call64From32() { MESOSPHERE_LOG("Stubbed Svc"#NAME"64From32 was called\n"); } \ + }; + #endif /* Set omit-frame-pointer to prevent GCC from emitting MOV X29, SP instructions. */ @@ -66,4 +74,3 @@ namespace ams::kern::svc { }(); } -#endif \ No newline at end of file diff --git a/libraries/libmesosphere/source/kern_k_process.cpp b/libraries/libmesosphere/source/kern_k_process.cpp new file mode 100644 index 000000000..127fe2915 --- /dev/null +++ b/libraries/libmesosphere/source/kern_k_process.cpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + void KProcess::SetPreemptionState() { + /* TODO */ + } + +} diff --git a/libraries/libmesosphere/source/kern_k_scheduler.cpp b/libraries/libmesosphere/source/kern_k_scheduler.cpp index b4f9993f2..5e4d18733 100644 --- a/libraries/libmesosphere/source/kern_k_scheduler.cpp +++ b/libraries/libmesosphere/source/kern_k_scheduler.cpp @@ -114,11 +114,11 @@ namespace ams::kern { for (size_t core_id = 0; core_id < cpu::NumCores; core_id++) { KThread *top_thread = priority_queue.GetScheduledFront(core_id); if (top_thread != nullptr) { - /* If the thread has no waiters, we might prefer a suggestion from the owner process to it. */ + /* If the thread has no waiters, we need to check if the process has a thread pinned by PreemptionState. */ if (top_thread->GetNumKernelWaiters() == 0) { if (KProcess *parent = top_thread->GetOwnerProcess(); parent != nullptr) { - if (KThread *suggested = parent->GetSuggestedTopThread(core_id); suggested != nullptr && suggested != top_thread) { - /* We prefer our parent's suggestion whenever possible. However, we also don't want to schedule un-runnable threads. */ + if (KThread *suggested = parent->GetPreemptionStatePinnedThread(core_id); suggested != nullptr && suggested != top_thread) { + /* We prefer our parent's pinned thread possible. However, we also don't want to schedule un-runnable threads. */ if (suggested->GetRawState() == KThread::ThreadState_Runnable) { top_thread = suggested; } else { diff --git a/libraries/libmesosphere/source/kern_kernel.cpp b/libraries/libmesosphere/source/kern_kernel.cpp index c72732547..3e397261d 100644 --- a/libraries/libmesosphere/source/kern_kernel.cpp +++ b/libraries/libmesosphere/source/kern_kernel.cpp @@ -42,7 +42,7 @@ namespace ams::kern { clc->current.scheduler = std::addressof(clc->scheduler); clc->current.interrupt_task_manager = std::addressof(clc->interrupt_task_manager); clc->current.core_id = core_id; - clc->current.exception_stack_bottom = GetVoidPointer(KMemoryLayout::GetExceptionStackBottomAddress(core_id)); + clc->current.exception_stack_top = GetVoidPointer(KMemoryLayout::GetExceptionStackTopAddress(core_id) - sizeof(KThread::StackParameters)); /* Clear debugging counters. */ clc->num_sw_interrupts = 0; diff --git a/mesosphere/kernel/source/arch/arm64/exception_vectors.s b/mesosphere/kernel/source/arch/arm64/exception_vectors.s index 89fd0de6d..12fc17086 100644 --- a/mesosphere/kernel/source/arch/arm64/exception_vectors.s +++ b/mesosphere/kernel/source/arch/arm64/exception_vectors.s @@ -70,72 +70,95 @@ vector_base _ZN3ams4kern16ExceptionVectorsEv /* Current EL, SP0 */ -.global unknown_exception -unknown_exception: vector_entry synch_sp0 /* Just infinite loop. */ - b unknown_exception + clrex + nop + b synch_sp0 check_vector_size synch_sp0 vector_entry irq_sp0 - b unknown_exception + clrex + nop + b irq_sp0 check_vector_size irq_sp0 vector_entry fiq_sp0 - b unknown_exception + clrex + nop + b fiq_sp0 check_vector_size fiq_sp0 vector_entry serror_sp0 - b unknown_exception + clrex + nop + b serror_sp0 check_vector_size serror_sp0 /* Current EL, SPx */ vector_entry synch_spx - b unknown_exception + clrex + b _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv check_vector_size synch_spx vector_entry irq_spx - b unknown_exception + b _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv check_vector_size irq_spx vector_entry fiq_spx - b unknown_exception + clrex + nop + b fiq_spx check_vector_size fiq_spx vector_entry serror_spx - b unknown_exception + clrex + nop + b _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv check_vector_size serror_spx /* Lower EL, A64 */ vector_entry synch_a64 - b unknown_exception + clrex + b _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv check_vector_size synch_a64 vector_entry irq_a64 - b unknown_exception + clrex + b _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv check_vector_size irq_a64 vector_entry fiq_a64 - b unknown_exception + clrex + nop + b fiq_a64 check_vector_size fiq_a64 vector_entry serror_a64 - b unknown_exception + clrex + nop + b _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv check_vector_size serror_a64 /* Lower EL, A32 */ vector_entry synch_a32 - b unknown_exception + clrex + b _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv check_vector_size synch_a32 vector_entry irq_a32 - b unknown_exception + clrex + b _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv check_vector_size irq_a32 vector_entry fiq_a32 - b unknown_exception + clrex + nop + b fiq_a32 check_vector_size fiq_a32 vector_entry serror_a32 - b unknown_exception - check_vector_size serror_a32 \ No newline at end of file + clrex + nop + b _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv + check_vector_size serror_a32