From 9e7b56b33c803f54b693036e411283a5425e521a Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 27 Jul 2021 14:05:32 -0700 Subject: [PATCH] kern: optimize hw-single-step management --- .../arch/arm64/kern_cpu_system_registers.hpp | 9 +++ .../arch/arm64/kern_exception_handlers.cpp | 48 ++++++++++++--- .../arch/arm64/svc/kern_svc_exception_asm.s | 38 ++---------- .../arch/arm64/svc/kern_svc_handlers_asm.s | 59 +++++-------------- .../source/kern_k_debug_base.cpp | 5 +- .../arch/arm64/kern_exception_handlers_asm.s | 47 +++------------ .../source/arch/arm64/kern_k_scheduler_asm.s | 21 ++++++- 7 files changed, 102 insertions(+), 125 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp index d8c936e25..d14f0a723 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu_system_registers.hpp @@ -265,6 +265,10 @@ namespace ams::kern::arch::arm64::cpu { return this->GetBits(12, 1) != 0; } + constexpr ALWAYS_INLINE bool GetSoftwareStep() const { + return this->GetBits(0, 1) != 0; + } + constexpr ALWAYS_INLINE decltype(auto) SetMde(bool set) { this->SetBit(15, set); return *this; @@ -274,6 +278,11 @@ namespace ams::kern::arch::arm64::cpu { this->SetBit(12, set); return *this; } + + constexpr ALWAYS_INLINE decltype(auto) SetSoftwareStep(bool set) { + this->SetBit(0, set); + return *this; + } }; MESOSPHERE_CPU_SYSREG_ACCESSOR_CLASS(MultiprocessorAffinity) { diff --git a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp index 67e6fbab5..714ce88f4 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp @@ -109,6 +109,15 @@ namespace ams::kern::arch::arm64 { break; } + /* If we should, clear the thread's state as single-step. */ + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + if (AMS_UNLIKELY(GetCurrentThread().IsSingleStep())) { + GetCurrentThread().ClearSingleStep(); + cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(false).Store(); + cpu::EnsureInstructionConsistency(); + } + #endif + /* If we should process the user exception (and it's not a breakpoint), try to enter. */ const bool is_software_break = (ec == EsrEc_Unknown || ec == EsrEc_IllegalExecution || ec == EsrEc_BkptInstruction || ec == EsrEc_BrkInstruction); const bool is_breakpoint = (ec == EsrEc_BreakPointEl0 || ec == EsrEc_SoftwareStepEl0 || ec == EsrEc_WatchPointEl0); @@ -290,16 +299,40 @@ namespace ams::kern::arch::arm64 { return; } - /* Print that an exception occurred. */ - MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId()); - + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) { - /* Print the current thread's registers. */ - KDebug::PrintRegister(); + if (ec != EsrEc_SoftwareStepEl0) { + /* If the exception wasn't single-step, print details. */ + MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId()); - /* Print a backtrace. */ - KDebug::PrintBacktrace(); + { + /* Print the current thread's registers. */ + KDebug::PrintRegister(); + + /* Print a backtrace. */ + KDebug::PrintBacktrace(); + } + } else { + /* If the exception was single-step and we have no debug object, we should just return. */ + if (AMS_UNLIKELY(!cur_process.IsAttachedToDebugger())) { + return; + } + } } + #else + { + /* Print that an exception occurred. */ + MESOSPHERE_RELEASE_LOG("Exception occurred. %016lx\n", GetCurrentProcess().GetProgramId()); + + { + /* Print the current thread's registers. */ + KDebug::PrintRegister(); + + /* Print a backtrace. */ + KDebug::PrintBacktrace(); + } + } + #endif /* If the SVC is handled, handle it. */ if (!svc::ResultNotHandled::Includes(result)) { @@ -559,6 +592,7 @@ namespace ams::kern::arch::arm64 { KDpcManager::HandleDpc(); } } + /* Note that we're no longer in an exception handler. */ GetCurrentThread().ClearInExceptionHandler(); } diff --git a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s index af3b79901..b6caa3799 100644 --- a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s +++ b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s @@ -15,39 +15,6 @@ */ #include -#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - .macro disable_single_step, scratch - /* Clear MDSCR_EL1.SS. */ - mrs \scratch, mdscr_el1 - bic \scratch, \scratch, #1 - msr mdscr_el1, \scratch - .endm - - .macro check_enable_single_step, scratch1, scratch2, spsr_value - /* Check if single-step is requested. */ - ldrb \scratch1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_SINGLE_STEP)] - tbz \scratch1, #0, .skip_single_step\@ - - /* If single-step is requested, enable the single-step machine by setting MDSCR_EL1.SS. */ - mrs \scratch2, mdscr_el1 - orr \scratch2, \scratch2, #1 - msr mdscr_el1, \scratch2 - - /* Since we're returning from an exception, set SPSR.SS so we actually advance an instruction. */ - orr \spsr_value, \spsr_value, #(1 << 21) - - isb - -.skip_single_step\@: - .endm -#else - .macro disable_single_step, scratch - .endm - - .macro check_enable_single_step, scratch1, scratch2, spsr_value - .endm -#endif - /* ams::kern::svc::CallReturnFromException64(Result result) */ .section .text._ZN3ams4kern3svc25CallReturnFromException64Ev, "ax", %progbits .global _ZN3ams4kern3svc25CallReturnFromException64Ev @@ -123,7 +90,10 @@ _ZN3ams4kern3svc14RestoreContextEm: ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - check_enable_single_step w0, x0, x10 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */ + orr x22, x22, #(1 << 21) + #endif msr sp_el0, x8 msr elr_el1, x9 diff --git a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s index a0e9f3360..c0544ee26 100644 --- a/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s +++ b/libraries/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s @@ -16,39 +16,6 @@ #include #include -#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - .macro disable_single_step, scratch - /* Clear MDSCR_EL1.SS. */ - mrs \scratch, mdscr_el1 - bic \scratch, \scratch, #1 - msr mdscr_el1, \scratch - .endm - - .macro check_enable_single_step, scratch1, scratch2, spsr_value - /* Check if single-step is requested. */ - ldrb \scratch1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_SINGLE_STEP)] - tbz \scratch1, #0, .skip_single_step\@ - - /* If single-step is requested, enable the single-step machine by setting MDSCR_EL1.SS. */ - mrs \scratch2, mdscr_el1 - orr \scratch2, \scratch2, #1 - msr mdscr_el1, \scratch2 - - /* Since we're returning from an SVC, make sure SPSR.SS is cleared so we break instantly on the instruction after the SVC. */ - bic \spsr_value, \spsr_value, #(1 << 21) - - isb - -.skip_single_step\@: - .endm -#else - .macro disable_single_step, scratch - .endm - - .macro check_enable_single_step, scratch1, scratch2, spsr_value - .endm -#endif - /* ams::kern::arch::arm64::SvcHandler64() */ .section .text._ZN3ams4kern4arch5arm6412SvcHandler64Ev, "ax", %progbits .global _ZN3ams4kern4arch5arm6412SvcHandler64Ev @@ -84,9 +51,6 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: stp x8, x9, [sp, #(EXCEPTION_CONTEXT_SP_PC)] stp x10, x11, [sp, #(EXCEPTION_CONTEXT_PSR_TPIDR)] - /* Disable single-step. */ - disable_single_step x8 - /* Check if the SVC index is out of range. */ mrs x8, esr_el1 and x8, x8, #0xFF @@ -191,7 +155,10 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - check_enable_single_step w0, x0, x10 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ + bic x10, x10, #(1 << 21) + #endif msr sp_el0, x8 msr elr_el1, x9 @@ -243,7 +210,10 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] ldr x18, [sp, #(EXCEPTION_CONTEXT_X18)] - check_enable_single_step w12, x12, x10 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ + bic x10, x10, #(1 << 21) + #endif msr sp_el0, x8 msr elr_el1, x9 @@ -302,9 +272,6 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: stp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)] stp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)] - /* Disable single-step. */ - disable_single_step x8 - /* Check if the SVC index is out of range. */ mrs x16, esr_el1 and x16, x16, #0xFF @@ -405,7 +372,10 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: ldp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - check_enable_single_step w0, x0, x20 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ + bic x20, x20, #(1 << 21) + #endif msr elr_el1, x17 msr spsr_el1, x20 @@ -451,7 +421,10 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: ldp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - check_enable_single_step w21, x21, x20 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ + bic x20, x20, #(1 << 21) + #endif msr elr_el1, x17 msr spsr_el1, x20 diff --git a/libraries/libmesosphere/source/kern_k_debug_base.cpp b/libraries/libmesosphere/source/kern_k_debug_base.cpp index 0d84f9835..544cde9ce 100644 --- a/libraries/libmesosphere/source/kern_k_debug_base.cpp +++ b/libraries/libmesosphere/source/kern_k_debug_base.cpp @@ -372,6 +372,7 @@ namespace ams::kern { new_state = state; } + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) /* Clear single step on all threads. */ { auto end = target->GetThreadList().end(); @@ -379,6 +380,7 @@ namespace ams::kern { it->ClearSingleStep(); } } + #endif /* Detach from the process. */ target->ClearDebugObject(new_state); @@ -900,8 +902,10 @@ namespace ams::kern { { auto end = process->GetThreadList().end(); for (auto it = process->GetThreadList().begin(); it != end; ++it) { + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) /* Clear the thread's single-step state. */ it->ClearSingleStep(); + #endif if (resume) { /* If the process isn't crashed, resume threads. */ @@ -993,7 +997,6 @@ namespace ams::kern { /* If the event is an exception, set the result and clear single step. */ if (event == ams::svc::DebugEvent_Exception) { GetCurrentThread().SetDebugExceptionResult(ResultSuccess()); - GetCurrentThread().ClearSingleStep(); } /* Exit our retry loop. */ diff --git a/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s b/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s index 4c73caff2..0f0c6a7a7 100644 --- a/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s +++ b/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s @@ -15,39 +15,6 @@ */ #include -#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - .macro disable_single_step, scratch - /* Clear MDSCR_EL1.SS. */ - mrs \scratch, mdscr_el1 - bic \scratch, \scratch, #1 - msr mdscr_el1, \scratch - .endm - - .macro check_enable_single_step, scratch1, scratch2, spsr_value - /* Check if single-step is requested. */ - ldrb \scratch1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_SINGLE_STEP)] - tbz \scratch1, #0, .skip_single_step\@ - - /* If single-step is requested, enable the single-step machine by setting MDSCR_EL1.SS. */ - mrs \scratch2, mdscr_el1 - orr \scratch2, \scratch2, #1 - msr mdscr_el1, \scratch2 - - /* Since we're returning from an exception, set SPSR.SS so we actually advance an instruction. */ - orr \spsr_value, \spsr_value, #(1 << 21) - - isb - -.skip_single_step\@: - .endm -#else - .macro disable_single_step, scratch - .endm - - .macro check_enable_single_step, scratch1, scratch2, spsr_value - .endm -#endif - /* ams::kern::arch::arm64::EL1IrqExceptionHandler() */ .section .text._ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv, "ax", %progbits .global _ZN3ams4kern4arch5arm6422EL1IrqExceptionHandlerEv @@ -133,8 +100,6 @@ _ZN3ams4kern4arch5arm6422EL0IrqExceptionHandlerEv: stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - disable_single_step x0 - /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)] mov x0, #1 @@ -145,7 +110,10 @@ _ZN3ams4kern4arch5arm6422EL0IrqExceptionHandlerEv: ldp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - check_enable_single_step w0, x0, x22 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */ + orr x22, x22, #(1 << 21) + #endif msr sp_el0, x20 msr elr_el1, x21 @@ -239,8 +207,6 @@ _ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv: stp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] str x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - disable_single_step x16 - /* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */ ldr x18, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CUR_THREAD)] mov x0, sp @@ -251,7 +217,10 @@ _ZN3ams4kern4arch5arm6430EL0SynchronousExceptionHandlerEv: ldp x21, x22, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x23, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - check_enable_single_step w0, x0, x22 + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an exception, set SPSR.SS so that we advance an instruction if single-stepping. */ + orr x22, x22, #(1 << 21) + #endif msr sp_el0, x20 msr elr_el1, x21 diff --git a/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s b/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s index ac5ebc64c..998022cd3 100644 --- a/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s +++ b/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s @@ -235,7 +235,26 @@ _ZN3ams4kern10KScheduler12ScheduleImplEv: mov x0, x22 RESTORE_THREAD_CONTEXT(x0, x1, x2, 9f) -9: /* We're done restoring the thread context, and can return safely. */ +9: /* Configure single-step, if we should. */ + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + + /* Get a reference to the new thread's stack parameters. */ + add x2, sp, #0x1000 + and x2, x2, #~(0x1000-1) + + /* Read the single-step flag. */ + ldurb w2, [x2, #-(THREAD_STACK_PARAMETERS_SIZE - THREAD_STACK_PARAMETERS_IS_SINGLE_STEP)] + + /* Update the single-step bit in mdscr_el1. */ + mrs x1, mdscr_el1 + bic x1, x1, #1 + orr x1, x1, x2 + msr mdscr_el1, x1 + + isb + #endif + + /* We're done restoring the thread context, and can return safely. */ ret 10: /* Our switch failed. */