diff --git a/thermosphere/src/asm_macros.s b/thermosphere/src/asm_macros.s index 8289a170e..5668eba2b 100644 --- a/thermosphere/src/asm_macros.s +++ b/thermosphere/src/asm_macros.s @@ -16,7 +16,7 @@ #define EXCEP_STACK_FRAME_SIZE 0x140 -#define CORECTX_USER_FRAME_OFFSET 0x000 +#define CORECTX_GUEST_FRAME_OFFSET 0x000 #define CORECTX_SCRATCH_OFFSET 0x008 #define CORECTX_CRASH_STACK_OFFSET 0x010 diff --git a/thermosphere/src/barrier.c b/thermosphere/src/barrier.c index 36cead568..0ba1528a7 100644 --- a/thermosphere/src/barrier.c +++ b/thermosphere/src/barrier.c @@ -14,7 +14,6 @@ * along with this program. If not, see . */ -#include #include "barrier.h" #include "core_ctx.h" #include "utils.h" diff --git a/thermosphere/src/barrier.h b/thermosphere/src/barrier.h index cf818f8a3..b14667427 100644 --- a/thermosphere/src/barrier.h +++ b/thermosphere/src/barrier.h @@ -27,4 +27,4 @@ void barrierInit(Barrier *barrier, u32 coreList); void barrierInitAllButSelf(Barrier *barrier); void barrierInitAll(Barrier *barrier); -void barrierWait(Barrier *barrier); \ No newline at end of file +void barrierWait(Barrier *barrier); diff --git a/thermosphere/src/core_ctx.h b/thermosphere/src/core_ctx.h index f904e5afb..f48d9c1c3 100644 --- a/thermosphere/src/core_ctx.h +++ b/thermosphere/src/core_ctx.h @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include "utils.h" #include "barrier.h" @@ -22,7 +23,7 @@ struct ExceptionStackFrame; typedef struct CoreCtx { - struct ExceptionStackFrame *userFrame; // @0x00 + struct ExceptionStackFrame *guestFrame; // @0x00 u64 scratch; // @0x08 u8 *crashStack; // @0x10 u64 kernelArgument; // @0x18 @@ -42,6 +43,9 @@ typedef struct CoreCtx { Barrier executedFunctionBarrier; // @0x50 bool executedFunctionSync; // @0x54 + // Debug features + bool wasPaused; // @0x55 + // Cache stuff u32 setWayCounter; // @0x58 } CoreCtx; @@ -49,6 +53,7 @@ typedef struct CoreCtx { static_assert(offsetof(CoreCtx, warmboot) == 0x2E, "Wrong definition for CoreCtx"); static_assert(offsetof(CoreCtx, emulPtimerCval) == 0x38, "Wrong definition for CoreCtx"); static_assert(offsetof(CoreCtx, executedFunctionSync) == 0x54, "Wrong definition for CoreCtx"); +static_assert(offsetof(CoreCtx, setWayCounter) == 0x58, "Wrong definition for CoreCtx"); extern CoreCtx g_coreCtxs[4]; register CoreCtx *currentCoreCtx asm("x18"); diff --git a/thermosphere/src/debug_pause.c b/thermosphere/src/debug_pause.c index ccef5e6a1..2ff103df0 100644 --- a/thermosphere/src/debug_pause.c +++ b/thermosphere/src/debug_pause.c @@ -14,46 +14,74 @@ * along with this program. If not, see . */ +#include + #include "debug_pause.h" #include "core_ctx.h" #include "irq.h" #include "spinlock.h" +#include "single_step.h" // Reminder: use these functions behind a lock static Barrier g_debugPauseBarrier; -static RecursiveSpinlock g_debugPauseContinueLocks[4]; +static atomic_uint g_debugPausePausedCoreList; +static atomic_uint g_debugPauseSingleStepCoreList; -void debugPauseSgiTopHalf(void) +void debugPauseSgiHandler(void) { + currentCoreCtx->wasPaused = true; barrierWait(&g_debugPauseBarrier); } -void debugPauseSgiBottomHalf(void) +void debugPauseWaitAndUpdateSingleStep(void) { - recursiveSpinlockLock(&g_debugPauseContinueLocks[currentCoreCtx->coreId]); - maskIrq(); // <- unlikely race condition here? If it happens, it shouldn't happen more than once/should be fine anyway - recursiveSpinlockUnlock(&g_debugPauseContinueLocks[currentCoreCtx->coreId]); + u32 coreId = currentCoreCtx->coreId; + if (atomic_load(&g_debugPausePausedCoreList) & BIT(coreId)) { + unmaskIrq(); + do { + __wfe(); + } while (atomic_load(&g_debugPausePausedCoreList) & BIT(coreId)); + maskIrq(); + } + + currentCoreCtx->wasPaused = false; + + // Single-step: if inactive and requested, start single step; cancel if active and not requested + u32 ssReqd = (atomic_load(&g_debugPauseSingleStepCoreList) & ~BIT(currentCoreCtx->coreId)) != 0; + SingleStepState singleStepState = singleStepGetNextState(currentCoreCtx->guestFrame); + if (ssReqd && singleStepState == SingleStepState_Inactive) { + singleStepSetNextState(currentCoreCtx->guestFrame, SingleStepState_ActiveNotPending); + } else if (!ssReqd && singleStepState != SingleStepState_Inactive) { + singleStepSetNextState(currentCoreCtx->guestFrame, SingleStepState_Inactive); + } } void debugPauseCores(u32 coreList) { - coreList &= ~BIT(currentCoreCtx->coreId); + // Since we're using a debugger lock, a simple stlr should be fine... + atomic_store(&g_debugPausePausedCoreList, coreList); - barrierInit(&g_debugPauseBarrier, coreList | BIT(currentCoreCtx->coreId)); - FOREACH_BIT (tmp, core, coreList) { - recursiveSpinlockLock(&g_debugPauseContinueLocks[core]); + if (coreList != BIT(currentCoreCtx->coreId)) { + // We need to notify other cores... + u32 otherCores = coreList & ~BIT(currentCoreCtx->coreId); + barrierInit(&g_debugPauseBarrier, otherCores | BIT(currentCoreCtx->coreId)); + generateSgiForList(ThermosphereSgi_DebugPause, otherCores); + barrierWait(&g_debugPauseBarrier); } - generateSgiForList(ThermosphereSgi_DebugPause, coreList); - barrierWait(&g_debugPauseBarrier); + if (coreList & BIT(currentCoreCtx->coreId)) { + currentCoreCtx->wasPaused = true; + } } -void debugUnpauseCores(u32 coreList) +void debugUnpauseCores(u32 coreList, u32 singleStepList) { - coreList &= ~BIT(currentCoreCtx->coreId); + singleStepList &= coreList; - FOREACH_BIT (tmp, core, coreList) { - recursiveSpinlockUnlock(&g_debugPauseContinueLocks[core]); - } + // Since we're using a debugger lock, a simple stlr should be fine... + atomic_store(&g_debugPauseSingleStepCoreList, singleStepList); + atomic_store(&g_debugPausePausedCoreList, 0); + + __sev(); } diff --git a/thermosphere/src/debug_pause.h b/thermosphere/src/debug_pause.h index 21ec9d9ee..0656813f4 100644 --- a/thermosphere/src/debug_pause.h +++ b/thermosphere/src/debug_pause.h @@ -18,14 +18,13 @@ #include "utils.h" -void debugPauseSgiTopHalf(void); -void debugPauseSgiBottomHalf(void); +void debugPauseSgiHandler(void); + +// Hypervisor interrupts will be serviced during the pause-wait +void debugPauseWaitAndUpdateSingleStep(void); // Note: these functions are not reentrant! (need a global debug lock...) -// These functions also run with interrupts unmasked (but we know we're in our code -- should be safe if we take care) -// while the core is paused. // "Pause" makes sure all cores reaches the pause function before proceeding. // "Unpause" doesn't synchronize, it just makes sure the core resumes & that "pause" can be called again. - void debugPauseCores(u32 coreList); -void debugUnpauseCores(u32 coreList); +void debugUnpauseCores(u32 coreList, u32 singleStepList); diff --git a/thermosphere/src/exception_vectors.s b/thermosphere/src/exception_vectors.s index 050145918..cb48aa0a9 100644 --- a/thermosphere/src/exception_vectors.s +++ b/thermosphere/src/exception_vectors.s @@ -104,7 +104,7 @@ vector_entry \name .if \type == EXCEPTION_TYPE_GUEST ldp x18, xzr, [sp, #EXCEP_STACK_FRAME_SIZE] - str x0, [x18, #CORECTX_USER_FRAME_OFFSET] + str x0, [x18, #CORECTX_GUEST_FRAME_OFFSET] mov w1, #1 .else mov w1, #0 diff --git a/thermosphere/src/exceptions.c b/thermosphere/src/exceptions.c index 0fe520d23..4a830784f 100644 --- a/thermosphere/src/exceptions.c +++ b/thermosphere/src/exceptions.c @@ -23,7 +23,8 @@ #include "core_ctx.h" #include "single_step.h" #include "data_abort.h" - +#include "spinlock.h" +#include "debug_pause.h" #include "timer.h" bool spsrEvaluateConditionCode(u64 spsr, u32 conditionCode) @@ -113,10 +114,15 @@ void exceptionEntryPostprocess(ExceptionStackFrame *frame, bool isLowerEl) // Called on exception return (avoids overflowing a vector section) void exceptionReturnPreprocess(ExceptionStackFrame *frame) { + if (currentCoreCtx->wasPaused && frame == currentCoreCtx->guestFrame) { + // Were we paused & are we about to return to the guest? + exceptionEnterInterruptibleHypervisorCode(frame); + debugPauseWaitAndUpdateSingleStep(); + } + // Update virtual counter currentCoreCtx->totalTimeInHypervisor += timerGetSystemTick() - frame->cntpct_el0; SET_SYSREG(cntvoff_el2, currentCoreCtx->totalTimeInHypervisor); - //DEBUG("pct %lu - vct %lu = voff %lu\n", timerGetSystemTick() - GET_SYSREG(cntvct_el0), GET_SYSREG(cntvoff_el2)); // Restore interrupt mask SET_SYSREG(cntp_ctl_el0, frame->cntp_ctl_el0); diff --git a/thermosphere/src/irq.c b/thermosphere/src/irq.c index d1df11cc4..26efd43e9 100644 --- a/thermosphere/src/irq.c +++ b/thermosphere/src/irq.c @@ -237,8 +237,7 @@ void handleIrqException(ExceptionStackFrame *frame, bool isLowerEl, bool isA32) // Nothing in particular to do here break; case ThermosphereSgi_DebugPause: - debugPauseSgiTopHalf(); - hasBottomHalf = true; + debugPauseSgiHandler(); break; case GIC_IRQID_MAINTENANCE: isMaintenanceInterrupt = true; @@ -280,10 +279,7 @@ void handleIrqException(ExceptionStackFrame *frame, bool isLowerEl, bool isA32) if (hasBottomHalf) { exceptionEnterInterruptibleHypervisorCode(frame); unmaskIrq(); - - if (irqId == ThermosphereSgi_DebugPause) { - debugPauseSgiBottomHalf(); - } else if (transportIface != NULL) { + if (transportIface != NULL) { transportInterfaceIrqHandlerBottomHalf(transportIface); } } diff --git a/thermosphere/src/main.c b/thermosphere/src/main.c index ddd4edda3..f3712339b 100644 --- a/thermosphere/src/main.c +++ b/thermosphere/src/main.c @@ -59,7 +59,7 @@ void testProcessDataCallback(TransportInterface *iface, void *p, size_t sz) { (void)iface; (void)sz; - debugUnpauseCores(BIT(0)); + debugUnpauseCores(BIT(0), BIT(0)); TestCtx *ctx = (TestCtx *)p; DEBUG("EL2 [core %u]: you typed: %s\n", currentCoreCtx->coreId, ctx->buf); } diff --git a/thermosphere/src/single_step.c b/thermosphere/src/single_step.c index e4ffa9a99..cbeb84202 100644 --- a/thermosphere/src/single_step.c +++ b/thermosphere/src/single_step.c @@ -28,7 +28,7 @@ SingleStepState singleStepGetNextState(ExceptionStackFrame *frame) if (!mdscrSS) { return SingleStepState_Inactive; } else { - return pstateSS ? SingleStepState_ActivePending : SingleStepState_ActiveNotPending; + return pstateSS ? SingleStepState_ActiveNotPending : SingleStepState_ActivePending; } } @@ -41,15 +41,16 @@ void singleStepSetNextState(ExceptionStackFrame *frame, SingleStepState state) // Unset mdscr_el1.ss mdscr &= ~MDSCR_SS; break; - case SingleStepState_ActivePending: + case SingleStepState_ActiveNotPending: // Set mdscr_el1.ss and pstate.ss mdscr |= MDSCR_SS; frame->spsr_el2 |= PSTATE_SS; break; - case SingleStepState_ActiveNotPending: + case SingleStepState_ActivePending: + // We never use this because pstate.ss is 0 by default... // Set mdscr_el1.ss and unset pstate.ss mdscr |= MDSCR_SS; - frame->spsr_el2 |= PSTATE_SS; + frame->spsr_el2 &= ~PSTATE_SS; break; default: break; @@ -65,7 +66,4 @@ void handleSingleStep(ExceptionStackFrame *frame, ExceptionSyndromeRegister esr) singleStepSetNextState(NULL, SingleStepState_Inactive); DEBUG("Single-step exeception ELR = 0x%016llx, ISV = %u, EX = %u\n", frame->elr_el2, (esr.iss >> 24) & 1, (esr.iss >> 6) & 1); - - // Hehe boi - //singleStepSetNextState(frame, SingleStepState_ActivePending); } diff --git a/thermosphere/src/single_step.h b/thermosphere/src/single_step.h index 05d93a857..5b90fe914 100644 --- a/thermosphere/src/single_step.h +++ b/thermosphere/src/single_step.h @@ -21,8 +21,8 @@ typedef enum SingleStepState { SingleStepState_Inactive = 0, // Single step disabled OR in the debugger - SingleStepState_ActivePending = 1, // Instruction not yet executed - SingleStepState_ActiveNotPending = 2, // Instruction executed, single-step exception is going to be generated soon + SingleStepState_ActiveNotPending = 1, // Instruction not yet executed + SingleStepState_ActivePending = 2, // Instruction executed or return-from-trap, single-step exception is going to be generated soon } SingleStepState; /// Get the single-step state machine state (state after eret) diff --git a/thermosphere/src/utils.h b/thermosphere/src/utils.h index 2a113cb3b..9bcf90d30 100644 --- a/thermosphere/src/utils.h +++ b/thermosphere/src/utils.h @@ -62,6 +62,21 @@ typedef enum ReadWriteDirection { DIRECTION_READWRITE = DIRECTION_READ | DIRECTION_WRITE, } ReadWriteDirection; +static inline void __wfe(void) +{ + __asm__ __volatile__ ("wfe" ::: "memory"); +} + +static inline void __sev(void) +{ + __asm__ __volatile__ ("sev" ::: "memory"); +} + +static inline void __sevl(void) +{ + __asm__ __volatile__ ("sevl" ::: "memory"); +} + /* Domains: - Inner shareable: typically cores within a cluster (maybe more) with L1+L2 caches