thermopshere: interrupt refactoring

This commit is contained in:
TuxSH 2020-02-16 01:19:19 +00:00
parent 1ee289f5f1
commit 2574f68484
14 changed files with 205 additions and 148 deletions

View file

@ -15,7 +15,6 @@
*/
#pragma once
#include <assert.h>
#include "utils.h"
#include "core_ctx.h"
@ -86,8 +85,8 @@ typedef struct ExceptionStackFrame {
u64 cntv_ctl_el0;
} ExceptionStackFrame;
static_assert(offsetof(ExceptionStackFrame, far_el2) == 0x120, "Wrong definition for ExceptionStackFrame");
static_assert(sizeof(ExceptionStackFrame) == 0x140, "Wrong size for ExceptionStackFrame");
//static_assert(offsetof(ExceptionStackFrame, far_el2) == 0x120, "Wrong definition for ExceptionStackFrame");
//static_assert(sizeof(ExceptionStackFrame) == 0x140, "Wrong size for ExceptionStackFrame");
static inline bool spsrIsA32(u64 spsr)
{

View file

@ -24,17 +24,11 @@ namespace ams::hvisor {
HwBreakpointManager HwBreakpointManager::instance{};
void HwBreakpointManager::ReloadOnAllCoresSgiHandler()
void HwBreakpointManager::Reload() const
{
// TODO
}
void HwBreakpointManager::ReloadOnAllCores() const
{
cpu::dmb();
// TODO
}
bool HwBreakpointManager::FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t, cpu::DebugRegisterPair::LoadStoreControl) const
{
return pair.vr == addr;

View file

@ -22,18 +22,15 @@ namespace ams::hvisor {
class HwBreakpointManager final : public HwStopPointManager {
SINGLETON(HwBreakpointManager);
protected:
virtual bool FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t, cpu::DebugRegisterPair::LoadStoreControl) const;
private:
bool FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t, cpu::DebugRegisterPair::LoadStoreControl) const final;
void Reload() const final;
public:
virtual void ReloadOnAllCores() const;
static void ReloadOnAllCoresSgiHandler();
cpu::DebugRegisterPair RetrieveWatchpointConfig(uintptr_t addr, cpu::DebugRegisterPair::LoadStoreControl direction) const;
int Add(uintptr_t addr);
int Remove(uintptr_t addr);
public:
constexpr HwBreakpointManager() : HwStopPointManager(MAX_BCR) {}
constexpr HwBreakpointManager() : HwStopPointManager(MAX_BCR, IrqManager::ReloadHwBreakpointsSgi) {}
};
}

View file

@ -15,6 +15,8 @@
*/
#include "hvisor_hw_stop_point_manager.hpp"
#include "cpu/hvisor_cpu_instructions.hpp"
#include "cpu/hvisor_cpu_interrupt_mask_guard.hpp"
#include <mutex>
#define _REENT_ONLY
@ -22,6 +24,16 @@
namespace ams::hvisor {
void HwStopPointManager::DoReloadOnAllCores() const
{
cpu::InterruptMaskGuard mg{};
cpu::dmb();
Reload();
m_reloadBarrier.Reset(getActiveCoreMask());
IrqManager::GenerateSgiForAllOthers(m_irqId);
m_reloadBarrier.Join();
}
cpu::DebugRegisterPair *HwStopPointManager::Allocate()
{
size_t pos = __builtin_ffs(m_freeBitmap);
@ -76,7 +88,7 @@ namespace ams::hvisor {
regs->vr = preconfiguredPair.vr;
regs->cr.enabled = true;
ReloadOnAllCores();
DoReloadOnAllCores();
return 0;
}
@ -98,7 +110,16 @@ namespace ams::hvisor {
m_freeBitmap |= m_usedBitmap;
m_usedBitmap = 0;
std::fill(m_stopPoints.begin(), m_stopPoints.end(), cpu::DebugRegisterPair{});
ReloadOnAllCores();
DoReloadOnAllCores();
}
std::optional<bool> HwStopPointManager::InterruptTopHalfHandler(u32 irqId, u32)
{
if (irqId != m_irqId) {
return {};
}
Reload();
return false;
}
}

View file

@ -17,37 +17,58 @@
#pragma once
#include "defines.hpp"
#include "hvisor_synchronization.hpp"
#include "cpu/hvisor_cpu_debug_register_pair.hpp"
#include "hvisor_irq_manager.hpp"
namespace ams::hvisor {
class HwStopPointManager {
class HwStopPointManager : public IInterruptTask {
NON_COPYABLE(HwStopPointManager);
NON_MOVEABLE(HwStopPointManager);
protected:
static constexpr size_t maxStopPoints = std::max(MAX_BCR, MAX_WCR);
protected:
mutable RecursiveSpinlock m_lock{};
mutable Barrier m_reloadBarrier{};
u16 m_freeBitmap;
u16 m_usedBitmap = 0;
std::array<cpu::DebugRegisterPair, maxStopPoints> m_stopPoints{};
IrqManager::ThermosphereSgi m_irqId;
protected:
void DoReloadOnAllCores() const;
cpu::DebugRegisterPair *Allocate();
void Free(size_t pos);
const cpu::DebugRegisterPair *Find(uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl dir) const;
virtual bool FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl direction) const = 0;
virtual void Reload() const = 0;
int AddImpl(uintptr_t addr, size_t size, cpu::DebugRegisterPair preconfiguredPair);
int RemoveImpl(uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl direction);
protected:
constexpr HwStopPointManager(size_t numStopPoints) : m_freeBitmap(MASK(numStopPoints)) {}
constexpr HwStopPointManager(size_t numStopPoints, IrqManager::ThermosphereSgi irqId) :
m_freeBitmap(MASK(numStopPoints)), m_irqId(irqId)
{
}
public:
virtual void ReloadOnAllCores() const = 0;
void RemoveAll();
std::optional<bool> InterruptTopHalfHandler(u32 irqId, u32) final;
void ReloadOnAllCores() const
{
m_lock.lock();
DoReloadOnAllCores();
m_lock.unlock();
}
void Initialize()
{
IrqManager::GetInstance().Register(*this, m_irqId, false);
}
};
}

View file

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include "defines.hpp"
#include <vapours/util/util_intrusive_list.hpp>
namespace ams::hvisor {
class IInterruptTask : public util::IntrusiveListBaseNode<IInterruptTask> {
NON_COPYABLE(IInterruptTask);
NON_MOVEABLE(IInterruptTask);
protected:
constexpr IInterruptTask() = default;
public:
virtual std::optional<bool> InterruptTopHalfHandler(u32 irqId, u32 srcCore) = 0;
virtual void InterruptBottomHalfHandler(u32 irqId, u32 srcCore) {}
};
}

View file

@ -17,6 +17,8 @@
#include <mutex>
#include "hvisor_irq_manager.hpp"
#include "hvisor_virtual_gic.hpp"
#include "cpu/hvisor_cpu_interrupt_mask_guard.hpp"
#include "platform/interrupt_config.h"
#include "core_ctx.h"
@ -24,7 +26,6 @@
#include "transport_interface.h"
#include "timer.h"
#include "vgic.h"
//#include "debug_manager.h"
namespace {
@ -161,21 +162,20 @@ namespace ams::hvisor {
std::scoped_lock lk{m_lock};
InitializeGic();
for (u32 i = 0; i < MaxSgi; i++) {
DoConfigureInterrupt(i, hostPriority, false);
}
DoConfigureInterrupt(GIC_IRQID_MAINTENANCE, hostPriority, true);
vgicInit();
VirtualGic::GetInstance().Initialize();
}
void IrqManager::ConfigureInterrupt(u32 id, u8 prio, bool isLevelSensitive)
void IrqManager::Register(IInterruptTask &task, u32 id, bool isLevelSensitive, u8 prio)
{
cpu::InterruptMaskGuard mg{};
std::scoped_lock lk{m_lock};
DoConfigureInterrupt(id, prio, isLevelSensitive);
if (!task.IsLinked()) {
m_interruptTaskList.push_back(task);
}
}
void IrqManager::SetInterruptAffinity(u32 id, u8 affinity)
@ -188,96 +188,53 @@ namespace ams::hvisor {
void IrqManager::HandleInterrupt(ExceptionStackFrame *frame)
{
// TODO refactor c parts
// Acknowledge the interrupt. Interrupt goes from pending to active.
u32 iar = AcknowledgeIrq();
u32 irqId = iar & 0x3FF;
u32 srcCore = (iar >> 10) & 7;
IInterruptTask *taskForBottomHalf;
//DEBUG("EL2 [core %d]: Received irq %x\n", (int)currentCoreCtx->coreId, irqId);
if (irqId == GicV2Distributor::spuriousIrqId) {
// Spurious interrupt received
return;
} else if (!checkGuestTimerInterrupts(frame, irqId)) {
// Deactivate the interrupt, return early
// Deactivate the interrupt, return ASAP
DropCurrentInterruptPriority(iar);
DeactivateCurrentInterrupt(iar);
return;
}
bool isGuestInterrupt = false;
bool isMaintenanceInterrupt = false;
bool isPaused = false;
bool hasDebugEvent = false;
switch (irqId) {
case ExecuteFunctionSgi:
executeFunctionInterruptHandler(srcCore);
break;
case VgicUpdateSgi:
// Nothing in particular to do here
break;
case DebugPauseSgi:
// TODO debugManagerPauseSgiHandler();
break;
case ReportDebuggerBreakSgi:
case DebuggerContinueSgi:
// See bottom halves
// Because exceptions (other debug events) are handling w/ interrupts off, if
// we get there, there's no race condition possible with debugManagerReportEvent
break;
case GIC_IRQID_MAINTENANCE:
isMaintenanceInterrupt = true;
break;
case TIMER_IRQID(CURRENT_TIMER):
timerInterruptHandler();
break;
default:
isGuestInterrupt = irqId >= 16;
break;
}
TransportInterface *transportIface = irqId >= 32 ? transportInterfaceIrqHandlerTopHalf(irqId) : NULL;
// Priority drop
DropCurrentInterruptPriority(iar);
isGuestInterrupt = isGuestInterrupt && transportIface == NULL && IsGuestInterrupt(irqId);
instance.m_lock.lock();
if (!isGuestInterrupt) {
if (isMaintenanceInterrupt) {
vgicMaintenanceInterruptHandler();
}
// Deactivate the interrupt
DeactivateCurrentInterrupt(iar);
} else {
vgicEnqueuePhysicalIrq(irqId);
// Everything else
std::scoped_lock lk{instance.m_lock};
VirtualGic &vgic = VirtualGic::GetInstance();
if (irqId >= 16 && IsGuestInterrupt(irqId)) {
// Guest interrupts
taskForBottomHalf = nullptr;
DropCurrentInterruptPriority(iar);
vgic.EnqueuePhysicalIrq(irqId);
} else {
// Host interrupts
// Try all handlers and see which one fits
for (IInterruptTask &task: instance.m_interruptTaskList) {
auto b = task.InterruptTopHalfHandler(irqId, srcCore);
if (b) {
taskForBottomHalf = *b ? &task : nullptr;
break;
}
}
DropCurrentInterruptPriority(iar);
DeactivateCurrentInterrupt(iar);
}
vgic.UpdateState();
}
// Update vgic state
vgicUpdateState();
instance.m_lock.unlock();
// TODO
/*isPaused = debugManagerIsCorePaused(currentCoreCtx->coreId);
hasDebugEvent = debugManagerHasDebugEvent(currentCoreCtx->coreId);
if (irqId == ThermosphereSgi_ReportDebuggerBreak) DEBUG("debug event=%d\n", (int)debugManagerGetDebugEvent(currentCoreCtx->coreId)->type);
// Bottom half part
if (transportIface != NULL) {
if (taskForBottomHalf != nullptr) {
// Unmasking the irq signal is left at the discretion of the bottom half handler
exceptionEnterInterruptibleHypervisorCode();
unmaskIrq();
transportInterfaceIrqHandlerBottomHalf(transportIface);
} else if (irqId == ThermosphereSgi_ReportDebuggerBreak && !hasDebugEvent) {
debugManagerReportEvent(DBGEVENT_DEBUGGER_BREAK);
} else if (irqId == DebuggerContinueSgi && isPaused) {
debugManagerUnpauseCores(BIT(currentCoreCtx->coreId));
}*/
taskForBottomHalf->InterruptBottomHalfHandler(irqId, srcCore);
}
}
}

View file

@ -18,6 +18,8 @@
#include "hvisor_gicv2.hpp"
#include "hvisor_synchronization.hpp"
#include "hvisor_i_interrupt_task.hpp"
#include "memory_map.h"
#include "exceptions.h" // TODO
@ -62,7 +64,10 @@ namespace ams::hvisor {
static void DeactivateCurrentInterrupt(u32 iar) { gicc->dir = iar; }
private:
using InterruptTaskList = util::IntrusiveListBaseTraits<IInterruptTask>::ListType;
mutable RecursiveSpinlock m_lock{};
InterruptTaskList m_interruptTaskList{};
u32 m_numSharedInterrupts = 0;
u8 m_priorityShift = 0;
u8 m_numPriorityLevels = 0;
@ -76,8 +81,12 @@ namespace ams::hvisor {
public:
enum ThermosphereSgi : u32 {
ExecuteFunctionSgi = 0,
VgicUpdateSgi,
VgicUpdateSgi = 0,
ReloadHwBreakpointsSgi,
ReloadWatchpointsSgi,
ApplyRevertSwBreakpointSgi,
DebugPauseSgi,
ReportDebuggerBreakSgi,
DebuggerContinueSgi,
@ -98,7 +107,7 @@ namespace ams::hvisor {
public:
void Initialize();
void ConfigureInterrupt(u32 id, u8 prio, bool isLevelSensitive);
void Register(IInterruptTask &task, u32 id, bool isLevelSensitive, u8 prio = IrqManager::hostPriority);
void SetInterruptAffinity(u32 id, u8 affinityMask);
public:

View file

@ -16,6 +16,7 @@
#include "hvisor_sw_breakpoint_manager.hpp"
#include "cpu/hvisor_cpu_instructions.hpp"
#include "cpu/hvisor_cpu_interrupt_mask_guard.hpp"
#include <mutex>
@ -59,7 +60,6 @@ namespace ams::hvisor {
size_t sz = guestReadWriteMemory(bp.address, 4, &bp.savedInstruction, &brkInst);
bp.applied = sz == 4;
m_triedToApplyOrRevertBreakpoint.store(true);
return sz == 4;
}
@ -68,10 +68,33 @@ namespace ams::hvisor {
Breakpoint &bp = m_breakpoints[id];
size_t sz = guestWriteMemory(bp.address, 4, &bp.savedInstruction);
bp.applied = sz != 4;
m_triedToApplyOrRevertBreakpoint.store(true);
return sz == 4;
}
std::optional<bool> SwBreakpointManager::InterruptTopHalfHandler(u32 irqId, u32)
{
if (irqId != IrqManager::ApplyRevertSwBreakpointSgi) {
return {};
}
m_applyBarrier.Join();
return false;
}
bool SwBreakpointManager::ApplyOrRevert(size_t id, bool apply)
{
cpu::InterruptMaskGuard mg{};
m_applyBarrier.Reset(getActiveCoreMask());
IrqManager::GenerateSgiForAllOthers(IrqManager::ApplyRevertSwBreakpointSgi);
if (apply) {
DoApply(id);
} else {
DoRevert(id);
}
m_applyBarrier.Join();
}
// TODO apply revert handlers
int SwBreakpointManager::Add(uintptr_t addr, bool persistent)
@ -103,7 +126,7 @@ namespace ams::hvisor {
bp.applied = false;
bp.uid = static_cast<u16>(0x2000 + m_bpUniqueCounter++);
return Apply(id) ? 0 : -EFAULT;
return ApplyOrRevert(id, true) ? 0 : -EFAULT;
}
int SwBreakpointManager::Remove(uintptr_t addr, bool keepPersistent)
@ -126,7 +149,7 @@ namespace ams::hvisor {
Breakpoint &bp = m_breakpoints[id];
bool ok = true;
if (!keepPersistent || !bp.persistent) {
ok = Revert(id);
ok = ApplyOrRevert(id, false);
}
for(size_t i = id; i < m_numBreakpoints - 1; i++) {
@ -146,7 +169,7 @@ namespace ams::hvisor {
for (size_t id = 0; id < m_numBreakpoints; id++) {
Breakpoint &bp = m_breakpoints[id];
if (!keepPersistent || !bp.persistent) {
ok = ok && Revert(id);
ok = ok && ApplyOrRevert(id, false);
}
}
@ -156,6 +179,4 @@ namespace ams::hvisor {
return ok ? 0 : -EFAULT;
}
}

View file

@ -17,13 +17,13 @@
#pragma once
#include "defines.hpp"
#include "hvisor_synchronization.hpp"
#include "hvisor_irq_manager.hpp"
#define MAX_SW_BREAKPOINTS 16
namespace ams::hvisor {
class SwBreakpointManager {
class SwBreakpointManager : public IInterruptTask {
SINGLETON(SwBreakpointManager);
private:
struct Breakpoint {
@ -36,7 +36,7 @@ namespace ams::hvisor {
private:
mutable RecursiveSpinlock m_lock{};
std::atomic<bool> m_triedToApplyOrRevertBreakpoint{};
mutable Barrier m_applyBarrier{};
u32 m_bpUniqueCounter = 0;
size_t m_numBreakpoints = 0;
@ -48,15 +48,18 @@ namespace ams::hvisor {
bool DoApply(size_t id);
bool DoRevert(size_t id);
// TODO apply, revert handler
bool Apply(size_t id);
bool Revert(size_t id);
bool ApplyOrRevert(size_t id, bool apply);
public:
int Add(uintptr_t addr, bool persistent);
int Remove(uintptr_t addr, bool keepPersistent);
int RemoveAll(bool keepPersistent);
std::optional<bool> InterruptTopHalfHandler(u32 irqId, u32) final;
void Initialize()
{
IrqManager::GetInstance().Register(*this, IrqManager::ApplyRevertSwBreakpointSgi, false);
}
public:
constexpr SwBreakpointManager() = default;
};

View file

@ -18,6 +18,8 @@
#include "hvisor_virtual_gic.hpp"
#include "cpu/hvisor_cpu_instructions.hpp"
#include "platform/interrupt_config.h" // TODO remove
#define GICDOFF(field) (offsetof(GicV2Distributor, field))
namespace ams::hvisor {
@ -625,8 +627,16 @@ namespace ams::hvisor {
gich->hcr.raw = hcr.raw;
}
void VirtualGic::MaintenanceInterruptHandler()
std::optional<bool> VirtualGic::InterruptTopHalfHandler(u32 irqId, u32)
{
if (irqId == IrqManager::VgicUpdateSgi) {
// This SGI is just there to trigger the state update
return false;
} else if (irqId != GIC_IRQID_MAINTENANCE) {
return {};
}
// Maintenance interrupt handler:
GicV2VirtualInterfaceController::MaintenanceIntStatRegister misr = { .raw = gich->misr.raw };
// Force GICV_CTRL to behave like ns-GICC_CTLR, with group 1 being replaced by group 0
@ -669,6 +679,7 @@ namespace ams::hvisor {
ENSURE2(!misr.lrenp, "List Register Entry Not Present maintenance interrupt!\n");
// The rest should be handled by the main loop...
return false;
}
void VirtualGic::EnqueuePhysicalIrq(u32 id)
@ -715,6 +726,11 @@ namespace ams::hvisor {
// All guest SPIs are initially configured as level-sensitive with no targets
}
auto &mgr = IrqManager::GetInstance();
mgr.Register(*this, GIC_IRQID_MAINTENANCE, true);
mgr.Register(*this, IrqManager::VgicUpdateSgi, false);
// Clear the list registers (they reset to 0, though)
for (u8 i = 0; i < m_numListRegisters; i++) {
gich->lr[i].raw = 0;

View file

@ -24,7 +24,7 @@
namespace ams::hvisor {
class VirtualGic final {
class VirtualGic final : public IInterruptTask {
SINGLETON(VirtualGic);
private:
@ -366,29 +366,19 @@ namespace ams::hvisor {
void PushListRegisters(VirqState *chosen[], size_t num);
bool UpdateListRegister(volatile GicV2VirtualInterfaceController::ListRegister *lr);
void UpdateState();
public:
static bool ValidateGicdRegisterAccess(size_t offset, size_t sz);
public:
void WriteGicdRegister(u32 val, size_t offset, size_t sz);
u32 ReadGicdRegister(size_t offset, size_t sz);
void MaintenanceInterruptHandler();
void EnqueuePhysicalIrq(u32 id);
// Must be called by irqManager only...
// not sure if I should have made IrqManager a friend of this class
void UpdateState();
std::optional<bool> InterruptTopHalfHandler(u32 irqId, u32) final;
void EnqueuePhysicalIrq(u32 id);
void Initialize();
};
}
/*bool vgicValidateGicdRegisterAccess(size_t offset, size_t sz);
void vgicWriteGicdRegister(u32 val, size_t offset, size_t sz);
u32 vgicReadGicdRegister(size_t offset, size_t sz);
void handleVgicdMmio(ExceptionStackFrame *frame, cpu::DataAbortIss dabtIss, size_t offset);
void vgicInit(void);
void vgicUpdateState(void);
void vgicMaintenanceInterruptHandler(void);
void vgicEnqueuePhysicalIrq(u16 irqId);*/

View file

@ -47,17 +47,11 @@ namespace ams::hvisor {
WatchpointManager WatchpointManager::instance{};
void WatchpointManager::ReloadOnAllCoresSgiHandler()
void WatchpointManager::Reload() const
{
// TODO
}
void WatchpointManager::ReloadOnAllCores() const
{
cpu::dmb();
// TODO
}
bool WatchpointManager::FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl direction) const
{
size_t off;

View file

@ -22,8 +22,9 @@ namespace ams::hvisor {
class WatchpointManager final : public HwStopPointManager {
SINGLETON(WatchpointManager);
protected:
virtual bool FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl direction) const;
private:
bool FindPredicate(const cpu::DebugRegisterPair &pair, uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl direction) const final;
void Reload() const final;
public:
virtual void ReloadOnAllCores() const;
@ -34,6 +35,6 @@ namespace ams::hvisor {
int Remove(uintptr_t addr, size_t size, cpu::DebugRegisterPair::LoadStoreControl direction);
public:
constexpr WatchpointManager() : HwStopPointManager(MAX_WCR) {}
constexpr WatchpointManager() : HwStopPointManager(MAX_WCR, IrqManager::ReloadWatchpointsSgi) {}
};
}