kern: fix multicore instruction cache invalidation

This commit is contained in:
Michael Scire 2020-08-03 18:39:32 -07:00 committed by SciresM
parent f058536b59
commit b5f2698bf0
3 changed files with 72 additions and 35 deletions

View file

@ -59,11 +59,6 @@ namespace ams::kern::arch::arm64::cpu {
InstructionMemoryBarrier(); InstructionMemoryBarrier();
} }
ALWAYS_INLINE void InvalidateEntireInstructionCache() {
__asm__ __volatile__("ic iallu" ::: "memory");
EnsureInstructionConsistency();
}
ALWAYS_INLINE void Yield() { ALWAYS_INLINE void Yield() {
__asm__ __volatile__("yield" ::: "memory"); __asm__ __volatile__("yield" ::: "memory");
} }
@ -179,6 +174,7 @@ namespace ams::kern::arch::arm64::cpu {
void ClearPageToZeroImpl(void *); void ClearPageToZeroImpl(void *);
void FlushEntireDataCacheSharedForInit(); void FlushEntireDataCacheSharedForInit();
void FlushEntireDataCacheLocalForInit(); void FlushEntireDataCacheLocalForInit();
void InvalidateEntireInstructionCacheForInit();
void StoreEntireCacheForInit(); void StoreEntireCacheForInit();
void FlushEntireDataCache(); void FlushEntireDataCache();
@ -188,6 +184,8 @@ namespace ams::kern::arch::arm64::cpu {
Result FlushDataCache(const void *addr, size_t size); Result FlushDataCache(const void *addr, size_t size);
Result InvalidateInstructionCache(void *addr, size_t size); Result InvalidateInstructionCache(void *addr, size_t size);
void InvalidateEntireInstructionCache();
ALWAYS_INLINE void ClearPageToZero(void *page) { ALWAYS_INLINE void ClearPageToZero(void *page) {
MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize)); MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
MESOSPHERE_ASSERT(page != nullptr); MESOSPHERE_ASSERT(page != nullptr);

View file

@ -155,31 +155,44 @@ namespace ams::kern::arch::arm64::cpu {
void RequestOperation(Operation op) { void RequestOperation(Operation op) {
KScopedLightLock lk(this->lock); KScopedLightLock lk(this->lock);
MESOSPHERE_ABORT_UNLESS(this->operation == Operation::Idle);
/* Send and wait for acknowledgement of request. */
{
KScopedLightLock cv_lk(this->cv_lock);
MESOSPHERE_ABORT_UNLESS(this->target_cores == 0);
/* Set operation. */
this->operation = op;
/* Create core masks for us to use. */ /* Create core masks for us to use. */
constexpr u64 AllCoresMask = (1ul << cpu::NumCores) - 1ul; constexpr u64 AllCoresMask = (1ul << cpu::NumCores) - 1ul;
const u64 other_cores_mask = AllCoresMask & ~(1ul << GetCurrentCoreId()); const u64 other_cores_mask = AllCoresMask & ~(1ul << GetCurrentCoreId());
if ((op == Operation::InvalidateInstructionCache) || (Kernel::GetState() == Kernel::State::Initializing)) { if ((op == Operation::InvalidateInstructionCache) || (Kernel::GetState() == Kernel::State::Initializing)) {
/* Check that there's no on-going operation. */
MESOSPHERE_ABORT_UNLESS(this->operation == Operation::Idle);
MESOSPHERE_ABORT_UNLESS(this->target_cores == 0);
/* Set operation. */
this->operation = op;
/* For certain operations, we want to send an interrupt. */ /* For certain operations, we want to send an interrupt. */
this->target_cores = other_cores_mask; this->target_cores = other_cores_mask;
DataSynchronizationBarrier();
const u64 target_mask = this->target_cores; const u64 target_mask = this->target_cores;
DataSynchronizationBarrier(); DataSynchronizationBarrier();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask); Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
this->ProcessOperation(); this->ProcessOperation();
while (this->target_cores != 0) { while (this->target_cores != 0) {
cpu::Yield(); cpu::Yield();
} }
/* Go idle again. */
this->operation = Operation::Idle;
} else { } else {
/* Lock condvar so that we can send and wait for acknowledgement of request. */
KScopedLightLock cv_lk(this->cv_lock);
/* Check that there's no on-going operation. */
MESOSPHERE_ABORT_UNLESS(this->operation == Operation::Idle);
MESOSPHERE_ABORT_UNLESS(this->target_cores == 0);
/* Set operation. */
this->operation = op;
/* Request all cores. */ /* Request all cores. */
this->target_cores = AllCoresMask; this->target_cores = AllCoresMask;
@ -188,11 +201,11 @@ namespace ams::kern::arch::arm64::cpu {
while (this->target_cores != 0) { while (this->target_cores != 0) {
this->cv.Wait(std::addressof(this->cv_lock)); this->cv.Wait(std::addressof(this->cv_lock));
} }
}
}
/* Go idle again. */ /* Go idle again. */
this->operation = Operation::Idle; this->operation = Operation::Idle;
} }
}
}; };
/* Instances of the interrupt handlers. */ /* Instances of the interrupt handlers. */
@ -285,6 +298,8 @@ namespace ams::kern::arch::arm64::cpu {
DataSynchronizationBarrier(); DataSynchronizationBarrier();
break; break;
} }
this->target_cores &= ~(1ul << GetCurrentCoreId());
} }
ALWAYS_INLINE void SetEventLocally() { ALWAYS_INLINE void SetEventLocally() {
@ -327,6 +342,14 @@ namespace ams::kern::arch::arm64::cpu {
return ResultSuccess(); return ResultSuccess();
} }
ALWAYS_INLINE void InvalidateEntireInstructionCacheLocalImpl() {
__asm__ __volatile__("ic iallu" ::: "memory");
}
ALWAYS_INLINE void InvalidateEntireInstructionCacheGlobalImpl() {
__asm__ __volatile__("ic ialluis" ::: "memory");
}
} }
void FlushEntireDataCacheSharedForInit() { void FlushEntireDataCacheSharedForInit() {
@ -337,11 +360,16 @@ namespace ams::kern::arch::arm64::cpu {
return PerformCacheOperationBySetWayLocal<true>(FlushDataCacheLineBySetWayImpl); return PerformCacheOperationBySetWayLocal<true>(FlushDataCacheLineBySetWayImpl);
} }
void InvalidateEntireInstructionCacheForInit() {
InvalidateEntireInstructionCacheLocalImpl();
EnsureInstructionConsistency();
}
void StoreEntireCacheForInit() { void StoreEntireCacheForInit() {
PerformCacheOperationBySetWayLocal<true>(StoreDataCacheLineBySetWayImpl); PerformCacheOperationBySetWayLocal<true>(StoreDataCacheLineBySetWayImpl);
PerformCacheOperationBySetWayShared<true>(StoreDataCacheLineBySetWayImpl); PerformCacheOperationBySetWayShared<true>(StoreDataCacheLineBySetWayImpl);
DataSynchronizationBarrierInnerShareable(); DataSynchronizationBarrierInnerShareable();
InvalidateEntireInstructionCache(); InvalidateEntireInstructionCacheForInit();
} }
void FlushEntireDataCache() { void FlushEntireDataCache() {
@ -401,6 +429,17 @@ namespace ams::kern::arch::arm64::cpu {
return ResultSuccess(); return ResultSuccess();
} }
void InvalidateEntireInstructionCache() {
KScopedCoreMigrationDisable dm;
/* Invalidate the instruction cache on all cores. */
InvalidateEntireInstructionCacheGlobalImpl();
EnsureInstructionConsistency();
/* Request the interrupt helper to invalidate, too. */
g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InvalidateInstructionCache);
}
void InitializeInterruptThreads(s32 core_id) { void InitializeInterruptThreads(s32 core_id) {
/* Initialize the cache operation handler. */ /* Initialize the cache operation handler. */
g_cache_operation_handler.Initialize(core_id); g_cache_operation_handler.Initialize(core_id);

View file

@ -81,7 +81,7 @@ namespace ams::kern::init::loader {
cpu::DataSynchronizationBarrier(); cpu::DataSynchronizationBarrier();
/* Invalidate entire instruction cache. */ /* Invalidate entire instruction cache. */
cpu::InvalidateEntireInstructionCache(); cpu::InvalidateEntireInstructionCacheForInit();
/* Invalidate entire TLB. */ /* Invalidate entire TLB. */
cpu::InvalidateEntireTlb(); cpu::InvalidateEntireTlb();