nold/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere synced 2024-12-22 04:11:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

kern: update for new hw maintenance semantics

Browse source
This commit is contained in:
Michael Scire 2022-03-22 21:33:43 -07:00 committed by SciresM
parent 6e17317d5d
commit 9d89835ff8
19 changed files with 386 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp
View file

@ -279,20 +279,21 @@ namespace ams::kern::arch::arm64::init {
/* Invalidate the entire tlb. */
cpu::DataSynchronizationBarrierInnerShareable();
cpu::InvalidateEntireTlbInnerShareable();
cpu::InvalidateEntireTlb();
/* Copy data, if we should. */
const u64 negative_block_size_for_mask = static_cast<u64>(-static_cast<s64>(block_size));
const u64 offset_mask = negative_block_size_for_mask & ((1ul << 48) - 1);
const KVirtualAddress copy_src_addr = KVirtualAddress(src_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
const KVirtualAddress copy_dst_addr = KVirtualAddress(dst_saved.GetRawAttributesUnsafeForSwap() & offset_mask);
if (block_size && do_copy) {
if (do_copy) {
u8 tmp[0x100];
for (size_t ofs = 0; ofs < block_size; ofs += sizeof(tmp)) {
std::memcpy(tmp, GetVoidPointer(copy_src_addr + ofs), sizeof(tmp));
std::memcpy(GetVoidPointer(copy_src_addr + ofs), GetVoidPointer(copy_dst_addr + ofs), sizeof(tmp));
std::memcpy(GetVoidPointer(copy_dst_addr + ofs), tmp, sizeof(tmp));
}
cpu::DataSynchronizationBarrierInnerShareable();
}
/* Swap the mappings. */
@ -339,7 +340,6 @@ namespace ams::kern::arch::arm64::init {
/* Can we make an L1 block? */
if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) {
*l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
cpu::DataSynchronizationBarrierInnerShareable();
virt_addr += L1BlockSize;
phys_addr += L1BlockSize;
@ -350,8 +350,8 @@ namespace ams::kern::arch::arm64::init {
/* If we don't already have an L2 table, we need to make a new one. */
if (!l1_entry->IsTable()) {
KPhysicalAddress new_table = AllocateNewPageTable(allocator);
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
cpu::DataSynchronizationBarrierInnerShareable();
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
}
L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr);
@ -365,14 +365,12 @@ namespace ams::kern::arch::arm64::init {
phys_addr += L2BlockSize;
size -= L2BlockSize;
}
cpu::DataSynchronizationBarrierInnerShareable();
continue;
}
/* Can we make an L2 block? */
if (util::IsAligned(GetInteger(virt_addr), L2BlockSize) && util::IsAligned(GetInteger(phys_addr), L2BlockSize) && size >= L2BlockSize) {
*l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
cpu::DataSynchronizationBarrierInnerShareable();
virt_addr += L2BlockSize;
phys_addr += L2BlockSize;
@ -383,8 +381,8 @@ namespace ams::kern::arch::arm64::init {
/* If we don't already have an L3 table, we need to make a new one. */
if (!l2_entry->IsTable()) {
KPhysicalAddress new_table = AllocateNewPageTable(allocator);
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
cpu::DataSynchronizationBarrierInnerShareable();
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever());
}
L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr);
@ -398,17 +396,18 @@ namespace ams::kern::arch::arm64::init {
phys_addr += L3BlockSize;
size -= L3BlockSize;
}
cpu::DataSynchronizationBarrierInnerShareable();
continue;
}
/* Make an L3 block. */
*l3_entry = L3PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, attr, PageTableEntry::SoftwareReservedBit_None, false);
cpu::DataSynchronizationBarrierInnerShareable();
virt_addr += L3BlockSize;
phys_addr += L3BlockSize;
size -= L3BlockSize;
}
/* Ensure data consistency after our mapping is added. */
cpu::DataSynchronizationBarrierInnerShareable();
}
KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const {
@ -556,9 +555,6 @@ namespace ams::kern::arch::arm64::init {
}
void Reprotect(KVirtualAddress virt_addr, size_t size, const PageTableEntry &attr_before, const PageTableEntry &attr_after) {
/* Ensure data consistency before we begin reprotection. */
cpu::DataSynchronizationBarrierInnerShareable();
/* Ensure that addresses and sizes are page aligned. */
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), PageSize));
MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, PageSize));
@ -699,7 +695,7 @@ namespace ams::kern::arch::arm64::init {
this->PhysicallyRandomize(virt_addr, size, L2BlockSize, do_copy);
this->PhysicallyRandomize(virt_addr, size, L3ContiguousBlockSize, do_copy);
this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
cpu::StoreEntireCacheForInit();
cpu::StoreCacheForInit(GetVoidPointer(virt_addr), size);
}
};

34
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp
View file

@ -48,6 +48,10 @@ namespace ams::kern::arch::arm64::cpu {
__asm__ __volatile__("dsb ish" ::: "memory");
}
ALWAYS_INLINE void DataSynchronizationBarrierInnerShareableStore() {
__asm__ __volatile__("dsb ishst" ::: "memory");
}
ALWAYS_INLINE void DataMemoryBarrier() {
__asm__ __volatile__("dmb sy" ::: "memory");
}
@ -56,16 +60,20 @@ namespace ams::kern::arch::arm64::cpu {
__asm__ __volatile__("dmb ish" ::: "memory");
}
ALWAYS_INLINE void DataMemoryBarrierInnerShareableStore() {
__asm__ __volatile__("dmb ishst" ::: "memory");
}
ALWAYS_INLINE void InstructionMemoryBarrier() {
__asm__ __volatile__("isb" ::: "memory");
}
ALWAYS_INLINE void EnsureInstructionConsistencyInnerShareable() {
ALWAYS_INLINE void EnsureInstructionConsistency() {
DataSynchronizationBarrierInnerShareable();
InstructionMemoryBarrier();
}
ALWAYS_INLINE void EnsureInstructionConsistency() {
ALWAYS_INLINE void EnsureInstructionConsistencyFullSystem() {
DataSynchronizationBarrier();
InstructionMemoryBarrier();
}
@ -182,28 +190,23 @@ namespace ams::kern::arch::arm64::cpu {
NOINLINE void SynchronizeAllCores();
/* Cache management helpers. */
void StoreEntireCacheForInit();
void FlushEntireCacheForInit();
void StoreCacheForInit(void *addr, size_t size);
void FlushEntireDataCache();
Result InvalidateDataCache(void *addr, size_t size);
Result StoreDataCache(const void *addr, size_t size);
Result FlushDataCache(const void *addr, size_t size);
Result InvalidateInstructionCache(void *addr, size_t size);
void InvalidateEntireInstructionCache();
void ClearPageToZeroImpl(void *);
ALWAYS_INLINE void ClearPageToZero(void * const page) {
MESOSPHERE_ASSERT(util::IsAligned(reinterpret_cast<uintptr_t>(page), PageSize));
MESOSPHERE_ASSERT(page != nullptr);
uintptr_t cur = reinterpret_cast<uintptr_t>(__builtin_assume_aligned(page, PageSize));
const uintptr_t last = cur + PageSize - DataCacheLineSize;
for (/* ... */; cur <= last; cur += DataCacheLineSize) {
__asm__ __volatile__("dc zva, %[cur]" :: [cur]"r"(cur) : "memory");
}
ClearPageToZeroImpl(page);
}
ALWAYS_INLINE void InvalidateTlbByAsid(u32 asid) {
@ -223,20 +226,15 @@ namespace ams::kern::arch::arm64::cpu {
EnsureInstructionConsistency();
}
ALWAYS_INLINE void InvalidateEntireTlbInnerShareable() {
__asm__ __volatile__("tlbi vmalle1is" ::: "memory");
EnsureInstructionConsistencyInnerShareable();
}
ALWAYS_INLINE void InvalidateEntireTlbDataOnly() {
__asm__ __volatile__("tlbi vmalle1is" ::: "memory");
DataSynchronizationBarrier();
DataSynchronizationBarrierInnerShareable();
}
ALWAYS_INLINE void InvalidateTlbByVaDataOnly(KProcessAddress virt_addr) {
const u64 value = ((GetInteger(virt_addr) >> 12) & 0xFFFFFFFFFFFul);
__asm__ __volatile__("tlbi vaae1is, %[value]" :: [value]"r"(value) : "memory");
DataSynchronizationBarrier();
DataSynchronizationBarrierInnerShareable();
}
ALWAYS_INLINE uintptr_t GetCurrentThreadPointerValue() {

2
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_interrupt_manager.hpp
View file

@ -96,8 +96,6 @@ namespace ams::kern::arch::arm64 {
}
static void HandleInterrupt(bool user_mode);
/* Implement more KInterruptManager functionality. */
private:
Result BindGlobal(KInterruptHandler *handler, s32 irq, s32 core_id, s32 priority, bool manual_clear, bool level);
Result BindLocal(KInterruptHandler *handler, s32 irq, s32 priority, bool manual_clear);

24
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table.hpp
View file

@ -174,7 +174,6 @@ namespace ams::kern::arch::arm64 {
static NOINLINE void Initialize(s32 core_id);
ALWAYS_INLINE void Activate(u32 proc_id) {
cpu::DataSynchronizationBarrier();
cpu::SwitchProcess(m_ttbr, proc_id);
}
@ -219,12 +218,13 @@ namespace ams::kern::arch::arm64 {
Result ChangePermissions(KProcessAddress virt_addr, size_t num_pages, PageTableEntry entry_template, DisableMergeAttribute disable_merge_attr, bool refresh_mapping, PageLinkedList *page_list, bool reuse_ll);
static ALWAYS_INLINE void PteDataSynchronizationBarrier() {
cpu::DataSynchronizationBarrierInnerShareable();
static ALWAYS_INLINE void PteDataMemoryBarrier() {
cpu::DataMemoryBarrierInnerShareableStore();
}
static ALWAYS_INLINE void ClearPageTable(KVirtualAddress table) {
cpu::ClearPageToZero(GetVoidPointer(table));
cpu::DataSynchronizationBarrierInnerShareable();
}
ALWAYS_INLINE void OnTableUpdated() const {
@ -239,22 +239,8 @@ namespace ams::kern::arch::arm64 {
cpu::InvalidateTlbByVaDataOnly(virt_addr);
}
ALWAYS_INLINE void NoteUpdated() const {
cpu::DataSynchronizationBarrier();
if (this->IsKernel()) {
this->OnKernelTableUpdated();
} else {
this->OnTableUpdated();
}
}
ALWAYS_INLINE void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const {
MESOSPHERE_ASSERT(this->IsKernel());
cpu::DataSynchronizationBarrier();
this->OnKernelTableSinglePageUpdated(virt_addr);
}
void NoteUpdated() const;
void NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const;
KVirtualAddress AllocatePageTable(PageLinkedList *page_list, bool reuse_ll) const {
KVirtualAddress table = this->GetPageTableManager().Allocate();

1
libraries/libmesosphere/include/mesosphere/arch/arm64/kern_userspace_memory_access.hpp
View file

@ -46,7 +46,6 @@ namespace ams::kern::arch::arm64 {
static bool StoreDataCache(uintptr_t start, uintptr_t end);
static bool FlushDataCache(uintptr_t start, uintptr_t end);
static bool InvalidateDataCache(uintptr_t start, uintptr_t end);
static bool InvalidateInstructionCache(uintptr_t start, uintptr_t end);
static bool ReadIoMemory32Bit(void *dst, const void *src, size_t size);
static bool ReadIoMemory16Bit(void *dst, const void *src, size_t size);

2
libraries/libmesosphere/include/mesosphere/kern_k_scheduler_impls.hpp
View file

@ -35,7 +35,7 @@ namespace ams::kern {
ALWAYS_INLINE void KScheduler::RescheduleOtherCores(u64 cores_needing_scheduling) {
if (const u64 core_mask = cores_needing_scheduling & ~(1ul << m_core_id); core_mask != 0) {
cpu::DataSynchronizationBarrier();
cpu::DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_Scheduler, core_mask);
}
}

136
libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp
View file

@ -176,7 +176,7 @@ namespace ams::kern::arch::arm64::cpu {
const u64 target_mask = m_target_cores.Load();
DataSynchronizationBarrier();
DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask);
this->ProcessOperation();
@ -213,32 +213,37 @@ namespace ams::kern::arch::arm64::cpu {
};
/* Instances of the interrupt handlers. */
KThreadTerminationInterruptHandler g_thread_termination_handler;
KCacheHelperInterruptHandler g_cache_operation_handler;
KPerformanceCounterInterruptHandler g_performance_counter_handler[cpu::NumCores];
constinit KThreadTerminationInterruptHandler g_thread_termination_handler;
constinit KCacheHelperInterruptHandler g_cache_operation_handler;
constinit KPerformanceCounterInterruptHandler g_performance_counter_handler[cpu::NumCores];
/* Expose this as a global, for asm to use. */
s32 g_all_core_sync_count;
constinit s32 g_all_core_sync_count;
template<bool Init, typename F>
template<typename F>
ALWAYS_INLINE void PerformCacheOperationBySetWayImpl(int level, F f) {
/* Used in multiple locations. */
const u64 level_sel_value = static_cast<u64>(level << 1);
/* Get the cache size id register value with interrupts disabled. */
u64 ccsidr_value;
if constexpr (Init) {
/* During init, we can just set the selection register directly. */
cpu::SetCsselrEl1(level_sel_value);
cpu::InstructionMemoryBarrier();
ccsidr_value = cpu::GetCcsidrEl1();
} else {
/* After init, we need to care about interrupts. */
{
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Configure the cache select register for our level. */
cpu::SetCsselrEl1(level_sel_value);
/* Ensure our configuration takes before reading the cache size id register. */
cpu::InstructionMemoryBarrier();
/* Get the cache size id register. */
ccsidr_value = cpu::GetCcsidrEl1();
}
/* Ensure that no memory inconsistencies occur between cache management invocations. */
cpu::DataSynchronizationBarrier();
/* Get cache size id info. */
CacheSizeIdRegisterAccessor ccsidr_el1(ccsidr_value);
const int num_sets = ccsidr_el1.GetNumberOfSets();
@ -266,13 +271,11 @@ namespace ams::kern::arch::arm64::cpu {
}
void StoreDataCacheBySetWay(int level) {
PerformCacheOperationBySetWayImpl<false>(level, StoreDataCacheLineBySetWayImpl);
cpu::DataSynchronizationBarrier();
PerformCacheOperationBySetWayImpl(level, StoreDataCacheLineBySetWayImpl);
}
void FlushDataCacheBySetWay(int level) {
PerformCacheOperationBySetWayImpl<false>(level, FlushDataCacheLineBySetWayImpl);
cpu::DataSynchronizationBarrier();
PerformCacheOperationBySetWayImpl(level, FlushDataCacheLineBySetWayImpl);
}
void KCacheHelperInterruptHandler::ProcessOperation() {
@ -284,9 +287,11 @@ namespace ams::kern::arch::arm64::cpu {
break;
case Operation::StoreDataCache:
StoreDataCacheBySetWay(0);
cpu::DataSynchronizationBarrier();
break;
case Operation::FlushDataCache:
FlushDataCacheBySetWay(0);
cpu::DataSynchronizationBarrier();
break;
}
}
@ -323,14 +328,6 @@ namespace ams::kern::arch::arm64::cpu {
R_SUCCEED();
}
ALWAYS_INLINE Result InvalidateInstructionCacheRange(uintptr_t start, uintptr_t end) {
MESOSPHERE_ASSERT(util::IsAligned(start, InstructionCacheLineSize));
MESOSPHERE_ASSERT(util::IsAligned(end, InstructionCacheLineSize));
R_UNLESS(UserspaceAccess::InvalidateInstructionCache(start, end), svc::ResultInvalidCurrentMemory());
EnsureInstructionConsistency();
R_SUCCEED();
}
ALWAYS_INLINE void InvalidateEntireInstructionCacheLocalImpl() {
__asm__ __volatile__("ic iallu" ::: "memory");
}
@ -341,26 +338,12 @@ namespace ams::kern::arch::arm64::cpu {
}
void StoreEntireCacheForInit() {
/* Store local. */
{
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
for (int level = 0; level != levels_of_unification; ++level) {
PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
}
}
/* Store shared. */
{
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency();
const int levels_of_unification = clidr_el1.GetLevelsOfUnification();
for (int level = levels_of_unification; level <= levels_of_coherency; ++level) {
PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
}
void StoreCacheForInit(void *addr, size_t size) {
/* Store the data cache for the specified range. */
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = start + size;
for (uintptr_t cur = start; cur < end; cur += DataCacheLineSize) {
__asm__ __volatile__("dc cvac, %[cur]" :: [cur]"r"(cur) : "memory");
}
/* Data synchronization barrier. */
@ -370,36 +353,7 @@ namespace ams::kern::arch::arm64::cpu {
InvalidateEntireInstructionCacheLocalImpl();
/* Ensure local instruction consistency. */
DataSynchronizationBarrierInnerShareable();
InstructionMemoryBarrier();
}
void FlushEntireCacheForInit() {
/* Flush data cache. */
{
/* Get levels of coherence/unificaiton. */
CacheLineIdRegisterAccessor clidr_el1;
const int levels_of_coherency = clidr_el1.GetLevelsOfCoherency();
/* Store cache from L1 up to (level of coherence - 1). */
for (int level = 0; level < levels_of_coherency - 1; ++level) {
PerformCacheOperationBySetWayImpl<true>(level, StoreDataCacheLineBySetWayImpl);
DataSynchronizationBarrier();
}
/* Flush cache from (level of coherence - 1) down to L0. */
for (int level = levels_of_coherency; level > 0; --level) {
PerformCacheOperationBySetWayImpl<true>(level - 1, FlushDataCacheLineBySetWayImpl);
DataSynchronizationBarrier();
}
}
/* Invalidate instruction cache. */
InvalidateEntireInstructionCacheLocalImpl();
EnsureInstructionConsistency();
/* Invalidate entire TLB. */
InvalidateEntireTlb();
}
void FlushEntireDataCache() {
@ -417,10 +371,17 @@ namespace ams::kern::arch::arm64::cpu {
for (int level = levels_of_coherency; level > 1; --level) {
FlushDataCacheBySetWay(level - 1);
}
/* Data synchronization barrier for full system. */
DataSynchronizationBarrier();
}
Result InvalidateDataCache(void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
__asm__ __volatile__("" ::: "memory");
GetCurrentThread().SetInCacheMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInCacheMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
const uintptr_t start = reinterpret_cast<uintptr_t>(addr);
const uintptr_t end = start + size;
uintptr_t aligned_start = util::AlignDown(start, DataCacheLineSize);
@ -444,7 +405,11 @@ namespace ams::kern::arch::arm64::cpu {
}
Result StoreDataCache(const void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
__asm__ __volatile__("" ::: "memory");
GetCurrentThread().SetInCacheMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInCacheMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, DataCacheLineSize);
@ -452,26 +417,17 @@ namespace ams::kern::arch::arm64::cpu {
}
Result FlushDataCache(const void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
/* Mark ourselves as in a cache maintenance operation, and prevent re-ordering. */
__asm__ __volatile__("" ::: "memory");
GetCurrentThread().SetInCacheMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInCacheMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), DataCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, DataCacheLineSize);
R_RETURN(FlushDataCacheRange(start, end));
}
Result InvalidateInstructionCache(void *addr, size_t size) {
KScopedCoreMigrationDisable dm;
const uintptr_t start = util::AlignDown(reinterpret_cast<uintptr_t>(addr), InstructionCacheLineSize);
const uintptr_t end = util::AlignUp( reinterpret_cast<uintptr_t>(addr) + size, InstructionCacheLineSize);
R_TRY(InvalidateInstructionCacheRange(start, end));
/* Request the interrupt helper to perform an instruction memory barrier. */
g_cache_operation_handler.RequestOperation(KCacheHelperInterruptHandler::Operation::InstructionMemoryBarrier);
R_SUCCEED();
}
void InvalidateEntireInstructionCache() {
KScopedCoreMigrationDisable dm;

135
libraries/libmesosphere/source/arch/arm64/kern_cpu_asm.s
View file

@ -61,3 +61,138 @@ _ZN3ams4kern4arch5arm643cpu23SynchronizeAllCoresImplEPii:
5:
stlr wzr, [x0]
ret
/* ams::kern::arch::arm64::cpu::ClearPageToZeroImpl(void *) */
.section .text._ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv, "ax", %progbits
.global _ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv
.type _ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv, %function
_ZN3ams4kern4arch5arm643cpu19ClearPageToZeroImplEPv:
/* Efficiently clear the page using dc zva. */
dc zva, x0
add x8, x0, #0x040
dc zva, x8
add x8, x0, #0x080
dc zva, x8
add x8, x0, #0x0c0
dc zva, x8
add x8, x0, #0x100
dc zva, x8
add x8, x0, #0x140
dc zva, x8
add x8, x0, #0x180
dc zva, x8
add x8, x0, #0x1c0
dc zva, x8
add x8, x0, #0x200
dc zva, x8
add x8, x0, #0x240
dc zva, x8
add x8, x0, #0x280
dc zva, x8
add x8, x0, #0x2c0
dc zva, x8
add x8, x0, #0x300
dc zva, x8
add x8, x0, #0x340
dc zva, x8
add x8, x0, #0x380
dc zva, x8
add x8, x0, #0x3c0
dc zva, x8
add x8, x0, #0x400
dc zva, x8
add x8, x0, #0x440
dc zva, x8
add x8, x0, #0x480
dc zva, x8
add x8, x0, #0x4c0
dc zva, x8
add x8, x0, #0x500
dc zva, x8
add x8, x0, #0x540
dc zva, x8
add x8, x0, #0x580
dc zva, x8
add x8, x0, #0x5c0
dc zva, x8
add x8, x0, #0x600
dc zva, x8
add x8, x0, #0x640
dc zva, x8
add x8, x0, #0x680
dc zva, x8
add x8, x0, #0x6c0
dc zva, x8
add x8, x0, #0x700
dc zva, x8
add x8, x0, #0x740
dc zva, x8
add x8, x0, #0x780
dc zva, x8
add x8, x0, #0x7c0
dc zva, x8
add x8, x0, #0x800
dc zva, x8
add x8, x0, #0x840
dc zva, x8
add x8, x0, #0x880
dc zva, x8
add x8, x0, #0x8c0
dc zva, x8
add x8, x0, #0x900
dc zva, x8
add x8, x0, #0x940
dc zva, x8
add x8, x0, #0x980
dc zva, x8
add x8, x0, #0x9c0
dc zva, x8
add x8, x0, #0xa00
dc zva, x8
add x8, x0, #0xa40
dc zva, x8
add x8, x0, #0xa80
dc zva, x8
add x8, x0, #0xac0
dc zva, x8
add x8, x0, #0xb00
dc zva, x8
add x8, x0, #0xb40
dc zva, x8
add x8, x0, #0xb80
dc zva, x8
add x8, x0, #0xbc0
dc zva, x8
add x8, x0, #0xc00
dc zva, x8
add x8, x0, #0xc40
dc zva, x8
add x8, x0, #0xc80
dc zva, x8
add x8, x0, #0xcc0
dc zva, x8
add x8, x0, #0xd00
dc zva, x8
add x8, x0, #0xd40
dc zva, x8
add x8, x0, #0xd80
dc zva, x8
add x8, x0, #0xdc0
dc zva, x8
add x8, x0, #0xe00
dc zva, x8
add x8, x0, #0xe40
dc zva, x8
add x8, x0, #0xe80
dc zva, x8
add x8, x0, #0xec0
dc zva, x8
add x8, x0, #0xf00
dc zva, x8
add x8, x0, #0xf40
dc zva, x8
add x8, x0, #0xf80
dc zva, x8
add x8, x0, #0xfc0
dc zva, x8
ret

12
libraries/libmesosphere/source/arch/arm64/kern_k_debug.cpp
View file

@ -257,21 +257,21 @@ namespace ams::kern::arch::arm64 {
#define MESOSPHERE_SET_HW_BREAK_POINT(ID, FLAGS, VALUE) \
({ \
cpu::SetDbgBcr##ID##El1(0); \
cpu::EnsureInstructionConsistency(); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgBvr##ID##El1(VALUE); \
cpu::EnsureInstructionConsistency(); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgBcr##ID##El1(FLAGS); \
cpu::EnsureInstructionConsistency(); \
cpu::EnsureInstructionConsistencyFullSystem(); \
})
#define MESOSPHERE_SET_HW_WATCH_POINT(ID, FLAGS, VALUE) \
({ \
cpu::SetDbgWcr##ID##El1(0); \
cpu::EnsureInstructionConsistency(); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgWvr##ID##El1(VALUE); \
cpu::EnsureInstructionConsistency(); \
cpu::EnsureInstructionConsistencyFullSystem(); \
cpu::SetDbgWcr##ID##El1(FLAGS); \
cpu::EnsureInstructionConsistency(); \
cpu::EnsureInstructionConsistencyFullSystem(); \
})
Result KDebug::SetHardwareBreakPoint(ams::svc::HardwareBreakPointRegisterName name, u64 flags, u64 value) {

51
libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp
View file

@ -158,6 +158,32 @@ namespace ams::kern::arch::arm64 {
}
ALWAYS_INLINE void KPageTable::NoteUpdated() const {
cpu::DataSynchronizationBarrierInnerShareableStore();
/* Mark ourselves as in a tlb maintenance operation. */
GetCurrentThread().SetInTlbMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInTlbMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
if (this->IsKernel()) {
this->OnKernelTableUpdated();
} else {
this->OnTableUpdated();
}
}
ALWAYS_INLINE void KPageTable::NoteSingleKernelPageUpdated(KProcessAddress virt_addr) const {
MESOSPHERE_ASSERT(this->IsKernel());
cpu::DataSynchronizationBarrierInnerShareableStore();
/* Mark ourselves as in a tlb maintenance operation. */
GetCurrentThread().SetInTlbMaintenanceOperation();
ON_SCOPE_EXIT { GetCurrentThread().ClearInTlbMaintenanceOperation(); __asm__ __volatile__("" ::: "memory"); };
this->OnKernelTableSinglePageUpdated(virt_addr);
}
void KPageTable::Initialize(s32 core_id) {
/* Nothing actually needed here. */
MESOSPHERE_UNUSED(core_id);
@ -412,9 +438,8 @@ namespace ams::kern::arch::arm64 {
/* Set the entry. */
l2_phys = GetPageTablePhysicalAddress(l2_virt);
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, l2_phys, this->IsKernel(), true);
PteDataSynchronizationBarrier();
} else {
l2_virt = GetPageTableVirtualAddress(l2_phys);
}
@ -477,9 +502,8 @@ namespace ams::kern::arch::arm64 {
/* Set the entry. */
l2_phys = GetPageTablePhysicalAddress(l2_virt);
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, l2_phys, this->IsKernel(), true);
PteDataSynchronizationBarrier();
l2_allocated = true;
} else {
l2_virt = GetPageTableVirtualAddress(l2_phys);
@ -505,9 +529,8 @@ namespace ams::kern::arch::arm64 {
/* Set the entry. */
l3_phys = GetPageTablePhysicalAddress(l3_virt);
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, l3_phys, this->IsKernel(), true);
PteDataSynchronizationBarrier();
l2_open_count++;
} else {
l3_virt = GetPageTableVirtualAddress(l3_phys);
@ -631,7 +654,7 @@ namespace ams::kern::arch::arm64 {
for (size_t i = 0; i < num_l2_blocks; i++) {
*impl.GetL2EntryFromTable(l2_virt, virt_addr + L2BlockSize * i) = InvalidL2PageTableEntry;
}
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
/* Close references to the L2 table. */
if (this->GetPageTableManager().IsInPageTableHeap(l2_virt)) {
@ -665,7 +688,7 @@ namespace ams::kern::arch::arm64 {
for (size_t i = 0; i < num_l3_blocks; i++) {
*impl.GetL3EntryFromTable(l3_virt, virt_addr + L3BlockSize * i) = InvalidL3PageTableEntry;
}
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
/* Close references to the L3 table. */
if (this->GetPageTableManager().IsInPageTableHeap(l3_virt)) {
@ -783,6 +806,9 @@ namespace ams::kern::arch::arm64 {
this->MergePages(orig_virt_addr + (num_pages - 1) * PageSize, page_list);
}
/* Wait for pending stores to complete. */
cpu::DataSynchronizationBarrierInnerShareableStore();
/* Open references to the pages, if we should. */
if (IsHeapPhysicalAddress(orig_phys_addr)) {
Kernel::GetMemoryManager().Open(orig_phys_addr, num_pages);
@ -878,6 +904,9 @@ namespace ams::kern::arch::arm64 {
this->MergePages(orig_virt_addr + (num_pages - 1) * PageSize, page_list);
}
/* Wait for pending stores to complete. */
cpu::DataSynchronizationBarrierInnerShareableStore();
/* We succeeded! We want to persist the reference to the pages. */
spg.CancelClose();
R_SUCCEED();
@ -967,7 +996,6 @@ namespace ams::kern::arch::arm64 {
auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_entry->IsHeadMergeDisabled(), head_entry->IsHeadAndBodyMergeDisabled(), tail_entry->IsTailMergeDisabled());
/* Merge! */
PteDataSynchronizationBarrier();
*l2_entry = L2PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
/* Note that we updated. */
@ -1049,7 +1077,6 @@ namespace ams::kern::arch::arm64 {
auto sw_reserved_bits = PageTableEntry::EncodeSoftwareReservedBits(head_entry->IsHeadMergeDisabled(), head_entry->IsHeadAndBodyMergeDisabled(), tail_entry->IsTailMergeDisabled());
/* Merge! */
/* NOTE: As of 13.1.0, Nintendo does not do: PteDataSynchronizationBarrier(); */
*l1_entry = L1PageTableEntry(PageTableEntry::BlockTag{}, phys_addr, PageTableEntry(entry_template), sw_reserved_bits, false);
/* Note that we updated. */
@ -1097,7 +1124,7 @@ namespace ams::kern::arch::arm64 {
this->GetPageTableManager().Open(l2_table, L1BlockSize / L2BlockSize);
/* Replace the L1 entry with one to the new table. */
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
*l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, l2_phys, this->IsKernel(), true);
this->NoteUpdated();
}
@ -1147,7 +1174,7 @@ namespace ams::kern::arch::arm64 {
this->GetPageTableManager().Open(l3_table, L2BlockSize / L3BlockSize);
/* Replace the L2 entry with one to the new table. */
PteDataSynchronizationBarrier();
PteDataMemoryBarrier();
*l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, l3_phys, this->IsKernel(), true);
this->NoteUpdated();
}

20
libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s
View file

@ -577,26 +577,6 @@ _ZN3ams4kern4arch5arm6415UserspaceAccess19InvalidateDataCacheEmm:
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::InvalidateInstructionCache(uintptr_t start, uintptr_t end) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess26InvalidateInstructionCacheEmm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess26InvalidateInstructionCacheEmm
.type _ZN3ams4kern4arch5arm6415UserspaceAccess26InvalidateInstructionCacheEmm, %function
.balign 0x10
_ZN3ams4kern4arch5arm6415UserspaceAccess26InvalidateInstructionCacheEmm:
/* Check if we have any work to do. */
cmp x1, x0
b.eq 2f
1: /* Loop, invalidating each cache line. */
ic ivau, x0
add x0, x0, #0x40
cmp x1, x0
b.ne 1b
2: /* We're done! */
mov x0, #1
ret
/* ams::kern::arch::arm64::UserspaceAccess::ReadIoMemory32Bit(void *dst, const void *src, size_t size) */
.section .text._ZN3ams4kern4arch5arm6415UserspaceAccess17ReadIoMemory32BitEPvPKvm, "ax", %progbits
.global _ZN3ams4kern4arch5arm6415UserspaceAccess17ReadIoMemory32BitEPvPKvm

3
libraries/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s
View file

@ -278,6 +278,9 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm:
.global _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm
.type _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm, %function
_ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm:
/* cpu::DataSynchronizationBarrier(); */
dsb sy
/* const u64 level_sel_value = level << 1; */
lsl x8, x0, #1

6
libraries/libmesosphere/source/kern_k_initial_process_reader.cpp
View file

@ -179,13 +179,7 @@ namespace ams::kern {
}
}
/* Flush caches. */
/* NOTE: This seems incorrect according to arm spec, which says not to flush via set/way after boot. */
/* However, Nintendo flushes the entire cache here and not doing so has caused reports of abort with ESR_EL1 */
/* as 0x02000000 (unknown abort) to occur. */
MESOSPHERE_UNUSED(params);
cpu::FlushEntireDataCache();
cpu::InvalidateEntireInstructionCache();
R_SUCCEED();
}

2
libraries/libmesosphere/source/kern_k_thread.cpp
View file

@ -1315,7 +1315,7 @@ namespace ams::kern {
/* If the thread is runnable, send a termination interrupt to other cores. */
if (this->GetState() == ThreadState_Runnable) {
if (const u64 core_mask = m_physical_affinity_mask.GetAffinityMask() & ~(1ul << GetCurrentCoreId()); core_mask != 0) {
cpu::DataSynchronizationBarrier();
cpu::DataSynchronizationBarrierInnerShareable();
Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_ThreadTerminate, core_mask);
}
}

7
libraries/libstratosphere/source/os/impl/os_cache_impl.os.horizon.hpp
View file

@ -32,6 +32,13 @@ namespace ams::os::impl {
/* Calculate cache line size. */
cache_line_size = 4 << ((cache_type_register >> 16) & 0xF);
/* Get the thread local region. */
auto * const tlr = svc::GetThreadLocalRegion();
/* Note to the kernel that we're performing cache maintenance, in case we get interrupted while touching cache lines. */
tlr->cache_maintenance_flag = 1;
ON_SCOPE_EXIT { tlr->cache_maintenance_flag = 0; }
/* Iterate, flushing cache lines. */
for (uintptr_t cur = reinterpret_cast<uintptr_t>(addr) & ~(cache_line_size - 1); cur < end_addr; cur += cache_line_size) {
__asm__ __volatile__ ("dc civac, %[cur]" :: [cur]"r"(cur));

18
libraries/libvapours/source/dd/impl/dd_cache_impl.os.horizon.hpp
View file

@ -31,6 +31,15 @@ namespace ams::dd::impl {
__asm__ __volatile__("mrs %[ctr_el0], ctr_el0" : [ctr_el0]"=r"(ctr_el0));
const uintptr_t cache_line_size = 4 << ((ctr_el0 >> 16) & 0xF);
#if defined(ATMOSPHERE_IS_STRATOSPHERE)
/* Get the thread local region. */
auto * const tlr = svc::GetThreadLocalRegion();
/* Note to the kernel that we're performing cache maintenance, in case we get interrupted while touching cache lines. */
tlr->cache_maintenance_flag = 1;
ON_SCOPE_EXIT { tlr->cache_maintenance_flag = 0; }
#endif
/* Invalidate the cache. */
const uintptr_t start_addr = reinterpret_cast<uintptr_t>(addr) & ~(cache_line_size - 1);
const uintptr_t end_addr = reinterpret_cast<uintptr_t>(addr) + size;
@ -62,6 +71,15 @@ namespace ams::dd::impl {
__asm__ __volatile__("mrs %[ctr_el0], ctr_el0" : [ctr_el0]"=r"(ctr_el0));
const uintptr_t cache_line_size = 4 << ((ctr_el0 >> 16) & 0xF);
#if defined(ATMOSPHERE_IS_STRATOSPHERE)
/* Get the thread local region. */
auto * const tlr = svc::GetThreadLocalRegion();
/* Note to the kernel that we're performing cache maintenance, in case we get interrupted while touching cache lines. */
tlr->cache_maintenance_flag = 1;
ON_SCOPE_EXIT { tlr->cache_maintenance_flag = 0; }
#endif
/* Invalidate the cache. */
const uintptr_t start_addr = reinterpret_cast<uintptr_t>(addr) & ~(cache_line_size - 1);
const uintptr_t end_addr = reinterpret_cast<uintptr_t>(addr) + size;

14
mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp
View file

@ -570,13 +570,13 @@ namespace ams::kern::init {
cpu::DebugFeatureRegisterAccessor aa64dfr0;
const auto num_watchpoints = aa64dfr0.GetNumWatchpoints();
const auto num_breakpoints = aa64dfr0.GetNumBreakpoints();
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
/* Clear the debug monitor register and the os lock access register. */
cpu::MonitorDebugSystemControlRegisterAccessor(0).Store();
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
cpu::OsLockAccessRegisterAccessor(0).Store();
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
/* Clear all debug watchpoints/breakpoints. */
#define FOR_I_IN_15_TO_1(HANDLER, ...) \
@ -620,22 +620,22 @@ namespace ams::kern::init {
#undef MESOSPHERE_INITIALIZE_BREAKPOINT_CASE
#undef FOR_I_IN_15_TO_1
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
/* Initialize the context id register to all 1s. */
cpu::ContextIdRegisterAccessor(0).SetProcId(std::numeric_limits<u32>::max()).Store();
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
/* Configure the debug monitor register. */
cpu::MonitorDebugSystemControlRegisterAccessor(0).SetMde(true).SetTdcc(true).Store();
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
}
void InitializeExceptionVectors() {
cpu::SetVbarEl1(reinterpret_cast<uintptr_t>(::ams::kern::ExceptionVectors));
cpu::SetTpidrEl1(0);
cpu::SetExceptionThreadStackTop(0);
cpu::EnsureInstructionConsistency();
cpu::EnsureInstructionConsistencyFullSystem();
}
size_t GetMiscUnknownDebugRegionSize() {

133
mesosphere/kernel/source/arch/arm64/init/start.s
View file

@ -84,17 +84,31 @@ _ZN3ams4kern4init10StartCore0Emm:
mov x20, x1
/* Check our current EL. We want to be executing out of EL1. */
/* If we're in EL2, we'll need to deprivilege ourselves. */
mrs x1, currentel
/* Check if we're EL1. */
cmp x1, #0x4
b.eq core0_el1
b.eq 2f
/* Check if we're EL2. */
cmp x1, #0x8
b.eq core0_el2
core0_el3:
b core0_el3
core0_el2:
b.eq 1f
0: /* We're EL3. This is a panic condition. */
b 0b
1: /* We're EL2. */
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
/* On NX board, this is a panic condition. */
b 1b
#else
/* Otherwise, deprivilege to EL2. */
/* TODO: Does N still have this? We need it for qemu emulation/unit testing, we should come up with a better solution maybe. */
bl _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
core0_el1:
#endif
2: /* We're EL1. */
/* Disable the MMU/Caches. */
bl _ZN3ams4kern4init19DisableMmuAndCachesEv
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
@ -103,14 +117,18 @@ core0_el1:
mov w1, #65000
smc #1
cmp x0, #0
0:
b.ne 0b
3:
b.ne 3b
/* Store the target firmware. */
adr x0, __metadata_target_firmware
str w1, [x0]
#endif
/* Get the unknown debug region. */
/* TODO: This is always zero in release kernels -- what is this? Is it the device tree buffer? */
mov x21, #0
/* We want to invoke kernel loader. */
adr x0, _start
adr x1, __metadata_kernel_layout
@ -126,7 +144,7 @@ core0_el1:
/* Next thing to do is to set up our memory management and slabheaps -- all the other core initialization. */
/* Call ams::kern::init::InitializeCore(uintptr_t, void **) */
mov x1, x0 /* Kernelldr returns a state object for the kernel to re-use. */
mov x0, xzr /* Official kernel always passes zero, when this is non-zero the address is mapped. */
mov x0, x21 /* Use the address we determined earlier. */
bl _ZN3ams4kern4init14InitializeCoreEmPPv
/* Get the init arguments for core 0. */
@ -144,17 +162,31 @@ _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE:
mov x20, x0
/* Check our current EL. We want to be executing out of EL1. */
/* If we're in EL2, we'll need to deprivilege ourselves. */
mrs x1, currentel
/* Check if we're EL1. */
cmp x1, #0x4
b.eq othercore_el1
b.eq 2f
/* Check if we're EL2. */
cmp x1, #0x8
b.eq othercore_el2
othercore_el3:
b othercore_el3
othercore_el2:
b.eq 1f
0: /* We're EL3. This is a panic condition. */
b 0b
1: /* We're EL2. */
#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
/* On NX board, this is a panic condition. */
b 1b
#else
/* Otherwise, deprivilege to EL2. */
/* TODO: Does N still have this? We need it for qemu emulation/unit testing, we should come up with a better solution maybe. */
bl _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
othercore_el1:
#endif
2: /* We're EL1. */
/* Disable the MMU/Caches. */
bl _ZN3ams4kern4init19DisableMmuAndCachesEv
/* Setup system registers using values from our KInitArguments. */
@ -171,21 +203,20 @@ othercore_el1:
mrs x1, midr_el1
ubfx x2, x1, #0x18, #0x8 /* Extract implementer bits. */
cmp x2, #0x41 /* Implementer::ArmLimited */
b.ne othercore_cpu_specific_setup_end
b.ne 4f
ubfx x2, x1, #0x4, #0xC /* Extract primary part number. */
cmp x2, #0xD07 /* PrimaryPartNumber::CortexA57 */
b.eq othercore_cpu_specific_setup_cortex_a57
b.eq 3f
cmp x2, #0xD03 /* PrimaryPartNumber::CortexA53 */
b.eq othercore_cpu_specific_setup_cortex_a53
b othercore_cpu_specific_setup_end
othercore_cpu_specific_setup_cortex_a57:
othercore_cpu_specific_setup_cortex_a53:
b.eq 3f
b 4f
3: /* We're running on a Cortex-A53/Cortex-A57. */
ldr x1, [x20, #(INIT_ARGUMENTS_CPUACTLR)]
msr cpuactlr_el1, x1
ldr x1, [x20, #(INIT_ARGUMENTS_CPUECTLR)]
msr cpuectlr_el1, x1
othercore_cpu_specific_setup_end:
4:
/* Ensure instruction consistency. */
dsb sy
isb
@ -237,7 +268,8 @@ _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE:
ldr x0, [x20, #(INIT_ARGUMENTS_ARGUMENT)]
br x1
/* TODO: Can we remove this while retaining QEMU support? */
#ifndef ATMOSPHERE_BOARD_NINTENDO_NX
/* ams::kern::init::JumpFromEL2ToEL1() */
.section .crt0.text._ZN3ams4kern4init16JumpFromEL2ToEL1Ev, "ax", %progbits
.global _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
@ -314,6 +346,7 @@ _ZN3ams4kern4init16JumpFromEL2ToEL1Ev:
msr spsr_el2, x0
eret
#endif
/* ams::kern::init::DisableMmuAndCaches() */
.section .crt0.text._ZN3ams4kern4init19DisableMmuAndCachesEv, "ax", %progbits
@ -341,6 +374,10 @@ _ZN3ams4kern4init19DisableMmuAndCachesEv:
and x0, x0, x1
msr sctlr_el1, x0
/* Ensure instruction consistency. */
dsb sy
isb
mov x30, x22
ret
@ -354,13 +391,10 @@ _ZN3ams4kern4arch5arm643cpu32FlushEntireDataCacheWithoutStackEv:
/* Ensure that the cache is coherent. */
bl _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv
dsb sy
bl _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv
dsb sy
bl _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv
dsb sy
/* Invalidate the entire TLB, and ensure instruction consistency. */
tlbi vmalle1is
@ -387,10 +421,10 @@ _ZN3ams4kern4arch5arm643cpu37FlushEntireDataCacheLocalWithoutStackEv:
mov x9, xzr
/* while (level <= levels_of_unification) { */
begin_flush_cache_local_loop:
cmp x9, x10
b.eq done_flush_cache_local_loop
b.eq 1f
0:
/* FlushEntireDataCacheImplWithoutStack(level); */
mov w0, w9
bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
@ -399,9 +433,13 @@ begin_flush_cache_local_loop:
add w9, w9, #1
/* } */
b begin_flush_cache_local_loop
cmp x9, x10
b.ne 0b
done_flush_cache_local_loop:
/* cpu::DataSynchronizationBarrier(); */
dsb sy
1:
mov x30, x24
ret
@ -423,21 +461,25 @@ _ZN3ams4kern4arch5arm643cpu38FlushEntireDataCacheSharedWithoutStackEv:
/* int level = levels_of_unification */
/* while (level <= levels_of_coherency) { */
begin_flush_cache_shared_loop:
cmp w9, w10
b.hi done_flush_cache_shared_loop
b.hi 1f
0:
/* FlushEntireDataCacheImplWithoutStack(level); */
mov w0, w9
bl _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv
/* level++; */
cmp w9, w10
add w9, w9, #1
/* } */
b begin_flush_cache_shared_loop
b.cc 0b
done_flush_cache_shared_loop:
/* cpu::DataSynchronizationBarrier(); */
dsb sy
1:
mov x30, x24
ret
@ -450,6 +492,9 @@ _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv:
lsl w6, w0, #1
sxtw x6, w6
/* cpu::DataSynchronizationBarrier(); */
dsb sy
/* cpu::SetCsselrEl1(level_sel_value); */
msr csselr_el1, x6
@ -479,17 +524,17 @@ _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv:
mov x5, #0
/* while (way <= num_ways) { */
begin_flush_cache_impl_way_loop:
0:
cmp w8, w5
b.lt done_flush_cache_impl_way_loop
b.lt 3f
/* int set = 0; */
mov x0, #0
/* while (set <= num_sets) { */
begin_flush_cache_impl_set_loop:
1:
cmp w3, w0
b.lt done_flush_cache_impl_set_loop
b.lt 2f
/* const u64 cisw_value = (static_cast<u64>(way) << way_shift) | (static_cast<u64>(set) << set_shift) | level_sel_value; */
lsl x2, x5, x7
@ -504,13 +549,13 @@ begin_flush_cache_impl_set_loop:
add x0, x0, #1
/* } */
b begin_flush_cache_impl_set_loop
done_flush_cache_impl_set_loop:
b 1b
2:
/* way++; */
add x5, x5, 1
/* } */
b begin_flush_cache_impl_way_loop
done_flush_cache_impl_way_loop:
b 0b
3:
ret

8
mesosphere/kernel_ldr/source/kern_init_loader.cpp
View file

@ -88,17 +88,17 @@ namespace ams::kern::init::loader {
cpu::MemoryAccessIndirectionRegisterAccessor(MairValue).Store();
cpu::TranslationControlRegisterAccessor(TcrValue).Store();
/* Ensure that our configuration takes before proceeding. */
cpu::EnsureInstructionConsistency();
/* Perform board-specific setup. */
PerformBoardSpecificSetup();
/* Ensure that the entire cache is flushed. */
cpu::FlushEntireCacheForInit();
/* Setup SCTLR_EL1. */
/* TODO: Define these bits properly elsewhere, document exactly what each bit set is doing .*/
constexpr u64 SctlrValue = 0x0000000034D5D925ul;
cpu::SetSctlrEl1(SctlrValue);
cpu::EnsureInstructionConsistency();
cpu::InstructionMemoryBarrier();
}
KVirtualAddress GetRandomKernelBaseAddress(KInitialPageTable &page_table, KPhysicalAddress phys_base_address, size_t kernel_size) {