diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index 48af48efe..44f6226c3 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -394,6 +394,77 @@ namespace ams::kern::arch::arm64::init { return l3_entry->GetBlock() + (GetInteger(virt_addr) & (L3BlockSize - 1)); } + KPhysicalAddress GetPhysicalAddressOfRandomizedRange(KVirtualAddress virt_addr, size_t size) const { + /* Define tracking variables for ourselves to use. */ + KPhysicalAddress min_phys_addr = Null; + KPhysicalAddress max_phys_addr = Null; + + /* Ensure the range we're querying is valid. */ + const KVirtualAddress end_virt_addr = virt_addr + size; + if (virt_addr > end_virt_addr) { + MESOSPHERE_INIT_ABORT_UNLESS(size == 0); + return min_phys_addr; + } + + auto UpdateExtents = [&](const KPhysicalAddress block, size_t block_size) ALWAYS_INLINE_LAMBDA { + /* Ensure that we are allowed to have the block here. */ + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), block_size)); + MESOSPHERE_INIT_ABORT_UNLESS(block_size <= GetInteger(end_virt_addr) - GetInteger(virt_addr)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(block), block_size)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, block_size)); + + const KPhysicalAddress block_end = block + block_size; + + /* We want to update min phys addr when it's 0 or > block. */ + /* This is equivalent in two's complement to (n - 1) >= block. */ + if ((GetInteger(min_phys_addr) - 1) >= GetInteger(block)) { + min_phys_addr = block; + } + + /* Update max phys addr when it's 0 or < block_end. */ + if (GetInteger(max_phys_addr) < GetInteger(block_end) || GetInteger(max_phys_addr) == 0) { + max_phys_addr = block_end; + } + + /* Traverse onwards. */ + virt_addr += block_size; + }; + + while (virt_addr < end_virt_addr) { + L1PageTableEntry *l1_entry = GetL1Entry(this->l1_table, virt_addr); + + /* If an L1 block is mapped, update. */ + if (l1_entry->IsBlock()) { + UpdateExtents(l1_entry->GetBlock(), L1BlockSize); + continue; + } + + /* Not a block, so we must have a table. */ + MESOSPHERE_INIT_ABORT_UNLESS(l1_entry->IsTable()); + + L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr); + if (l2_entry->IsBlock()) { + UpdateExtents(l2_entry->GetBlock(), l2_entry->IsContiguous() ? L2ContiguousBlockSize : L2BlockSize); + continue; + } + + /* Not a block, so we must have a table. */ + MESOSPHERE_INIT_ABORT_UNLESS(l2_entry->IsTable()); + + /* We must have a mapped l3 entry to inspect. */ + L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr); + MESOSPHERE_INIT_ABORT_UNLESS(l3_entry->IsBlock()); + + UpdateExtents(l3_entry->GetBlock(), l3_entry->IsContiguous() ? L3ContiguousBlockSize : L3BlockSize); + } + + /* Ensure we got the right range. */ + MESOSPHERE_INIT_ABORT_UNLESS(GetInteger(max_phys_addr) - GetInteger(min_phys_addr) == size); + + /* Write the address that we found. */ + return min_phys_addr; + } + bool IsFree(KVirtualAddress virt_addr, size_t size) { /* Ensure that addresses and sizes are page aligned. */ MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), PageSize)); diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp index 03311a65d..5cd51a910 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp @@ -156,6 +156,7 @@ namespace ams::kern::arch::arm64::cpu { void ClearPageToZeroImpl(void *); void FlushEntireDataCacheSharedForInit(); void FlushEntireDataCacheLocalForInit(); + void StoreEntireCacheForInit(); void FlushEntireDataCache(); diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp index 9ed30784e..f1997ecb8 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_spin_lock.hpp @@ -26,14 +26,14 @@ namespace ams::kern::arch::arm64 { constexpr KNotAlignedSpinLock() : packed_tickets(0) { /* ... */ } void Lock() { - u32 tmp0, tmp1; + u32 tmp0, tmp1, tmp2; __asm__ __volatile__( " prfm pstl1keep, %[packed_tickets]\n" "1:\n" " ldaxr %w[tmp0], %[packed_tickets]\n" - " add %w[tmp0], %w[tmp0], #0x10000\n" - " stxr %w[tmp1], %w[tmp0], %[packed_tickets]\n" + " add %w[tmp2], %w[tmp0], #0x10000\n" + " stxr %w[tmp1], %w[tmp2], %[packed_tickets]\n" " cbnz %w[tmp1], 1b\n" " \n" " and %w[tmp1], %w[tmp0], #0xFFFF\n" @@ -46,7 +46,7 @@ namespace ams::kern::arch::arm64 { " cmp %w[tmp1], %w[tmp0], lsr #16\n" " b.ne 2b\n" "3:\n" - : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [packed_tickets]"+Q"(this->packed_tickets) + : [tmp0]"=&r"(tmp0), [tmp1]"=&r"(tmp1), [tmp2]"=&r"(tmp2), [packed_tickets]"+Q"(this->packed_tickets) : : "cc", "memory" ); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_bitmap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_bitmap.hpp index f75e4a700..56ef731e1 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_bitmap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_bitmap.hpp @@ -51,11 +51,11 @@ namespace ams::kern { u64 selected = 0; u64 cur_num_bits = BITSIZEOF(bitmap) / 2; - u64 cur_mask = (1ull << cur_num_bits) / 2; + u64 cur_mask = (1ull << cur_num_bits) - 1; while (cur_num_bits) { - const u64 high = (bitmap >> 0) & cur_mask; - const u64 low = (bitmap >> cur_num_bits) & cur_mask; + const u64 low = (bitmap >> 0) & cur_mask; + const u64 high = (bitmap >> cur_num_bits) & cur_mask; bool choose_low; if (high == 0) { diff --git a/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp b/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp index 733ba1744..60fd0473e 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp @@ -333,6 +333,13 @@ namespace ams::kern::arch::arm64::cpu { return PerformCacheOperationBySetWayLocal(FlushDataCacheLineBySetWayImpl); } + void StoreEntireCacheForInit() { + PerformCacheOperationBySetWayLocal(StoreDataCacheLineBySetWayImpl); + PerformCacheOperationBySetWayShared(StoreDataCacheLineBySetWayImpl); + DataSynchronizationBarrierInnerShareable(); + InvalidateEntireInstructionCache(); + } + void FlushEntireDataCache() { return PerformCacheOperationBySetWayShared(FlushDataCacheLineBySetWayImpl); } diff --git a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp index 829be5ab5..cdeb77909 100644 --- a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp +++ b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp @@ -324,7 +324,7 @@ namespace ams::kern::board::nintendo::nx { EnsureRandomGeneratorInitialized(); - return GenerateRandomU64(); + return GenerateRandomU64FromGenerator(); } void KSystemControl::SleepSystem() { diff --git a/mesosphere/build_mesosphere.py b/mesosphere/build_mesosphere.py index eb53e7ee6..9bc8fe511 100644 --- a/mesosphere/build_mesosphere.py +++ b/mesosphere/build_mesosphere.py @@ -2,6 +2,20 @@ import sys, os from struct import pack as pk, unpack as up +ATMOSPHERE_TARGET_FIRMWARE_100 = 1 +ATMOSPHERE_TARGET_FIRMWARE_200 = 2 +ATMOSPHERE_TARGET_FIRMWARE_300 = 3 +ATMOSPHERE_TARGET_FIRMWARE_400 = 4 +ATMOSPHERE_TARGET_FIRMWARE_500 = 5 +ATMOSPHERE_TARGET_FIRMWARE_600 = 6 +ATMOSPHERE_TARGET_FIRMWARE_620 = 7 +ATMOSPHERE_TARGET_FIRMWARE_700 = 8 +ATMOSPHERE_TARGET_FIRMWARE_800 = 9 +ATMOSPHERE_TARGET_FIRMWARE_810 = 10 +ATMOSPHERE_TARGET_FIRMWARE_900 = 11 +ATMOSPHERE_TARGET_FIRMWARE_910 = 12 +ATMOSPHERE_TARGET_FIRMWARE_1000 = 13 + def align_up(val, algn): val += algn - 1 return val - (val % algn) @@ -18,7 +32,7 @@ def main(argc, argv): kernel_metadata_offset = 4 assert (kernel_metadata_offset <= len(kernel) - 0x40) assert (kernel[kernel_metadata_offset:kernel_metadata_offset + 4] == b'MSS0') - kernel_end = up('= len(kernel)) embedded_ini = b'' @@ -36,8 +50,8 @@ def main(argc, argv): with open('mesosphere.bin', 'wb') as f: f.write(kernel[:kernel_metadata_offset + 4]) - f.write(pk(' InitialProcessBinarySizeMax); + static_assert(KernelResourceRegionSize + ExtraKernelResourceSize > InitialProcessBinarySizeMax); + + /* 10.0.0 reduced the kernel resource region size by 64K. */ + if (kern::GetTargetFirmware() >= kern::TargetFirmware_10_0_0) { + resource_region_size -= KernelResourceReduction_10_0_0; + } + return resource_region_size; + } + /* Page table attributes. */ constexpr PageTableEntry KernelRoDataAttribute(PageTableEntry::Permission_KernelR, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable); constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable); @@ -138,16 +153,15 @@ namespace ams::kern::init { MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(stack_region_start), StackRegionSize, KMemoryRegionType_KernelStack)); /* Decide if Kernel should have enlarged resource region (slab region + page table heap region). */ - const bool use_extra_resources = KSystemControl::Init::ShouldIncreaseThreadResourceLimit(); - const size_t resource_region_size = KernelResourceRegionSize + (use_extra_resources ? ExtraKernelResourceSize : 0); + const size_t resource_region_size = GetResourceRegionSize(); /* Determine the size of the slab region. */ const size_t slab_region_size = CalculateTotalSlabHeapSize(); MESOSPHERE_INIT_ABORT_UNLESS(slab_region_size <= resource_region_size); /* Setup the slab region. */ - const KPhysicalAddress code_start_phys_addr = ttbr1_table.GetPhysicalAddress(code_start_virt_addr); - const KPhysicalAddress code_end_phys_addr = code_start_phys_addr + (code_end_virt_addr - code_start_virt_addr); + const KPhysicalAddress code_start_phys_addr = ttbr1_table.GetPhysicalAddressOfRandomizedRange(code_start_virt_addr, code_region_size); + const KPhysicalAddress code_end_phys_addr = code_start_phys_addr + code_region_size; const KPhysicalAddress slab_start_phys_addr = code_end_phys_addr; const KPhysicalAddress slab_end_phys_addr = slab_start_phys_addr + slab_region_size; constexpr size_t SlabRegionAlign = KernelAslrAlignment; @@ -155,9 +169,6 @@ namespace ams::kern::init { const KVirtualAddress slab_region_start = KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegion(slab_region_needed_size, SlabRegionAlign, KMemoryRegionType_Kernel) + (GetInteger(code_end_phys_addr) % SlabRegionAlign); MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(slab_region_start), slab_region_size, KMemoryRegionType_KernelSlab)); - /* Set the slab region's pair region. */ - KMemoryLayout::GetVirtualMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_KernelSlab)->SetPairAddress(GetInteger(slab_start_phys_addr)); - /* Setup the temp region. */ constexpr size_t TempRegionSize = 128_MB; constexpr size_t TempRegionAlign = KernelAslrAlignment; @@ -206,15 +217,20 @@ namespace ams::kern::init { SetupDramPhysicalMemoryRegions(); /* Insert a physical region for the kernel code region. */ - MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(code_start_phys_addr), (code_end_virt_addr - code_start_virt_addr), KMemoryRegionType_DramKernelCode)); - KMemoryLayout::GetPhysicalMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_DramKernelCode)->SetPairAddress(code_start_virt_addr); + MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(code_start_phys_addr), code_region_size, KMemoryRegionType_DramKernelCode)); /* Insert a physical region for the kernel slab region. */ MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(slab_start_phys_addr), slab_region_size, KMemoryRegionType_DramKernelSlab)); - KMemoryLayout::GetPhysicalMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_DramKernelSlab)->SetPairAddress(GetInteger(slab_region_start)); - /* Map and clear the slab region. */ + /* Map the slab region. */ ttbr1_table.Map(slab_region_start, slab_region_size, slab_start_phys_addr, KernelRwDataAttribute, g_initial_page_allocator); + + /* Physically randomize the slab region. */ + /* NOTE: Nintendo does this only on 10.0.0+ */ + ttbr1_table.PhysicallyRandomize(slab_region_start, slab_region_size, false); + cpu::StoreEntireCacheForInit(); + + /* Clear the slab region. */ std::memset(GetVoidPointer(slab_region_start), 0, slab_region_size); /* Determine size available for kernel page table heaps, requiring > 8 MB. */ diff --git a/mesosphere/kernel/source/arch/arm64/init/start.s b/mesosphere/kernel/source/arch/arm64/init/start.s index af91ca038..4114cc300 100644 --- a/mesosphere/kernel/source/arch/arm64/init/start.s +++ b/mesosphere/kernel/source/arch/arm64/init/start.s @@ -65,7 +65,7 @@ __metadata_kernel_layout: .type _ZN3ams4kern17GetTargetFirmwareEv, %function _ZN3ams4kern17GetTargetFirmwareEv: adr x0, __metadata_target_firmware - ldr x0, [x0] + ldr w0, [x0] ret /* ams::kern::init::StartCore0(uintptr_t, uintptr_t) */ @@ -101,6 +101,15 @@ core0_el1: add x2, x0, x2 LOAD_FROM_LABEL(x3, __metadata_kernelldr_offset) add x3, x0, x3 + + /* If kernelldr is ours, set its target firmware. */ + ldr w4, [x3, #4] + LOAD_IMMEDIATE_32(w5, 0x30444C4D) + cmp w4, w5 + b.ne 1f + LOAD_FROM_LABEL(x4, __metadata_target_firmware) + str w4, [x3, #8] +1: blr x3 /* At this point kernelldr has been invoked, and we are relocated at a random virtual address. */ diff --git a/mesosphere/kernel_ldr/source/arch/arm64/start.s b/mesosphere/kernel_ldr/source/arch/arm64/start.s index 9862239c9..3a27f40c3 100644 --- a/mesosphere/kernel_ldr/source/arch/arm64/start.s +++ b/mesosphere/kernel_ldr/source/arch/arm64/start.s @@ -109,7 +109,7 @@ _main: .type _ZN3ams4kern17GetTargetFirmwareEv, %function _ZN3ams4kern17GetTargetFirmwareEv: adr x0, __metadata_target_firmware - ldr x0, [x0] + ldr w0, [x0] ret .balign 8 diff --git a/mesosphere/kernel_ldr/source/kern_init_loader.cpp b/mesosphere/kernel_ldr/source/kern_init_loader.cpp index c0305832a..6fc9136cb 100644 --- a/mesosphere/kernel_ldr/source/kern_init_loader.cpp +++ b/mesosphere/kernel_ldr/source/kern_init_loader.cpp @@ -304,7 +304,7 @@ namespace ams::kern::init::loader { ttbr1_table.Map(virtual_base_address + ro_offset, ro_end_offset - ro_offset, base_address + ro_offset, KernelRwDataAttribute, g_initial_page_allocator); ttbr1_table.Map(virtual_base_address + rw_offset, bss_end_offset - rw_offset, base_address + rw_offset, KernelRwDataAttribute, g_initial_page_allocator); - /* On 10.0.0+, Physicaly randomize the kernel region. */ + /* On 10.0.0+, Physically randomize the kernel region. */ if (kern::GetTargetFirmware() >= kern::TargetFirmware_10_0_0) { ttbr1_table.PhysicallyRandomize(virtual_base_address + rx_offset, bss_end_offset - rx_offset, true); }