kern/ldr: move crt0 into .rodata

fusee/program/source/fusee_stratosphere.cpp

@@ -24,6 +24,9 @@ namespace ams::nxboot {
 
     namespace {
 
+        constexpr u32 MesoshereMetadataLayout0Magic = util::FourCC<'M','S','S','0'>::Code;
+        constexpr u32 MesoshereMetadataLayout1Magic = util::FourCC<'M','S','S','1'>::Code;
+
         struct InitialProcessBinaryHeader {
             static constexpr u32 Magic = util::FourCC<'I','N','I','1'>::Code;
 
@@ -1011,7 +1014,20 @@ namespace ams::nxboot {
         }
 
         /* Set the embedded ini pointer. */
+        const u32 magic = *reinterpret_cast<const u32 *>(payload_data + 4);
+        if (magic == MesoshereMetadataLayout0Magic) {
             std::memcpy(payload_data + 8, std::addressof(meso_size), sizeof(meso_size));
+        } else if (magic == MesoshereMetadataLayout1Magic) {
+            if (const u32 meta_offset = *reinterpret_cast<const u32 *>(payload_data + 8); meta_offset <= meso_size - sizeof(meso_size)) {
+                s64 relative_offset = meso_size - meta_offset;
+                std::memcpy(payload_data + meta_offset, std::addressof(relative_offset), sizeof(relative_offset));
+            } else {
+                ShowFatalError("Invalid mesosphere metadata layout!\n");
+            }
+        } else {
+            ShowFatalError("Unknown mesosphere metadata version!\n");
+        }
+
 
         /* Get the ini pointer. */
         InitialProcessBinaryHeader * const ini = reinterpret_cast<InitialProcessBinaryHeader *>(payload_data + meso_size);

libraries/libmesosphere/include/mesosphere/init/kern_init_layout.hpp

@@ -31,8 +31,22 @@ namespace ams::kern::init {
         u32 dynamic_offset;
         u32 init_array_offset;
         u32 init_array_end_offset;
+        u32 sysreg_offset;
     };
     static_assert(util::is_pod<KernelLayout>::value);
-    static_assert(sizeof(KernelLayout) == 0x30);
+    static_assert(sizeof(KernelLayout) == 0x34);
+
+    #if defined(ATMOSPHERE_ARCH_ARM64)
+    struct KernelSystemRegisters {
+        u64 ttbr0_el1;
+        u64 ttbr1_el1;
+        u64 tcr_el1;
+        u64 mair_el1;
+        u64 sctlr_el1;
+    };
+    #else
+    struct KernelSystemRegisters {
+    };
+    #endif
 
 }

mesosphere/build_mesosphere.py

@@ -17,11 +17,16 @@ def main(argc, argv):
         kernel_ldr = f.read()
     with open(argv[2], 'rb') as f:
         kernel = f.read()
-    kernel_metadata_offset = 4
+    kernel_metaptr_offset = 4
+    assert (kernel_metaptr_offset <= len(kernel) - 0x40)
+    assert (kernel[kernel_metaptr_offset:kernel_metaptr_offset + 4] == b'MSS1')
+    kernel_metadata_offset = up('<I', kernel[kernel_metaptr_offset+4:kernel_metaptr_offset+8])[0]
     assert (kernel_metadata_offset <= len(kernel) - 0x40)
-    assert (kernel[kernel_metadata_offset:kernel_metadata_offset + 4] == b'MSS0')
 
-    bss_start, bss_end, kernel_end = up('<III', kernel[kernel_metadata_offset + 0x30:kernel_metadata_offset + 0x3C])
+    bss_start, bss_end, kernel_end = up('<III', kernel[kernel_metadata_offset + 0x2C:kernel_metadata_offset + 0x38])
+    bss_start  += kernel_metadata_offset + 0x14
+    bss_end    += kernel_metadata_offset + 0x14
+    kernel_end += kernel_metadata_offset + 0x14
     assert (bss_end >= bss_start)
     assert (bss_end == kernel_end)
 
@@ -53,9 +58,9 @@ def main(argc, argv):
     mesosphere_end    = align_up(kernel_ldr_end, 0x1000)
 
     with open(argv[3], 'wb') as f:
-        f.write(kernel[:kernel_metadata_offset + 4])
+        f.write(kernel[:kernel_metadata_offset])
-        f.write(pk('<QQI', embedded_ini_offset, kernel_ldr_offset, atmosphere_target_firmware(13, 0, 0)))
+        f.write(pk('<QQI', embedded_ini_offset - (kernel_metadata_offset), kernel_ldr_offset  - (kernel_metadata_offset + 8), atmosphere_target_firmware(17, 0, 0)))
-        f.write(kernel[kernel_metadata_offset + 0x18:])
+        f.write(kernel[kernel_metadata_offset + 0x14:])
         f.seek(embedded_ini_offset)
         f.write(embedded_ini_header)
         f.write(embedded_kips)

mesosphere/kernel/kernel.ld

@@ -16,9 +16,25 @@ SECTIONS
 	. = __start__;
 	__code_start = . ;
 
-	.crt0 :
+	.start :
 	{
-		KEEP (*(.crt0 .crt0.*))
+		KEEP (*(.start .start.*))
+		. = ALIGN(8);
+	} :code
+
+	/* .sleep. */
+    .sleep :
+	{
+		KEEP( *(.sleep .sleep.*) )
+		. = ALIGN(8);
+	} :code
+
+	/* .vectors. */
+	. = ALIGN(2K);
+	__vectors_start__ = . ;
+    .vectors :
+	{
+		KEEP( *(.vectors) )
 		. = ALIGN(8);
 	} :code
 
@@ -51,28 +67,16 @@ SECTIONS
 		. = ALIGN(8);
 	} :code
 
-	/* .sleep. */
-	. = ALIGN(4K);
-	__sleep_start__ = . ;
-    .sleep :
-	{
-		KEEP( *(.sleep .sleep.*) )
-		. = ALIGN(8);
-	} :code
-
-	/* .vectors. */
-	. = ALIGN(2K);
-	__vectors_start__ = . ;
-    .vectors :
-	{
-		KEEP( *(.vectors) )
-		. = ALIGN(8);
-	} :code
-
 	/* =========== RODATA section =========== */
 	. = ALIGN(0x1000);
 	__rodata_start = . ;
 
+	.rodata.text.crt0 :
+	{
+		KEEP (*(.crt0 .crt0.*))
+		. = ALIGN(8);
+	} :code
+
 	.rodata :
 	{
 		*(.rodata .rodata.* .gnu.linkonce.r.*)

mesosphere/kernel/source/arch/arm64/init/start.s

@@ -33,7 +33,37 @@
     adr reg, label;                 \
     ldr reg, [reg]
 
+#define LOAD_RELATIVE_FROM_LABEL(reg, reg2, label) \
+    adr reg2, label;                               \
+    ldr reg, [reg2];                               \
+    add reg, reg, reg2
 
+#define INDIRECT_RELATIVE_CALL(reg, reg2, label) \
+    adr reg, label;                              \
+    add reg, reg, reg2;                          \
+    blr reg
+
+#define SETUP_SYSTEM_REGISTER(_reg, _sr)              \
+    LOAD_FROM_LABEL(_reg, __sysreg_constant_ ## _sr); \
+    msr _sr, _reg
+
+.section    .start, "ax", %progbits
+.global     _start
+_start:
+    b _ZN3ams4kern4init10StartCore0Emm
+__metadata_magic_number:
+    .ascii "MSS1"                     /* Magic, if executed as gadget "adds w13, w26, #0x4d4, lsl #12" */
+__metadata_offset:
+    .word __metadata_begin - _start
+
+#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
+.global     _ZN3ams4kern17GetTargetFirmwareEv
+.type       _ZN3ams4kern17GetTargetFirmwareEv, %function
+_ZN3ams4kern17GetTargetFirmwareEv:
+    adr x0, __metadata_target_firmware
+    ldr w0, [x0]
+    ret
+#endif
 
 .section    .crt0.text.start, "ax", %progbits
 
@@ -45,43 +75,6 @@ _ZN3ams4kern4init31IdentityMappedFunctionAreaBeginEv:
 
 /*  ================ Functions after this line remain identity-mapped after initialization finishes. ================ */
 
-.global     _start
-_start:
-    b _ZN3ams4kern4init10StartCore0Emm
-__metadata_begin:
-    .ascii "MSS0"                     /* Magic */
-__metadata_ini_offset:
-    .quad  0                          /* INI1 base address. */
-__metadata_kernelldr_offset:
-    .quad  0                          /* Kernel Loader base address. */
-__metadata_target_firmware:
-    .word  0xCCCCCCCC                 /* Target firmware. */
-__metadata_kernel_layout:
-    .word _start - _start             /* rx_offset */
-    .word __rodata_start     - _start /* rx_end_offset */
-    .word __rodata_start     - _start /* ro_offset */
-    .word __data_start       - _start /* ro_end_offset */
-    .word __data_start       - _start /* rw_offset */
-    .word __bss_start__      - _start /* rw_end_offset */
-    .word __bss_start__      - _start /* bss_offset */
-    .word __bss_end__        - _start /* bss_end_offset */
-    .word __end__            - _start /* resource_offset */
-    .word _DYNAMIC           - _start /* dynamic_offset */
-    .word __init_array_start - _start /* init_array_offset */
-    .word __init_array_end   - _start /* init_array_end_offset */
-.if (. - __metadata_begin) != 0x48
-    .error "Incorrect Mesosphere Metadata"
-.endif
-
-#ifdef ATMOSPHERE_BOARD_NINTENDO_NX
-.global     _ZN3ams4kern17GetTargetFirmwareEv
-.type       _ZN3ams4kern17GetTargetFirmwareEv, %function
-_ZN3ams4kern17GetTargetFirmwareEv:
-    adr x0, __metadata_target_firmware
-    ldr w0, [x0]
-    ret
-#endif
-
 /* ams::kern::init::StartCore0(uintptr_t, uintptr_t) */
 .section    .crt0.text._ZN3ams4kern4init10StartCore0Emm, "ax", %progbits
 .global     _ZN3ams4kern4init10StartCore0Emm
@@ -139,28 +132,31 @@ _ZN3ams4kern4init10StartCore0Emm:
     /* Get the unknown debug region. */
     /* TODO: This is always zero in release kernels -- what is this? Is it the device tree buffer? */
     mov x21, #0
+    nop
 
     /* We want to invoke kernel loader. */
     adr x0, _start
     adr x1, __metadata_kernel_layout
-    LOAD_FROM_LABEL(x2, __metadata_ini_offset)
+    LOAD_RELATIVE_FROM_LABEL(x2, x4, __metadata_ini_offset)
-    add x2, x0, x2
+    LOAD_RELATIVE_FROM_LABEL(x3, x4, __metadata_kernelldr_offset)
-    LOAD_FROM_LABEL(x3, __metadata_kernelldr_offset)
-    add x3, x0, x3
 
     /* Invoke kernel loader. */
     blr x3
 
+    /* Save the offset to virtual address from this page's physical address for our use. */
+    mov x24, x1
+
     /* At this point kernelldr has been invoked, and we are relocated at a random virtual address. */
     /* Next thing to do is to set up our memory management and slabheaps -- all the other core initialization. */
     /* Call ams::kern::init::InitializeCore(uintptr_t, void **) */
     mov x1, x0  /* Kernelldr returns a state object for the kernel to re-use. */
     mov x0, x21 /* Use the address we determined earlier. */
-    bl _ZN3ams4kern4init20InitializeCorePhase1EmPPv
+    INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init20InitializeCorePhase1EmPPv)
 
     /* Get the init arguments for core 0. */
     mov x0, xzr
-    bl _ZN3ams4kern4init16GetInitArgumentsEi
+    nop
+    INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init16GetInitArgumentsEi)
 
     /* Setup the stack pointer. */
     ldr x2, [x0, #(INIT_ARGUMENTS_SP)]
@@ -168,15 +164,24 @@ _ZN3ams4kern4init10StartCore0Emm:
 
     /* Perform further initialization with the stack pointer set up, as required. */
     /* This will include e.g. unmapping the identity mapping. */
-    bl _ZN3ams4kern4init20InitializeCorePhase2Ev
+    INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init20InitializeCorePhase2Ev)
 
     /* Get the init arguments for core 0. */
     mov x0, xzr
-    bl _ZN3ams4kern4init16GetInitArgumentsEi
+    nop
+    INDIRECT_RELATIVE_CALL(x16, x24, _ZN3ams4kern4init16GetInitArgumentsEi)
 
-    /* Invoke the entrypoint. */
+    /* Retrieve entrypoint and argument. */
     ldr x1, [x0, #(INIT_ARGUMENTS_ENTRYPOINT)]
     ldr x0, [x0, #(INIT_ARGUMENTS_ARGUMENT)]
+
+    /* Set sctlr_el1 and ensure instruction consistency. */
+    SETUP_SYSTEM_REGISTER(x3, sctlr_el1)
+
+    dsb sy
+    isb
+
+    /* Invoke the entrypoint. */
     blr x1
 
 0:  /* If we return here, something has gone wrong, so wait forever. */
@@ -218,15 +223,11 @@ _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE:
     /* Disable the MMU/Caches. */
     bl _ZN3ams4kern4init19DisableMmuAndCachesEv
 
-    /* Setup system registers using values from our KInitArguments. */
+    /* Setup system registers using values from constants table. */
-    ldr x1, [x20, #(INIT_ARGUMENTS_TTBR0)]
+    SETUP_SYSTEM_REGISTER(x1, ttbr0_el1)
-    msr ttbr0_el1, x1
+    SETUP_SYSTEM_REGISTER(x1, ttbr1_el1)
-    ldr x1, [x20, #(INIT_ARGUMENTS_TTBR1)]
+    SETUP_SYSTEM_REGISTER(x1, tcr_el1)
-    msr ttbr1_el1, x1
+    SETUP_SYSTEM_REGISTER(x1, mair_el1)
-    ldr x1, [x20, #(INIT_ARGUMENTS_TCR)]
-    msr tcr_el1, x1
-    ldr x1, [x20, #(INIT_ARGUMENTS_MAIR)]
-    msr mair_el1, x1
 
     /* Perform cpu-specific setup. */
     mrs x1, midr_el1
@@ -251,13 +252,12 @@ _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE:
     isb
 
     /* Load remaining needed fields from the init args. */
-    ldr x3, [x20, #(INIT_ARGUMENTS_SCTLR)]
     ldr x2, [x20, #(INIT_ARGUMENTS_SP)]
     ldr x1, [x20, #(INIT_ARGUMENTS_ENTRYPOINT)]
     ldr x0, [x20, #(INIT_ARGUMENTS_ARGUMENT)]
 
     /* Set sctlr_el1 and ensure instruction consistency. */
-    msr sctlr_el1, x3
+    SETUP_SYSTEM_REGISTER(x3, sctlr_el1)
 
     dsb sy
     isb
@@ -271,6 +271,35 @@ _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE:
 0:  /* If we return here, something has gone wrong, so wait forever. */
     b 0b
 
+/* Nintendo places the metadata after StartOthercore. */
+.align 8
+
+__metadata_begin:
+__metadata_ini_offset:
+    .quad  0                          /* INI1 base address. */
+__metadata_kernelldr_offset:
+    .quad  0                          /* Kernel Loader base address. */
+__metadata_target_firmware:
+    .word  0xCCCCCCCC                 /* Target firmware. */
+__metadata_kernel_layout:
+    .word _start                  - __metadata_kernel_layout /* rx_offset */
+    .word __rodata_start          - __metadata_kernel_layout /* rx_end_offset */
+    .word __rodata_start          - __metadata_kernel_layout /* ro_offset */
+    .word __data_start            - __metadata_kernel_layout /* ro_end_offset */
+    .word __data_start            - __metadata_kernel_layout /* rw_offset */
+    .word __bss_start__           - __metadata_kernel_layout /* rw_end_offset */
+    .word __bss_start__           - __metadata_kernel_layout /* bss_offset */
+    .word __bss_end__             - __metadata_kernel_layout /* bss_end_offset */
+    .word __end__                 - __metadata_kernel_layout /* resource_offset */
+    .word _DYNAMIC                - __metadata_kernel_layout /* dynamic_offset */
+    .word __init_array_start      - __metadata_kernel_layout /* init_array_offset */
+    .word __init_array_end        - __metadata_kernel_layout /* init_array_end_offset */
+    .word __sysreg_constant_begin - __metadata_kernel_layout /* sysreg_offset */
+.if (. - __metadata_begin) != 0x48
+    .error "Incorrect Mesosphere Metadata"
+.endif
+
+
 /* TODO: Can we remove this while retaining QEMU support? */
 #ifndef ATMOSPHERE_BOARD_NINTENDO_NX
 /* ams::kern::init::JumpFromEL2ToEL1() */
@@ -564,6 +593,24 @@ _ZN3ams4kern4arch5arm643cpu36FlushEntireDataCacheImplWithoutStackEv:
     ret
 
 
+/* System register values. */
+.align 8
+
+__sysreg_constant_begin:
+__sysreg_constant_ttbr0_el1:
+    .quad  0 /* ttbr0_e11. */
+__sysreg_constant_ttbr1_el1:
+    .quad  0 /* ttbr1_e11. */
+__sysreg_constant_tcr_el1:
+    .quad  0 /* tcr_e11. */
+__sysreg_constant_mair_el1:
+    .quad  0 /* mair_e11. */
+__sysreg_constant_sctlr_el1:
+    .quad  0 /* sctlr_e11. */
+.if (. - __sysreg_constant_begin) != 0x28
+    .error "Incorrect System Registers"
+.endif
+
 /*  ================ Functions before this line remain identity-mapped after initialization finishes. ================ */
 
 /* ams::kern::init::IdentityMappedFunctionAreaEnd() */

mesosphere/kernel_ldr/source/arch/arm64/start.s

@@ -116,13 +116,29 @@ _main:
     /* X0 is now the saved state. */
     /* We will return this to the kernel. */
 
-    /* Return to the newly-relocated kernel. */
+    /* Adjust return address to point to the relocated kernel. */
     ldr x1, [sp, #0x18] /* Return address to Kernel */
     ldr x2, [sp, #0x00] /* Relocated kernel base address diff. */
     add x1, x2, x1
+
+    /* Translate the relocated address back to a physical address. */
+    and x4, x1, #0xFFF
+    sub x3, x1, x4
+    at  s1e1r, x3
+    isb
+    mrs x3, par_el1
+1:
+    tbnz w3, #0, 1b
+    and x3, x3, #0xFFFFFFFFF000
+    add x3, x3, x4
+
+    /* Return the difference between relocated and physical in x1. */
+    sub x1, x1, x3
+
+    /* Setup stack, and return to the kernel. */
     ldr x2, [sp, #0x20]
     mov sp, x2
-    br  x1
+    br  x3
 
 #ifdef ATMOSPHERE_BOARD_NINTENDO_NX
 .global     _ZN3ams4kern17GetTargetFirmwareEv

mesosphere/kernel_ldr/source/kern_init_loader.cpp

@@ -50,6 +50,25 @@ namespace ams::kern::init::loader {
         constinit void *g_final_state[2];
 
         void RelocateKernelPhysically(uintptr_t &base_address, KernelLayout *&layout) {
+            /* Adjust layout to be correct. */
+            {
+                const ptrdiff_t layout_offset = reinterpret_cast<uintptr_t>(layout) - base_address;
+                layout->rx_offset             += layout_offset;
+                layout->rx_end_offset         += layout_offset;
+                layout->ro_offset             += layout_offset;
+                layout->ro_end_offset         += layout_offset;
+                layout->rw_offset             += layout_offset;
+                layout->rw_end_offset         += layout_offset;
+                layout->bss_offset            += layout_offset;
+                layout->bss_end_offset        += layout_offset;
+                layout->resource_offset       += layout_offset;
+                layout->dynamic_offset        += layout_offset;
+                layout->init_array_offset     += layout_offset;
+                layout->init_array_end_offset += layout_offset;
+                layout->sysreg_offset         += layout_offset;
+            }
+
+            /* Relocate the kernel if necessary. */
             KPhysicalAddress correct_base = KSystemControl::Init::GetKernelPhysicalBaseAddress(base_address);
             if (correct_base != base_address) {
                 const uintptr_t diff = GetInteger(correct_base) - base_address;
@@ -62,7 +81,7 @@ namespace ams::kern::init::loader {
             }
         }
 
-        void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageAllocator &allocator) {
+        void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageAllocator &allocator, KernelSystemRegisters *sysregs) {
             /* Map in an RWX identity mapping for the kernel. */
             constexpr PageTableEntry KernelRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
             init_pt.Map(base_address, kernel_size, base_address, KernelRWXIdentityAttribute, allocator, 0);
@@ -96,9 +115,17 @@ namespace ams::kern::init::loader {
 
             /* Setup SCTLR_EL1. */
             /* TODO: Define these bits properly elsewhere, document exactly what each bit set is doing .*/
-            constexpr u64 SctlrValue = 0x0000000034D5D925ul;
+            constexpr u64 SctlrValue = 0x0000000034D5D92Dul;
             cpu::SetSctlrEl1(SctlrValue);
             cpu::InstructionMemoryBarrier();
+
+            /* Setup the system registers for other cores. */
+            /* NOTE: sctlr_el1 on other cores has the WXN bit set (0x80000); this will be set before KernelMain() on this core. */
+            sysregs->ttbr0_el1 = init_pt.GetTtbr0L1TableAddress();
+            sysregs->ttbr1_el1 = init_pt.GetTtbr1L1TableAddress();
+            sysregs->tcr_el1   = TcrValue;
+            sysregs->mair_el1  = MairValue;
+            sysregs->sctlr_el1 = SctlrValue | 0x80000;
         }
 
         KVirtualAddress GetRandomKernelBaseAddress(KInitialPageTable &page_table, KPhysicalAddress phys_base_address, size_t kernel_size) {
@@ -159,6 +186,7 @@ namespace ams::kern::init::loader {
         const uintptr_t dynamic_offset        = layout->dynamic_offset;
         const uintptr_t init_array_offset     = layout->init_array_offset;
         const uintptr_t init_array_end_offset = layout->init_array_end_offset;
+        const uintptr_t sysreg_offset         = layout->sysreg_offset;
 
         /* Determine the size of the resource region. */
         const size_t resource_region_size = KMemoryLayout::GetResourceRegionSizeForInit(KSystemControl::Init::ShouldIncreaseThreadResourceLimit());
@@ -199,7 +227,7 @@ namespace ams::kern::init::loader {
         KInitialPageTable init_pt(KernelBaseRangeStart, KernelBaseRangeLast, g_initial_page_allocator);
 
         /* Setup initial identity mapping. TTBR1 table passed by reference. */
-        SetupInitialIdentityMapping(init_pt, base_address, bss_end_offset, resource_end_address, InitialPageTableRegionSizeMax, g_initial_page_allocator);
+        SetupInitialIdentityMapping(init_pt, base_address, bss_end_offset, resource_end_address, InitialPageTableRegionSizeMax, g_initial_page_allocator, reinterpret_cast<KernelSystemRegisters *>(base_address + sysreg_offset));
 
         /* Generate a random slide for the kernel's base address. */
         const KVirtualAddress virtual_base_address = GetRandomKernelBaseAddress(init_pt, base_address, bss_end_offset);