kern: finish implementing assembly init routines

libraries/config/common.mk

@@ -76,14 +76,14 @@ TARGET       := $(notdir $(CURDIR))
 BUILD        := build
 DATA         := data
 INCLUDES     := include
-SOURCES      ?= $(foreach d,$(filter-out source/arch source/board,$(wildcard source)),$(call DIR_WILDCARD,$d) $d)
+SOURCES      ?= source $(foreach d,$(filter-out source/arch source/board source,$(wildcard source)),$(call DIR_WILDCARD,$d) $d)
 
 ifneq ($(strip $(wildcard source/$(ATMOSPHERE_ARCH_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_ARCH_DIR))
+SOURCES += source/$(ATMOSPHERE_ARCH_DIR) $(call DIR_WILDCARD,source/$(ATMOSPHERE_ARCH_DIR))
 endif
 ifneq ($(strip $(wildcard source/$(ATMOSPHERE_BOARD_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_BOARD_DIR))
+SOURCES += source/$(ATMOSPHERE_BOARD_DIR $(call DIR_WILDCARD,source/$(ATMOSPHERE_BOARD_DIR))
 endif
 ifneq ($(strip $(wildcard source/$(ATMOSPHERE_OS_DIR)/.*)),)
-SOURCES += $(call DIR_WILDCARD,source/$(ATMOSPHERE_OS_DIR))
+SOURCES += source/$(ATMOSPHERE_OS_DIR $(call DIR_WILDCARD,source/$(ATMOSPHERE_OS_DIR))
 endif

libraries/libmesosphere/include/mesosphere.hpp

@@ -32,6 +32,7 @@
 #include "mesosphere/init/kern_init_elf.hpp"
 #include "mesosphere/init/kern_init_layout.hpp"
 #include "mesosphere/init/kern_init_page_table_select.hpp"
+#include "mesosphere/init/kern_init_arguments_select.hpp"
 
 /* Core functionality. */
 #include "mesosphere/kern_select_interrupts.hpp"

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2018-2019 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+namespace ams::kern::init {
+
+    struct KInitArguments {
+        u64 ttbr0;
+        u64 ttbr1;
+        u64 tcr;
+        u64 mair;
+        u64 cpuactlr;
+        u64 cpuectlr;
+        u64 sctlr;
+        u64 sp;
+        u64 entrypoint;
+        u64 argument;
+        u64 setup_function;
+    };
+
+}

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -495,4 +495,31 @@ namespace ams::kern::init {
 
     };
 
+    class KInitialPageAllocator : public KInitialPageTable::IPageAllocator {
+        private:
+            uintptr_t next_address;
+        public:
+            constexpr ALWAYS_INLINE KInitialPageAllocator() : next_address(Null<uintptr_t>) { /* ... */ }
+
+            ALWAYS_INLINE void Initialize(uintptr_t address) {
+                this->next_address = address;
+            }
+
+            ALWAYS_INLINE uintptr_t GetFinalState() {
+                const uintptr_t final_address = this->next_address;
+                this->next_address = Null<uintptr_t>;
+                return final_address;
+            }
+        public:
+            virtual KPhysicalAddress Allocate() override {
+                MESOSPHERE_ABORT_UNLESS(this->next_address != Null<uintptr_t>);
+                const uintptr_t allocated = this->next_address;
+                this->next_address += PageSize;
+                std::memset(reinterpret_cast<void *>(allocated), 0, PageSize);
+                return allocated;
+            }
+
+            /* No need to override free. The default does nothing, and so would we. */
+    };
+
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp

@@ -19,6 +19,19 @@
 
 namespace ams::kern::arm64::cpu {
 
+#if defined(ATMOSPHERE_CPU_ARM_CORTEX_A57) || defined(ATMOSPHERE_CPU_ARM_CORTEX_A53)
+    constexpr inline size_t InstructionCacheLineSize = 0x40;
+    constexpr inline size_t DataCacheLineSize        = 0x40;
+#else
+    #error "Unknown CPU for cache line sizes"
+#endif
+
+#if defined(ATMOSPHERE_BOARD_NINTENDO_SWITCH)
+    static constexpr size_t NumCores = 4;
+#else
+    #error "Unknown Board for cpu::NumCores"
+#endif
+
     /* Helpers for managing memory state. */
     ALWAYS_INLINE void DataSynchronizationBarrier() {
         __asm__ __volatile__("dsb sy" ::: "memory");

libraries/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2018-2019 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_k_typed_address.hpp>
+
+#ifdef ATMOSPHERE_ARCH_ARM64
+    #include "../arch/arm64/init/kern_k_init_arguments.hpp"
+#else
+    #error "Unknown architecture for KInitArguments"
+#endif
+
+namespace ams::kern::init {
+
+    KPhysicalAddress GetInitArgumentsAddress(s32 core_id);
+
+}

mesosphere/kernel/kernel.ld

@@ -18,7 +18,7 @@ SECTIONS
 
 	.crt0 :
 	{
-		KEEP (*(.crt0))
+		KEEP (*(.crt0 .crt0.*))
 		. = ALIGN(8);
 	} :code
 

mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2018-2019 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+namespace ams::kern::init {
+
+    namespace {
+
+        /* Global Allocator. */
+        KInitialPageAllocator g_initial_page_allocator;
+
+        /* Global initial arguments array. */
+        KInitArguments g_init_arguments[cpu::NumCores];
+
+    }
+
+    void InitializeCore(uintptr_t arg0, uintptr_t initial_page_allocator_state) {
+        /* TODO */
+    }
+
+    KPhysicalAddress GetInitArgumentsAddress(s32 core) {
+        return KPhysicalAddress(std::addressof(g_init_arguments[core]));
+    }
+
+    void InitializeDebugRegisters() {
+        /* TODO */
+    }
+
+    void InitializeExceptionVectors() {
+        /* TODO */
+    }
+
+}

mesosphere/kernel/source/arch/arm64/init/start.s

@@ -94,9 +94,116 @@ core0_el1:
     add x3, x0, x3
     blr x3
 
-    /* TODO: Finish post-kernelldr init code. */
+    /* At this point kernelldr has been invoked, and we are relocated at a random virtual address. */
-1:
+    /* Next thing to do is to set up our memory management and slabheaps -- all the other core initialization. */
-    b 1b
+    /* Call ams::kern::init::InitializeCore(uintptr_t, uintptr_t) */
+    mov x1, x0  /* Kernelldr returns a KInitialPageAllocator state for the kernel to re-use. */
+    mov x0, xzr /* Official kernel always passes zero, when this is non-zero the address is mapped. */
+    bl _ZN3ams4kern4init14InitializeCoreEmm
+
+    /* Get the init arguments for core 0. */
+    mov x0, xzr
+    bl _ZN3ams4kern4init23GetInitArgumentsAddressEi
+
+    bl _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE
+
+/* ams::kern::init::StartOtherCore(const ams::kern::init::KInitArguments *) */
+.section    .crt0.text._ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE, "ax", %progbits
+.global     _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE
+.type       _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE, %function
+_ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE:
+    /* Preserve the KInitArguments pointer in a register. */
+    mov x20, x0
+
+    /* Check our current EL. We want to be executing out of EL1. */
+    /* If we're in EL2, we'll need to deprivilege ourselves. */
+    mrs x1, currentel
+    cmp x1, #0x4
+    b.eq othercore_el1
+    cmp x1, #0x8
+    b.eq othercore_el2
+othercore_el3:
+    b othercore_el3
+othercore_el2:
+    bl _ZN3ams4kern4init16JumpFromEL2ToEL1Ev
+othercore_el1:
+    bl _ZN3ams4kern4init19DisableMmuAndCachesEv
+
+    /* Setup system registers using values from our KInitArguments. */
+    ldr x1, [x20, #0x00]
+    msr ttbr0_el1, x1
+    ldr x1, [x20, #0x08]
+    msr ttbr1_el1, x1
+    ldr x1, [x20, #0x10]
+    msr tcr_el1, x1
+    ldr x1, [x20, #0x18]
+    msr mair_el1, x1
+
+    /* Perform cpu-specific setup. */
+    mrs x1, midr_el1
+    ubfx x2, x1, #0x18, #0x8 /* Extract implementer bits. */
+    cmp x2, #0x41            /* Implementer::ArmLimited */
+    b.ne othercore_cpu_specific_setup_end
+    ubfx x2, x1, #0x4, #0xC  /* Extract primary part number. */
+    cmp x2, #0xD07           /* PrimaryPartNumber::CortexA57 */
+    b.eq othercore_cpu_specific_setup_cortex_a57
+    cmp x2, #0xD03           /* PrimaryPartNumber::CortexA53 */
+    b.eq othercore_cpu_specific_setup_cortex_a53
+    b othercore_cpu_specific_setup_end
+othercore_cpu_specific_setup_cortex_a57:
+othercore_cpu_specific_setup_cortex_a53:
+    ldr x1, [x20, #0x20]
+    msr cpuactlr_el1, x1
+    ldr x1, [x20, #0x28]
+    msr cpuectlr_el1, x1
+
+othercore_cpu_specific_setup_end:
+    /* Ensure instruction consistency. */
+    dsb sy
+    isb
+
+    /* Set sctlr_el1 and ensure instruction consistency. */
+    ldr x1, [x20, #0x30]
+    msr sctlr_el1, x1
+
+    dsb sy
+    isb
+
+    /* Jump to the virtual address equivalent to ams::kern::init::InvokeEntrypoint */
+    ldr x1, [x20, #0x50]
+    adr x2, _ZN3ams4kern4init14StartOtherCoreEPKNS1_14KInitArgumentsE
+    sub x1, x1, x2
+    adr x2, _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE
+    add x1, x1, x2
+    mov x0, x20
+    br x1
+
+/* ams::kern::init::InvokeEntrypoint(const ams::kern::init::KInitArguments *) */
+.section    .crt0.text._ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE, "ax", %progbits
+.global     _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE
+.type       _ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE, %function
+_ZN3ams4kern4init16InvokeEntrypointEPKNS1_14KInitArgumentsE:
+    /* Preserve the KInitArguments pointer in a register. */
+    mov x20, x0
+
+    /* Clear CPACR_EL1. This will prevent classes of traps (SVE, etc). */
+    msr cpacr_el1, xzr
+    isb
+
+    /* Setup the stack pointer. */
+    ldr x1, [x20, #0x38]
+    mov sp, x1
+
+    /* Ensure that system debug registers are setup. */
+    bl _ZN3ams4kern4init24InitializeDebugRegistersEv
+
+    /* Ensure that the exception vectors are setup. */
+    bl _ZN3ams4kern4init26InitializeExceptionVectorsEv
+
+    /* Jump to the entrypoint. */
+    ldr x1, [x20, #0x40]
+    ldr x0, [x20, #0x48]
+    br x1
 
 
 /* ams::kern::init::JumpFromEL2ToEL1() */

mesosphere/kernel_ldr/source/arch/arm64/start.s

@@ -75,10 +75,10 @@ _start:
     bl _ZN3ams4kern4init6loader4MainEmPNS1_12KernelLayoutEm
     str x0, [sp, #0x00]
 
-    /* Call ams::kern::init::loader::Finalize() */
+    /* Call ams::kern::init::loader::GetFinalPageAllocatorState() */
-    bl _ZN3ams4kern4init6loader8FinalizeEv
+    bl _ZN3ams4kern4init6loader26GetFinalPageAllocatorStateEv
 
-    /* X0 is now the next address for the page allocator. */
+    /* X0 is now the saved state for the page allocator. */
     /* We will return this to the kernel. */
 
     /* Return to the newly-relocated kernel. */

mesosphere/kernel_ldr/source/kern_init_loader.cpp

@@ -34,33 +34,6 @@ namespace ams::kern::init::loader {
 
         constexpr size_t InitialPageTableRegionSize = 0x200000;
 
-        class KInitialPageAllocator : public KInitialPageTable::IPageAllocator {
-            private:
-                uintptr_t next_address;
-            public:
-                constexpr ALWAYS_INLINE KInitialPageAllocator() : next_address(Null<uintptr_t>) { /* ... */ }
-
-                ALWAYS_INLINE void Initialize(uintptr_t address) {
-                    this->next_address = address;
-                }
-
-                ALWAYS_INLINE uintptr_t Finalize() {
-                    const uintptr_t final_address = this->next_address;
-                    this->next_address = Null<uintptr_t>;
-                    return final_address;
-                }
-            public:
-                virtual KPhysicalAddress Allocate() override {
-                    MESOSPHERE_ABORT_UNLESS(this->next_address != Null<uintptr_t>);
-                    const uintptr_t allocated = this->next_address;
-                    this->next_address += PageSize;
-                    std::memset(reinterpret_cast<void *>(allocated), 0, PageSize);
-                    return allocated;
-                }
-
-                /* No need to override free. The default does nothing, and so would we. */
-        };
-
         /* Global Allocator. */
         KInitialPageAllocator g_initial_page_allocator;
 
@@ -335,8 +308,8 @@ namespace ams::kern::init::loader {
         return GetInteger(virtual_base_address) - base_address;
     }
 
-    uintptr_t Finalize() {
+    uintptr_t GetFinalPageAllocatorState() {
-        return g_initial_page_allocator.Finalize();
+        return g_initial_page_allocator.GetFinalState();
     }
 
 }