Rewrite relocation stuff

exosphere/linker.ld

@@ -40,25 +40,6 @@ SECTIONS
         . = ALIGN(64);
     } >fake AT>fake
 
-    .warm_crt0 :
-    {
-        . = ALIGN(64);
-        __warmboot_crt0_lma__   = LOADADDR(.warm_crt0);
-        __warmboot_crt0_start__ = ABSOLUTE(.);
-        KEEP (*(.warm_crt0.text.start)) /* Should be first */
-        KEEP (*(.warm_crt0.text*))
-        KEEP (build/warmboot_init.o(.text*))
-        *(.warm_crt0.rodata*)
-        build/warmboot_init.o(.rodata*)
-        *(.warm_crt0.data*)
-        build/warmboot_init.o(.data*)
-        . = ALIGN(8);
-        *(.warm_crt0.bss*)
-        build/warmboot_init.o(.bss*)
-        . = ALIGN(64);
-        __warmboot_crt0_end__ = ABSOLUTE(.);
-    } >warmboot_crt0 AT>fake
-
     .pk2ldr :
     {
         . = ALIGN(4096);
@@ -89,6 +70,25 @@ SECTIONS
         __vectors_end__ = ABSOLUTE(.);
     } >evt AT>fake
 
+    .warm_crt0 :
+    {
+        . = ALIGN(64);
+        __warmboot_crt0_lma__   = LOADADDR(.warm_crt0);
+        __warmboot_crt0_start__ = ABSOLUTE(.);
+        KEEP (*(.warm_crt0.text.start)) /* Should be first */
+        KEEP (*(.warm_crt0.text*))
+        KEEP (build/warmboot_init.o(.text*))
+        *(.warm_crt0.rodata*)
+        build/warmboot_init.o(.rodata*)
+        *(.warm_crt0.data*)
+        build/warmboot_init.o(.data*)
+        . = ALIGN(8);
+        *(.warm_crt0.bss*)
+        build/warmboot_init.o(.bss*)
+        . = ALIGN(64);
+        __warmboot_crt0_end__ = ABSOLUTE(.);
+    } >warmboot_crt0 AT>fake
+
     .text :
     {
         . = ALIGN(256);
@@ -209,12 +209,12 @@ SECTIONS
     {
         . = ALIGN(8);
         __main_bss_start__  = ABSOLUTE(.);
+        __loaded_end_lma__  = LOADADDR(.bss);
         *(.dynbss)
         *(.bss .bss.* .gnu.linkonce.b.*)
         *(COMMON)
         . = ALIGN(8);
         __main_end__ = ABSOLUTE(.);
-        __end_lma__  = LOADADDR(.bss) + __main_end__ - __main_bss_start__;
     } >main AT>fake
 
     __end__ = ABSOLUTE(.) ;

exosphere/src/coldboot_init.c

@@ -6,13 +6,6 @@
 
 extern const uint8_t __start_cold[];
 
-extern const uint8_t __warmboot_crt0_start__[], __warmboot_crt0_end__[];
-extern const uint8_t __main_start__[], __main_bss_start__[], __main_end__[];
-extern const uint8_t __pk2ldr_start__[], __pk2ldr_bss_start__[], __pk2ldr_end__[];
-extern const uint8_t __vectors_start__[], __vectors_end__[];
-
-extern const size_t __warmboot_crt0_offset, __main_offset, __pk2ldr_offset, __vectors_offset;
-
 /* warmboot_init.c */
 void set_memory_registers_enable_mmu(void);
 
@@ -100,68 +93,57 @@ static void configure_ttbls(void) {
     tzram_map_all_segments(mmu_l3_tbl);
 }
 
-__attribute__((noinline)) static void copy_lma_to_vma(const void *vma, size_t offset, size_t size) {
+static void translate_warmboot_func_list(coldboot_crt0_reloc_list_t *reloc_list) {
-    uint64_t *p_vma = (uint64_t *)vma;
+    coldboot_crt0_reloc_t *warmboot_crt0_reloc = &reloc_list->relocs[0];
-    const uint64_t *p_lma = (const uint64_t *)(__start_cold + offset);
+    coldboot_crt0_reloc_t *main_reloc = &reloc_list->relocs[reloc_list->nb_relocs_pre_mmu_init];
-    for (size_t i = 0; i < size / 8; i++) {
+
-        p_vma[i] = p_lma[i];
+    /* The main segment immediately follows the warmboot crt0 in TZRAM, in the same page. */
+    uintptr_t main_pa = (uintptr_t)warmboot_crt0_reloc->vma | ((uintptr_t)main_reloc->vma & ~0xFFF);
+    for(size_t i = 0; i < reloc_list->func_list->nb_funcs; i++) {
+        if(reloc_list->func_list->addrs[i] >= 0x1F0000000ull) {
+            reloc_list->func_list->addrs[i] = main_pa + reloc_list->func_list->addrs[i] - (uintptr_t)main_reloc->vma;
+        }
     }
 }
 
-FAR_REACHING static void copy_warmboot_crt0(void) {
+static void do_relocation(const coldboot_crt0_reloc_list_t *reloc_list, size_t index) {
-    copy_lma_to_vma(__warmboot_crt0_start__, __warmboot_crt0_offset, __warmboot_crt0_end__ - __warmboot_crt0_start__);
+    uint64_t *p_vma = (uint64_t *)reloc_list->relocs[index].vma;
-}
+    const uint64_t *p_lma = (const uint64_t *)(reloc_list->reloc_base + reloc_list->relocs[index].reloc_offset);
+    size_t size = reloc_list->relocs[index].end_vma - reloc_list->relocs[index].vma;
 
-FAR_REACHING static void copy_other_sections(void) {
+    for(size_t i = 0; i < size / 8; i++) {
-    copy_lma_to_vma(__main_start__, __main_offset, __main_end__ - __main_start__);
+        p_vma[i] = reloc_list->relocs[index].reloc_offset != 0 ? p_lma[i] : 0;
-    copy_lma_to_vma(__pk2ldr_start__, __pk2ldr_offset, __pk2ldr_end__ - __pk2ldr_start__);
+    }
-    copy_lma_to_vma(__vectors_start__, __vectors_offset, __vectors_end__ - __vectors_start__);
-}
-
-FAR_REACHING static void set_memory_registers_enable_mmu_tzram_pa(void) {
-    volatile uintptr_t v = (uintptr_t)set_memory_registers_enable_mmu; 
-    ((void (*)(void))v)();
-}
-
-FAR_REACHING static void flush_dcache_all_tzram_pa(void) {
-    uintptr_t pa = TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN);
-    uintptr_t main_pa = pa | ((uintptr_t)__main_start__ & 0xFFF);
-    uintptr_t v = (uintptr_t)flush_dcache_all - (uintptr_t)__main_start__ + (uintptr_t)main_pa;
-    ((void (*)(void))v)();
-}
-
-FAR_REACHING static void invalidate_icache_all_tzram_pa(void) {
-    uintptr_t pa = TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN);
-    uintptr_t main_pa = pa | ((uintptr_t)__main_start__ & 0xFFF);
-    uintptr_t v = (uintptr_t)invalidate_icache_all - (uintptr_t)__main_start__ + (uintptr_t)main_pa;
-    ((void (*)(void))v)();
-}
-
-FAR_REACHING static void clear_bss(void) {
-    volatile uintptr_t v = (uintptr_t)memset;
-    ((void (*)(void *, int, size_t))v)((void *)__pk2ldr_bss_start__, 0, __pk2ldr_end__ - __pk2ldr_bss_start__);
-    ((void (*)(void *, int, size_t))v)((void *)__main_bss_start__, 0, __main_end__ - __main_bss_start__);
 }
 
 uintptr_t get_coldboot_crt0_stack_address(void) {
     return TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_CORE3_STACK) + 0x800;
 }
 
-void coldboot_init(void) {
+void coldboot_init(coldboot_crt0_reloc_list_t *reloc_list) {
+    /* Custom approach */
+    reloc_list->reloc_base = (uintptr_t)__start_cold;
+
     /* TODO: Set NX BOOTLOADER clock time field */
 
-    copy_warmboot_crt0();
+    /* This at least copies .warm_crt0 to its VMA. */
+    for(size_t i = 0; i < reloc_list->nb_relocs_pre_mmu_init; i++) {
+        do_relocation(reloc_list, i);
+    }
     /* At this point, we can (and will) access functions located in .warm_crt0 */
 
+    translate_warmboot_func_list(reloc_list);
+
     /* TODO: initialize DMA controllers, etc. */
     configure_ttbls();
-    set_memory_registers_enable_mmu_tzram_pa();
+    reloc_list->func_list->funcs.set_memory_registers_enable_mmu();
 
-    copy_other_sections();
+    /* Copy or clear the remaining sections */
+    for(size_t i = 0; i < reloc_list->nb_relocs_post_mmu_init; i++) {
+        do_relocation(reloc_list, reloc_list->nb_relocs_pre_mmu_init + i);
+    }
 
-    flush_dcache_all_tzram_pa();
+    reloc_list->func_list->funcs.flush_dcache_all();
-    invalidate_icache_all_tzram_pa();
+    reloc_list->func_list->funcs.invalidate_icache_all();
     /* At this point we can access all the mapped segments (all other functions, data...) normally */
-
-    clear_bss();
 }

exosphere/src/package2.c

@@ -372,7 +372,7 @@ uintptr_t get_pk2ldr_stack_address(void) {
 
 /* This function is called during coldboot init, and validates a package2. */
 /* This package2 is read into memory by a concurrent BPMP bootloader. */
-void load_package2(void) {
+void load_package2(coldboot_crt0_reloc_list_t *reloc_list) {
     
     /* Setup the Security Engine. */
     setup_se();
@@ -393,7 +393,7 @@ void load_package2(void) {
     randomcache_init();
 
     /* memclear the initial copy of Exosphere running in IRAM (relocated to TZRAM by earlier code). */
-    memset(__start_cold_addr, 0, __bin_size);
+    memset((void *)reloc_list->reloc_base, 0, reloc_list->loaded_bin_size);
 
     /* Let NX Bootloader know that we're running. */
     MAILBOX_NX_BOOTLOADER_IS_SECMON_AWAKE = 1;

exosphere/src/package2.h

@@ -3,7 +3,7 @@
 
 /* This is code responsible for validating a package2. Actual file reading is done by bootloader. */
 
-#include <stdint.h>
+#include "utils.h"
 #include "bootconfig.h"
 #include "memory_map.h"
 
@@ -77,6 +77,6 @@ typedef struct {
 } package2_header_t;
 #pragma pack(pop)
 
-void load_package2(void);
+void load_package2(coldboot_crt0_reloc_list_t *reloc_list);
 
 #endif

exosphere/src/start.s

@@ -80,11 +80,11 @@ __start_cold:
     bl   get_coldboot_crt0_stack_address
     mov  sp, x0
     mov  fp, #0
-    bl   coldboot_init
 
-    adr  x0, __start_cold
+    adrp x19, g_coldboot_crt0_relocation_list
-    ldr  x1, =__start_cold_addr
+    add  x19, x19, #:lo12:g_coldboot_crt0_relocation_list
-    str  x0, [x1]
+    mov  x0, x19
+    bl   coldboot_init
 
     ldr  x16, =__jump_to_main_cold
     br   x16
@@ -100,6 +100,9 @@ __start_warm:
     bl   get_warmboot_crt0_stack_address
     mov  sp, x0
     mov  fp, #0
+
+    adrp x0, g_warmboot_crt0_main_func_list
+    add  x0, x0, #:lo12:g_warmboot_crt0_main_func_list
     bl   warmboot_init
     ldr  x16, =__jump_to_main_warm
     br   x16
@@ -202,32 +205,29 @@ __jump_to_lower_el:
     isb
     eret
 
+/* Custom stuff */
 .align      3
-.section    .cold_crt0.rodata.reloc_constants, "a", %progbits
+.section    .cold_crt0.data.g_coldboot_crt0_relocation_list, "aw", %progbits
-.global     __warmboot_crt0_offset
+.global     g_coldboot_crt0_relocation_list
-__warmboot_crt0_offset:
+g_coldboot_crt0_relocation_list:
-    .quad __warmboot_crt0_lma__
+    .quad   0, __loaded_end_lma__  /* __start_cold, to be set & loaded size */
-
+    .quad   1, 5                   /* number of sections to relocate/clear before & after mmu init */
-.global     __main_offset
+    .quad   g_warmboot_crt0_main_func_list
-__main_offset:
+    /* Relocations */
-    .quad __main_lma__
+    .quad   __warmboot_crt0_start__, __warmboot_crt0_end__, __warmboot_crt0_lma__
-
+    .quad   __main_start__, __main_bss_start__, __main_lma__
-.global     __pk2ldr_offset
+    .quad   __pk2ldr_start__, __pk2ldr_bss_start__, __pk2ldr_lma__
-__pk2ldr_offset:
+    .quad   __vectors_start__, __vectors_end__, __vectors_lma__
-    .quad __pk2ldr_lma__
+    /* BSS clears */
-
+    .quad   __main_bss_start__, __main_end__, 0
-.global     __vectors_offset
+    .quad   __pk2ldr_bss_start__, __pk2ldr_end__, 0
-__vectors_offset:
-    .quad __vectors_lma__
 
 .align      3
-.section    .rodata.__bin_size, "a", %progbits
+.section    .warm_crt0.data.g_warmboot_crt0_main_func_list, "aw", %progbits
-.global     __bin_size
+.global     g_warmboot_crt0_main_func_list
-__bin_size:
+g_warmboot_crt0_main_func_list:
-    .quad   __end_lma__
+    .quad   3   /* Number of functions */
-
+    /* Functions */
-.align      3
+    .quad   set_memory_registers_enable_mmu
-.section    .bss.__start_cold_addr, "w", %nobits
+    .quad   flush_dcache_all
-.global     __start_cold_addr
+    .quad   invalidate_icache_all
-__start_cold_addr:
-    .space  8

exosphere/src/utils.h

@@ -18,6 +18,37 @@
 #define ALINLINE        __attribute__((always_inline))
 #define FAR_REACHING    __attribute__((target("cmodel=large"), noinline))
 
+/* Custom stuff below */
+
+/* For warmboot (and coldboot crt0) */
+typedef struct {
+    size_t      nb_funcs;
+    union {
+        struct {
+            void (*set_memory_registers_enable_mmu)(void);
+            void (*flush_dcache_all)(void);
+            void (*invalidate_icache_all)(void);
+        } funcs;
+        uintptr_t addrs[3];
+    };
+} warmboot_func_list_t;
+
+/* For coldboot */
+typedef struct {
+    uint8_t     *vma;
+    uint8_t     *end_vma;
+    uintptr_t   reloc_offset;
+} coldboot_crt0_reloc_t;
+
+typedef struct {
+    uintptr_t               reloc_base;
+    size_t                  loaded_bin_size;
+    size_t                  nb_relocs_pre_mmu_init;     /* first is always warmboot_crt0 */
+    size_t                  nb_relocs_post_mmu_init;    /* first is always main segment excl. .bss */
+    warmboot_func_list_t    *func_list;
+    coldboot_crt0_reloc_t   relocs[];
+} coldboot_crt0_reloc_list_t;
+
 __attribute__ ((noreturn)) void panic(uint32_t code);
 __attribute__ ((noreturn)) void generic_panic(void);
 bool overlaps(uint64_t as, uint64_t ae, uint64_t bs, uint64_t be);

exosphere/src/warmboot_init.c

@@ -2,27 +2,10 @@
 #include "memory_map.h"
 #include "arm.h"
 
-extern const uint8_t __main_start__[];
-
 /* start.s */
 void __set_memory_registers(uintptr_t ttbr0, uintptr_t vbar, uint64_t cpuectlr, uint32_t scr,
                             uint32_t tcr, uint32_t cptr, uint64_t mair, uint32_t sctlr);
 
-/*
-FAR_REACHING static void flush_dcache_all_tzram_pa(void) {
-    uintptr_t pa = TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN);
-    uintptr_t main_pa = pa | ((uintptr_t)__main_start__ & 0xFFF);
-    uintptr_t v = (uintptr_t)flush_dcache_all - (uintptr_t)__main_start__ + (uintptr_t)main_pa;
-    ((void (*)(void))v)();
-}
-
-FAR_REACHING static void invalidate_icache_all_inner_shareable_tzram_pa(void) {
-    uintptr_t pa = TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN);
-    uintptr_t main_pa = pa | ((uintptr_t)__main_start__ & 0xFFF);
-    uintptr_t v = (uintptr_t)invalidate_icache_all_inner_shareable - (uintptr_t)__main_start__ + (uintptr_t)main_pa;
-    ((void (*)(void))v)();
-}*/
-
 uintptr_t get_warmboot_crt0_stack_address(void) {
     return TZRAM_GET_SEGMENT_PA(TZRAM_SEGMENT_ID_CORE012_STACK) + 0x800;
 }
@@ -80,6 +63,6 @@ void set_memory_registers_enable_mmu(void) {
     __set_memory_registers(ttbr0, vbar, cpuectlr, scr, tcr, cptr, mair, sctlr);
 }
 
-void warmboot_init(void) {
+void warmboot_init(warmboot_func_list_t *func_list) {
-    /* TODO: Implement. */
+    (void)func_list;
 }