diff --git a/.gitignore b/.gitignore index 9a9a07d7b..f767b059c 100644 --- a/.gitignore +++ b/.gitignore @@ -51,9 +51,9 @@ Module.symvers Mkfile.old dkms.conf -.*/ +.**/ -exosphere/bpmpfw/out/ -exosphere/bpmpfw/build -exosphere/build -exosphere/out +exosphere/bpmpfw/out/** +exosphere/bpmpfw/build/** +exosphere/build/** +exosphere/out/** diff --git a/exosphere/src/coldboot_init.c b/exosphere/src/coldboot_init.c index b62650c08..645f01e4b 100644 --- a/exosphere/src/coldboot_init.c +++ b/exosphere/src/coldboot_init.c @@ -41,7 +41,7 @@ static void configure_ttbls(void) { identity_map_all_mappings(mmu_l1_tbl, mmu_l3_tbl); mmio_map_all_devices(mmu_l3_tbl); lp0_map_all_plaintext_ram_segments(mmu_l3_tbl); - lp0_map_all_ciphertext_ram_segments(mmu_l3_tbl); + warmboot_map_all_ram_segments(mmu_l3_tbl); tzram_map_all_segments(mmu_l3_tbl); } diff --git a/exosphere/src/memory_map.h b/exosphere/src/memory_map.h index 1bb9dd964..fe728ca58 100644 --- a/exosphere/src/memory_map.h +++ b/exosphere/src/memory_map.h @@ -82,6 +82,8 @@ static const struct { #define WARMBOOT_RAM_SEGMENT_BASE (LP0_ENTRY_RAM_SEGMENT_BASE + 0x000047000) /* increment seems to be arbitrary ? */ #define TZRAM_SEGMENT_BASE (MMIO_BASE + 0x0001E0000) +#define IDENTIY_MAPPING_ID_MAX 3 + #define MMIO_DEVID_GICD 0 #define MMIO_DEVID_GICC 1 #define MMIO_DEVID_UART_A 2 @@ -100,13 +102,16 @@ static const struct { #define MMIO_DEVID_GPIO 15 #define MMIO_DEVID_DTV_I2C234 16 #define MMIO_DEVID_EXCEPTION_VECTORS 17 +#define MMIO_DEVID_MAX 18 #define LP0_ENTRY_RAM_SEGMENT_ID_DECRYPTED_TZRAM 0 #define LP0_ENTRY_RAM_SEGMENT_ID_LP0_ENTRY_CODE 1 #define LP0_ENTRY_RAM_SEGMENT_ID_CURRENT_TZRAM 2 +#define LP0_ENTRY_RAM_SEGMENT_ID_MAX 3 #define WARMBOOT_RAM_SEGMENT_ID_SE_STATE 0 #define WARMBOOT_RAM_SEGMENT_ID_TZRAM 1 +#define WARMBOOT_RAM_SEGMENT_ID_MAX 2 #define TZRAM_SEGMENT_ID_WARMBOOT_CRT0_AND_MAIN 0 #define TZRAM_SEGMENT_ID_PK2LDR 1 @@ -116,12 +121,13 @@ static const struct { #define TZRAM_SEGEMENT_ID_SECMON_EVT 5 #define TZRAM_SEGMENT_ID_L2_TRANSLATION_TABLE 6 #define TZRAM_SEGMENT_ID_L3_TRANSLATION_TABLE 7 +#define TZRAM_SEGMENT_ID_MAX 8 /**********************************************************************************************/ -static inline void identity_map_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *mmu_l3_tbl) { +INLINE_UNROLL static inline void identity_map_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *mmu_l3_tbl) { static uint64_t base_attributes = MMU_PTE_BLOCK_INNER_SHAREBLE | ATTRIB_MEMTYPE_NORMAL; - for(size_t i = 0; i < sizeof(g_identity_mappings) / sizeof(g_identity_mappings[0]); i++) { + for(size_t i = 0; i < IDENTIY_MAPPING_ID_MAX; i++) { uint64_t attributes = base_attributes | g_identity_mappings[i].attributes; if(g_identity_mappings[i].is_block_range) { mmu_map_block_range(1, mmu_l1_tbl, g_identity_mappings[i].address, g_identity_mappings[i].address, @@ -134,8 +140,8 @@ static inline void identity_map_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *m } } -static inline void identity_unmap_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *mmu_l3_tbl) { - for(size_t i = 0; i < sizeof(g_identity_mappings) / sizeof(g_identity_mappings[0]); i++) { +INLINE_UNROLL static inline void identity_unmap_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t *mmu_l3_tbl) { + for(size_t i = 0; i < IDENTIY_MAPPING_ID_MAX; i++) { if(g_identity_mappings[i].is_block_range) { mmu_unmap_range(1, mmu_l1_tbl, g_identity_mappings[i].address, g_identity_mappings[i].size); } @@ -147,12 +153,12 @@ static inline void identity_unmap_all_mappings(uintptr_t *mmu_l1_tbl, uintptr_t /**********************************************************************************************/ -static inline uintptr_t mmio_get_device_pa(unsigned int device_id) { +INLINE_UNROLL static inline uintptr_t mmio_get_device_pa(unsigned int device_id) { return g_devices[device_id].pa; } #ifndef MEMORY_MAP_USE_IDENTIY_MAPPING -static inline uintptr_t mmio_get_device_address(unsigned int device_id) { +INLINE_UNROLL static inline const uintptr_t mmio_get_device_address(unsigned int device_id) { size_t offset = 0; for(unsigned int i = 0; i < device_id; i++) { offset += g_devices[i].size; @@ -163,16 +169,16 @@ static inline uintptr_t mmio_get_device_address(unsigned int device_id) { } #else -static inline uintptr_t mmio_get_device_address(unsigned int device_id) { +INLINE_UNROLL static inline uintptr_t mmio_get_device_address(unsigned int device_id) { return mmio_get_device_pa(device_id); } #endif -static inline void mmio_map_all_devices(uintptr_t *mmu_l3_tbl) { +INLINE_UNROLL static inline void mmio_map_all_devices(uintptr_t *mmu_l3_tbl) { static const uint64_t secure_device_attributes = MMU_PTE_BLOCK_XN | MMU_PTE_BLOCK_INNER_SHAREBLE | ATTRIB_MEMTYPE_DEVICE; static const uint64_t device_attributes = MMU_PTE_BLOCK_NS | secure_device_attributes; - for(size_t i = 0, offset = 0; i < sizeof(g_devices) / sizeof(g_devices[0]); i++) { + for(size_t i = 0, offset = 0; i < MMIO_DEVID_MAX; i++) { uint64_t attributes = g_devices[i].is_secure ? secure_device_attributes : device_attributes; mmu_map_page_range(mmu_l3_tbl, MMIO_BASE + offset, g_devices[i].pa, g_devices[i].size, attributes); @@ -181,8 +187,8 @@ static inline void mmio_map_all_devices(uintptr_t *mmu_l3_tbl) { } } -static inline void mmio_unmap_all_devices(uintptr_t *mmu_l3_tbl) { - for(size_t i = 0, offset = 0; i < sizeof(g_devices) / sizeof(g_devices[0]); i++) { +INLINE_UNROLL static inline void mmio_unmap_all_devices(uintptr_t *mmu_l3_tbl) { + for(size_t i = 0, offset = 0; i < MMIO_DEVID_MAX; i++) { mmu_unmap_range(3, mmu_l3_tbl, MMIO_BASE + offset, g_devices[i].size); offset += g_devices[i].size; @@ -192,22 +198,22 @@ static inline void mmio_unmap_all_devices(uintptr_t *mmu_l3_tbl) { /**********************************************************************************************/ -static inline uintptr_t lp0_get_plaintext_ram_segment_pa(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t lp0_get_plaintext_ram_segment_pa(unsigned int segment_id) { return g_lp0_entry_ram_segments[segment_id].pa; } #ifndef MEMORY_MAP_USE_IDENTIY_MAPPING -static inline uintptr_t lp0_get_plaintext_ram_segment_address(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t lp0_get_plaintext_ram_segment_address(unsigned int segment_id) { return LP0_ENTRY_RAM_SEGMENT_BASE + 0x10000 * segment_id; } #else -static inline uintptr_t lp0_get_plaintext_ram_segment_address(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t lp0_get_plaintext_ram_segment_address(unsigned int segment_id) { return lp0_get_plaintext_ram_segment_pa(segment_id); } #endif -static inline void lp0_map_all_plaintext_ram_segments(uintptr_t *mmu_l3_tbl) { - for(size_t i = 0, offset = 0; i < sizeof(g_lp0_entry_ram_segments) / sizeof(g_lp0_entry_ram_segments[0]); i++) { +INLINE_UNROLL static inline void lp0_map_all_plaintext_ram_segments(uintptr_t *mmu_l3_tbl) { + for(size_t i = 0, offset = 0; i < LP0_ENTRY_RAM_SEGMENT_ID_MAX; i++) { uint64_t attributes = MMU_PTE_BLOCK_XN | MMU_PTE_BLOCK_INNER_SHAREBLE | g_lp0_entry_ram_segments[i].attributes; mmu_map_page_range(mmu_l3_tbl, LP0_ENTRY_RAM_SEGMENT_BASE + offset, g_lp0_entry_ram_segments[i].pa, g_lp0_entry_ram_segments[i].size, attributes); @@ -215,8 +221,8 @@ static inline void lp0_map_all_plaintext_ram_segments(uintptr_t *mmu_l3_tbl) { } } -static inline void lp0_unmap_all_plaintext_ram_segments(uintptr_t *mmu_l3_tbl) { - for(size_t i = 0, offset = 0; i < sizeof(g_lp0_entry_ram_segments) / sizeof(g_lp0_entry_ram_segments[0]); i++) { +INLINE_UNROLL static inline void lp0_unmap_all_plaintext_ram_segments(uintptr_t *mmu_l3_tbl) { + for(size_t i = 0, offset = 0; i < LP0_ENTRY_RAM_SEGMENT_ID_MAX; i++) { mmu_unmap_range(3, mmu_l3_tbl, LP0_ENTRY_RAM_SEGMENT_BASE + offset, g_lp0_entry_ram_segments[i].size); offset += 0x10000; @@ -225,12 +231,12 @@ static inline void lp0_unmap_all_plaintext_ram_segments(uintptr_t *mmu_l3_tbl) { /**********************************************************************************************/ -static inline uintptr_t lp0_get_ciphertext_ram_segment_pa(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t warmboot_get_ram_segment_pa(unsigned int segment_id) { return g_warmboot_ram_segments[segment_id].pa; } #ifndef MEMORY_MAP_USE_IDENTIY_MAPPING -static inline uintptr_t lp0_get_ciphertext_ram_segment_address(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t warmboot_get_ram_segment_address(unsigned int segment_id) { size_t offset = 0; for(unsigned int i = 0; i < segment_id; i++) { offset += g_warmboot_ram_segments[i].size; @@ -239,13 +245,13 @@ static inline uintptr_t lp0_get_ciphertext_ram_segment_address(unsigned int segm return WARMBOOT_RAM_SEGMENT_BASE + offset; } #else -static inline uintptr_t lp0_get_ciphertext_ram_segment_address(unsigned int segment_id) { - return lp0_get_ciphertext_ram_segment_pa(segment_id); +INLINE_UNROLL static inline uintptr_t warmboot_get_ram_segment_address(unsigned int segment_id) { + return warmboot_get_ram_segment_pa(segment_id); } #endif -static inline void lp0_map_all_ciphertext_ram_segments(uintptr_t *mmu_l3_tbl) { - for(size_t i = 0, offset = 0; i < sizeof(g_warmboot_ram_segments) / sizeof(g_warmboot_ram_segments[0]); i++) { +INLINE_UNROLL static inline void warmboot_map_all_ram_segments(uintptr_t *mmu_l3_tbl) { + for(size_t i = 0, offset = 0; i < WARMBOOT_RAM_SEGMENT_ID_MAX; i++) { uint64_t attributes = MMU_PTE_BLOCK_XN | MMU_PTE_BLOCK_INNER_SHAREBLE | g_warmboot_ram_segments[i].attributes; mmu_map_page_range(mmu_l3_tbl, WARMBOOT_RAM_SEGMENT_BASE + offset, g_warmboot_ram_segments[i].pa, g_warmboot_ram_segments[i].size, attributes); @@ -253,8 +259,8 @@ static inline void lp0_map_all_ciphertext_ram_segments(uintptr_t *mmu_l3_tbl) { } } -static inline void lp0_unmap_all_ciphertext_ram_segments(uintptr_t *mmu_l3_tbl) { - for(size_t i = 0, offset = 0; i < sizeof(g_warmboot_ram_segments) / sizeof(g_warmboot_ram_segments[0]); i++) { +INLINE_UNROLL static inline void warmboot_unmap_all_ram_segments(uintptr_t *mmu_l3_tbl) { + for(size_t i = 0, offset = 0; i < WARMBOOT_RAM_SEGMENT_ID_MAX; i++) { mmu_unmap_range(3, mmu_l3_tbl, WARMBOOT_RAM_SEGMENT_BASE + offset, g_warmboot_ram_segments[i].size); offset += g_warmboot_ram_segments[i].size; @@ -263,12 +269,12 @@ static inline void lp0_unmap_all_ciphertext_ram_segments(uintptr_t *mmu_l3_tbl) /**********************************************************************************************/ -static inline uintptr_t tzram_get_segment_pa(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t tzram_get_segment_pa(unsigned int segment_id) { return 0x7C010000 + g_tzram_segments[segment_id].tzram_offset; } #ifndef MEMORY_MAP_USE_IDENTIY_MAPPING -static inline uintptr_t tzram_get_segment_address(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t tzram_get_segment_address(unsigned int segment_id) { size_t offset = 0; for(unsigned int i = 0; i < segment_id; i++) { offset += g_tzram_segments[i].increment; @@ -277,14 +283,14 @@ static inline uintptr_t tzram_get_segment_address(unsigned int segment_id) { return TZRAM_SEGMENT_BASE + offset; } #else -static inline uintptr_t tzram_get_segment_address(unsigned int segment_id) { +INLINE_UNROLL static inline uintptr_t tzram_get_segment_address(unsigned int segment_id) { return tzram_get_segment_pa(segment_id); } #endif -static inline void tzram_map_all_segments(uintptr_t *mmu_l3_tbl) { +INLINE_UNROLL static inline void tzram_map_all_segments(uintptr_t *mmu_l3_tbl) { /* Except the SPL userpage */ - for(size_t i = 0, offset = 0; i < sizeof(g_tzram_segments) / sizeof(g_tzram_segments[0]); i++) { + for(size_t i = 0, offset = 0; i < TZRAM_SEGMENT_ID_MAX; i++) { uint64_t attributes = (g_tzram_segments[i].is_code_segment ? 0 : MMU_PTE_BLOCK_XN) | MMU_PTE_BLOCK_INNER_SHAREBLE | ATTRIB_MEMTYPE_NORMAL; if(g_tzram_segments[i].map_size == 0) { continue; @@ -295,9 +301,9 @@ static inline void tzram_map_all_segments(uintptr_t *mmu_l3_tbl) { } } -static inline void tzram_unmap_all_segments(uintptr_t *mmu_l3_tbl) { +INLINE_UNROLL static inline void tzram_unmap_all_segments(uintptr_t *mmu_l3_tbl) { /* Except the SPL userpage */ - for(size_t i = 0, offset = 0; i < sizeof(g_warmboot_ram_segments) / sizeof(g_warmboot_ram_segments[0]); i++) { + for(size_t i = 0, offset = 0; i < TZRAM_SEGMENT_ID_MAX; i++) { if(g_tzram_segments[i].map_size == 0) { continue; } diff --git a/exosphere/src/utils.h b/exosphere/src/utils.h index c731d2743..3dc79bcee 100644 --- a/exosphere/src/utils.h +++ b/exosphere/src/utils.h @@ -5,8 +5,13 @@ #include #include -#define BIT(x) (1u << (x)) -#define BITL(x) (1ull << (x)) +#define BIT(n) (1u << (n)) +#define BITL(n) (1ull << (n)) + +#define ALIGN(m) __attribute__((aligned(m))) +#define PACKED __attribute__((packed)) + +#define INLINE_UNROLL __attribute__((always_inline, optimize("unroll-all-loops"))) void panic(uint32_t code); void generic_panic(void);