2018-02-27 01:41:31 +00:00
|
|
|
#include <stdbool.h>
|
2018-02-18 02:50:39 +00:00
|
|
|
#include <stdint.h>
|
2018-02-27 21:29:47 +00:00
|
|
|
#include "arm.h"
|
2018-02-23 02:08:57 +00:00
|
|
|
#include "cpu_context.h"
|
2018-02-27 01:41:31 +00:00
|
|
|
#include "flow.h"
|
2018-02-18 02:50:39 +00:00
|
|
|
#include "pmc.h"
|
2018-02-24 00:09:34 +00:00
|
|
|
#include "timers.h"
|
2018-02-28 00:10:51 +00:00
|
|
|
#include "smc_api.h"
|
2018-02-24 00:09:34 +00:00
|
|
|
#include "utils.h"
|
2018-03-03 02:43:46 +00:00
|
|
|
#include "synchronization.h"
|
2018-02-28 00:59:50 +00:00
|
|
|
#include "preprocessor.h"
|
|
|
|
|
|
|
|
#define SAVE_SYSREG64(reg) do { __asm__ __volatile__ ("mrs %0, " #reg : "=r"(temp_reg) :: "memory"); g_cpu_contexts[current_core].reg = temp_reg; } while(false)
|
|
|
|
#define SAVE_SYSREG32(reg) do { __asm__ __volatile__ ("mrs %0, " #reg : "=r"(temp_reg) :: "memory"); g_cpu_contexts[current_core].reg = (uint32_t)temp_reg; } while(false)
|
|
|
|
#define SAVE_BP_REG(i, _) SAVE_SYSREG64(DBGBVR##i##_EL1); SAVE_SYSREG64(DBGBCR##i##_EL1);
|
|
|
|
#define SAVE_WP_REG(i, _) SAVE_SYSREG64(DBGBVR##i##_EL1); SAVE_SYSREG64(DBGBCR##i##_EL1);
|
|
|
|
|
|
|
|
#define RESTORE_SYSREG64(reg) do { temp_reg = g_cpu_contexts[current_core].reg; __asm__ __volatile__ ("msr " #reg ", %0" :: "r"(temp_reg) : "memory"); } while(false)
|
|
|
|
#define RESTORE_SYSREG32(reg) RESTORE_SYSREG64(reg)
|
2018-03-02 22:16:54 +00:00
|
|
|
#define RESTORE_BP_REG(i, _) RESTORE_SYSREG64(DBGBVR##i##_EL1); RESTORE_SYSREG64(DBGBCR##i##_EL1);
|
|
|
|
#define RESTORE_WP_REG(i, _) RESTORE_SYSREG64(DBGBVR##i##_EL1); RESTORE_SYSREG64(DBGBCR##i##_EL1);
|
2018-02-18 02:50:39 +00:00
|
|
|
|
2018-03-03 02:43:46 +00:00
|
|
|
/* start.s */
|
2018-03-11 11:53:52 +00:00
|
|
|
void __attribute__((noreturn)) __jump_to_lower_el(uint64_t arg, uintptr_t ep, uint32_t spsr);
|
2018-03-03 02:43:46 +00:00
|
|
|
|
2018-02-26 08:42:21 +00:00
|
|
|
static saved_cpu_context_t g_cpu_contexts[NUM_CPU_CORES] = {0};
|
2018-02-18 02:50:39 +00:00
|
|
|
|
2018-03-01 00:40:09 +00:00
|
|
|
void use_core_entrypoint_and_argument(uint32_t core, uintptr_t *entrypoint_addr, uint64_t *argument) {
|
|
|
|
saved_cpu_context_t *ctx = &g_cpu_contexts[core];
|
|
|
|
if(ctx->ELR_EL3 == 0 || ctx->is_active) {
|
2018-03-03 18:31:22 +00:00
|
|
|
panic(0xF7F00007); /* invalid context */
|
2018-03-01 00:40:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
*entrypoint_addr = ctx->ELR_EL3;
|
|
|
|
*argument = ctx->argument;
|
|
|
|
|
|
|
|
ctx->ELR_EL3 = 0;
|
|
|
|
ctx->argument = 0;
|
2018-03-03 18:31:22 +00:00
|
|
|
ctx->is_active = 1;
|
2018-03-01 00:40:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void set_core_entrypoint_and_argument(uint32_t core, uintptr_t entrypoint_addr, uint64_t argument) {
|
2018-02-18 02:50:39 +00:00
|
|
|
g_cpu_contexts[core].ELR_EL3 = entrypoint_addr;
|
2018-02-26 10:00:02 +00:00
|
|
|
g_cpu_contexts[core].argument = argument;
|
2018-02-18 02:50:39 +00:00
|
|
|
}
|
|
|
|
|
2018-05-11 12:07:37 +00:00
|
|
|
static __attribute__((target("cmodel=large"), noinline))
|
|
|
|
critical_section_t *get_boot_critical_section(void) {
|
|
|
|
return &g_boot_critical_section;
|
|
|
|
}
|
|
|
|
|
2018-03-03 18:31:22 +00:00
|
|
|
void __attribute__((noreturn)) core_jump_to_lower_el(void) {
|
2018-03-03 02:43:46 +00:00
|
|
|
uintptr_t ep;
|
|
|
|
uint64_t arg;
|
|
|
|
unsigned int core_id = get_core_id();
|
2018-03-11 11:53:52 +00:00
|
|
|
uint32_t spsr = get_spsr();
|
2018-05-11 12:07:37 +00:00
|
|
|
critical_section_t *critsec = get_boot_critical_section();
|
2018-03-03 02:43:46 +00:00
|
|
|
|
|
|
|
use_core_entrypoint_and_argument(core_id, &ep, &arg);
|
2018-05-11 12:07:37 +00:00
|
|
|
critical_section_leave(critsec);
|
|
|
|
flush_dcache_range(critsec, (uint8_t *)critsec + sizeof(critical_section_t));
|
|
|
|
/* already does a dsb sy */
|
2018-03-03 02:43:46 +00:00
|
|
|
__sev();
|
|
|
|
|
2018-03-11 11:53:52 +00:00
|
|
|
/* Nintendo hardcodes EL1, but we can boot fine using other EL1/EL2 modes as well */
|
|
|
|
__jump_to_lower_el(arg, ep, 0b1111 << 6 | (spsr & 0b1101)); /* only keep EL, SPSel, set DAIF */
|
2018-03-03 02:43:46 +00:00
|
|
|
}
|
|
|
|
|
2018-03-01 00:40:09 +00:00
|
|
|
uint32_t cpu_on(uint32_t core, uintptr_t entrypoint_addr, uint64_t argument) {
|
2018-02-18 02:50:39 +00:00
|
|
|
/* Is core valid? */
|
|
|
|
if (core >= NUM_CPU_CORES) {
|
|
|
|
return 0xFFFFFFFE;
|
|
|
|
}
|
2018-03-11 11:53:52 +00:00
|
|
|
|
2018-02-18 02:50:39 +00:00
|
|
|
/* Is core already on? */
|
|
|
|
if (g_cpu_contexts[core].is_active) {
|
|
|
|
return 0xFFFFFFFC;
|
|
|
|
}
|
2018-03-03 18:31:22 +00:00
|
|
|
|
2018-02-26 17:11:49 +00:00
|
|
|
set_core_entrypoint_and_argument(core, entrypoint_addr, argument);
|
2018-03-03 18:31:22 +00:00
|
|
|
|
2018-02-18 02:50:39 +00:00
|
|
|
const uint32_t status_masks[NUM_CPU_CORES] = {0x4000, 0x200, 0x400, 0x800};
|
|
|
|
const uint32_t toggle_vals[NUM_CPU_CORES] = {0xE, 0x9, 0xA, 0xB};
|
2018-03-11 11:53:52 +00:00
|
|
|
|
2018-02-18 02:50:39 +00:00
|
|
|
/* Check if we're already in the correct state. */
|
|
|
|
if ((APBDEV_PMC_PWRGATE_STATUS_0 & status_masks[core]) != status_masks[core]) {
|
|
|
|
uint32_t counter = 5001;
|
2018-03-11 11:53:52 +00:00
|
|
|
|
2018-02-18 02:50:39 +00:00
|
|
|
/* Poll the start bit until 0 */
|
|
|
|
while (APBDEV_PMC_PWRGATE_TOGGLE_0 & 0x100) {
|
|
|
|
wait(1);
|
|
|
|
counter--;
|
|
|
|
if (counter < 1) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
2018-03-11 11:53:52 +00:00
|
|
|
|
2018-02-18 02:50:39 +00:00
|
|
|
/* Program PWRGATE_TOGGLE with the START bit set to 1, selecting CE[N] */
|
|
|
|
APBDEV_PMC_PWRGATE_TOGGLE_0 = toggle_vals[core] | 0x100;
|
2018-03-11 11:53:52 +00:00
|
|
|
|
2018-02-18 02:50:39 +00:00
|
|
|
/* Poll until we're in the correct state. */
|
|
|
|
counter = 5001;
|
|
|
|
while (counter > 0) {
|
|
|
|
if ((APBDEV_PMC_PWRGATE_STATUS_0 & status_masks[core]) == status_masks[core]) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
wait(1);
|
|
|
|
counter--;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-02-27 01:41:31 +00:00
|
|
|
void power_down_current_core(void) {
|
|
|
|
unsigned int current_core = get_core_id();
|
|
|
|
flow_set_csr(current_core, 0);
|
2018-03-09 22:56:16 +00:00
|
|
|
flow_set_halt_events(current_core, false);
|
2018-02-27 01:41:31 +00:00
|
|
|
flow_set_cc4_ctrl(current_core, 0);
|
|
|
|
save_current_core_context();
|
|
|
|
g_cpu_contexts[current_core].is_active = 0;
|
|
|
|
flush_dcache_all();
|
2018-02-28 00:35:35 +00:00
|
|
|
finalize_powerdown();
|
2018-02-27 01:41:31 +00:00
|
|
|
}
|
|
|
|
|
2018-02-25 19:00:50 +00:00
|
|
|
uint32_t cpu_off(void) {
|
2018-02-28 00:10:51 +00:00
|
|
|
unsigned int current_core = get_core_id();
|
|
|
|
if (current_core == 3) {
|
2018-02-27 01:41:31 +00:00
|
|
|
power_down_current_core();
|
|
|
|
} else {
|
2018-02-28 00:35:35 +00:00
|
|
|
clear_priv_smc_in_progress();
|
2018-02-28 00:10:51 +00:00
|
|
|
call_with_stack_pointer(get_exception_entry_stack_address(current_core), power_down_current_core);
|
2018-02-27 01:41:31 +00:00
|
|
|
}
|
|
|
|
while (true) {
|
|
|
|
/* Wait forever. */
|
|
|
|
}
|
2018-02-25 19:00:50 +00:00
|
|
|
return 0;
|
2018-02-27 01:41:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void save_current_core_context(void) {
|
2018-02-27 23:40:05 +00:00
|
|
|
unsigned int current_core = get_core_id();
|
|
|
|
uint64_t temp_reg = 1;
|
|
|
|
/* Write 1 to OS lock .*/
|
|
|
|
__asm__ __volatile__ ("msr oslar_el1, %0" : : "r"(temp_reg));
|
2018-02-28 00:59:50 +00:00
|
|
|
|
2018-02-27 23:40:05 +00:00
|
|
|
/* Save system registers. */
|
2018-02-28 00:59:50 +00:00
|
|
|
|
|
|
|
SAVE_SYSREG32(OSDTRRX_EL1);
|
|
|
|
SAVE_SYSREG32(MDSCR_EL1);
|
|
|
|
SAVE_SYSREG32(OSECCR_EL1);
|
|
|
|
SAVE_SYSREG32(MDCCINT_EL1);
|
|
|
|
SAVE_SYSREG32(DBGCLAIMCLR_EL1);
|
|
|
|
SAVE_SYSREG32(DBGVCR32_EL2);
|
|
|
|
SAVE_SYSREG32(SDER32_EL3);
|
|
|
|
SAVE_SYSREG32(MDCR_EL2);
|
|
|
|
SAVE_SYSREG32(MDCR_EL3);
|
2018-03-11 11:53:52 +00:00
|
|
|
SAVE_SYSREG32(SPSR_EL3);
|
2018-02-28 00:59:50 +00:00
|
|
|
|
|
|
|
EVAL(REPEAT(6, SAVE_BP_REG, ~));
|
|
|
|
EVAL(REPEAT(4, SAVE_WP_REG, ~));
|
2018-02-27 23:40:05 +00:00
|
|
|
|
|
|
|
/* Mark context as saved. */
|
|
|
|
g_cpu_contexts[current_core].is_saved = 1;
|
2018-02-25 19:00:50 +00:00
|
|
|
}
|
2018-02-27 01:41:31 +00:00
|
|
|
|
|
|
|
void restore_current_core_context(void) {
|
2018-02-27 23:40:05 +00:00
|
|
|
unsigned int current_core = get_core_id();
|
|
|
|
uint64_t temp_reg;
|
2018-02-28 00:59:50 +00:00
|
|
|
|
2018-02-27 23:40:05 +00:00
|
|
|
if (g_cpu_contexts[current_core].is_saved) {
|
2018-02-28 00:59:50 +00:00
|
|
|
RESTORE_SYSREG32(OSDTRRX_EL1);
|
|
|
|
RESTORE_SYSREG32(MDSCR_EL1);
|
|
|
|
RESTORE_SYSREG32(OSECCR_EL1);
|
|
|
|
RESTORE_SYSREG32(MDCCINT_EL1);
|
|
|
|
RESTORE_SYSREG32(DBGCLAIMCLR_EL1);
|
|
|
|
RESTORE_SYSREG32(DBGVCR32_EL2);
|
|
|
|
RESTORE_SYSREG32(SDER32_EL3);
|
|
|
|
RESTORE_SYSREG32(MDCR_EL2);
|
|
|
|
RESTORE_SYSREG32(MDCR_EL3);
|
2018-03-11 11:53:52 +00:00
|
|
|
RESTORE_SYSREG32(SPSR_EL3);
|
2018-02-28 00:59:50 +00:00
|
|
|
|
|
|
|
EVAL(REPEAT(6, RESTORE_BP_REG, ~));
|
|
|
|
EVAL(REPEAT(4, RESTORE_WP_REG, ~));
|
2018-03-03 18:31:22 +00:00
|
|
|
|
2018-02-27 23:40:05 +00:00
|
|
|
g_cpu_contexts[current_core].is_saved = 0;
|
|
|
|
}
|
2018-02-27 21:29:47 +00:00
|
|
|
}
|
2018-02-28 00:10:51 +00:00
|
|
|
|
2018-03-03 18:31:22 +00:00
|
|
|
bool is_core_active(uint32_t core) {
|
|
|
|
return g_cpu_contexts[core].is_active != 0;
|
|
|
|
}
|
|
|
|
|
2018-03-02 21:44:21 +00:00
|
|
|
void set_core_is_active(uint32_t core, bool is_active) {
|
|
|
|
g_cpu_contexts[core].is_active = (is_active) ? 1 : 0;
|
|
|
|
}
|
|
|
|
|
2018-02-28 00:10:51 +00:00
|
|
|
void set_current_core_active(void) {
|
2018-03-02 21:44:21 +00:00
|
|
|
set_core_is_active(get_core_id(), true);
|
2018-02-28 00:10:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void set_current_core_inactive(void) {
|
2018-03-02 21:44:21 +00:00
|
|
|
set_core_is_active(get_core_id(), false);
|
2018-02-28 00:10:51 +00:00
|
|
|
}
|
|
|
|
|