bdk: smmu: refactor and update driver

- Allow ASID to be configured
- Allow 34-bit PAs
- Use special type for setting PDE/PTE config
- Initialize all pages as non accessible
- Add function for mapping 4MB regions directly
- Add SMMU heap reset function
- Correct address load OP to 32-bit and remove alignment on SMMU enable payload
- Refactor all defines

bdk/mem/smmu.c

@@ -18,6 +18,7 @@
 
 #include <string.h>
 
+#include <soc/bpmp.h>
 #include <soc/ccplex.h>
 #include <soc/timer.h>
 #include <soc/t210.h>
@@ -25,27 +26,39 @@
 #include <mem/smmu.h>
 #include <memory_map.h>
 
-#define SMMU_PAGE_SHIFT 12
+/*! SMMU register defines */
-#define SMMU_PAGE_SIZE  (1 << SMMU_PAGE_SHIFT)
+#define SMMU_ASID(asid)          (((asid) << 24u) | ((asid) << 16u) | ((asid) << 8u) | (asid))
+#define SMMU_ENABLE              BIT(31)
+#define SMMU_TLB_ACTIVE_LINES(l) ((l) << 0u)
+#define SMMU_TLB_RR_ARBITRATION  BIT(28)
+#define SMMU_TLB_HIT_UNDER_MISS  BIT(29)
+#define SMMU_TLB_STATS_ENABLE    BIT(31)
+#define SMUU_PTC_INDEX_MAP(m)    ((m) << 0u)
+#define SMUU_PTC_LINE_MASK(m)    ((m) << 8u)
+#define SMUU_PTC_REQ_LIMIT(l)    ((l) << 24u)
+#define SMUU_PTC_CACHE_ENABLE    BIT(29)
+#define SMUU_PTC_STATS_ENABLE    BIT(31)
+
+/*! Page table defines */
+#define SMMU_4MB_REGION 0
+#define SMMU_PAGE_TABLE 1
 #define SMMU_PDIR_COUNT 1024
-#define SMMU_PDIR_SIZE  (sizeof(u32) * SMMU_PDIR_COUNT)
 #define SMMU_PTBL_COUNT 1024
-#define SMMU_PTBL_SIZE  (sizeof(u32) * SMMU_PTBL_COUNT)
+#define SMMU_PAGE_SHIFT 12u
-#define SMMU_PDIR_SHIFT 12
+#define SMMU_PTN_SHIFT  SMMU_PAGE_SHIFT
-#define SMMU_PDE_SHIFT  12
+#define SMMU_PDN_SHIFT  22u
-#define SMMU_PTE_SHIFT  12
+#define SMMU_ADDR_TO_PFN(addr) ((addr) >> SMMU_PAGE_SHIFT)
-#define SMMU_PFN_MASK   0x000FFFFF
+#define SMMU_ADDR_TO_PTN(addr) ((addr) >> SMMU_PTN_SHIFT)
-#define SMMU_ADDR_TO_PFN(addr) ((addr) >> 12)
+#define SMMU_ADDR_TO_PDN(addr) ((addr) >> SMMU_PDN_SHIFT)
-#define SMMU_ADDR_TO_PDN(addr) ((addr) >> 22)
+#define SMMU_PTN_TO_ADDR(ptn)  ((ptn)  << SMMU_PTN_SHIFT)
-#define SMMU_PDN_TO_ADDR(addr) ((pdn) << 22)
+#define SMMU_PDN_TO_ADDR(pdn)  ((pdn)  << SMMU_PDN_SHIFT)
-#define SMMU_MK_PDIR(page, attr) (((page) >> SMMU_PDIR_SHIFT) | (attr))
+#define SMMU_PTB(page, attr) (((attr) << 29u) | ((page) >> SMMU_PAGE_SHIFT))
-#define SMMU_MK_PDE(page, attr)  (((page) >> SMMU_PDE_SHIFT) | (attr))
 
-u8 *_pageheap = (u8 *)SMMU_HEAP_ADDR;
+static void *smmu_heap = (void *)SMMU_HEAP_ADDR;
 
-// Enabling SMMU requires a TZ secure write: MC(MC_SMMU_CONFIG) = 1;
+// Enabling SMMU requires a TZ (EL3) secure write. MC(MC_SMMU_CONFIG) = 1;
-u8 smmu_payload[] __attribute__((aligned(16))) = {
+static const u8 smmu_enable_payload[] = {
-	0xC1, 0x00, 0x00, 0x58, // 0x00: LDR  X1, =0x70019010
+	0xC1, 0x00, 0x00, 0x18, // 0x00: LDR  W1, =0x70019010
 	0x20, 0x00, 0x80, 0xD2, // 0x04: MOV  X0, #0x1
 	0x20, 0x00, 0x00, 0xB9, // 0x08: STR  W0, [X1]
 	0x1F, 0x71, 0x08, 0xD5, // 0x0C: IC   IALLUIS
@@ -56,17 +69,22 @@ u8 smmu_payload[] __attribute__((aligned(16))) = {
 
 void *smmu_page_zalloc(u32 num)
 {
-	u8 *res = _pageheap;
+	void *page = smmu_heap;
-	_pageheap += SZ_PAGE * num;
+	memset(page, 0, SZ_PAGE * num);
-	memset(res, 0, SZ_PAGE * num);
+
-	return res;
+	smmu_heap += SZ_PAGE * num;
+
+	return page;
 }
 
-static u32 *_smmu_pdir_alloc()
+static pde_t *_smmu_pdir_alloc()
 {
-	u32 *pdir = (u32 *)smmu_page_zalloc(1);
+	pde_t *pdir = (pde_t *)smmu_page_zalloc(1);
-	for (int pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
+
-		pdir[pdn] = _PDE_VACANT(pdn);
+	// Initialize pdes with no permissions.
+	for (u32 pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
+		pdir[pdn].huge.page = pdn;
+
 	return pdir;
 }
 
@@ -77,9 +95,12 @@ static void _smmu_flush_regs()
 
 void smmu_flush_all()
 {
+
+	// Flush the entire page table cache.
 	MC(MC_SMMU_PTC_FLUSH) = 0;
 	_smmu_flush_regs();
 
+	// Flush the entire table.
 	MC(MC_SMMU_TLB_FLUSH) = 0;
 	_smmu_flush_regs();
 }
@@ -88,8 +109,8 @@ void smmu_init()
 {
 	MC(MC_SMMU_PTB_ASID)   = 0;
 	MC(MC_SMMU_PTB_DATA)   = 0;
-	MC(MC_SMMU_TLB_CONFIG) = 0x30000030;
+	MC(MC_SMMU_TLB_CONFIG) = SMMU_TLB_HIT_UNDER_MISS | SMMU_TLB_RR_ARBITRATION | SMMU_TLB_ACTIVE_LINES(48);
-	MC(MC_SMMU_PTC_CONFIG) = 0x28000F3F;
+	MC(MC_SMMU_PTC_CONFIG) = SMUU_PTC_CACHE_ENABLE | SMUU_PTC_REQ_LIMIT(8) | SMUU_PTC_LINE_MASK(0xF) | SMUU_PTC_INDEX_MAP(0x3F);
 	MC(MC_SMMU_PTC_FLUSH)  = 0;
 	MC(MC_SMMU_TLB_FLUSH)  = 0;
 }
@@ -101,7 +122,8 @@ void smmu_enable()
 	if (enabled)
 		return;
 
-	ccplex_boot_cpu0((u32)smmu_payload, false);
+	// Launch payload on CCPLEX in order to set SMMU enable bit.
+	ccplex_boot_cpu0((u32)smmu_enable_payload, false);
 	msleep(100);
 	ccplex_powergate_cpu0();
 
@@ -110,63 +132,114 @@ void smmu_enable()
 	enabled = true;
 }
 
-u32 *smmu_init_domain4(u32 dev_base, u32 asid)
+void smmu_reset_heap()
 {
-	u32 *pdir = _smmu_pdir_alloc();
+	smmu_heap = (void *)SMMU_HEAP_ADDR;
-
-	MC(MC_SMMU_PTB_ASID) = asid;
-	MC(MC_SMMU_PTB_DATA) = SMMU_MK_PDIR((u32)pdir, _PDIR_ATTR);
-	_smmu_flush_regs();
-
-	MC(dev_base) = 0x80000000 | (asid << 24) | (asid << 16) | (asid << 8) | (asid);
-	_smmu_flush_regs();
-
-	return pdir;
 }
 
-u32 *smmu_get_pte(u32 *pdir, u32 iova)
+void *smmu_init_domain(u32 dev_base, u32 asid)
 {
-	u32 ptn = SMMU_ADDR_TO_PFN(iova);
+	void *ptb = _smmu_pdir_alloc();
-	u32 pdn = SMMU_ADDR_TO_PDN(iova);
-	u32 *ptbl;
 
-	if (pdir[pdn] != _PDE_VACANT(pdn))
+	MC(MC_SMMU_PTB_ASID) = asid;
-		ptbl = (u32 *)((pdir[pdn] & SMMU_PFN_MASK) << SMMU_PDIR_SHIFT);
+	MC(MC_SMMU_PTB_DATA) = SMMU_PTB((u32)ptb, SMMU_ATTR_ALL);
+	_smmu_flush_regs();
+
+	// Use the same macro for both quad and single domains. Reserved bits are not set anyway.
+	MC(dev_base) = SMMU_ENABLE | SMMU_ASID(asid);
+	_smmu_flush_regs();
+
+	return ptb;
+}
+
+void smmu_deinit_domain(u32 dev_base, u32 asid)
+{
+	MC(MC_SMMU_PTB_ASID) = asid;
+	MC(MC_SMMU_PTB_DATA) = 0;
+	MC(dev_base)         = 0;
+	_smmu_flush_regs();
+}
+
+void smmu_domain_bypass(u32 dev_base, bool bypass)
+{
+	if (bypass)
+	{
+		smmu_flush_all();
+		bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
+		MC(dev_base) &= ~SMMU_ENABLE;
+	}
 	else
 	{
-		ptbl = (u32 *)smmu_page_zalloc(1);
+		bpmp_mmu_maintenance(BPMP_MMU_MAINT_CLN_INV_WAY, false);
+		MC(dev_base) |=  SMMU_ENABLE;
+		smmu_flush_all();
+	}
+	_smmu_flush_regs();
+}
+
+static pte_t *_smmu_get_pte(pde_t *pdir, u32 iova)
+{
+	u32 pdn = SMMU_ADDR_TO_PDN(iova);
+	pte_t *ptbl;
+
+	// Get 4MB page table or initialize one.
+	if (pdir[pdn].tbl.attr)
+		ptbl = (pte_t *)(SMMU_PTN_TO_ADDR(pdir[pdn].tbl.table));
+	else
+	{
+		// Allocate page table.
+		ptbl = (pte_t *)smmu_page_zalloc(1);
+
+		// Get address.
 		u32 addr = SMMU_PDN_TO_ADDR(pdn);
-		for (int pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SMMU_PAGE_SIZE)
+
-			ptbl[pn] = _PTE_VACANT(addr);
+		// Initialize page table with no permissions.
-		pdir[pdn] = SMMU_MK_PDE((u32)ptbl, _PDE_ATTR | _PDE_NEXT);
+		for (u32 pn = 0; pn < SMMU_PTBL_COUNT; pn++, addr += SZ_PAGE)
+			ptbl[pn].page = SMMU_ADDR_TO_PFN(addr);
+
+		// Set page table to the page directory.
+		pdir[pdn].tbl.table = SMMU_ADDR_TO_PTN((u32)ptbl);
+		pdir[pdn].tbl.next  = SMMU_PAGE_TABLE;
+		pdir[pdn].tbl.attr  = SMMU_ATTR_ALL;
+
 		smmu_flush_all();
 	}
 
-	return &ptbl[ptn % SMMU_PTBL_COUNT];
+	return &ptbl[SMMU_ADDR_TO_PTN(iova) % SMMU_PTBL_COUNT];
 }
 
-void smmu_map(u32 *pdir, u32 addr, u32 page, int cnt, u32 attr)
+void smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr)
 {
-	for (int i = 0; i < cnt; i++)
+	// Map pages to page table entries. VA/PA should be aligned to 4KB.
+	for (u32 i = 0; i < pages; i++)
 	{
-		u32 *pte = smmu_get_pte(pdir, addr);
+		pte_t *pte = _smmu_get_pte((pde_t *)ptb, iova);
-		*pte = SMMU_ADDR_TO_PFN(page) | attr;
+
-		addr += SZ_PAGE;
+		pte->page = SMMU_ADDR_TO_PFN(iopa);
-		page += SZ_PAGE;
+		pte->attr = attr;
+
+		iova += SZ_PAGE;
+		iopa += SZ_PAGE;
 	}
+
 	smmu_flush_all();
 }
 
-u32 *smmu_init_domain(u32 asid)
+void smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr)
 {
-	return smmu_init_domain4(asid, 1);
+	pde_t *pdir = (pde_t *)ptb;
-}
 
-void smmu_deinit_domain(u32 asid)
+	// Map 4MB regions to page directory entries. VA/PA should be aligned to 4MB.
-{
+	for (u32 i = 0; i < regions; i++)
-	MC(MC_SMMU_PTB_ASID)  = 1;
+	{
-	MC(MC_SMMU_PTB_DATA)  = 0;
+		u32 pdn = SMMU_ADDR_TO_PDN(iova);
-	MC(asid)              = 0;
+		pdir[pdn].huge.page = SMMU_ADDR_TO_PDN(iopa);
-	_smmu_flush_regs();
+		pdir[pdn].huge.next = SMMU_4MB_REGION;
-}
+		pdir[pdn].huge.attr = attr;
 
+		iova += SZ_4M;
+		iopa += SZ_4M;
+	}
+
+	smmu_flush_all();
+}

bdk/mem/smmu.h

@@ -15,32 +15,57 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <assert.h>
+
 #include <utils/types.h>
 
 #define MC_SMMU_AVPC_ASID            0x23C
 #define MC_SMMU_TSEC_ASID            0x294
 
-#define SMMU_PDE_NEXT_SHIFT 28
+#define SMMU_NS    BIT(0)
-#define MC_SMMU_PTB_DATA_0_ASID_NONSECURE_SHIFT 29
+#define SMMU_WRITE BIT(1)
-#define MC_SMMU_PTB_DATA_0_ASID_WRITABLE_SHIFT  30
+#define SMMU_READ  BIT(2)
-#define MC_SMMU_PTB_DATA_0_ASID_READABLE_SHIFT  31
+#define SMMU_ATTR_ALL (SMMU_READ | SMMU_WRITE | SMMU_NS)
-#define _READABLE  (1 << MC_SMMU_PTB_DATA_0_ASID_READABLE_SHIFT)
+
-#define _WRITABLE  (1 << MC_SMMU_PTB_DATA_0_ASID_WRITABLE_SHIFT)
+typedef struct _pde_t {
-#define _NONSECURE (1 << MC_SMMU_PTB_DATA_0_ASID_NONSECURE_SHIFT)
+	union {
-#define _PDE_NEXT  (1 << SMMU_PDE_NEXT_SHIFT)
+		union {
-#define _MASK_ATTR (_READABLE | _WRITABLE | _NONSECURE)
+			struct {
-#define _PDIR_ATTR (_READABLE | _WRITABLE | _NONSECURE)
+				u32 table:22;
-#define _PDE_ATTR  (_READABLE | _WRITABLE | _NONSECURE)
+				u32 rsvd:6;
-#define _PDE_VACANT(pdn) (((pdn) << 10) | _PDE_ATTR)
+				u32 next:1;
-#define _PTE_ATTR  (_READABLE | _WRITABLE | _NONSECURE)
+				u32 attr:3;
-#define _PTE_VACANT(addr) (((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR)
+			} tbl;
+
+			struct {
+				u32 rsvd_:10;
+				u32 page:12;
+				u32 rsvd:6;
+				u32 next:1;
+				u32 attr:3;
+			} huge;
+		};
+
+		u32 pde;
+	};
+} pde_t;
+
+typedef struct _pte_t {
+	u32 page:22;
+	u32 rsvd:7;
+	u32 attr:3;
+} pte_t;
+
+static_assert(sizeof(pde_t) == sizeof(u32), "pde_t size is wrong!");
+static_assert(sizeof(pte_t) == sizeof(u32), "pte_t size is wrong!");
 
 void *smmu_page_zalloc(u32 num);
 void  smmu_flush_all();
 void  smmu_init();
 void  smmu_enable();
-u32  *smmu_init_domain4(u32 dev_base, u32 asid);
+void  smmu_reset_heap();
-u32  *smmu_get_pte(u32 *pdir, u32 iova);
+void *smmu_init_domain(u32 dev_base, u32 asid);
-void  smmu_map(u32 *pdir, u32 addr, u32 page, int cnt, u32 attr);
+void  smmu_deinit_domain(u32 dev_base, u32 asid);
-u32  *smmu_init_domain(u32 asid);
+void  smmu_domain_bypass(u32 dev_base, bool bypass);
-void  smmu_deinit_domain(u32 asid);
+void  smmu_map(void *ptb, u32 iova, u64 iopa, u32 pages, u32 attr);
+void  smmu_map_huge(void *ptb, u32 iova, u64 iopa, u32 regions, u32 attr);

bdk/sec/tsec.c

@@ -70,8 +70,9 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
 	int res = 0;
 	u8 *fwbuf = NULL;
 	u32 type = tsec_ctxt->type;
-	u32 *pdir, *car, *fuse, *pmc, *flowctrl, *se, *mc, *iram, *evec;
+	u32 *car, *fuse, *pmc, *flowctrl, *se, *mc, *iram, *evec;
 	u32 *pkg11_magic_off;
+	void *ptb;
 
 	bpmp_mmu_disable();
 	bpmp_freq_t prev_fid = bpmp_clk_rate_set(BPMP_CLK_NORMAL);
@@ -145,7 +146,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
 	if (type == TSEC_FW_TYPE_EMU)
 	{
 		// Init SMMU translation for TSEC.
-		pdir = smmu_init_domain(MC_SMMU_TSEC_ASID);
+		ptb = smmu_init_domain(MC_SMMU_TSEC_ASID, 1);
 		smmu_init();
 
 		// Enable SMMU.
@@ -155,7 +156,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
 		car = smmu_page_zalloc(1);
 		memcpy(car, (void *)CLOCK_BASE, SZ_PAGE);
 		car[CLK_RST_CONTROLLER_CLK_SOURCE_TSEC / 4] = CLK_SRC_DIV(2);
-		smmu_map(pdir, CLOCK_BASE, (u32)car, 1, _WRITABLE | _READABLE | _NONSECURE);
+		smmu_map(ptb, CLOCK_BASE, (u32)car, 1, SMMU_WRITE | SMMU_READ | SMMU_NS);
 
 		// Fuse driver.
 		fuse = smmu_page_zalloc(1);
@@ -163,38 +164,38 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
 		fuse[0x82C / 4] = 0;
 		fuse[0x9E0 / 4] = (1 << (TSEC_HOS_KB_620 + 2)) - 1;
 		fuse[0x9E4 / 4] = (1 << (TSEC_HOS_KB_620 + 2)) - 1;
-		smmu_map(pdir, (FUSE_BASE - 0x800), (u32)fuse, 1, _READABLE | _NONSECURE);
+		smmu_map(ptb, (FUSE_BASE - 0x800), (u32)fuse, 1, SMMU_READ | SMMU_NS);
 
 		// Power management controller.
 		pmc = smmu_page_zalloc(1);
-		smmu_map(pdir, RTC_BASE, (u32)pmc, 1, _READABLE | _NONSECURE);
+		smmu_map(ptb, RTC_BASE, (u32)pmc, 1, SMMU_READ | SMMU_NS);
 
 		// Flow control.
 		flowctrl = smmu_page_zalloc(1);
-		smmu_map(pdir, FLOW_CTLR_BASE, (u32)flowctrl, 1, _WRITABLE | _NONSECURE);
+		smmu_map(ptb, FLOW_CTLR_BASE, (u32)flowctrl, 1, SMMU_WRITE | SMMU_NS);
 
 		// Security engine.
 		se = smmu_page_zalloc(1);
 		memcpy(se, (void *)SE_BASE, SZ_PAGE);
-		smmu_map(pdir, SE_BASE, (u32)se, 1, _READABLE | _WRITABLE | _NONSECURE);
+		smmu_map(ptb, SE_BASE, (u32)se, 1, SMMU_READ | SMMU_WRITE | SMMU_NS);
 
 		// Memory controller.
 		mc = smmu_page_zalloc(1);
 		memcpy(mc, (void *)MC_BASE, SZ_PAGE);
 		mc[MC_IRAM_BOM / 4] = 0;
 		mc[MC_IRAM_TOM / 4] = DRAM_START;
-		smmu_map(pdir, MC_BASE, (u32)mc, 1, _READABLE | _NONSECURE);
+		smmu_map(ptb, MC_BASE, (u32)mc, 1, SMMU_READ | SMMU_NS);
 
 		// IRAM
 		iram = smmu_page_zalloc(0x30);
 		memcpy(iram, tsec_ctxt->pkg1, 0x30000);
 		// PKG1.1 magic offset.
-		pkg11_magic_off = (u32 *)(iram + ((tsec_ctxt->pkg11_off + 0x20) / 4));
+		pkg11_magic_off = (u32 *)(iram + ((tsec_ctxt->pkg11_off + 0x20) / sizeof(u32)));
-		smmu_map(pdir, 0x40010000, (u32)iram, 0x30, _READABLE | _WRITABLE | _NONSECURE);
+		smmu_map(ptb, 0x40010000, (u32)iram, 0x30, SMMU_READ | SMMU_WRITE | SMMU_NS);
 
 		// Exception vectors
 		evec = smmu_page_zalloc(1);
-		smmu_map(pdir, EXCP_VEC_BASE, (u32)evec, 1, _READABLE | _WRITABLE | _NONSECURE);
+		smmu_map(ptb, EXCP_VEC_BASE, (u32)evec, 1, SMMU_READ | SMMU_WRITE | SMMU_NS);
 	}
 
 	// Execute firmware.
@@ -229,7 +230,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
 		if (kidx != 8)
 		{
 			res = -6;
-			smmu_deinit_domain(MC_SMMU_TSEC_ASID);
+			smmu_deinit_domain(MC_SMMU_TSEC_ASID, 1);
 
 			goto out_free;
 		}
@@ -240,7 +241,7 @@ int tsec_query(void *tsec_keys, tsec_ctxt_t *tsec_ctxt)
 		memcpy(tsec_keys, &key, 0x20);
 		memcpy(tsec_ctxt->pkg1, iram, 0x30000);
 
-		smmu_deinit_domain(MC_SMMU_TSEC_ASID);
+		smmu_deinit_domain(MC_SMMU_TSEC_ASID, 1);
 
 		// for (int i = 0; i < kidx; i++)
 		// 	gfx_printf("key %08X\n", key[i]);