From 27de1978c331d4e7571a5d0187ef1802d65dbf65 Mon Sep 17 00:00:00 2001 From: Swathi Sridhar Date: Fri, 1 May 2020 15:40:20 -0700 Subject: [PATCH] ANDROID: GKI: iommu/io-pgtable-arm: LPAE related updates by vendor This is a snapshot of iommu/io-pgtable-arm as of commit 79efc458af96. Most of the changes are related to LPAE. The Signed-off-bys of all the commits has been copied over. Signed-off-by: Sudarshan Rajagopalan Signed-off-by: Prakash Gupta Signed-off-by: Vinayak Menon Signed-off-by: Swathi Sridhar Bug: 155522481 Signed-off-by: Mark Salyzyn Signed-off-by: Saravana Kannan Change-Id: I375f3c4d43c23818ae2e2c1963cda474328e677f --- drivers/iommu/io-pgtable-arm.c | 797 +++++++++++++++++++++++++++------ 1 file changed, 652 insertions(+), 145 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index d04aa36be377..7e6b3459aeb1 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -21,10 +21,10 @@ #define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt #include -#include #include #include #include +#include #include #include #include @@ -32,7 +32,7 @@ #include -#define ARM_LPAE_MAX_ADDR_BITS 52 +#define ARM_LPAE_MAX_ADDR_BITS 48 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 @@ -69,6 +69,9 @@ #define ARM_LPAE_PGD_IDX(l,d) \ ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) +#define ARM_LPAE_LVL_MASK(l, d) \ + ((l) == ARM_LPAE_START_LVL(d) ? (1 << (d)->pgd_bits) - 1 : \ + (1 << (d)->bits_per_level) - 1) #define ARM_LPAE_LVL_IDX(a,l,d) \ (((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) @@ -86,8 +89,7 @@ #define ARM_LPAE_PTE_TYPE_TABLE 3 #define ARM_LPAE_PTE_TYPE_PAGE 3 -#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12) - +#define ARM_LPAE_PTE_SH_MASK (((arm_lpae_iopte)0x3) << 8) #define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) @@ -106,8 +108,11 @@ #define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55) /* Stage-1 PTE */ +#define ARM_LPAE_PTE_AP_PRIV_RW (((arm_lpae_iopte)0) << 6) #define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) -#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_AP_PRIV_RO (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_AP_RO (((arm_lpae_iopte)3) << 6) +#define ARM_LPAE_PTE_ATTRINDX_MASK 0x7 #define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 #define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) @@ -161,19 +166,25 @@ #define ARM_LPAE_TCR_PS_42_BIT 0x3ULL #define ARM_LPAE_TCR_PS_44_BIT 0x4ULL #define ARM_LPAE_TCR_PS_48_BIT 0x5ULL -#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL #define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define ARM_LPAE_MAIR1_ATTR_SHIFT(n) ((n-4) << 3) #define ARM_LPAE_MAIR_ATTR_MASK 0xff #define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 #define ARM_LPAE_MAIR_ATTR_NC 0x44 #define ARM_LPAE_MAIR_ATTR_WBRWA 0xff +#define ARM_LPAE_MAIR_ATTR_UPSTREAM 0xf4 +#define ARM_LPAE_MAIR_ATTR_LLC_NWA 0xe4 #define ARM_LPAE_MAIR_ATTR_IDX_NC 0 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 #define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 +#define ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM 3 +#define ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA 0x4ULL /* IOPTE accessors */ -#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d)) +#define iopte_deref(pte, d) \ + (__va(iopte_val(pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ + & ~(ARM_LPAE_GRANULE(d) - 1ULL))) #define iopte_type(pte,l) \ (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) @@ -185,38 +196,95 @@ (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \ (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK)) +#define iopte_to_pfn(pte, d) \ + (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift) + +#define pfn_to_iopte(pfn, d) \ + (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) + struct arm_lpae_io_pgtable { struct io_pgtable iop; int levels; + unsigned int pgd_bits; size_t pgd_size; unsigned long pg_shift; unsigned long bits_per_level; void *pgd; + void *pgd_ttbr1; }; typedef u64 arm_lpae_iopte; -static arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr, - struct arm_lpae_io_pgtable *data) -{ - arm_lpae_iopte pte = paddr; +/* + * We'll use some ignored bits in table entries to keep track of the number + * of page mappings beneath the table. The maximum number of entries + * beneath any table mapping in armv8 is 8192 (which is possible at the + * 2nd- and 3rd-level when using a 64K granule size). The bits at our + * disposal are: + * + * 4k granule: [54..52], [11..2] + * 64k granule: [54..52], [15..2] + * + * [54..52], [11..2] is enough bits for tracking table mappings at any + * level for any granule, so we'll use those. + */ +#define BOTTOM_IGNORED_MASK 0x3ff +#define BOTTOM_IGNORED_SHIFT 2 +#define BOTTOM_IGNORED_NUM_BITS 10 +#define TOP_IGNORED_MASK 0x7ULL +#define TOP_IGNORED_SHIFT 52 +#define IOPTE_RESERVED_MASK ((BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT) | \ + (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT)) - /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ - return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK; +static arm_lpae_iopte iopte_val(arm_lpae_iopte table_pte) +{ + return table_pte & ~IOPTE_RESERVED_MASK; } -static phys_addr_t iopte_to_paddr(arm_lpae_iopte pte, - struct arm_lpae_io_pgtable *data) +static arm_lpae_iopte _iopte_bottom_ignored_val(arm_lpae_iopte table_pte) { - u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK; + return (table_pte & (BOTTOM_IGNORED_MASK << BOTTOM_IGNORED_SHIFT)) + >> BOTTOM_IGNORED_SHIFT; +} - if (data->pg_shift < 16) - return paddr; +static arm_lpae_iopte _iopte_top_ignored_val(arm_lpae_iopte table_pte) +{ + return (table_pte & (TOP_IGNORED_MASK << TOP_IGNORED_SHIFT)) + >> TOP_IGNORED_SHIFT; +} - /* Rotate the packed high-order bits back to the top */ - return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4); +static int iopte_tblcnt(arm_lpae_iopte table_pte) +{ + return (_iopte_bottom_ignored_val(table_pte) | + (_iopte_top_ignored_val(table_pte) << BOTTOM_IGNORED_NUM_BITS)); +} + +static void iopte_tblcnt_set(arm_lpae_iopte *table_pte, int val) +{ + arm_lpae_iopte pte = iopte_val(*table_pte); + + pte |= ((val & BOTTOM_IGNORED_MASK) << BOTTOM_IGNORED_SHIFT) | + (((val & (TOP_IGNORED_MASK << BOTTOM_IGNORED_NUM_BITS)) + >> BOTTOM_IGNORED_NUM_BITS) << TOP_IGNORED_SHIFT); + *table_pte = pte; +} + +static void iopte_tblcnt_sub(arm_lpae_iopte *table_ptep, int cnt) +{ + arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep); + + current_cnt -= cnt; + iopte_tblcnt_set(table_ptep, current_cnt); +} + +static void iopte_tblcnt_add(arm_lpae_iopte *table_ptep, int cnt) +{ + arm_lpae_iopte current_cnt = iopte_tblcnt(*table_ptep); + + current_cnt += cnt; + iopte_tblcnt_set(table_ptep, current_cnt); } static bool selftest_running = false; @@ -226,22 +294,27 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages) return (dma_addr_t)virt_to_phys(pages); } +static inline void pgtable_dma_sync_single_for_device( + struct io_pgtable_cfg *cfg, + dma_addr_t addr, size_t size, + enum dma_data_direction dir) +{ + if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) + dma_sync_single_for_device(cfg->iommu_dev, addr, size, + dir); +} + static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, - struct io_pgtable_cfg *cfg) + struct io_pgtable_cfg *cfg, void *cookie) { struct device *dev = cfg->iommu_dev; - int order = get_order(size); - struct page *p; dma_addr_t dma; - void *pages; + void *pages = io_pgtable_alloc_pages_exact(cfg, cookie, size, + gfp | __GFP_ZERO); - VM_BUG_ON((gfp & __GFP_HIGHMEM)); - p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE, - gfp | __GFP_ZERO, order); - if (!p) + if (!pages) return NULL; - pages = page_address(p); if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) { dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) @@ -261,23 +334,23 @@ out_unmap: dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); out_free: - __free_pages(p, order); + io_pgtable_free_pages_exact(cfg, cookie, pages, size); return NULL; } static void __arm_lpae_free_pages(void *pages, size_t size, - struct io_pgtable_cfg *cfg) + struct io_pgtable_cfg *cfg, void *cookie) { if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA)) dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), size, DMA_TO_DEVICE); - free_pages((unsigned long)pages, get_order(size)); + io_pgtable_free_pages_exact(cfg, cookie, pages, size); } static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, struct io_pgtable_cfg *cfg) { - dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep), + pgtable_dma_sync_single_for_device(cfg, __arm_lpae_dma_addr(ptep), sizeof(*ptep), DMA_TO_DEVICE); } @@ -294,9 +367,11 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, unsigned long iova, size_t size, int lvl, arm_lpae_iopte *ptep); + static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, phys_addr_t paddr, arm_lpae_iopte prot, - int lvl, arm_lpae_iopte *ptep) + int lvl, arm_lpae_iopte *ptep, + bool flush) { arm_lpae_iopte pte = prot; @@ -308,50 +383,48 @@ static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, else pte |= ARM_LPAE_PTE_TYPE_BLOCK; - pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; - pte |= paddr_to_iopte(paddr, data); + pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_OS; + pte |= pfn_to_iopte(paddr >> data->pg_shift, data); - __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); + if (flush) + __arm_lpae_set_pte(ptep, pte, &data->iop.cfg); + else + *ptep = pte; } static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, - arm_lpae_iopte *ptep) + arm_lpae_iopte *ptep, arm_lpae_iopte *prev_ptep, + bool flush) { arm_lpae_iopte pte = *ptep; - if (iopte_leaf(pte, lvl)) { - /* We require an unmap first */ - WARN_ON(!selftest_running); + /* We require an unmap first */ + if (pte & ARM_LPAE_PTE_VALID) { + WARN_RATELIMIT(1, "map without unmap\n"); return -EEXIST; - } else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { - /* - * We need to unmap and free the old table before - * overwriting it with a block entry. - */ - arm_lpae_iopte *tblp; - size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); - - tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); - if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) - return -EINVAL; } - __arm_lpae_init_pte(data, paddr, prot, lvl, ptep); + __arm_lpae_init_pte(data, paddr, prot, lvl, ptep, flush); + + if (prev_ptep) + iopte_tblcnt_add(prev_ptep, 1); return 0; } static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, arm_lpae_iopte *ptep, arm_lpae_iopte curr, - struct io_pgtable_cfg *cfg) + struct io_pgtable_cfg *cfg, + int ref_count) { arm_lpae_iopte old, new; new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) new |= ARM_LPAE_PTE_NSTABLE; + iopte_tblcnt_set(&new, ref_count); /* * Ensure the table itself is visible before its PTE can be. @@ -374,21 +447,69 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, return old; } +struct map_state { + unsigned long iova_end; + unsigned int pgsize; + arm_lpae_iopte *pgtable; + arm_lpae_iopte *prev_pgtable; + arm_lpae_iopte *pte_start; + unsigned int num_pte; +}; +/* map state optimization works at level 3 (the 2nd-to-last level) */ +#define MAP_STATE_LVL 3 + static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, size_t size, arm_lpae_iopte prot, - int lvl, arm_lpae_iopte *ptep) + int lvl, arm_lpae_iopte *ptep, + arm_lpae_iopte *prev_ptep, struct map_state *ms) { arm_lpae_iopte *cptep, pte; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); size_t tblsz = ARM_LPAE_GRANULE(data); struct io_pgtable_cfg *cfg = &data->iop.cfg; + void *cookie = data->iop.cookie; + arm_lpae_iopte *pgtable = ptep; /* Find our entry at the current level */ ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); /* If we can install a leaf entry at this level, then do so */ - if (size == block_size && (size & cfg->pgsize_bitmap)) - return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); + if (size == block_size && (size & cfg->pgsize_bitmap)) { + if (!ms) + return arm_lpae_init_pte(data, iova, paddr, prot, lvl, + ptep, prev_ptep, true); + + if (lvl == MAP_STATE_LVL) { + if (ms->pgtable) + pgtable_dma_sync_single_for_device(cfg, + __arm_lpae_dma_addr(ms->pte_start), + ms->num_pte * sizeof(*ptep), + DMA_TO_DEVICE); + + ms->iova_end = round_down(iova, SZ_2M) + SZ_2M; + ms->pgtable = pgtable; + ms->prev_pgtable = prev_ptep; + ms->pgsize = size; + ms->pte_start = ptep; + ms->num_pte = 1; + } else { + /* + * We have some map state from previous page + * mappings, but we're about to set up a block + * mapping. Flush out the previous page mappings. + */ + if (ms->pgtable) + pgtable_dma_sync_single_for_device(cfg, + __arm_lpae_dma_addr(ms->pte_start), + ms->num_pte * sizeof(*ptep), + DMA_TO_DEVICE); + memset(ms, 0, sizeof(*ms)); + ms = NULL; + } + + return arm_lpae_init_pte(data, iova, paddr, prot, lvl, + ptep, prev_ptep, ms == NULL); + } /* We can't allocate tables at the final level */ if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) @@ -397,13 +518,14 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = READ_ONCE(*ptep); if (!pte) { - cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg); + cptep = __arm_lpae_alloc_pages(tblsz, GFP_ATOMIC, cfg, cookie); if (!cptep) return -ENOMEM; - pte = arm_lpae_install_table(cptep, ptep, 0, cfg); + pte = arm_lpae_install_table(cptep, ptep, 0, cfg, 0); if (pte) - __arm_lpae_free_pages(cptep, tblsz, cfg); + __arm_lpae_free_pages(cptep, tblsz, cfg, cookie); + } else if (!(cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) && !(pte & ARM_LPAE_PTE_SW_SYNC)) { __arm_lpae_sync_pte(ptep, cfg); @@ -418,7 +540,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, } /* Rinse, repeat */ - return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); + return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep, + ptep, ms); } static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, @@ -430,8 +553,12 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, data->iop.fmt == ARM_32_LPAE_S1) { pte = ARM_LPAE_PTE_nG; - if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) - pte |= ARM_LPAE_PTE_AP_RDONLY; + if (prot & IOMMU_WRITE) + pte |= (prot & IOMMU_PRIV) ? ARM_LPAE_PTE_AP_PRIV_RW + : ARM_LPAE_PTE_AP_UNPRIV; + else + pte |= (prot & IOMMU_PRIV) ? ARM_LPAE_PTE_AP_PRIV_RO + : ARM_LPAE_PTE_AP_RO; if (!(prot & IOMMU_PRIV)) pte |= ARM_LPAE_PTE_AP_UNPRIV; @@ -442,6 +569,12 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, else if (prot & IOMMU_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_USE_UPSTREAM_HINT) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_USE_LLC_NWA) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA + << ARM_LPAE_PTE_ATTRINDX_SHIFT); } else { pte = ARM_LPAE_PTE_HAP_FAULT; if (prot & IOMMU_READ) @@ -479,7 +612,8 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return -ERANGE; prot = arm_lpae_prot_to_pte(data, iommu_prot); - ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); + ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, NULL, + NULL); /* * Synchronise all PTE updates for the new mapping before there's * a chance for anything to kick off a table walk for the new iova. @@ -489,11 +623,97 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return ret; } +static int arm_lpae_map_sg(struct io_pgtable_ops *ops, unsigned long iova, + struct scatterlist *sg, unsigned int nents, + int iommu_prot, size_t *size) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte *ptep = data->pgd; + int lvl = ARM_LPAE_START_LVL(data); + arm_lpae_iopte prot; + struct scatterlist *s; + size_t mapped = 0; + int i; + int ret = -EINVAL; + unsigned int min_pagesz; + struct io_pgtable_cfg *cfg = &data->iop.cfg; + struct map_state ms; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + goto out_err; + + prot = arm_lpae_prot_to_pte(data, iommu_prot); + + min_pagesz = 1 << __ffs(cfg->pgsize_bitmap); + + memset(&ms, 0, sizeof(ms)); + + for_each_sg(sg, s, nents, i) { + phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; + size_t size = s->length; + + /* + * We are mapping on IOMMU page boundaries, so offset within + * the page must be 0. However, the IOMMU may support pages + * smaller than PAGE_SIZE, so s->offset may still represent + * an offset of that boundary within the CPU page. + */ + if (!IS_ALIGNED(s->offset, min_pagesz)) + goto out_err; + + while (size) { + size_t pgsize = iommu_pgsize( + cfg->pgsize_bitmap, iova | phys, size); + + if (ms.pgtable && (iova < ms.iova_end)) { + arm_lpae_iopte *ptep = ms.pgtable + + ARM_LPAE_LVL_IDX(iova, MAP_STATE_LVL, + data); + arm_lpae_init_pte( + data, iova, phys, prot, MAP_STATE_LVL, + ptep, ms.prev_pgtable, false); + ms.num_pte++; + } else { + ret = __arm_lpae_map(data, iova, phys, pgsize, + prot, lvl, ptep, NULL, &ms); + if (ret) + goto out_err; + } + + iova += pgsize; + mapped += pgsize; + phys += pgsize; + size -= pgsize; + } + } + + if (ms.pgtable) + pgtable_dma_sync_single_for_device(cfg, + __arm_lpae_dma_addr(ms.pte_start), + ms.num_pte * sizeof(*ms.pte_start), + DMA_TO_DEVICE); + + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return mapped; + +out_err: + /* Return the size of the partial mapping so that they can be undone */ + *size = mapped; + return ret; +} + static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte *start, *end; unsigned long table_size; + void *cookie = data->iop.cookie; if (lvl == ARM_LPAE_START_LVL(data)) table_size = data->pgd_size; @@ -517,7 +737,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } - __arm_lpae_free_pages(start, table_size, &data->iop.cfg); + __arm_lpae_free_pages(start, table_size, &data->iop.cfg, cookie); } static void arm_lpae_free_pgtable(struct io_pgtable *iop) @@ -525,6 +745,8 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop) struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); + __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), + data->pgd_ttbr1); kfree(data); } @@ -539,18 +761,22 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, size_t tablesz = ARM_LPAE_GRANULE(data); size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data); int i, unmap_idx = -1; + void *cookie = data->iop.cookie; + int child_cnt = 0; + + size = iommu_pgsize(data->iop.cfg.pgsize_bitmap, iova, size); if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) return 0; - tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); + tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg, cookie); if (!tablep) return 0; /* Bytes unmapped */ if (size == split_sz) unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data); - blk_paddr = iopte_to_paddr(blk_pte, data); + blk_paddr = iopte_to_pfn(blk_pte, data) << data->pg_shift; pte = iopte_prot(blk_pte); for (i = 0; i < tablesz / sizeof(pte); i++, blk_paddr += split_sz) { @@ -558,12 +784,14 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, if (i == unmap_idx) continue; - __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]); + __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i], + true); + child_cnt++; } - pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); + pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg, child_cnt); if (pte != blk_pte) { - __arm_lpae_free_pages(tablep, tablesz, cfg); + __arm_lpae_free_pages(tablep, tablesz, cfg, cookie); /* * We may race against someone unmapping another part of this * block, but anything else is invalid. We can't misinterpret @@ -603,16 +831,44 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ - io_pgtable_tlb_add_flush(iop, iova, size, - ARM_LPAE_GRANULE(data), false); - io_pgtable_tlb_sync(iop); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); - } else { - io_pgtable_tlb_add_flush(iop, iova, size, size, true); } return size; + } else if ((lvl == ARM_LPAE_MAX_LEVELS - 2) && !iopte_leaf(pte, lvl)) { + arm_lpae_iopte *table = iopte_deref(pte, data); + arm_lpae_iopte *table_base = table; + int tl_offset = ARM_LPAE_LVL_IDX(iova, lvl + 1, data); + int entry_size = ARM_LPAE_GRANULE(data); + int max_entries = ARM_LPAE_BLOCK_SIZE(lvl, data) >> + data->pg_shift; + int entries = min_t(int, size / entry_size, + max_entries - tl_offset); + int table_len = entries * sizeof(*table); + + /* + * This isn't a block mapping so it must be a table mapping + * and since it's the 2nd-to-last level the next level has + * to be all page mappings. Zero them all out in one fell + * swoop. + */ + + table += tl_offset; + + memset(table, 0, table_len); + pgtable_dma_sync_single_for_device(&iop->cfg, + __arm_lpae_dma_addr(table), + table_len, DMA_TO_DEVICE); + + iopte_tblcnt_sub(ptep, entries); + if (!iopte_tblcnt(*ptep)) { + /* no valid mappings left under this table. free it. */ + __arm_lpae_set_pte(ptep, 0, &iop->cfg); + __arm_lpae_free_pgtable(data, lvl + 1, table_base); + } + + return entries * entry_size; } else if (iopte_leaf(pte, lvl)) { /* * Insert a table at the next level to map the old region, @@ -628,8 +884,9 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, } static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, - size_t size) + size_t size) { + size_t unmapped = 0; struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); @@ -637,49 +894,133 @@ static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias))) return 0; - return __arm_lpae_unmap(data, iova, size, lvl, ptep); + while (unmapped < size) { + size_t ret, size_to_unmap, remaining; + + remaining = (size - unmapped); + size_to_unmap = iommu_pgsize(data->iop.cfg.pgsize_bitmap, iova, + remaining); + size_to_unmap = size_to_unmap >= SZ_2M ? + size_to_unmap : + min_t(unsigned long, remaining, + (ALIGN(iova + 1, SZ_2M) - iova)); + ret = __arm_lpae_unmap(data, iova, size_to_unmap, lvl, ptep); + if (ret == 0) + break; + unmapped += ret; + iova += ret; + } + + if (unmapped) + io_pgtable_tlb_flush_all(&data->iop); + + return unmapped; +} + +static int arm_lpae_iova_to_pte(struct arm_lpae_io_pgtable *data, + unsigned long iova, int *plvl_ret, + arm_lpae_iopte *ptep_ret) +{ + arm_lpae_iopte pte, *ptep = data->pgd; + *plvl_ret = ARM_LPAE_START_LVL(data); + *ptep_ret = 0; + + do { + /* Valid IOPTE pointer? */ + if (!ptep) + return -EINVAL; + + /* Grab the IOPTE we're interested in */ + pte = *(ptep + ARM_LPAE_LVL_IDX(iova, *plvl_ret, data)); + + /* Valid entry? */ + if (!pte) + return -EINVAL; + + /* Leaf entry? */ + if (iopte_leaf(pte, *plvl_ret)) + goto found_translation; + + /* Take it to the next level */ + ptep = iopte_deref(pte, data); + } while (++(*plvl_ret) < ARM_LPAE_MAX_LEVELS); + + /* Ran out of page tables to walk */ + return -EINVAL; + +found_translation: + *ptep_ret = pte; + return 0; +} + +static uint64_t arm_lpae_iova_get_pte(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte pte; + int lvl; + + if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) + return pte; + + return 0; } static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); - arm_lpae_iopte pte, *ptep = data->pgd; - int lvl = ARM_LPAE_START_LVL(data); + arm_lpae_iopte pte; + int lvl; + phys_addr_t phys = 0; - do { - /* Valid IOPTE pointer? */ - if (!ptep) - return 0; + if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) { + iova &= ((1 << ARM_LPAE_LVL_SHIFT(lvl, data)) - 1); + phys = ((phys_addr_t)iopte_to_pfn(pte, data) + << data->pg_shift) | iova; + } - /* Grab the IOPTE we're interested in */ - ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); - pte = READ_ONCE(*ptep); + return phys; +} - /* Valid entry? */ - if (!pte) - return 0; +static bool __arm_lpae_is_iova_coherent(struct arm_lpae_io_pgtable *data, + arm_lpae_iopte *ptep) +{ + if (data->iop.fmt == ARM_64_LPAE_S1 || + data->iop.fmt == ARM_32_LPAE_S1) { + int attr_idx = (*ptep & (ARM_LPAE_PTE_ATTRINDX_MASK << + ARM_LPAE_PTE_ATTRINDX_SHIFT)) >> + ARM_LPAE_PTE_ATTRINDX_SHIFT; + if ((attr_idx == ARM_LPAE_MAIR_ATTR_IDX_CACHE) && + (((*ptep & ARM_LPAE_PTE_SH_MASK) == ARM_LPAE_PTE_SH_IS) + || + (*ptep & ARM_LPAE_PTE_SH_MASK) == ARM_LPAE_PTE_SH_OS)) + return true; + } else { + if (*ptep & ARM_LPAE_PTE_MEMATTR_OIWB) + return true; + } - /* Leaf entry? */ - if (iopte_leaf(pte,lvl)) - goto found_translation; + return false; +} - /* Take it to the next level */ - ptep = iopte_deref(pte, data); - } while (++lvl < ARM_LPAE_MAX_LEVELS); +static bool arm_lpae_is_iova_coherent(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte pte; + int lvl; + bool ret = false; - /* Ran out of page tables to walk */ - return 0; + if (!arm_lpae_iova_to_pte(data, iova, &lvl, &pte)) + ret = __arm_lpae_is_iova_coherent(data, &pte); -found_translation: - iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); - return iopte_to_paddr(pte, data) | iova; + return ret; } static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) { - unsigned long granule, page_sizes; - unsigned int max_addr_bits = 48; + unsigned long granule; /* * We need to restrict the supported page sizes to match the @@ -699,24 +1040,17 @@ static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) switch (granule) { case SZ_4K: - page_sizes = (SZ_4K | SZ_2M | SZ_1G); + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); break; case SZ_16K: - page_sizes = (SZ_16K | SZ_32M); + cfg->pgsize_bitmap &= (SZ_16K | SZ_32M); break; case SZ_64K: - max_addr_bits = 52; - page_sizes = (SZ_64K | SZ_512M); - if (cfg->oas > 48) - page_sizes |= 1ULL << 42; /* 4TB */ + cfg->pgsize_bitmap &= (SZ_64K | SZ_512M); break; default: - page_sizes = 0; + cfg->pgsize_bitmap = 0; } - - cfg->pgsize_bitmap &= page_sizes; - cfg->ias = min(cfg->ias, max_addr_bits); - cfg->oas = min(cfg->oas, max_addr_bits); } static struct arm_lpae_io_pgtable * @@ -753,12 +1087,16 @@ arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) /* Calculate the actual size of our pgd (without concatenation) */ pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); + data->pgd_bits = pgd_bits; data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); data->iop.ops = (struct io_pgtable_ops) { .map = arm_lpae_map, + .map_sg = arm_lpae_map_sg, .unmap = arm_lpae_unmap, .iova_to_phys = arm_lpae_iova_to_phys, + .is_iova_coherent = arm_lpae_is_iova_coherent, + .iova_to_pte = arm_lpae_iova_get_pte, }; return data; @@ -770,7 +1108,11 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) u64 reg; struct arm_lpae_io_pgtable *data; - if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA)) + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS + | IO_PGTABLE_QUIRK_NO_DMA + | IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT + | IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE + | IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -778,9 +1120,32 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) return NULL; /* TCR */ - reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | - (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + if (cfg->quirks & IO_PGTABLE_QUIRK_NO_DMA) + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + else if ((cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) && + (cfg->quirks & IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE)) + reg = (ARM_LPAE_TCR_SH_NS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + else if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + else if ((cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA) && + (cfg->quirks & IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE)) + reg = (ARM_LPAE_TCR_SH_NS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WB << ARM_LPAE_TCR_ORGN0_SHIFT); + else if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA) + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WB << ARM_LPAE_TCR_ORGN0_SHIFT); + else + reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_ORGN0_SHIFT); switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: @@ -813,9 +1178,6 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); break; - case 52: - reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_IPS_SHIFT); - break; default: goto out_free_data; } @@ -832,24 +1194,39 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) (ARM_LPAE_MAIR_ATTR_WBRWA << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | (ARM_LPAE_MAIR_ATTR_DEVICE - << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)) | + (ARM_LPAE_MAIR_ATTR_UPSTREAM + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM)); cfg->arm_lpae_s1_cfg.mair[0] = reg; - cfg->arm_lpae_s1_cfg.mair[1] = 0; + + reg = ARM_LPAE_MAIR_ATTR_LLC_NWA + << ARM_LPAE_MAIR1_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA); + + cfg->arm_lpae_s1_cfg.mair[1] = reg; /* Looking good; allocate a pgd */ - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, + cfg, cookie); if (!data->pgd) goto out_free_data; + data->pgd_ttbr1 = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, + cfg, cookie); + if (!data->pgd_ttbr1) + goto out_free_pgd; + /* Ensure the empty pgd is visible before any actual TTBR write */ wmb(); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); - cfg->arm_lpae_s1_cfg.ttbr[1] = 0; + cfg->arm_lpae_s1_cfg.ttbr[1] = virt_to_phys(data->pgd_ttbr1); return &data->iop; +out_free_pgd: + __arm_lpae_free_pages(data->pgd, data->pgd_size, cfg, cookie); + out_free_data: kfree(data); return NULL; @@ -862,7 +1239,7 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) struct arm_lpae_io_pgtable *data; /* The NS quirk doesn't apply at stage 2 */ - if (cfg->quirks & ~IO_PGTABLE_QUIRK_NO_DMA) + if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -923,9 +1300,6 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) case 48: reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); break; - case 52: - reg |= (ARM_LPAE_TCR_PS_52_BIT << ARM_LPAE_TCR_PS_SHIFT); - break; default: goto out_free_data; } @@ -935,7 +1309,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s2_cfg.vtcr = reg; /* Allocate pgd pages */ - data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, + cfg, cookie); if (!data->pgd) goto out_free_data; @@ -1018,7 +1393,6 @@ static void dummy_tlb_add_flush(unsigned long iova, size_t size, size_t granule, bool leaf, void *cookie) { WARN_ON(cookie != cfg_cookie); - WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); } static void dummy_tlb_sync(void *cookie) @@ -1051,6 +1425,43 @@ static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) -EFAULT; \ }) +/* + * Returns true if there's any mapping in the given iova range in ops. + */ +static bool arm_lpae_range_has_mapping(struct io_pgtable_ops *ops, + unsigned long iova_start, size_t size) +{ + unsigned long iova = iova_start; + + while (iova < (iova_start + size)) { + if (ops->iova_to_phys(ops, iova + 42)) + return true; + iova += SZ_4K; + } + return false; +} + +/* + * Returns true if the iova range is successfully mapped to the contiguous + * phys range in ops. + */ +static bool arm_lpae_range_has_specific_mapping(struct io_pgtable_ops *ops, + const unsigned long iova_start, + const phys_addr_t phys_start, + const size_t size) +{ + unsigned long iova = iova_start; + phys_addr_t phys = phys_start; + + while (iova < (iova_start + size)) { + if (ops->iova_to_phys(ops, iova + 42) != (phys + 42)) + return false; + iova += SZ_4K; + phys += SZ_4K; + } + return true; +} + static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) { static const enum io_pgtable_fmt fmts[] = { @@ -1058,14 +1469,16 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) ARM_64_LPAE_S2, }; - int i, j; + int i, j, k; unsigned long iova; size_t size; struct io_pgtable_ops *ops; - selftest_running = true; for (i = 0; i < ARRAY_SIZE(fmts); ++i) { + unsigned long test_sg_sizes[] = { SZ_4K, SZ_64K, SZ_2M, + SZ_1M * 12, SZ_1M * 20 }; + cfg_cookie = cfg; ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); if (!ops) { @@ -1074,16 +1487,11 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) } /* - * Initial sanity checks. - * Empty page tables shouldn't provide any translations. + * Initial sanity checks. Empty page tables shouldn't + * provide any translations. TODO: check entire supported + * range for these ops rather than first 2G */ - if (ops->iova_to_phys(ops, 42)) - return __FAIL(ops, i); - - if (ops->iova_to_phys(ops, SZ_1G + 42)) - return __FAIL(ops, i); - - if (ops->iova_to_phys(ops, SZ_2G + 42)) + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) return __FAIL(ops, i); /* @@ -1104,7 +1512,8 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) IOMMU_READ | IOMMU_NOEXEC)) return __FAIL(ops, i); - if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + if (!arm_lpae_range_has_specific_mapping(ops, iova, + iova, size)) return __FAIL(ops, i); iova += SZ_1G; @@ -1115,11 +1524,15 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) if (ops->unmap(ops, SZ_1G + size, size) != size) return __FAIL(ops, i); + if (arm_lpae_range_has_mapping(ops, SZ_1G + size, size)) + return __FAIL(ops, i); + /* Remap of partial unmap */ if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) return __FAIL(ops, i); - if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) + if (!arm_lpae_range_has_specific_mapping(ops, SZ_1G + size, + size, size)) return __FAIL(ops, i); /* Full unmap */ @@ -1140,9 +1553,105 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) return __FAIL(ops, i); + if (ops->unmap(ops, iova, size) != size) + return __FAIL(ops, i); + iova += SZ_1G; } + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + + if ((cfg->pgsize_bitmap & SZ_2M) && + (cfg->pgsize_bitmap & SZ_4K)) { + /* mixed block + page mappings */ + iova = 0; + if (ops->map(ops, iova, iova, SZ_2M, IOMMU_READ)) + return __FAIL(ops, i); + + if (ops->map(ops, iova + SZ_2M, iova + SZ_2M, SZ_4K, + IOMMU_READ)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + SZ_2M + 42) != + (iova + SZ_2M + 42)) + return __FAIL(ops, i); + + /* unmap both mappings at once */ + if (ops->unmap(ops, iova, SZ_2M + SZ_4K) != + (SZ_2M + SZ_4K)) + return __FAIL(ops, i); + + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + } + + /* map_sg */ + for (j = 0; j < ARRAY_SIZE(test_sg_sizes); ++j) { + size_t mapped; + size_t unused; + struct page *page; + phys_addr_t page_phys; + struct sg_table table; + struct scatterlist *sg; + unsigned long total_size = test_sg_sizes[j]; + int chunk_size = 1UL << find_first_bit( + &cfg->pgsize_bitmap, BITS_PER_LONG); + int nents = total_size / chunk_size; + int ret; + + if (total_size < chunk_size) + continue; + + page = alloc_pages(GFP_KERNEL, get_order(chunk_size)); + if (!page) + return -ENOMEM; + page_phys = page_to_phys(page); + + iova = 0; + ret = sg_alloc_table(&table, nents, GFP_KERNEL); + if (ret) + return ret; + for_each_sg(table.sgl, sg, table.nents, k) + sg_set_page(sg, page, chunk_size, 0); + + mapped = ops->map_sg(ops, iova, table.sgl, table.nents, + IOMMU_READ | IOMMU_WRITE, &unused); + + if (mapped != total_size) + return __FAIL(ops, i); + + if (!arm_lpae_range_has_mapping(ops, iova, total_size)) + return __FAIL(ops, i); + + if (arm_lpae_range_has_mapping(ops, iova + total_size, + SZ_2G - (iova + total_size))) + return __FAIL(ops, i); + + for_each_sg(table.sgl, sg, table.nents, k) { + dma_addr_t newphys = + ops->iova_to_phys(ops, iova + 42); + if (newphys != (page_phys + 42)) + return __FAIL(ops, i); + iova += chunk_size; + } + + if (ops->unmap(ops, 0, total_size) != total_size) + return __FAIL(ops, i); + + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + + sg_free_table(&table); + __free_pages(page, get_order(chunk_size)); + } + + if (arm_lpae_range_has_mapping(ops, 0, SZ_2G)) + return __FAIL(ops, i); + free_io_pgtable_ops(ops); } @@ -1154,8 +1663,6 @@ static int __init arm_lpae_do_selftests(void) { static const unsigned long pgsize[] = { SZ_4K | SZ_2M | SZ_1G, - SZ_16K | SZ_32M, - SZ_64K | SZ_512M, }; static const unsigned int ias[] = {