diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index dc54f1e1a676..0e0baa944c2c 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -145,6 +145,7 @@ struct rk_iommu { u32 version; bool shootdown_entire; bool iommu_enabled; + bool need_res_map; }; struct rk_iommudata { @@ -155,6 +156,8 @@ struct rk_iommudata { static struct device *dma_dev; static struct rk_iommu *rk_iommu_from_dev(struct device *dev); +static char reserve_range[PAGE_SIZE] __aligned(PAGE_SIZE); +static phys_addr_t res_page; static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, unsigned int count) @@ -299,6 +302,8 @@ static inline u32 rk_mk_dte_v2(dma_addr_t pt_dma) #define RK_PTE_PAGE_READABLE_V2 BIT(1) #define RK_PTE_PAGE_WRITABLE_V2 BIT(2) +#define RK_PTE_PAGE_REPRESENT BIT(3) + static inline phys_addr_t rk_pte_page_address(u32 pte) { return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK; @@ -320,12 +325,19 @@ static inline bool rk_pte_is_page_valid(u32 pte) return pte & RK_PTE_PAGE_VALID; } +static inline bool rk_pte_is_page_represent(u32 pte) +{ + return pte & RK_PTE_PAGE_REPRESENT; +} + /* TODO: set cache flags per prot IOMMU_CACHE */ static u32 rk_mk_pte(phys_addr_t page, int prot) { u32 flags = 0; flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0; flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0; + flags |= (prot & IOMMU_PRIV) ? RK_PTE_PAGE_REPRESENT : 0; + page &= RK_PTE_PAGE_ADDRESS_MASK; return page | flags | RK_PTE_PAGE_VALID; } @@ -336,6 +348,12 @@ static u32 rk_mk_pte_v2(phys_addr_t page, int prot) flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE_V2 : 0; flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE_V2 : 0; + /* If BIT(3) set, don't break iommu_map if BIT(0) set. + * Means we can reupdate a page that already presented. We can use + * this bit to reupdate a pre-mapped 4G range. + */ + flags |= (prot & IOMMU_PRIV) ? RK_PTE_PAGE_REPRESENT : 0; + page = (page & PAGE_DESC_LO_MASK) | ((page & PAGE_DESC_HI_MASK1) >> PAGE_DESC_HI_SHIFT1) | (page & PAGE_DESC_HI_MASK2) >> PAGE_DESC_HI_SHIFT2; @@ -346,7 +364,7 @@ static u32 rk_mk_pte_v2(phys_addr_t page, int prot) static u32 rk_mk_pte_invalid(u32 pte) { - return pte & ~RK_PTE_PAGE_VALID; + return pte & ~(RK_PTE_PAGE_VALID | RK_PTE_PAGE_REPRESENT); } /* @@ -986,10 +1004,11 @@ done: static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, dma_addr_t pte_dma, - size_t size) + size_t size, struct rk_iommu *iommu) { unsigned int pte_count; unsigned int pte_total = size / SPAGE_SIZE; + int prot = IOMMU_READ | IOMMU_WRITE | IOMMU_PRIV; assert_spin_locked(&rk_domain->dt_lock); @@ -998,7 +1017,15 @@ static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, if (!rk_pte_is_page_valid(pte)) break; - pte_addr[pte_count] = rk_mk_pte_invalid(pte); + if (iommu && iommu->need_res_map) { + if (iommu->version >= 0x2) + pte_addr[pte_count] = rk_mk_pte_v2(res_page, + prot); + else + pte_addr[pte_count] = rk_mk_pte(res_page, prot); + } else { + pte_addr[pte_count] = rk_mk_pte_invalid(pte); + } } rk_table_flush(rk_domain, pte_dma, pte_count); @@ -1006,6 +1033,23 @@ static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, return pte_count * SPAGE_SIZE; } +static struct rk_iommu *rk_iommu_get(struct rk_iommu_domain *rk_domain) +{ + unsigned long flags; + struct list_head *pos; + struct rk_iommu *iommu = NULL; + + spin_lock_irqsave(&rk_domain->iommus_lock, flags); + list_for_each(pos, &rk_domain->iommus) { + iommu = list_entry(pos, struct rk_iommu, node); + if (iommu->need_res_map) + break; + } + spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); + + return iommu; +} + static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, dma_addr_t pte_dma, dma_addr_t iova, phys_addr_t paddr, size_t size, int prot) @@ -1019,12 +1063,15 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, for (pte_count = 0; pte_count < pte_total; pte_count++) { u32 pte = pte_addr[pte_count]; - if (rk_pte_is_page_valid(pte)) + if (rk_pte_is_page_valid(pte) && !rk_pte_is_page_represent(pte)) goto unwind; - pte_addr[pte_count] = rk_mk_pte(paddr, prot); - - paddr += SPAGE_SIZE; + if (prot & IOMMU_PRIV) { + pte_addr[pte_count] = rk_mk_pte(res_page, prot); + } else { + pte_addr[pte_count] = rk_mk_pte(paddr, prot); + paddr += SPAGE_SIZE; + } } rk_table_flush(rk_domain, pte_dma, pte_total); @@ -1043,7 +1090,7 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, unwind: /* Unmap the range of iovas that we just mapped */ rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, - pte_count * SPAGE_SIZE); + pte_count * SPAGE_SIZE, NULL); iova += pte_count * SPAGE_SIZE; page_phys = rk_pte_page_address(pte_addr[pte_count]); @@ -1066,12 +1113,15 @@ static int rk_iommu_map_iova_v2(struct rk_iommu_domain *rk_domain, u32 *pte_addr for (pte_count = 0; pte_count < pte_total; pte_count++) { u32 pte = pte_addr[pte_count]; - if (rk_pte_is_page_valid(pte)) + if (rk_pte_is_page_valid(pte) && !rk_pte_is_page_represent(pte)) goto unwind; - pte_addr[pte_count] = rk_mk_pte_v2(paddr, prot); - - paddr += SPAGE_SIZE; + if (prot & IOMMU_PRIV) { + pte_addr[pte_count] = rk_mk_pte_v2(res_page, prot); + } else { + pte_addr[pte_count] = rk_mk_pte_v2(paddr, prot); + paddr += SPAGE_SIZE; + } } rk_table_flush(rk_domain, pte_dma, pte_total); @@ -1090,7 +1140,7 @@ static int rk_iommu_map_iova_v2(struct rk_iommu_domain *rk_domain, u32 *pte_addr unwind: /* Unmap the range of iovas that we just mapped */ rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, - pte_count * SPAGE_SIZE); + pte_count * SPAGE_SIZE, NULL); iova += pte_count * SPAGE_SIZE; page_phys = rk_pte_page_address_v2(pte_addr[pte_count]); @@ -1184,6 +1234,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, u32 dte; u32 *pte_addr; size_t unmap_size; + struct rk_iommu *iommu = rk_iommu_get(rk_domain); spin_lock_irqsave(&rk_domain->dt_lock, flags); @@ -1204,7 +1255,8 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, pt_phys = rk_dte_pt_address(dte); pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); - unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); + unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size, + iommu); spin_unlock_irqrestore(&rk_domain->dt_lock, flags); @@ -1224,6 +1276,7 @@ static size_t rk_iommu_unmap_v2(struct iommu_domain *domain, unsigned long _iova u32 dte; u32 *pte_addr; size_t unmap_size; + struct rk_iommu *iommu = rk_iommu_get(rk_domain); spin_lock_irqsave(&rk_domain->dt_lock, flags); @@ -1244,7 +1297,8 @@ static size_t rk_iommu_unmap_v2(struct iommu_domain *domain, unsigned long _iova pt_phys = rk_dte_pt_address_v2(dte); pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); - unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); + unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size, + iommu); spin_unlock_irqrestore(&rk_domain->dt_lock, flags); @@ -1812,6 +1866,9 @@ static int rk_iommu_probe(struct platform_device *pdev) iommu->cmd_retry = device_property_read_bool(dev, "rockchip,enable-cmd-retry"); + iommu->need_res_map = device_property_read_bool(dev, + "rockchip,reserve-map"); + /* * iommu clocks should be present for all new devices and devicetrees * but there are older devicetrees without clocks out in the wild. @@ -1882,6 +1939,12 @@ static int rk_iommu_probe(struct platform_device *pdev) } skip_request_irq: + if (!res_page && iommu->need_res_map) { + res_page = __pa_symbol(reserve_range); + + pr_info("%s,%d, res_page = 0x%pa\n", __func__, __LINE__, &res_page); + } + return 0; err_remove_sysfs: iommu_device_sysfs_remove(&iommu->iommu);