mirror of
https://github.com/hardkernel/linux.git
synced 2026-06-07 03:15:31 +09:00
iommu/rockchip: Backport from UPSTREAM linux-5.15
Change-Id: I110dbe10b49d6e61a7e48a0f85864389641526de Signed-off-by: Simon Xue <xxm@rock-chips.com>
This commit is contained in:
Simon Xue
@@ -87,30 +87,6 @@
|
||||
*/
|
||||
#define RK_IOMMU_PGSIZE_BITMAP 0x007ff000
|
||||
|
||||
#define DT_LO_MASK 0xfffff000
|
||||
#define DT_HI_MASK GENMASK_ULL(39, 32)
|
||||
#define DT_SHIFT 28
|
||||
|
||||
#define DTE_BASE_HI_MASK GENMASK(11, 4)
|
||||
|
||||
#define PAGE_DESC_LO_MASK 0xfffff000
|
||||
#define PAGE_DESC_HI1_LOWER 32
|
||||
#define PAGE_DESC_HI1_UPPER 35
|
||||
#define PAGE_DESC_HI2_LOWER 36
|
||||
#define PAGE_DESC_HI2_UPPER 39
|
||||
#define PAGE_DESC_HI_MASK1 GENMASK_ULL(PAGE_DESC_HI1_UPPER, PAGE_DESC_HI1_LOWER)
|
||||
#define PAGE_DESC_HI_MASK2 GENMASK_ULL(PAGE_DESC_HI2_UPPER, PAGE_DESC_HI2_LOWER)
|
||||
|
||||
#define DTE_HI1_LOWER 8
|
||||
#define DTE_HI1_UPPER 11
|
||||
#define DTE_HI2_LOWER 4
|
||||
#define DTE_HI2_UPPER 7
|
||||
#define DTE_HI_MASK1 GENMASK(DTE_HI1_UPPER, DTE_HI1_LOWER)
|
||||
#define DTE_HI_MASK2 GENMASK(DTE_HI2_UPPER, DTE_HI2_LOWER)
|
||||
|
||||
#define PAGE_DESC_HI_SHIFT1 (PAGE_DESC_HI1_LOWER - DTE_HI1_LOWER)
|
||||
#define PAGE_DESC_HI_SHIFT2 (PAGE_DESC_HI2_LOWER - DTE_HI2_LOWER)
|
||||
|
||||
struct rk_iommu_domain {
|
||||
struct list_head iommus;
|
||||
u32 *dt; /* page directory table */
|
||||
@@ -122,8 +98,13 @@ struct rk_iommu_domain {
|
||||
struct iommu_domain domain;
|
||||
};
|
||||
|
||||
struct rockchip_iommu_data {
|
||||
u32 version;
|
||||
struct rk_iommu_ops {
|
||||
phys_addr_t (*pt_address)(u32 dte);
|
||||
u32 (*mk_dtentries)(dma_addr_t pt_dma);
|
||||
u32 (*mk_ptentries)(phys_addr_t page, int prot);
|
||||
phys_addr_t (*dte_addr_phys)(u32 addr);
|
||||
u32 (*dma_addr_dte)(dma_addr_t dt_dma);
|
||||
u64 dma_bit_mask;
|
||||
};
|
||||
|
||||
struct rk_iommu {
|
||||
@@ -142,7 +123,6 @@ struct rk_iommu {
|
||||
struct list_head node; /* entry in rk_iommu_domain.iommus */
|
||||
struct iommu_domain *domain; /* domain to which iommu is attached */
|
||||
struct iommu_group *group;
|
||||
u32 version;
|
||||
bool shootdown_entire;
|
||||
bool iommu_enabled;
|
||||
bool need_res_map;
|
||||
@@ -155,6 +135,7 @@ struct rk_iommudata {
|
||||
};
|
||||
|
||||
static struct device *dma_dev;
|
||||
static const struct rk_iommu_ops *rk_ops;
|
||||
static struct rk_iommu *rk_iommu_from_dev(struct device *dev);
|
||||
static char reserve_range[PAGE_SIZE] __aligned(PAGE_SIZE);
|
||||
static phys_addr_t res_page;
|
||||
@@ -216,6 +197,11 @@ static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
|
||||
#define RK_DTE_PT_ADDRESS_MASK 0xfffff000
|
||||
#define RK_DTE_PT_VALID BIT(0)
|
||||
|
||||
static inline phys_addr_t rk_dte_pt_address(u32 dte)
|
||||
{
|
||||
return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK;
|
||||
}
|
||||
|
||||
/*
|
||||
* In v2:
|
||||
* 31:12 - PT address bit 31:0
|
||||
@@ -224,20 +210,21 @@ static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
|
||||
* 3: 1 - Reserved
|
||||
* 0 - 1 if PT @ PT address is valid
|
||||
*/
|
||||
#define RK_DTE_PT_ADDRESS_MASK_V2 0xfffffff0
|
||||
|
||||
static inline phys_addr_t rk_dte_pt_address(u32 dte)
|
||||
{
|
||||
return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK;
|
||||
}
|
||||
#define RK_DTE_PT_ADDRESS_MASK_V2 GENMASK_ULL(31, 4)
|
||||
#define DTE_HI_MASK1 GENMASK(11, 8)
|
||||
#define DTE_HI_MASK2 GENMASK(7, 4)
|
||||
#define DTE_HI_SHIFT1 24 /* shift bit 8 to bit 32 */
|
||||
#define DTE_HI_SHIFT2 32 /* shift bit 4 to bit 36 */
|
||||
#define PAGE_DESC_HI_MASK1 GENMASK_ULL(35, 32)
|
||||
#define PAGE_DESC_HI_MASK2 GENMASK_ULL(39, 36)
|
||||
|
||||
static inline phys_addr_t rk_dte_pt_address_v2(u32 dte)
|
||||
{
|
||||
u64 dte_v2 = dte;
|
||||
|
||||
dte_v2 = ((dte_v2 & DTE_HI_MASK2) << PAGE_DESC_HI_SHIFT2) |
|
||||
((dte_v2 & DTE_HI_MASK1) << PAGE_DESC_HI_SHIFT1) |
|
||||
(dte_v2 & PAGE_DESC_LO_MASK);
|
||||
dte_v2 = ((dte_v2 & DTE_HI_MASK2) << DTE_HI_SHIFT2) |
|
||||
((dte_v2 & DTE_HI_MASK1) << DTE_HI_SHIFT1) |
|
||||
(dte_v2 & RK_DTE_PT_ADDRESS_MASK);
|
||||
|
||||
return (phys_addr_t)dte_v2;
|
||||
}
|
||||
@@ -254,9 +241,9 @@ static inline u32 rk_mk_dte(dma_addr_t pt_dma)
|
||||
|
||||
static inline u32 rk_mk_dte_v2(dma_addr_t pt_dma)
|
||||
{
|
||||
pt_dma = (pt_dma & PAGE_DESC_LO_MASK) |
|
||||
((pt_dma & PAGE_DESC_HI_MASK1) >> PAGE_DESC_HI_SHIFT1) |
|
||||
(pt_dma & PAGE_DESC_HI_MASK2) >> PAGE_DESC_HI_SHIFT2;
|
||||
pt_dma = (pt_dma & RK_DTE_PT_ADDRESS_MASK) |
|
||||
((pt_dma & PAGE_DESC_HI_MASK1) >> DTE_HI_SHIFT1) |
|
||||
(pt_dma & PAGE_DESC_HI_MASK2) >> DTE_HI_SHIFT2;
|
||||
|
||||
return (pt_dma & RK_DTE_PT_ADDRESS_MASK_V2) | RK_DTE_PT_VALID;
|
||||
}
|
||||
@@ -287,44 +274,13 @@ static inline u32 rk_mk_dte_v2(dma_addr_t pt_dma)
|
||||
#define RK_PTE_PAGE_READABLE BIT(1)
|
||||
#define RK_PTE_PAGE_VALID BIT(0)
|
||||
|
||||
/*
|
||||
* In v2:
|
||||
* 31:12 - Page address bit 31:0
|
||||
* 11:9 - Page address bit 34:32
|
||||
* 8:4 - Page address bit 39:35
|
||||
* 3 - Security
|
||||
* 2 - Writable
|
||||
* 1 - Readable
|
||||
* 0 - 1 if Page @ Page address is valid
|
||||
*/
|
||||
#define RK_PTE_PAGE_ADDRESS_MASK_V2 0xfffffff0
|
||||
#define RK_PTE_PAGE_FLAGS_MASK_V2 0x0000000e
|
||||
#define RK_PTE_PAGE_READABLE_V2 BIT(1)
|
||||
#define RK_PTE_PAGE_WRITABLE_V2 BIT(2)
|
||||
|
||||
#define RK_PTE_PAGE_REPRESENT BIT(3)
|
||||
|
||||
static inline phys_addr_t rk_pte_page_address(u32 pte)
|
||||
{
|
||||
return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK;
|
||||
}
|
||||
|
||||
static inline phys_addr_t rk_pte_page_address_v2(u32 pte)
|
||||
{
|
||||
u64 pte_v2 = pte;
|
||||
|
||||
pte_v2 = ((pte_v2 & DTE_HI_MASK2) << PAGE_DESC_HI_SHIFT2) |
|
||||
((pte_v2 & DTE_HI_MASK1) << PAGE_DESC_HI_SHIFT1) |
|
||||
(pte_v2 & PAGE_DESC_LO_MASK);
|
||||
|
||||
return (phys_addr_t)pte_v2;
|
||||
}
|
||||
|
||||
static inline bool rk_pte_is_page_valid(u32 pte)
|
||||
{
|
||||
return pte & RK_PTE_PAGE_VALID;
|
||||
}
|
||||
|
||||
#define RK_PTE_PAGE_REPRESENT BIT(3)
|
||||
|
||||
static inline bool rk_pte_is_page_represent(u32 pte)
|
||||
{
|
||||
return pte & RK_PTE_PAGE_REPRESENT;
|
||||
@@ -337,7 +293,6 @@ static u32 rk_mk_pte(phys_addr_t page, int prot)
|
||||
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0;
|
||||
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0;
|
||||
flags |= (prot & IOMMU_PRIV) ? RK_PTE_PAGE_REPRESENT : 0;
|
||||
|
||||
page &= RK_PTE_PAGE_ADDRESS_MASK;
|
||||
return page | flags | RK_PTE_PAGE_VALID;
|
||||
}
|
||||
@@ -346,20 +301,16 @@ static u32 rk_mk_pte_v2(phys_addr_t page, int prot)
|
||||
{
|
||||
u32 flags = 0;
|
||||
|
||||
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE_V2 : 0;
|
||||
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE_V2 : 0;
|
||||
/* If BIT(3) set, don't break iommu_map if BIT(0) set.
|
||||
* Means we can reupdate a page that already presented. We can use
|
||||
* this bit to reupdate a pre-mapped 4G range.
|
||||
*/
|
||||
flags |= (prot & IOMMU_PRIV) ? RK_PTE_PAGE_REPRESENT : 0;
|
||||
|
||||
page = (page & PAGE_DESC_LO_MASK) |
|
||||
((page & PAGE_DESC_HI_MASK1) >> PAGE_DESC_HI_SHIFT1) |
|
||||
(page & PAGE_DESC_HI_MASK2) >> PAGE_DESC_HI_SHIFT2;
|
||||
page &= RK_PTE_PAGE_ADDRESS_MASK_V2;
|
||||
flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0;
|
||||
flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0;
|
||||
|
||||
return page | flags | RK_PTE_PAGE_VALID;
|
||||
return rk_mk_dte_v2(page) | flags;
|
||||
}
|
||||
|
||||
static u32 rk_mk_pte_invalid(u32 pte)
|
||||
@@ -615,7 +566,7 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu)
|
||||
int ret, i;
|
||||
u32 dte_addr;
|
||||
bool val;
|
||||
u32 address_mask;
|
||||
u32 dte_address_mask;
|
||||
|
||||
if (iommu->reset_disabled)
|
||||
return 0;
|
||||
@@ -632,14 +583,11 @@ static int rk_iommu_force_reset(struct rk_iommu *iommu)
|
||||
* In v2: upper 7 nybbles are read back.
|
||||
*/
|
||||
for (i = 0; i < iommu->num_mmu; i++) {
|
||||
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY);
|
||||
dte_address_mask = rk_ops->pt_address(DTE_ADDR_DUMMY);
|
||||
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_address_mask);
|
||||
|
||||
if (iommu->version >= 0x2)
|
||||
address_mask = RK_DTE_PT_ADDRESS_MASK_V2;
|
||||
else
|
||||
address_mask = RK_DTE_PT_ADDRESS_MASK;
|
||||
ret = readx_poll_timeout(rk_iommu_read_dte_addr, iommu->bases[i], dte_addr,
|
||||
dte_addr == (DTE_ADDR_DUMMY & address_mask),
|
||||
dte_addr == dte_address_mask,
|
||||
RK_MMU_POLL_PERIOD_US, RK_MMU_POLL_TIMEOUT_US);
|
||||
if (ret) {
|
||||
dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n");
|
||||
@@ -663,6 +611,33 @@ read_wa:
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline phys_addr_t rk_dte_addr_phys(u32 addr)
|
||||
{
|
||||
return (phys_addr_t)addr;
|
||||
}
|
||||
|
||||
static inline u32 rk_dma_addr_dte(dma_addr_t dt_dma)
|
||||
{
|
||||
return dt_dma;
|
||||
}
|
||||
|
||||
#define DT_HI_MASK GENMASK_ULL(39, 32)
|
||||
#define DTE_BASE_HI_MASK GENMASK(11, 4)
|
||||
#define DT_SHIFT 28
|
||||
|
||||
static inline phys_addr_t rk_dte_addr_phys_v2(u32 addr)
|
||||
{
|
||||
u64 addr64 = addr;
|
||||
return (phys_addr_t)(addr64 & RK_DTE_PT_ADDRESS_MASK) |
|
||||
((addr64 & DTE_BASE_HI_MASK) << DT_SHIFT);
|
||||
}
|
||||
|
||||
static inline u32 rk_dma_addr_dte_v2(dma_addr_t dt_dma)
|
||||
{
|
||||
return (dt_dma & RK_DTE_PT_ADDRESS_MASK) |
|
||||
((dt_dma & DT_HI_MASK) >> DT_SHIFT);
|
||||
}
|
||||
|
||||
static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
|
||||
{
|
||||
void __iomem *base = iommu->bases[index];
|
||||
@@ -682,11 +657,7 @@ static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
|
||||
page_offset = rk_iova_page_offset(iova);
|
||||
|
||||
mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR);
|
||||
mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr;
|
||||
if (iommu->version >= 0x2) {
|
||||
mmu_dte_addr_phys = (mmu_dte_addr_phys & DT_LO_MASK) |
|
||||
((mmu_dte_addr_phys & DTE_BASE_HI_MASK) << DT_SHIFT);
|
||||
}
|
||||
mmu_dte_addr_phys = rk_ops->dte_addr_phys(mmu_dte_addr);
|
||||
|
||||
dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index);
|
||||
dte_addr = phys_to_virt(dte_addr_phys);
|
||||
@@ -695,20 +666,14 @@ static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
|
||||
if (!rk_dte_is_pt_valid(dte))
|
||||
goto print_it;
|
||||
|
||||
if (iommu->version >= 0x2)
|
||||
pte_addr_phys = rk_dte_pt_address_v2(dte) + (pte_index * 4);
|
||||
else
|
||||
pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4);
|
||||
pte_addr_phys = rk_ops->pt_address(dte) + (pte_index * 4);
|
||||
pte_addr = phys_to_virt(pte_addr_phys);
|
||||
pte = *pte_addr;
|
||||
|
||||
if (!rk_pte_is_page_valid(pte))
|
||||
goto print_it;
|
||||
|
||||
if (iommu->version >= 0x2)
|
||||
page_addr_phys = rk_pte_page_address_v2(pte) + page_offset;
|
||||
else
|
||||
page_addr_phys = rk_pte_page_address(pte) + page_offset;
|
||||
page_addr_phys = rk_ops->pt_address(pte) + page_offset;
|
||||
page_flags = pte & RK_PTE_PAGE_FLAGS_MASK;
|
||||
|
||||
print_it:
|
||||
@@ -847,41 +812,13 @@ static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain,
|
||||
if (!rk_dte_is_pt_valid(dte))
|
||||
goto out;
|
||||
|
||||
pt_phys = rk_dte_pt_address(dte);
|
||||
pt_phys = rk_ops->pt_address(dte);
|
||||
page_table = (u32 *)phys_to_virt(pt_phys);
|
||||
pte = page_table[rk_iova_pte_index(iova)];
|
||||
if (!rk_pte_is_page_valid(pte))
|
||||
goto out;
|
||||
|
||||
phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova);
|
||||
out:
|
||||
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
|
||||
|
||||
return phys;
|
||||
}
|
||||
|
||||
static phys_addr_t rk_iommu_iova_to_phys_v2(struct iommu_domain *domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
|
||||
unsigned long flags;
|
||||
phys_addr_t pt_phys, phys = 0;
|
||||
u32 dte, pte;
|
||||
u32 *page_table;
|
||||
|
||||
spin_lock_irqsave(&rk_domain->dt_lock, flags);
|
||||
|
||||
dte = rk_domain->dt[rk_iova_dte_index(iova)];
|
||||
if (!rk_dte_is_pt_valid(dte))
|
||||
goto out;
|
||||
|
||||
pt_phys = rk_dte_pt_address_v2(dte);
|
||||
page_table = (u32 *)phys_to_virt(pt_phys);
|
||||
pte = page_table[rk_iova_pte_index(iova)];
|
||||
if (!rk_pte_is_page_valid(pte))
|
||||
goto out;
|
||||
|
||||
phys = rk_pte_page_address_v2(pte) + rk_iova_page_offset(iova);
|
||||
phys = rk_ops->pt_address(pte) + rk_iova_page_offset(iova);
|
||||
out:
|
||||
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
|
||||
|
||||
@@ -957,52 +894,13 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
dte = rk_mk_dte(pt_dma);
|
||||
dte = rk_ops->mk_dtentries(pt_dma);
|
||||
*dte_addr = dte;
|
||||
|
||||
rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES);
|
||||
rk_table_flush(rk_domain,
|
||||
rk_domain->dt_dma + dte_index * sizeof(u32), 1);
|
||||
done:
|
||||
pt_phys = rk_dte_pt_address(dte);
|
||||
return (u32 *)phys_to_virt(pt_phys);
|
||||
}
|
||||
|
||||
static u32 *rk_dte_get_page_table_v2(struct rk_iommu_domain *rk_domain,
|
||||
dma_addr_t iova)
|
||||
{
|
||||
u32 *page_table, *dte_addr;
|
||||
u32 dte_index, dte;
|
||||
phys_addr_t pt_phys;
|
||||
dma_addr_t pt_dma;
|
||||
|
||||
assert_spin_locked(&rk_domain->dt_lock);
|
||||
|
||||
dte_index = rk_iova_dte_index(iova);
|
||||
dte_addr = &rk_domain->dt[dte_index];
|
||||
dte = *dte_addr;
|
||||
if (rk_dte_is_pt_valid(dte))
|
||||
goto done;
|
||||
|
||||
page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
|
||||
if (!page_table)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE);
|
||||
if (dma_mapping_error(dma_dev, pt_dma)) {
|
||||
dev_err(dma_dev, "DMA mapping error while allocating page table\n");
|
||||
free_page((unsigned long)page_table);
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
dte = rk_mk_dte_v2(pt_dma);
|
||||
*dte_addr = dte;
|
||||
|
||||
rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES);
|
||||
rk_table_flush(rk_domain,
|
||||
rk_domain->dt_dma + dte_index * sizeof(u32), 1);
|
||||
done:
|
||||
pt_phys = rk_dte_pt_address_v2(dte);
|
||||
pt_phys = rk_ops->pt_address(dte);
|
||||
return (u32 *)phys_to_virt(pt_phys);
|
||||
}
|
||||
|
||||
@@ -1021,15 +919,11 @@ static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain,
|
||||
if (!rk_pte_is_page_valid(pte))
|
||||
break;
|
||||
|
||||
if (iommu && iommu->need_res_map) {
|
||||
if (iommu->version >= 0x2)
|
||||
pte_addr[pte_count] = rk_mk_pte_v2(res_page,
|
||||
if (iommu && iommu->need_res_map)
|
||||
pte_addr[pte_count] = rk_ops->mk_ptentries(res_page,
|
||||
prot);
|
||||
else
|
||||
pte_addr[pte_count] = rk_mk_pte(res_page, prot);
|
||||
} else {
|
||||
else
|
||||
pte_addr[pte_count] = rk_mk_pte_invalid(pte);
|
||||
}
|
||||
}
|
||||
|
||||
rk_table_flush(rk_domain, pte_dma, pte_count);
|
||||
@@ -1071,9 +965,10 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
|
||||
goto unwind;
|
||||
|
||||
if (prot & IOMMU_PRIV) {
|
||||
pte_addr[pte_count] = rk_mk_pte(res_page, prot);
|
||||
pte_addr[pte_count] = rk_ops->mk_ptentries(res_page, prot);
|
||||
} else {
|
||||
pte_addr[pte_count] = rk_mk_pte(paddr, prot);
|
||||
pte_addr[pte_count] = rk_ops->mk_ptentries(paddr, prot);
|
||||
|
||||
paddr += SPAGE_SIZE;
|
||||
}
|
||||
}
|
||||
@@ -1095,55 +990,7 @@ unwind:
|
||||
pte_count * SPAGE_SIZE, NULL);
|
||||
|
||||
iova += pte_count * SPAGE_SIZE;
|
||||
page_phys = rk_pte_page_address(pte_addr[pte_count]);
|
||||
pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n",
|
||||
&iova, &page_phys, &paddr, prot);
|
||||
|
||||
return -EADDRINUSE;
|
||||
}
|
||||
|
||||
static int rk_iommu_map_iova_v2(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
|
||||
dma_addr_t pte_dma, dma_addr_t iova,
|
||||
phys_addr_t paddr, size_t size, int prot)
|
||||
{
|
||||
unsigned int pte_count;
|
||||
unsigned int pte_total = size / SPAGE_SIZE;
|
||||
phys_addr_t page_phys;
|
||||
|
||||
assert_spin_locked(&rk_domain->dt_lock);
|
||||
|
||||
for (pte_count = 0; pte_count < pte_total; pte_count++) {
|
||||
u32 pte = pte_addr[pte_count];
|
||||
|
||||
if (rk_pte_is_page_valid(pte) && !rk_pte_is_page_represent(pte))
|
||||
goto unwind;
|
||||
|
||||
if (prot & IOMMU_PRIV) {
|
||||
pte_addr[pte_count] = rk_mk_pte_v2(res_page, prot);
|
||||
} else {
|
||||
pte_addr[pte_count] = rk_mk_pte_v2(paddr, prot);
|
||||
paddr += SPAGE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
rk_table_flush(rk_domain, pte_dma, pte_total);
|
||||
|
||||
/*
|
||||
* Zap the first and last iova to evict from iotlb any previously
|
||||
* mapped cachelines holding stale values for its dte and pte.
|
||||
* We only zap the first and last iova, since only they could have
|
||||
* dte or pte shared with an existing mapping.
|
||||
*/
|
||||
rk_iommu_zap_iova_first_last(rk_domain, iova, size);
|
||||
|
||||
return 0;
|
||||
unwind:
|
||||
/* Unmap the range of iovas that we just mapped */
|
||||
rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma,
|
||||
pte_count * SPAGE_SIZE, NULL);
|
||||
|
||||
iova += pte_count * SPAGE_SIZE;
|
||||
page_phys = rk_pte_page_address_v2(pte_addr[pte_count]);
|
||||
page_phys = rk_ops->pt_address(pte_addr[pte_count]);
|
||||
pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n",
|
||||
&iova, &page_phys, &paddr, prot);
|
||||
|
||||
@@ -1178,7 +1025,7 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
|
||||
dte = rk_domain->dt[rk_iova_dte_index(iova)];
|
||||
pte_index = rk_iova_pte_index(iova);
|
||||
pte_addr = &page_table[pte_index];
|
||||
pte_dma = rk_dte_pt_address(dte) + pte_index * sizeof(u32);
|
||||
pte_dma = rk_ops->pt_address(dte) + pte_index * sizeof(u32);
|
||||
ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova,
|
||||
paddr, size, prot);
|
||||
|
||||
@@ -1187,43 +1034,6 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int rk_iommu_map_v2(struct iommu_domain *domain, unsigned long _iova,
|
||||
phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
|
||||
{
|
||||
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
|
||||
unsigned long flags;
|
||||
dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
|
||||
u32 *page_table, *pte_addr;
|
||||
u32 dte, pte_index;
|
||||
int ret;
|
||||
|
||||
spin_lock_irqsave(&rk_domain->dt_lock, flags);
|
||||
|
||||
/*
|
||||
* pgsize_bitmap specifies iova sizes that fit in one page table
|
||||
* (1024 4-KiB pages = 4 MiB).
|
||||
* So, size will always be 4096 <= size <= 4194304.
|
||||
* Since iommu_map() guarantees that both iova and size will be
|
||||
* aligned, we will always only be mapping from a single dte here.
|
||||
*/
|
||||
page_table = rk_dte_get_page_table_v2(rk_domain, iova);
|
||||
if (IS_ERR(page_table)) {
|
||||
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
|
||||
return PTR_ERR(page_table);
|
||||
}
|
||||
|
||||
dte = rk_domain->dt[rk_iova_dte_index(iova)];
|
||||
pte_index = rk_iova_pte_index(iova);
|
||||
pte_addr = &page_table[pte_index];
|
||||
pte_dma = rk_dte_pt_address_v2(dte) + pte_index * sizeof(u32);
|
||||
ret = rk_iommu_map_iova_v2(rk_domain, pte_addr, pte_dma, iova,
|
||||
paddr, size, prot);
|
||||
|
||||
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
|
||||
size_t size, struct iommu_iotlb_gather *gather)
|
||||
{
|
||||
@@ -1252,49 +1062,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
|
||||
return 0;
|
||||
}
|
||||
|
||||
pt_phys = rk_dte_pt_address(dte);
|
||||
pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);
|
||||
pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32);
|
||||
unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size,
|
||||
iommu);
|
||||
|
||||
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
|
||||
|
||||
/* Shootdown iotlb entries for iova range that was just unmapped */
|
||||
rk_iommu_zap_iova(rk_domain, iova, unmap_size);
|
||||
|
||||
return unmap_size;
|
||||
}
|
||||
|
||||
static size_t rk_iommu_unmap_v2(struct iommu_domain *domain, unsigned long _iova,
|
||||
size_t size, struct iommu_iotlb_gather *gather)
|
||||
{
|
||||
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
|
||||
unsigned long flags;
|
||||
dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
|
||||
phys_addr_t pt_phys;
|
||||
u32 dte;
|
||||
u32 *pte_addr;
|
||||
size_t unmap_size;
|
||||
struct rk_iommu *iommu = rk_iommu_get(rk_domain);
|
||||
|
||||
spin_lock_irqsave(&rk_domain->dt_lock, flags);
|
||||
|
||||
/*
|
||||
* pgsize_bitmap specifies iova sizes that fit in one page table
|
||||
* (1024 4-KiB pages = 4 MiB).
|
||||
* So, size will always be 4096 <= size <= 4194304.
|
||||
* Since iommu_unmap() guarantees that both iova and size will be
|
||||
* aligned, we will always only be unmapping from a single dte here.
|
||||
*/
|
||||
dte = rk_domain->dt[rk_iova_dte_index(iova)];
|
||||
/* Just return 0 if iova is unmapped */
|
||||
if (!rk_dte_is_pt_valid(dte)) {
|
||||
spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
|
||||
return 0;
|
||||
}
|
||||
|
||||
pt_phys = rk_dte_pt_address_v2(dte);
|
||||
pt_phys = rk_ops->pt_address(dte);
|
||||
pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);
|
||||
pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32);
|
||||
unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size,
|
||||
@@ -1383,7 +1151,6 @@ static int rk_iommu_enable(struct rk_iommu *iommu)
|
||||
struct iommu_domain *domain = iommu->domain;
|
||||
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
|
||||
int ret, i;
|
||||
u32 dt_v2;
|
||||
u32 auto_gate;
|
||||
|
||||
ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
|
||||
@@ -1399,14 +1166,8 @@ static int rk_iommu_enable(struct rk_iommu *iommu)
|
||||
goto out_disable_stall;
|
||||
|
||||
for (i = 0; i < iommu->num_mmu; i++) {
|
||||
if (iommu->version >= 0x2) {
|
||||
dt_v2 = (rk_domain->dt_dma & DT_LO_MASK) |
|
||||
((rk_domain->dt_dma & DT_HI_MASK) >> DT_SHIFT);
|
||||
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dt_v2);
|
||||
} else {
|
||||
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
|
||||
rk_domain->dt_dma);
|
||||
}
|
||||
rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
|
||||
rk_ops->dma_addr_dte(rk_domain->dt_dma));
|
||||
rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
|
||||
rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);
|
||||
|
||||
@@ -1564,8 +1325,6 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
|
||||
goto err_free_dt;
|
||||
}
|
||||
|
||||
rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES);
|
||||
|
||||
spin_lock_init(&rk_domain->iommus_lock);
|
||||
spin_lock_init(&rk_domain->dt_lock);
|
||||
INIT_LIST_HEAD(&rk_domain->iommus);
|
||||
@@ -1597,7 +1356,7 @@ static void rk_iommu_domain_free(struct iommu_domain *domain)
|
||||
for (i = 0; i < NUM_DT_ENTRIES; i++) {
|
||||
u32 dte = rk_domain->dt[i];
|
||||
if (rk_dte_is_pt_valid(dte)) {
|
||||
phys_addr_t pt_phys = rk_dte_pt_address(dte);
|
||||
phys_addr_t pt_phys = rk_ops->pt_address(dte);
|
||||
u32 *page_table = phys_to_virt(pt_phys);
|
||||
dma_unmap_single(dma_dev, pt_phys,
|
||||
SPAGE_SIZE, DMA_TO_DEVICE);
|
||||
@@ -1609,37 +1368,6 @@ static void rk_iommu_domain_free(struct iommu_domain *domain)
|
||||
SPAGE_SIZE, DMA_TO_DEVICE);
|
||||
free_page((unsigned long)rk_domain->dt);
|
||||
|
||||
if (domain->type == IOMMU_DOMAIN_DMA)
|
||||
iommu_put_dma_cookie(&rk_domain->domain);
|
||||
kfree(rk_domain);
|
||||
}
|
||||
|
||||
static void rk_iommu_domain_free_v2(struct iommu_domain *domain)
|
||||
{
|
||||
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
|
||||
int i;
|
||||
|
||||
WARN_ON(!list_empty(&rk_domain->iommus));
|
||||
|
||||
for (i = 0; i < NUM_DT_ENTRIES; i++) {
|
||||
u32 dte = rk_domain->dt[i];
|
||||
|
||||
if (rk_dte_is_pt_valid(dte)) {
|
||||
phys_addr_t pt_phys = rk_dte_pt_address_v2(dte);
|
||||
u32 *page_table = phys_to_virt(pt_phys);
|
||||
|
||||
dma_unmap_single(dma_dev, pt_phys,
|
||||
SPAGE_SIZE, DMA_TO_DEVICE);
|
||||
free_page((unsigned long)page_table);
|
||||
}
|
||||
}
|
||||
|
||||
dma_unmap_single(dma_dev, rk_domain->dt_dma,
|
||||
SPAGE_SIZE, DMA_TO_DEVICE);
|
||||
free_page((unsigned long)rk_domain->dt);
|
||||
|
||||
if (domain->type == IOMMU_DOMAIN_DMA)
|
||||
iommu_put_dma_cookie(&rk_domain->domain);
|
||||
kfree(rk_domain);
|
||||
}
|
||||
|
||||
@@ -1749,7 +1477,7 @@ void rockchip_iommu_unmask_irq(struct device *dev)
|
||||
}
|
||||
EXPORT_SYMBOL(rockchip_iommu_unmask_irq);
|
||||
|
||||
static struct iommu_ops rk_iommu_ops = {
|
||||
static const struct iommu_ops rk_iommu_ops = {
|
||||
.domain_alloc = rk_iommu_domain_alloc,
|
||||
.domain_free = rk_iommu_domain_free,
|
||||
.attach_dev = rk_iommu_attach_device,
|
||||
@@ -1766,68 +1494,34 @@ static struct iommu_ops rk_iommu_ops = {
|
||||
.of_xlate = rk_iommu_of_xlate,
|
||||
};
|
||||
|
||||
static struct iommu_ops rk_iommu_ops_v2 = {
|
||||
.domain_alloc = rk_iommu_domain_alloc,
|
||||
.domain_free = rk_iommu_domain_free_v2,
|
||||
.attach_dev = rk_iommu_attach_device,
|
||||
.detach_dev = rk_iommu_detach_device,
|
||||
.map = rk_iommu_map_v2,
|
||||
.unmap = rk_iommu_unmap_v2,
|
||||
.flush_iotlb_all = rk_iommu_flush_tlb_all,
|
||||
.probe_device = rk_iommu_probe_device,
|
||||
.release_device = rk_iommu_release_device,
|
||||
.iova_to_phys = rk_iommu_iova_to_phys_v2,
|
||||
.is_attach_deferred = rk_iommu_is_attach_deferred,
|
||||
.device_group = rk_iommu_device_group,
|
||||
.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
|
||||
.of_xlate = rk_iommu_of_xlate,
|
||||
};
|
||||
|
||||
static const struct rockchip_iommu_data iommu_data_v1 = {
|
||||
.version = 0x1,
|
||||
};
|
||||
|
||||
static const struct rockchip_iommu_data iommu_data_v2 = {
|
||||
.version = 0x2,
|
||||
};
|
||||
|
||||
static const struct of_device_id rk_iommu_dt_ids[] = {
|
||||
{
|
||||
.compatible = "rockchip,iommu",
|
||||
.data = &iommu_data_v1,
|
||||
}, {
|
||||
.compatible = "rockchip,iommu-v2",
|
||||
.data = &iommu_data_v2,
|
||||
},
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
|
||||
static int rk_iommu_probe(struct platform_device *pdev)
|
||||
{
|
||||
struct device *dev = &pdev->dev;
|
||||
struct rk_iommu *iommu;
|
||||
struct resource *res;
|
||||
const struct rk_iommu_ops *ops;
|
||||
int num_res = pdev->num_resources;
|
||||
int err, i;
|
||||
const struct of_device_id *match;
|
||||
struct rockchip_iommu_data *data;
|
||||
|
||||
iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
|
||||
if (!iommu)
|
||||
return -ENOMEM;
|
||||
|
||||
match = of_match_device(rk_iommu_dt_ids, dev);
|
||||
if (!match)
|
||||
return -EINVAL;
|
||||
|
||||
data = (struct rockchip_iommu_data *)match->data;
|
||||
iommu->version = data->version;
|
||||
dev_info(dev, "version = %x\n", iommu->version);
|
||||
|
||||
platform_set_drvdata(pdev, iommu);
|
||||
iommu->dev = dev;
|
||||
iommu->num_mmu = 0;
|
||||
|
||||
ops = of_device_get_match_data(dev);
|
||||
if (!rk_ops)
|
||||
rk_ops = ops;
|
||||
|
||||
/*
|
||||
* That should not happen unless different versions of the
|
||||
* hardware block are embedded the same SoC
|
||||
*/
|
||||
if (WARN_ON(rk_ops != ops))
|
||||
return -EINVAL;
|
||||
|
||||
iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases),
|
||||
GFP_KERNEL);
|
||||
if (!iommu->bases)
|
||||
@@ -1894,10 +1588,8 @@ static int rk_iommu_probe(struct platform_device *pdev)
|
||||
if (err)
|
||||
goto err_put_group;
|
||||
|
||||
if (iommu->version >= 0x2)
|
||||
iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops_v2);
|
||||
else
|
||||
iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops);
|
||||
iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops);
|
||||
|
||||
iommu_device_set_fwnode(&iommu->iommu, &dev->of_node->fwnode);
|
||||
|
||||
err = iommu_device_register(&iommu->iommu);
|
||||
@@ -1912,10 +1604,7 @@ static int rk_iommu_probe(struct platform_device *pdev)
|
||||
if (!dma_dev)
|
||||
dma_dev = &pdev->dev;
|
||||
|
||||
if (iommu->version >= 0x2)
|
||||
bus_set_iommu(&platform_bus_type, &rk_iommu_ops_v2);
|
||||
else
|
||||
bus_set_iommu(&platform_bus_type, &rk_iommu_ops);
|
||||
bus_set_iommu(&platform_bus_type, &rk_iommu_ops);
|
||||
|
||||
pm_runtime_enable(dev);
|
||||
|
||||
@@ -1943,6 +1632,8 @@ skip_request_irq:
|
||||
pr_info("%s,%d, res_page = 0x%pa\n", __func__, __LINE__, &res_page);
|
||||
}
|
||||
|
||||
dma_set_mask_and_coherent(dev, rk_ops->dma_bit_mask);
|
||||
|
||||
return 0;
|
||||
err_remove_sysfs:
|
||||
iommu_device_sysfs_remove(&iommu->iommu);
|
||||
@@ -2005,6 +1696,37 @@ static const struct dev_pm_ops rk_iommu_pm_ops = {
|
||||
pm_runtime_force_resume)
|
||||
};
|
||||
|
||||
static struct rk_iommu_ops iommu_data_ops_v1 = {
|
||||
.pt_address = &rk_dte_pt_address,
|
||||
.mk_dtentries = &rk_mk_dte,
|
||||
.mk_ptentries = &rk_mk_pte,
|
||||
.dte_addr_phys = &rk_dte_addr_phys,
|
||||
.dma_addr_dte = &rk_dma_addr_dte,
|
||||
.dma_bit_mask = DMA_BIT_MASK(32),
|
||||
};
|
||||
|
||||
static struct rk_iommu_ops iommu_data_ops_v2 = {
|
||||
.pt_address = &rk_dte_pt_address_v2,
|
||||
.mk_dtentries = &rk_mk_dte_v2,
|
||||
.mk_ptentries = &rk_mk_pte_v2,
|
||||
.dte_addr_phys = &rk_dte_addr_phys_v2,
|
||||
.dma_addr_dte = &rk_dma_addr_dte_v2,
|
||||
.dma_bit_mask = DMA_BIT_MASK(40),
|
||||
};
|
||||
|
||||
static const struct of_device_id rk_iommu_dt_ids[] = {
|
||||
{ .compatible = "rockchip,iommu",
|
||||
.data = &iommu_data_ops_v1,
|
||||
},
|
||||
{ .compatible = "rockchip,iommu-v2",
|
||||
.data = &iommu_data_ops_v2,
|
||||
},
|
||||
{ .compatible = "rockchip,rk3568-iommu",
|
||||
.data = &iommu_data_ops_v2,
|
||||
},
|
||||
{ /* sentinel */ }
|
||||
};
|
||||
|
||||
static struct platform_driver rk_iommu_driver = {
|
||||
.probe = rk_iommu_probe,
|
||||
.shutdown = rk_iommu_shutdown,
|
||||
|
||||
Reference in New Issue
Block a user