From 2349ea26cbe4573ec1771f133e5c1f207c6f3494 Mon Sep 17 00:00:00 2001 From: Yandong Lin Date: Mon, 3 Jul 2023 20:32:01 +0800 Subject: [PATCH] iommu/rockchip: support av1d iommu Signed-off-by: Yandong Lin Change-Id: Ic7cbb95b83cef361ea4d62c969d5585318ec0dac --- drivers/iommu/Makefile | 4 +- drivers/iommu/rockchip-iommu-av1d.c | 795 ++++++++++++++++++++++++++ drivers/iommu/rockchip-iommu.c | 7 +- include/soc/rockchip/rockchip_iommu.h | 9 + 4 files changed, 813 insertions(+), 2 deletions(-) create mode 100644 drivers/iommu/rockchip-iommu-av1d.c diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index cc9f381013c3..4881d3232fe2 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -19,7 +19,9 @@ obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o -obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o +obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip_iommu.o +rockchip_iommu-objs := rockchip-iommu.o +rockchip_iommu-$(CONFIG_ROCKCHIP_MPP_AV1DEC) += rockchip-iommu-av1d.o obj-$(CONFIG_SUN50I_IOMMU) += sun50i-iommu.o obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o diff --git a/drivers/iommu/rockchip-iommu-av1d.c b/drivers/iommu/rockchip-iommu-av1d.c new file mode 100644 index 000000000000..6286e2cac08d --- /dev/null +++ b/drivers/iommu/rockchip-iommu-av1d.c @@ -0,0 +1,795 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Compatible with the IOMMU of av1 decode + * + * Module Authors: Yandong Lin + * Simon Xue + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct av1d_iommu { + struct device *dev; + void __iomem **bases; + int num_mmu; + int num_irq; + struct clk_bulk_data *clocks; + int num_clocks; + struct iommu_device iommu; + struct iommu_domain *domain; /* domain to which iommu is attached */ + struct iommu_group *group; + + u32 *dt; /* page directory table */ + dma_addr_t dt_dma; + spinlock_t iommus_lock; /* lock for iommus list */ + spinlock_t dt_lock; /* lock for modifying page directory table */ + /* for av1 iommu */ + u64 *pta; /* page directory table */ + dma_addr_t pta_dma; +}; + +#define RK_IOMMU_AV1 0xa +#define NUM_DT_ENTRIES 1024 +#define NUM_PT_ENTRIES 1024 + +#define SPAGE_ORDER 12 +#define SPAGE_SIZE (1 << SPAGE_ORDER) + +/* av1 iommu regs address */ +#define AV1_CLOCK_CTRL_BASE 0x0 +#define AV1_IDLE_ST_BASE 0x4 +#define AV1_MMU_CONFIG0_BASE 0x184 +#define AV1_MMU_CONFIG1_BASE 0x1ac +#define AV1_MMU_AHB_EXCEPTION_BASE 0x380 +#define AV1_MMU_AHB_STATUS_BASE 0x384 +#define AV1_MMU_AHB_CONTROL_BASE 0x388 +#define AV1_MMU_AHB_TBL_ARRAY_BASE_L_BASE 0x38C +#define AV1_MMU_AHB_TBL_ARRAY_BASE_H_BASE 0x390 +#define AV1_MMU_AHB_CTX_PD_BASE 0x3b4 +#define AV1_MMU_BUTT_BASE 0xffff + +/* MMU register offsets */ +#define AV1_MMU_FLUSH_BASE 0x184 +#define AV1_MMU_BIT_FLUSH BIT(4) + +#define AV1_MMU_PAGE_FAULT_ADDR 0x380 +#define AV1_MMU_STATUS_BASE 0x384 /* IRQ status */ + +#define AV1_MMU_EN_BASE 0x388 +#define AV1_MMU_BIT_ENABLE BIT(0) + +#define AV1_MMU_OUT_OF_BOUND BIT(28) +/* Irq mask */ +#define AV1_MMU_IRQ_MASK 0x7 + +#define AV1_DTE_PT_ADDRESS_MASK 0xffffffc0 +#define AV1_DTE_PT_VALID BIT(0) + +#define AV1_PAGE_DESC_LO_MASK 0xfffff000 +#define AV1_PAGE_DESC_HI_MASK GENMASK_ULL(39, 32) +#define AV1_PAGE_DESC_HI_SHIFT (32-4) + +#define AV1_IOMMU_PGSIZE_BITMAP 0x007ff000 + +static inline phys_addr_t av1_dte_pt_address(u32 dte) +{ + return (phys_addr_t)dte & AV1_DTE_PT_ADDRESS_MASK; +} + +static inline u32 av1_mk_dte(dma_addr_t pt_dma) +{ + return (pt_dma) | AV1_DTE_PT_VALID; +} + +#define AV1_PTE_PAGE_ADDRESS_MASK 0xfffffff0 +#define AV1_PTE_PAGE_WRITABLE BIT(2) +#define AV1_PTE_PAGE_VALID BIT(0) + +static struct device *dma_dev; + +static inline phys_addr_t av1_pte_page_address(u32 pte) +{ + u64 pte_av1 = pte; + + pte_av1 = ((pte_av1 & AV1_PAGE_DESC_HI_MASK) << AV1_PAGE_DESC_HI_SHIFT) | + (pte_av1 & AV1_PAGE_DESC_LO_MASK); + + return (phys_addr_t)pte_av1; +} + +static u32 av1_mk_pte(phys_addr_t page, int prot) +{ + u32 flags = 0; + + flags |= (prot & IOMMU_WRITE) ? AV1_PTE_PAGE_WRITABLE : 0; + page = (page & AV1_PAGE_DESC_LO_MASK) | + ((page & AV1_PAGE_DESC_HI_MASK) >> AV1_PAGE_DESC_HI_SHIFT); + page &= AV1_PTE_PAGE_ADDRESS_MASK; + + return page | flags | AV1_PTE_PAGE_VALID; +} + +#define AV1_DTE_PT_VALID BIT(0) + +static inline bool av1_dte_is_pt_valid(u32 dte) +{ + return dte & AV1_DTE_PT_VALID; +} + +static inline bool av1_pte_is_page_valid(u32 pte) +{ + return pte & AV1_PTE_PAGE_VALID; +} + +static u32 av1_mk_pte_invalid(u32 pte) +{ + return pte & ~AV1_PTE_PAGE_VALID; +} + +#define AV1_MASTER_TLB_MASK GENMASK_ULL(31, 10) +/* mode 0 : 4k */ +#define AV1_PTA_4K_MODE 0 + +static u64 av1_mk_pta(dma_addr_t dt_dma) +{ + u64 val = (dt_dma & AV1_MASTER_TLB_MASK) | AV1_PTA_4K_MODE; + + return val; +} + +static void av1d_iommu_disable(struct av1d_iommu *iommu) +{ + int i; + + /* Ignore error while disabling, just keep going */ + WARN_ON(clk_bulk_enable(iommu->num_clocks, iommu->clocks)); + for (i = 0; i < iommu->num_mmu; i++) + writel(0, iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE); + + clk_bulk_disable(iommu->num_clocks, iommu->clocks); +} + +static int av1_iommu_disable(struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + + if (!av1d_iommu->domain) + return 0; + + av1d_iommu_disable(av1d_iommu); + + return 0; +} + +static int av1d_iommu_enable(struct av1d_iommu *av1d_iommu) +{ + int ret, i; + + ret = clk_bulk_enable(av1d_iommu->num_clocks, av1d_iommu->clocks); + if (ret) + return ret; + + for (i = 0; i < av1d_iommu->num_mmu; i++) { + u32 val = readl(av1d_iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE); + + if (!(val & AV1_MMU_BIT_ENABLE)) { + writel(av1d_iommu->pta_dma, + av1d_iommu->bases[i] + AV1_MMU_AHB_TBL_ARRAY_BASE_L_BASE); + writel(AV1_MMU_OUT_OF_BOUND, av1d_iommu->bases[i] + AV1_MMU_CONFIG1_BASE); + writel(AV1_MMU_BIT_ENABLE, + av1d_iommu->bases[i] + AV1_MMU_AHB_EXCEPTION_BASE); + writel(AV1_MMU_BIT_ENABLE, av1d_iommu->bases[i] + AV1_MMU_AHB_CONTROL_BASE); + } + } + clk_bulk_disable(av1d_iommu->num_clocks, av1d_iommu->clocks); + return ret; +} + +static int av1_iommu_enable(struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + + if (!av1d_iommu->domain) + return 0; + + return av1d_iommu_enable(av1d_iommu); +} + +static inline void av1_table_flush(dma_addr_t dma, unsigned int count) +{ + size_t size = count * sizeof(u32); /* count of u32 entry */ + + dma_sync_single_for_device(dma_dev, dma, size, DMA_TO_DEVICE); +} + +#define AV1_IOVA_DTE_MASK 0xffc00000 +#define AV1_IOVA_DTE_SHIFT 22 +#define AV1_IOVA_PTE_MASK 0x003ff000 +#define AV1_IOVA_PTE_SHIFT 12 +#define AV1_IOVA_PAGE_MASK 0x00000fff +#define AV1_IOVA_PAGE_SHIFT 0 + +static u32 av1_iova_dte_index(dma_addr_t iova) +{ + return (u32)(iova & AV1_IOVA_DTE_MASK) >> AV1_IOVA_DTE_SHIFT; +} + +static u32 av1_iova_pte_index(dma_addr_t iova) +{ + return (u32)(iova & AV1_IOVA_PTE_MASK) >> AV1_IOVA_PTE_SHIFT; +} + +static u32 av1_iova_page_offset(dma_addr_t iova) +{ + return (u32)(iova & AV1_IOVA_PAGE_MASK) >> AV1_IOVA_PAGE_SHIFT; +} + +static u32 av1_iommu_read(void __iomem *base, u32 offset) +{ + return readl(base + offset); +} + +static void av1_iommu_write(void __iomem *base, u32 offset, u32 value) +{ + writel(value, base + offset); +} + + +static void av1_iommu_flush_tlb_all(struct iommu_domain *domain, struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + unsigned long flags; + int i, ret; + + if (WARN_ON(!av1d_iommu)) + return; + + spin_lock_irqsave(&av1d_iommu->iommus_lock, flags); + ret = pm_runtime_get_if_in_use(av1d_iommu->dev); + if (WARN_ON_ONCE(ret < 0)) { + spin_unlock_irqrestore(&av1d_iommu->iommus_lock, flags); + return; + } + if (ret) { + WARN_ON(clk_bulk_enable(av1d_iommu->num_clocks, av1d_iommu->clocks)); + for (i = 0; i < av1d_iommu->num_mmu; i++) { + writel(AV1_MMU_BIT_FLUSH, + av1d_iommu->bases[i] + AV1_MMU_FLUSH_BASE); + writel(0, av1d_iommu->bases[i] + AV1_MMU_FLUSH_BASE); + } + clk_bulk_disable(av1d_iommu->num_clocks, av1d_iommu->clocks); + pm_runtime_put(av1d_iommu->dev); + } + spin_unlock_irqrestore(&av1d_iommu->iommus_lock, flags); +} + +static irqreturn_t av1_iommu_irq(int irq, void *dev_id) +{ + struct av1d_iommu *av1d_iommu = dev_id; + u32 int_status; + dma_addr_t iova; + irqreturn_t ret = IRQ_NONE; + int i, err; + + err = pm_runtime_get_if_in_use(av1d_iommu->dev); + if (!err || WARN_ON_ONCE(err < 0)) + return ret; + + if (WARN_ON(clk_bulk_enable(av1d_iommu->num_clocks, av1d_iommu->clocks))) + goto out; + + for (i = 0; i < av1d_iommu->num_mmu; i++) { + int_status = av1_iommu_read(av1d_iommu->bases[i], AV1_MMU_STATUS_BASE); + if (int_status & AV1_MMU_IRQ_MASK) { + dev_err(av1d_iommu->dev, "unexpected int_status=%08x\n", int_status); + iova = av1_iommu_read(av1d_iommu->bases[i], AV1_MMU_PAGE_FAULT_ADDR); + + if (av1d_iommu->domain) + report_iommu_fault(av1d_iommu->domain, av1d_iommu->dev, + iova, int_status); + else + dev_err(av1d_iommu->dev, + "Page fault while av1d_iommu not attached to domain?\n"); + } + av1_iommu_write(av1d_iommu->bases[i], AV1_MMU_STATUS_BASE, 0); + ret = IRQ_HANDLED; + } + + clk_bulk_disable(av1d_iommu->num_clocks, av1d_iommu->clocks); + +out: + pm_runtime_put(av1d_iommu->dev); + return ret; +} + +static int av1d_iommu_init(struct av1d_iommu *av1d_iommu) +{ + if (!dma_dev) + return -ENAVAIL; + + /* + * av132xx iommus use a 2 level pagetable. + * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. + * Allocate one 4 KiB page for each table. + */ + av1d_iommu->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); + if (!av1d_iommu->dt) + return -1; + + av1d_iommu->dt_dma = dma_map_single(dma_dev, av1d_iommu->dt, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, av1d_iommu->dt_dma)) { + dev_err(dma_dev, "DMA map error for DT\n"); + goto err_free_dt; + } + + av1d_iommu->pta = (u64 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); + if (!av1d_iommu->pta) + goto err_unmap_dt; + + av1d_iommu->pta_dma = dma_map_single(dma_dev, av1d_iommu->pta, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, av1d_iommu->pta_dma)) { + dev_err(dma_dev, "DMA map error for PTA\n"); + goto err_free_pta; + } + av1d_iommu->pta[0] = av1_mk_pta(av1d_iommu->dt_dma); + + av1_table_flush(av1d_iommu->pta_dma, 1024); + av1_table_flush(av1d_iommu->dt_dma, NUM_DT_ENTRIES); + + spin_lock_init(&av1d_iommu->iommus_lock); + spin_lock_init(&av1d_iommu->dt_lock); + + return 0; +err_free_pta: + free_page((unsigned long)av1d_iommu->pta); +err_unmap_dt: + dma_unmap_single(dma_dev, av1d_iommu->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); +err_free_dt: + free_page((unsigned long)av1d_iommu->dt); + + return -1; +} + +static phys_addr_t av1_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova, + struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + unsigned long flags; + phys_addr_t pt_phys, phys = 0; + u32 dte, pte; + u32 *page_table; + + spin_lock_irqsave(&av1d_iommu->dt_lock, flags); + + dte = av1d_iommu->dt[av1_iova_dte_index(iova)]; + if (!av1_dte_is_pt_valid(dte)) + goto out; + + pt_phys = av1_dte_pt_address(dte); + page_table = (u32 *)phys_to_virt(pt_phys); + pte = page_table[av1_iova_pte_index(iova)]; + if (!av1_pte_is_page_valid(pte)) + goto out; + + phys = av1_pte_page_address(pte) + av1_iova_page_offset(iova); +out: + spin_unlock_irqrestore(&av1d_iommu->dt_lock, flags); + + return phys; +} + +static u32 *av1_dte_get_page_table(struct av1d_iommu *av1d_iommu, dma_addr_t iova) +{ + u32 *page_table, *dte_addr; + u32 dte_index, dte; + phys_addr_t pt_phys; + dma_addr_t pt_dma; + + assert_spin_locked(&av1d_iommu->dt_lock); + + dte_index = av1_iova_dte_index(iova); + dte_addr = &av1d_iommu->dt[dte_index]; + dte = *dte_addr; + if (av1_dte_is_pt_valid(dte)) + goto done; + + page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32); + if (!page_table) + return ERR_PTR(-ENOMEM); + + pt_dma = dma_map_single(dma_dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev, pt_dma)) { + dev_err(dma_dev, "DMA mapping error while allocating page table\n"); + free_page((unsigned long)page_table); + return ERR_PTR(-ENOMEM); + } + + dte = av1_mk_dte(pt_dma); + *dte_addr = dte; + + av1_table_flush(pt_dma, NUM_PT_ENTRIES); + av1_table_flush(av1d_iommu->dt_dma + dte_index * sizeof(u32), 1); +done: + pt_phys = av1_dte_pt_address(dte); + + return (u32 *)phys_to_virt(pt_phys); +} + +static size_t av1_iommu_unmap_iova(struct av1d_iommu *av1d_iommu, + u32 *pte_addr, dma_addr_t pte_dma, + size_t size) +{ + unsigned int pte_count; + unsigned int pte_total = size / SPAGE_SIZE; + + assert_spin_locked(&av1d_iommu->dt_lock); + + for (pte_count = 0; pte_count < pte_total; pte_count++) { + u32 pte = pte_addr[pte_count]; + + if (!av1_pte_is_page_valid(pte)) + break; + + pte_addr[pte_count] = av1_mk_pte_invalid(pte); + } + + av1_table_flush(pte_dma, pte_count); + + return pte_count * SPAGE_SIZE; +} + +static int av1_iommu_map_iova(struct av1d_iommu *av1_iommu, u32 *pte_addr, + dma_addr_t pte_dma, dma_addr_t iova, + phys_addr_t paddr, size_t size, int prot) +{ + unsigned int pte_count; + unsigned int pte_total = size / SPAGE_SIZE; + phys_addr_t page_phys; + + assert_spin_locked(&av1_iommu->dt_lock); + + for (pte_count = 0; pte_count < pte_total; pte_count++) { + u32 pte = pte_addr[pte_count]; + + if (av1_pte_is_page_valid(pte)) + goto unwind; + + pte_addr[pte_count] = av1_mk_pte(paddr, prot); + + paddr += SPAGE_SIZE; + } + + av1_table_flush(pte_dma, pte_total); + + return 0; +unwind: + /* Unmap the range of iovas that we just mapped */ + av1_iommu_unmap_iova(av1_iommu, pte_addr, pte_dma, + pte_count * SPAGE_SIZE); + + iova += pte_count * SPAGE_SIZE; + page_phys = av1_pte_page_address(pte_addr[pte_count]); + pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n", + &iova, &page_phys, &paddr, prot); + + return -EADDRINUSE; +} + +static size_t av1_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, + size_t size, struct iommu_iotlb_gather *gather, struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + unsigned long flags; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; + phys_addr_t pt_phys; + u32 dte; + u32 *pte_addr; + size_t unmap_size; + + spin_lock_irqsave(&av1d_iommu->dt_lock, flags); + + dte = av1d_iommu->dt[av1_iova_dte_index(iova)]; + /* Just return 0 if iova is unmapped */ + if (!av1_dte_is_pt_valid(dte)) { + spin_unlock_irqrestore(&av1d_iommu->dt_lock, flags); + return 0; + } + + pt_phys = av1_dte_pt_address(dte); + pte_addr = (u32 *)phys_to_virt(pt_phys) + av1_iova_pte_index(iova); + pte_dma = pt_phys + av1_iova_pte_index(iova) * sizeof(u32); + unmap_size = av1_iommu_unmap_iova(av1d_iommu, pte_addr, pte_dma, size); + + spin_unlock_irqrestore(&av1d_iommu->dt_lock, flags); + + return unmap_size; +} + +static int av1_iommu_map(struct iommu_domain *domain, unsigned long _iova, phys_addr_t paddr, + size_t size, int prot, gfp_t gfp, struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + unsigned long flags; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; + u32 *page_table, *pte_addr; + u32 dte, pte_index; + int ret; + + if (WARN_ON(!av1d_iommu)) + return -1; + + spin_lock_irqsave(&av1d_iommu->dt_lock, flags); + + page_table = av1_dte_get_page_table(av1d_iommu, iova); + if (IS_ERR(page_table)) { + spin_unlock_irqrestore(&av1d_iommu->dt_lock, flags); + return PTR_ERR(page_table); + } + + dte = av1d_iommu->dt[av1_iova_dte_index(iova)]; + pte_index = av1_iova_pte_index(iova); + pte_addr = &page_table[pte_index]; + pte_dma = av1_dte_pt_address(dte) + pte_index * sizeof(u32); + ret = av1_iommu_map_iova(av1d_iommu, pte_addr, pte_dma, iova, + paddr, size, prot); + + spin_unlock_irqrestore(&av1d_iommu->dt_lock, flags); + + return ret; +} + +static void av1_iommu_detach_device(struct iommu_domain *domain, struct device *dev) +{ + struct av1d_iommu *iommu = (struct av1d_iommu *)dev_get_platdata(dev); + int ret; + + if (WARN_ON(!iommu)) + return; + + dev_dbg(dev, "Detaching from iommu domain\n"); + + if (!iommu->domain) + return; + + ret = pm_runtime_get_if_in_use(iommu->dev); + WARN_ON_ONCE(ret < 0); + if (ret > 0) { + av1d_iommu_disable(iommu); + pm_runtime_put(iommu->dev); + } + iommu->domain = NULL; +} + +static int av1_iommu_attach_device(struct iommu_domain *domain, struct device *dev) +{ + struct av1d_iommu *iommu = (struct av1d_iommu *)dev_get_platdata(dev); + int ret; + + if (WARN_ON(!iommu)) + return -ENODEV; + + if (iommu->domain) + av1_iommu_detach_device(iommu->domain, dev); + + iommu->domain = domain; + + /* Attach NULL for disable iommu */ + if (!domain) + return 0; + + ret = pm_runtime_get_if_in_use(iommu->dev); + if (!ret || WARN_ON_ONCE(ret < 0)) + return 0; + + ret = av1d_iommu_enable(iommu); + if (ret) + av1_iommu_detach_device(iommu->domain, dev); + + pm_runtime_put(iommu->dev); + + return ret; +} + +static void av1_iommu_domain_free(struct iommu_domain *domain, struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + int i; + + for (i = 0; i < NUM_DT_ENTRIES; i++) { + u32 dte = av1d_iommu->dt[i]; + + if (av1_dte_is_pt_valid(dte)) { + phys_addr_t pt_phys = av1_dte_pt_address(dte); + u32 *page_table = phys_to_virt(pt_phys); + + dma_unmap_single(dma_dev, pt_phys, + SPAGE_SIZE, DMA_TO_DEVICE); + free_page((unsigned long)page_table); + } + } + + dma_unmap_single(dma_dev, av1d_iommu->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); + free_page((unsigned long)av1d_iommu->dt); + + dma_unmap_single(dma_dev, av1d_iommu->pta_dma, + SPAGE_SIZE, DMA_TO_DEVICE); + free_page((unsigned long)av1d_iommu->pta); + + kfree(av1d_iommu); +} + +static int av1_iommu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct av1d_iommu *iommu; + struct resource *res; + int num_res = pdev->num_resources; + int err, i; + + iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); + if (!iommu) + return -ENOMEM; + + iommu->dev = dev; + iommu->num_mmu = 0; + + iommu->bases = devm_kcalloc(dev, num_res, sizeof(*iommu->bases), + GFP_KERNEL); + if (!iommu->bases) + return -ENOMEM; + + for (i = 0; i < num_res; i++) { + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + if (!res) + continue; + iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(iommu->bases[i])) + continue; + iommu->num_mmu++; + } + if (iommu->num_mmu == 0) { + dev_err(dev, "get iommu base failed\n"); + return PTR_ERR(iommu->bases[0]); + } + + iommu->num_irq = platform_irq_count(pdev); + if (iommu->num_irq < 0) { + dev_err(dev, "get av1 iommu irq num failed\n"); + return iommu->num_irq; + } + + err = devm_clk_bulk_get_all(dev, &iommu->clocks); + if (err >= 0) + iommu->num_clocks = err; + else if (err == -ENOENT) + iommu->num_clocks = 0; + else { + dev_err(dev, "get av1 iommu clocks failed\n"); + return err; + } + + err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks); + if (err) + return err; + + if (!dma_dev) + dma_dev = &pdev->dev; + + pm_runtime_enable(dev); + + for (i = 0; i < iommu->num_irq; i++) { + int irq = platform_get_irq(pdev, i); + + if (irq < 0) { + err = -ENODEV; + dev_err(dev, "av1 iommu get irq failed\n"); + goto err_disable_runtime; + } + + err = devm_request_irq(iommu->dev, irq, av1_iommu_irq, + IRQF_SHARED, dev_name(dev), iommu); + if (err) { + dev_err(dev, "av1 iommu request irq handle failed\n"); + goto err_disable_runtime; + } + + } + + err = av1d_iommu_init(iommu); + if (err) { + dev_err(dev, "av1 iommu init failed err %d\n", err); + goto err_disable_runtime; + } + + /* store av1d-iommu data to dev->platform_data */ + dev->platform_data = iommu; + + return 0; +err_disable_runtime: + pm_runtime_disable(dev); + clk_bulk_unprepare(iommu->num_clocks, iommu->clocks); + + return err; +} + +static void av1_iommu_shutdown(struct platform_device *pdev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(&pdev->dev); + int i; + + for (i = 0; i < av1d_iommu->num_irq; i++) { + int irq = platform_get_irq(pdev, i); + + devm_free_irq(av1d_iommu->dev, irq, av1d_iommu); + } + + pm_runtime_force_suspend(&pdev->dev); +} + +static int __maybe_unused av1_iommu_suspend(struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + + if (!av1d_iommu->domain) + return 0; + + av1d_iommu_disable(av1d_iommu); + return 0; +} + +static int __maybe_unused av1_iommu_resume(struct device *dev) +{ + struct av1d_iommu *av1d_iommu = (struct av1d_iommu *)dev_get_platdata(dev); + + if (!av1d_iommu->domain) + return 0; + + return av1d_iommu_enable(av1d_iommu); +} + +static struct third_iommu_ops_wrap av1d_iommu_ops = { + .attach_dev = av1_iommu_attach_device, + .detach_dev = av1_iommu_detach_device, + .map = av1_iommu_map, + .unmap = av1_iommu_unmap, + .flush_iotlb_all = av1_iommu_flush_tlb_all, + .iova_to_phys = av1_iommu_iova_to_phys, + .free = av1_iommu_domain_free, + .enable = av1_iommu_enable, + .disable = av1_iommu_disable, + .shutdown = av1_iommu_shutdown, + .suspend = av1_iommu_suspend, + .resume = av1_iommu_resume, + .probe = av1_iommu_probe, +}; + +struct third_iommu_ops_wrap *av1d_iommu_get_ops(void) +{ + return &av1d_iommu_ops; +} diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 131a7a5d55d6..e940646dde61 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1597,7 +1597,9 @@ static int rk_iommu_probe(struct platform_device *pdev) return -EINVAL; if (device_property_match_string(dev, "compatible", "rockchip,iommu-av1d") >= 0) { - iommu->opt_ops = NULL; + iommu->opt_ops = av1d_iommu_get_ops(); + if (iommu->opt_ops) + dev_info(dev, "av1d iommu enabled\n"); goto alloc_group; } @@ -1825,6 +1827,9 @@ static const struct of_device_id rk_iommu_dt_ids[] = { { .compatible = "rockchip,rk3568-iommu", .data = &iommu_data_ops_v2, }, + { .compatible = "rockchip,iommu-av1d", + .data = &iommu_data_ops_v2, + }, { /* sentinel */ } }; diff --git a/include/soc/rockchip/rockchip_iommu.h b/include/soc/rockchip/rockchip_iommu.h index 4ed4328078b0..bb86f5a1c26b 100644 --- a/include/soc/rockchip/rockchip_iommu.h +++ b/include/soc/rockchip/rockchip_iommu.h @@ -77,4 +77,13 @@ static inline int rockchip_iommu_force_reset(struct device *dev) } #endif +#if IS_ENABLED(CONFIG_ROCKCHIP_MPP_AV1DEC) +struct third_iommu_ops_wrap *av1d_iommu_get_ops(void); +#else +static inline struct third_iommu_ops_wrap *av1d_iommu_get_ops(void) +{ + return NULL; +} +#endif + #endif