diff --git a/drivers/video/rockchip/rga3/include/rga_dma_buf.h b/drivers/video/rockchip/rga3/include/rga_dma_buf.h index 1271621ac0e4..05baf63c9c92 100644 --- a/drivers/video/rockchip/rga3/include/rga_dma_buf.h +++ b/drivers/video/rockchip/rga3/include/rga_dma_buf.h @@ -18,11 +18,13 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr, int rga_virtual_memory_check(void *vaddr, u32 w, u32 h, u32 format, int fd); int rga_dma_memory_check(struct rga_dma_buffer *rga_dma_buffer, struct rga_img_info_t *img); -int rga_iommu_map_virt_addr(struct rga_memory_parm *memory_parm, - struct rga_dma_buffer *virt_dma_buf, - struct device *rga_dev, - struct mm_struct *mm); -void rga_iommu_unmap_virt_addr(struct rga_dma_buffer *virt_addr); +int rga_iommu_map_sgt(struct sg_table *sgt, size_t size, + struct rga_dma_buffer *buffer, + struct device *rga_dev); +int rga_iommu_map(phys_addr_t paddr, size_t size, + struct rga_dma_buffer *buffer, + struct device *rga_dev); +void rga_iommu_unmap(struct rga_dma_buffer *buffer); int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buffer, enum dma_data_direction dir, struct device *rga_dev); diff --git a/drivers/video/rockchip/rga3/include/rga_drv.h b/drivers/video/rockchip/rga3/include/rga_drv.h index 199166f4cc9c..261a11aa3916 100644 --- a/drivers/video/rockchip/rga3/include/rga_drv.h +++ b/drivers/video/rockchip/rga3/include/rga_drv.h @@ -145,7 +145,6 @@ struct rga_dma_buffer { void *vmap_ptr; struct iommu_domain *domain; - struct rga_iommu_dma_cookie *cookie; enum dma_data_direction dir; diff --git a/drivers/video/rockchip/rga3/rga_dma_buf.c b/drivers/video/rockchip/rga3/rga_dma_buf.c index 21050bbddd45..3f8779e95a0e 100644 --- a/drivers/video/rockchip/rga3/rga_dma_buf.c +++ b/drivers/video/rockchip/rga3/rga_dma_buf.c @@ -13,30 +13,15 @@ #include "rga_job.h" #include "rga_debugger.h" -#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) && \ - LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0) -#include -#endif - -/** - * rga_dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API - * page flags. - * @dir: Direction of DMA transfer - * @coherent: Is the DMA master cache-coherent? - * - * Return: corresponding IOMMU API page protection flags - */ -static int rga_dma_info_to_prot(enum dma_data_direction dir, bool coherent) +static int rga_dma_info_to_prot(enum dma_data_direction dir) { - int prot = coherent ? IOMMU_CACHE : 0; - switch (dir) { case DMA_BIDIRECTIONAL: - return prot | IOMMU_READ | IOMMU_WRITE; + return IOMMU_READ | IOMMU_WRITE; case DMA_TO_DEVICE: - return prot | IOMMU_READ; + return IOMMU_READ; case DMA_FROM_DEVICE: - return prot | IOMMU_WRITE; + return IOMMU_WRITE; default: return 0; } @@ -257,33 +242,13 @@ static dma_addr_t rga_iommu_dma_alloc_iova(struct iommu_domain *domain, return (dma_addr_t)iova << shift; } -static void rga_iommu_dma_free_iova(struct rga_iommu_dma_cookie *cookie, - dma_addr_t iova, size_t size) +static void rga_iommu_dma_free_iova(struct iommu_domain *domain, + dma_addr_t iova, size_t size) { + struct rga_iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; - free_iova_fast(iovad, iova_pfn(iovad, iova), - size >> iova_shift(iovad)); -} - -static inline size_t rga_iommu_map_sg(struct iommu_domain *domain, - unsigned long iova, struct scatterlist *sg, - unsigned int nents, int prot) -{ -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) - return iommu_map_sg_atomic(domain, iova, sg, nents, prot); -#else - return iommu_map_sg(domain, iova, sg, nents, prot); -#endif -} - -static inline bool rga_dev_is_dma_coherent(struct device *dev) -{ -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) - return dev_is_dma_coherent(dev); -#else - return dev->archdata.dma_coherent; -#endif + free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); } static inline struct iommu_domain *rga_iommu_get_dma_domain(struct device *dev) @@ -291,64 +256,103 @@ static inline struct iommu_domain *rga_iommu_get_dma_domain(struct device *dev) return iommu_get_domain_for_dev(dev); } -void rga_iommu_unmap_virt_addr(struct rga_dma_buffer *virt_dma_buf) +void rga_iommu_unmap(struct rga_dma_buffer *buffer) { - if (virt_dma_buf == NULL) + if (buffer == NULL) return; - if (virt_dma_buf->iova == 0) + if (buffer->iova == 0) return; - iommu_unmap(virt_dma_buf->domain, virt_dma_buf->iova, virt_dma_buf->size); - rga_iommu_dma_free_iova(virt_dma_buf->cookie, virt_dma_buf->iova, virt_dma_buf->size); + iommu_unmap(buffer->domain, buffer->iova, buffer->size); + rga_iommu_dma_free_iova(buffer->domain, buffer->iova, buffer->size); } -int rga_iommu_map_virt_addr(struct rga_memory_parm *memory_parm, - struct rga_dma_buffer *virt_dma_buf, - struct device *rga_dev, - struct mm_struct *mm) +int rga_iommu_map_sgt(struct sg_table *sgt, size_t size, + struct rga_dma_buffer *buffer, + struct device *rga_dev) { - unsigned long size; - size_t map_size; - bool coherent; - int ioprot; struct iommu_domain *domain = NULL; struct rga_iommu_dma_cookie *cookie; struct iova_domain *iovad; dma_addr_t iova; - struct sg_table *sgt = NULL; + size_t map_size; + unsigned long align_size; - coherent = rga_dev_is_dma_coherent(rga_dev); - domain = rga_iommu_get_dma_domain(rga_dev); - ioprot = rga_dma_info_to_prot(DMA_BIDIRECTIONAL, coherent); - - cookie = domain->iova_cookie; - iovad = &cookie->iovad; - size = iova_align(iovad, virt_dma_buf->size); - sgt = virt_dma_buf->sgt; if (sgt == NULL) { pr_err("can not map iommu, because sgt is null!\n"); - return -EFAULT; + return -EINVAL; } - if (DEBUGGER_EN(MSG)) - pr_info("iova_align size = %ld", size); + domain = rga_iommu_get_dma_domain(rga_dev); + cookie = domain->iova_cookie; + iovad = &cookie->iovad; + align_size = iova_align(iovad, size); - iova = rga_iommu_dma_alloc_iova(domain, size, rga_dev->coherent_dma_mask, rga_dev); + if (DEBUGGER_EN(MSG)) + pr_info("iova_align size = %ld", align_size); + + iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev); if (!iova) { pr_err("rga_iommu_dma_alloc_iova failed"); return -ENOMEM; } - map_size = rga_iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, ioprot); - if (map_size < size) { + map_size = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, + rga_dma_info_to_prot(DMA_BIDIRECTIONAL)); + if (map_size < align_size) { pr_err("iommu can not map sgt to iova"); + rga_iommu_dma_free_iova(domain, iova, align_size); return -EINVAL; } - virt_dma_buf->cookie = cookie; - virt_dma_buf->domain = domain; - virt_dma_buf->iova = iova; - virt_dma_buf->size = size; + buffer->domain = domain; + buffer->iova = iova; + buffer->size = align_size; + + return 0; +} + +int rga_iommu_map(phys_addr_t paddr, size_t size, + struct rga_dma_buffer *buffer, + struct device *rga_dev) +{ + int ret; + struct iommu_domain *domain = NULL; + struct rga_iommu_dma_cookie *cookie; + struct iova_domain *iovad; + dma_addr_t iova; + unsigned long align_size; + + if (paddr == 0) { + pr_err("can not map iommu, because phys_addr is 0!\n"); + return -EINVAL; + } + + domain = rga_iommu_get_dma_domain(rga_dev); + cookie = domain->iova_cookie; + iovad = &cookie->iovad; + align_size = iova_align(iovad, size); + + if (DEBUGGER_EN(MSG)) + pr_info("iova_align size = %ld", align_size); + + iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev); + if (!iova) { + pr_err("rga_iommu_dma_alloc_iova failed"); + return -ENOMEM; + } + + ret = iommu_map(domain, iova, paddr, align_size, + rga_dma_info_to_prot(DMA_BIDIRECTIONAL)); + if (ret) { + pr_err("iommu can not map phys_addr to iova"); + rga_iommu_dma_free_iova(domain, iova, align_size); + return ret; + } + + buffer->domain = domain; + buffer->iova = iova; + buffer->size = align_size; return 0; } diff --git a/drivers/video/rockchip/rga3/rga_mm.c b/drivers/video/rockchip/rga3/rga_mm.c index 405de758578a..9e7c48859ccc 100644 --- a/drivers/video/rockchip/rga3/rga_mm.c +++ b/drivers/video/rockchip/rga3/rga_mm.c @@ -313,6 +313,11 @@ static int rga_mm_check_range_sgt(struct sg_table *sgt) return 1; } +static inline int rga_mm_check_range_phys_addr(phys_addr_t paddr, size_t size) +{ + return ((paddr + size) <= 0xffffffff); +} + static inline bool rga_mm_check_contiguous_sgt(struct sg_table *sgt) { if (sgt->orig_nents == 1) @@ -426,7 +431,7 @@ static void rga_mm_unmap_virt_addr(struct rga_internal_buffer *internal_buffer) for (i = 0; i < internal_buffer->dma_buffer_size; i++) if (internal_buffer->dma_buffer[i].core == RGA3_SCHEDULER_CORE0 || internal_buffer->dma_buffer[i].core == RGA3_SCHEDULER_CORE1) - rga_iommu_unmap_virt_addr(&internal_buffer->dma_buffer[i]); + rga_iommu_unmap(&internal_buffer->dma_buffer[i]); else if (internal_buffer->dma_buffer[i].core != 0) dma_unmap_sg(internal_buffer->dma_buffer[i].dev, internal_buffer->dma_buffer[i].sgt->sgl, @@ -436,6 +441,7 @@ static void rga_mm_unmap_virt_addr(struct rga_internal_buffer *internal_buffer) for (i = 0; i < internal_buffer->dma_buffer_size; i++) rga_free_sgt(&internal_buffer->dma_buffer[i]); kfree(internal_buffer->dma_buffer); + internal_buffer->dma_buffer = NULL; internal_buffer->dma_buffer_size = 0; rga_free_virt_addr(&internal_buffer->virt_addr); @@ -511,10 +517,10 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer, if (scheduler->core == RGA3_SCHEDULER_CORE0 || scheduler->core == RGA3_SCHEDULER_CORE1) { - ret = rga_iommu_map_virt_addr(&internal_buffer->memory_parm, - &internal_buffer->dma_buffer[i], - scheduler->dev, - internal_buffer->current_mm); + ret = rga_iommu_map_sgt(internal_buffer->dma_buffer[i].sgt, + internal_buffer->dma_buffer[i].size, + &internal_buffer->dma_buffer[i], + scheduler->dev); if (ret < 0) { pr_err("%s core[%d] iommu_map virtual address error!\n", __func__, scheduler->core); @@ -545,7 +551,7 @@ unmap_virt_addr: for (i = 0; i < internal_buffer->dma_buffer_size; i++) if (internal_buffer->dma_buffer[i].core == RGA3_SCHEDULER_CORE0 || internal_buffer->dma_buffer[i].core == RGA3_SCHEDULER_CORE1) - rga_iommu_unmap_virt_addr(&internal_buffer->dma_buffer[i]); + rga_iommu_unmap(&internal_buffer->dma_buffer[i]); else if (internal_buffer->dma_buffer[i].core != 0) dma_unmap_sg(internal_buffer->dma_buffer[i].dev, internal_buffer->dma_buffer[i].sgt->sgl, @@ -565,6 +571,85 @@ put_current_mm: return ret; } +static void rga_mm_unmap_phys_addr(struct rga_internal_buffer *internal_buffer) +{ + int i; + + WARN_ON(internal_buffer->dma_buffer == NULL); + + for (i = 0; i < internal_buffer->dma_buffer_size; i++) { + if (internal_buffer->dma_buffer[i].core == RGA2_SCHEDULER_CORE0) + continue; + + rga_iommu_unmap(&internal_buffer->dma_buffer[i]); + } + + internal_buffer->phys_addr = 0; + + kfree(internal_buffer->dma_buffer); + internal_buffer->dma_buffer = NULL; + internal_buffer->dma_buffer_size = 0; +} + +static int rga_mm_map_phys_addr(struct rga_external_buffer *external_buffer, + struct rga_internal_buffer *internal_buffer, + struct rga_job *job) +{ + int ret, i; + struct rga_scheduler_t *scheduler = NULL; + + internal_buffer->dma_buffer_size = job ? 1 : rga_drvdata->num_of_scheduler; + internal_buffer->dma_buffer = kcalloc(internal_buffer->dma_buffer_size, + sizeof(struct rga_dma_buffer), GFP_KERNEL); + if (internal_buffer->dma_buffer == NULL) { + pr_err("%s alloc internal_buffer error!\n", __func__); + return -ENOMEM; + } + + internal_buffer->phys_addr = external_buffer->memory; + + if (internal_buffer->memory_parm.size) + internal_buffer->size = internal_buffer->memory_parm.size; + else + internal_buffer->size = + rga_image_size_cal(internal_buffer->memory_parm.width, + internal_buffer->memory_parm.height, + internal_buffer->memory_parm.format, + NULL, NULL, NULL); + + internal_buffer->mm_flag |= RGA_MEM_PHYSICAL_CONTIGUOUS; + if (rga_mm_check_range_phys_addr(internal_buffer->phys_addr, internal_buffer->size)) + internal_buffer->mm_flag |= RGA_MEM_UNDER_4G; + + for (i = 0; i < internal_buffer->dma_buffer_size; i++) { + scheduler = job ? job->scheduler : rga_drvdata->scheduler[i]; + + /* RGA_MMU can directly access physical addresses. */ + if (scheduler->core == RGA2_SCHEDULER_CORE0) + continue; + + ret = rga_iommu_map(internal_buffer->phys_addr, + internal_buffer->size, + &internal_buffer->dma_buffer[i], + scheduler->dev); + if (ret < 0) { + pr_err("%s core[%d] map phys_addr error!\n", + __func__, scheduler->core); + goto UNMAP_PHYS_ADDR; + } + + internal_buffer->dma_buffer[i].core = scheduler->core; + internal_buffer->dma_buffer[i].dev = scheduler->dev; + } + + return 0; + +UNMAP_PHYS_ADDR: + rga_mm_unmap_phys_addr(internal_buffer); + + return ret; +} + static int rga_mm_unmap_buffer(struct rga_internal_buffer *internal_buffer) { switch (internal_buffer->type) { @@ -576,7 +661,7 @@ static int rga_mm_unmap_buffer(struct rga_internal_buffer *internal_buffer) rga_mm_unmap_virt_addr(internal_buffer); break; case RGA_PHYSICAL_ADDRESS: - internal_buffer->phys_addr = 0; + rga_mm_unmap_phys_addr(internal_buffer); break; default: pr_err("Illegal external buffer!\n"); @@ -626,16 +711,13 @@ static int rga_mm_map_buffer(struct rga_external_buffer *external_buffer, case RGA_PHYSICAL_ADDRESS: internal_buffer->type = RGA_PHYSICAL_ADDRESS; - internal_buffer->phys_addr = external_buffer->memory; + ret = rga_mm_map_phys_addr(external_buffer, internal_buffer, job); + if (ret < 0) { + pr_err("%s iommu_map physical address error!\n", __func__); + return ret; + } - if (internal_buffer->memory_parm.size) - internal_buffer->size = internal_buffer->memory_parm.size; - else - internal_buffer->size = - rga_image_size_cal(internal_buffer->memory_parm.width, - internal_buffer->memory_parm.height, - internal_buffer->memory_parm.format, - NULL, NULL, NULL); + internal_buffer->mm_flag |= RGA_MEM_NEED_USE_IOMMU; break; default: pr_err("Illegal external buffer!\n"); @@ -1176,7 +1258,16 @@ static int rga_mm_get_buffer_info(struct rga_job *job, break; case RGA_PHYSICAL_ADDRESS: - addr = internal_buffer->phys_addr; + if (job->core == RGA3_SCHEDULER_CORE0 || + job->core == RGA3_SCHEDULER_CORE1) { + addr = rga_mm_lookup_iova(internal_buffer, job->core); + if (addr == 0) { + pr_err("core[%d] lookup phys_addr iova error!\n", job->core); + return -EINVAL; + } + } else { + addr = internal_buffer->phys_addr; + } break; default: pr_err("Illegal external buffer!\n");