From d4a3012c2476400a7fa77dd204c2266c375b8961 Mon Sep 17 00:00:00 2001 From: Yu Qiaowei Date: Wed, 16 Apr 2025 15:16:49 +0800 Subject: [PATCH] video: rockchip: rga3: remove dependency on CONFIG_IOMMU_IOVA Since 'CONFIG_IOMMU_IOVA' is often disabled on 32-bit systems, causing iommu_map API calls to fail, the DMA mapping API is used instead. Change-Id: I7226ddf6a04e0af4c6b1b029bb5a32ede04c0810 Signed-off-by: Yu Qiaowei --- .../video/rockchip/rga3/include/rga_dma_buf.h | 18 +- drivers/video/rockchip/rga3/include/rga_drv.h | 11 +- drivers/video/rockchip/rga3/include/rga_mm.h | 2 +- drivers/video/rockchip/rga3/rga_debugger.c | 4 +- drivers/video/rockchip/rga3/rga_dma_buf.c | 248 +++++------------- drivers/video/rockchip/rga3/rga_mm.c | 202 +++++++------- 6 files changed, 192 insertions(+), 293 deletions(-) diff --git a/drivers/video/rockchip/rga3/include/rga_dma_buf.h b/drivers/video/rockchip/rga3/include/rga_dma_buf.h index fefea23914e2..05a801c0705c 100644 --- a/drivers/video/rockchip/rga3/include/rga_dma_buf.h +++ b/drivers/video/rockchip/rga3/include/rga_dma_buf.h @@ -26,18 +26,18 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr, int rga_virtual_memory_check(void *vaddr, u32 w, u32 h, u32 format, int fd); int rga_dma_memory_check(struct rga_dma_buffer *rga_dma_buffer, struct rga_img_info_t *img); -int rga_iommu_map_sgt(struct sg_table *sgt, size_t size, - struct rga_dma_buffer *buffer, - struct device *rga_dev); -int rga_iommu_map(phys_addr_t paddr, size_t size, - struct rga_dma_buffer *buffer, - struct device *rga_dev); -void rga_iommu_unmap(struct rga_dma_buffer *buffer); +int rga_dma_map_phys_addr(phys_addr_t phys_addr, size_t size, struct rga_dma_buffer *buffer, + enum dma_data_direction dir, struct device *map_dev); +void rga_dma_unmap_phys_addr(struct rga_dma_buffer *buffer); + +int rga_dma_map_sgt(struct sg_table *sgt, struct rga_dma_buffer *buffer, + enum dma_data_direction dir, struct device *map_dev); +void rga_dma_unmap_sgt(struct rga_dma_buffer *buffer); int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buffer, - enum dma_data_direction dir, struct device *rga_dev); + enum dma_data_direction dir, struct device *map_dev); int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer, - enum dma_data_direction dir, struct device *rga_dev); + enum dma_data_direction dir, struct device *map_dev); void rga_dma_unmap_buf(struct rga_dma_buffer *rga_dma_buffer); void rga_dma_sync_flush_range(void *pstart, void *pend, struct rga_scheduler_t *scheduler); diff --git a/drivers/video/rockchip/rga3/include/rga_drv.h b/drivers/video/rockchip/rga3/include/rga_drv.h index f934e38c6a83..0d41d4be2e8a 100644 --- a/drivers/video/rockchip/rga3/include/rga_drv.h +++ b/drivers/video/rockchip/rga3/include/rga_drv.h @@ -182,8 +182,11 @@ struct rga_dma_buffer { */ size_t offset; - /* The scheduler of the mapping */ - struct rga_scheduler_t *scheduler; + /* + * The device used by dma-buf mapping, which usually corresponds to the + * default domain or the current device. + */ + struct device *map_dev; }; struct rga_virt_addr { @@ -192,6 +195,7 @@ struct rga_virt_addr { struct page **pages; int pages_order; int page_count; + /* Actual effective size */ unsigned long size; /* The offset of the first page of the virtual address */ @@ -227,6 +231,9 @@ struct rga_internal_buffer { struct kref refcount; struct rga_session *session; + + /* The scheduler of the mapping */ + struct rga_scheduler_t *scheduler; }; struct rga_scheduler_t; diff --git a/drivers/video/rockchip/rga3/include/rga_mm.h b/drivers/video/rockchip/rga3/include/rga_mm.h index d68fd75dcb45..a75f2470b928 100644 --- a/drivers/video/rockchip/rga3/include/rga_mm.h +++ b/drivers/video/rockchip/rga3/include/rga_mm.h @@ -42,7 +42,7 @@ static inline bool rga_mm_is_invalid_dma_buffer(struct rga_dma_buffer *buffer) if (buffer == NULL) return true; - return buffer->scheduler == NULL ? true : false; + return buffer->map_dev == NULL ? true : false; } struct rga_internal_buffer *rga_mm_lookup_handle(struct rga_mm *mm_session, uint32_t handle); diff --git a/drivers/video/rockchip/rga3/rga_debugger.c b/drivers/video/rockchip/rga3/rga_debugger.c index 93c5fcf3ec48..572035977338 100644 --- a/drivers/video/rockchip/rga3/rga_debugger.c +++ b/drivers/video/rockchip/rga3/rga_debugger.c @@ -286,7 +286,7 @@ static int rga_mm_session_show(struct seq_file *m, void *data) (unsigned long)dump_buffer->dma_buffer->iova, dump_buffer->dma_buffer->sgt, dump_buffer->dma_buffer->size, - dump_buffer->dma_buffer->scheduler->core); + dump_buffer->scheduler->core); if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) seq_printf(m, "\t is contiguous, pa = 0x%lx\n", @@ -310,7 +310,7 @@ static int rga_mm_session_show(struct seq_file *m, void *data) (unsigned long)dump_buffer->dma_buffer->offset, dump_buffer->dma_buffer->sgt, dump_buffer->dma_buffer->size, - dump_buffer->dma_buffer->scheduler->core); + dump_buffer->scheduler->core); if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) seq_printf(m, "\t is contiguous, pa = 0x%lx\n", diff --git a/drivers/video/rockchip/rga3/rga_dma_buf.c b/drivers/video/rockchip/rga3/rga_dma_buf.c index e72470e5c5f5..96609d0784fe 100644 --- a/drivers/video/rockchip/rga3/rga_dma_buf.c +++ b/drivers/video/rockchip/rga3/rga_dma_buf.c @@ -11,20 +11,6 @@ #include "rga_job.h" #include "rga_debugger.h" -static int rga_dma_info_to_prot(enum dma_data_direction dir) -{ - switch (dir) { - case DMA_BIDIRECTIONAL: - return IOMMU_READ | IOMMU_WRITE; - case DMA_TO_DEVICE: - return IOMMU_READ; - case DMA_FROM_DEVICE: - return IOMMU_WRITE; - default: - return 0; - } -} - int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr, unsigned long v_addr, int format, uint32_t w, uint32_t h, unsigned long *StartAddr, unsigned long *size) @@ -198,165 +184,6 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr, return pageCount; } -static dma_addr_t rga_iommu_dma_alloc_iova(struct iommu_domain *domain, - size_t size, u64 dma_limit, - struct device *dev) -{ - struct rga_iommu_dma_cookie *cookie = (void *)domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; - unsigned long shift, iova_len, iova = 0; - - shift = iova_shift(iovad); - iova_len = size >> shift; - -#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0)) - /* - * Freeing non-power-of-two-sized allocations back into the IOVA caches - * will come back to bite us badly, so we have to waste a bit of space - * rounding up anything cacheable to make sure that can't happen. The - * order of the unadjusted size will still match upon freeing. - */ - if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) - iova_len = roundup_pow_of_two(iova_len); -#endif - -#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0)) - dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); -#else - if (dev->bus_dma_mask) - dma_limit &= dev->bus_dma_mask; -#endif - - if (domain->geometry.force_aperture) - dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end); - -#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 19, 111) && \ - LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0)) - iova = alloc_iova_fast(iovad, iova_len, - min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn), - true); -#else - iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true); -#endif - - return (dma_addr_t)iova << shift; -} - -static void rga_iommu_dma_free_iova(struct iommu_domain *domain, - dma_addr_t iova, size_t size) -{ - struct rga_iommu_dma_cookie *cookie = (void *)domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; - - free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); -} - -static inline struct iommu_domain *rga_iommu_get_dma_domain(struct device *dev) -{ - return iommu_get_domain_for_dev(dev); -} - -void rga_iommu_unmap(struct rga_dma_buffer *buffer) -{ - if (buffer == NULL) - return; - if (buffer->iova == 0) - return; - - iommu_unmap(buffer->domain, buffer->iova, buffer->size); - rga_iommu_dma_free_iova(buffer->domain, buffer->iova, buffer->size); -} - -int rga_iommu_map_sgt(struct sg_table *sgt, size_t size, - struct rga_dma_buffer *buffer, - struct device *rga_dev) -{ - struct iommu_domain *domain = NULL; - struct rga_iommu_dma_cookie *cookie; - struct iova_domain *iovad; - dma_addr_t iova; - size_t map_size; - unsigned long align_size; - - if (sgt == NULL) { - rga_err("can not map iommu, because sgt is null!\n"); - return -EINVAL; - } - - domain = rga_iommu_get_dma_domain(rga_dev); - cookie = (void *)domain->iova_cookie; - iovad = &cookie->iovad; - align_size = iova_align(iovad, size); - - if (DEBUGGER_EN(MSG)) - rga_log("iova_align size = %ld", align_size); - - iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev); - if (!iova) { - rga_err("rga_iommu_dma_alloc_iova failed"); - return -ENOMEM; - } - - map_size = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents, - rga_dma_info_to_prot(DMA_BIDIRECTIONAL)); - if (map_size < align_size) { - rga_err("iommu can not map sgt to iova"); - rga_iommu_dma_free_iova(domain, iova, align_size); - return -EINVAL; - } - - buffer->domain = domain; - buffer->iova = iova; - buffer->size = align_size; - - return 0; -} - -int rga_iommu_map(phys_addr_t paddr, size_t size, - struct rga_dma_buffer *buffer, - struct device *rga_dev) -{ - int ret; - struct iommu_domain *domain = NULL; - struct rga_iommu_dma_cookie *cookie; - struct iova_domain *iovad; - dma_addr_t iova; - unsigned long align_size; - - if (paddr == 0) { - rga_err("can not map iommu, because phys_addr is 0!\n"); - return -EINVAL; - } - - domain = rga_iommu_get_dma_domain(rga_dev); - cookie = (void *)domain->iova_cookie; - iovad = &cookie->iovad; - align_size = iova_align(iovad, size); - - if (DEBUGGER_EN(MSG)) - rga_log("iova_align size = %ld", align_size); - - iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev); - if (!iova) { - rga_err("rga_iommu_dma_alloc_iova failed"); - return -ENOMEM; - } - - ret = iommu_map(domain, iova, paddr, align_size, - rga_dma_info_to_prot(DMA_BIDIRECTIONAL)); - if (ret) { - rga_err("iommu can not map phys_addr to iova"); - rga_iommu_dma_free_iova(domain, iova, align_size); - return ret; - } - - buffer->domain = domain; - buffer->iova = iova; - buffer->size = align_size; - - return 0; -} - int rga_virtual_memory_check(void *vaddr, u32 w, u32 h, u32 format, int fd) { int bits = 32; @@ -424,8 +251,67 @@ int rga_dma_memory_check(struct rga_dma_buffer *rga_dma_buffer, struct rga_img_i return ret; } +int rga_dma_map_phys_addr(phys_addr_t phys_addr, size_t size, struct rga_dma_buffer *buffer, + enum dma_data_direction dir, struct device *map_dev) +{ + dma_addr_t addr; + + addr = dma_map_resource(map_dev, phys_addr, size, dir, 0); + if (addr == DMA_MAPPING_ERROR) { + rga_err("dma_map_resouce failed!\n"); + return -EINVAL; + } + + buffer->dma_addr = addr; + buffer->dir = dir; + buffer->size = size; + buffer->map_dev = map_dev; + + return 0; +} + +void rga_dma_unmap_phys_addr(struct rga_dma_buffer *buffer) +{ + dma_unmap_resource(buffer->map_dev, buffer->dma_addr, buffer->size, buffer->dir, 0); +} + +int rga_dma_map_sgt(struct sg_table *sgt, struct rga_dma_buffer *buffer, + enum dma_data_direction dir, struct device *map_dev) +{ + int i, ret = 0; + struct scatterlist *sg = NULL; + + ret = dma_map_sg(map_dev, sgt->sgl, sgt->orig_nents, dir); + if (ret <= 0) { + rga_err("dma_map_sg failed! ret = %d\n", ret); + return ret < 0 ? ret : -EINVAL; + } + sgt->nents = ret; + + buffer->sgt = sgt; + buffer->dma_addr = sg_dma_address(sgt->sgl); + buffer->dir = dir; + buffer->size = 0; + for_each_sgtable_sg(sgt, sg, i) + buffer->size += sg_dma_len(sg); + buffer->map_dev = map_dev; + + return 0; +} + +void rga_dma_unmap_sgt(struct rga_dma_buffer *buffer) +{ + if (!buffer->sgt) + return; + + dma_unmap_sg(buffer->map_dev, + buffer->sgt->sgl, + buffer->sgt->orig_nents, + buffer->dir); +} + int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buffer, - enum dma_data_direction dir, struct device *rga_dev) + enum dma_data_direction dir, struct device *map_dev) { struct dma_buf_attachment *attach = NULL; struct sg_table *sgt = NULL; @@ -439,7 +325,7 @@ int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buff return -EINVAL; } - attach = dma_buf_attach(dma_buf, rga_dev); + attach = dma_buf_attach(dma_buf, map_dev); if (IS_ERR(attach)) { ret = PTR_ERR(attach); rga_err("Failed to attach dma_buf, ret[%d]\n", ret); @@ -461,6 +347,7 @@ int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buff rga_dma_buffer->size = 0; for_each_sgtable_sg(sgt, sg, i) rga_dma_buffer->size += sg_dma_len(sg); + rga_dma_buffer->map_dev = map_dev; return ret; @@ -475,7 +362,7 @@ err_get_attach: } int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer, - enum dma_data_direction dir, struct device *rga_dev) + enum dma_data_direction dir, struct device *map_dev) { struct dma_buf *dma_buf = NULL; struct dma_buf_attachment *attach = NULL; @@ -490,7 +377,7 @@ int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer, return ret; } - attach = dma_buf_attach(dma_buf, rga_dev); + attach = dma_buf_attach(dma_buf, map_dev); if (IS_ERR(attach)) { ret = PTR_ERR(attach); rga_err("Failed to attach dma_buf, ret[%d]\n", ret); @@ -512,6 +399,7 @@ int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer, rga_dma_buffer->size = 0; for_each_sgtable_sg(sgt, sg, i) rga_dma_buffer->size += sg_dma_len(sg); + rga_dma_buffer->map_dev = map_dev; return ret; @@ -551,12 +439,12 @@ int rga_dma_free(struct rga_dma_buffer *buffer) return -EINVAL; } - dma_free_coherent(buffer->scheduler->dev, buffer->size, buffer->vaddr, buffer->dma_addr); + dma_free_coherent(buffer->map_dev, buffer->size, buffer->vaddr, buffer->dma_addr); buffer->vaddr = NULL; buffer->dma_addr = 0; buffer->iova = 0; buffer->size = 0; - buffer->scheduler = NULL; + buffer->map_dev = NULL; kfree(buffer); @@ -581,7 +469,7 @@ struct rga_dma_buffer *rga_dma_alloc_coherent(struct rga_scheduler_t *scheduler, buffer->size = align_size; buffer->dma_addr = dma_addr; - buffer->scheduler = scheduler; + buffer->map_dev = scheduler->dev; if (scheduler->data->mmu == RGA_IOMMU) buffer->iova = buffer->dma_addr; diff --git a/drivers/video/rockchip/rga3/rga_mm.c b/drivers/video/rockchip/rga3/rga_mm.c index 740009008e59..eb34fde9d769 100644 --- a/drivers/video/rockchip/rga3/rga_mm.c +++ b/drivers/video/rockchip/rga3/rga_mm.c @@ -179,11 +179,10 @@ static struct sg_table *rga_alloc_sgt(struct rga_virt_addr *virt_addr) } /* get sg form pages. */ - /* iova requires minimum page alignment, so sgt cannot have offset */ ret = sg_alloc_table_from_pages(sgt, virt_addr->pages, virt_addr->page_count, - 0, + virt_addr->offset, virt_addr->size, GFP_KERNEL); if (ret) { @@ -232,7 +231,6 @@ static int rga_alloc_virt_addr(struct rga_virt_addr **virt_addr_p, unsigned int count; int img_size; size_t offset; - unsigned long size; struct page **pages = NULL; struct rga_virt_addr *virt_addr = NULL; @@ -246,10 +244,9 @@ static int rga_alloc_virt_addr(struct rga_virt_addr **virt_addr_p, offset = viraddr & (~PAGE_MASK); count = RGA_GET_PAGE_COUNT(img_size + offset); - size = count * PAGE_SIZE; - if (!size) { - rga_err("failed to calculating buffer size! size = %ld, count = %d, offset = %ld\n", - size, count, (unsigned long)offset); + if (!count) { + rga_err("failed to calculating buffer size! img_size = %d, count = %d, offset = %ld\n", + img_size, count, (unsigned long)offset); rga_dump_memory_parm(memory_parm); return -EFAULT; } @@ -292,7 +289,7 @@ static int rga_alloc_virt_addr(struct rga_virt_addr **virt_addr_p, virt_addr->pages = pages; virt_addr->pages_order = order; virt_addr->page_count = count; - virt_addr->size = size; + virt_addr->size = img_size; virt_addr->offset = offset; virt_addr->result = result; @@ -398,18 +395,17 @@ static int rga_mm_map_dma_buffer(struct rga_external_buffer *external_buffer, return ex_buffer_size == 0 ? -EINVAL : ex_buffer_size; } - /* - * dma-buf api needs to use default_domain of main dev, - * and not IOMMU for devices without iommu_info ptr. - */ - map_dev = scheduler->iommu_info ? scheduler->iommu_info->default_dev : scheduler->dev; - buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (buffer == NULL) { rga_err("%s alloc internal_buffer error!\n", __func__); return -ENOMEM; } + /* + * dma-buf api needs to use default_domain of main dev, + * and not IOMMU for devices without iommu_info ptr. + */ + map_dev = scheduler->iommu_info ? scheduler->iommu_info->default_dev : scheduler->dev; switch (external_buffer->type) { case RGA_DMA_BUFFER: ret = rga_dma_map_fd((int)external_buffer->memory, @@ -440,8 +436,6 @@ static int rga_mm_map_dma_buffer(struct rga_external_buffer *external_buffer, goto unmap_buffer; } - buffer->scheduler = scheduler; - if (scheduler->data->mmu == RGA_IOMMU) buffer->iova = buffer->dma_addr; @@ -473,6 +467,8 @@ static int rga_mm_map_dma_buffer(struct rga_external_buffer *external_buffer, internal_buffer->dma_buffer = buffer; internal_buffer->mm_flag = mm_flag; internal_buffer->phys_addr = phys_addr ? phys_addr : 0; + internal_buffer->size = buffer->size - buffer->offset; + internal_buffer->scheduler = scheduler; return 0; @@ -492,19 +488,7 @@ static void rga_mm_unmap_virt_addr(struct rga_internal_buffer *internal_buffer) if (rga_mm_is_invalid_dma_buffer(internal_buffer->dma_buffer)) return; - switch (internal_buffer->dma_buffer->scheduler->data->mmu) { - case RGA_IOMMU: - rga_iommu_unmap(internal_buffer->dma_buffer); - break; - case RGA_MMU: - dma_unmap_sg(internal_buffer->dma_buffer->scheduler->dev, - internal_buffer->dma_buffer->sgt->sgl, - internal_buffer->dma_buffer->sgt->orig_nents, - DMA_BIDIRECTIONAL); - break; - default: - break; - } + rga_dma_unmap_sgt(internal_buffer->dma_buffer); if (internal_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS && internal_buffer->phys_addr > 0) @@ -532,6 +516,7 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer, struct sg_table *sgt; struct rga_virt_addr *virt_addr; struct rga_dma_buffer *buffer; + struct device *map_dev; struct rga_scheduler_t *scheduler; scheduler = job ? job->scheduler : @@ -578,6 +563,11 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer, } mm_flag |= RGA_MEM_PHYSICAL_CONTIGUOUS; + } else if (scheduler->data->mmu == RGA_NONE_MMU) { + rga_err("Current %s[%d] cannot support physically discontinuous virtual address!\n", + rga_get_mmu_type_str(scheduler->data->mmu), scheduler->data->mmu); + ret = -EOPNOTSUPP; + goto free_sgt; } /* @@ -601,47 +591,28 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer, goto free_sgt; } - switch (scheduler->data->mmu) { - case RGA_IOMMU: - ret = rga_iommu_map_sgt(sgt, virt_addr->size, buffer, scheduler->dev); - if (ret < 0) { - rga_err("%s core[%d] iommu_map virtual address error!\n", - __func__, scheduler->core); - goto free_dma_buffer; - } - - buffer->dma_addr = buffer->iova; - - break; - case RGA_MMU: - ret = dma_map_sg(scheduler->dev, sgt->sgl, sgt->orig_nents, DMA_BIDIRECTIONAL); - if (ret == 0) { - rga_err("%s core[%d] dma_map_sgt error! va = 0x%lx, nents = %d\n", - __func__, scheduler->core, - (unsigned long)virt_addr->addr, sgt->orig_nents); - ret = -EINVAL; - goto free_dma_buffer; - } - break; - default: - if (mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) - break; - - rga_err("Current %s[%d] cannot support physically discontinuous virtual address!\n", - rga_get_mmu_type_str(scheduler->data->mmu), scheduler->data->mmu); - ret = -EOPNOTSUPP; + /* + * dma-buf api needs to use default_domain of main dev, + * and not IOMMU for devices without iommu_info ptr. + */ + map_dev = scheduler->iommu_info ? scheduler->iommu_info->default_dev : scheduler->dev; + ret = rga_dma_map_sgt(sgt, buffer, DMA_BIDIRECTIONAL, map_dev); + if (ret < 0) { + rga_err("%s core[%d] rga map sgt failed! va = 0x%lx, orig_nents = %d\n", + __func__, scheduler->core, + (unsigned long)virt_addr->addr, sgt->orig_nents); goto free_dma_buffer; } - buffer->sgt = sgt; - buffer->offset = virt_addr->offset; - buffer->size = virt_addr->size; - buffer->scheduler = scheduler; + if (scheduler->data->mmu == RGA_IOMMU) + buffer->iova = buffer->dma_addr; internal_buffer->virt_addr = virt_addr; internal_buffer->dma_buffer = buffer; internal_buffer->mm_flag = mm_flag; internal_buffer->phys_addr = phys_addr ? phys_addr + virt_addr->offset : 0; + internal_buffer->size = buffer->size - buffer->offset; + internal_buffer->scheduler = scheduler; return 0; @@ -661,16 +632,12 @@ put_current_mm: static void rga_mm_unmap_phys_addr(struct rga_internal_buffer *internal_buffer) { - WARN_ON(internal_buffer->dma_buffer == NULL); + if (internal_buffer->dma_buffer != NULL) { + rga_dma_unmap_phys_addr(internal_buffer->dma_buffer); + kfree(internal_buffer->dma_buffer); + internal_buffer->dma_buffer = NULL; + } - if (rga_mm_is_invalid_dma_buffer(internal_buffer->dma_buffer)) - return; - - if (internal_buffer->dma_buffer->scheduler->data->mmu == RGA_IOMMU) - rga_iommu_unmap(internal_buffer->dma_buffer); - - kfree(internal_buffer->dma_buffer); - internal_buffer->dma_buffer = NULL; internal_buffer->phys_addr = 0; internal_buffer->size = 0; } @@ -683,7 +650,8 @@ static int rga_mm_map_phys_addr(struct rga_external_buffer *external_buffer, phys_addr_t phys_addr; int buffer_size; uint32_t mm_flag = 0; - struct rga_dma_buffer *buffer; + struct rga_dma_buffer *buffer = NULL; + struct device *map_dev; struct rga_scheduler_t *scheduler; scheduler = job ? job->scheduler : @@ -717,26 +685,34 @@ static int rga_mm_map_phys_addr(struct rga_external_buffer *external_buffer, return -EINVAL; } - buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); - if (buffer == NULL) { - rga_err("%s alloc internal dma buffer error!\n", __func__); - return -ENOMEM; - } - if (scheduler->data->mmu == RGA_IOMMU) { - ret = rga_iommu_map(phys_addr, buffer_size, buffer, scheduler->dev); + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (buffer == NULL) { + rga_err("%s alloc internal dma buffer error!\n", __func__); + return -ENOMEM; + } + + /* + * dma-buf api needs to use default_domain of main dev, + * and not IOMMU for devices without iommu_info ptr. + */ + map_dev = scheduler->iommu_info ? + scheduler->iommu_info->default_dev : scheduler->dev; + ret = rga_dma_map_phys_addr(phys_addr, buffer_size, buffer, + DMA_BIDIRECTIONAL, map_dev); if (ret < 0) { rga_err("%s core[%d] map phys_addr error!\n", __func__, scheduler->core); goto free_dma_buffer; } + + buffer->iova = buffer->dma_addr; } - buffer->scheduler = scheduler; - - internal_buffer->phys_addr = phys_addr; - internal_buffer->size = buffer_size; - internal_buffer->mm_flag = mm_flag; internal_buffer->dma_buffer = buffer; + internal_buffer->mm_flag = mm_flag; + internal_buffer->phys_addr = phys_addr; + internal_buffer->size = buffer ? buffer->size : buffer_size; + internal_buffer->scheduler = scheduler; return 0; @@ -785,8 +761,6 @@ static int rga_mm_map_buffer(struct rga_external_buffer *external_buffer, if (ret < 0) return ret; - internal_buffer->size = internal_buffer->dma_buffer->size - - internal_buffer->dma_buffer->offset; internal_buffer->mm_flag |= RGA_MEM_NEED_USE_IOMMU; break; case RGA_VIRTUAL_ADDRESS: @@ -796,8 +770,6 @@ static int rga_mm_map_buffer(struct rga_external_buffer *external_buffer, if (ret < 0) return ret; - internal_buffer->size = internal_buffer->virt_addr->size - - internal_buffer->virt_addr->offset; internal_buffer->mm_flag |= RGA_MEM_NEED_USE_IOMMU; break; case RGA_PHYSICAL_ADDRESS: @@ -1002,7 +974,7 @@ void rga_mm_dump_buffer(struct rga_internal_buffer *dump_buffer) (unsigned long)dump_buffer->dma_buffer->offset, dump_buffer->dma_buffer->sgt, dump_buffer->dma_buffer->size, - dump_buffer->dma_buffer->scheduler->core); + dump_buffer->scheduler->core); if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) rga_log("is contiguous, pa = 0x%lx\n", @@ -1026,7 +998,7 @@ void rga_mm_dump_buffer(struct rga_internal_buffer *dump_buffer) (unsigned long)dump_buffer->dma_buffer->offset, dump_buffer->dma_buffer->sgt, dump_buffer->dma_buffer->size, - dump_buffer->dma_buffer->scheduler->core); + dump_buffer->scheduler->core); if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) rga_buf_log(dump_buffer, "is contiguous, pa = 0x%lx\n", @@ -1034,6 +1006,16 @@ void rga_mm_dump_buffer(struct rga_internal_buffer *dump_buffer) break; case RGA_PHYSICAL_ADDRESS: rga_buf_log(dump_buffer, "pa = 0x%lx\n", (unsigned long)dump_buffer->phys_addr); + + if (rga_mm_is_invalid_dma_buffer(dump_buffer->dma_buffer)) + break; + + rga_buf_log(dump_buffer, "iova = 0x%lx, dma_addr = 0x%lx, offset = 0x%lx, size = %ld, map_core = 0x%x\n", + (unsigned long)dump_buffer->dma_buffer->iova, + (unsigned long)dump_buffer->dma_buffer->dma_addr, + (unsigned long)dump_buffer->dma_buffer->offset, + dump_buffer->dma_buffer->size, + dump_buffer->scheduler->core); break; default: rga_buf_err(dump_buffer, "Illegal buffer! type= %d\n", dump_buffer->type); @@ -1129,7 +1111,7 @@ static int rga_mm_sgt_to_page_table(struct sg_table *sg, * The length of each sgl is expected to be obtained here, not * the length of the entire dma_buf, so sg_dma_len() is not used. */ - len = sgl->length >> PAGE_SHIFT; + len = (sgl->length + (PAGE_SIZE - 1)) >> PAGE_SHIFT; if (use_dma_address) /* @@ -1148,6 +1130,8 @@ static int rga_mm_sgt_to_page_table(struct sg_table *sg, else Address = sg_phys(sgl); + Address &= PAGE_MASK; + for (i = 0; i < len; i++) { if (mapped_size + i >= pageCount) { break_flag = 1; @@ -1335,16 +1319,26 @@ static int rga_mm_sync_dma_sg_for_device(struct rga_internal_buffer *buffer, struct rga_scheduler_t *scheduler; ktime_t timestamp = ktime_get(); - scheduler = buffer->dma_buffer->scheduler; + scheduler = buffer->scheduler; if (scheduler == NULL) { rga_job_err(job, "%s(%d), failed to get scheduler, core = 0x%x\n", __func__, __LINE__, job->core); return -EFAULT; } - if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS && - scheduler->data->mmu != RGA_IOMMU) { - dma_sync_single_for_device(scheduler->dev, buffer->phys_addr, buffer->size, dir); + if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) { + if (scheduler->data->mmu == RGA_IOMMU) { + if (rga_mm_is_invalid_dma_buffer(buffer->dma_buffer)) { + rga_job_err(job, "invalid dma-buffer with IOMMU device!\n"); + return -EFAULT; + } + + dma_sync_single_for_device(buffer->dma_buffer->map_dev, + buffer->dma_buffer->iova, buffer->dma_buffer->size, dir); + } else { + dma_sync_single_for_device(scheduler->dev, + buffer->phys_addr, buffer->size, dir); + } } else { sgt = rga_mm_lookup_sgt(buffer); if (sgt == NULL) { @@ -1372,16 +1366,26 @@ static int rga_mm_sync_dma_sg_for_cpu(struct rga_internal_buffer *buffer, struct rga_scheduler_t *scheduler; ktime_t timestamp = ktime_get(); - scheduler = buffer->dma_buffer->scheduler; + scheduler = buffer->scheduler; if (scheduler == NULL) { rga_job_err(job, "%s(%d), failed to get scheduler, core = 0x%x\n", __func__, __LINE__, job->core); return -EFAULT; } - if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS && - scheduler->data->mmu != RGA_IOMMU) { - dma_sync_single_for_cpu(scheduler->dev, buffer->phys_addr, buffer->size, dir); + if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) { + if (scheduler->data->mmu == RGA_IOMMU) { + if (rga_mm_is_invalid_dma_buffer(buffer->dma_buffer)) { + rga_job_err(job, "invalid dma-buffer with IOMMU device!\n"); + return -EFAULT; + } + + dma_sync_single_for_cpu(buffer->dma_buffer->map_dev, + buffer->dma_buffer->iova, buffer->dma_buffer->size, dir); + } else { + dma_sync_single_for_cpu(scheduler->dev, + buffer->phys_addr, buffer->size, dir); + } } else { sgt = rga_mm_lookup_sgt(buffer); if (sgt == NULL) {