shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-05 10:31:46 +09:00
Code Issues Packages Projects Releases Wiki Activity

video: rockchip: rga3: remove dependency on CONFIG_IOMMU_IOVA

Browse Source 
Since 'CONFIG_IOMMU_IOVA' is often disabled on 32-bit systems, causing
iommu_map API calls to fail, the DMA mapping API is used instead.

Change-Id: I7226ddf6a04e0af4c6b1b029bb5a32ede04c0810
Signed-off-by: Yu Qiaowei <cerf.yu@rock-chips.com>
This commit is contained in:
Yu Qiaowei
2025-04-16 15:16:49 +08:00
committed by Tao Huang
parent b6558f3a76
commit d4a3012c24
6 changed files with 192 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
drivers/video/rockchip/rga3/include/rga_dma_buf.h
View File

@@ -26,18 +26,18 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr,
int rga_virtual_memory_check(void *vaddr, u32 w, u32 h, u32 format, int fd);
int rga_dma_memory_check(struct rga_dma_buffer *rga_dma_buffer, struct rga_img_info_t *img);
int rga_iommu_map_sgt(struct sg_table *sgt, size_t size,
struct rga_dma_buffer *buffer,
struct device *rga_dev);
int rga_iommu_map(phys_addr_t paddr, size_t size,
struct rga_dma_buffer *buffer,
struct device *rga_dev);
void rga_iommu_unmap(struct rga_dma_buffer *buffer);
int rga_dma_map_phys_addr(phys_addr_t phys_addr, size_t size, struct rga_dma_buffer *buffer,
enum dma_data_direction dir, struct device *map_dev);
void rga_dma_unmap_phys_addr(struct rga_dma_buffer *buffer);
int rga_dma_map_sgt(struct sg_table *sgt, struct rga_dma_buffer *buffer,
enum dma_data_direction dir, struct device *map_dev);
void rga_dma_unmap_sgt(struct rga_dma_buffer *buffer);
int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buffer,
enum dma_data_direction dir, struct device *rga_dev);
enum dma_data_direction dir, struct device *map_dev);
int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
enum dma_data_direction dir, struct device *rga_dev);
enum dma_data_direction dir, struct device *map_dev);
void rga_dma_unmap_buf(struct rga_dma_buffer *rga_dma_buffer);
void rga_dma_sync_flush_range(void *pstart, void *pend, struct rga_scheduler_t *scheduler);

11
drivers/video/rockchip/rga3/include/rga_drv.h
View File

@@ -182,8 +182,11 @@ struct rga_dma_buffer {
*/
size_t offset;
/* The scheduler of the mapping */
struct rga_scheduler_t *scheduler;
/*
* The device used by dma-buf mapping, which usually corresponds to the
* default domain or the current device.
*/
struct device *map_dev;
};
struct rga_virt_addr {
@@ -192,6 +195,7 @@ struct rga_virt_addr {
struct page **pages;
int pages_order;
int page_count;
/* Actual effective size */
unsigned long size;
/* The offset of the first page of the virtual address */
@@ -227,6 +231,9 @@ struct rga_internal_buffer {
struct kref refcount;
struct rga_session *session;
/* The scheduler of the mapping */
struct rga_scheduler_t *scheduler;
};
struct rga_scheduler_t;

2
drivers/video/rockchip/rga3/include/rga_mm.h
View File

@@ -42,7 +42,7 @@ static inline bool rga_mm_is_invalid_dma_buffer(struct rga_dma_buffer *buffer)
if (buffer == NULL)
return true;
return buffer->scheduler == NULL ? true : false;
return buffer->map_dev == NULL ? true : false;
}
struct rga_internal_buffer *rga_mm_lookup_handle(struct rga_mm *mm_session, uint32_t handle);

4
drivers/video/rockchip/rga3/rga_debugger.c
View File

@@ -286,7 +286,7 @@ static int rga_mm_session_show(struct seq_file *m, void *data)
(unsigned long)dump_buffer->dma_buffer->iova,
dump_buffer->dma_buffer->sgt,
dump_buffer->dma_buffer->size,
dump_buffer->dma_buffer->scheduler->core);
dump_buffer->scheduler->core);
if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS)
seq_printf(m, "\t is contiguous, pa = 0x%lx\n",
@@ -310,7 +310,7 @@ static int rga_mm_session_show(struct seq_file *m, void *data)
(unsigned long)dump_buffer->dma_buffer->offset,
dump_buffer->dma_buffer->sgt,
dump_buffer->dma_buffer->size,
dump_buffer->dma_buffer->scheduler->core);
dump_buffer->scheduler->core);
if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS)
seq_printf(m, "\t is contiguous, pa = 0x%lx\n",

248
drivers/video/rockchip/rga3/rga_dma_buf.c
View File

@@ -11,20 +11,6 @@
#include "rga_job.h"
#include "rga_debugger.h"
static int rga_dma_info_to_prot(enum dma_data_direction dir)
{
switch (dir) {
case DMA_BIDIRECTIONAL:
return IOMMU_READ | IOMMU_WRITE;
case DMA_TO_DEVICE:
return IOMMU_READ;
case DMA_FROM_DEVICE:
return IOMMU_WRITE;
default:
return 0;
}
}
int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr,
unsigned long v_addr, int format, uint32_t w,
uint32_t h, unsigned long *StartAddr, unsigned long *size)
@@ -198,165 +184,6 @@ int rga_buf_size_cal(unsigned long yrgb_addr, unsigned long uv_addr,
return pageCount;
}
static dma_addr_t rga_iommu_dma_alloc_iova(struct iommu_domain *domain,
size_t size, u64 dma_limit,
struct device *dev)
{
struct rga_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
unsigned long shift, iova_len, iova = 0;
shift = iova_shift(iovad);
iova_len = size >> shift;
#if (LINUX_VERSION_CODE < KERNEL_VERSION(6, 1, 0))
/*
* Freeing non-power-of-two-sized allocations back into the IOVA caches
* will come back to bite us badly, so we have to waste a bit of space
* rounding up anything cacheable to make sure that can't happen. The
* order of the unadjusted size will still match upon freeing.
*/
if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
iova_len = roundup_pow_of_two(iova_len);
#endif
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0))
dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
#else
if (dev->bus_dma_mask)
dma_limit &= dev->bus_dma_mask;
#endif
if (domain->geometry.force_aperture)
dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
#if (LINUX_VERSION_CODE > KERNEL_VERSION(4, 19, 111) && \
LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
iova = alloc_iova_fast(iovad, iova_len,
min_t(dma_addr_t, dma_limit >> shift, iovad->end_pfn),
true);
#else
iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift, true);
#endif
return (dma_addr_t)iova << shift;
}
static void rga_iommu_dma_free_iova(struct iommu_domain *domain,
dma_addr_t iova, size_t size)
{
struct rga_iommu_dma_cookie *cookie = (void *)domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad));
}
static inline struct iommu_domain *rga_iommu_get_dma_domain(struct device *dev)
{
return iommu_get_domain_for_dev(dev);
}
void rga_iommu_unmap(struct rga_dma_buffer *buffer)
{
if (buffer == NULL)
return;
if (buffer->iova == 0)
return;
iommu_unmap(buffer->domain, buffer->iova, buffer->size);
rga_iommu_dma_free_iova(buffer->domain, buffer->iova, buffer->size);
}
int rga_iommu_map_sgt(struct sg_table *sgt, size_t size,
struct rga_dma_buffer *buffer,
struct device *rga_dev)
{
struct iommu_domain *domain = NULL;
struct rga_iommu_dma_cookie *cookie;
struct iova_domain *iovad;
dma_addr_t iova;
size_t map_size;
unsigned long align_size;
if (sgt == NULL) {
rga_err("can not map iommu, because sgt is null!\n");
return -EINVAL;
}
domain = rga_iommu_get_dma_domain(rga_dev);
cookie = (void *)domain->iova_cookie;
iovad = &cookie->iovad;
align_size = iova_align(iovad, size);
if (DEBUGGER_EN(MSG))
rga_log("iova_align size = %ld", align_size);
iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev);
if (!iova) {
rga_err("rga_iommu_dma_alloc_iova failed");
return -ENOMEM;
}
map_size = iommu_map_sg(domain, iova, sgt->sgl, sgt->orig_nents,
rga_dma_info_to_prot(DMA_BIDIRECTIONAL));
if (map_size < align_size) {
rga_err("iommu can not map sgt to iova");
rga_iommu_dma_free_iova(domain, iova, align_size);
return -EINVAL;
}
buffer->domain = domain;
buffer->iova = iova;
buffer->size = align_size;
return 0;
}
int rga_iommu_map(phys_addr_t paddr, size_t size,
struct rga_dma_buffer *buffer,
struct device *rga_dev)
{
int ret;
struct iommu_domain *domain = NULL;
struct rga_iommu_dma_cookie *cookie;
struct iova_domain *iovad;
dma_addr_t iova;
unsigned long align_size;
if (paddr == 0) {
rga_err("can not map iommu, because phys_addr is 0!\n");
return -EINVAL;
}
domain = rga_iommu_get_dma_domain(rga_dev);
cookie = (void *)domain->iova_cookie;
iovad = &cookie->iovad;
align_size = iova_align(iovad, size);
if (DEBUGGER_EN(MSG))
rga_log("iova_align size = %ld", align_size);
iova = rga_iommu_dma_alloc_iova(domain, align_size, rga_dev->coherent_dma_mask, rga_dev);
if (!iova) {
rga_err("rga_iommu_dma_alloc_iova failed");
return -ENOMEM;
}
ret = iommu_map(domain, iova, paddr, align_size,
rga_dma_info_to_prot(DMA_BIDIRECTIONAL));
if (ret) {
rga_err("iommu can not map phys_addr to iova");
rga_iommu_dma_free_iova(domain, iova, align_size);
return ret;
}
buffer->domain = domain;
buffer->iova = iova;
buffer->size = align_size;
return 0;
}
int rga_virtual_memory_check(void *vaddr, u32 w, u32 h, u32 format, int fd)
{
int bits = 32;
@@ -424,8 +251,67 @@ int rga_dma_memory_check(struct rga_dma_buffer *rga_dma_buffer, struct rga_img_i
return ret;
}
int rga_dma_map_phys_addr(phys_addr_t phys_addr, size_t size, struct rga_dma_buffer *buffer,
enum dma_data_direction dir, struct device *map_dev)
{
dma_addr_t addr;
addr = dma_map_resource(map_dev, phys_addr, size, dir, 0);
if (addr == DMA_MAPPING_ERROR) {
rga_err("dma_map_resouce failed!\n");
return -EINVAL;
}
buffer->dma_addr = addr;
buffer->dir = dir;
buffer->size = size;
buffer->map_dev = map_dev;
return 0;
}
void rga_dma_unmap_phys_addr(struct rga_dma_buffer *buffer)
{
dma_unmap_resource(buffer->map_dev, buffer->dma_addr, buffer->size, buffer->dir, 0);
}
int rga_dma_map_sgt(struct sg_table *sgt, struct rga_dma_buffer *buffer,
enum dma_data_direction dir, struct device *map_dev)
{
int i, ret = 0;
struct scatterlist *sg = NULL;
ret = dma_map_sg(map_dev, sgt->sgl, sgt->orig_nents, dir);
if (ret <= 0) {
rga_err("dma_map_sg failed! ret = %d\n", ret);
return ret < 0 ? ret : -EINVAL;
}
sgt->nents = ret;
buffer->sgt = sgt;
buffer->dma_addr = sg_dma_address(sgt->sgl);
buffer->dir = dir;
buffer->size = 0;
for_each_sgtable_sg(sgt, sg, i)
buffer->size += sg_dma_len(sg);
buffer->map_dev = map_dev;
return 0;
}
void rga_dma_unmap_sgt(struct rga_dma_buffer *buffer)
{
if (!buffer->sgt)
return;
dma_unmap_sg(buffer->map_dev,
buffer->sgt->sgl,
buffer->sgt->orig_nents,
buffer->dir);
}
int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buffer,
enum dma_data_direction dir, struct device *rga_dev)
enum dma_data_direction dir, struct device *map_dev)
{
struct dma_buf_attachment *attach = NULL;
struct sg_table *sgt = NULL;
@@ -439,7 +325,7 @@ int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buff
return -EINVAL;
}
attach = dma_buf_attach(dma_buf, rga_dev);
attach = dma_buf_attach(dma_buf, map_dev);
if (IS_ERR(attach)) {
ret = PTR_ERR(attach);
rga_err("Failed to attach dma_buf, ret[%d]\n", ret);
@@ -461,6 +347,7 @@ int rga_dma_map_buf(struct dma_buf *dma_buf, struct rga_dma_buffer *rga_dma_buff
rga_dma_buffer->size = 0;
for_each_sgtable_sg(sgt, sg, i)
rga_dma_buffer->size += sg_dma_len(sg);
rga_dma_buffer->map_dev = map_dev;
return ret;
@@ -475,7 +362,7 @@ err_get_attach:
}
int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
enum dma_data_direction dir, struct device *rga_dev)
enum dma_data_direction dir, struct device *map_dev)
{
struct dma_buf *dma_buf = NULL;
struct dma_buf_attachment *attach = NULL;
@@ -490,7 +377,7 @@ int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
return ret;
}
attach = dma_buf_attach(dma_buf, rga_dev);
attach = dma_buf_attach(dma_buf, map_dev);
if (IS_ERR(attach)) {
ret = PTR_ERR(attach);
rga_err("Failed to attach dma_buf, ret[%d]\n", ret);
@@ -512,6 +399,7 @@ int rga_dma_map_fd(int fd, struct rga_dma_buffer *rga_dma_buffer,
rga_dma_buffer->size = 0;
for_each_sgtable_sg(sgt, sg, i)
rga_dma_buffer->size += sg_dma_len(sg);
rga_dma_buffer->map_dev = map_dev;
return ret;
@@ -551,12 +439,12 @@ int rga_dma_free(struct rga_dma_buffer *buffer)
return -EINVAL;
}
dma_free_coherent(buffer->scheduler->dev, buffer->size, buffer->vaddr, buffer->dma_addr);
dma_free_coherent(buffer->map_dev, buffer->size, buffer->vaddr, buffer->dma_addr);
buffer->vaddr = NULL;
buffer->dma_addr = 0;
buffer->iova = 0;
buffer->size = 0;
buffer->scheduler = NULL;
buffer->map_dev = NULL;
kfree(buffer);
@@ -581,7 +469,7 @@ struct rga_dma_buffer *rga_dma_alloc_coherent(struct rga_scheduler_t *scheduler,
buffer->size = align_size;
buffer->dma_addr = dma_addr;
buffer->scheduler = scheduler;
buffer->map_dev = scheduler->dev;
if (scheduler->data->mmu == RGA_IOMMU)
buffer->iova = buffer->dma_addr;

202
drivers/video/rockchip/rga3/rga_mm.c
View File

@@ -179,11 +179,10 @@ static struct sg_table *rga_alloc_sgt(struct rga_virt_addr *virt_addr)
}
/* get sg form pages. */
/* iova requires minimum page alignment, so sgt cannot have offset */
ret = sg_alloc_table_from_pages(sgt,
virt_addr->pages,
virt_addr->page_count,
0,
virt_addr->offset,
virt_addr->size,
GFP_KERNEL);
if (ret) {
@@ -232,7 +231,6 @@ static int rga_alloc_virt_addr(struct rga_virt_addr **virt_addr_p,
unsigned int count;
int img_size;
size_t offset;
unsigned long size;
struct page **pages = NULL;
struct rga_virt_addr *virt_addr = NULL;
@@ -246,10 +244,9 @@ static int rga_alloc_virt_addr(struct rga_virt_addr **virt_addr_p,
offset = viraddr & (~PAGE_MASK);
count = RGA_GET_PAGE_COUNT(img_size + offset);
size = count * PAGE_SIZE;
if (!size) {
rga_err("failed to calculating buffer size! size = %ld, count = %d, offset = %ld\n",
size, count, (unsigned long)offset);
if (!count) {
rga_err("failed to calculating buffer size! img_size = %d, count = %d, offset = %ld\n",
img_size, count, (unsigned long)offset);
rga_dump_memory_parm(memory_parm);
return -EFAULT;
}
@@ -292,7 +289,7 @@ static int rga_alloc_virt_addr(struct rga_virt_addr **virt_addr_p,
virt_addr->pages = pages;
virt_addr->pages_order = order;
virt_addr->page_count = count;
virt_addr->size = size;
virt_addr->size = img_size;
virt_addr->offset = offset;
virt_addr->result = result;
@@ -398,18 +395,17 @@ static int rga_mm_map_dma_buffer(struct rga_external_buffer *external_buffer,
return ex_buffer_size == 0 ? -EINVAL : ex_buffer_size;
}
/*
* dma-buf api needs to use default_domain of main dev,
* and not IOMMU for devices without iommu_info ptr.
*/
map_dev = scheduler->iommu_info ? scheduler->iommu_info->default_dev : scheduler->dev;
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
if (buffer == NULL) {
rga_err("%s alloc internal_buffer error!\n", __func__);
return -ENOMEM;
}
/*
* dma-buf api needs to use default_domain of main dev,
* and not IOMMU for devices without iommu_info ptr.
*/
map_dev = scheduler->iommu_info ? scheduler->iommu_info->default_dev : scheduler->dev;
switch (external_buffer->type) {
case RGA_DMA_BUFFER:
ret = rga_dma_map_fd((int)external_buffer->memory,
@@ -440,8 +436,6 @@ static int rga_mm_map_dma_buffer(struct rga_external_buffer *external_buffer,
goto unmap_buffer;
}
buffer->scheduler = scheduler;
if (scheduler->data->mmu == RGA_IOMMU)
buffer->iova = buffer->dma_addr;
@@ -473,6 +467,8 @@ static int rga_mm_map_dma_buffer(struct rga_external_buffer *external_buffer,
internal_buffer->dma_buffer = buffer;
internal_buffer->mm_flag = mm_flag;
internal_buffer->phys_addr = phys_addr ? phys_addr : 0;
internal_buffer->size = buffer->size - buffer->offset;
internal_buffer->scheduler = scheduler;
return 0;
@@ -492,19 +488,7 @@ static void rga_mm_unmap_virt_addr(struct rga_internal_buffer *internal_buffer)
if (rga_mm_is_invalid_dma_buffer(internal_buffer->dma_buffer))
return;
switch (internal_buffer->dma_buffer->scheduler->data->mmu) {
case RGA_IOMMU:
rga_iommu_unmap(internal_buffer->dma_buffer);
break;
case RGA_MMU:
dma_unmap_sg(internal_buffer->dma_buffer->scheduler->dev,
internal_buffer->dma_buffer->sgt->sgl,
internal_buffer->dma_buffer->sgt->orig_nents,
DMA_BIDIRECTIONAL);
break;
default:
break;
}
rga_dma_unmap_sgt(internal_buffer->dma_buffer);
if (internal_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS &&
internal_buffer->phys_addr > 0)
@@ -532,6 +516,7 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer,
struct sg_table *sgt;
struct rga_virt_addr *virt_addr;
struct rga_dma_buffer *buffer;
struct device *map_dev;
struct rga_scheduler_t *scheduler;
scheduler = job ? job->scheduler :
@@ -578,6 +563,11 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer,
}
mm_flag |= RGA_MEM_PHYSICAL_CONTIGUOUS;
} else if (scheduler->data->mmu == RGA_NONE_MMU) {
rga_err("Current %s[%d] cannot support physically discontinuous virtual address!\n",
rga_get_mmu_type_str(scheduler->data->mmu), scheduler->data->mmu);
ret = -EOPNOTSUPP;
goto free_sgt;
}
/*
@@ -601,47 +591,28 @@ static int rga_mm_map_virt_addr(struct rga_external_buffer *external_buffer,
goto free_sgt;
}
switch (scheduler->data->mmu) {
case RGA_IOMMU:
ret = rga_iommu_map_sgt(sgt, virt_addr->size, buffer, scheduler->dev);
if (ret < 0) {
rga_err("%s core[%d] iommu_map virtual address error!\n",
__func__, scheduler->core);
goto free_dma_buffer;
}
buffer->dma_addr = buffer->iova;
break;
case RGA_MMU:
ret = dma_map_sg(scheduler->dev, sgt->sgl, sgt->orig_nents, DMA_BIDIRECTIONAL);
if (ret == 0) {
rga_err("%s core[%d] dma_map_sgt error! va = 0x%lx, nents = %d\n",
__func__, scheduler->core,
(unsigned long)virt_addr->addr, sgt->orig_nents);
ret = -EINVAL;
goto free_dma_buffer;
}
break;
default:
if (mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS)
break;
rga_err("Current %s[%d] cannot support physically discontinuous virtual address!\n",
rga_get_mmu_type_str(scheduler->data->mmu), scheduler->data->mmu);
ret = -EOPNOTSUPP;
/*
* dma-buf api needs to use default_domain of main dev,
* and not IOMMU for devices without iommu_info ptr.
*/
map_dev = scheduler->iommu_info ? scheduler->iommu_info->default_dev : scheduler->dev;
ret = rga_dma_map_sgt(sgt, buffer, DMA_BIDIRECTIONAL, map_dev);
if (ret < 0) {
rga_err("%s core[%d] rga map sgt failed! va = 0x%lx, orig_nents = %d\n",
__func__, scheduler->core,
(unsigned long)virt_addr->addr, sgt->orig_nents);
goto free_dma_buffer;
}
buffer->sgt = sgt;
buffer->offset = virt_addr->offset;
buffer->size = virt_addr->size;
buffer->scheduler = scheduler;
if (scheduler->data->mmu == RGA_IOMMU)
buffer->iova = buffer->dma_addr;
internal_buffer->virt_addr = virt_addr;
internal_buffer->dma_buffer = buffer;
internal_buffer->mm_flag = mm_flag;
internal_buffer->phys_addr = phys_addr ? phys_addr + virt_addr->offset : 0;
internal_buffer->size = buffer->size - buffer->offset;
internal_buffer->scheduler = scheduler;
return 0;
@@ -661,16 +632,12 @@ put_current_mm:
static void rga_mm_unmap_phys_addr(struct rga_internal_buffer *internal_buffer)
{
WARN_ON(internal_buffer->dma_buffer == NULL);
if (internal_buffer->dma_buffer != NULL) {
rga_dma_unmap_phys_addr(internal_buffer->dma_buffer);
kfree(internal_buffer->dma_buffer);
internal_buffer->dma_buffer = NULL;
}
if (rga_mm_is_invalid_dma_buffer(internal_buffer->dma_buffer))
return;
if (internal_buffer->dma_buffer->scheduler->data->mmu == RGA_IOMMU)
rga_iommu_unmap(internal_buffer->dma_buffer);
kfree(internal_buffer->dma_buffer);
internal_buffer->dma_buffer = NULL;
internal_buffer->phys_addr = 0;
internal_buffer->size = 0;
}
@@ -683,7 +650,8 @@ static int rga_mm_map_phys_addr(struct rga_external_buffer *external_buffer,
phys_addr_t phys_addr;
int buffer_size;
uint32_t mm_flag = 0;
struct rga_dma_buffer *buffer;
struct rga_dma_buffer *buffer = NULL;
struct device *map_dev;
struct rga_scheduler_t *scheduler;
scheduler = job ? job->scheduler :
@@ -717,26 +685,34 @@ static int rga_mm_map_phys_addr(struct rga_external_buffer *external_buffer,
return -EINVAL;
}
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
if (buffer == NULL) {
rga_err("%s alloc internal dma buffer error!\n", __func__);
return -ENOMEM;
}
if (scheduler->data->mmu == RGA_IOMMU) {
ret = rga_iommu_map(phys_addr, buffer_size, buffer, scheduler->dev);
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
if (buffer == NULL) {
rga_err("%s alloc internal dma buffer error!\n", __func__);
return -ENOMEM;
}
/*
* dma-buf api needs to use default_domain of main dev,
* and not IOMMU for devices without iommu_info ptr.
*/
map_dev = scheduler->iommu_info ?
scheduler->iommu_info->default_dev : scheduler->dev;
ret = rga_dma_map_phys_addr(phys_addr, buffer_size, buffer,
DMA_BIDIRECTIONAL, map_dev);
if (ret < 0) {
rga_err("%s core[%d] map phys_addr error!\n", __func__, scheduler->core);
goto free_dma_buffer;
}
buffer->iova = buffer->dma_addr;
}
buffer->scheduler = scheduler;
internal_buffer->phys_addr = phys_addr;
internal_buffer->size = buffer_size;
internal_buffer->mm_flag = mm_flag;
internal_buffer->dma_buffer = buffer;
internal_buffer->mm_flag = mm_flag;
internal_buffer->phys_addr = phys_addr;
internal_buffer->size = buffer ? buffer->size : buffer_size;
internal_buffer->scheduler = scheduler;
return 0;
@@ -785,8 +761,6 @@ static int rga_mm_map_buffer(struct rga_external_buffer *external_buffer,
if (ret < 0)
return ret;
internal_buffer->size = internal_buffer->dma_buffer->size -
internal_buffer->dma_buffer->offset;
internal_buffer->mm_flag |= RGA_MEM_NEED_USE_IOMMU;
break;
case RGA_VIRTUAL_ADDRESS:
@@ -796,8 +770,6 @@ static int rga_mm_map_buffer(struct rga_external_buffer *external_buffer,
if (ret < 0)
return ret;
internal_buffer->size = internal_buffer->virt_addr->size -
internal_buffer->virt_addr->offset;
internal_buffer->mm_flag |= RGA_MEM_NEED_USE_IOMMU;
break;
case RGA_PHYSICAL_ADDRESS:
@@ -1002,7 +974,7 @@ void rga_mm_dump_buffer(struct rga_internal_buffer *dump_buffer)
(unsigned long)dump_buffer->dma_buffer->offset,
dump_buffer->dma_buffer->sgt,
dump_buffer->dma_buffer->size,
dump_buffer->dma_buffer->scheduler->core);
dump_buffer->scheduler->core);
if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS)
rga_log("is contiguous, pa = 0x%lx\n",
@@ -1026,7 +998,7 @@ void rga_mm_dump_buffer(struct rga_internal_buffer *dump_buffer)
(unsigned long)dump_buffer->dma_buffer->offset,
dump_buffer->dma_buffer->sgt,
dump_buffer->dma_buffer->size,
dump_buffer->dma_buffer->scheduler->core);
dump_buffer->scheduler->core);
if (dump_buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS)
rga_buf_log(dump_buffer, "is contiguous, pa = 0x%lx\n",
@@ -1034,6 +1006,16 @@ void rga_mm_dump_buffer(struct rga_internal_buffer *dump_buffer)
break;
case RGA_PHYSICAL_ADDRESS:
rga_buf_log(dump_buffer, "pa = 0x%lx\n", (unsigned long)dump_buffer->phys_addr);
if (rga_mm_is_invalid_dma_buffer(dump_buffer->dma_buffer))
break;
rga_buf_log(dump_buffer, "iova = 0x%lx, dma_addr = 0x%lx, offset = 0x%lx, size = %ld, map_core = 0x%x\n",
(unsigned long)dump_buffer->dma_buffer->iova,
(unsigned long)dump_buffer->dma_buffer->dma_addr,
(unsigned long)dump_buffer->dma_buffer->offset,
dump_buffer->dma_buffer->size,
dump_buffer->scheduler->core);
break;
default:
rga_buf_err(dump_buffer, "Illegal buffer! type= %d\n", dump_buffer->type);
@@ -1129,7 +1111,7 @@ static int rga_mm_sgt_to_page_table(struct sg_table *sg,
* The length of each sgl is expected to be obtained here, not
* the length of the entire dma_buf, so sg_dma_len() is not used.
*/
len = sgl->length >> PAGE_SHIFT;
len = (sgl->length + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
if (use_dma_address)
/*
@@ -1148,6 +1130,8 @@ static int rga_mm_sgt_to_page_table(struct sg_table *sg,
else
Address = sg_phys(sgl);
Address &= PAGE_MASK;
for (i = 0; i < len; i++) {
if (mapped_size + i >= pageCount) {
break_flag = 1;
@@ -1335,16 +1319,26 @@ static int rga_mm_sync_dma_sg_for_device(struct rga_internal_buffer *buffer,
struct rga_scheduler_t *scheduler;
ktime_t timestamp = ktime_get();
scheduler = buffer->dma_buffer->scheduler;
scheduler = buffer->scheduler;
if (scheduler == NULL) {
rga_job_err(job, "%s(%d), failed to get scheduler, core = 0x%x\n",
__func__, __LINE__, job->core);
return -EFAULT;
}
if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS &&
scheduler->data->mmu != RGA_IOMMU) {
dma_sync_single_for_device(scheduler->dev, buffer->phys_addr, buffer->size, dir);
if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) {
if (scheduler->data->mmu == RGA_IOMMU) {
if (rga_mm_is_invalid_dma_buffer(buffer->dma_buffer)) {
rga_job_err(job, "invalid dma-buffer with IOMMU device!\n");
return -EFAULT;
}
dma_sync_single_for_device(buffer->dma_buffer->map_dev,
buffer->dma_buffer->iova, buffer->dma_buffer->size, dir);
} else {
dma_sync_single_for_device(scheduler->dev,
buffer->phys_addr, buffer->size, dir);
}
} else {
sgt = rga_mm_lookup_sgt(buffer);
if (sgt == NULL) {
@@ -1372,16 +1366,26 @@ static int rga_mm_sync_dma_sg_for_cpu(struct rga_internal_buffer *buffer,
struct rga_scheduler_t *scheduler;
ktime_t timestamp = ktime_get();
scheduler = buffer->dma_buffer->scheduler;
scheduler = buffer->scheduler;
if (scheduler == NULL) {
rga_job_err(job, "%s(%d), failed to get scheduler, core = 0x%x\n",
__func__, __LINE__, job->core);
return -EFAULT;
}
if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS &&
scheduler->data->mmu != RGA_IOMMU) {
dma_sync_single_for_cpu(scheduler->dev, buffer->phys_addr, buffer->size, dir);
if (buffer->mm_flag & RGA_MEM_PHYSICAL_CONTIGUOUS) {
if (scheduler->data->mmu == RGA_IOMMU) {
if (rga_mm_is_invalid_dma_buffer(buffer->dma_buffer)) {
rga_job_err(job, "invalid dma-buffer with IOMMU device!\n");
return -EFAULT;
}
dma_sync_single_for_cpu(buffer->dma_buffer->map_dev,
buffer->dma_buffer->iova, buffer->dma_buffer->size, dir);
} else {
dma_sync_single_for_cpu(scheduler->dev,
buffer->phys_addr, buffer->size, dir);
}
} else {
sgt = rga_mm_lookup_sgt(buffer);
if (sgt == NULL) {