diff --git a/drivers/video/rockchip/vcodec/vcodec_iommu_drm.c b/drivers/video/rockchip/vcodec/vcodec_iommu_drm.c index 0368ffa7688b..90d2c002991e 100644 --- a/drivers/video/rockchip/vcodec/vcodec_iommu_drm.c +++ b/drivers/video/rockchip/vcodec/vcodec_iommu_drm.c @@ -25,6 +25,14 @@ #include "vcodec_iommu_ops.h" +/* + * there two type drm buffer, one is through import interface and + * the other is through alloc. there are some differents between + * import and alloc buffer when free them. this flag is used to + * inform driver which type of this buffer. + */ +#define VCODEC_DRM_BUFFER_ALLOC 0x00000001 + struct vcodec_drm_buffer { struct list_head list; struct dma_buf *dma_buf; @@ -32,6 +40,7 @@ struct vcodec_drm_buffer { dma_addr_t iova; unsigned long phys; }; + void *cpu_addr; unsigned long size; int index; struct dma_buf_attachment *attach; @@ -41,6 +50,7 @@ struct vcodec_drm_buffer { struct kref ref; struct vcodec_iommu_session_info *session_info; ktime_t last_used; + int flags; }; struct vcodec_iommu_drm_info { @@ -88,6 +98,182 @@ vcodec_drm_get_buffer_fd_no_lock(struct vcodec_iommu_session_info *session_info, return NULL; } +static void *vcodec_drm_sgt_map_kernel(struct vcodec_drm_buffer *drm_buffer) +{ + struct vcodec_iommu_session_info *session_info = + drm_buffer->session_info; + struct device *dev = session_info->dev; + struct scatterlist *sgl, *sg; + int nr_pages = PAGE_ALIGN(drm_buffer->size) >> PAGE_SHIFT; + int i = 0, j = 0, k = 0; + struct page *page; + + drm_buffer->pages = kmalloc_array(nr_pages, sizeof(*drm_buffer->pages), + GFP_KERNEL); + if (!(drm_buffer->pages)) { + dev_err(dev, "drm map alloc %d pages failed!\n", nr_pages); + + return NULL; + } + + sgl = drm_buffer->copy_sgt->sgl; + + for_each_sg(sgl, sg, drm_buffer->copy_sgt->nents, i) { + page = sg_page(sg); + for (j = 0; j < sg->length / PAGE_SIZE; j++) + drm_buffer->pages[k++] = page++; + } + + return vmap(drm_buffer->pages, nr_pages, VM_MAP, + pgprot_noncached(PAGE_KERNEL)); +} + +static void vcodec_drm_sgt_unmap_kernel(struct vcodec_drm_buffer *drm_buffer) +{ + vunmap(drm_buffer->cpu_addr); + kfree(drm_buffer->pages); +} + +static void vcodec_dma_unmap_sg(struct iommu_domain *domain, + dma_addr_t dma_addr) +{ + struct iova_domain *iovad = domain->iova_cookie; + unsigned long shift = iova_shift(iovad); + unsigned long pfn = dma_addr >> shift; + struct iova *iova = find_iova(iovad, pfn); + size_t size; + + if (WARN_ON(!iova)) + return; + + size = iova_size(iova) << shift; + size -= iommu_unmap(domain, pfn << shift, size); + /* ...and if we can't, then something is horribly, horribly wrong */ + WARN_ON(size > 0); + __free_iova(iovad, iova); +} + +static void vcodec_drm_clear_map(struct kref *ref) +{ + struct vcodec_drm_buffer *drm_buffer = + container_of(ref, struct vcodec_drm_buffer, ref); + struct vcodec_iommu_session_info *session_info = + drm_buffer->session_info; + struct vcodec_iommu_info *iommu_info = session_info->iommu_info; + struct vcodec_iommu_drm_info *drm_info = iommu_info->private; + struct sg_table *table; + struct scatterlist *sg; + struct page *page; + int i; + + mutex_lock(&iommu_info->iommu_mutex); + drm_info = session_info->iommu_info->private; + + if (drm_buffer->cpu_addr) { + vcodec_drm_sgt_unmap_kernel(drm_buffer); + drm_buffer->cpu_addr = NULL; + } + + vcodec_dma_unmap_sg(drm_info->domain, drm_buffer->iova); + if (drm_buffer->flags & VCODEC_DRM_BUFFER_ALLOC) { + table = drm_buffer->copy_sgt; + for_each_sg(table->sgl, sg, table->nents, i) { + page = sg_page(sg); + __free_pages(page, compound_order(page)); + } + } + sg_free_table(drm_buffer->copy_sgt); + kfree(drm_buffer->copy_sgt); + + if (drm_buffer->attach) { + dma_buf_unmap_attachment(drm_buffer->attach, drm_buffer->sgt, + DMA_BIDIRECTIONAL); + dma_buf_detach(drm_buffer->dma_buf, drm_buffer->attach); + dma_buf_put(drm_buffer->dma_buf); + drm_buffer->attach = NULL; + } + + mutex_unlock(&iommu_info->iommu_mutex); +} + +static void* +vcodec_drm_map_kernel(struct vcodec_iommu_session_info *session_info, int idx) +{ + struct device *dev = session_info->dev; + struct vcodec_drm_buffer *drm_buffer; + + mutex_lock(&session_info->list_mutex); + drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx); + mutex_unlock(&session_info->list_mutex); + + if (!drm_buffer) { + dev_err(dev, "can not find %d buffer in list\n", idx); + return NULL; + } + + if (!drm_buffer->cpu_addr) + drm_buffer->cpu_addr = + vcodec_drm_sgt_map_kernel(drm_buffer); + + kref_get(&drm_buffer->ref); + + return drm_buffer->cpu_addr; +} + +static int +vcodec_drm_unmap_kernel(struct vcodec_iommu_session_info *session_info, int idx) +{ + struct device *dev = session_info->dev; + struct vcodec_drm_buffer *drm_buffer; + + mutex_lock(&session_info->list_mutex); + drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx); + mutex_unlock(&session_info->list_mutex); + + if (!drm_buffer) { + dev_err(dev, "can not find %d buffer in list\n", idx); + + return -EINVAL; + } + + if (drm_buffer->cpu_addr) { + vcodec_drm_sgt_unmap_kernel(drm_buffer); + drm_buffer->cpu_addr = NULL; + } + + kref_put(&drm_buffer->ref, vcodec_drm_clear_map); + return 0; +} + +static void +vcodec_drm_remove_extra_buffer_no_lock(struct vcodec_iommu_session_info + *session_info) +{ + struct vcodec_drm_buffer *oldest_buffer = NULL, *loop_buffer = NULL; + struct vcodec_drm_buffer *n; + ktime_t oldest_time = ktime_set(0, 0); + + if (session_info->buffer_nums > BUFFER_LIST_MAX_NUMS) { + list_for_each_entry_safe(loop_buffer, n, + &session_info->buffer_list, list) { + if (loop_buffer->flags & VCODEC_DRM_BUFFER_ALLOC) + continue; + + if (ktime_to_ns(oldest_time) == 0 || + ktime_after(oldest_time, + loop_buffer->last_used)) { + oldest_time = loop_buffer->last_used; + oldest_buffer = loop_buffer; + } + } + kref_put(&oldest_buffer->ref, vcodec_drm_clear_map); + dma_buf_put(oldest_buffer->dma_buf); + list_del_init(&oldest_buffer->list); + kfree(oldest_buffer); + session_info->buffer_nums--; + } +} + static void vcodec_drm_detach(struct vcodec_iommu_info *iommu_info) { struct vcodec_iommu_drm_info *drm_info = iommu_info->private; @@ -271,7 +457,7 @@ static dma_addr_t vcodec_dma_map_sg(struct iommu_domain *domain, } iova = alloc_iova(iovad, iova_align(iovad, iova_len) >> shift, - mask >> shift, true); + mask >> shift, true); if (!iova) goto out_restore_sg; @@ -292,51 +478,6 @@ out_restore_sg: return 0; } -static void vcodec_dma_unmap_sg(struct iommu_domain *domain, - dma_addr_t dma_addr) -{ - struct iova_domain *iovad = domain->iova_cookie; - unsigned long shift = iova_shift(iovad); - unsigned long pfn = dma_addr >> shift; - struct iova *iova = find_iova(iovad, pfn); - size_t size; - - if (WARN_ON(!iova)) - return; - - size = iova_size(iova) << shift; - size -= iommu_unmap(domain, pfn << shift, size); - /* ...and if we can't, then something is horribly, horribly wrong */ - WARN_ON(size > 0); - __free_iova(iovad, iova); -} - -static void vcodec_drm_clear_map(struct kref *ref) -{ - struct vcodec_drm_buffer *drm_buffer = - container_of(ref, struct vcodec_drm_buffer, ref); - struct vcodec_iommu_session_info *session_info = - drm_buffer->session_info; - struct vcodec_iommu_info *iommu_info = session_info->iommu_info; - struct vcodec_iommu_drm_info *drm_info = iommu_info->private; - - mutex_lock(&iommu_info->iommu_mutex); - drm_info = session_info->iommu_info->private; - - if (drm_buffer->attach) { - vcodec_dma_unmap_sg(drm_info->domain, drm_buffer->iova); - sg_free_table(drm_buffer->copy_sgt); - kfree(drm_buffer->copy_sgt); - dma_buf_unmap_attachment(drm_buffer->attach, drm_buffer->sgt, - DMA_BIDIRECTIONAL); - dma_buf_detach(drm_buffer->dma_buf, drm_buffer->attach); - dma_buf_put(drm_buffer->dma_buf); - drm_buffer->attach = NULL; - } - - mutex_unlock(&iommu_info->iommu_mutex); -} - static void vcdoec_drm_dump_info(struct vcodec_iommu_session_info *session_info) { struct vcodec_drm_buffer *drm_buffer = NULL, *n; @@ -478,14 +619,12 @@ static int vcodec_drm_import(struct vcodec_iommu_session_info *session_info, int fd) { struct vcodec_drm_buffer *drm_buffer = NULL, *n; - struct vcodec_drm_buffer *oldest_buffer = NULL, *loop_buffer = NULL; struct vcodec_iommu_info *iommu_info = session_info->iommu_info; struct vcodec_iommu_drm_info *drm_info = iommu_info->private; struct device *dev = session_info->dev; struct dma_buf_attachment *attach; struct sg_table *sgt; struct dma_buf *dma_buf; - ktime_t oldest_time = ktime_set(0, 0); struct scatterlist *sg, *s; int i; int ret = 0; @@ -578,22 +717,7 @@ static int vcodec_drm_import(struct vcodec_iommu_session_info *session_info, session_info->buffer_nums++; vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL, "buffer nums %d\n", session_info->buffer_nums); - if (session_info->buffer_nums > BUFFER_LIST_MAX_NUMS) { - list_for_each_entry_safe(loop_buffer, n, - &session_info->buffer_list, list) { - if (ktime_to_ns(oldest_time) == 0 || - ktime_after(oldest_time, - loop_buffer->last_used)) { - oldest_time = loop_buffer->last_used; - oldest_buffer = loop_buffer; - } - } - kref_put(&oldest_buffer->ref, vcodec_drm_clear_map); - dma_buf_put(oldest_buffer->dma_buf); - list_del_init(&oldest_buffer->list); - kfree(oldest_buffer); - session_info->buffer_nums--; - } + vcodec_drm_remove_extra_buffer_no_lock(session_info); drm_buffer->index = session_info->max_idx; list_add_tail(&drm_buffer->list, &session_info->buffer_list); session_info->max_idx++; @@ -618,6 +742,103 @@ fail_out: return ret; } +static int vcodec_drm_alloc(struct vcodec_iommu_session_info *session_info, + unsigned long size, + unsigned long align) +{ + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + long size_remaining = PAGE_ALIGN(size); + struct vcodec_drm_buffer *drm_buffer; + struct vcodec_iommu_info *iommu_info = session_info->iommu_info; + struct vcodec_iommu_drm_info *drm_info = iommu_info->private; + int i; + + if (align > PAGE_SIZE) + return -EINVAL; + + if (size / PAGE_SIZE > totalram_pages / 2) + return -ENOMEM; + + drm_buffer = kzalloc(sizeof(*drm_buffer), GFP_KERNEL); + if (!drm_buffer) + return -ENOMEM; + + drm_buffer->session_info = session_info; + drm_buffer->last_used = ktime_set(0, 0); + + kref_init(&drm_buffer->ref); + + INIT_LIST_HEAD(&pages); + + i = 0; + while (size_remaining > 0) { + gfp_t gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | + __GFP_NORETRY) & ~__GFP_DIRECT_RECLAIM; + page = alloc_pages(gfp_flags | __GFP_COMP, 8); + if (!page) + goto free_pages; + + size_remaining -= PAGE_SIZE << compound_order(page); + list_add_tail(&page->lru, &pages); + i++; + } + + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto free_pages; + + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_table; + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, PAGE_SIZE << compound_order(page), 0); + sg = sg_next(sg); + list_del(&page->lru); + } + + mutex_lock(&iommu_info->iommu_mutex); + drm_info = session_info->iommu_info->private; + drm_buffer->copy_sgt = table; + + vcodec_dma_map_sg(drm_info->domain, drm_buffer->copy_sgt->sgl, + drm_buffer->copy_sgt->nents, + IOMMU_READ | IOMMU_WRITE); + drm_buffer->iova = sg_dma_address(drm_buffer->copy_sgt->sgl); + drm_buffer->size = size; + drm_buffer->flags = VCODEC_DRM_BUFFER_ALLOC; + + mutex_unlock(&iommu_info->iommu_mutex); + + INIT_LIST_HEAD(&drm_buffer->list); + mutex_lock(&session_info->list_mutex); + session_info->buffer_nums++; + vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL, + "buffer nums %d\n", session_info->buffer_nums); + vcodec_drm_remove_extra_buffer_no_lock(session_info); + drm_buffer->index = session_info->max_idx; + list_add_tail(&drm_buffer->list, &session_info->buffer_list); + session_info->max_idx++; + if ((session_info->max_idx & 0xfffffff) == 0) + session_info->max_idx = 0; + mutex_unlock(&session_info->list_mutex); + + return drm_buffer->index; + +free_table: + kfree(table); +free_pages: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + __free_pages(page, 8); + + kfree(drm_buffer); + + return -ENOMEM; +} + static int vcodec_drm_create(struct vcodec_iommu_info *iommu_info) { struct vcodec_iommu_drm_info *drm_info; @@ -682,9 +903,12 @@ static int vcodec_drm_destroy(struct vcodec_iommu_info *iommu_info) static struct vcodec_iommu_ops drm_ops = { .create = vcodec_drm_create, + .alloc = vcodec_drm_alloc, .import = vcodec_drm_import, .free = vcodec_drm_free, .free_fd = vcodec_drm_free_fd, + .map_kernel = vcodec_drm_map_kernel, + .unmap_kernel = vcodec_drm_unmap_kernel, .map_iommu = vcodec_drm_map_iommu, .unmap_iommu = vcodec_drm_unmap_iommu, .destroy = vcodec_drm_destroy, diff --git a/drivers/video/rockchip/vcodec/vcodec_iommu_ion.c b/drivers/video/rockchip/vcodec/vcodec_iommu_ion.c index 00145e562dcf..3b97e5458463 100644 --- a/drivers/video/rockchip/vcodec/vcodec_iommu_ion.c +++ b/drivers/video/rockchip/vcodec/vcodec_iommu_ion.c @@ -59,7 +59,6 @@ vcodec_ion_clear_session(struct vcodec_iommu_session_info *session_info) static int vcodec_ion_attach(struct vcodec_iommu_info *iommu_info) { struct vcodec_iommu_ion_info *ion_info = iommu_info->private; - int ret; mutex_lock(&iommu_info->iommu_mutex); @@ -74,7 +73,7 @@ static int vcodec_ion_attach(struct vcodec_iommu_info *iommu_info) mutex_unlock(&iommu_info->iommu_mutex); - return ret; + return 0; } static void vcodec_ion_detach(struct vcodec_iommu_info *iommu_info) @@ -208,6 +207,80 @@ vcodec_ion_import(struct vcodec_iommu_session_info *session_info, int fd) return ion_buffer->index; } +static int +vcodec_ion_unmap_kernel(struct vcodec_iommu_session_info *session_info, + int idx) +{ + struct vcodec_ion_buffer *ion_buffer; + + mutex_lock(&session_info->list_mutex); + ion_buffer = vcodec_ion_get_buffer_no_lock(session_info, idx); + mutex_unlock(&session_info->list_mutex); + + if (!ion_buffer) { + pr_err("%s can not find %d buffer in list\n", __func__, idx); + + return -EINVAL; + } + + return 0; +} + +static void* +vcodec_ion_map_kernel(struct vcodec_iommu_session_info *session_info, int idx) +{ + struct vcodec_ion_buffer *ion_buffer; + struct vcodec_iommu_info *iommu_info = session_info->iommu_info; + struct vcodec_iommu_ion_info *ion_info = iommu_info->private; + + rockchip_iovmm_invalidate_tlb(session_info->dev); + + mutex_lock(&session_info->list_mutex); + ion_buffer = vcodec_ion_get_buffer_no_lock(session_info, idx); + mutex_unlock(&session_info->list_mutex); + + if (!ion_buffer) { + pr_err("%s can not find %d buffer in list\n", __func__, idx); + + return NULL; + } + + return ion_map_kernel(ion_info->ion_client, ion_buffer->handle); +} + +static int vcodec_ion_alloc(struct vcodec_iommu_session_info *session_info, + unsigned long size, + unsigned long align) +{ + struct vcodec_ion_buffer *ion_buffer = NULL; + struct vcodec_iommu_info *iommu_info = session_info->iommu_info; + struct vcodec_iommu_ion_info *ion_info = iommu_info->private; + unsigned int heap_id_mask; + + if (iommu_info->mmu_dev) + heap_id_mask = ION_HEAP_TYPE_SYSTEM; + else + heap_id_mask = ION_HEAP_TYPE_DMA; + + ion_buffer = kzalloc(sizeof(*ion_buffer), GFP_KERNEL); + if (!ion_buffer) + return -ENOMEM; + + ion_buffer->handle = ion_alloc(ion_info->ion_client, size, + align, heap_id_mask, 0); + + INIT_LIST_HEAD(&ion_buffer->list); + mutex_lock(&session_info->list_mutex); + ion_buffer->index = session_info->max_idx; + list_add_tail(&ion_buffer->list, &session_info->buffer_list); + session_info->max_idx++; + if ((session_info->max_idx & 0xfffffff) == 0) + session_info->max_idx = 0; + mutex_unlock(&session_info->list_mutex); + + return ion_buffer->index; +} + static int vcodec_ion_create(struct vcodec_iommu_info *iommu_info) { struct vcodec_iommu_ion_info *ion_info; @@ -229,9 +302,12 @@ static int vcodec_ion_create(struct vcodec_iommu_info *iommu_info) static struct vcodec_iommu_ops ion_ops = { .create = vcodec_ion_create, .destroy = vcodec_ion_destroy, + .alloc = vcodec_ion_alloc, .import = vcodec_ion_import, .free = vcodec_ion_free, .free_fd = NULL, + .map_kernel = vcodec_ion_map_kernel, + .unmap_kernel = vcodec_ion_unmap_kernel, .map_iommu = vcodec_ion_map_iommu, .unmap_iommu = vcodec_ion_unmap_iommu, .dump = NULL, diff --git a/drivers/video/rockchip/vcodec/vcodec_iommu_ops.c b/drivers/video/rockchip/vcodec/vcodec_iommu_ops.c index e9fd423b7aeb..891eb0f3aaf7 100644 --- a/drivers/video/rockchip/vcodec/vcodec_iommu_ops.c +++ b/drivers/video/rockchip/vcodec/vcodec_iommu_ops.c @@ -41,6 +41,41 @@ int vcodec_iommu_create(struct vcodec_iommu_info *iommu_info) return iommu_info->ops->create(iommu_info); } +int vcodec_iommu_alloc(struct vcodec_iommu_info *iommu_info, + struct vpu_session *session, + unsigned long size, + unsigned long align) +{ + struct vcodec_iommu_session_info *session_info = NULL; + + if (!iommu_info || !iommu_info->ops->alloc || !session) + return -EINVAL; + + session_info = vcodec_iommu_get_session_info(iommu_info, session); + if (!session_info) { + session_info = kzalloc(sizeof(*session_info), GFP_KERNEL); + if (!session_info) + return -ENOMEM; + + INIT_LIST_HEAD(&session_info->head); + INIT_LIST_HEAD(&session_info->buffer_list); + mutex_init(&session_info->list_mutex); + session_info->max_idx = 0; + session_info->session = session; + session_info->mmu_dev = iommu_info->mmu_dev; + session_info->dev = iommu_info->dev; + session_info->iommu_info = iommu_info; + session_info->buffer_nums = 0; + mutex_lock(&iommu_info->list_mutex); + list_add_tail(&session_info->head, &iommu_info->session_list); + mutex_unlock(&iommu_info->list_mutex); + } + + session_info->debug_level = iommu_info->debug_level; + + return iommu_info->ops->alloc(session_info, size, align); +} + int vcodec_iommu_import(struct vcodec_iommu_info *iommu_info, struct vpu_session *session, int fd) { diff --git a/drivers/video/rockchip/vcodec/vcodec_iommu_ops.h b/drivers/video/rockchip/vcodec/vcodec_iommu_ops.h index 3e65d3aa0edd..f9ea7c493c7e 100644 --- a/drivers/video/rockchip/vcodec/vcodec_iommu_ops.h +++ b/drivers/video/rockchip/vcodec/vcodec_iommu_ops.h @@ -46,9 +46,16 @@ struct vcodec_iommu_session_info; struct vcodec_iommu_ops { int (*create)(struct vcodec_iommu_info *iommu_info); + int (*alloc)(struct vcodec_iommu_session_info *session_info, + unsigned long size, + unsigned long align); int (*import)(struct vcodec_iommu_session_info *session_info, int fd); int (*free)(struct vcodec_iommu_session_info *session_info, int idx); int (*free_fd)(struct vcodec_iommu_session_info *session_info, int fd); + void* (*map_kernel)(struct vcodec_iommu_session_info *session_info, + int idx); + int (*unmap_kernel)(struct vcodec_iommu_session_info *session_info, + int idx); int (*map_iommu)(struct vcodec_iommu_session_info *session_info, int idx, dma_addr_t *iova, unsigned long *size); @@ -98,6 +105,10 @@ struct vcodec_iommu_info *vcodec_iommu_info_create(struct device *dev, int vcodec_iommu_info_destroy(struct vcodec_iommu_info *iommu_info); int vcodec_iommu_create(struct vcodec_iommu_info *iommu_info); +int vcodec_iommu_alloc(struct vcodec_iommu_info *iommu_info, + struct vpu_session *session, + unsigned long size, + unsigned long align); int vcodec_iommu_import(struct vcodec_iommu_info *iommu_info, struct vpu_session *session, int fd); int vcodec_iommu_free(struct vcodec_iommu_info *iommu_info,