video: rockchip: vpu: add alloc function

Change-Id: I61a3f556bf1aee5bac92b10eb4a9b7a2c6367178
Signed-off-by: Jung Zhao <jung.zhao@rock-chips.com>
This commit is contained in:
Jung Zhao
2017-12-21 15:55:19 +08:00
committed by Tao Huang
parent fa6d15fc93
commit b359b5f287
4 changed files with 412 additions and 66 deletions

View File

@@ -25,6 +25,14 @@
#include "vcodec_iommu_ops.h"
/*
* there two type drm buffer, one is through import interface and
* the other is through alloc. there are some differents between
* import and alloc buffer when free them. this flag is used to
* inform driver which type of this buffer.
*/
#define VCODEC_DRM_BUFFER_ALLOC 0x00000001
struct vcodec_drm_buffer {
struct list_head list;
struct dma_buf *dma_buf;
@@ -32,6 +40,7 @@ struct vcodec_drm_buffer {
dma_addr_t iova;
unsigned long phys;
};
void *cpu_addr;
unsigned long size;
int index;
struct dma_buf_attachment *attach;
@@ -41,6 +50,7 @@ struct vcodec_drm_buffer {
struct kref ref;
struct vcodec_iommu_session_info *session_info;
ktime_t last_used;
int flags;
};
struct vcodec_iommu_drm_info {
@@ -88,6 +98,182 @@ vcodec_drm_get_buffer_fd_no_lock(struct vcodec_iommu_session_info *session_info,
return NULL;
}
static void *vcodec_drm_sgt_map_kernel(struct vcodec_drm_buffer *drm_buffer)
{
struct vcodec_iommu_session_info *session_info =
drm_buffer->session_info;
struct device *dev = session_info->dev;
struct scatterlist *sgl, *sg;
int nr_pages = PAGE_ALIGN(drm_buffer->size) >> PAGE_SHIFT;
int i = 0, j = 0, k = 0;
struct page *page;
drm_buffer->pages = kmalloc_array(nr_pages, sizeof(*drm_buffer->pages),
GFP_KERNEL);
if (!(drm_buffer->pages)) {
dev_err(dev, "drm map alloc %d pages failed!\n", nr_pages);
return NULL;
}
sgl = drm_buffer->copy_sgt->sgl;
for_each_sg(sgl, sg, drm_buffer->copy_sgt->nents, i) {
page = sg_page(sg);
for (j = 0; j < sg->length / PAGE_SIZE; j++)
drm_buffer->pages[k++] = page++;
}
return vmap(drm_buffer->pages, nr_pages, VM_MAP,
pgprot_noncached(PAGE_KERNEL));
}
static void vcodec_drm_sgt_unmap_kernel(struct vcodec_drm_buffer *drm_buffer)
{
vunmap(drm_buffer->cpu_addr);
kfree(drm_buffer->pages);
}
static void vcodec_dma_unmap_sg(struct iommu_domain *domain,
dma_addr_t dma_addr)
{
struct iova_domain *iovad = domain->iova_cookie;
unsigned long shift = iova_shift(iovad);
unsigned long pfn = dma_addr >> shift;
struct iova *iova = find_iova(iovad, pfn);
size_t size;
if (WARN_ON(!iova))
return;
size = iova_size(iova) << shift;
size -= iommu_unmap(domain, pfn << shift, size);
/* ...and if we can't, then something is horribly, horribly wrong */
WARN_ON(size > 0);
__free_iova(iovad, iova);
}
static void vcodec_drm_clear_map(struct kref *ref)
{
struct vcodec_drm_buffer *drm_buffer =
container_of(ref, struct vcodec_drm_buffer, ref);
struct vcodec_iommu_session_info *session_info =
drm_buffer->session_info;
struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
struct sg_table *table;
struct scatterlist *sg;
struct page *page;
int i;
mutex_lock(&iommu_info->iommu_mutex);
drm_info = session_info->iommu_info->private;
if (drm_buffer->cpu_addr) {
vcodec_drm_sgt_unmap_kernel(drm_buffer);
drm_buffer->cpu_addr = NULL;
}
vcodec_dma_unmap_sg(drm_info->domain, drm_buffer->iova);
if (drm_buffer->flags & VCODEC_DRM_BUFFER_ALLOC) {
table = drm_buffer->copy_sgt;
for_each_sg(table->sgl, sg, table->nents, i) {
page = sg_page(sg);
__free_pages(page, compound_order(page));
}
}
sg_free_table(drm_buffer->copy_sgt);
kfree(drm_buffer->copy_sgt);
if (drm_buffer->attach) {
dma_buf_unmap_attachment(drm_buffer->attach, drm_buffer->sgt,
DMA_BIDIRECTIONAL);
dma_buf_detach(drm_buffer->dma_buf, drm_buffer->attach);
dma_buf_put(drm_buffer->dma_buf);
drm_buffer->attach = NULL;
}
mutex_unlock(&iommu_info->iommu_mutex);
}
static void*
vcodec_drm_map_kernel(struct vcodec_iommu_session_info *session_info, int idx)
{
struct device *dev = session_info->dev;
struct vcodec_drm_buffer *drm_buffer;
mutex_lock(&session_info->list_mutex);
drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
mutex_unlock(&session_info->list_mutex);
if (!drm_buffer) {
dev_err(dev, "can not find %d buffer in list\n", idx);
return NULL;
}
if (!drm_buffer->cpu_addr)
drm_buffer->cpu_addr =
vcodec_drm_sgt_map_kernel(drm_buffer);
kref_get(&drm_buffer->ref);
return drm_buffer->cpu_addr;
}
static int
vcodec_drm_unmap_kernel(struct vcodec_iommu_session_info *session_info, int idx)
{
struct device *dev = session_info->dev;
struct vcodec_drm_buffer *drm_buffer;
mutex_lock(&session_info->list_mutex);
drm_buffer = vcodec_drm_get_buffer_no_lock(session_info, idx);
mutex_unlock(&session_info->list_mutex);
if (!drm_buffer) {
dev_err(dev, "can not find %d buffer in list\n", idx);
return -EINVAL;
}
if (drm_buffer->cpu_addr) {
vcodec_drm_sgt_unmap_kernel(drm_buffer);
drm_buffer->cpu_addr = NULL;
}
kref_put(&drm_buffer->ref, vcodec_drm_clear_map);
return 0;
}
static void
vcodec_drm_remove_extra_buffer_no_lock(struct vcodec_iommu_session_info
*session_info)
{
struct vcodec_drm_buffer *oldest_buffer = NULL, *loop_buffer = NULL;
struct vcodec_drm_buffer *n;
ktime_t oldest_time = ktime_set(0, 0);
if (session_info->buffer_nums > BUFFER_LIST_MAX_NUMS) {
list_for_each_entry_safe(loop_buffer, n,
&session_info->buffer_list, list) {
if (loop_buffer->flags & VCODEC_DRM_BUFFER_ALLOC)
continue;
if (ktime_to_ns(oldest_time) == 0 ||
ktime_after(oldest_time,
loop_buffer->last_used)) {
oldest_time = loop_buffer->last_used;
oldest_buffer = loop_buffer;
}
}
kref_put(&oldest_buffer->ref, vcodec_drm_clear_map);
dma_buf_put(oldest_buffer->dma_buf);
list_del_init(&oldest_buffer->list);
kfree(oldest_buffer);
session_info->buffer_nums--;
}
}
static void vcodec_drm_detach(struct vcodec_iommu_info *iommu_info)
{
struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
@@ -271,7 +457,7 @@ static dma_addr_t vcodec_dma_map_sg(struct iommu_domain *domain,
}
iova = alloc_iova(iovad, iova_align(iovad, iova_len) >> shift,
mask >> shift, true);
mask >> shift, true);
if (!iova)
goto out_restore_sg;
@@ -292,51 +478,6 @@ out_restore_sg:
return 0;
}
static void vcodec_dma_unmap_sg(struct iommu_domain *domain,
dma_addr_t dma_addr)
{
struct iova_domain *iovad = domain->iova_cookie;
unsigned long shift = iova_shift(iovad);
unsigned long pfn = dma_addr >> shift;
struct iova *iova = find_iova(iovad, pfn);
size_t size;
if (WARN_ON(!iova))
return;
size = iova_size(iova) << shift;
size -= iommu_unmap(domain, pfn << shift, size);
/* ...and if we can't, then something is horribly, horribly wrong */
WARN_ON(size > 0);
__free_iova(iovad, iova);
}
static void vcodec_drm_clear_map(struct kref *ref)
{
struct vcodec_drm_buffer *drm_buffer =
container_of(ref, struct vcodec_drm_buffer, ref);
struct vcodec_iommu_session_info *session_info =
drm_buffer->session_info;
struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
mutex_lock(&iommu_info->iommu_mutex);
drm_info = session_info->iommu_info->private;
if (drm_buffer->attach) {
vcodec_dma_unmap_sg(drm_info->domain, drm_buffer->iova);
sg_free_table(drm_buffer->copy_sgt);
kfree(drm_buffer->copy_sgt);
dma_buf_unmap_attachment(drm_buffer->attach, drm_buffer->sgt,
DMA_BIDIRECTIONAL);
dma_buf_detach(drm_buffer->dma_buf, drm_buffer->attach);
dma_buf_put(drm_buffer->dma_buf);
drm_buffer->attach = NULL;
}
mutex_unlock(&iommu_info->iommu_mutex);
}
static void vcdoec_drm_dump_info(struct vcodec_iommu_session_info *session_info)
{
struct vcodec_drm_buffer *drm_buffer = NULL, *n;
@@ -478,14 +619,12 @@ static int vcodec_drm_import(struct vcodec_iommu_session_info *session_info,
int fd)
{
struct vcodec_drm_buffer *drm_buffer = NULL, *n;
struct vcodec_drm_buffer *oldest_buffer = NULL, *loop_buffer = NULL;
struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
struct device *dev = session_info->dev;
struct dma_buf_attachment *attach;
struct sg_table *sgt;
struct dma_buf *dma_buf;
ktime_t oldest_time = ktime_set(0, 0);
struct scatterlist *sg, *s;
int i;
int ret = 0;
@@ -578,22 +717,7 @@ static int vcodec_drm_import(struct vcodec_iommu_session_info *session_info,
session_info->buffer_nums++;
vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL,
"buffer nums %d\n", session_info->buffer_nums);
if (session_info->buffer_nums > BUFFER_LIST_MAX_NUMS) {
list_for_each_entry_safe(loop_buffer, n,
&session_info->buffer_list, list) {
if (ktime_to_ns(oldest_time) == 0 ||
ktime_after(oldest_time,
loop_buffer->last_used)) {
oldest_time = loop_buffer->last_used;
oldest_buffer = loop_buffer;
}
}
kref_put(&oldest_buffer->ref, vcodec_drm_clear_map);
dma_buf_put(oldest_buffer->dma_buf);
list_del_init(&oldest_buffer->list);
kfree(oldest_buffer);
session_info->buffer_nums--;
}
vcodec_drm_remove_extra_buffer_no_lock(session_info);
drm_buffer->index = session_info->max_idx;
list_add_tail(&drm_buffer->list, &session_info->buffer_list);
session_info->max_idx++;
@@ -618,6 +742,103 @@ fail_out:
return ret;
}
static int vcodec_drm_alloc(struct vcodec_iommu_session_info *session_info,
unsigned long size,
unsigned long align)
{
struct sg_table *table;
struct scatterlist *sg;
struct list_head pages;
struct page *page, *tmp_page;
long size_remaining = PAGE_ALIGN(size);
struct vcodec_drm_buffer *drm_buffer;
struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
struct vcodec_iommu_drm_info *drm_info = iommu_info->private;
int i;
if (align > PAGE_SIZE)
return -EINVAL;
if (size / PAGE_SIZE > totalram_pages / 2)
return -ENOMEM;
drm_buffer = kzalloc(sizeof(*drm_buffer), GFP_KERNEL);
if (!drm_buffer)
return -ENOMEM;
drm_buffer->session_info = session_info;
drm_buffer->last_used = ktime_set(0, 0);
kref_init(&drm_buffer->ref);
INIT_LIST_HEAD(&pages);
i = 0;
while (size_remaining > 0) {
gfp_t gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
__GFP_NORETRY) & ~__GFP_DIRECT_RECLAIM;
page = alloc_pages(gfp_flags | __GFP_COMP, 8);
if (!page)
goto free_pages;
size_remaining -= PAGE_SIZE << compound_order(page);
list_add_tail(&page->lru, &pages);
i++;
}
table = kmalloc(sizeof(*table), GFP_KERNEL);
if (!table)
goto free_pages;
if (sg_alloc_table(table, i, GFP_KERNEL))
goto free_table;
sg = table->sgl;
list_for_each_entry_safe(page, tmp_page, &pages, lru) {
sg_set_page(sg, page, PAGE_SIZE << compound_order(page), 0);
sg = sg_next(sg);
list_del(&page->lru);
}
mutex_lock(&iommu_info->iommu_mutex);
drm_info = session_info->iommu_info->private;
drm_buffer->copy_sgt = table;
vcodec_dma_map_sg(drm_info->domain, drm_buffer->copy_sgt->sgl,
drm_buffer->copy_sgt->nents,
IOMMU_READ | IOMMU_WRITE);
drm_buffer->iova = sg_dma_address(drm_buffer->copy_sgt->sgl);
drm_buffer->size = size;
drm_buffer->flags = VCODEC_DRM_BUFFER_ALLOC;
mutex_unlock(&iommu_info->iommu_mutex);
INIT_LIST_HEAD(&drm_buffer->list);
mutex_lock(&session_info->list_mutex);
session_info->buffer_nums++;
vpu_iommu_debug(session_info->debug_level, DEBUG_IOMMU_NORMAL,
"buffer nums %d\n", session_info->buffer_nums);
vcodec_drm_remove_extra_buffer_no_lock(session_info);
drm_buffer->index = session_info->max_idx;
list_add_tail(&drm_buffer->list, &session_info->buffer_list);
session_info->max_idx++;
if ((session_info->max_idx & 0xfffffff) == 0)
session_info->max_idx = 0;
mutex_unlock(&session_info->list_mutex);
return drm_buffer->index;
free_table:
kfree(table);
free_pages:
list_for_each_entry_safe(page, tmp_page, &pages, lru)
__free_pages(page, 8);
kfree(drm_buffer);
return -ENOMEM;
}
static int vcodec_drm_create(struct vcodec_iommu_info *iommu_info)
{
struct vcodec_iommu_drm_info *drm_info;
@@ -682,9 +903,12 @@ static int vcodec_drm_destroy(struct vcodec_iommu_info *iommu_info)
static struct vcodec_iommu_ops drm_ops = {
.create = vcodec_drm_create,
.alloc = vcodec_drm_alloc,
.import = vcodec_drm_import,
.free = vcodec_drm_free,
.free_fd = vcodec_drm_free_fd,
.map_kernel = vcodec_drm_map_kernel,
.unmap_kernel = vcodec_drm_unmap_kernel,
.map_iommu = vcodec_drm_map_iommu,
.unmap_iommu = vcodec_drm_unmap_iommu,
.destroy = vcodec_drm_destroy,

View File

@@ -59,7 +59,6 @@ vcodec_ion_clear_session(struct vcodec_iommu_session_info *session_info)
static int vcodec_ion_attach(struct vcodec_iommu_info *iommu_info)
{
struct vcodec_iommu_ion_info *ion_info = iommu_info->private;
int ret;
mutex_lock(&iommu_info->iommu_mutex);
@@ -74,7 +73,7 @@ static int vcodec_ion_attach(struct vcodec_iommu_info *iommu_info)
mutex_unlock(&iommu_info->iommu_mutex);
return ret;
return 0;
}
static void vcodec_ion_detach(struct vcodec_iommu_info *iommu_info)
@@ -208,6 +207,80 @@ vcodec_ion_import(struct vcodec_iommu_session_info *session_info, int fd)
return ion_buffer->index;
}
static int
vcodec_ion_unmap_kernel(struct vcodec_iommu_session_info *session_info,
int idx)
{
struct vcodec_ion_buffer *ion_buffer;
mutex_lock(&session_info->list_mutex);
ion_buffer = vcodec_ion_get_buffer_no_lock(session_info, idx);
mutex_unlock(&session_info->list_mutex);
if (!ion_buffer) {
pr_err("%s can not find %d buffer in list\n", __func__, idx);
return -EINVAL;
}
return 0;
}
static void*
vcodec_ion_map_kernel(struct vcodec_iommu_session_info *session_info, int idx)
{
struct vcodec_ion_buffer *ion_buffer;
struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
struct vcodec_iommu_ion_info *ion_info = iommu_info->private;
rockchip_iovmm_invalidate_tlb(session_info->dev);
mutex_lock(&session_info->list_mutex);
ion_buffer = vcodec_ion_get_buffer_no_lock(session_info, idx);
mutex_unlock(&session_info->list_mutex);
if (!ion_buffer) {
pr_err("%s can not find %d buffer in list\n", __func__, idx);
return NULL;
}
return ion_map_kernel(ion_info->ion_client, ion_buffer->handle);
}
static int vcodec_ion_alloc(struct vcodec_iommu_session_info *session_info,
unsigned long size,
unsigned long align)
{
struct vcodec_ion_buffer *ion_buffer = NULL;
struct vcodec_iommu_info *iommu_info = session_info->iommu_info;
struct vcodec_iommu_ion_info *ion_info = iommu_info->private;
unsigned int heap_id_mask;
if (iommu_info->mmu_dev)
heap_id_mask = ION_HEAP_TYPE_SYSTEM;
else
heap_id_mask = ION_HEAP_TYPE_DMA;
ion_buffer = kzalloc(sizeof(*ion_buffer), GFP_KERNEL);
if (!ion_buffer)
return -ENOMEM;
ion_buffer->handle = ion_alloc(ion_info->ion_client, size,
align, heap_id_mask, 0);
INIT_LIST_HEAD(&ion_buffer->list);
mutex_lock(&session_info->list_mutex);
ion_buffer->index = session_info->max_idx;
list_add_tail(&ion_buffer->list, &session_info->buffer_list);
session_info->max_idx++;
if ((session_info->max_idx & 0xfffffff) == 0)
session_info->max_idx = 0;
mutex_unlock(&session_info->list_mutex);
return ion_buffer->index;
}
static int vcodec_ion_create(struct vcodec_iommu_info *iommu_info)
{
struct vcodec_iommu_ion_info *ion_info;
@@ -229,9 +302,12 @@ static int vcodec_ion_create(struct vcodec_iommu_info *iommu_info)
static struct vcodec_iommu_ops ion_ops = {
.create = vcodec_ion_create,
.destroy = vcodec_ion_destroy,
.alloc = vcodec_ion_alloc,
.import = vcodec_ion_import,
.free = vcodec_ion_free,
.free_fd = NULL,
.map_kernel = vcodec_ion_map_kernel,
.unmap_kernel = vcodec_ion_unmap_kernel,
.map_iommu = vcodec_ion_map_iommu,
.unmap_iommu = vcodec_ion_unmap_iommu,
.dump = NULL,

View File

@@ -41,6 +41,41 @@ int vcodec_iommu_create(struct vcodec_iommu_info *iommu_info)
return iommu_info->ops->create(iommu_info);
}
int vcodec_iommu_alloc(struct vcodec_iommu_info *iommu_info,
struct vpu_session *session,
unsigned long size,
unsigned long align)
{
struct vcodec_iommu_session_info *session_info = NULL;
if (!iommu_info || !iommu_info->ops->alloc || !session)
return -EINVAL;
session_info = vcodec_iommu_get_session_info(iommu_info, session);
if (!session_info) {
session_info = kzalloc(sizeof(*session_info), GFP_KERNEL);
if (!session_info)
return -ENOMEM;
INIT_LIST_HEAD(&session_info->head);
INIT_LIST_HEAD(&session_info->buffer_list);
mutex_init(&session_info->list_mutex);
session_info->max_idx = 0;
session_info->session = session;
session_info->mmu_dev = iommu_info->mmu_dev;
session_info->dev = iommu_info->dev;
session_info->iommu_info = iommu_info;
session_info->buffer_nums = 0;
mutex_lock(&iommu_info->list_mutex);
list_add_tail(&session_info->head, &iommu_info->session_list);
mutex_unlock(&iommu_info->list_mutex);
}
session_info->debug_level = iommu_info->debug_level;
return iommu_info->ops->alloc(session_info, size, align);
}
int vcodec_iommu_import(struct vcodec_iommu_info *iommu_info,
struct vpu_session *session, int fd)
{

View File

@@ -46,9 +46,16 @@ struct vcodec_iommu_session_info;
struct vcodec_iommu_ops {
int (*create)(struct vcodec_iommu_info *iommu_info);
int (*alloc)(struct vcodec_iommu_session_info *session_info,
unsigned long size,
unsigned long align);
int (*import)(struct vcodec_iommu_session_info *session_info, int fd);
int (*free)(struct vcodec_iommu_session_info *session_info, int idx);
int (*free_fd)(struct vcodec_iommu_session_info *session_info, int fd);
void* (*map_kernel)(struct vcodec_iommu_session_info *session_info,
int idx);
int (*unmap_kernel)(struct vcodec_iommu_session_info *session_info,
int idx);
int (*map_iommu)(struct vcodec_iommu_session_info *session_info,
int idx,
dma_addr_t *iova, unsigned long *size);
@@ -98,6 +105,10 @@ struct vcodec_iommu_info *vcodec_iommu_info_create(struct device *dev,
int vcodec_iommu_info_destroy(struct vcodec_iommu_info *iommu_info);
int vcodec_iommu_create(struct vcodec_iommu_info *iommu_info);
int vcodec_iommu_alloc(struct vcodec_iommu_info *iommu_info,
struct vpu_session *session,
unsigned long size,
unsigned long align);
int vcodec_iommu_import(struct vcodec_iommu_info *iommu_info,
struct vpu_session *session, int fd);
int vcodec_iommu_free(struct vcodec_iommu_info *iommu_info,