From 008f4b8c9b1f67eb7e99e8b45f197baa6c4cd02b Mon Sep 17 00:00:00 2001 From: Yu Qiaowei Date: Wed, 16 Mar 2022 10:15:40 +0800 Subject: [PATCH] video: rockchip: rga3: Add CONFIG_ROCKCHIP_RGA_ASYNC Asynchronous mode is disabled when CONFIG_SYNC_FILE is disabled. Signed-off-by: Yu Qiaowei Change-Id: I91da6cdb0221d6c99f61e41e34d0770f049d1d57 --- drivers/video/rockchip/rga3/Kconfig | 7 + drivers/video/rockchip/rga3/Makefile | 3 +- drivers/video/rockchip/rga3/include/rga_drv.h | 19 +- .../video/rockchip/rga3/include/rga_fence.h | 85 +++++++- drivers/video/rockchip/rga3/rga_drv.c | 19 +- drivers/video/rockchip/rga3/rga_fence.c | 100 +++++----- drivers/video/rockchip/rga3/rga_job.c | 188 ++++++++++-------- 7 files changed, 260 insertions(+), 161 deletions(-) diff --git a/drivers/video/rockchip/rga3/Kconfig b/drivers/video/rockchip/rga3/Kconfig index 6c40b9b03a91..c8c96b2d67ff 100644 --- a/drivers/video/rockchip/rga3/Kconfig +++ b/drivers/video/rockchip/rga3/Kconfig @@ -7,6 +7,13 @@ menuconfig ROCKCHIP_MULTI_RGA if ROCKCHIP_MULTI_RGA +config ROCKCHIP_RGA_ASYNC + bool "Enable async mode" + depends on SYNC_FILE + default y + help + Asynchronous calls will be supported. + config ROCKCHIP_RGA_PROC_FS bool "Enable RGA procfs" select ROCKCHIP_RGA_DEBUGGER diff --git a/drivers/video/rockchip/rga3/Makefile b/drivers/video/rockchip/rga3/Makefile index 16bcf4f88b36..143f35f0605a 100644 --- a/drivers/video/rockchip/rga3/Makefile +++ b/drivers/video/rockchip/rga3/Makefile @@ -2,7 +2,8 @@ ccflags-y += -I$(srctree)/$(src)/include -rga3-y := rga_drv.o rga_common.o rga3_reg_info.o rga_dma_buf.o rga_fence.o rga_job.o rga_hw_config.o rga2_reg_info.o rga2_mmu_info.o rga_policy.o rga_mm.o +rga3-y := rga_drv.o rga_common.o rga3_reg_info.o rga_dma_buf.o rga_job.o rga_hw_config.o rga2_reg_info.o rga2_mmu_info.o rga_policy.o rga_mm.o +rga3-$(CONFIG_ROCKCHIP_RGA_ASYNC) += rga_fence.o rga3-$(CONFIG_ROCKCHIP_RGA_DEBUGGER) += rga_debugger.o obj-$(CONFIG_ROCKCHIP_MULTI_RGA) += rga3.o diff --git a/drivers/video/rockchip/rga3/include/rga_drv.h b/drivers/video/rockchip/rga3/include/rga_drv.h index 406720c9c5c4..7640cc633e4b 100644 --- a/drivers/video/rockchip/rga3/include/rga_drv.h +++ b/drivers/video/rockchip/rga3/include/rga_drv.h @@ -123,19 +123,6 @@ enum iommu_dma_cookie_type { IOMMU_DMA_MSI_COOKIE, }; -struct rga_fence_context { - unsigned int context; - unsigned int seqno; - spinlock_t spinlock; -}; - -struct rga_fence_waiter { - /* Base sync driver waiter structure */ - struct dma_fence_cb waiter; - - struct rga_job *job; -}; - struct rga_iommu_dma_cookie { enum iommu_dma_cookie_type type; @@ -376,8 +363,6 @@ struct rga_pending_ctx_manager { struct rga_drvdata_t { struct miscdevice miscdev; - struct rga_fence_context *fence_ctx; - /* used by rga2's mmu lock */ struct mutex lock; @@ -392,6 +377,10 @@ struct rga_drvdata_t { /* rga_job pending manager, import by RGA_START_CONFIG */ struct rga_pending_ctx_manager *pend_ctx_manager; +#ifdef CONFIG_ROCKCHIP_RGA_ASYNC + struct rga_fence_context *fence_ctx; +#endif + #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER struct rga_debugger *debugger; #endif diff --git a/drivers/video/rockchip/rga3/include/rga_fence.h b/drivers/video/rockchip/rga3/include/rga_fence.h index 6ab3ccf447de..9fc7f45f5dd9 100644 --- a/drivers/video/rockchip/rga3/include/rga_fence.h +++ b/drivers/video/rockchip/rga3/include/rga_fence.h @@ -8,21 +8,88 @@ #ifndef __LINUX_RGA_FENCE_H_ #define __LINUX_RGA_FENCE_H_ -#include "rga_drv.h" +struct rga_fence_context { + unsigned int context; + unsigned int seqno; + spinlock_t spinlock; +}; -struct rga_fence_context *rga_fence_context_alloc(void); +struct rga_fence_waiter { + /* Base sync driver waiter structure */ + struct dma_fence_cb waiter; -void rga_fence_context_free(struct rga_fence_context *fence_ctx); + void *private; +}; -int rga_out_fence_alloc(struct rga_job *job); +#ifdef CONFIG_ROCKCHIP_RGA_ASYNC +int rga_fence_context_init(struct rga_fence_context **ctx); +void rga_fence_context_remove(struct rga_fence_context **ctx); -int rga_out_fence_get_fd(struct rga_job *job); +struct dma_fence *rga_dma_fence_alloc(spinlock_t *lock); +int rga_dma_fence_get_fd(struct dma_fence *fence); +struct dma_fence *rga_get_dma_fence_from_fd(int fence_fd); +int rga_dma_fence_wait(struct dma_fence *fence); +int rga_dma_fence_add_callback(struct dma_fence *fence, dma_fence_func_t func, void *private); -struct dma_fence *rga_get_input_fence(int in_fence_fd); -int rga_wait_input_fence(struct dma_fence *in_fence); +static inline void rga_dma_fence_put(struct dma_fence *fence) +{ + if (fence) + dma_fence_put(fence); +} -int rga_add_dma_fence_callback(struct rga_job *job, - struct dma_fence *in_fence, dma_fence_func_t func); +static inline void rga_dma_fence_signal(struct dma_fence *fence) +{ + if (fence) + dma_fence_signal(fence); +} + +static inline int rga_dma_fence_get_status(struct dma_fence *fence) +{ + return dma_fence_get_status(fence); +} + +#else +static inline struct dma_fence *rga_dma_fence_alloc(spinlock_t *lock) +{ + return NULL; +} + +static inline int rga_dma_fence_get_fd(struct dma_fence *fence) +{ + return 0; +} + +static inline struct dma_fence *rga_get_dma_fence_from_fd(int fence_fd) +{ + return NULL; +} + +static inline int rga_dma_fence_wait(struct dma_fence *fence) +{ + return 0; +} + +static inline int rga_dma_fence_add_callback(struct dma_fence *fence, + dma_fence_func_t func, + void *private) +{ + return 0; +} + +static inline void rga_dma_fence_put(struct dma_fence *fence) +{ +} + +static inline void rga_dma_fence_signal(struct dma_fence *fence) +{ +} + +static inline int rga_dma_fence_get_status(struct dma_fence *fence) +{ + return 0; +} + +#endif /* #ifdef CONFIG_SYNC_FILE */ #endif /* __LINUX_RGA_FENCE_H_ */ diff --git a/drivers/video/rockchip/rga3/rga_drv.c b/drivers/video/rockchip/rga3/rga_drv.c index 1607f1dbc194..942fddd8c9d5 100644 --- a/drivers/video/rockchip/rga3/rga_drv.c +++ b/drivers/video/rockchip/rga3/rga_drv.c @@ -475,6 +475,8 @@ static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) struct rga_hw_versions_t hw_versions; struct rga_internal_ctx_t ctx; + memset(&ctx, 0x0, sizeof(ctx)); + if (!rga) { pr_err("rga_drvdata is null, rga is not init\n"); return -ENODEV; @@ -1194,13 +1196,6 @@ static int __init rga_init(void) rga_init_timer(); - rga_drvdata->fence_ctx = rga_fence_context_alloc(); - if (IS_ERR(rga_drvdata->fence_ctx)) { - pr_err("failed to allocate fence context for RGA\n"); - ret = PTR_ERR(rga_drvdata->fence_ctx); - return ret; - } - ret = misc_register(&rga_dev); if (ret) { pr_err("cannot register miscdev (%d)\n", ret); @@ -1211,6 +1206,10 @@ static int __init rga_init(void) rga_ctx_manager_init(&rga_drvdata->pend_ctx_manager); +#ifdef CONFIG_ROCKCHIP_RGA_ASYNC + rga_fence_context_init(&rga_drvdata->fence_ctx); +#endif + #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER rga_debugger_init(&rga_drvdata->debugger); #endif @@ -1230,14 +1229,16 @@ static void __exit rga_exit(void) rga_debugger_remove(&rga_drvdata->debugger); #endif +#ifdef CONFIG_ROCKCHIP_RGA_ASYNC + rga_fence_context_remove(&rga_drvdata->fence_ctx); +#endif + rga_mm_remove(&rga_drvdata->mm); rga_ctx_manager_remove(&rga_drvdata->pend_ctx_manager); wake_lock_destroy(&rga_drvdata->wake_lock); - rga_fence_context_free(rga_drvdata->fence_ctx); - rga_cancel_timer(); platform_driver_unregister(&rga3_core0_driver); diff --git a/drivers/video/rockchip/rga3/rga_fence.c b/drivers/video/rockchip/rga3/rga_fence.c index 34bfd0a09a94..f25a4cd382c3 100644 --- a/drivers/video/rockchip/rga3/rga_fence.c +++ b/drivers/video/rockchip/rga3/rga_fence.c @@ -11,6 +11,7 @@ #include #include +#include "rga_drv.h" #include "rga_fence.h" static const char *rga_fence_get_name(struct dma_fence *fence) @@ -23,55 +24,66 @@ static const struct dma_fence_ops rga_fence_ops = { .get_timeline_name = rga_fence_get_name, }; -struct rga_fence_context *rga_fence_context_alloc(void) +int rga_fence_context_init(struct rga_fence_context **ctx) { struct rga_fence_context *fence_ctx = NULL; - fence_ctx = kzalloc(sizeof(*fence_ctx), GFP_KERNEL); - if (!fence_ctx) - return ERR_PTR(-ENOMEM); + fence_ctx = kzalloc(sizeof(struct rga_fence_context), GFP_KERNEL); + if (!fence_ctx) { + pr_err("can not kzalloc for rga_fence_context!\n"); + return -ENOMEM; + } fence_ctx->context = dma_fence_context_alloc(1); spin_lock_init(&fence_ctx->spinlock); - return fence_ctx; -} - -void rga_fence_context_free(struct rga_fence_context *fence_ctx) -{ - kfree(fence_ctx); -} - -int rga_out_fence_alloc(struct rga_job *job) -{ - struct rga_fence_context *fence_ctx = rga_drvdata->fence_ctx; - struct dma_fence *fence = NULL; - - fence = kzalloc(sizeof(*fence), GFP_KERNEL); - if (!fence) - return -ENOMEM; - - dma_fence_init(fence, &rga_fence_ops, &job->fence_lock, - fence_ctx->context, ++fence_ctx->seqno); - - job->out_fence = fence; + *ctx = fence_ctx; return 0; } -int rga_out_fence_get_fd(struct rga_job *job) +void rga_fence_context_remove(struct rga_fence_context **ctx) +{ + if (*ctx == NULL) + return; + + kfree(*ctx); + *ctx = NULL; +} + +struct dma_fence *rga_dma_fence_alloc(spinlock_t *lock) +{ + struct rga_fence_context *fence_ctx = rga_drvdata->fence_ctx; + struct dma_fence *fence = NULL; + + if (fence_ctx == NULL) { + pr_err("fence_context is NULL!\n"); + return ERR_PTR(-EINVAL); + } + + fence = kzalloc(sizeof(*fence), GFP_KERNEL); + if (!fence) + return ERR_PTR(-ENOMEM); + + dma_fence_init(fence, &rga_fence_ops, lock, + fence_ctx->context, ++fence_ctx->seqno); + + return fence; +} + +int rga_dma_fence_get_fd(struct dma_fence *fence) { struct sync_file *sync_file = NULL; int fence_fd = -1; - if (!job->out_fence) + if (!fence) return -EINVAL; fence_fd = get_unused_fd_flags(O_CLOEXEC); if (fence_fd < 0) return fence_fd; - sync_file = sync_file_create(job->out_fence); + sync_file = sync_file_create(fence); if (!sync_file) return -ENOMEM; @@ -80,34 +92,32 @@ int rga_out_fence_get_fd(struct rga_job *job) return fence_fd; } -struct dma_fence *rga_get_input_fence(int in_fence_fd) +struct dma_fence *rga_get_dma_fence_from_fd(int fence_fd) { - struct dma_fence *in_fence; + struct dma_fence *fence; - in_fence = sync_file_get_fence(in_fence_fd); + fence = sync_file_get_fence(fence_fd); + if (!fence) + pr_err("can not get fence from fd\n"); - if (!in_fence) - pr_err("can not get in-fence from fd\n"); - - return in_fence; + return fence; } -int rga_wait_input_fence(struct dma_fence *in_fence) +int rga_dma_fence_wait(struct dma_fence *fence) { int ret = 0; - ret = dma_fence_wait(in_fence, true); + ret = dma_fence_wait(fence, true); - dma_fence_put(in_fence); + dma_fence_put(fence); return ret; } -int rga_add_dma_fence_callback(struct rga_job *job, struct dma_fence *in_fence, - dma_fence_func_t func) +int rga_dma_fence_add_callback(struct dma_fence *fence, dma_fence_func_t func, void *private) { - struct rga_fence_waiter *waiter; int ret; + struct rga_fence_waiter *waiter = NULL; waiter = kmalloc(sizeof(*waiter), GFP_KERNEL); if (!waiter) { @@ -115,16 +125,14 @@ int rga_add_dma_fence_callback(struct rga_job *job, struct dma_fence *in_fence, return -ENOMEM; } - waiter->job = job; + waiter->private = private; - ret = dma_fence_add_callback(in_fence, &waiter->waiter, func); + ret = dma_fence_add_callback(fence, &waiter->waiter, func); if (ret == -ENOENT) { pr_err("'input fence' has been already signaled."); goto err_free_waiter; } else if (ret == -EINVAL) { - pr_err - ("%s: failed to add callback to dma_fence, err: %d\n", - __func__, ret); + pr_err("%s: failed to add callback to dma_fence, err: %d\n", __func__, ret); goto err_free_waiter; } diff --git a/drivers/video/rockchip/rga3/rga_job.c b/drivers/video/rockchip/rga3/rga_job.c index f5c3d1b48322..48370a0427eb 100644 --- a/drivers/video/rockchip/rga3/rga_job.c +++ b/drivers/video/rockchip/rga3/rga_job.c @@ -125,8 +125,7 @@ static void rga_job_put_current_mm(struct rga_job *job) static void rga_job_free(struct rga_job *job) { - if (job->out_fence) - dma_fence_put(job->out_fence); + rga_dma_fence_put(job->out_fence); if (~job->flags & RGA_JOB_USE_HANDLE) rga_job_put_current_mm(job); @@ -330,8 +329,7 @@ static int rga_internal_ctx_signal(struct rga_scheduler_t *scheduler, struct rga spin_unlock_irqrestore(&ctx->lock, flags); if (finished_job_count >= ctx->cmd_num) { - if (ctx->out_fence) - dma_fence_signal(ctx->out_fence); + rga_dma_fence_signal(ctx->out_fence); job->flags |= RGA_JOB_DONE; @@ -450,8 +448,7 @@ next_job: if (job->use_batch_mode) { rga_internal_ctx_signal(rga_scheduler, job); } else { - if (job->out_fence) - dma_fence_signal(job->out_fence); + rga_dma_fence_signal(job->out_fence); job->flags |= RGA_JOB_DONE; @@ -491,8 +488,7 @@ static void rga_job_finish_and_next(struct rga_scheduler_t *rga_scheduler, if (job->use_batch_mode) rga_internal_ctx_signal(rga_scheduler, job); else { - if (job->out_fence) - dma_fence_signal(job->out_fence); + rga_dma_fence_signal(job->out_fence); job->flags |= RGA_JOB_DONE; @@ -550,8 +546,7 @@ static void rga_job_timeout_clean(struct rga_scheduler_t *scheduler) if (job->use_batch_mode) rga_internal_ctx_signal(scheduler, job); else { - if (job->out_fence) - dma_fence_signal(job->out_fence); + rga_dma_fence_signal(job->out_fence); rga_job_cleanup(job); } @@ -678,28 +673,6 @@ static inline int rga_job_wait(struct rga_scheduler_t *rga_scheduler, return ret; } -static void rga_input_fence_signaled(struct dma_fence *fence, - struct dma_fence_cb *_waiter) -{ - struct rga_fence_waiter *waiter = (struct rga_fence_waiter *)_waiter; - struct rga_scheduler_t *scheduler = NULL; - - ktime_t now; - - now = ktime_get(); - - if (DEBUGGER_EN(TIME)) - pr_err("rga job wait in_fence signal use time = %lld\n", - ktime_us_delta(now, waiter->job->timestamp)); - - scheduler = rga_job_schedule(waiter->job); - - if (scheduler == NULL) - pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__); - - kfree(waiter); -} - uint32_t rga_internal_ctx_alloc_to_get_idr_id(void) { struct rga_pending_ctx_manager *ctx_manager; @@ -928,11 +901,82 @@ int rga_job_cancel_by_user_ctx(uint32_t ctx_id) return ret; } +static int rga_job_alloc_release_fence(struct dma_fence **release_fence, spinlock_t *lock) +{ + struct dma_fence *fence; + + fence = rga_dma_fence_alloc(lock); + if (IS_ERR(fence)) { + pr_err("Can not alloc release fence!\n"); + return IS_ERR(fence); + } + + *release_fence = fence; + + return rga_dma_fence_get_fd(fence); +} + +static void rga_job_fence_signaled_callback(struct dma_fence *fence, struct dma_fence_cb *_waiter) +{ + struct rga_fence_waiter *waiter = (struct rga_fence_waiter *)_waiter; + struct rga_scheduler_t *scheduler = NULL; + struct rga_job *job; + ktime_t now; + + job = (struct rga_job *)waiter->private; + + now = ktime_get(); + if (DEBUGGER_EN(TIME)) + pr_err("rga job wait acquire_fence signal use time = %lld\n", + ktime_us_delta(now, job->timestamp)); + + scheduler = rga_job_schedule(job); + if (scheduler == NULL) { + pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__); + rga_job_free(job); + } + + kfree(waiter); +} + +static int rga_job_add_acquire_fence_callback(int acquire_fence_fd, void *private, + dma_fence_func_t cb_func) +{ + int ret; + struct dma_fence *acquire_fence = NULL; + + if (DEBUGGER_EN(MSG)) + pr_info("acquire_fence_fd = %d", acquire_fence_fd); + + acquire_fence = rga_get_dma_fence_from_fd(acquire_fence_fd); + if (IS_ERR(acquire_fence)) { + pr_err("%s: failed to get acquire dma_fence\n", __func__); + return PTR_ERR(acquire_fence); + } + /* close acquire fence fd */ + ksys_close(acquire_fence_fd); + + ret = rga_dma_fence_get_status(acquire_fence); + if (ret == 0) { + ret = rga_dma_fence_add_callback(acquire_fence, cb_func, private); + if (ret < 0) { + if (ret == -ENOENT) + return 1; + + pr_err("%s: failed to add fence callback\n", __func__); + return ret; + } + } else { + return ret; + } + + return 0; +} + int rga_job_commit(struct rga_req *rga_command_base, struct rga_internal_ctx_t *ctx) { struct rga_job *job = NULL; struct rga_scheduler_t *scheduler = NULL; - struct dma_fence *in_fence; int ret = 0; job = rga_job_alloc(rga_command_base); @@ -952,85 +996,63 @@ int rga_job_commit(struct rga_req *rga_command_base, struct rga_internal_ctx_t * if (ret < 0) { pr_err("%s: failed to get dma buf from fd\n", __func__); - rga_job_free(job); - return ret; + goto free_job; } if (ctx->sync_mode == RGA_BLIT_ASYNC) { + if (!IS_ENABLED(CONFIG_ROCKCHIP_RGA_ASYNC)) { + pr_err("Unsupported ASYNC mode, please enable CONFIG_ROCKCHIP_RGA_ASYNC\n"); + ret = -EFAULT; + goto free_job; + } + job->flags |= RGA_JOB_ASYNC; if (ctx->out_fence) { job->out_fence = ctx->out_fence; + rga_command_base->out_fence_fd = ctx->out_fence_fd; } else { - ret = rga_out_fence_alloc(job); - if (ret) { - rga_job_free(job); - return ret; + rga_command_base->out_fence_fd = + rga_job_alloc_release_fence(&job->out_fence, + &job->fence_lock); + if (rga_command_base->out_fence_fd < 0) { + pr_err("Failed to alloc release fence fd!\n"); + goto free_job; } /* on batch mode, only first job need to alloc fence */ if (ctx->use_batch_mode) ctx->out_fence = job->out_fence; + + ctx->out_fence_fd = rga_command_base->out_fence_fd; } - rga_command_base->out_fence_fd = rga_out_fence_get_fd(job); - ctx->out_fence_fd = rga_command_base->out_fence_fd; - - if (DEBUGGER_EN(MSG)) - pr_info("in_fence_fd = %d", - rga_command_base->in_fence_fd); - - /* if input fence is valiable */ - if (rga_command_base->in_fence_fd > 0) { - in_fence = rga_get_input_fence( - rga_command_base->in_fence_fd); - if (!in_fence) { - pr_err("%s: failed to get input dma_fence\n", - __func__); - rga_job_free(job); - return ret; - } - - /* close input fence fd */ - ksys_close(rga_command_base->in_fence_fd); - - ret = dma_fence_get_status(in_fence); - /* ret = 1: fence has been signaled */ + if (job->rga_command_base.in_fence_fd > 0) { + ret = rga_job_add_acquire_fence_callback(job->rga_command_base.in_fence_fd, + (void *)job, + rga_job_fence_signaled_callback); if (ret == 1) { + /* It means has been already signaled. */ scheduler = rga_job_schedule(job); if (scheduler == NULL) { pr_err("failed to get scheduler, %s(%d)\n", - __func__, __LINE__); + __func__, __LINE__); ret = -EINVAL; goto invalid_job; } - /* if input fence is valid */ - } else if (ret == 0) { - ret = rga_add_dma_fence_callback(job, - in_fence, rga_input_fence_signaled); - if (ret < 0) { - pr_err("%s: failed to add fence callback\n", - __func__); - rga_job_free(job); - return ret; - } } else { - pr_err("%s: fence status error\n", __func__); - rga_job_free(job); - return ret; + pr_err("Failed to wait acquire fence fd[%d]!\n", + job->rga_command_base.in_fence_fd); + goto free_job; } } else { scheduler = rga_job_schedule(job); if (scheduler == NULL) { - pr_err("failed to get scheduler, %s(%d)\n", - __func__, __LINE__); + pr_err("failed to get scheduler, %s(%d)\n", __func__, __LINE__); ret = -EINVAL; goto invalid_job; } } - - return ret; - /* RGA_BLIT_SYNC: wait until job finish */ } else if (ctx->sync_mode == RGA_BLIT_SYNC) { scheduler = rga_job_schedule(job); @@ -1057,6 +1079,10 @@ int rga_job_commit(struct rga_req *rga_command_base, struct rga_internal_ctx_t * } return ret; +free_job: + rga_job_free(job); + return ret; + invalid_job: rga_invalid_job_abort(job); return ret;