diff --git a/drivers/video/rockchip/rga3/include/rga.h b/drivers/video/rockchip/rga3/include/rga.h index f5bf5ac8fe1b..92c82a2c5316 100644 --- a/drivers/video/rockchip/rga3/include/rga.h +++ b/drivers/video/rockchip/rga3/include/rga.h @@ -16,6 +16,11 @@ #define RGA_IOC_IMPORT_BUFFER RGA_IOWR(0x3, struct rga_buffer_pool) #define RGA_IOC_RELEASE_BUFFER RGA_IOW(0x4, struct rga_buffer_pool) +#define RGA_START_CONFIG RGA_IOR(0x5, uint32_t) +#define RGA_END_CONFIG RGA_IOWR(0x6, struct rga_user_ctx_t) +#define RGA_CMD_CONFIG RGA_IOWR(0x7, struct rga_user_ctx_t) +#define RGA_CANCEL_CONFIG RGA_IOWR(0x8, uint32_t) + #define RGA_BLIT_SYNC 0x5017 #define RGA_BLIT_ASYNC 0x5018 #define RGA_FLUSH 0x5019 @@ -27,6 +32,8 @@ #define RGA_IMPORT_DMA 0x601d #define RGA_RELEASE_DMA 0x601e +#define RGA_CMD_NUM_MAX 1 + #define RGA_OUT_OF_RESOURCES -10 #define RGA_MALLOC_ERROR -11 @@ -667,12 +674,40 @@ struct rga3_req { u8 rgb2yuv_mode; }; +struct rga_video_frame_info { + uint32_t x_offset; + uint32_t y_offset; + uint32_t width; + uint32_t height; + uint32_t format; + uint32_t vir_w; + uint32_t vir_h; + uint32_t rd_mode; +}; + struct rga_mpi_job_t { struct dma_buf *dma_buf_src0; struct dma_buf *dma_buf_src1; struct dma_buf *dma_buf_dst; + + struct rga_video_frame_info src; + struct rga_video_frame_info dst; + + int ctx_id; }; -int rga_mpi_commit(struct rga_req *cmd, struct rga_mpi_job_t *mpi_job); +struct rga_user_ctx_t { + uint64_t cmd_ptr; + uint32_t cmd_num; + uint32_t id; + uint32_t sync_mode; + uint32_t out_fence_fd; + + uint8_t mpi_config_flags; + + uint8_t reservr[127]; +}; + +int rga_mpi_commit(struct rga_mpi_job_t *mpi_job); #endif /*_RGA_DRIVER_H_*/ diff --git a/drivers/video/rockchip/rga3/include/rga_debugger.h b/drivers/video/rockchip/rga3/include/rga_debugger.h index 015688aafcc1..f3f248205e9e 100644 --- a/drivers/video/rockchip/rga3/include/rga_debugger.h +++ b/drivers/video/rockchip/rga3/include/rga_debugger.h @@ -128,6 +128,7 @@ static inline int rga_procfs_init(void) #endif /* #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER */ void rga_cmd_print_debug_info(struct rga_req *req); +void rga_ctx_cache_cmd_debug_info(struct seq_file *m, struct rga_req *req); #endif /* #ifndef _RGA_DEBUGGER_H_ */ diff --git a/drivers/video/rockchip/rga3/include/rga_drv.h b/drivers/video/rockchip/rga3/include/rga_drv.h index 313fbd3f0122..b891fc88c788 100644 --- a/drivers/video/rockchip/rga3/include/rga_drv.h +++ b/drivers/video/rockchip/rga3/include/rga_drv.h @@ -113,6 +113,11 @@ enum { RGA_NONE_CORE = 0x0, }; +enum { + RGA_CMD_SLAVE = 1, + RGA_CMD_MASTER = 2, +}; + enum iommu_dma_cookie_type { IOMMU_DMA_IOVA_COOKIE, IOMMU_DMA_MSI_COOKIE, @@ -286,10 +291,11 @@ struct rga_job { ktime_t timestamp; ktime_t running_time; unsigned int flags; - int job_id; + int ctx_id; int priority; int core; int ret; + bool use_batch_mode; }; struct rga_scheduler_t; @@ -329,6 +335,44 @@ struct rga_scheduler_t { struct rga_timer timer; }; +struct rga_internal_ctx_t { + struct rga_req *cached_cmd; + int cmd_num; + int flags; + int id; + + uint8_t mpi_config_flags; + uint32_t sync_mode; + + uint32_t finished_job_count; + + bool use_batch_mode; + bool is_running; + + struct dma_fence *out_fence; + int32_t out_fence_fd; + + spinlock_t lock; + struct kref refcount; + + pid_t pid; + /* TODO: add some common work */ +}; + +struct rga_pending_ctx_manager { + struct mutex lock; + + /* + * @ctx_id_idr: + * + * Mapping of ctx id to object pointers. Used by the GEM + * subsystem. Protected by @lock. + */ + struct idr ctx_id_idr; + + int ctx_count; +}; + struct rga_drvdata_t { struct miscdevice miscdev; @@ -345,6 +389,9 @@ struct rga_drvdata_t { struct rga_mm *mm; + /* rga_job pending manager, import by RGA_START_CONFIG */ + struct rga_pending_ctx_manager *pend_ctx_manager; + #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER struct rga_debugger *debugger; #endif diff --git a/drivers/video/rockchip/rga3/include/rga_job.h b/drivers/video/rockchip/rga3/include/rga_job.h index 10b0a9253350..a5dcf6e3ed37 100644 --- a/drivers/video/rockchip/rga3/include/rga_job.h +++ b/drivers/video/rockchip/rga3/include/rga_job.h @@ -24,13 +24,24 @@ enum job_flags { struct rga_scheduler_t *rga_job_get_scheduler(int core); void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret); -int rga_job_commit(struct rga_req *rga_command_base, int flags); +int rga_job_commit(struct rga_req *rga_command_base, struct rga_internal_ctx_t *ctx); int rga_job_mpi_commit(struct rga_req *rga_command_base, - struct rga_mpi_job_t *mpi_job, int flags); + struct rga_mpi_job_t *mpi_job, struct rga_internal_ctx_t *ctx); int rga_job_assign(struct rga_job *job); +int rga_ctx_manager_init(struct rga_pending_ctx_manager **ctx_manager_session); +int rga_ctx_manager_remove(struct rga_pending_ctx_manager **ctx_manager_session); + +struct rga_internal_ctx_t * + rga_internal_ctx_lookup(struct rga_pending_ctx_manager *ctx_manager, uint32_t id); +uint32_t rga_internal_ctx_alloc_to_get_idr_id(void); +void rga_internel_ctx_kref_release(struct kref *ref); +int rga_job_config_by_user_ctx(struct rga_user_ctx_t *user_ctx); +int rga_job_commit_by_user_ctx(struct rga_user_ctx_t *user_ctx); +int rga_job_cancel_by_user_ctx(uint32_t ctx_id); + struct rga_job * rga_scheduler_get_pending_job_list(struct rga_scheduler_t *scheduler); diff --git a/drivers/video/rockchip/rga3/rga_debugger.c b/drivers/video/rockchip/rga3/rga_debugger.c index 457e4b12fe73..d431256b4580 100644 --- a/drivers/video/rockchip/rga3/rga_debugger.c +++ b/drivers/video/rockchip/rga3/rga_debugger.c @@ -246,12 +246,63 @@ static int rga_mm_session_show(struct seq_file *m, void *data) return 0; } +static int rga_ctx_manager_show(struct seq_file *m, void *data) +{ + int id, i; + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + struct rga_req *cached_cmd; + unsigned long flags; + int cmd_num = 0; + int finished_job_count = 0; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + seq_puts(m, "rga internal ctx dump:\n"); + seq_printf(m, "ctx count = %d\n", ctx_manager->ctx_count); + seq_puts(m, "===============================================================\n"); + + mutex_lock(&ctx_manager->lock); + + idr_for_each_entry(&ctx_manager->ctx_id_idr, ctx, id) { + seq_printf(m, "------------------ ctx: %d ------------------\n", ctx->id); + + spin_lock_irqsave(&ctx->lock, flags); + + cmd_num = ctx->cmd_num; + finished_job_count = ctx->finished_job_count; + cached_cmd = ctx->cached_cmd; + + spin_unlock_irqrestore(&ctx->lock, flags); + + if (cached_cmd == NULL) { + seq_puts(m, "\t can not find cached cmd from id\n"); + continue; + } + + seq_printf(m, "\t set cmd num: %d, finish job sum: %d\n", + cmd_num, finished_job_count); + + seq_puts(m, "\t cmd dump:\n\n"); + + for (i = 0; i < ctx->cmd_num; i++) + rga_ctx_cache_cmd_debug_info(m, &(cached_cmd[i])); + + } + + mutex_unlock(&ctx_manager->lock); + + return 0; +} + + struct rga_debugger_list rga_debugger_root_list[] = { {"debug", rga_debug_show, rga_debug_write, NULL}, {"driver_version", rga_version_show, NULL, NULL}, {"load", rga_load_show, NULL, NULL}, {"scheduler_status", rga_scheduler_show, NULL, NULL}, {"mm_session", rga_mm_session_show, NULL, NULL}, + {"ctx_manager", rga_ctx_manager_show, NULL, NULL}, }; static ssize_t rga_debugger_write(struct file *file, const char __user *ubuf, @@ -523,9 +574,43 @@ CREATE_FAIL: } #endif /* #ifdef CONFIG_ROCKCHIP_RGA_PROC_FS */ +void rga_ctx_cache_cmd_debug_info(struct seq_file *m, struct rga_req *req) +{ + seq_printf(m, "\t\t rotate_mode = %d\n", req->rotate_mode); + seq_printf(m, "\t\t src: y = %lx uv = %lx v = %lx aw = %d ah = %d vw = %d vh = %d\n", + (unsigned long)req->src.yrgb_addr, (unsigned long)req->src.uv_addr, + (unsigned long)req->src.v_addr, req->src.act_w, req->src.act_h, + req->src.vir_w, req->src.vir_h); + seq_printf(m, "\t\t src: xoff = %d, yoff = %d, format = 0x%x, rd_mode = %d\n", + req->src.x_offset, req->src.y_offset, req->src.format, req->src.rd_mode); + + if (req->pat.yrgb_addr != 0 || req->pat.uv_addr != 0 + || req->pat.v_addr != 0) { + seq_printf(m, "\t\t pat: y=%lx uv=%lx v=%lx aw=%d ah=%d vw=%d vh=%d\n", + (unsigned long)req->pat.yrgb_addr, (unsigned long)req->pat.uv_addr, + (unsigned long)req->pat.v_addr, req->pat.act_w, req->pat.act_h, + req->pat.vir_w, req->pat.vir_h); + seq_printf(m, "\t\t xoff = %d yoff = %d, format = 0x%x, rd_mode = %d\n", + req->pat.x_offset, req->pat.y_offset, req->pat.format, req->pat.rd_mode); + } + + seq_printf(m, "\t\t dst: y=%lx uv=%lx v=%lx aw=%d ah=%d vw=%d vh=%d\n", + (unsigned long)req->dst.yrgb_addr, (unsigned long)req->dst.uv_addr, + (unsigned long)req->dst.v_addr, req->dst.act_w, req->dst.act_h, + req->dst.vir_w, req->dst.vir_h); + seq_printf(m, "\t\t dst: xoff = %d, yoff = %d, format = 0x%x, rd_mode = %d\n", + req->dst.x_offset, req->dst.y_offset, req->dst.format, req->dst.rd_mode); + + seq_printf(m, "\t\t mmu: mmu_flag=%x en=%x\n", + req->mmu_info.mmu_flag, req->mmu_info.mmu_en); + seq_printf(m, "\t\t alpha: rop_mode = %x\n", req->alpha_rop_mode); + seq_printf(m, "\t\t yuv2rgb mode is %x\n", req->yuv2rgb_mode); + seq_printf(m, "\t\t set core = %d, priority = %d, in_fence_fd = %d\n", + req->core, req->priority, req->in_fence_fd); +} + void rga_cmd_print_debug_info(struct rga_req *req) { - pr_info("============= start ==============\n"); pr_info("render_mode = %d, bitblit_mode=%d, rotate_mode = %d\n", req->render_mode, req->bsfilter_flag, req->rotate_mode); diff --git a/drivers/video/rockchip/rga3/rga_dma_buf.c b/drivers/video/rockchip/rga3/rga_dma_buf.c index 4fa5fe82e2f1..5dec21062de1 100644 --- a/drivers/video/rockchip/rga3/rga_dma_buf.c +++ b/drivers/video/rockchip/rga3/rga_dma_buf.c @@ -934,13 +934,14 @@ static void rga_dma_put_channel_info(struct rga_dma_buffer_t **rga_dma_buffer, s return; rga_dma_unmap_buffer(buffer); - if (*dma_buf) + if (*dma_buf && buffer->use_dma_buf == false) { dma_buf_put(*dma_buf); + *dma_buf = NULL; + } kfree(buffer); *rga_dma_buffer = NULL; - *dma_buf = NULL; } int rga_dma_buf_get(struct rga_job *job) diff --git a/drivers/video/rockchip/rga3/rga_drv.c b/drivers/video/rockchip/rga3/rga_drv.c index 327f3eb1ea4a..19eba6c2f5e0 100644 --- a/drivers/video/rockchip/rga3/rga_drv.c +++ b/drivers/video/rockchip/rga3/rga_drv.c @@ -41,24 +41,71 @@ static const struct rga_backend_ops rga2_ops = { .soft_reset = rga2_soft_reset }; -int rga_mpi_commit(struct rga_req *cmd, struct rga_mpi_job_t *mpi_job) +int rga_mpi_commit(struct rga_mpi_job_t *mpi_job) { - int ret; + int ret = 0; + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + struct rga_req *cached_cmd; + unsigned long flags; + int i; - if (DEBUGGER_EN(MSG)) - rga_cmd_print_debug_info(cmd); + ctx_manager = rga_drvdata->pend_ctx_manager; - ret = rga_job_mpi_commit(cmd, mpi_job, RGA_BLIT_SYNC); - if (ret < 0) { - if (ret == -ERESTARTSYS) { - if (DEBUGGER_EN(MSG)) - pr_err("%s, commit mpi job failed, by a software interrupt.\n", - __func__); - } else { - pr_err("%s, commit mpi job failed\n", __func__); + ctx = rga_internal_ctx_lookup(ctx_manager, mpi_job->ctx_id); + if (IS_ERR_OR_NULL(ctx)) { + pr_err("can not find internal ctx from id[%d]", mpi_job->ctx_id); + return -EINVAL; + } + + spin_lock_irqsave(&ctx->lock, flags); + + ctx->sync_mode = RGA_BLIT_SYNC; + /* TODO: batch mode need mpi async mode */ + ctx->use_batch_mode = false; + + cached_cmd = ctx->cached_cmd; + + spin_unlock_irqrestore(&ctx->lock, flags); + + /* change src cmd config by mpi frame info */ + if (!ctx->mpi_config_flags) { + cached_cmd->src.x_offset = mpi_job->src.x_offset; + cached_cmd->src.y_offset = mpi_job->src.y_offset; + cached_cmd->src.act_w = mpi_job->src.width; + cached_cmd->src.act_h = mpi_job->src.height; + cached_cmd->src.vir_w = mpi_job->src.vir_w; + cached_cmd->src.vir_h = mpi_job->src.vir_h; + cached_cmd->src.rd_mode = mpi_job->src.rd_mode; + cached_cmd->src.format = mpi_job->src.format; + } + + /* copy dst info to mpi job */ + mpi_job->dst.x_offset = cached_cmd->dst.x_offset; + mpi_job->dst.y_offset = cached_cmd->dst.y_offset; + mpi_job->dst.width = cached_cmd->dst.act_w; + mpi_job->dst.height = cached_cmd->dst.act_h; + mpi_job->dst.vir_w = cached_cmd->dst.vir_w; + mpi_job->dst.vir_h = cached_cmd->dst.vir_h; + mpi_job->dst.rd_mode = cached_cmd->dst.rd_mode; + mpi_job->dst.format = cached_cmd->dst.format; + + for (i = 0; i < ctx->cmd_num; i++) { + if (DEBUGGER_EN(MSG)) + rga_cmd_print_debug_info(&(cached_cmd[i])); + + ret = rga_job_mpi_commit(&(cached_cmd[i]), mpi_job, ctx); + if (ret < 0) { + if (ret == -ERESTARTSYS) { + if (DEBUGGER_EN(MSG)) + pr_err("%s, commit mpi job failed, by a software interrupt.\n", + __func__); + } else { + pr_err("%s, commit mpi job failed\n", __func__); + } + + return ret; } - - return ret; } return ret; @@ -67,12 +114,16 @@ EXPORT_SYMBOL_GPL(rga_mpi_commit); int rga_kernel_commit(struct rga_req *cmd) { - int ret; + int ret = 0; + struct rga_internal_ctx_t ctx; if (DEBUGGER_EN(MSG)) rga_cmd_print_debug_info(cmd); - ret = rga_job_commit(cmd, RGA_BLIT_SYNC); + ctx.sync_mode = RGA_BLIT_SYNC; + ctx.use_batch_mode = false; + + ret = rga_job_commit(cmd, &ctx); if (ret < 0) { if (ret == -ERESTARTSYS) { if (DEBUGGER_EN(MSG)) @@ -302,6 +353,113 @@ err_free_external_buffer: return ret; } +static long rga_ioctl_cmd_start(unsigned long arg) +{ + uint32_t rga_user_ctx_id; + int ret = 0; + + rga_user_ctx_id = rga_internal_ctx_alloc_to_get_idr_id(); + + if (copy_to_user((void *)arg, &rga_user_ctx_id, sizeof(uint32_t))) + ret = -EFAULT; + + return ret; +} + +static long rga_ioctl_cmd_config(unsigned long arg) +{ + struct rga_user_ctx_t rga_user_ctx; + int ret = 0; + + if (unlikely(copy_from_user(&rga_user_ctx, (struct rga_user_ctx_t *)arg, + sizeof(rga_user_ctx)))) { + pr_err("rga_user_ctx copy_from_user failed!\n"); + return -EFAULT; + } + + if (rga_user_ctx.cmd_num > RGA_CMD_NUM_MAX) { + pr_err("Cannot import more than %d buffers at a time!\n", + RGA_CMD_NUM_MAX); + return -EFBIG; + } + + if (rga_user_ctx.cmd_ptr == 0) { + pr_err("Cmd is NULL"); + return -EINVAL; + } + + if (rga_user_ctx.id <= 0) { + pr_err("ctx id[%d] is invalid", rga_user_ctx.id); + return -EINVAL; + } + + if (DEBUGGER_EN(MSG)) + pr_err("config cmd id = %d", rga_user_ctx.id); + + /* find internal_ctx to set cmd by user ctx (internal ctx id) */ + ret = rga_job_config_by_user_ctx(&rga_user_ctx); + if (ret < 0) { + pr_err("config ctx id[%d] failed!\n", rga_user_ctx.id); + return -EFAULT; + } + + return ret; +} + +static long rga_ioctl_cmd_end(unsigned long arg) +{ + struct rga_user_ctx_t rga_user_ctx; + int ret = 0; + + if (unlikely(copy_from_user(&rga_user_ctx, (struct rga_user_ctx_t *)arg, + sizeof(rga_user_ctx)))) { + pr_err("rga_user_ctx copy_from_user failed!\n"); + return -EFAULT; + } + + if (DEBUGGER_EN(MSG)) + pr_err("config end id = %d", rga_user_ctx.id); + + /* find internal_ctx to set cmd by user ctx (internal ctx id) */ + ret = rga_job_commit_by_user_ctx(&rga_user_ctx); + if (ret < 0) { + pr_err("commit ctx id[%d] failed!\n", rga_user_ctx.id); + return -EFAULT; + } + + if (copy_to_user((struct rga_user_ctx_t *)arg, + &rga_user_ctx, sizeof(struct rga_user_ctx_t))) { + pr_err("copy_to_user failed\n"); + return -EFAULT; + } + + return ret; +} + +static long rga_ioctl_cmd_cancel(unsigned long arg) +{ + uint32_t rga_user_ctx_id; + int ret = 0; + + if (unlikely(copy_from_user(&rga_user_ctx_id, (uint32_t *)arg, + sizeof(uint32_t)))) { + pr_err("rga_user_ctx_id copy_from_user failed!\n"); + return -EFAULT; + } + + if (DEBUGGER_EN(MSG)) + pr_err("config cancel id = %d", rga_user_ctx_id); + + /* find internal_ctx to set cmd by user ctx (internal ctx id) */ + ret = rga_job_cancel_by_user_ctx(rga_user_ctx_id); + if (ret < 0) { + pr_err("cancel ctx id[%d] failed!\n", rga_user_ctx_id); + return -EFAULT; + } + + return ret; +} + static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) { struct rga_drvdata_t *rga = rga_drvdata; @@ -312,6 +470,7 @@ static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) char version[16] = { 0 }; struct rga_version_t driver_version; struct rga_hw_versions_t hw_versions; + struct rga_internal_ctx_t ctx; if (!rga) { pr_err("rga_drvdata is null, rga is not init\n"); @@ -334,7 +493,10 @@ static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) if (DEBUGGER_EN(MSG)) rga_cmd_print_debug_info(&req_rga); - ret = rga_job_commit(&req_rga, cmd); + ctx.sync_mode = cmd; + ctx.use_batch_mode = false; + + ret = rga_job_commit(&req_rga, &ctx); if (ret < 0) { if (ret == -ERESTARTSYS) { if (DEBUGGER_EN(MSG)) @@ -438,6 +600,26 @@ static long rga_ioctl(struct file *file, uint32_t cmd, unsigned long arg) break; + case RGA_START_CONFIG: + ret = rga_ioctl_cmd_start(arg); + + break; + + case RGA_END_CONFIG: + ret = rga_ioctl_cmd_end(arg); + + break; + + case RGA_CMD_CONFIG: + ret = rga_ioctl_cmd_config(arg); + + break; + + case RGA_CANCEL_CONFIG: + ret = rga_ioctl_cmd_cancel(arg); + + break; + case RGA_IMPORT_DMA: case RGA_RELEASE_DMA: default: @@ -497,6 +679,31 @@ static int rga_open(struct inode *inode, struct file *file) static int rga_release(struct inode *inode, struct file *file) { + pid_t pid; + int ctx_id; + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + + pid = current->pid; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + mutex_lock(&ctx_manager->lock); + + idr_for_each_entry(&ctx_manager->ctx_id_idr, ctx, ctx_id) { + + mutex_unlock(&ctx_manager->lock); + + if (pid == ctx->pid) { + pr_err("[pid:%d] destroy ctx[%d] when the user exits", pid, ctx->id); + kref_put(&ctx->refcount, rga_internel_ctx_kref_release); + } + + mutex_lock(&ctx_manager->lock); + } + + mutex_unlock(&ctx_manager->lock); + return 0; } @@ -986,6 +1193,8 @@ static int __init rga_init(void) rga_mm_init(&rga_drvdata->mm); + rga_ctx_manager_init(&rga_drvdata->pend_ctx_manager); + #ifdef CONFIG_ROCKCHIP_RGA_DEBUGGER rga_debugger_init(&rga_drvdata->debugger); #endif @@ -1007,6 +1216,8 @@ static void __exit rga_exit(void) rga_mm_remove(&rga_drvdata->mm); + rga_ctx_manager_remove(&rga_drvdata->pend_ctx_manager); + wake_lock_destroy(&rga_drvdata->wake_lock); rga_fence_context_free(rga_drvdata->fence_ctx); diff --git a/drivers/video/rockchip/rga3/rga_job.c b/drivers/video/rockchip/rga3/rga_job.c index 09a1b1f1eb35..946cc2682e9e 100644 --- a/drivers/video/rockchip/rga3/rga_job.c +++ b/drivers/video/rockchip/rga3/rga_job.c @@ -236,7 +236,114 @@ static struct rga_job *rga_job_alloc(struct rga_req *rga_command_base) return job; } -static void print_job_info(struct rga_job *job) +struct rga_internal_ctx_t * +rga_internal_ctx_lookup(struct rga_pending_ctx_manager *ctx_manager, uint32_t id) +{ + struct rga_internal_ctx_t *ctx = NULL; + + mutex_lock(&ctx_manager->lock); + + ctx = idr_find(&ctx_manager->ctx_id_idr, id); + + mutex_unlock(&ctx_manager->lock); + + return ctx; +} + +/* + * Called at driver close to release the internal ctx's id references. + */ +static int rga_internal_ctx_free_remove_idr_cb(int id, void *ptr, void *data) +{ + struct rga_internal_ctx_t *ctx = ptr; + + idr_remove(&rga_drvdata->pend_ctx_manager->ctx_id_idr, ctx->id); + if (ctx->cached_cmd != NULL) { + kfree(ctx->cached_cmd); + ctx->cached_cmd = NULL; + } + kfree(ctx); + + return 0; +} + +static int rga_internal_ctx_free_remove_idr(struct rga_internal_ctx_t *ctx) +{ + struct rga_pending_ctx_manager *ctx_manager; + struct rga_req *cached_cmd; + unsigned long flags; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + if (IS_ERR_OR_NULL(ctx)) { + pr_err("ctx already freed"); + return -EFAULT; + } + + mutex_lock(&ctx_manager->lock); + + ctx_manager->ctx_count--; + idr_remove(&ctx_manager->ctx_id_idr, ctx->id); + + mutex_unlock(&ctx_manager->lock); + + spin_lock_irqsave(&ctx->lock, flags); + + cached_cmd = ctx->cached_cmd; + + spin_unlock_irqrestore(&ctx->lock, flags); + + if (cached_cmd != NULL) + kfree(cached_cmd); + + kfree(ctx); + + return 0; +} + +static int rga_internal_ctx_signal(struct rga_scheduler_t *scheduler, struct rga_job *job) +{ + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + int finished_job_count; + unsigned long flags; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + ctx = rga_internal_ctx_lookup(ctx_manager, job->ctx_id); + if (IS_ERR_OR_NULL(ctx)) { + pr_err("can not find internal ctx from id[%d]", job->ctx_id); + return -EINVAL; + } + + spin_lock_irqsave(&ctx->lock, flags); + + finished_job_count = ++ctx->finished_job_count; + + spin_unlock_irqrestore(&ctx->lock, flags); + + if (finished_job_count >= ctx->cmd_num) { + if (ctx->out_fence) + dma_fence_signal(ctx->out_fence); + + job->flags |= RGA_JOB_DONE; + + if (job->flags & RGA_JOB_ASYNC) + rga_job_cleanup(job); + + wake_up(&scheduler->job_done_wq); + + spin_lock_irqsave(&ctx->lock, flags); + + ctx->is_running = false; + + spin_unlock_irqrestore(&ctx->lock, flags); + } + + return 0; +} + +static void rga_job_dump_info(struct rga_job *job) { pr_info("job: priority = %d, core = %d\n", job->priority, job->core); @@ -281,7 +388,7 @@ static int rga_job_run(struct rga_job *job, struct rga_scheduler_t *scheduler) /* for debug */ if (DEBUGGER_EN(MSG)) - print_job_info(job); + rga_job_dump_info(job); return ret; @@ -333,13 +440,17 @@ next_job: else rga_dma_put_info(job); - if (job->out_fence) - dma_fence_signal(job->out_fence); + if (job->use_batch_mode) { + rga_internal_ctx_signal(rga_scheduler, job); + } else { + if (job->out_fence) + dma_fence_signal(job->out_fence); - if (job->flags & RGA_JOB_ASYNC) - rga_job_cleanup(job); - else { job->flags |= RGA_JOB_DONE; + + if (job->flags & RGA_JOB_ASYNC) + rga_job_cleanup(job); + wake_up(&rga_scheduler->job_done_wq); } @@ -347,23 +458,11 @@ next_job: } } -void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret) +static void rga_job_finish_and_next(struct rga_scheduler_t *rga_scheduler, + struct rga_job *job, int ret) { - struct rga_job *job; - unsigned long flags; - ktime_t now = ktime_get(); - spin_lock_irqsave(&rga_scheduler->irq_lock, flags); - - job = rga_scheduler->running_job; - rga_scheduler->running_job = NULL; - - rga_scheduler->timer.busy_time += ktime_us_delta(now, job->timestamp); - - spin_unlock_irqrestore(&rga_scheduler->irq_lock, flags); - - job->flags |= RGA_JOB_DONE; job->ret = ret; if (DEBUGGER_EN(TIME)) @@ -381,19 +480,43 @@ void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret) else rga_dma_put_info(job); - if (job->out_fence) - dma_fence_signal(job->out_fence); + if (job->use_batch_mode) + rga_internal_ctx_signal(rga_scheduler, job); + else { + if (job->out_fence) + dma_fence_signal(job->out_fence); - if (job->flags & RGA_JOB_ASYNC) - rga_job_cleanup(job); + job->flags |= RGA_JOB_DONE; - wake_up(&rga_scheduler->job_done_wq); + if (job->flags & RGA_JOB_ASYNC) + rga_job_cleanup(job); + + wake_up(&rga_scheduler->job_done_wq); + } rga_job_next(rga_scheduler); rga_power_disable(rga_scheduler); } +void rga_job_done(struct rga_scheduler_t *rga_scheduler, int ret) +{ + struct rga_job *job; + unsigned long flags; + ktime_t now = ktime_get(); + + spin_lock_irqsave(&rga_scheduler->irq_lock, flags); + + job = rga_scheduler->running_job; + rga_scheduler->running_job = NULL; + + rga_scheduler->timer.busy_time += ktime_us_delta(now, job->timestamp); + + spin_unlock_irqrestore(&rga_scheduler->irq_lock, flags); + + rga_job_finish_and_next(rga_scheduler, job, ret); +} + static void rga_job_timeout_clean(struct rga_scheduler_t *scheduler) { unsigned long flags; @@ -416,10 +539,14 @@ static void rga_job_timeout_clean(struct rga_scheduler_t *scheduler) else rga_dma_put_info(job); - if (job->out_fence) - dma_fence_signal(job->out_fence); + if (job->use_batch_mode) + rga_internal_ctx_signal(scheduler, job); + else { + if (job->out_fence) + dma_fence_signal(job->out_fence); - rga_job_cleanup(job); + rga_job_cleanup(job); + } rga_power_disable(scheduler); } else { @@ -450,6 +577,7 @@ static struct rga_scheduler_t *rga_job_schedule(struct rga_job *job) return NULL; } + /* Only async will timeout clean */ rga_job_timeout_clean(scheduler); spin_lock_irqsave(&scheduler->irq_lock, flags); @@ -564,7 +692,235 @@ static void rga_input_fence_signaled(struct dma_fence *fence, kfree(waiter); } -int rga_job_commit(struct rga_req *rga_command_base, int flags) +uint32_t rga_internal_ctx_alloc_to_get_idr_id(void) +{ + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (ctx == NULL) { + pr_err("can not kzalloc for rga_pending_ctx_manager\n"); + return -ENOMEM; + } + + ctx_manager = rga_drvdata->pend_ctx_manager; + if (ctx_manager == NULL) { + pr_err("rga_pending_ctx_manager is null!\n"); + kfree(ctx); + return -EFAULT; + } + + spin_lock_init(&ctx->lock); + + /* + * Get the user-visible handle using idr. Preload and perform + * allocation under our spinlock. + */ + + mutex_lock(&ctx_manager->lock); + + idr_preload(GFP_KERNEL); + ctx->id = idr_alloc(&ctx_manager->ctx_id_idr, ctx, 1, 0, GFP_KERNEL); + idr_preload_end(); + + ctx_manager->ctx_count++; + + kref_init(&ctx->refcount); + ctx->pid = current->pid; + + mutex_unlock(&ctx_manager->lock); + + if (ctx->id < 0) + pr_err("[pid: %d]alloc ctx_id failed", ctx->pid); + + return (uint32_t)ctx->id; +} + +int rga_job_config_by_user_ctx(struct rga_user_ctx_t *user_ctx) +{ + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + struct rga_req *cached_cmd; + int ret = 0; + bool first_config = false; + unsigned long flags; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + ctx = rga_internal_ctx_lookup(ctx_manager, user_ctx->id); + if (IS_ERR_OR_NULL(ctx)) { + pr_err("can not find internal ctx from id[%d]", user_ctx->id); + return -EINVAL; + } + + spin_lock_irqsave(&ctx->lock, flags); + + cached_cmd = ctx->cached_cmd; + + spin_unlock_irqrestore(&ctx->lock, flags); + + if (cached_cmd == NULL) { + cached_cmd = kmalloc_array(user_ctx->cmd_num, sizeof(struct rga_req), GFP_KERNEL); + if (cached_cmd == NULL) { + pr_err("cmd_cached list alloc error!\n"); + return -ENOMEM; + } + + first_config = true; + } + + if (unlikely(copy_from_user(cached_cmd, (struct rga_req *)(unsigned long)user_ctx->cmd_ptr, + sizeof(struct rga_req) * user_ctx->cmd_num))) { + pr_err("rga_user_ctx cmd list copy_from_user failed\n"); + if (first_config) + kfree(cached_cmd); + return -EFAULT; + } + + spin_lock_irqsave(&ctx->lock, flags); + + ctx->sync_mode = user_ctx->sync_mode; + ctx->cmd_num = user_ctx->cmd_num; + ctx->cached_cmd = cached_cmd; + ctx->mpi_config_flags = user_ctx->mpi_config_flags; + + spin_unlock_irqrestore(&ctx->lock, flags); + + return ret; +} + +int rga_job_commit_by_user_ctx(struct rga_user_ctx_t *user_ctx) +{ + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + struct rga_req *cached_cmd; + int i; + int ret = 0; + unsigned long flags; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + ctx = rga_internal_ctx_lookup(ctx_manager, user_ctx->id); + if (IS_ERR_OR_NULL(ctx)) { + pr_err("can not find internal ctx from id[%d]", user_ctx->id); + return -EINVAL; + } + + spin_lock_irqsave(&ctx->lock, flags); + + if (ctx->is_running) { + pr_err("can not re-config when ctx is running"); + spin_unlock_irqrestore(&ctx->lock, flags); + return -EFAULT; + } + + /* Reset */ + ctx->finished_job_count = 0; + + cached_cmd = ctx->cached_cmd; + if (cached_cmd == NULL) { + pr_err("can not find cached cmd from id[%d]", user_ctx->id); + spin_unlock_irqrestore(&ctx->lock, flags); + return -EINVAL; + } + + ctx->use_batch_mode = true; + ctx->is_running = true; + + spin_unlock_irqrestore(&ctx->lock, flags); + + for (i = 0; i < ctx->cmd_num; i++) { + ret = rga_job_commit(&(cached_cmd[i]), ctx); + if (ret < 0) { + pr_err("rga_job_commit failed\n"); + return -EFAULT; + } + } + + user_ctx->out_fence_fd = ctx->out_fence_fd; + + return ret; +} + +void rga_internel_ctx_kref_release(struct kref *ref) +{ + struct rga_internal_ctx_t *ctx; + struct rga_scheduler_t *scheduler = NULL; + struct rga_job *job_pos, *job_q, *job; + int i; + bool need_reset = false; + unsigned long flags; + ktime_t now = ktime_get(); + + ctx = container_of(ref, struct rga_internal_ctx_t, refcount); + + spin_lock_irqsave(&ctx->lock, flags); + + if (!ctx->is_running || ctx->finished_job_count >= ctx->cmd_num) { + spin_unlock_irqrestore(&ctx->lock, flags); + goto free_ctx; + } + + spin_unlock_irqrestore(&ctx->lock, flags); + + for (i = 0; i < rga_drvdata->num_of_scheduler; i++) { + scheduler = rga_drvdata->rga_scheduler[i]; + + spin_lock_irqsave(&scheduler->irq_lock, flags); + + list_for_each_entry_safe(job_pos, job_q, &scheduler->todo_list, head) { + if (ctx->id == job_pos->ctx_id) { + job = job_pos; + list_del_init(&job_pos->head); + + scheduler->job_count--; + } + } + + if (scheduler->running_job) { + job = scheduler->running_job; + + if (job->ctx_id == ctx->id) { + scheduler->running_job = NULL; + scheduler->timer.busy_time += ktime_us_delta(now, job->timestamp); + need_reset = true; + } + } + + spin_unlock_irqrestore(&scheduler->irq_lock, flags); + + if (need_reset) { + pr_info("reset core[%d] by user cancel", scheduler->core); + scheduler->ops->soft_reset(scheduler); + + rga_job_finish_and_next(scheduler, job, 0); + } + } + +free_ctx: + rga_internal_ctx_free_remove_idr(ctx); +} + +int rga_job_cancel_by_user_ctx(uint32_t ctx_id) +{ + struct rga_pending_ctx_manager *ctx_manager; + struct rga_internal_ctx_t *ctx; + int ret = 0; + + ctx_manager = rga_drvdata->pend_ctx_manager; + + ctx = rga_internal_ctx_lookup(ctx_manager, ctx_id); + if (IS_ERR_OR_NULL(ctx)) { + pr_err("can not find internal ctx from id[%d]", ctx_id); + return -EINVAL; + } + + kref_put(&ctx->refcount, rga_internel_ctx_kref_release); + + return ret; +} + +int rga_job_commit(struct rga_req *rga_command_base, struct rga_internal_ctx_t *ctx) { struct rga_job *job = NULL; struct rga_scheduler_t *scheduler = NULL; @@ -577,6 +933,9 @@ int rga_job_commit(struct rga_req *rga_command_base, int flags) return -ENOMEM; } + job->use_batch_mode = ctx->use_batch_mode; + job->ctx_id = ctx->id; + /* * because fd can not pass on other thread, * so need to get dma_buf first. @@ -589,14 +948,25 @@ int rga_job_commit(struct rga_req *rga_command_base, int flags) return ret; } - if (flags == RGA_BLIT_ASYNC) { - ret = rga_out_fence_alloc(job); - if (ret) { - rga_job_free(job); - return ret; - } + if (ctx->sync_mode == RGA_BLIT_ASYNC) { job->flags |= RGA_JOB_ASYNC; + + if (ctx->out_fence) { + job->out_fence = ctx->out_fence; + } else { + ret = rga_out_fence_alloc(job); + if (ret) { + rga_job_free(job); + return ret; + } + + /* on batch mode, only first job need to alloc fence */ + if (ctx->use_batch_mode) + ctx->out_fence = job->out_fence; + } + rga_command_base->out_fence_fd = rga_out_fence_get_fd(job); + ctx->out_fence_fd = rga_command_base->out_fence_fd; if (DEBUGGER_EN(MSG)) pr_err("in_fence_fd = %d", @@ -652,8 +1022,9 @@ int rga_job_commit(struct rga_req *rga_command_base, int flags) } return ret; - /* sync mode: wait utill job finish */ - } else if (flags == RGA_BLIT_SYNC) { + + /* RGA_BLIT_SYNC: wait until job finish */ + } else if (ctx->sync_mode == RGA_BLIT_SYNC) { scheduler = rga_job_schedule(job); if (scheduler == NULL) { pr_err("failed to get scheduler, %s(%d)\n", __func__, @@ -689,7 +1060,7 @@ running_job_abort: } int rga_job_mpi_commit(struct rga_req *rga_command_base, - struct rga_mpi_job_t *mpi_job, int flags) + struct rga_mpi_job_t *mpi_job, struct rga_internal_ctx_t *ctx) { struct rga_job *job = NULL; struct rga_scheduler_t *scheduler = NULL; @@ -707,10 +1078,12 @@ int rga_job_mpi_commit(struct rga_req *rga_command_base, job->dma_buf_dst = mpi_job->dma_buf_dst; } - if (flags == RGA_BLIT_ASYNC) { - //TODO: + job->ctx_id = ctx->id; + + if (ctx->sync_mode == RGA_BLIT_ASYNC) { + //TODO: mpi async mode pr_err("rk-debug TODO\n"); - } else if (flags == RGA_BLIT_SYNC) { + } else if (ctx->sync_mode == RGA_BLIT_SYNC) { scheduler = rga_job_schedule(job); if (scheduler == NULL) { pr_err("failed to get scheduler, %s(%d)\n", __func__, @@ -743,3 +1116,40 @@ running_job_abort: rga_running_job_abort(job, scheduler); return ret; } + +int rga_ctx_manager_init(struct rga_pending_ctx_manager **ctx_manager_session) +{ + struct rga_pending_ctx_manager *ctx_manager = NULL; + + *ctx_manager_session = kzalloc(sizeof(struct rga_pending_ctx_manager), GFP_KERNEL); + if (*ctx_manager_session == NULL) { + pr_err("can not kzalloc for rga_pending_ctx_manager\n"); + return -ENOMEM; + } + + ctx_manager = *ctx_manager_session; + + mutex_init(&ctx_manager->lock); + + idr_init_base(&ctx_manager->ctx_id_idr, 1); + + return 0; +} + +int rga_ctx_manager_remove(struct rga_pending_ctx_manager **ctx_manager_session) +{ + struct rga_pending_ctx_manager *ctx_manager = *ctx_manager_session; + + mutex_lock(&ctx_manager->lock); + + idr_for_each(&ctx_manager->ctx_id_idr, &rga_internal_ctx_free_remove_idr_cb, ctx_manager); + idr_destroy(&ctx_manager->ctx_id_idr); + + mutex_unlock(&ctx_manager->lock); + + kfree(*ctx_manager_session); + + *ctx_manager_session = NULL; + + return 0; +}