diff --git a/drivers/media/common/codec_mm/Makefile b/drivers/media/common/codec_mm/Makefile index 2ffed88ce..fee33e1f6 100644 --- a/drivers/media/common/codec_mm/Makefile +++ b/drivers/media/common/codec_mm/Makefile @@ -17,5 +17,6 @@ $(MEDIA_MODULE_NAME)-$(CONFIG_AMLOGIC_MEDIA_CODEC_MM) += common/codec_mm/dmabuf_ $(MEDIA_MODULE_NAME)-$(CONFIG_AMLOGIC_MEDIA_CODEC_MM) += common/codec_mm/codec_mm_track.o $(MEDIA_MODULE_NAME)-$(CONFIG_AMLOGIC_MEDIA_CODEC_MM) += common/codec_mm/codec_mm_track_kps.o $(MEDIA_MODULE_NAME)-$(CONFIG_AMLOGIC_MEDIA_CODEC_MM) += common/codec_mm/codec_mm_state.o +$(MEDIA_MODULE_NAME)-$(CONFIG_AMLOGIC_MEDIA_CODEC_MM) += common/codec_mm/codec_mm_sys.o include $(srctree)/$(COMMON_DRIVERS_DIR)/drivers/media/common/codec_mm/configs/Makefile diff --git a/drivers/media/common/codec_mm/codec_mm.c b/drivers/media/common/codec_mm/codec_mm.c index 3be8cfebd..124015295 100644 --- a/drivers/media/common/codec_mm/codec_mm.c +++ b/drivers/media/common/codec_mm/codec_mm.c @@ -41,7 +41,6 @@ #if IS_ENABLED(CONFIG_AMLOGIC_CPU_INFO) #include #else - unsigned char get_meson_cpu_version(int level) { return 0; @@ -51,11 +50,13 @@ unsigned char get_meson_cpu_version(int level) #if IS_BUILTIN(CONFIG_AMLOGIC_MEDIA_MODULE) #include #endif +#define CONFIG_CODEC_MM_EXT_POOL #include "codec_mm_priv.h" #include "codec_mm_scatter_priv.h" #include "codec_mm_keeper_priv.h" #include "codec_mm_track_priv.h" +#include "codec_mm_sys_priv.h" #include #include #include @@ -73,10 +74,6 @@ void tee_unprotect_tvp_mem(u32 handle) { } -void tee_unprotect_mem(u32 handle) -{ -} - u32 tee_register_mem(u32 type, phys_addr_t pa, size_t size) { pr_info("no tee config\n"); @@ -95,11 +92,6 @@ struct mm_struct *aml_init_mm; struct device *codec_dev; #ifdef CONFIG_ARM64 -pte_t * (*aml__pte_offset_map)(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); - -unsigned long (*aml_get_pfnblock_flags_mask)(const struct page *page, - unsigned long pfn, unsigned long mask); - static void aml_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { @@ -128,9 +120,7 @@ static void aml_set_pte_at(struct mm_struct *mm, unsigned long addr, aml_mte_sync_tags(old_pte, pte); } - /* - * __check_racy_pte_update(mm, ptep, pte); - */ + __check_racy_pte_update(mm, ptep, pte); set_pte(ptep, pte); } @@ -141,7 +131,7 @@ static bool is_cma_page(struct page *page) if (!page) return false; - migrate_type = aml_get_pfnblock_flags_mask(page, page_to_pfn(page), MIGRATETYPE_MASK); + migrate_type = get_pageblock_migratetype(page); if (is_migrate_cma(migrate_type) || is_migrate_isolate(migrate_type)) { return true; @@ -173,7 +163,7 @@ int cma_mmu_op(struct page *page, int count, bool set) } if (!aml_init_mm || !aml_mte_sync_tags) { - pr_debug("%s, no cma mmu operation.\n", __func__); + pr_err("%s, no cma mmu operation.\n", __func__); return -EINVAL; } @@ -197,11 +187,11 @@ int cma_mmu_op(struct page *page, int count, bool set) if (pmd_none(*pmd)) break; - pte = aml__pte_offset_map(pmd, addr, NULL); + pte = pte_offset_map(pmd, addr); if (set) aml_set_pte_at(mm, addr, pte, mk_pte(page, pgprot_tagged(PAGE_KERNEL))); else - __pte_clear(mm, addr, pte); + pte_clear(mm, addr, pte); pte_unmap(pte); #ifdef CONFIG_ARM pr_debug("%s, add:%lx, pgd:%p %x, pmd:%p %x, pte:%p %x\n", @@ -225,6 +215,9 @@ int cma_mmu_op(struct page *page, int count, bool set) #endif #include +static bool secure_mem_ctrl; +static u32 secure_mem_align2n; + #define TVP_POOL_NAME "TVP_POOL" #define CMA_RES_POOL_NAME "CMA_RES" @@ -273,6 +266,10 @@ static void dump_mem_infos(struct seq_file *m); static int dump_free_mem_infos(void *buf, int size); static int secure_vdec_res_setup(struct reserved_mem *rmem); +static int codec_mm_res_setup(struct reserved_mem *rmem); +#ifdef CONFIG_CODEC_MM_EXT_POOL +static int codec_mm_res_ext_setup(struct reserved_mem *rmem); +#endif static inline u32 codec_mm_align_up2n(u32 addr, u32 alg2n) { @@ -435,6 +432,12 @@ struct codec_mm_mgt_s { struct extpool_mgt_s tvp_pool; struct extpool_mgt_s cma_res_pool; struct reserved_mem rmem; +#ifdef CONFIG_CODEC_MM_EXT_POOL + struct reserved_mem rmem_ext; + struct gen_pool *res_ext_pool; + int total_reserved_ext_size; + int alloced_res_ext_size; +#endif int total_codec_mem_size; int total_alloced_size; int total_cma_size; @@ -470,6 +473,10 @@ struct codec_mm_mgt_s { struct codec_state_node cs; void *trk_h; struct work_struct tvp_alloc_wrk; + ulong *tee_sectbl_bitmap; + u32 tee_sectbl_size; + /* spin lock for sectbl bitmap update */ + spinlock_t tee_sectbl_lock; }; #define PHY_OFF() offsetof(struct codec_mm_s, phy_addr) @@ -507,6 +514,34 @@ static struct codec_mm_mgt_s *get_mem_mgt(void) return &mgt; }; +void set_sectbl_bitmap(ulong phy_addr, int buffer_size) +{ + unsigned long bitmap_no, bitmap_count; + unsigned long flags; + struct codec_mm_mgt_s *mgt = get_mem_mgt(); + + bitmap_no = (phy_addr - cma_get_base(dev_get_cma_area(mgt->dev))) >> secure_mem_align2n; + bitmap_count = ALIGN(buffer_size, 1UL << secure_mem_align2n) >> secure_mem_align2n; + + spin_lock_irqsave(&mgt->tee_sectbl_lock, flags); + bitmap_set(mgt->tee_sectbl_bitmap, bitmap_no, bitmap_count); + spin_unlock_irqrestore(&mgt->tee_sectbl_lock, flags); +} + +void clear_sectbl_bitmap(ulong phy_addr, int buffer_size) +{ + unsigned long bitmap_no, bitmap_count; + unsigned long flags; + struct codec_mm_mgt_s *mgt = get_mem_mgt(); + + bitmap_no = (phy_addr - cma_get_base(dev_get_cma_area(mgt->dev))) >> secure_mem_align2n; + bitmap_count = ALIGN(buffer_size, 1UL << secure_mem_align2n) >> secure_mem_align2n; + + spin_lock_irqsave(&mgt->tee_sectbl_lock, flags); + bitmap_clear(mgt->tee_sectbl_bitmap, bitmap_no, bitmap_count); + spin_unlock_irqrestore(&mgt->tee_sectbl_lock, flags); +} + static void *codec_mm_extpool_alloc(struct extpool_mgt_s *tvp_pool, void **from_pool, int size) { @@ -860,6 +895,44 @@ void codec_mm_dev_set_dma_mask(u64 bits) } EXPORT_SYMBOL(codec_mm_dev_set_dma_mask); +#ifdef CONFIG_CODEC_MM_EXT_POOL +static int codec_mm_alloc_first(struct codec_mm_mgt_s *mgt, + struct codec_mm_s *mem) +{ + if (!(mem->flags & CODEC_MM_FLAGS_RESERVED_EXT)) + return 0; + + if (mgt->total_reserved_ext_size > mem->buffer_size && + mem->align2n <= RESERVE_MM_ALIGNED_2N) { + int aligned_buffer_size = ALIGN(mem->buffer_size, + (1 << RESERVE_MM_ALIGNED_2N)); + if (mem->flags & CODEC_MM_FLAGS_TVP) + // to do.. + ; + mem->mem_handle = (void *)gen_pool_alloc(mgt->res_ext_pool, + aligned_buffer_size); + mem->from_flags = + AMPORTS_MEM_FLAGS_FROM_GET_FROM_REVERSED_EXT; + if (mem->mem_handle) { + /*default is no map */ + mem->vbuffer = NULL; + mem->phy_addr = (unsigned long)mem->mem_handle; + mem->buffer_size = aligned_buffer_size; + if (debug_mode & 0x10) + pr_info("alloc flags:%d,align=%d,pages:%d,s:%d\n", + mem->flags, + mem->align2n, + mem->page_count, + mem->buffer_size); + return 0; + } else { + return -ENOMEM; + } + } + return -ENOMEM; +} +#endif + static int codec_mm_alloc_in(struct codec_mm_mgt_s *mgt, struct codec_mm_s *mem) { int try_alloced_from_sys = 0; @@ -887,12 +960,18 @@ static int codec_mm_alloc_in(struct codec_mm_mgt_s *mgt, struct codec_mm_s *mem) (mem->page_count <= mgt->alloc_from_sys_pages_max); int can_from_tvp = (mem->flags & CODEC_MM_FLAGS_TVP); + if (can_from_tvp) { can_from_sys = 0; can_from_res = 0; can_from_cma = 0; } +#ifdef CONFIG_CODEC_MM_EXT_POOL + if (mem->flags & CODEC_MM_FLAGS_RESERVED_EXT) + return codec_mm_alloc_first(mgt, mem); +#endif + have_space = codec_mm_alloc_pre_check_in(mgt, mem->buffer_size, mem->flags); if (!have_space) return -10001; @@ -942,7 +1021,6 @@ static int codec_mm_alloc_in(struct codec_mm_mgt_s *mgt, struct codec_mm_s *mem) PAGE_SHIFT, false); mem->from_flags = AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA; if (mem->mem_handle) { - mem->vbuffer = mem->mem_handle; mem->phy_addr = page_to_phys((struct page *)mem->mem_handle); if (!mgt->tvp_enable) { @@ -951,6 +1029,10 @@ static int codec_mm_alloc_in(struct codec_mm_mgt_s *mgt, struct codec_mm_s *mem) mem->page_count << PAGE_SHIFT, DMA_FROM_DEVICE); } + mem->vbuffer = (mem->flags & + CODEC_MM_FLAGS_CPU) ? + codec_mm_map_phyaddr(mem) : + NULL; #ifdef CONFIG_ARM64 if (mem->flags & CODEC_MM_FLAGS_CMA_CLEAR) { /*dma_clear_buffer((struct page *)*/ @@ -983,7 +1065,7 @@ static int codec_mm_alloc_in(struct codec_mm_mgt_s *mgt, struct codec_mm_s *mem) if (can_from_res) { if (mgt->cma_res_pool.total_size > 0 && (mgt->cma_res_pool.alloced_size + - mem->buffer_size) < mgt->cma_res_pool.total_size) { + mem->buffer_size) <= mgt->cma_res_pool.total_size) { /* *from cma res first. */ @@ -1167,6 +1249,14 @@ static void codec_mm_free_in(struct codec_mm_mgt_s *mgt, if (!mgt->fastplay_enable && codec_mm_extpool_pool_release(&mgt->cma_res_pool) == 0) pr_info("disable cma_res mem\n"); } +#ifdef CONFIG_CODEC_MM_EXT_POOL + else if (mem->from_flags == + AMPORTS_MEM_FLAGS_FROM_GET_FROM_REVERSED_EXT) { + gen_pool_free(mgt->res_ext_pool, + (unsigned long)mem->mem_handle, + mem->buffer_size); + } +#endif spin_lock_irqsave(&mgt->lock, flags); if (!(mem->flags & CODEC_MM_FLAGS_FOR_LOCAL_MGR)) mgt->total_alloced_size -= mem->buffer_size; @@ -1190,6 +1280,12 @@ static void codec_mm_free_in(struct codec_mm_mgt_s *mgt, } else if (mem->from_flags == AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES) { mgt->cma_res_pool.alloced_size -= mem->buffer_size; } +#ifdef CONFIG_CODEC_MM_EXT_POOL + else if (mem->from_flags == + AMPORTS_MEM_FLAGS_FROM_GET_FROM_REVERSED_EXT) { + mgt->alloced_res_ext_size -= mem->buffer_size; + } +#endif spin_unlock_irqrestore(&mgt->lock, flags); if (mem->from_flags == AMPORTS_MEM_FLAGS_FROM_GET_FROM_TVP && @@ -1334,6 +1430,11 @@ struct codec_mm_s *codec_mm_alloc(const char *owner, int size, case AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES: mgt->cma_res_pool.alloced_size += mem->buffer_size; break; +#ifdef CONFIG_CODEC_MM_EXT_POOL + case AMPORTS_MEM_FLAGS_FROM_GET_FROM_REVERSED_EXT: + mgt->alloced_res_ext_size += mem->buffer_size; + break; +#endif default: pr_err("error alloc flags %d\n", mem->from_flags); } @@ -1482,16 +1583,8 @@ void *codec_mm_dma_alloc_coherent(ulong *handle, spin_lock_irqsave(&mgt->lock, flags); mem->mem_id = mgt->global_memid++; - if (s_cma) { + if (s_cma) mgt->alloced_cma_size += buf_size; - } else if (s_res) { - if (mgt->cma_res_pool.total_size > 0) - mgt->cma_res_pool.total_size += buf_size; - else - mgt->alloced_res_size += buf_size; - } else { - mgt->alloced_sys_size += buf_size; - } mgt->alloced_from_coherent += buf_size; mgt->total_alloced_size += buf_size; if (mgt->total_alloced_size > mgt->max_used_mem_size) @@ -1531,13 +1624,6 @@ void codec_mm_dma_free_coherent(ulong handle) if (mem->flags & 2) mgt->alloced_cma_size -= mem->buffer_size; - else if (mem->flags & 1) - if (mgt->cma_res_pool.total_size > 0) - mgt->cma_res_pool.total_size += mem->buffer_size; - else - mgt->alloced_res_size += mem->buffer_size; - else - mgt->alloced_sys_size -= mem->buffer_size; mgt->alloced_from_coherent -= mem->buffer_size; mgt->total_alloced_size -= mem->buffer_size; list_del(&mem->list); @@ -1751,14 +1837,21 @@ static int codec_mm_tvp_pool_protect(struct extpool_mgt_s *tvp_pool) for (i = 0; i < tvp_pool->slot_num; i++) { if (tvp_pool->mm[i]->tvp_handle == -1) { - ret = tee_protect_tvp_mem - (tvp_pool->mm[i]->phy_addr, - tvp_pool->mm[i]->buffer_size, - &tvp_pool->mm[i]->tvp_handle); - pr_info("protect tvp %d %d ret %d %lx %x\n", - i, tvp_pool->mm[i]->tvp_handle, ret, - tvp_pool->mm[i]->phy_addr, - tvp_pool->mm[i]->buffer_size); + if (secure_mem_ctrl) { + ret = tee_sectbl_secmem_set(tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size, 1); + set_sectbl_bitmap(tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size); + tvp_pool->mm[i]->tvp_handle = i; + } else { + ret = tee_protect_tvp_mem + (tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size, + &tvp_pool->mm[i]->tvp_handle); + } + pr_info("protect tvp %d %d ret %d %lx %x\n", i, tvp_pool->mm[i]->tvp_handle, + ret, tvp_pool->mm[i]->phy_addr, tvp_pool->mm[i]->buffer_size); + if (ret) break; } else { @@ -1769,6 +1862,15 @@ static int codec_mm_tvp_pool_protect(struct extpool_mgt_s *tvp_pool) return ret; } +static bool codec_mm_from_reserved(int flags) +{ + if (flags == AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES || + flags == AMPORTS_MEM_FLAGS_FROM_GET_FROM_REVERSED) + return true; + + return false; +} + static int codec_mm_extpool_pool_release_inner(int slot_num_start, struct extpool_mgt_s *tvp_pool) { @@ -1780,10 +1882,17 @@ static int codec_mm_extpool_pool_release_inner(int slot_num_start, int slot_mem_size = 0; if (gpool) { - if (tvp_pool->mm[i] && tvp_pool->mm[i]->tvp_handle > 0) { + if (tvp_pool->mm[i] && tvp_pool->mm[i]->tvp_handle >= 0) { pr_info("unprotect tvp %d handle is %d\n", i, tvp_pool->mm[i]->tvp_handle); - tee_unprotect_tvp_mem(tvp_pool->mm[i]->tvp_handle); + if (secure_mem_ctrl) { + tee_sectbl_secmem_set(tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size, 0); + clear_sectbl_bitmap(tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size); + } else { + tee_unprotect_tvp_mem(tvp_pool->mm[i]->tvp_handle); + } tvp_pool->mm[i]->tvp_handle = -1; } slot_mem_size = gen_pool_size(gpool); @@ -1791,8 +1900,7 @@ static int codec_mm_extpool_pool_release_inner(int slot_num_start, if (tvp_pool->mm[i]) { struct page *mm = tvp_pool->mm[i]->mem_handle; - if (tvp_pool->mm[i]->from_flags == - AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES) + if (codec_mm_from_reserved(tvp_pool->mm[i]->from_flags)) mm = phys_to_page((unsigned long)mm); cma_mmu_op(mm, tvp_pool->mm[i]->page_count, @@ -1824,8 +1932,7 @@ static int codec_mm_tvp_pool_alloc_by_type(struct extpool_mgt_s *tvp_pool, if (mem) { struct page *mm = mem->mem_handle; - if (mem->from_flags == - AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES) + if (codec_mm_from_reserved(mem->from_flags)) mm = phys_to_page((unsigned long)mm); cma_mmu_op(mm, mem->page_count, 0); ret = codec_mm_init_tvp_pool(tvp_pool, mem); @@ -1853,6 +1960,11 @@ static int codec_mm_tvp_pool_alloc_by_type(struct extpool_mgt_s *tvp_pool, CODEC_MM_FLAGS_RESERVED); if (mem) { + struct page *mm = mem->mem_handle; + + if (codec_mm_from_reserved(mem->from_flags)) + mm = phys_to_page((unsigned long)mm); + cma_mmu_op(mm, mem->page_count, 0); ret = codec_mm_init_tvp_pool(tvp_pool, mem); if (ret < 0) { codec_mm_release(mem, TVP_POOL_NAME); @@ -2118,6 +2230,11 @@ int codec_mm_extpool_pool_alloc(struct extpool_mgt_s *tvp_pool, CODEC_MM_FLAGS_RESERVED); if (mem) { + struct page *mm = mem->mem_handle; + + if (codec_mm_from_reserved(mem->from_flags)) + mm = phys_to_page((unsigned long)mm); + cma_mmu_op(mm, mem->page_count, 0); ret = codec_mm_init_tvp_pool(tvp_pool, mem); if (ret < 0) { codec_mm_release(mem, TVP_POOL_NAME); @@ -2166,8 +2283,7 @@ int codec_mm_extpool_pool_alloc(struct extpool_mgt_s *tvp_pool, if (mem) { struct page *mm = mem->mem_handle; - if (mem->from_flags == - AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES) + if (codec_mm_from_reserved(mem->from_flags)) mm = phys_to_page((unsigned long)mm); if (for_tvp) { cma_mmu_op(mm, @@ -2254,8 +2370,7 @@ static int codec_mm_extpool_pool_release(struct extpool_mgt_s *tvp_pool) if (tvp_pool->mm[i]) { struct page *mm = tvp_pool->mm[i]->mem_handle; - if (tvp_pool->mm[i]->from_flags == - AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES) + if (codec_mm_from_reserved(tvp_pool->mm[i]->from_flags)) mm = phys_to_page((unsigned long)mm); cma_mmu_op(mm, tvp_pool->mm[i]->page_count, @@ -2318,12 +2433,19 @@ static int codec_mm_tvp_pool_unprotect_and_release(struct extpool_mgt_s *tvp_poo pr_info("unprotect tvp %d handle is %d\n", i, tvp_pool->mm[i]->tvp_handle); - if (tvp_pool->mm[i]->tvp_handle > 0) { - tee_unprotect_tvp_mem(tvp_pool->mm[i]->tvp_handle); + if (tvp_pool->mm[i]->tvp_handle >= 0) { + if (secure_mem_ctrl) { + tee_sectbl_secmem_set(tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size, 0); + clear_sectbl_bitmap(tvp_pool->mm[i]->phy_addr, + tvp_pool->mm[i]->buffer_size); + } else { + tee_unprotect_tvp_mem(tvp_pool->mm[i]->tvp_handle); + } tvp_pool->mm[i]->tvp_handle = -1; } - if (tvp_pool->mm[i]->from_flags == - AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES) + + if (codec_mm_from_reserved(tvp_pool->mm[i]->from_flags)) mm = phys_to_page((unsigned long)mm); cma_mmu_op(mm, tvp_pool->mm[i]->page_count, 1); codec_mm_release(tvp_pool->mm[i], TVP_POOL_NAME); @@ -2571,6 +2693,13 @@ static void dump_mem_infos(struct seq_file *m) (mgt->phys_vmaped_page_cnt << PAGE_SHIFT) / SZ_1M); pbuf += get_string_offset(&buf_len, &tsize, s); +#ifdef CONFIG_CODEC_MM_EXT_POOL + s = snprintf(pbuf, buf_size - tsize, + "\tRES_EXT:%d MB\n", + mgt->alloced_res_ext_size / SZ_1M); + pbuf += get_string_offset(&buf_len, &tsize, s); +#endif + if (mgt->res_pool) { s = snprintf(pbuf, buf_size - tsize, "\t[%d]RES size:%d MB,alloced:%d MB free:%d MB\n", @@ -2714,6 +2843,32 @@ static int dump_free_mem_infos(void *buf, int size) return 0; } +void set_secure_controller_mode(struct codec_mm_mgt_s *mgt) +{ + u32 temp[2] = { 0 }; + int ret = of_property_read_u32_array(mgt->dev->of_node, "secure_ctrl_para", + temp, ARRAY_SIZE(temp)); + + if (ret) + return; + + secure_mem_ctrl = temp[0]; + if (secure_mem_ctrl) { + phys_addr_t cma_base = cma_get_base(dev_get_cma_area(mgt->dev)); + ulong cma_size = cma_get_size(dev_get_cma_area(mgt->dev)); + + secure_mem_align2n = temp[1]; + tee_sectbl_mem_map(cma_base, cma_size, 1 << secure_mem_align2n, 0, 0, 0); + + mgt->tee_sectbl_size = (cma_size >> secure_mem_align2n) >> 3; + mgt->tee_sectbl_bitmap = vzalloc(ALIGN(mgt->tee_sectbl_size, 4)); + spin_lock_init(&mgt->tee_sectbl_lock); + + pr_info("codec_mm secure_mem_align2n is %d tee_sectbl_size is %d bytes\n", + secure_mem_align2n, mgt->tee_sectbl_size); + } +} + static void codec_mm_tvp_segment_init(void) { u32 size = 0; @@ -3179,12 +3334,39 @@ int codec_mm_mgt_init(struct device *dev) #endif } +#ifdef CONFIG_CODEC_MM_EXT_POOL + if (mgt->rmem_ext.size > 0) { + unsigned long aligned_addr; + int aligned_size; + /*mem aligned, */ + mgt->res_ext_pool = gen_pool_create(RESERVE_MM_ALIGNED_2N, -1); + if (!mgt->res_ext_pool) + return -ENOMEM; + aligned_addr = ((unsigned long)mgt->rmem_ext.base + + ((1 << RESERVE_MM_ALIGNED_2N) - 1)) & + (~((1 << RESERVE_MM_ALIGNED_2N) - 1)); + aligned_size = mgt->rmem_ext.size - + (int)(aligned_addr - (unsigned long)mgt->rmem_ext.base); + ret = gen_pool_add(mgt->res_ext_pool, + aligned_addr, aligned_size, -1); + if (ret < 0) { + gen_pool_destroy(mgt->res_ext_pool); + return -1; + } + pr_debug("add reserve ext memory %p(aligned %p) size=%x(aligned %x)\n", + (void *)mgt->rmem_ext.base, (void *)aligned_addr, + (int)mgt->rmem_ext.size, (int)aligned_size); + mgt->total_reserved_ext_size = aligned_size; + } +#endif + mgt->total_cma_size = codec_mm_get_cma_size_int_byte(mgt->dev); mgt->total_codec_mem_size += mgt->total_cma_size; if (get_meson_cpu_version(MESON_CPU_VERSION_LVL_MAJOR) < MESON_CPU_MAJOR_ID_G12A) tvp_dynamic_increase_disable = 1; default_tvp_4k_size = 0; + set_secure_controller_mode(mgt); codec_mm_tvp_segment_init(); default_cma_res_size = mgt->total_cma_size; mgt->global_memid = 0; @@ -3494,17 +3676,49 @@ static int codec_mm_mem_dump(unsigned long addr, int isphy, int len) return 0; } -static ssize_t debug_store(const struct class *class, - const struct class_attribute *attr, - const char *buf, size_t size) +static void dump_tee_sectbl_info(void) { - unsigned int val; + u32 ret; + struct tee_sectbl_info tbl_info = { 0 }; + + if (!secure_mem_ctrl) + return; + + ret = tee_sectbl_get_info(&tbl_info); + if (ret) { + pr_info("get tee_sectbl info fail ret %u\n", ret); + return; + } + + pr_info("codec_mm tee_info addr 0x%llx size 0x%llx blk_size 0x%x secure_bits %u sectbl_crc 0x%x\n", + tbl_info.tbl0_sta, tbl_info.tbl0_size, tbl_info.tbl0_blk_size, + tbl_info.secure_bits, tbl_info.sectbl_crc); +} + +static void dump_tee_sectbl_bitmap(void) +{ + int i; + struct codec_mm_mgt_s *mgt = get_mem_mgt(); + + if (!secure_mem_ctrl) + return; + + pr_info("Start dump tee sectbl bitmap\n"); + for (i = 0; i < mgt->tee_sectbl_size / 4; i++) + pr_err("line%d 0x%08lx\n", i, *(mgt->tee_sectbl_bitmap + i)); + pr_info("End dump tee sectbl bitmap\n"); +} + +static ssize_t debug_store(const struct class *class, + const struct class_attribute *attr, + const char *buf, size_t size) +{ + unsigned int val, tmp; ssize_t ret; val = -1; - /*ret = sscanf(buf, "%d", &val);*/ - ret = kstrtoint(buf, 0, &val); - if (ret != 0) + ret = sscanf(buf, "%d %d", &val, &tmp); + if (ret == 0) return -EINVAL; switch (val) { case 0: @@ -3540,6 +3754,12 @@ static ssize_t debug_store(const struct class *class, case 12: dump_free_mem_infos(NULL, 0); break; + case 13: + dump_tee_sectbl_info(); + break; + case 14: + dump_tee_sectbl_bitmap(); + break; case 20: { int cmd = 0, len = 0; unsigned int addr; @@ -3560,6 +3780,20 @@ static ssize_t debug_store(const struct class *class, case 200: codec_mm_dbuf_walk(NULL); break; + case 900: { + uint cmd, buffer_page_num, buffer_num, intrval_ms, times; + uint wait_size, tvp_mode; + + ret = sscanf(buf, "%d %d %d %d %d %d %d", + &cmd, ×, &buffer_page_num, &buffer_num, + &intrval_ms, &wait_size, &tvp_mode); + pr_info("buf num:%d, page num:%d, intrval_ms:%d, times:%d, wait size:%d, tvp:%d\n", + buffer_num, buffer_page_num, intrval_ms, + times, wait_size, tvp_mode); + codec_mm_scatter_simu_thread_test(times, buffer_page_num, buffer_num, + intrval_ms, wait_size, tvp_mode); + } + break; default: pr_err("unknown cmd! %d\n", val); } @@ -4014,24 +4248,6 @@ unsigned long (*aml_syms_lookup)(const char *name); static struct kprobe kp_lookup_name = { .symbol_name = "kallsyms_lookup_name", }; - -static void *get_symbol_addr(const char *symbol_name) -{ - struct kprobe kp = { - .symbol_name = symbol_name, - }; - int ret; - - ret = register_kprobe(&kp); - if (ret < 0) { - pr_err("register_kprobe:%s failed, returned %d\n", symbol_name, ret); - return NULL; - } - pr_debug("symbol_name:%s addr=%px\n", symbol_name, kp.addr); - unregister_kprobe(&kp); - - return kp.addr; -} #endif int __nocfi get_mte_sync_tags_hook_kprobe(void *data) @@ -4040,6 +4256,7 @@ int __nocfi get_mte_sync_tags_hook_kprobe(void *data) struct cma *cma = NULL; struct page *page = NULL; #if defined(CONFIG_ARM64) + struct task_struct *task = NULL; int ret; ret = register_kprobe(&kp_lookup_name); @@ -4052,12 +4269,18 @@ int __nocfi get_mte_sync_tags_hook_kprobe(void *data) aml_syms_lookup = (unsigned long (*)(const char *name))kp_lookup_name.addr; aml_init_mm = (struct mm_struct *)aml_syms_lookup("init_mm"); + if (!aml_init_mm) { + rcu_read_lock(); + for_each_process(task) { + if (task->pid == 1) { + aml_init_mm = task->active_mm; + break; + } + } + rcu_read_unlock(); + } aml_mte_sync_tags = (void (*)(pte_t old_pte, pte_t pte))aml_syms_lookup("mte_sync_tags"); - pr_debug("aml_init_mm: %px, aml_mte_sync_tags: %px\n", aml_init_mm, aml_mte_sync_tags); - aml__pte_offset_map = (pte_t * (*)(pmd_t *pmd, unsigned long addr, - pmd_t *pmdvalp))get_symbol_addr("__pte_offset_map"); - aml_get_pfnblock_flags_mask = (unsigned long (*)(const struct page *page, - unsigned long pfn, unsigned long mask))get_symbol_addr("get_pfnblock_flags_mask"); + pr_info("aml_init_mm: %px, aml_mte_sync_tags: %px\n", aml_init_mm, aml_mte_sync_tags); #endif cma = dev_get_cma_area(codec_dev); @@ -4097,6 +4320,57 @@ bool is_2k_platform(void) return false; } +static inline void codec_mm_parse_reserved_mem(struct platform_device *pdev, char *name, + int (*vdec_res_setup)(struct reserved_mem *rmem)) +{ + int r; + struct reserved_mem *mem = NULL; + int region_index = 0; + struct device_node *search_target = NULL; + + while (1) { + search_target = of_parse_phandle(pdev->dev.of_node, + "memory-region", + region_index); + if (!search_target) + break; + + if (!strcmp(search_target->name, name)) { + mem = of_reserved_mem_lookup(search_target); + if (mem) { + r = vdec_res_setup(mem); + if (r) + pr_err("%s vdec_res_setup res %x\n", name, r); + r = of_reserved_mem_device_init_by_idx(&pdev->dev, + pdev->dev.of_node, region_index); + if (r) + pr_err("%s vdec_res_setup device init failed\n", name); + } + break; + } + region_index++; + } +} + +u32 codec_mm_get_property_from_dts(char *property_name) +{ + struct device_node *pnode = NULL; + u32 ret = -1; + + pnode = of_find_node_by_name(NULL, "codec_mm"); + if (!pnode) { + pr_info("not find codec_mm node\n"); + return -EINVAL; + } + + if (of_property_read_u32(pnode, property_name, &ret)) { + pr_info("read format in dts failed, ret = %d\n", ret); + return -EINVAL; + } + + return ret; +} + u64 codec_mm_managed_max_addr(void) { u64 addr = 0; @@ -4126,46 +4400,42 @@ u64 codec_mm_secure_vdec_max_addr(void) return sec_vdec_addr + sec_vdec_size; } +static void prepare_full_pagemap(struct device_node *of_node) +{ +#if IS_MODULE(CONFIG_AMLOGIC_MEDIA_MODULE) && \ + IS_ENABLED(CONFIG_KALLSYMS_ALL) && \ + !IS_ENABLED(CONFIG_DEBUG_SPINLOCK) + u32 random = 0; + int ret; + + if (!of_node) + return; + + ret = of_property_read_u32(of_node, "random_access", &random); + if (!ret && random) { + if (kthread_run(get_mte_sync_tags_hook_kprobe, NULL, + "AML_CODEC_MM") == ERR_PTR(-ENOMEM)) { + pr_err("Failed to start kernel thread AML_CODEC_MM\n"); + } + } + +#endif +} + static int codec_mm_probe(struct platform_device *pdev) { int r; - struct reserved_mem *mem = NULL; - int secure_region_index = 0; - struct device_node *search_target = NULL; struct codec_mm_mgt_s *mgt = NULL; - while (1) { - search_target = of_parse_phandle(pdev->dev.of_node, - "memory-region", - secure_region_index); - if (!search_target) - break; - - if (!strcmp(search_target->name, "linux,secure_vdec_reserved")) { - mem = of_reserved_mem_lookup(search_target); - if (mem) { - r = secure_vdec_res_setup(mem); - if (r) - pr_err("secure_vdec_res_setup res %x\n", r); - r = of_reserved_mem_device_init_by_idx(&pdev->dev, - pdev->dev.of_node, secure_region_index); - if (r) - pr_err("secure_vdec_res_setup device init failed\n"); - } - break; - } - secure_region_index++; - } + codec_mm_parse_reserved_mem(pdev, "linux,secure_vdec_reserved", secure_vdec_res_setup); + codec_mm_parse_reserved_mem(pdev, "linux,codec_mm_reserved", codec_mm_res_setup); +#ifdef CONFIG_CODEC_MM_EXT_POOL + codec_mm_parse_reserved_mem(pdev, "linux,codec_mm_reserved_ext", codec_mm_res_ext_setup); +#endif mgt = get_mem_mgt(); - pdev->dev.platform_data = mgt; - r = of_reserved_mem_device_init(&pdev->dev); - if (r == 0) - pr_debug("%s mem init done\n", __func__); - codec_mm_mgt_init(&pdev->dev); - codec_mm_dev_set_dma_mask(DMA_BIT_MASK(64)); r = class_register(&codec_mm_class); if (r) { pr_err("vdec class create fail.\n"); @@ -4173,9 +4443,11 @@ static int codec_mm_probe(struct platform_device *pdev) } r = of_reserved_mem_device_init(&pdev->dev); if (r == 0) - pr_debug("codec_mm reserved memory probed done\n"); + pr_debug("codec_mm cma memory probed done\n"); pr_info("%s ok\n", __func__); + codec_mm_mgt_init(&pdev->dev); + codec_mm_dev_set_dma_mask(DMA_BIT_MASK(64)); #if IS_MODULE(CONFIG_AMLOGIC_MEDIA_MODULE) && \ IS_ENABLED(CONFIG_KALLSYMS_ALL) && \ @@ -4191,11 +4463,10 @@ static int codec_mm_probe(struct platform_device *pdev) "trigger", codec_mm_trigger, CONFIG_FOR_RW | CONFIG_FOR_T); codec_state_register(&mgt->cs, &codec_mm_cs_ops); -#if IS_MODULE(CONFIG_AMLOGIC_MEDIA_MODULE) && \ - IS_ENABLED(CONFIG_KALLSYMS_ALL) && \ - !IS_ENABLED(CONFIG_DEBUG_SPINLOCK) - kthread_run(get_mte_sync_tags_hook_kprobe, NULL, "AML_CODEC_MM"); -#endif + + init_alloc_page_boost_task(); + prepare_full_pagemap(pdev->dev.of_node); + return 0; } @@ -4348,6 +4619,34 @@ static int __init secure_vdec_res_setup(struct reserved_mem *rmem) RESERVEDMEM_OF_DECLARE(secure_vdec_reserved, "amlogic, secure-vdec-reserved", secure_vdec_res_setup); +#ifdef CONFIG_CODEC_MM_EXT_POOL +static int codec_mm_reserved_ext_init(struct reserved_mem *rmem, struct device *dev) +{ + struct codec_mm_mgt_s *mgt = get_mem_mgt(); + + mgt->rmem_ext = *rmem; + pr_debug("%s %p->%p\n", __func__, + (void *)mgt->rmem_ext.base, + (void *)mgt->rmem_ext.base + mgt->rmem_ext.size); + return 0; +} + +static const struct reserved_mem_ops codec_mm_rmem_ext_vdec_ops = { + .device_init = codec_mm_reserved_ext_init, +}; + +static int codec_mm_res_ext_setup(struct reserved_mem *rmem) +{ + rmem->ops = &codec_mm_rmem_ext_vdec_ops; + pr_debug("vdec: reserved mem setup\n"); + + return 0; +} + +RESERVEDMEM_OF_DECLARE(codec_mm_reserved_ext, "amlogic, codec-mm-reserved_ext", + codec_mm_res_ext_setup); +#endif + module_param(debug_mode, uint, 0664); MODULE_PARM_DESC(debug_mode, "\n debug module\n"); __module_param(debug_sc_mode, uint, 0664); diff --git a/drivers/media/common/codec_mm/codec_mm_priv.h b/drivers/media/common/codec_mm/codec_mm_priv.h index e533965fe..e6cd66b49 100644 --- a/drivers/media/common/codec_mm/codec_mm_priv.h +++ b/drivers/media/common/codec_mm/codec_mm_priv.h @@ -39,7 +39,6 @@ do { \ void dma_clear_buffer(struct page *page, size_t size); -u32 codec_mm_get_sc_debug_mode(void); u32 codec_mm_get_keep_debug_mode(void); bool codec_mm_scatter_available_check(int size); void codec_mm_scatter_level_decrease(int size); @@ -48,4 +47,5 @@ void codec_mm_set_min_linear_size(int min_mem_val); int codec_mm_get_min_linear_size(void); int codec_mm_get_scatter_watermark(void); bool is_2k_platform(void); +u32 codec_mm_get_property_from_dts(char *property_name); #endif /**/ diff --git a/drivers/media/common/codec_mm/codec_mm_scatter.c b/drivers/media/common/codec_mm/codec_mm_scatter.c index 024864168..220a425e6 100644 --- a/drivers/media/common/codec_mm/codec_mm_scatter.c +++ b/drivers/media/common/codec_mm/codec_mm_scatter.c @@ -31,10 +31,12 @@ #include #include #include +#include #include #include "codec_mm_priv.h" #include "codec_mm_scatter_priv.h" +#include "codec_mm_sys_priv.h" #define KERNEL_ATRACE_TAG KERNEL_ATRACE_TAG_CODEC_MM #include @@ -48,6 +50,14 @@ }) #endif +#ifndef MAX +#define MAX(a, b) ({ \ + const typeof(a) _a = a; \ + const typeof(b) _b = b; \ + _a > _b ? _a : _b; \ + }) +#endif + #define PAGE_COUNT(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT) u32 scatter_align_pages_size = 128; @@ -80,6 +90,11 @@ module_param(scatter_debug_mode, int, 0644); #define MAX_SID (SID_MASK - 1) #define MAX_HASH_SID (MAX_SID - 1) +#define SCATTER_ALLOC_FROM_DEFAULT 0 +#define SCATTER_ALLOC_FROM_SYS 1 +#define SCATTER_ALLOC_FROM_CMA 2 +#define SCATTER_ALLOC_FROM_MIXED 3 + #define ADDR_SEED(paddr) (((paddr) >> (MAX_ADDR_SHIFT << 1)) +\ ((paddr) >> MAX_ADDR_SHIFT) +\ (paddr)) @@ -104,7 +119,7 @@ module_param(scatter_debug_mode, int, 0644); #define INVALID_ID(mms, id) (!(mms) ||\ !(mms)->pages_list ||\ - (mms)->page_cnt <= (id) ||\ + (mms)->queue ? 0 : (mms)->page_cnt <= (id) ||\ (id) < 0) #define SID_OF_ONEPAGE(sid) ((sid) == ONE_PAGE_SID) @@ -239,6 +254,25 @@ struct codec_mm_scatter_mgt { struct codec_mm_scatter *try_free_cache_sc; }; +static int codec_mm_page_alloc_all_locked + (struct codec_mm_scatter_mgt *smgt, + phy_addr_type *pages, + int num, + int iscache); + +static inline int codec_mm_scatter_map_del_locked + (struct codec_mm_scatter_mgt *smgt, + struct codec_mm_scatter *mms); + +static int codec_mm_scatter_free_page_id_locked + (struct codec_mm_scatter_mgt *smgt, + struct codec_mm_scatter *mms, int id); + +struct codec_mm_scatter *codec_mm_scatter_alloc_new + (struct codec_mm_scatter_mgt *smgt, + int max_page, + int page_num); + static inline int scatter_get_unit_pages(void) { return scatter_align_pages_size; @@ -341,11 +375,6 @@ static inline void codec_mm_scatter_lock(struct codec_mm_scatter *mms) mutex_lock(&mms->mutex); } -static inline int codec_mm_scatter_trylock(struct codec_mm_scatter *mms) -{ - return mutex_trylock(&mms->mutex); -} - static inline void codec_mm_list_lock(struct codec_mm_scatter_mgt *smgt) { spin_lock(&smgt->list_lock); @@ -361,6 +390,259 @@ static inline void codec_mm_list_unlock(struct codec_mm_scatter_mgt *smgt) spin_unlock(&smgt->list_lock); } +static inline int codec_mm_queue_head_trylock(struct codec_mm_cache_queue *queue) +{ + return mutex_trylock(&queue->head_lock); +} + +static inline void codec_mm_queue_head_lock(struct codec_mm_cache_queue *queue) +{ + mutex_lock(&queue->head_lock); +} + +static inline void codec_mm_queue_head_unlock(struct codec_mm_cache_queue *queue) +{ + mutex_unlock(&queue->head_lock); +} + +static inline int codec_mm_queue_tail_trylock(struct codec_mm_cache_queue *queue) +{ + return spin_trylock(&queue->tail_lock); +} + +static inline void codec_mm_queue_tail_lock(struct codec_mm_cache_queue *queue) +{ + spin_lock(&queue->tail_lock); +} + +static inline void codec_mm_queue_tail_unlock(struct codec_mm_cache_queue *queue) +{ + spin_unlock(&queue->tail_lock); +} + +static int codec_mm_cache_queue_init(struct codec_mm_scatter *cache) +{ + struct codec_mm_cache_queue *queue; + + if (!cache->queue) { + queue = kmalloc(sizeof(*queue), GFP_KERNEL); + if (!queue) + return -ENOMEM; + } else { + queue = cache->queue; + } + + queue->head = 0; + queue->tail = 0; + mutex_init(&queue->head_lock); + spin_lock_init(&queue->tail_lock); + + cache->queue = queue; + + return 0; +} + +static void codec_mm_cache_queue_release(struct codec_mm_scatter *cache) +{ + kfree(cache->queue); + cache->queue = NULL; +} + +static inline u32 codec_mm_queue_len(struct codec_mm_cache_queue *queue) +{ + return (queue->head - queue->tail + CACHE_QUEUE_LEN) % CACHE_QUEUE_LEN; +} + +static inline void codec_mm_cache_queue_head_add + (struct codec_mm_cache_queue *queue, + u32 num) +{ + WARN_ON(codec_mm_queue_len(queue) + num > CACHE_QUEUE_LEN); + queue->head = (queue->head + num) % CACHE_QUEUE_LEN; +} + +static inline void codec_mm_cache_queue_tail_add + (struct codec_mm_cache_queue *queue, + u32 num) +{ + WARN_ON(codec_mm_queue_len(queue) < num); + queue->tail = (queue->tail + num) % CACHE_QUEUE_LEN; +} + +static int codec_mm_cache_queue_produce + (struct codec_mm_scatter_mgt *smgt, + struct codec_mm_scatter *cache, + struct codec_mm_cache_queue *queue, + u32 supply_num, + phy_addr_type *producer) +{ + int num = MIN(CACHE_QUEUE_LEN - codec_mm_queue_len(cache->queue) - + scatter_get_unit_pages(), supply_num); + int ret; + int alloced = 0; + + if (producer) { /* get page from producer */ + if (queue->head + num <= CACHE_QUEUE_LEN) { + memcpy(&cache->pages_list[cache->queue->head], producer, + num * sizeof(phy_addr_type)); + } else { + memcpy(&cache->pages_list[cache->queue->head], producer, + (CACHE_QUEUE_LEN - queue->head) * sizeof(phy_addr_type)); + memcpy(&cache->pages_list[0], producer + CACHE_QUEUE_LEN - queue->head, + (queue->head + num - CACHE_QUEUE_LEN) * sizeof(phy_addr_type)); + } + memset(producer, 0, num * sizeof(phy_addr_type)); + + codec_mm_cache_queue_head_add(cache->queue, num); + ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_SMGT); + codec_mm_list_lock(smgt); + ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_SMGT_LOCK); + smgt->cached_pages += num; + ATRACE_COUNTER("mmu cache size", smgt->cached_pages); + codec_mm_list_unlock(smgt); + ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_SMGT_LOCK_DONE); + + return num; + } + + /* no producer, get page from slot/system */ + while (alloced < num) { + int try_num = MIN(CACHE_QUEUE_LEN - cache->queue->head, num - alloced); + + ret = codec_mm_page_alloc_all_locked(smgt, + &cache->pages_list[cache->queue->head], + try_num, true); + if (ret <= 0) + break; + codec_mm_cache_queue_head_add(cache->queue, ret); + alloced += ret; + } + + codec_mm_list_lock(smgt); + smgt->cached_pages += alloced; + ATRACE_COUNTER("mmu cache size", smgt->cached_pages); + codec_mm_list_unlock(smgt); + + return alloced; +} + +static int codec_mm_cache_queue_consume + (struct codec_mm_scatter *cache, + struct codec_mm_cache_queue *queue, + u32 need_num, + phy_addr_type *consumer) +{ + int num = MIN(codec_mm_queue_len(queue), need_num); + + if (queue->tail + num <= CACHE_QUEUE_LEN) { + memmove(consumer, &cache->pages_list[cache->queue->tail], + num * sizeof(phy_addr_type)); + memset(&cache->pages_list[cache->queue->tail], 0, + num * sizeof(phy_addr_type)); + } else { + memmove(consumer, &cache->pages_list[cache->queue->tail], + (CACHE_QUEUE_LEN - queue->tail) * sizeof(phy_addr_type)); + memset(&cache->pages_list[cache->queue->tail], 0, + (CACHE_QUEUE_LEN - queue->tail) * sizeof(phy_addr_type)); + + memmove(consumer + CACHE_QUEUE_LEN - queue->tail, &cache->pages_list[0], + (queue->tail + num - CACHE_QUEUE_LEN) * sizeof(phy_addr_type)); + memset(&cache->pages_list[0], 0, + (queue->tail + num - CACHE_QUEUE_LEN) * sizeof(phy_addr_type)); + } + + codec_mm_cache_queue_tail_add(cache->queue, num); + return num; +} + +static int codec_mm_scatter_cache_creat(struct codec_mm_scatter_mgt *smgt, + int cache_id) +{ + struct codec_mm_scatter *cache; + + pr_info("[%s]creat cache[%d]\n", __func__, cache_id); + cache = codec_mm_scatter_alloc_new(smgt, CACHE_QUEUE_LEN, 0); + if (cache) { + codec_mm_cache_queue_init(cache); + codec_mm_list_lock(smgt); + list_del_init(&cache->list); + if (smgt->cache_scs[cache_id]) { + codec_mm_scatter_map_del_locked(smgt, cache); + SC_FREE(cache); + codec_mm_list_unlock(smgt); + return 0; + } + smgt->cache_scs[cache_id] = cache; + codec_mm_list_unlock(smgt); + return 0; + } + + pr_err("[%s]creat cache[%d] failed\n", __func__, cache_id); + return -2; +} + +static int codec_mm_scatter_cache_less_page(struct codec_mm_scatter_mgt *smgt, + struct codec_mm_scatter *cache, + int free_page_num) +{ + int idx, ret, need_free, real_free; + + codec_mm_queue_tail_lock(cache->queue); + real_free = MIN(free_page_num, codec_mm_queue_len(cache->queue)); + need_free = real_free; + idx = cache->queue->tail; + for (; need_free > 0; need_free--) { + ret = codec_mm_scatter_free_page_id_locked(smgt, cache, idx++ % CACHE_QUEUE_LEN); + if (ret < 0) + pr_err("[%s]free cache[%5d]:%ld failed, ret:%d\n", + __func__, idx, cache->pages_list[idx], ret); + } + codec_mm_cache_queue_tail_add(cache->queue, real_free); + codec_mm_queue_tail_unlock(cache->queue); + + codec_mm_list_lock(smgt); + smgt->alloced_page_num -= real_free; + smgt->cached_pages -= real_free; + ATRACE_COUNTER("mmu cache size", smgt->cached_pages); + codec_mm_list_unlock(smgt); + + return 0; +} + +static void codec_mm_scatter_cache_destroy(struct codec_mm_scatter_mgt *smgt, + int cache_id) +{ + int free_len; + struct codec_mm_scatter *cache; + + /* remove form smgt */ + codec_mm_list_lock(smgt); + cache = smgt->cache_scs[cache_id]; + if (!cache) { + codec_mm_list_unlock(smgt); + return; + } + smgt->cache_scs[cache_id] = NULL; + if (!list_empty(&cache->list)) + list_del(&cache->list); + smgt->scatters_cnt--; + codec_mm_scatter_map_del_locked(smgt, cache); + codec_mm_list_unlock(smgt); + + /* free page */ + codec_mm_queue_head_lock(cache->queue); + codec_mm_queue_tail_lock(cache->queue); + free_len = codec_mm_queue_len(cache->queue); + codec_mm_queue_tail_unlock(cache->queue); + codec_mm_queue_head_unlock(cache->queue); + if (free_len) + codec_mm_scatter_cache_less_page(smgt, cache, free_len); + + /* free cache */ + codec_mm_cache_queue_release(cache); + SC_FREE(cache); +} + static int codec_mm_scatter_alloc_want_pages_in(struct codec_mm_scatter_mgt *smgt, struct codec_mm_scatter *mms, @@ -504,13 +786,14 @@ static int codec_mm_set_slot_in_hash(struct codec_mm_scatter_mgt *smgt, { page_sid_type sid = SLOT_TO_SID(slot); - if (sid > MAX_SID) { + if (sid >= MAX_SID) { ERR_LOG("ERROR sid %d", sid); return -1; } slot->sid = sid; INIT_LIST_HEAD(&slot->sid_list); INIT_LIST_HEAD(&slot->free_list); + slot->on_alloc_free = -1; /* The init state be held until switch to alloc state. */ codec_mm_list_lock(smgt); if (!smgt->slot_list_map[sid]) { smgt->slot_list_map[sid] = slot; @@ -548,7 +831,7 @@ codec_mm_find_slot_in_hash(struct codec_mm_scatter_mgt *smgt, codec_mm_list_lock(smgt); fslot = smgt->slot_list_map[sid]; if (!fslot) { - ERR_LOG("not valid sid %d\n", + ERR_LOG("not valid sid %x\n", (int)sid); goto err; } @@ -643,19 +926,6 @@ static int codec_mm_slot_free(struct codec_mm_scatter_mgt *smgt, return 0; } -static int codec_mm_slot_try_free(struct codec_mm_scatter_mgt *smgt, - struct codec_mm_slot *slot) -{ - if (smgt->keep_size_PAGE > 0) { - /*delay free, when size < delay_free_M MB */ - int free_pages = smgt->total_page_num - smgt->alloced_page_num; - - if (free_pages < smgt->keep_size_PAGE) - return -1; - } - return codec_mm_slot_free(smgt, slot); -} - static inline int codec_mm_slot_init_bitmap(struct codec_mm_slot *slot) { slot->alloced_page_num = 0; @@ -729,9 +999,9 @@ codec_mm_slot_alloc(struct codec_mm_scatter_mgt *smgt, int size, int flags) try_alloc_size = cma_free_size; } if (try_alloc_size <= 0 || - /* not enough cma for scatter */ - (!smgt->tvp_mode && !codec_mm_scatter_available_check(try_alloc_size))) { - INFO_LOG("No cma can be used for scatter"); + /* not enough cma for scatter */ + (!smgt->tvp_mode && !codec_mm_scatter_available_check(try_alloc_size))) { + pr_inf("No cma can be used for scatter"); break; } if (alloc_unit_pages) { @@ -871,7 +1141,7 @@ static int codec_mm_slot_free_page(struct codec_mm_scatter_mgt *smgt, codec_mm_list_lock(smgt); slot->on_alloc_free++; if (!test_and_clear_bit(bit, slot->pagemap)) { - ERR_LOG("ERROR,page is ready free before!!! page=%p\n", + ERR_LOG("ERROR,page is ready free before!!! page=%px\n", (void *)phy_addr); slot->on_alloc_free--; codec_mm_list_unlock(smgt); @@ -963,7 +1233,6 @@ static int codec_mm_page_free_to_slot(struct codec_mm_scatter_mgt *smgt, { struct codec_mm_slot *slot; int ret; - int slot_free = 0; slot = codec_mm_find_slot_in_hash(smgt, sid, addr); if (!slot) @@ -973,40 +1242,77 @@ static int codec_mm_page_free_to_slot(struct codec_mm_scatter_mgt *smgt, ERR_LOG("free slot addr error =%p ret=%d\n", (void *)addr, ret); - if (slot->alloced_page_num == 0) - slot_free = (codec_mm_slot_try_free(smgt, slot) == 0); - - if (!slot_free) { /*move to have free list. */ - codec_mm_list_lock(smgt); - if (list_empty(&slot->free_list) && - slot->alloced_page_num < slot->page_num) { - DBG_LOG("add to free %p, %d,%d,%d\n", - slot, slot->page_num, - slot->alloced_page_num, ret); - list_add_tail(&slot->free_list, &smgt->free_list); - } - codec_mm_list_unlock(smgt); + /*move to have free list. */ + codec_mm_list_lock(smgt); + if (list_empty(&slot->free_list) && + slot->alloced_page_num < slot->page_num) { + DBG_LOG("add to free %px, %d,%d,%d\n", + slot, slot->page_num, + slot->alloced_page_num, ret); + list_add_tail(&slot->free_list, &smgt->free_list); } + codec_mm_list_unlock(smgt); return 0; } -static int codec_mm_page_alloc_from_one_pages(struct codec_mm_scatter_mgt *smgt, - phy_addr_type *pages, int num) +#define MAX_PAGE_ALLOC_ONCE 4096 +struct page *page_array[MAX_PAGE_ALLOC_ONCE]; +static DEFINE_MUTEX(alloc_page_array_mutex); + +static int multi_task_alloc_pages(struct codec_mm_scatter_mgt *smgt, + unsigned long nr_pages, phy_addr_type *pages, gfp_t gfp) { - int neednum = num; + int neednum = nr_pages; int alloced = 0; + int idx; while (neednum > 0) { /*one page alloc */ + ulong paddr; + page_sid_type sid; + int page_num = MIN(neednum, MAX_PAGE_ALLOC_ONCE); + ulong ret = -1; + + mutex_lock(&alloc_page_array_mutex); + ret = aml_alloc_pages_array(gfp, page_num, page_array); + if (!ret) { + for (idx = 0; idx < page_num; idx++) { + paddr = page_to_phys(page_array[idx]); + sid = ONE_PAGE_SID; + paddr |= sid; + pages[alloced++] = paddr; + neednum--; + + /* Invalidate range of cache lines. */ + dma_sync_single_for_device(smgt->dev, + page_to_phys(page_array[idx]), + PAGE_SIZE, + DMA_FROM_DEVICE); + } + } else { + /*can't alloced memory from ONEPAGE alloc */ + WAR_LOG("Out of memory OnePage alloc =%d,%ld\n", + alloced, nr_pages); + break; + } + mutex_unlock(&alloc_page_array_mutex); + } + + return alloced; +} + +static int single_task_alloc_pages(struct codec_mm_scatter_mgt *smgt, + unsigned long nr_pages, phy_addr_type *pages, gfp_t gfp) +{ + int neednum = nr_pages; + int alloced = 0; + + while (neednum > 0) { /*one page alloc */ void *vpage; ulong page; page_sid_type sid; - if (smgt->config_alloc_flags & SC_ALLOC_SYS_DMA32) { - vpage = (void *)__get_free_page(GFP_KERNEL | GFP_DMA32); - } else { - vpage = (void *)__get_free_page(GFP_KERNEL); - } + vpage = (void *)__get_free_page(gfp); if (vpage) { page = virt_to_phys(vpage); sid = ONE_PAGE_SID; @@ -1021,21 +1327,40 @@ static int codec_mm_page_alloc_from_one_pages(struct codec_mm_scatter_mgt *smgt, DMA_FROM_DEVICE); } else { /*can't alloced memory from ONEPAGE alloc */ - WAR_LOG("Out of memory OnePage alloc =%d,%d\n", - alloced, num); + WAR_LOG("Out of memory OnePage alloc =%d,%ld\n", + alloced, nr_pages); break; } } + + return alloced; +} + +int scatter_multi_alloc = 1; + +static int codec_mm_page_alloc_from_one_pages(struct codec_mm_scatter_mgt *smgt, + phy_addr_type *pages, int num) +{ + int alloced = 0; + gfp_t gfp = (smgt->config_alloc_flags & SC_ALLOC_SYS_DMA32) ? + (GFP_KERNEL | GFP_DMA32) : GFP_KERNEL; + + if (scatter_multi_alloc) + alloced = multi_task_alloc_pages(smgt, num, pages, gfp); + else + alloced = single_task_alloc_pages(smgt, num, pages, gfp); + codec_mm_list_lock(smgt); smgt->total_page_num += alloced; smgt->one_page_cnt += alloced; smgt->alloced_page_num += alloced; codec_mm_list_unlock(smgt); + return alloced; } static int codec_mm_page_alloc_from_slot(struct codec_mm_scatter_mgt *smgt, - phy_addr_type *pages, int num) + phy_addr_type *pages, int num, int is_cache) { #define INSTR1 "alloc default cma size fail,try %d pages\n" struct codec_mm_slot *slot = NULL; @@ -1058,8 +1383,18 @@ static int codec_mm_page_alloc_from_slot(struct codec_mm_scatter_mgt *smgt, if (smgt->total_page_num <= 0 || /*no codec mm. */ smgt->alloced_page_num == smgt->total_page_num || list_empty(&smgt->free_list)) { + u32 alloc_pages = 0; + int alloc_unit_pages = scatter_get_unit_pages(); + + if (is_cache && !smgt->force_cache_on) { + // min slot size as 4M + alloc_unit_pages = MAX(alloc_unit_pages, 4 * SZ_1M >> PAGE_SHIFT); + alloc_pages = ALIGN(neednum, alloc_unit_pages); + pr_dbg("alloc pages is %d cache_pages is %d keep_size_page is %d\n", + alloc_pages, smgt->cached_pages, smgt->keep_size_PAGE); + } /*codec_mm_scatter_info_dump(NULL, 0);*/ - slot = codec_mm_slot_alloc(smgt, 0, 0); + slot = codec_mm_slot_alloc(smgt, alloc_pages * PAGE_SIZE, 0); if (!slot) { u32 alloc_pages = smgt->try_alloc_in_cma_page_cnt / 4; @@ -1076,7 +1411,9 @@ static int codec_mm_page_alloc_from_slot(struct codec_mm_scatter_mgt *smgt, } } codec_mm_list_lock(smgt); - if (slot && slot->on_alloc_free != 0) { + if (slot && slot->on_alloc_free == -1) { + slot->on_alloc_free = 0; + } else if (slot && slot->on_alloc_free != 0) { ERR_LOG("the slot on alloc/free1: %d\n", slot->on_alloc_free); slot = NULL; /*slot used on another alloc. */ @@ -1171,8 +1508,8 @@ codec_mm_get_next_cache_scatter(struct codec_mm_scatter_mgt *smgt, int more_pages = flags & 1; int id = -1; - if (smgt->cache_scs[0]->page_cnt >= - smgt->cache_scs[1]->page_cnt) + if (codec_mm_queue_len(smgt->cache_scs[0]->queue) >= + codec_mm_queue_len(smgt->cache_scs[1]->queue)) id = more_pages ? 0 : 1; else id = more_pages ? 1 : 0; @@ -1199,31 +1536,27 @@ int codec_mm_page_alloc_from_free_scatter(struct codec_mm_scatter_mgt *smgt, int alloced = 0; mms = src_mms; - if (!mms) + if (!mms || !mms->queue) return 0; ATRACE_COUNTER("mmu alloc", MMU_ALLOC_from_cache_scatter_1); if (!wait) { - if (!codec_mm_scatter_trylock(mms)) + if (!codec_mm_queue_tail_trylock(mms->queue)) return 0;/*mms is locked try another.*/ } else { - codec_mm_scatter_lock(mms); + codec_mm_queue_tail_lock(mms->queue); } ATRACE_COUNTER("mmu alloc", MMU_ALLOC_from_cache_scatter_2); - alloced = min(mms->page_cnt, need); - if (alloced > 0) { - mms->page_cnt -= alloced; - mms->page_tail -= alloced; - memcpy(pages, &mms->pages_list[mms->page_tail + 1], - alloced * sizeof(phy_addr_type)); - memset(&mms->pages_list[mms->page_tail + 1], 0, - alloced * sizeof(phy_addr_type)); - } + alloced = MIN(codec_mm_queue_len(mms->queue), need); + if (alloced > 0) + alloced = codec_mm_cache_queue_consume(mms, mms->queue, alloced, pages); + codec_mm_queue_tail_unlock(mms->queue); + ATRACE_COUNTER("mmu alloc", MMU_ALLOC_from_cache_scatter_3); codec_mm_list_lock(smgt); ATRACE_COUNTER("mmu alloc", MMU_ALLOC_from_cache_scatter_4); smgt->cached_pages -= alloced; + ATRACE_COUNTER("mmu cache size", smgt->cached_pages); codec_mm_list_unlock(smgt); - codec_mm_scatter_unlock(mms); ATRACE_COUNTER("mmu alloc", MMU_ALLOC_from_cache_scatter_5); return alloced; } @@ -1289,7 +1622,7 @@ static int codec_mm_page_alloc_all_locked(struct codec_mm_scatter_mgt *smgt, MMU_ALLOC_from_slot); new_alloc = codec_mm_page_alloc_from_slot(smgt, pages + - alloced, num - alloced); + alloced, num - alloced, iscache); ATRACE_COUNTER("mmu alloc", MMU_ALLOC_from_slot_end); if (new_alloc <= 0) can_from_slot = 0; @@ -1318,37 +1651,29 @@ static int codec_mm_pages_free_to_scatter(struct codec_mm_scatter_mgt *smgt, if (src_mms->page_used >= src_mms->page_cnt) return -1; /*no need free. */ - if (!dst_mms || src_mms == dst_mms) + if (!dst_mms || src_mms == dst_mms || !dst_mms->queue) return 0; ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_DTS); if (!wait) { - if (!codec_mm_scatter_trylock(dst_mms)) + if (!codec_mm_queue_head_trylock(dst_mms->queue)) return -2;/*mms is locked try another.*/ } else { - codec_mm_scatter_lock(dst_mms); + codec_mm_queue_head_lock(dst_mms->queue); } ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_DTS_LOCK); - moved = min(src_mms->page_cnt - src_mms->page_used, - dst_mms->page_max_cnt - dst_mms->page_cnt); + moved = MIN(src_mms->page_cnt - src_mms->page_used, + CACHE_QUEUE_LEN - codec_mm_queue_len(dst_mms->queue) - + scatter_get_unit_pages()); left = src_mms->page_cnt - moved; - if (moved > 0) { - memcpy(&dst_mms->pages_list[dst_mms->page_tail + 1], - &src_mms->pages_list[left], - moved * sizeof(phy_addr_type)); - memset(&src_mms->pages_list[left], 0, - sizeof(phy_addr_type) * moved); - } - dst_mms->page_cnt += moved; - dst_mms->page_tail += moved; + if (moved > 0) + moved = codec_mm_cache_queue_produce(smgt, + dst_mms, dst_mms->queue, moved, + &src_mms->pages_list[left]); + src_mms->page_cnt -= moved; src_mms->page_tail -= moved; - ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_SMGT); - codec_mm_list_lock(smgt); - ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_SMGT_LOCK); - smgt->cached_pages += moved; - codec_mm_list_unlock(smgt); - ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_SMGT_LOCK_DONE); - codec_mm_scatter_unlock(dst_mms); + + codec_mm_queue_head_unlock(dst_mms->queue); ATRACE_COUNTER("mmu_free", MMU_FREE_SCATTER_START_DTS_LOCK_DONE); return moved; } @@ -1424,9 +1749,9 @@ static int codec_mm_scatter_free_pages_in_locked(struct codec_mm_scatter *mms, ret = codec_mm_scatter_free_page_id_locked(smgt, mms, i); if (ret < 0) { if (not_continue_print) { - ERR_LOG("page free error.%d,id=%d, addr:%d\n", + ERR_LOG("page free error.%d,id=%d, addr:%lx\n", ret, i, - (int)mms->pages_list[i]); + mms->pages_list[i]); codec_mm_dump_scatter(mms, NULL, 0); } not_continue_print = 0; @@ -1438,8 +1763,6 @@ static int codec_mm_scatter_free_pages_in_locked(struct codec_mm_scatter *mms, } codec_mm_list_lock(smgt); smgt->alloced_page_num -= free_num; - if (is_cache_sc(smgt, mms)) - smgt->cached_pages -= free_num; codec_mm_list_unlock(smgt); return 0; } @@ -1589,10 +1912,13 @@ int free_scatter_pages_for_expected_slot(struct codec_mm_scatter *mms) int tail_index; int moved = 0; struct codec_mm_scatter *try_free_cache_mms = smgt->try_free_cache_sc; + int last, len; if (!try_free_cache_mms) return 0; + if (mms->queue) + goto queue_mode; codec_mm_scatter_lock(mms); if (mms->page_cnt == 0) { codec_mm_scatter_unlock(mms); @@ -1627,13 +1953,51 @@ int free_scatter_pages_for_expected_slot(struct codec_mm_scatter *mms) mms->page_tail -= moved; mms->page_used -= moved; - codec_mm_list_lock(smgt); - if (is_cache_sc(smgt, mms)) - smgt->cached_pages -= moved; - codec_mm_list_unlock(smgt); - codec_mm_scatter_unlock(mms); return moved; + +queue_mode: + codec_mm_scatter_lock(try_free_cache_mms); + codec_mm_queue_tail_lock(mms->queue); + if (codec_mm_queue_len(mms->queue) == 0) { + codec_mm_queue_tail_unlock(mms->queue); + codec_mm_scatter_unlock(try_free_cache_mms); + return 0; + } + index = mms->queue->tail; + last = mms->queue->head; + len = codec_mm_queue_len(mms->queue); + for (; len > 0; len -= pages_count) { + if (INVALID_ID(mms, index)) { + pr_err("mms %p free pages index %d invalid\n", mms, index); + break; + } + + sid = PAGE_SID_OF_MMS(mms, index); + if (compare_expected_slot_sid(smgt, sid)) { + memcpy(&try_free_cache_mms->pages_list[try_free_cache_mms->page_cnt], + &mms->pages_list[index], sizeof(phy_addr_type) * pages_count); + try_free_cache_mms->page_cnt += pages_count; + try_free_cache_mms->page_tail += pages_count; + + moved += pages_count; + codec_mm_cache_queue_consume(mms, mms->queue, + pages_count, &mms->pages_list[index]); + } + index += pages_count; + index %= CACHE_QUEUE_LEN; + } + + codec_mm_list_lock(smgt); + if (is_cache_sc(smgt, mms)) { + smgt->cached_pages -= moved; + ATRACE_COUNTER("mmu cache size", smgt->cached_pages); + } + codec_mm_list_unlock(smgt); + codec_mm_queue_tail_unlock(mms->queue); + codec_mm_scatter_unlock(try_free_cache_mms); + + return moved; } static inline void codec_mm_swap_cache_pages(struct codec_mm_scatter_mgt *smgt) @@ -1645,7 +2009,9 @@ static inline void codec_mm_swap_cache_pages(struct codec_mm_scatter_mgt *smgt) if (scatter_pages_swap_thrld == 0 || smgt->total_page_num - smgt->alloced_page_num < - PAGE_COUNT(scatter_pages_swap_thrld * SZ_1M)) + PAGE_COUNT(scatter_pages_swap_thrld * SZ_1M) || + !time_after64(get_jiffies_64(), + smgt->delay_free_timeout_jiffies64)) return; mms = codec_mm_get_next_cache_scatter(smgt, NULL, 1); @@ -2144,14 +2510,6 @@ int codec_mm_scatter_alloc_want_pages_in(struct codec_mm_scatter_mgt *smgt, return -1; } } - ATRACE_COUNTER("mmu alloc", MMU_ALLOC_LIST_LOCK_START); - codec_mm_list_lock(smgt); - ATRACE_COUNTER("mmu alloc", MMU_ALLOC_LIST_LOCK); - if (ret > 0 && is_cache_sc(smgt, mms)) - smgt->cached_pages += ret; - - codec_mm_list_unlock(smgt); - ATRACE_COUNTER("mmu alloc", MMU_ALLOC_LIST_LOCK_END); codec_mm_scatter_unlock(mms); if (smgt->cached_pages < smgt->keep_size_PAGE / 2) { /*try alloc more cache.*/ @@ -2163,7 +2521,7 @@ int codec_mm_scatter_alloc_want_pages_in(struct codec_mm_scatter_mgt *smgt, int codec_mm_scatter_alloc_want_pages(struct codec_mm_scatter *mms, int want_pages) { - struct codec_mm_scatter_mgt *smgt = (struct codec_mm_scatter_mgt *)mms->manager; + struct codec_mm_scatter_mgt *smgt; int ret; u64 startus; int need_pages = codec_mm_scatter_align_count(want_pages); @@ -2173,7 +2531,7 @@ int codec_mm_scatter_alloc_want_pages(struct codec_mm_scatter *mms, __func__, need_pages, want_pages); if (!mms) return -1; - + smgt = (struct codec_mm_scatter_mgt *)mms->manager; startus = codec_mm_get_current_us(); if (sc_swap_thrld && smgt->total_page_num - smgt->alloced_page_num > PAGE_COUNT(sc_swap_thrld * SZ_1M)) { @@ -2568,6 +2926,8 @@ int codec_mm_dump_scatter(struct codec_mm_scatter *mms, void *buf, int size) int tsize = 0; int s; int i; + int unit = scatter_get_unit_pages(); + int len; if (!pbuf) pbuf = sbuf; @@ -2579,19 +2939,38 @@ int codec_mm_dump_scatter(struct codec_mm_scatter *mms, void *buf, int size) pbuf += s; \ } while (0) - BUFPRINT("scatter info:%p\n", mms); - BUFPRINT("\tsize:%d\n", (int)(mms->page_cnt * PAGE_SIZE)); - BUFPRINT("\tmax:%d\n", mms->page_max_cnt); - BUFPRINT("\tpage_cnt:%d\n", mms->page_cnt); - BUFPRINT("\tpage_used:%d\n", mms->page_used); - BUFPRINT("\tpage_tail:%d\n", mms->page_tail); - BUFPRINT("\tuser_cnt:%d\n", atomic_read(&mms->user_cnt)); - BUFPRINT("\ttofree_jiffies:%ld\n", mms->tofree_jiffies); + if (!mms->queue) { + BUFPRINT("scatter info:%px\n", mms); + BUFPRINT("\tsize:%d\n", (int)(mms->page_cnt * PAGE_SIZE)); + BUFPRINT("\tmax:%d\n", mms->page_max_cnt); + BUFPRINT("\tpage_cnt:%d\n", mms->page_cnt); + BUFPRINT("\tpage_used:%d\n", mms->page_used); + BUFPRINT("\tpage_tail:%d\n", mms->page_tail); + BUFPRINT("\tuser_cnt:%d\n", atomic_read(&mms->user_cnt)); + BUFPRINT("\ttofree_jiffies:%ld\n", mms->tofree_jiffies); - i = 0; - while (i < mms->page_cnt && i < 16) - BUFPRINT(":%x", (u32)mms->pages_list[i++]); - BUFPRINT("\n"); + i = 0; + while (i < mms->page_cnt) { + BUFPRINT(":%lx", mms->pages_list[i]); + i += unit; + }; + BUFPRINT("\n"); + } else { + BUFPRINT("cache info:%px\n", mms); + BUFPRINT("\tsize:%d\n", (int)(codec_mm_queue_len(mms->queue) * PAGE_SIZE)); + BUFPRINT("\tmax:%d\n", mms->page_max_cnt); + BUFPRINT("\tpage_cnt:%d\n", codec_mm_queue_len(mms->queue)); + BUFPRINT("\tuser_cnt:%d\n", atomic_read(&mms->user_cnt)); + BUFPRINT("\tqueue head:%d\n", mms->queue->head); + BUFPRINT("\tqueue tail:%d\n", mms->queue->tail); + BUFPRINT("\ttofree_jiffies:%ld\n", mms->tofree_jiffies); + i = mms->queue->tail; + len = codec_mm_queue_len(mms->queue); + while ((len -= unit) > 0) { + BUFPRINT("%lx|", mms->pages_list[i % CACHE_QUEUE_LEN]); + i += unit; + } + } #undef BUFPRINT if (!buf) INFO_LOG("%s", sbuf); @@ -2858,10 +3237,11 @@ static void codec_mm_scatter_cache_manage(struct codec_mm_scatter_mgt *smgt) (smgt->force_cache_on &&/*on star cache*/ (smgt->total_page_num < smgt->force_cache_page_cnt)) ) {/*first 500ms ,alloc double.*/ + int once_alloc = 1024; + mms = codec_mm_get_next_cache_scatter(smgt, NULL, 0); if (mms) { int need; - int once_alloc = 1024;/*once 4M*/ if (smgt->force_cache_on) { once_alloc = 4096; @@ -2873,22 +3253,22 @@ static void codec_mm_scatter_cache_manage(struct codec_mm_scatter_mgt *smgt) smgt->alloced_page_num; } } - need = mms->page_cnt + once_alloc; - if ((need - mms->page_cnt) > once_alloc) - need = mms->page_cnt + once_alloc; - if (need > smgt->force_cache_page_cnt) + need = codec_mm_queue_len(mms->queue) + once_alloc; + if (smgt->force_cache_on && need > smgt->force_cache_page_cnt) need = smgt->force_cache_page_cnt; if (need > mms->page_max_cnt) need = mms->page_max_cnt; - if (need > mms->page_cnt) { - ATRACE_COUNTER - ("mmu alloc", - MMU_ALLOC_SCATTER_ALLOC_WANT_PAGE_IN_2) - ; - alloced = - !codec_mm_scatter_alloc_want_pages_in - (smgt, mms, need); - ATRACE_COUNTER("mmu alloc", INSTR3); + if (need > codec_mm_queue_len(mms->queue)) { + ATRACE_COUNTER("mmu alloc", + MMU_ALLOC_SCATTER_ALLOC_WANT_PAGE_IN_2); + codec_mm_queue_head_lock(mms->queue); + alloced = codec_mm_cache_queue_produce(smgt, + mms, + mms->queue, + need - codec_mm_queue_len(mms->queue), + NULL); + codec_mm_queue_head_unlock(mms->queue); + ATRACE_COUNTER("mmu alloc", INSTR3); } else { alloced = 0; } @@ -2896,12 +3276,14 @@ static void codec_mm_scatter_cache_manage(struct codec_mm_scatter_mgt *smgt) alloced = 0; } /*wake up wait.*/ - if (alloced && - smgt->force_cache_on && - smgt->cached_pages >= smgt->force_cache_page_cnt) { - smgt->force_cache_on = 0; - smgt->delay_free_timeout_jiffies64 = - get_jiffies_64() + 2000 * HZ / 1000; + if (alloced && smgt->force_cache_on) { + smgt->force_cache_page_cnt -= once_alloc; + if (smgt->force_cache_page_cnt <= 0) { + pr_dbg("scatter force_cache off\n"); + smgt->force_cache_on = 0; + smgt->delay_free_timeout_jiffies64 = + get_jiffies_64() + 2000 * HZ / 1000; + } } } else if ((smgt->cached_pages > (smgt->keep_size_PAGE + 1000)) && @@ -2941,7 +3323,7 @@ static void codec_mm_scatter_cache_manage(struct codec_mm_scatter_mgt *smgt) if (smgt->keep_size_PAGE == 0 && smgt->delay_free_on) { mms = codec_mm_get_next_cache_scatter(smgt, NULL, 1); if (mms) {/*only free some 1M cache*/ - codec_mm_scatter_less_pages(mms, 256); + codec_mm_scatter_cache_less_page(smgt, mms, 256); } } } @@ -2954,7 +3336,7 @@ static int codec_mm_scatter_scatter_arrange(struct codec_mm_scatter_mgt *smgt) int n = 0; if (smgt->delay_free_on > 0 && !smgt->try_free_cache_sc) { - mms = codec_mm_scatter_alloc_new(smgt, 16384, 0); + mms = codec_mm_scatter_alloc_new(smgt, CACHE_QUEUE_LEN, 0); if (mms) { codec_mm_list_lock(smgt); list_del_init(&mms->list); @@ -2971,21 +3353,9 @@ static int codec_mm_scatter_scatter_arrange(struct codec_mm_scatter_mgt *smgt) if (smgt->delay_free_on > 0 && !smgt->cache_scs[0]) { /*cache1*/ - mms = codec_mm_scatter_alloc_new(smgt, 16384, 0); - if (mms) { - codec_mm_list_lock(smgt); - list_del_init(&mms->list); - smgt->cache_scs[0] = mms; - codec_mm_list_unlock(smgt); - } + codec_mm_scatter_cache_creat(smgt, 0); /*cache2*/ - mms = codec_mm_scatter_alloc_new(smgt, 16384, 0); - if (mms) { - codec_mm_list_lock(smgt); - list_del_init(&mms->list); - smgt->cache_scs[1] = mms; - codec_mm_list_unlock(smgt); - } + codec_mm_scatter_cache_creat(smgt, 1); } if (smgt->delay_free_on <= 0 && smgt->cache_scs[0] && time_after64(get_jiffies_64(), @@ -2999,18 +3369,11 @@ static int codec_mm_scatter_scatter_arrange(struct codec_mm_scatter_mgt *smgt) } mms0 = smgt->cache_scs[0]; mms1 = smgt->cache_scs[1]; - smgt->cache_scs[0] = NULL; - smgt->cache_scs[1] = NULL; - smgt->cached_pages = 0; codec_mm_list_unlock(smgt); - if (mms0) { - codec_mm_scatter_dec_owner_user(mms0, 0); - codec_mm_scatter_free_on_nouser(smgt, mms0); - } - if (mms1) { - codec_mm_scatter_dec_owner_user(mms1, 0); - codec_mm_scatter_free_on_nouser(smgt, mms1); - } + if (mms0) + codec_mm_scatter_cache_destroy(smgt, 0); + if (mms1) + codec_mm_scatter_cache_destroy(smgt, 1); } if (smgt->delay_free_on <= 0 && smgt->try_free_cache_sc) { @@ -3117,26 +3480,12 @@ static int codec_mm_scatter_free_all_ignorecache_in mutex_lock(&smgt->monitor_lock); pr_info("force free all scatter ignorecache!\n"); /*free cache: always*/ - do { - struct codec_mm_scatter *mms; - /*clear cache first. */ - /*disabled free on first. */ - smgt->delay_free_on = 0; - codec_mm_list_lock(smgt); - mms = smgt->cache_scs[0]; - smgt->cache_scs[0] = smgt->cache_scs[1]; - smgt->cache_scs[1] = NULL; - smgt->cached_pages = 0; - codec_mm_list_unlock(smgt); - if (mms) { - pr_info("cache_sc page_max %d, page_used %d\n", - mms->page_max_cnt, mms->page_used); - codec_mm_scatter_dec_owner_user(mms, 0); - codec_mm_scatter_free_on_nouser(smgt, mms); - } - /*alloced again on timer?*/ - /* check again. */ - } while (smgt->cache_scs[0]); + /*clear cache first. */ + /*disabled free on first. */ + smgt->delay_free_on = 0; + codec_mm_scatter_cache_destroy(smgt, 0); + codec_mm_scatter_cache_destroy(smgt, 1); + /* free cache nouser * if flags==2 force */ @@ -3201,6 +3550,46 @@ static void codec_mm_scatter_monitor(struct work_struct *work) mutex_unlock(&smgt->monitor_lock); } +static u32 codec_mm_scatter_get_reserved_block_size(void) +{ + u32 ret, reserved_block_size; + int alloc_mode = + codec_mm_get_property_from_dts("scatter_alloc_mode"); + u32 total_size = codec_mm_get_total_size() / SZ_1M; + + if (alloc_mode < 0) + alloc_mode = SCATTER_ALLOC_FROM_DEFAULT; + + switch (alloc_mode) { + case SCATTER_ALLOC_FROM_SYS: + ret = total_size; + break; + + case SCATTER_ALLOC_FROM_CMA: + ret = 0; + break; + + case SCATTER_ALLOC_FROM_MIXED: + reserved_block_size = + codec_mm_get_property_from_dts("reserved_block_size"); + if (reserved_block_size > total_size || + reserved_block_size < 0) { + ret = 300; + pr_info("reserved_block_size is %d, reset as 300.\n", + reserved_block_size); + } else { + ret = reserved_block_size; + } + break; + + case SCATTER_ALLOC_FROM_DEFAULT: + default: + ret = 300; + } + + return ret; +} + static int codec_mm_scatter_mgt_alloc_in(struct codec_mm_scatter_mgt **psmgt) { struct codec_mm_scatter_mgt *smgt; @@ -3219,9 +3608,9 @@ static int codec_mm_scatter_mgt_alloc_in(struct codec_mm_scatter_mgt **psmgt) smgt->try_alloc_in_sys_page_cnt_max = MAX_SYS_BLOCK_PAGE; smgt->try_alloc_in_sys_page_cnt = MAX_SYS_BLOCK_PAGE; smgt->try_alloc_in_sys_page_cnt_min = MIN_SYS_BLOCK_PAGE; - smgt->reserved_block_mm_M = 300; + smgt->reserved_block_mm_M = codec_mm_scatter_get_reserved_block_size(); smgt->keep_size_PAGE = is_2k_platform() ? - 4 * SZ_1M >> PAGE_SHIFT : 20 * SZ_1M >> PAGE_SHIFT; + 4 * SZ_1M >> PAGE_SHIFT : 20 * SZ_1M >> PAGE_SHIFT; smgt->alloc_from_cma_first = 1; smgt->enable_slot_from_sys = 0; smgt->expected_slot_sid[0] = MAX_SID; @@ -3269,6 +3658,7 @@ static struct mconfig codec_mm_sc_configs[] = { MC_PU32("watermark_for_high", &g_scatter.watermark_for_high), MC_PU32("scatter_align_pages_size", &scatter_align_pages_size), MC_PU32("scatter_swap_threshold_size_M", &scatter_swap_threshold_size), + MC_PU32("scatter_multi_alloc", &scatter_multi_alloc), }; static struct mconfig_node codec_mm_sc; @@ -3386,3 +3776,48 @@ int codec_mm_scatter_test(int mode, int p1, int p2) } EXPORT_SYMBOL(codec_mm_scatter_test); +int codec_mm_scatter_simu_thread_test(uint time, + uint buffer_page_num, uint buffer_num, uint intrval_ms, + uint wait_size, uint tvp_mode) +{ + struct codec_mm_scatter *sc[30]; + int start_free_id; + + codec_mm_scatter_mgt_delay_free_switch(1, 2000, wait_size * SZ_1M, tvp_mode); + msleep(20); + + while (time--) { + int i; + ktime_t start_t, alloc_t, free_tail_t; + + for (i = 0; i < MIN(buffer_num, 30); i++) { + start_t = ktime_get(); + sc[i] = codec_mm_scatter_alloc(buffer_page_num + 64, + buffer_page_num, tvp_mode); + if (!sc[i]) + break; + alloc_t = ktime_get(); + + start_free_id = buffer_page_num * (3 + get_random_u32() % 4) / 10; + codec_mm_scatter_free_tail_pages_fast(sc[i], start_free_id); + free_tail_t = ktime_get(); + msleep(intrval_ms); + pr_crit("[%s]sc[%2d]alloc page_num:%d|%d (%lld-%lld)\n", + __func__, i, sc[i]->page_cnt, start_free_id, + ktime_us_delta(alloc_t, start_t), + ktime_us_delta(free_tail_t, alloc_t)); + } + + for (i = 0; i < MIN(buffer_num, 30); i++) { + start_t = ktime_get(); + codec_mm_scatter_dec_owner_user(sc[i], 0); + } + } + + codec_mm_scatter_mgt_delay_free_switch(0, 0, 0, tvp_mode); + msleep(500); + + return 0; +} +EXPORT_SYMBOL(codec_mm_scatter_simu_thread_test); + diff --git a/drivers/media/common/codec_mm/codec_mm_scatter_priv.h b/drivers/media/common/codec_mm/codec_mm_scatter_priv.h index 3e247360f..ddfa27275 100644 --- a/drivers/media/common/codec_mm/codec_mm_scatter_priv.h +++ b/drivers/media/common/codec_mm/codec_mm_scatter_priv.h @@ -43,6 +43,9 @@ int codec_mm_dump_all_scatters(void); int codec_mm_scatter_mgt_test(void); int codec_mm_scatter_test(int mode, int p1, int p2); +int codec_mm_scatter_simu_thread_test(uint time, + uint buffer_page_num, uint buffer_num, uint intrval_ms, + uint wait_size, uint tvp_mode); int codec_mm_dump_all_hash_table(void); int codec_mm_scatter_mask_for_keep(void *sc_mm); diff --git a/drivers/media/common/codec_mm/codec_mm_sys.c b/drivers/media/common/codec_mm/codec_mm_sys.c new file mode 100644 index 000000000..9fd293fc8 --- /dev/null +++ b/drivers/media/common/codec_mm/codec_mm_sys.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: (GPL-2.0+ OR MIT) +/* + * Copyright (c) 2019 Amlogic, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "codec_mm_priv.h" + +static struct list_head work_list; +static spinlock_t work_list_lock; /* protect job list */ + +#define MAX_JOB_NUMBER 12 +#define TASK_ARRAY_NUMBER 342 + +struct work_alloc_pages { + struct list_head list; + struct page **page[TASK_ARRAY_NUMBER]; + struct task_struct *host; + gfp_t gfp; + int array_len; +}; + +struct alloc_page_pcp { + struct completion start; + struct completion end; + struct task_struct *task; + int cpu; + int ret; +}; + +#define ALLOC_PAGE_NUM_KTHREAD 3 +static DEFINE_MUTEX(alloc_page_mutex); +struct work_alloc_pages alloc_pages_job[MAX_JOB_NUMBER] = {}; +struct alloc_page_pcp alloc_kthread_array[ALLOC_PAGE_NUM_KTHREAD]; + +unsigned long aml_alloc_pages_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array) +{ + int cpu, cpus, i = 0, j = 0, k = 0, ret = 0, enomem = 0, einv = 0; + atomic_t ok; + unsigned long flags; + struct alloc_page_pcp *work; + unsigned int max_count = MAX_JOB_NUMBER * TASK_ARRAY_NUMBER; + + mutex_lock(&alloc_page_mutex); + + cpus = num_online_cpus() - 1; + + if (nr_pages > max_count) { + pr_err("nr pages too large: %ld > max(%d)\n", nr_pages, max_count); + return -ENOMEM; + } + + spin_lock(&work_list_lock); + if (!list_empty(&work_list)) + INIT_LIST_HEAD(&work_list); + for (k = 0; k < nr_pages; i++) { + INIT_LIST_HEAD(&alloc_pages_job[i].list); + for (j = 0; j < TASK_ARRAY_NUMBER && k < nr_pages; j++) + alloc_pages_job[i].page[j] = &page_array[k++]; + alloc_pages_job[i].array_len = j; + alloc_pages_job[i].host = current; + alloc_pages_job[i].gfp = gfp; + list_add(&alloc_pages_job[i].list, &work_list); + } + spin_unlock(&work_list_lock); + i = 0; + + atomic_set(&ok, 0); + local_irq_save(flags); + for (cpu = 0; cpu < ALLOC_PAGE_NUM_KTHREAD; cpu++) { + work = &alloc_kthread_array[cpu]; + complete(&work->start); + i++; + if (i == cpus) { + pr_debug("sched work to %d cpu\n", i); + break; + } + } + local_irq_restore(flags); + + for (cpu = 0; cpu < i; cpu++) { + work = &alloc_kthread_array[cpu]; + wait_for_completion(&work->end); + if (work->ret) { + if (work->ret == -EINTR) + einv++; + else + enomem++; + } + } + + if (einv) + ret = -EINVAL; + else if (enomem) + ret = -ENOMEM; + else + ret = 0; + + if (ret < 0) + pr_err("%s, failed, ret:%d, ok:%d\n", + __func__, ret, atomic_read(&ok)); + + mutex_unlock(&alloc_page_mutex); + + return ret; +} +EXPORT_SYMBOL(aml_alloc_pages_array); + +static int alloc_page_boost_work_func(void *alloc_page_data) +{ + struct alloc_page_pcp *c_work; + struct work_alloc_pages *job; + int ret = -1, i = 0; + struct page *page; + gfp_t gfp; + + c_work = (struct alloc_page_pcp *)alloc_page_data; + for (;;) { + ret = wait_for_completion_interruptible(&c_work->start); + if (ret < 0) { + pr_err("%s wait for task %d is %d\n", + __func__, c_work->cpu, ret); + continue; + } +again: + spin_lock(&work_list_lock); + if (list_empty(&work_list)) { + /* NO job todo ? */ + spin_unlock(&work_list_lock); + pr_debug("%s,%d, list empty\n", __func__, __LINE__); + goto next; + } + job = list_first_entry(&work_list, struct work_alloc_pages, list); + list_del(&job->list); + spin_unlock(&work_list_lock); + gfp = job->gfp; + + if (fatal_signal_pending(job->host)) { + c_work->ret = -EINTR; + goto next; + } + for (i = 0; i < job->array_len; i++) { + page = alloc_pages(gfp, 0); + if (page) { + if (!job->page[i]) + pr_err("job i: %d, array_len: %d\n", i, job->array_len); + *job->page[i] = page; + } else { + c_work->ret = -ENOMEM; + pr_err("alloc page failed\n"); + break; + } + } + if (i >= job->array_len) + goto again; +next: + complete(&c_work->end); + if (kthread_should_stop()) { + pr_err("%s task exit\n", __func__); + break; + } + } + return 0; +} + +int init_alloc_page_boost_task(void) +{ + int cpu; + struct task_struct *task; + struct alloc_page_pcp *work; + char task_name[20] = {}; + + spin_lock_init(&work_list_lock); + INIT_LIST_HEAD(&work_list); + + for (cpu = 0; cpu < ALLOC_PAGE_NUM_KTHREAD; cpu++) { + memset(task_name, 0, sizeof(task_name)); + sprintf(task_name, "alloc_page_task%d", cpu); + work = &alloc_kthread_array[cpu]; + init_completion(&work->start); + init_completion(&work->end); + work->cpu = cpu; + task = kthread_create(alloc_page_boost_work_func, work, task_name); + if (!IS_ERR(task)) { + set_user_nice(task, -17); + work->task = task; + pr_debug("create alloc page task%p, for cpu %d\n", task, cpu); + wake_up_process(task); + } else { + pr_err("create task for cpu %d fail:%p\n", cpu, task); + return -1; + } + } + + return 0; +} + +void aml_free_pages_array(unsigned long nr_pages, struct page **page_array) +{ + unsigned int count = 0; + struct page *page; + unsigned int i = 0; + + while (nr_pages) { + page = page_array[i]; + if (!page) { + nr_pages--; + i++; + continue; + } + if (!put_page_testzero(page)) { + pr_info("%s, page %d are still in use\n", __func__, i); + count++; + break; + } + __free_page(page); + nr_pages--; + i++; + } + WARN(count != 0, "%d pages are still in use!\n", count); +} +EXPORT_SYMBOL(aml_free_pages_array); diff --git a/drivers/media/common/codec_mm/codec_mm_sys_priv.h b/drivers/media/common/codec_mm/codec_mm_sys_priv.h new file mode 100644 index 000000000..2bd02d729 --- /dev/null +++ b/drivers/media/common/codec_mm/codec_mm_sys_priv.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: (GPL-2.0+ OR MIT) */ +/* + * Copyright (c) 2019 Amlogic, Inc. All rights reserved. + */ + +#ifndef _CODEC_MM_SYS_H +#define _CODEC_MM_SYS_H + +int init_alloc_page_boost_task(void); + +unsigned long aml_alloc_pages_array(gfp_t gfp, unsigned long nr_pages, struct page **page_array); + +void aml_free_pages_array(unsigned long nr_pages, struct page **page_array); + +#endif //_CODEC_MM_SYS_H + diff --git a/include/linux/amlogic/major.h b/include/linux/amlogic/major.h index 95a37dab9..c1bd535be 100644 --- a/include/linux/amlogic/major.h +++ b/include/linux/amlogic/major.h @@ -48,4 +48,5 @@ #define AMEDIA_INFO_MAJOR (30 + (AML_BASE)) #define VICP_MAJOR (31 + (AML_BASE)) #define DI_PROCESS_MAJOR (32 + (AML_BASE)) +#define AML_DHP_MAJOR (33 + (AML_BASE)) #endif diff --git a/include/linux/amlogic/media/codec_mm/codec_mm.h b/include/linux/amlogic/media/codec_mm/codec_mm.h index 9b022cdc7..bc1b24817 100644 --- a/include/linux/amlogic/media/codec_mm/codec_mm.h +++ b/include/linux/amlogic/media/codec_mm/codec_mm.h @@ -17,15 +17,15 @@ #define CODEC_MM_FLAGS_CPU 2 #define CODEC_MM_FLAGS_TVP 4 -#define CODEC_MM_FLAGS_RESERVED 0x100 +#define CODEC_MM_FLAGS_RESERVED 0x100 #define CODEC_MM_FLAGS_CMA 0x200 - /* *alloc from cma first, *cma->then ..reserved. *for less memory fragment; */ -#define CODEC_MM_FLAGS_CMA_FIRST 0x400 +#define CODEC_MM_FLAGS_CMA_FIRST 0x400 +#define CODEC_MM_FLAGS_RESERVED_EXT 0x800 /* *flags can used for DRM: @@ -88,6 +88,7 @@ struct codec_mm_s { #define AMPORTS_MEM_FLAGS_FROM_GET_FROM_TVP 5 #define AMPORTS_MEM_FLAGS_FROM_GET_FROM_CMA_RES 6 #define AMPORTS_MEM_FLAGS_FROM_GET_FROM_COHERENT 7 +#define AMPORTS_MEM_FLAGS_FROM_GET_FROM_REVERSED_EXT 8 int from_flags; /*may can be shared on many user..*/ /*decoder/di/ppmgr,*/ @@ -149,6 +150,7 @@ void codec_mm_unmap_phyaddr(u8 *vaddr); void codec_mm_dma_flush(void *vaddr, int size, enum dma_data_direction dir); void codec_mm_prealloc_tvp_pool(void); +void dump_mem_infos_external(void); int codec_mm_get_total_size(void); int codec_mm_get_free_size(void); @@ -176,6 +178,7 @@ int codec_mm_alloc_cma_size(void); !IS_ENABLED(CONFIG_DEBUG_SPINLOCK) int cma_mmu_op(struct page *page, int count, bool set); void codec_mm_dev_set_dma_mask(u64 bits); + #else static inline int cma_mmu_op(struct page *page, int count, bool set) { diff --git a/include/linux/amlogic/media/codec_mm/codec_mm_scatter.h b/include/linux/amlogic/media/codec_mm/codec_mm_scatter.h index 5e4ca245f..08a4b9ce6 100644 --- a/include/linux/amlogic/media/codec_mm/codec_mm_scatter.h +++ b/include/linux/amlogic/media/codec_mm/codec_mm_scatter.h @@ -18,8 +18,19 @@ #define SC_ALLOC_SYS_DMA32 BIT(0) #define SC_OWEN_NAME_LEN 8 +#define CACHE_QUEUE_LEN 16384 + #define PAGE_INDEX(page) ((page) >> PAGE_SHIFT) +struct codec_mm_cache_queue { + int head; + int tail; + /* cache produce lock */ + struct mutex head_lock; + /* cache consume lock */ + spinlock_t tail_lock; +}; + struct codec_mm_scatter { void *manager; phy_addr_type *pages_list; @@ -27,11 +38,10 @@ struct codec_mm_scatter { int page_cnt; /*page num */ int page_tail; /*last page in list */ int page_used; - /* spin lock */ - spinlock_t lock; + struct codec_mm_cache_queue *queue; atomic_t user_cnt; unsigned long tofree_jiffies; - /* mutex lock */ + /* user scatter mutex lock */ struct mutex mutex; struct list_head list; /*hold list. */ };