diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 1bbe4a34ef4c..9594367a4ef0 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -39,6 +41,105 @@ static unsigned kcopyd_subjob_size_kb = DEFAULT_SUB_JOB_SIZE_KB; module_param(kcopyd_subjob_size_kb, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(kcopyd_subjob_size_kb, "Sub-job size for dm-kcopyd clients"); +static bool rsm_enabled; +static phys_addr_t rsm_mem_base, rsm_mem_size; + +#ifndef MODULE +static DEFINE_SPINLOCK(rsm_lock); +static int *rsm_mem; +static int rsm_page_cnt; +static int rsm_tbl_idx; +static struct reserved_mem *rmem; + +static void __init kcopyd_rsm_init(void) +{ + static struct device_node *rsm_node; + int ret = 0; + + if (!rsm_enabled) + return; + + rsm_node = of_find_compatible_node(NULL, NULL, "mediatek,dm_ota"); + if (!rsm_node) { + ret = -ENODEV; + goto out; + } + + rmem = of_reserved_mem_lookup(rsm_node); + if (!rmem) { + ret = -EINVAL; + goto out_put_node; + } + + rsm_mem_base = rmem->base; + rsm_mem_size = rmem->size; + rsm_page_cnt = rsm_mem_size / PAGE_SIZE; + rsm_mem = kcalloc(rsm_page_cnt, sizeof(int), GFP_KERNEL); + if (!rsm_mem) + ret = -ENOMEM; + +out_put_node: + of_node_put(rsm_node); +out: + if (ret) + pr_warn("kcopyd: failed to init rsm: %d", ret); +} + +static int __init kcopyd_rsm_enable(char *str) +{ + rsm_enabled = true; + + return 0; +} +early_param("mtk_kcopyd_quirk", kcopyd_rsm_enable); + +static void kcopyd_rsm_get_page(struct page **p) +{ + int i; + unsigned long flags; + + *p = NULL; + spin_lock_irqsave(&rsm_lock, flags); + for (i = 0 ; i < rsm_page_cnt ; i++) { + rsm_tbl_idx = (rsm_tbl_idx + 1 == rsm_page_cnt) ? 0 : rsm_tbl_idx + 1; + + if (rsm_mem[rsm_tbl_idx] == 0) { + rsm_mem[rsm_tbl_idx] = 1; + *p = virt_to_page(phys_to_virt(rsm_mem_base + PAGE_SIZE + * rsm_tbl_idx)); + break; + } + } + spin_unlock_irqrestore(&rsm_lock, flags); +} + +static void kcopyd_rsm_drop_page(struct page **p) +{ + u64 off; + unsigned long flags; + + if (*p) { + off = page_to_phys(*p) - rsm_mem_base; + spin_lock_irqsave(&rsm_lock, flags); + rsm_mem[off >> PAGE_SHIFT] = 0; + spin_unlock_irqrestore(&rsm_lock, flags); + *p = NULL; + } +} + +static void kcopyd_rsm_destroy(void) +{ + if (rsm_enabled) + kfree(rsm_mem); +} + +#else +#define kcopyd_rsm_destroy(...) +#define kcopyd_rsm_drop_page(...) +#define kcopyd_rsm_get_page(...) +#define kcopyd_rsm_init(...) +#endif + static unsigned dm_get_kcopyd_subjob_size(void) { unsigned sub_job_size_kb; @@ -211,7 +312,7 @@ static void wake(struct dm_kcopyd_client *kc) /* * Obtain one page for the use of kcopyd. */ -static struct page_list *alloc_pl(gfp_t gfp) +static struct page_list *alloc_pl(gfp_t gfp, unsigned long job_flags) { struct page_list *pl; @@ -219,7 +320,12 @@ static struct page_list *alloc_pl(gfp_t gfp) if (!pl) return NULL; - pl->page = alloc_page(gfp); + if (rsm_enabled && test_bit(DM_KCOPYD_SNAP_MERGE, &job_flags)) { + kcopyd_rsm_get_page(&pl->page); + } else { + pl->page = alloc_page(gfp); + } + if (!pl->page) { kfree(pl); return NULL; @@ -230,7 +336,14 @@ static struct page_list *alloc_pl(gfp_t gfp) static void free_pl(struct page_list *pl) { - __free_page(pl->page); + struct page *p = pl->page; + phys_addr_t pa = page_to_phys(p); + + if (rsm_enabled && pa >= rsm_mem_base && pa < rsm_mem_base + rsm_mem_size) + kcopyd_rsm_drop_page(&pl->page); + else + __free_page(pl->page); + kfree(pl); } @@ -258,14 +371,15 @@ static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) } static int kcopyd_get_pages(struct dm_kcopyd_client *kc, - unsigned int nr, struct page_list **pages) + unsigned int nr, struct page_list **pages, + unsigned long job_flags) { struct page_list *pl; *pages = NULL; do { - pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM); + pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM, job_flags); if (unlikely(!pl)) { /* Use reserved pages */ pl = kc->pages; @@ -309,7 +423,7 @@ static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages) struct page_list *pl = NULL, *next; for (i = 0; i < nr_pages; i++) { - next = alloc_pl(GFP_KERNEL); + next = alloc_pl(GFP_KERNEL, 0); if (!next) { if (pl) drop_pages(pl); @@ -395,6 +509,8 @@ int __init dm_kcopyd_init(void) zero_page_list.next = &zero_page_list; zero_page_list.page = ZERO_PAGE(0); + kcopyd_rsm_init(); + return 0; } @@ -402,6 +518,7 @@ void dm_kcopyd_exit(void) { kmem_cache_destroy(_job_cache); _job_cache = NULL; + kcopyd_rsm_destroy(); } /* @@ -586,7 +703,7 @@ static int run_pages_job(struct kcopyd_job *job) int r; unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9); - r = kcopyd_get_pages(job->kc, nr_pages, &job->pages); + r = kcopyd_get_pages(job->kc, nr_pages, &job->pages, job->flags); if (!r) { /* this job is ready for io */ push(&job->kc->io_jobs, job); diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 41735a25d50a..b2d8b37b202d 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1117,7 +1117,8 @@ static void snapshot_merge_next_chunks(struct dm_snapshot *s) for (i = 0; i < linear_chunks; i++) __check_for_conflicting_io(s, old_chunk + i); - dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); + dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 1 << DM_KCOPYD_SNAP_MERGE, + merge_callback, s); return; shut: diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h index e42de7750c88..3a594ab9b563 100644 --- a/include/linux/dm-kcopyd.h +++ b/include/linux/dm-kcopyd.h @@ -21,6 +21,7 @@ #define DM_KCOPYD_IGNORE_ERROR 1 #define DM_KCOPYD_WRITE_SEQ 2 +#define DM_KCOPYD_SNAP_MERGE 3 struct dm_kcopyd_throttle { unsigned throttle;