diff --git a/drivers/memory_ext/page_trace.c b/drivers/memory_ext/page_trace.c index dab0a63c6..32d391890 100644 --- a/drivers/memory_ext/page_trace.c +++ b/drivers/memory_ext/page_trace.c @@ -46,6 +46,8 @@ #include #include #include +#include +#include #endif #ifdef CONFIG_NUMA @@ -129,6 +131,7 @@ static struct fun_symbol common_func[] = { {"cma_alloc", 1, 0}, {"dma_alloc_from_contiguous", 1, 0}, {"dma_alloc_contiguous", 1, 0}, + {"__dma_direct_alloc_pages", 1, 0}, {"alloc_slab_page", 1, 0}, {"aml_cma_alloc_post_hook", 1, 0}, #ifdef CONFIG_ARM @@ -156,13 +159,18 @@ static struct fun_symbol common_func[] = { {"kmalloc_order", 1, 0}, {"kmalloc_order_trace", 1, 0}, {"aml_slub_alloc_large", 1, 0}, -#if IS_MODULE(CONFIG_AMLOGIC_PAGE_TRACE) - {"alloc_pages_ret_handler", 1, 0}, - {"comp_alloc_ret_handler", 1, 0}, {"__kretprobe_trampoline_handler", 1, 0}, {"trampoline_probe_handler", 1, 0}, + {"__traceiter_mm_page_alloc", 1, 0}, + {"page_alloc_callback", 1, 0}, + {"set_page_trace", 1, 0}, + {"set_page_trace.cfi_jt", 1, 0}, + {"reset_page_trace", 1, 0}, + {"replace_page_trace", 1, 0}, + {"page_alloc_callback", 1, 0}, + {"page_free_callback", 1, 0}, + {"migrate_page_callback", 1, 0}, {"kretprobe_trampoline", 1, 0}, -#endif {"module_alloc", 1, 0}, {"load_module", 1, 0}, #ifdef CONFIG_NUMA @@ -208,162 +216,23 @@ static struct kprobe kp_lookup_name = { /* ----------------------------------- */ -static char func_comp_alloc[NAME_MAX] = "compaction_alloc"; -static char func_alloc_pages[NAME_MAX] = "__alloc_pages"; -#if CONFIG_AMLOGIC_KERNEL_VERSION >= 14515 -static char func_free_prep[NAME_MAX] = "free_unref_page_prepare"; -#else -static char func_free_prep[NAME_MAX] = "free_pcp_prepare"; -#endif -static char func_free_ok[NAME_MAX] = "__free_pages_ok"; - -/* per-instance private data */ -struct kretp_data { - struct page *page; - gfp_t gfp; - unsigned int order; - unsigned long count; -}; - -/* Here we use the entry_handler to timestamp function entry */ -static int comp_alloc_entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +static void __maybe_unused page_free_callback(void *data, struct page *page, + unsigned int order) { - struct kretp_data *data; - - data = (struct kretp_data *)ri->data; - data->page = (struct page *)regs->regs[0]; - return 0; + reset_page_trace(page, order); } -NOKPROBE_SYMBOL(comp_alloc_entry_handler); - -static int alloc_pages_entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +static void __maybe_unused page_alloc_callback(void *data, struct page *page, + unsigned int order, gfp_t gfp_flags, int migratetype) { - struct kretp_data *data; - - data = (struct kretp_data *)ri->data; - data->gfp = (gfp_t)regs->regs[0]; - data->order = (unsigned int)regs->regs[1]; - return 0; + set_page_trace(page, order, gfp_flags, NULL); } -NOKPROBE_SYMBOL(alloc_pages_entry_handler); - -static int free_prep_entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +static void __maybe_unused migrate_page_callback(void *data, struct page *old_page, + struct page *new_page) { - struct kretp_data *data; - - data = (struct kretp_data *)ri->data; - data->page = (struct page *)regs->regs[0]; -#if CONFIG_AMLOGIC_KERNEL_VERSION >= 14515 - data->order = (unsigned int)regs->regs[2]; -#else - data->order = (unsigned int)regs->regs[1]; -#endif - return 0; + replace_page_trace(new_page, old_page); } - -NOKPROBE_SYMBOL(free_prep_entry_handler); - -static int free_ok_entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) -{ - struct kretp_data *data; - - data = (struct kretp_data *)ri->data; - data->page = (struct page *)regs->regs[0]; - data->order = (unsigned int)regs->regs[1]; - return 0; -} - -NOKPROBE_SYMBOL(free_ok_entry_handler); - -/* - * Return-probe handler: Log the return value and duration. Duration may turn - * out to be zero consistently, depending upon the granularity of time - * accounting on the platform. - */ -static int comp_alloc_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) -{ -// unsigned long retval = regs_return_value(regs); - struct page *ret_page = (struct page *)regs_return_value(regs); - struct kretp_data *data = (struct kretp_data *)ri->data; - - replace_page_trace(ret_page, data->page); - - return 0; -} - -NOKPROBE_SYMBOL(comp_alloc_ret_handler); - -static int alloc_pages_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) -{ -// unsigned long retval = regs_return_value(regs); - struct page *ret_page = (struct page *)regs_return_value(regs); - struct kretp_data *data = (struct kretp_data *)ri->data; - - set_page_trace(ret_page, data->order, data->gfp, NULL); - - return 0; -} - -NOKPROBE_SYMBOL(alloc_pages_ret_handler); - -static int free_prep_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) -{ - bool retval = (bool)regs_return_value(regs); - struct kretp_data *data = (struct kretp_data *)ri->data; - - if (retval) - reset_page_trace(data->page, data->order); - - return 0; -} - -NOKPROBE_SYMBOL(free_prep_ret_handler); - -static int free_ok_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) -{ - struct kretp_data *data = (struct kretp_data *)ri->data; - - if (PageBuddy(data->page)) - reset_page_trace(data->page, data->order); - - return 0; -} - -NOKPROBE_SYMBOL(free_ok_ret_handler); - -static struct kretprobe comp_alloc_kretprobe = { - .handler = comp_alloc_ret_handler, - .entry_handler = comp_alloc_entry_handler, - .data_size = sizeof(struct kretp_data), - /* Probe up to 20 instances concurrently. */ - .maxactive = 20, -}; - -static struct kretprobe alloc_pages_kretprobe = { - .handler = alloc_pages_ret_handler, - .entry_handler = alloc_pages_entry_handler, - .data_size = sizeof(struct kretp_data), - /* Probe up to 20 instances concurrently. */ - .maxactive = 20, -}; - -static struct kretprobe free_prep_kretprobe = { - .handler = free_prep_ret_handler, - .entry_handler = free_prep_entry_handler, - .data_size = sizeof(struct kretp_data), - /* Probe up to 20 instances concurrently. */ - .maxactive = 20, -}; - -static struct kretprobe free_ok_kretprobe = { - .handler = free_ok_ret_handler, - .entry_handler = free_ok_entry_handler, - .data_size = sizeof(struct kretp_data), - /* Probe up to 20 instances concurrently. */ - .maxactive = 20, -}; #endif /* ----------------------------------- */ @@ -517,19 +386,23 @@ static int __nocfi setup_common_caller(unsigned long kaddr) return -1; } +static int max_high; + static void dump_common_caller(void) { int i; for (i = 0; i < COMMON_CALLER_SIZE; i++) { if (common_caller[i].func_start_addr) - pr_debug("%2d, addr:%lx + %4lx, %ps\n", i, + pr_debug("dump: %2d, addr:%lx + %4lx, %ps\n", i, common_caller[i].func_start_addr, common_caller[i].size, (void *)common_caller[i].func_start_addr); else break; } + max_high = i - 1; + pr_info("max high: %d\n", max_high); } static int sym_cmp(const void *x1, const void *x2) @@ -678,6 +551,7 @@ static int __nocfi match_common_caller(void) addr = aml_kallsyms_lookup_name(s->name); if (addr) setup_common_caller(addr); + pr_debug("page_trace_addr: %s, %lx\n", s->name, addr); } return 0; } @@ -701,7 +575,7 @@ static int find_static_common_symbol(void *data) static int is_common_caller(struct alloc_caller *caller, unsigned long pc) { int ret = 0; - int low = 0, high = COMMON_CALLER_SIZE - 1, mid; + int low = 0, high = max_high, mid; unsigned long add_l, add_h; while (1) { @@ -724,8 +598,8 @@ static int is_common_caller(struct alloc_caller *caller, unsigned long pc) /* fix range */ if (high < 0) high = 0; - if (low > (COMMON_CALLER_SIZE - 1)) - low = COMMON_CALLER_SIZE - 1; + if (low > max_high) + low = max_high; } return ret; } @@ -2123,41 +1997,18 @@ static int __init page_trace_module_init(void) #endif #if IS_MODULE(CONFIG_AMLOGIC_PAGE_TRACE) - comp_alloc_kretprobe.kp.symbol_name = func_comp_alloc; - ret = register_kretprobe(&comp_alloc_kretprobe); - if (ret < 0) { - pr_err("register_kretprobe failed, returned %d\n", ret); - return ret; - } - pr_debug("Planted return probe at %s: %px\n", - comp_alloc_kretprobe.kp.symbol_name, comp_alloc_kretprobe.kp.addr); + ret = register_trace_mm_page_alloc(page_alloc_callback, NULL); + if (ret < 0) + pr_err("register_trace_mm_page_alloc fail ret=%d\n", ret); - alloc_pages_kretprobe.kp.symbol_name = func_alloc_pages; - ret = register_kretprobe(&alloc_pages_kretprobe); - if (ret < 0) { - pr_err("register_kretprobe failed, returned %d\n", ret); - return ret; - } - pr_debug("Planted return probe at %s: %px\n", - alloc_pages_kretprobe.kp.symbol_name, alloc_pages_kretprobe.kp.addr); + ret = register_trace_mm_page_free(page_free_callback, NULL); + if (ret < 0) + pr_err("register_trace_mm_page_free fail ret=%d\n", ret); - free_prep_kretprobe.kp.symbol_name = func_free_prep; - ret = register_kretprobe(&free_prep_kretprobe); - if (ret < 0) { - pr_err("register_kretprobe failed, returned %d\n", ret); - return ret; - } - pr_debug("Planted return probe at %s: %px\n", - free_prep_kretprobe.kp.symbol_name, free_prep_kretprobe.kp.addr); + ret = register_trace_android_vh_look_around_migrate_page(migrate_page_callback, NULL); + if (ret < 0) + pr_err("register_trace migrate page fail ret=%d\n", ret); - free_ok_kretprobe.kp.symbol_name = func_free_ok; - ret = register_kretprobe(&free_ok_kretprobe); - if (ret < 0) { - pr_err("register_kretprobe failed, returned %d\n", ret); - return ret; - } - pr_debug("Planted return probe at %s: %px\n", - free_ok_kretprobe.kp.symbol_name, free_ok_kretprobe.kp.addr); #endif return 0; @@ -2177,18 +2028,6 @@ static void __exit page_trace_module_exit(void) unregister_kprobe(&kp_dump_header); pr_debug("kprobe at %p unregistered\n", kp_dump_header.addr); - unregister_kretprobe(&comp_alloc_kretprobe); - pr_debug("kretprobe at %p unregistered\n", comp_alloc_kretprobe.kp.addr); - - unregister_kretprobe(&alloc_pages_kretprobe); - pr_debug("kretprobe at %p unregistered\n", alloc_pages_kretprobe.kp.addr); - - unregister_kretprobe(&free_prep_kretprobe); - pr_debug("kretprobe at %p unregistered\n", free_prep_kretprobe.kp.addr); - - unregister_kretprobe(&free_ok_kretprobe); - pr_debug("kretprobe at %p unregistered\n", free_ok_kretprobe.kp.addr); - if (!trace_buffer) vfree(trace_buffer);