diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index fd8484856d25..80126b5c2568 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -3,6 +3,7 @@ #define _LINUX_MM_TYPES_H #include +#include #include #include @@ -16,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -618,16 +621,16 @@ struct mm_struct { struct { /* this mm_struct is on lru_gen_mm_list */ struct list_head list; +#ifdef CONFIG_MEMCG + /* points to the memcg of "owner" above */ + struct mem_cgroup *memcg; +#endif /* * Set when switching to this mm_struct, as a hint of * whether it has been used since the last time per-node * page table walkers cleared the corresponding bits. */ - unsigned long bitmap; -#ifdef CONFIG_MEMCG - /* points to the memcg of "owner" above */ - struct mem_cgroup *memcg; -#endif + nodemask_t nodes; } lru_gen; #endif /* CONFIG_LRU_GEN */ @@ -676,7 +679,7 @@ void lru_gen_migrate_mm(struct mm_struct *mm); static inline void lru_gen_init_mm(struct mm_struct *mm) { INIT_LIST_HEAD(&mm->lru_gen.list); - mm->lru_gen.bitmap = 0; + nodes_clear(mm->lru_gen.nodes); #ifdef CONFIG_MEMCG mm->lru_gen.memcg = NULL; #endif @@ -689,7 +692,7 @@ static inline void lru_gen_use_mm(struct mm_struct *mm) * used since the last time it cleared the bitmap. So it might be worth * walking the page tables of this mm_struct to clear the accessed bit. */ - WRITE_ONCE(mm->lru_gen.bitmap, -1); + nodes_setall(mm->lru_gen.nodes); } #else /* !CONFIG_LRU_GEN */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index dced694881a7..1e37e837b605 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -391,7 +391,7 @@ struct lru_gen_struct { /* the multi-gen LRU lists, lazily sorted on eviction */ struct list_head lists[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the multi-gen LRU sizes, eventually consistent */ - long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; + unsigned long nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* the exponential moving average of refaulted */ unsigned long avg_refaulted[ANON_AND_FILE][MAX_NR_TIERS]; /* the exponential moving average of evicted+protected */ @@ -442,6 +442,8 @@ struct lru_gen_mm_walk { unsigned long max_seq; /* the next address within an mm to scan */ unsigned long next_addr; + /* Unused -- for ABI compatibility */ + unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)]; /* to batch promoted pages */ int nr_pages[MAX_NR_GENS][ANON_AND_FILE][MAX_NR_ZONES]; /* to batch the mm stats */ @@ -449,7 +451,7 @@ struct lru_gen_mm_walk { /* total batched items */ int batched; bool can_swap; - bool force_scan; + bool full_scan; }; void lru_gen_init_lruvec(struct lruvec *lruvec); diff --git a/mm/vmscan.c b/mm/vmscan.c index 7c4d5b882960..bb80ecfa2871 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -2949,12 +2949,12 @@ static bool should_skip_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) int type; unsigned long size = 0; struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); - int key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); + int key = pgdat->node_id; - if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) + if (!walk->full_scan && !node_isset(key, mm->lru_gen.nodes)) return true; - clear_bit(key, &mm->lru_gen.bitmap); + node_clear(key, mm->lru_gen.nodes); for (type = !walk->can_swap; type < ANON_AND_FILE; type++) { size += type ? get_mm_counter(mm, MM_FILEPAGES) : @@ -3016,7 +3016,7 @@ static bool iterate_mm_list(struct lruvec *lruvec, struct lru_gen_mm_walk *walk, /* force scan for those added after the last iteration */ if (!mm_state->tail || mm_state->tail == &mm->lru_gen.list) { mm_state->tail = mm_state->head; - walk->force_scan = true; + walk->full_scan = true; } if (should_skip_mm(mm, walk)) @@ -3630,7 +3630,7 @@ restart: walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos); } #endif - if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) + if (!walk->full_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i)) continue; walk->mm_stats[MM_NONLEAF_FOUND]++; @@ -3777,7 +3777,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, bool can_swap) if (type == LRU_GEN_ANON && !can_swap) goto done; - /* prevent cold/hot inversion if force_scan is true */ + /* prevent cold/hot inversion if full_scan is true */ for (zone = 0; zone < MAX_NR_ZONES; zone++) { struct list_head *head = &lrugen->lists[old_gen][type][zone]; @@ -3846,7 +3846,7 @@ next: return success; } -static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) +static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool full_scan) { int prev, next; int type, zone; @@ -3861,7 +3861,7 @@ static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) if (get_nr_gens(lruvec, type) != MAX_NR_GENS) continue; - VM_WARN_ON_ONCE(!force_scan && (type == LRU_GEN_FILE || can_swap)); + VM_WARN_ON_ONCE(!full_scan && (type == LRU_GEN_FILE || can_swap)); while (!inc_min_seq(lruvec, type, can_swap)) { spin_unlock_irq(&pgdat->lru_lock); @@ -3904,7 +3904,7 @@ static void inc_max_seq(struct lruvec *lruvec, bool can_swap, bool force_scan) } static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, - struct scan_control *sc, bool can_swap, bool force_scan) + struct scan_control *sc, bool can_swap, bool full_scan) { bool success; struct lru_gen_mm_walk *walk; @@ -3925,7 +3925,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, * handful of PTEs. Spreading the work out over a period of time usually * is less efficient, but it avoids bursty page faults. */ - if (!force_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { + if (!full_scan && !(arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK))) { success = iterate_mm_list_nowalk(lruvec, max_seq); goto done; } @@ -3939,7 +3939,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, walk->lruvec = lruvec; walk->max_seq = max_seq; walk->can_swap = can_swap; - walk->force_scan = force_scan; + walk->full_scan = full_scan; do { success = iterate_mm_list(lruvec, walk, &mm); @@ -3959,7 +3959,7 @@ done: VM_WARN_ON_ONCE(max_seq != READ_ONCE(lrugen->max_seq)); - inc_max_seq(lruvec, can_swap, force_scan); + inc_max_seq(lruvec, can_swap, full_scan); /* either this sees any waiters or they will see updated max_seq */ if (wq_has_sleeper(&lruvec->mm_state.wait)) wake_up_all(&lruvec->mm_state.wait); @@ -3986,7 +3986,8 @@ static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsig gen = lru_gen_from_seq(seq); for (zone = 0; zone < MAX_NR_ZONES; zone++) - size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + size += max_t(long, READ_ONCE(lrugen->nr_pages[gen][type][zone]), + 0); total += size; if (seq == max_seq) @@ -5099,7 +5100,8 @@ static int lru_gen_seq_show(struct seq_file *m, void *v) char mark = full && seq < min_seq[type] ? 'x' : ' '; for (zone = 0; zone < MAX_NR_ZONES; zone++) - size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); + size += max_t(long, READ_ONCE(lrugen->nr_pages[gen][type][zone]), + 0); seq_printf(m, " %10lu%c", size, mark); } @@ -5121,7 +5123,7 @@ static const struct seq_operations lru_gen_seq_ops = { }; static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, - bool can_swap, bool force_scan) + bool can_swap, bool full_scan) { DEFINE_MAX_SEQ(lruvec); DEFINE_MIN_SEQ(lruvec); @@ -5132,10 +5134,10 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq, struct scan_contr if (seq > max_seq) return -EINVAL; - if (!force_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) + if (!full_scan && min_seq[!can_swap] + MAX_NR_GENS - 1 <= max_seq) return -ERANGE; - try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, force_scan); + try_to_inc_max_seq(lruvec, max_seq, sc, can_swap, full_scan); return 0; }