shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-07 11:26:02 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge 1c52283265 ("Merge branch 'akpm' (patches from Andrew)") into android-mainline

Browse Source 
Steps on the way to 5.17-rc1

Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: I4bdb637ccb17b6c11b714c156d62aba7462ed28c
This commit is contained in:
Greg Kroah-Hartman
2022-02-16 19:46:04 +01:00
parent e4058ca11d 1c52283265
commit 20f48a2ca0
9 changed files with 61 additions and 407 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
Documentation/vm/frontswap.rst
View File

@@ -39,12 +39,6 @@ a disk write and, if the data is later read back, a disk read are avoided.
If a store returns failure, transcendent memory has rejected the data, and the
page can be written to swap as usual.
If a backend chooses, frontswap can be configured as a "writethrough
cache" by calling frontswap_writethrough(). In this mode, the reduction
in swap device writes is lost (and also a non-trivial performance advantage)
in order to allow the backend to arbitrarily "reclaim" space used to
store frontswap pages to more completely manage its memory usage.
Note that if a page is stored and the page already exists in transcendent memory
(a "duplicate" store), either the store succeeds and the data is overwritten,
or the store fails AND the page is invalidated. This ensures stale data may
@@ -261,19 +255,6 @@ the old data and ensure that it is no longer accessible. Since the
swap subsystem then writes the new data to the read swap device,
this is the correct course of action to ensure coherency.
* What is frontswap_shrink for?
When the (non-frontswap) swap subsystem swaps out a page to a real
swap device, that page is only taking up low-value pre-allocated disk
space. But if frontswap has placed a page in transcendent memory, that
page may be taking up valuable real estate. The frontswap_shrink
routine allows code outside of the swap subsystem to force pages out
of the memory managed by frontswap and back into kernel-addressable memory.
For example, in RAMster, a "suction driver" thread will attempt
to "repatriate" pages sent to a remote machine back to the local machine;
this is driven using the frontswap_shrink mechanism when memory pressure
subsides.
* Why does the frontswap patch create the new include file swapfile.h?
The frontswap code depends on some swap-subsystem-internal data

35
include/linux/frontswap.h
View File

@@ -7,31 +7,17 @@
#include <linux/bitops.h>
#include <linux/jump_label.h>
/*
* Return code to denote that requested number of
* frontswap pages are unused(moved to page cache).
* Used in shmem_unuse and try_to_unuse.
*/
#define FRONTSWAP_PAGES_UNUSED 2
struct frontswap_ops {
void (*init)(unsigned); /* this swap type was just swapon'ed */
int (*store)(unsigned, pgoff_t, struct page *); /* store a page */
int (*load)(unsigned, pgoff_t, struct page *); /* load a page */
void (*invalidate_page)(unsigned, pgoff_t); /* page no longer needed */
void (*invalidate_area)(unsigned); /* swap type just swapoff'ed */
struct frontswap_ops *next; /* private pointer to next ops */
};
extern void frontswap_register_ops(struct frontswap_ops *ops);
extern void frontswap_shrink(unsigned long);
extern unsigned long frontswap_curr_pages(void);
extern void frontswap_writethrough(bool);
#define FRONTSWAP_HAS_EXCLUSIVE_GETS
extern void frontswap_tmem_exclusive_gets(bool);
int frontswap_register_ops(const struct frontswap_ops *ops);
extern bool __frontswap_test(struct swap_info_struct *, pgoff_t);
extern void __frontswap_init(unsigned type, unsigned long *map);
extern void frontswap_init(unsigned type, unsigned long *map);
extern int __frontswap_store(struct page *page);
extern int __frontswap_load(struct page *page);
extern void __frontswap_invalidate_page(unsigned, pgoff_t);
@@ -45,11 +31,6 @@ static inline bool frontswap_enabled(void)
return static_branch_unlikely(&frontswap_enabled_key);
}
static inline bool frontswap_test(struct swap_info_struct *sis, pgoff_t offset)
{
return __frontswap_test(sis, offset);
}
static inline void frontswap_map_set(struct swap_info_struct *p,
unsigned long *map)
{
@@ -68,11 +49,6 @@ static inline bool frontswap_enabled(void)
return false;
}
static inline bool frontswap_test(struct swap_info_struct *sis, pgoff_t offset)
{
return false;
}
static inline void frontswap_map_set(struct swap_info_struct *p,
unsigned long *map)
{
@@ -112,11 +88,4 @@ static inline void frontswap_invalidate_area(unsigned type)
__frontswap_invalidate_area(type);
}
static inline void frontswap_init(unsigned type, unsigned long *map)
{
#ifdef CONFIG_FRONTSWAP
__frontswap_init(type, map);
#endif
}
#endif /* _LINUX_FRONTSWAP_H */

3
include/linux/shmem_fs.h
View File

@@ -83,8 +83,7 @@ extern void shmem_unlock_mapping(struct address_space *mapping);
extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask);
extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end);
extern int shmem_unuse(unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse);
int shmem_unuse(unsigned int type);
extern bool shmem_is_huge(struct vm_area_struct *vma,
struct inode *inode, pgoff_t index);

3
include/linux/swapfile.h
View File

@@ -6,10 +6,7 @@
* these were static in swapfile.c but frontswap.c needs them and we don't
* want to expose them to the dozens of source files that include swap.h
*/
extern spinlock_t swap_lock;
extern struct plist_head swap_active_head;
extern struct swap_info_struct *swap_info[];
extern int try_to_unuse(unsigned int, bool, unsigned long);
extern unsigned long generic_max_swapfile_size(void);
extern unsigned long max_swapfile_size(void);

18
mm/Kconfig
View File

@@ -445,20 +445,7 @@ config HAVE_SETUP_PER_CPU_AREA
bool
config FRONTSWAP
bool "Enable frontswap to cache swap pages if tmem is present"
depends on SWAP
help
Frontswap is so named because it can be thought of as the opposite
of a "backing" store for a swap device. The data is stored into
"transcendent memory", memory that is not directly accessible or
addressable by the kernel and is of unknown and possibly
time-varying size. When space in transcendent memory is available,
a significant swap I/O reduction may be achieved. When none is
available, all frontswap calls are reduced to a single pointer-
compare-against-NULL resulting in a negligible performance hit
and swap data is stored as normal on the matching swap device.
If unsure, say Y to enable frontswap.
bool
config CMA
bool "Contiguous Memory Allocator"
@@ -523,7 +510,8 @@ config MEM_SOFT_DIRTY
config ZSWAP
bool "Compressed cache for swap pages (EXPERIMENTAL)"
depends on FRONTSWAP && CRYPTO=y
depends on SWAP && CRYPTO=y
select FRONTSWAP
select ZPOOL
help
A lightweight compressed cache for swap pages. It takes

259
mm/frontswap.c
View File

@@ -27,27 +27,7 @@ DEFINE_STATIC_KEY_FALSE(frontswap_enabled_key);
* may be registered, but implementations can never deregister. This
* is a simple singly-linked list of all registered implementations.
*/
static struct frontswap_ops *frontswap_ops __read_mostly;
#define for_each_frontswap_ops(ops) \
for ((ops) = frontswap_ops; (ops); (ops) = (ops)->next)
/*
* If enabled, frontswap_store will return failure even on success. As
* a result, the swap subsystem will always write the page to swap, in
* effect converting frontswap into a writethrough cache. In this mode,
* there is no direct reduction in swap writes, but a frontswap backend
* can unilaterally "reclaim" any pages in use with no data loss, thus
* providing increases control over maximum memory usage due to frontswap.
*/
static bool frontswap_writethrough_enabled __read_mostly;
/*
* If enabled, the underlying tmem implementation is capable of doing
* exclusive gets, so frontswap_load, on a successful tmem_get must
* mark the page as no longer in frontswap AND mark it dirty.
*/
static bool frontswap_tmem_exclusive_gets_enabled __read_mostly;
static const struct frontswap_ops *frontswap_ops __read_mostly;
#ifdef CONFIG_DEBUG_FS
/*
@@ -114,87 +94,22 @@ static inline void inc_frontswap_invalidates(void) { }
/*
* Register operations for frontswap
*/
void frontswap_register_ops(struct frontswap_ops *ops)
int frontswap_register_ops(const struct frontswap_ops *ops)
{
DECLARE_BITMAP(a, MAX_SWAPFILES);
DECLARE_BITMAP(b, MAX_SWAPFILES);
struct swap_info_struct *si;
unsigned int i;
bitmap_zero(a, MAX_SWAPFILES);
bitmap_zero(b, MAX_SWAPFILES);
spin_lock(&swap_lock);
plist_for_each_entry(si, &swap_active_head, list) {
if (!WARN_ON(!si->frontswap_map))
__set_bit(si->type, a);
}
spin_unlock(&swap_lock);
/* the new ops needs to know the currently active swap devices */
for_each_set_bit(i, a, MAX_SWAPFILES)
ops->init(i);
/*
* Setting frontswap_ops must happen after the ops->init() calls
* above; cmpxchg implies smp_mb() which will ensure the init is
* complete at this point.
*/
do {
ops->next = frontswap_ops;
} while (cmpxchg(&frontswap_ops, ops->next, ops) != ops->next);
if (frontswap_ops)
return -EINVAL;
frontswap_ops = ops;
static_branch_inc(&frontswap_enabled_key);
spin_lock(&swap_lock);
plist_for_each_entry(si, &swap_active_head, list) {
if (si->frontswap_map)
__set_bit(si->type, b);
}
spin_unlock(&swap_lock);
/*
* On the very unlikely chance that a swap device was added or
* removed between setting the "a" list bits and the ops init
* calls, we re-check and do init or invalidate for any changed
* bits.
*/
if (unlikely(!bitmap_equal(a, b, MAX_SWAPFILES))) {
for (i = 0; i < MAX_SWAPFILES; i++) {
if (!test_bit(i, a) && test_bit(i, b))
ops->init(i);
else if (test_bit(i, a) && !test_bit(i, b))
ops->invalidate_area(i);
}
}
return 0;
}
EXPORT_SYMBOL(frontswap_register_ops);
/*
* Enable/disable frontswap writethrough (see above).
*/
void frontswap_writethrough(bool enable)
{
frontswap_writethrough_enabled = enable;
}
EXPORT_SYMBOL(frontswap_writethrough);
/*
* Enable/disable frontswap exclusive gets (see above).
*/
void frontswap_tmem_exclusive_gets(bool enable)
{
frontswap_tmem_exclusive_gets_enabled = enable;
}
EXPORT_SYMBOL(frontswap_tmem_exclusive_gets);
/*
* Called when a swap device is swapon'd.
*/
void __frontswap_init(unsigned type, unsigned long *map)
void frontswap_init(unsigned type, unsigned long *map)
{
struct swap_info_struct *sis = swap_info[type];
struct frontswap_ops *ops;
VM_BUG_ON(sis == NULL);
@@ -210,20 +125,16 @@ void __frontswap_init(unsigned type, unsigned long *map)
* p->frontswap set to something valid to work properly.
*/
frontswap_map_set(sis, map);
for_each_frontswap_ops(ops)
ops->init(type);
frontswap_ops->init(type);
}
EXPORT_SYMBOL(__frontswap_init);
bool __frontswap_test(struct swap_info_struct *sis,
static bool __frontswap_test(struct swap_info_struct *sis,
pgoff_t offset)
{
if (sis->frontswap_map)
return test_bit(offset, sis->frontswap_map);
return false;
}
EXPORT_SYMBOL(__frontswap_test);
static inline void __frontswap_set(struct swap_info_struct *sis,
pgoff_t offset)
@@ -253,7 +164,6 @@ int __frontswap_store(struct page *page)
int type = swp_type(entry);
struct swap_info_struct *sis = swap_info[type];
pgoff_t offset = swp_offset(entry);
struct frontswap_ops *ops;
VM_BUG_ON(!frontswap_ops);
VM_BUG_ON(!PageLocked(page));
@@ -267,28 +177,19 @@ int __frontswap_store(struct page *page)
*/
if (__frontswap_test(sis, offset)) {
__frontswap_clear(sis, offset);
for_each_frontswap_ops(ops)
ops->invalidate_page(type, offset);
frontswap_ops->invalidate_page(type, offset);
}
/* Try to store in each implementation, until one succeeds. */
for_each_frontswap_ops(ops) {
ret = ops->store(type, offset, page);
if (!ret) /* successful store */
break;
}
ret = frontswap_ops->store(type, offset, page);
if (ret == 0) {
__frontswap_set(sis, offset);
inc_frontswap_succ_stores();
} else {
inc_frontswap_failed_stores();
}
if (frontswap_writethrough_enabled)
/* report failure so swap also writes to swap device */
ret = -1;
return ret;
}
EXPORT_SYMBOL(__frontswap_store);
/*
* "Get" data from frontswap associated with swaptype and offset that were
@@ -302,7 +203,6 @@ int __frontswap_load(struct page *page)
int type = swp_type(entry);
struct swap_info_struct *sis = swap_info[type];
pgoff_t offset = swp_offset(entry);
struct frontswap_ops *ops;
VM_BUG_ON(!frontswap_ops);
VM_BUG_ON(!PageLocked(page));
@@ -312,21 +212,11 @@ int __frontswap_load(struct page *page)
return -1;
/* Try loading from each implementation, until one succeeds. */
for_each_frontswap_ops(ops) {
ret = ops->load(type, offset, page);
if (!ret) /* successful load */
break;
}
if (ret == 0) {
ret = frontswap_ops->load(type, offset, page);
if (ret == 0)
inc_frontswap_loads();
if (frontswap_tmem_exclusive_gets_enabled) {
SetPageDirty(page);
__frontswap_clear(sis, offset);
}
}
return ret;
}
EXPORT_SYMBOL(__frontswap_load);
/*
* Invalidate any data from frontswap associated with the specified swaptype
@@ -335,7 +225,6 @@ EXPORT_SYMBOL(__frontswap_load);
void __frontswap_invalidate_page(unsigned type, pgoff_t offset)
{
struct swap_info_struct *sis = swap_info[type];
struct frontswap_ops *ops;
VM_BUG_ON(!frontswap_ops);
VM_BUG_ON(sis == NULL);
@@ -343,12 +232,10 @@ void __frontswap_invalidate_page(unsigned type, pgoff_t offset)
if (!__frontswap_test(sis, offset))
return;
for_each_frontswap_ops(ops)
ops->invalidate_page(type, offset);
frontswap_ops->invalidate_page(type, offset);
__frontswap_clear(sis, offset);
inc_frontswap_invalidates();
}
EXPORT_SYMBOL(__frontswap_invalidate_page);
/*
* Invalidate all data from frontswap associated with all offsets for the
@@ -357,7 +244,6 @@ EXPORT_SYMBOL(__frontswap_invalidate_page);
void __frontswap_invalidate_area(unsigned type)
{
struct swap_info_struct *sis = swap_info[type];
struct frontswap_ops *ops;
VM_BUG_ON(!frontswap_ops);
VM_BUG_ON(sis == NULL);
@@ -365,123 +251,10 @@ void __frontswap_invalidate_area(unsigned type)
if (sis->frontswap_map == NULL)
return;
for_each_frontswap_ops(ops)
ops->invalidate_area(type);
frontswap_ops->invalidate_area(type);
atomic_set(&sis->frontswap_pages, 0);
bitmap_zero(sis->frontswap_map, sis->max);
}
EXPORT_SYMBOL(__frontswap_invalidate_area);
static unsigned long __frontswap_curr_pages(void)
{
unsigned long totalpages = 0;
struct swap_info_struct *si = NULL;
assert_spin_locked(&swap_lock);
plist_for_each_entry(si, &swap_active_head, list)
totalpages += atomic_read(&si->frontswap_pages);
return totalpages;
}
static int __frontswap_unuse_pages(unsigned long total, unsigned long *unused,
int *swapid)
{
int ret = -EINVAL;
struct swap_info_struct *si = NULL;
int si_frontswap_pages;
unsigned long total_pages_to_unuse = total;
unsigned long pages = 0, pages_to_unuse = 0;
assert_spin_locked(&swap_lock);
plist_for_each_entry(si, &swap_active_head, list) {
si_frontswap_pages = atomic_read(&si->frontswap_pages);
if (total_pages_to_unuse < si_frontswap_pages) {
pages = pages_to_unuse = total_pages_to_unuse;
} else {
pages = si_frontswap_pages;
pages_to_unuse = 0; /* unuse all */
}
/* ensure there is enough RAM to fetch pages from frontswap */
if (security_vm_enough_memory_mm(current->mm, pages)) {
ret = -ENOMEM;
continue;
}
vm_unacct_memory(pages);
*unused = pages_to_unuse;
*swapid = si->type;
ret = 0;
break;
}
return ret;
}
/*
* Used to check if it's necessary and feasible to unuse pages.
* Return 1 when nothing to do, 0 when need to shrink pages,
* error code when there is an error.
*/
static int __frontswap_shrink(unsigned long target_pages,
unsigned long *pages_to_unuse,
int *type)
{
unsigned long total_pages = 0, total_pages_to_unuse;
assert_spin_locked(&swap_lock);
total_pages = __frontswap_curr_pages();
if (total_pages <= target_pages) {
/* Nothing to do */
*pages_to_unuse = 0;
return 1;
}
total_pages_to_unuse = total_pages - target_pages;
return __frontswap_unuse_pages(total_pages_to_unuse, pages_to_unuse, type);
}
/*
* Frontswap, like a true swap device, may unnecessarily retain pages
* under certain circumstances; "shrink" frontswap is essentially a
* "partial swapoff" and works by calling try_to_unuse to attempt to
* unuse enough frontswap pages to attempt to -- subject to memory
* constraints -- reduce the number of pages in frontswap to the
* number given in the parameter target_pages.
*/
void frontswap_shrink(unsigned long target_pages)
{
unsigned long pages_to_unuse = 0;
int type, ret;
/*
* we don't want to hold swap_lock while doing a very
* lengthy try_to_unuse, but swap_list may change
* so restart scan from swap_active_head each time
*/
spin_lock(&swap_lock);
ret = __frontswap_shrink(target_pages, &pages_to_unuse, &type);
spin_unlock(&swap_lock);
if (ret == 0)
try_to_unuse(type, true, pages_to_unuse);
return;
}
EXPORT_SYMBOL(frontswap_shrink);
/*
* Count and return the number of frontswap pages across all
* swap devices. This is exported so that backend drivers can
* determine current usage without reading debugfs.
*/
unsigned long frontswap_curr_pages(void)
{
unsigned long totalpages = 0;
spin_lock(&swap_lock);
totalpages = __frontswap_curr_pages();
spin_unlock(&swap_lock);
return totalpages;
}
EXPORT_SYMBOL(frontswap_curr_pages);
static int __init init_frontswap(void)
{

33
mm/shmem.c
View File

@@ -36,7 +36,6 @@
#include <linux/uio.h>
#include <linux/khugepaged.h>
#include <linux/hugetlb.h>
#include <linux/frontswap.h>
#include <linux/fs_parser.h>
#include <linux/swapfile.h>
@@ -1152,7 +1151,7 @@ static void shmem_evict_inode(struct inode *inode)
static int shmem_find_swap_entries(struct address_space *mapping,
pgoff_t start, unsigned int nr_entries,
struct page **entries, pgoff_t *indices,
unsigned int type, bool frontswap)
unsigned int type)
{
XA_STATE(xas, &mapping->i_pages, start);
struct page *page;
@@ -1173,9 +1172,6 @@ static int shmem_find_swap_entries(struct address_space *mapping,
entry = radix_to_swp_entry(page);
if (swp_type(entry) != type)
continue;
if (frontswap &&
!frontswap_test(swap_info[type], swp_offset(entry)))
continue;
indices[ret] = xas.xa_index;
entries[ret] = page;
@@ -1228,26 +1224,20 @@ static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
/*
* If swap found in inode, free it and move page from swapcache to filecache.
*/
static int shmem_unuse_inode(struct inode *inode, unsigned int type,
bool frontswap, unsigned long *fs_pages_to_unuse)
static int shmem_unuse_inode(struct inode *inode, unsigned int type)
{
struct address_space *mapping = inode->i_mapping;
pgoff_t start = 0;
struct pagevec pvec;
pgoff_t indices[PAGEVEC_SIZE];
bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0);
int ret = 0;
pagevec_init(&pvec);
do {
unsigned int nr_entries = PAGEVEC_SIZE;
if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE)
nr_entries = *fs_pages_to_unuse;
pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
pvec.pages, indices,
type, frontswap);
pvec.pages, indices, type);
if (pvec.nr == 0) {
ret = 0;
break;
@@ -1257,14 +1247,6 @@ static int shmem_unuse_inode(struct inode *inode, unsigned int type,
if (ret < 0)
break;
if (frontswap_partial) {
*fs_pages_to_unuse -= ret;
if (*fs_pages_to_unuse == 0) {
ret = FRONTSWAP_PAGES_UNUSED;
break;
}
}
start = indices[pvec.nr - 1];
} while (true);
@@ -1276,8 +1258,7 @@ static int shmem_unuse_inode(struct inode *inode, unsigned int type,
* device 'type' back into memory, so the swap device can be
* unused.
*/
int shmem_unuse(unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse)
int shmem_unuse(unsigned int type)
{
struct shmem_inode_info *info, *next;
int error = 0;
@@ -1300,8 +1281,7 @@ int shmem_unuse(unsigned int type, bool frontswap,
atomic_inc(&info->stop_eviction);
mutex_unlock(&shmem_swaplist_mutex);
error = shmem_unuse_inode(&info->vfs_inode, type, frontswap,
fs_pages_to_unuse);
error = shmem_unuse_inode(&info->vfs_inode, type);
cond_resched();
mutex_lock(&shmem_swaplist_mutex);
@@ -4015,8 +3995,7 @@ int __init shmem_init(void)
return 0;
}
int shmem_unuse(unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse)
int shmem_unuse(unsigned int type)
{
return 0;
}

90
mm/swapfile.c
View File

@@ -49,7 +49,7 @@ static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
unsigned char);
static void free_swap_count_continuations(struct swap_info_struct *);
DEFINE_SPINLOCK(swap_lock);
static DEFINE_SPINLOCK(swap_lock);
static unsigned int nr_swapfiles;
atomic_long_t nr_swap_pages;
/*
@@ -71,7 +71,7 @@ static const char Unused_offset[] = "Unused swap offset entry ";
* all active swap_info_structs
* protected with swap_lock, and ordered by priority.
*/
PLIST_HEAD(swap_active_head);
static PLIST_HEAD(swap_active_head);
/*
* all available (active, not full) swap_info_structs
@@ -1923,8 +1923,7 @@ out:
static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
unsigned long addr, unsigned long end,
unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse)
unsigned int type)
{
struct page *page;
swp_entry_t entry;
@@ -1945,9 +1944,6 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
continue;
offset = swp_offset(entry);
if (frontswap && !frontswap_test(si, offset))
continue;
pte_unmap(pte);
swap_map = &si->swap_map[offset];
page = lookup_swap_cache(entry, vma, addr);
@@ -1979,11 +1975,6 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
try_to_free_swap(page);
unlock_page(page);
put_page(page);
if (*fs_pages_to_unuse && !--(*fs_pages_to_unuse)) {
ret = FRONTSWAP_PAGES_UNUSED;
goto out;
}
try_next:
pte = pte_offset_map(pmd, addr);
} while (pte++, addr += PAGE_SIZE, addr != end);
@@ -1996,8 +1987,7 @@ out:
static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
unsigned long addr, unsigned long end,
unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse)
unsigned int type)
{
pmd_t *pmd;
unsigned long next;
@@ -2009,8 +1999,7 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
next = pmd_addr_end(addr, end);
if (pmd_none_or_trans_huge_or_clear_bad(pmd))
continue;
ret = unuse_pte_range(vma, pmd, addr, next, type,
frontswap, fs_pages_to_unuse);
ret = unuse_pte_range(vma, pmd, addr, next, type);
if (ret)
return ret;
} while (pmd++, addr = next, addr != end);
@@ -2019,8 +2008,7 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
unsigned long addr, unsigned long end,
unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse)
unsigned int type)
{
pud_t *pud;
unsigned long next;
@@ -2031,8 +2019,7 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
next = pud_addr_end(addr, end);
if (pud_none_or_clear_bad(pud))
continue;
ret = unuse_pmd_range(vma, pud, addr, next, type,
frontswap, fs_pages_to_unuse);
ret = unuse_pmd_range(vma, pud, addr, next, type);
if (ret)
return ret;
} while (pud++, addr = next, addr != end);
@@ -2041,8 +2028,7 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
unsigned long addr, unsigned long end,
unsigned int type, bool frontswap,
unsigned long *fs_pages_to_unuse)
unsigned int type)
{
p4d_t *p4d;
unsigned long next;
@@ -2053,16 +2039,14 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
next = p4d_addr_end(addr, end);
if (p4d_none_or_clear_bad(p4d))
continue;
ret = unuse_pud_range(vma, p4d, addr, next, type,
frontswap, fs_pages_to_unuse);
ret = unuse_pud_range(vma, p4d, addr, next, type);
if (ret)
return ret;
} while (p4d++, addr = next, addr != end);
return 0;
}
static int unuse_vma(struct vm_area_struct *vma, unsigned int type,
bool frontswap, unsigned long *fs_pages_to_unuse)
static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
{
pgd_t *pgd;
unsigned long addr, end, next;
@@ -2076,16 +2060,14 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type,
next = pgd_addr_end(addr, end);
if (pgd_none_or_clear_bad(pgd))
continue;
ret = unuse_p4d_range(vma, pgd, addr, next, type,
frontswap, fs_pages_to_unuse);
ret = unuse_p4d_range(vma, pgd, addr, next, type);
if (ret)
return ret;
} while (pgd++, addr = next, addr != end);
return 0;
}
static int unuse_mm(struct mm_struct *mm, unsigned int type,
bool frontswap, unsigned long *fs_pages_to_unuse)
static int unuse_mm(struct mm_struct *mm, unsigned int type)
{
struct vm_area_struct *vma;
int ret = 0;
@@ -2093,8 +2075,7 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type,
mmap_read_lock(mm);
for (vma = mm->mmap; vma; vma = vma->vm_next) {
if (vma->anon_vma) {
ret = unuse_vma(vma, type, frontswap,
fs_pages_to_unuse);
ret = unuse_vma(vma, type);
if (ret)
break;
}
@@ -2110,7 +2091,7 @@ static int unuse_mm(struct mm_struct *mm, unsigned int type,
* if there are no inuse entries after prev till end of the map.
*/
static unsigned int find_next_to_unuse(struct swap_info_struct *si,
unsigned int prev, bool frontswap)
unsigned int prev)
{
unsigned int i;
unsigned char count;
@@ -2124,8 +2105,7 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
for (i = prev + 1; i < si->max; i++) {
count = READ_ONCE(si->swap_map[i]);
if (count && swap_count(count) != SWAP_MAP_BAD)
if (!frontswap || frontswap_test(si, i))
break;
break;
if ((i % LATENCY_LIMIT) == 0)
cond_resched();
}
@@ -2136,12 +2116,7 @@ static unsigned int find_next_to_unuse(struct swap_info_struct *si,
return i;
}
/*
* If the boolean frontswap is true, only unuse pages_to_unuse pages;
* pages_to_unuse==0 means all pages; ignored if frontswap is false
*/
int try_to_unuse(unsigned int type, bool frontswap,
unsigned long pages_to_unuse)
static int try_to_unuse(unsigned int type)
{
struct mm_struct *prev_mm;
struct mm_struct *mm;
@@ -2155,13 +2130,10 @@ int try_to_unuse(unsigned int type, bool frontswap,
if (!READ_ONCE(si->inuse_pages))
return 0;
if (!frontswap)
pages_to_unuse = 0;
retry:
retval = shmem_unuse(type, frontswap, &pages_to_unuse);
retval = shmem_unuse(type);
if (retval)
goto out;
return retval;
prev_mm = &init_mm;
mmget(prev_mm);
@@ -2178,11 +2150,10 @@ retry:
spin_unlock(&mmlist_lock);
mmput(prev_mm);
prev_mm = mm;
retval = unuse_mm(mm, type, frontswap, &pages_to_unuse);
retval = unuse_mm(mm, type);
if (retval) {
mmput(prev_mm);
goto out;
return retval;
}
/*
@@ -2199,7 +2170,7 @@ retry:
i = 0;
while (READ_ONCE(si->inuse_pages) &&
!signal_pending(current) &&
(i = find_next_to_unuse(si, i, frontswap)) != 0) {
(i = find_next_to_unuse(si, i)) != 0) {
entry = swp_entry(type, i);
page = find_get_page(swap_address_space(entry), i);
@@ -2217,14 +2188,6 @@ retry:
try_to_free_swap(page);
unlock_page(page);
put_page(page);
/*
* For frontswap, we just need to unuse pages_to_unuse, if
* it was specified. Need not check frontswap again here as
* we already zeroed out pages_to_unuse if not frontswap.
*/
if (pages_to_unuse && --pages_to_unuse == 0)
goto out;
}
/*
@@ -2242,10 +2205,10 @@ retry:
if (READ_ONCE(si->inuse_pages)) {
if (!signal_pending(current))
goto retry;
retval = -EINTR;
return -EINTR;
}
out:
return (retval == FRONTSWAP_PAGES_UNUSED) ? 0 : retval;
return 0;
}
/*
@@ -2463,7 +2426,8 @@ static void enable_swap_info(struct swap_info_struct *p, int prio,
struct swap_cluster_info *cluster_info,
unsigned long *frontswap_map)
{
frontswap_init(p->type, frontswap_map);
if (IS_ENABLED(CONFIG_FRONTSWAP))
frontswap_init(p->type, frontswap_map);
spin_lock(&swap_lock);
spin_lock(&p->lock);
setup_swap_info(p, prio, swap_map, cluster_info);
@@ -2576,7 +2540,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
disable_swap_slots_cache_lock();
set_current_oom_origin();
err = try_to_unuse(p->type, false, 0); /* force unuse all pages */
err = try_to_unuse(p->type);
clear_current_oom_origin();
if (err) {

8
mm/zswap.c
View File

@@ -1378,7 +1378,7 @@ static void zswap_frontswap_init(unsigned type)
zswap_trees[type] = tree;
}
static struct frontswap_ops zswap_frontswap_ops = {
static const struct frontswap_ops zswap_frontswap_ops = {
.store = zswap_frontswap_store,
.load = zswap_frontswap_load,
.invalidate_page = zswap_frontswap_invalidate_page,
@@ -1475,11 +1475,15 @@ static int __init init_zswap(void)
if (!shrink_wq)
goto fallback_fail;
frontswap_register_ops(&zswap_frontswap_ops);
ret = frontswap_register_ops(&zswap_frontswap_ops);
if (ret)
goto destroy_wq;
if (zswap_debugfs_init())
pr_warn("debugfs initialization failed\n");
return 0;
destroy_wq:
destroy_workqueue(shrink_wq);
fallback_fail:
if (pool)
zswap_pool_destroy(pool);