swapfile.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *  linux/mm/swapfile.c
   4 *
   5 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   6 *  Swap reorganised 29.12.95, Stephen Tweedie
   7 */
   8
   9#include <linux/blkdev.h>
  10#include <linux/mm.h>
  11#include <linux/sched/mm.h>
  12#include <linux/sched/task.h>
  13#include <linux/hugetlb.h>
  14#include <linux/mman.h>
  15#include <linux/slab.h>
  16#include <linux/kernel_stat.h>
  17#include <linux/swap.h>
  18#include <linux/vmalloc.h>
  19#include <linux/pagemap.h>
  20#include <linux/namei.h>
  21#include <linux/shmem_fs.h>
  22#include <linux/blk-cgroup.h>
  23#include <linux/random.h>
  24#include <linux/writeback.h>
  25#include <linux/proc_fs.h>
  26#include <linux/seq_file.h>
  27#include <linux/init.h>
  28#include <linux/ksm.h>
  29#include <linux/rmap.h>
  30#include <linux/security.h>
  31#include <linux/backing-dev.h>
  32#include <linux/mutex.h>
  33#include <linux/capability.h>
  34#include <linux/syscalls.h>
  35#include <linux/memcontrol.h>
  36#include <linux/poll.h>
  37#include <linux/oom.h>
  38#include <linux/swapfile.h>
  39#include <linux/export.h>
  40#include <linux/sort.h>
  41#include <linux/completion.h>
  42#include <linux/suspend.h>
  43#include <linux/zswap.h>
  44#include <linux/plist.h>
  45
  46#include <asm/tlbflush.h>
  47#include <linux/leafops.h>
  48#include <linux/swap_cgroup.h>
  49#include "swap_table.h"
  50#include "internal.h"
  51#include "swap.h"
  52
  53static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
  54				 unsigned char);
  55static void free_swap_count_continuations(struct swap_info_struct *);
  56static void swap_entries_free(struct swap_info_struct *si,
  57			      struct swap_cluster_info *ci,
  58			      swp_entry_t entry, unsigned int nr_pages);
  59static void swap_range_alloc(struct swap_info_struct *si,
  60			     unsigned int nr_entries);
  61static bool folio_swapcache_freeable(struct folio *folio);
  62static void move_cluster(struct swap_info_struct *si,
  63			 struct swap_cluster_info *ci, struct list_head *list,
  64			 enum swap_cluster_flags new_flags);
  65
  66static DEFINE_SPINLOCK(swap_lock);
  67static unsigned int nr_swapfiles;
  68atomic_long_t nr_swap_pages;
  69/*
  70 * Some modules use swappable objects and may try to swap them out under
  71 * memory pressure (via the shrinker). Before doing so, they may wish to
  72 * check to see if any swap space is available.
  73 */
  74EXPORT_SYMBOL_GPL(nr_swap_pages);
  75/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
  76long total_swap_pages;
  77#define DEF_SWAP_PRIO  -1
  78unsigned long swapfile_maximum_size;
  79#ifdef CONFIG_MIGRATION
  80bool swap_migration_ad_supported;
  81#endif	/* CONFIG_MIGRATION */
  82
  83static const char Bad_file[] = "Bad swap file entry ";
  84static const char Unused_file[] = "Unused swap file entry ";
  85static const char Bad_offset[] = "Bad swap offset entry ";
  86static const char Unused_offset[] = "Unused swap offset entry ";
  87
  88/*
  89 * all active swap_info_structs
  90 * protected with swap_lock, and ordered by priority.
  91 */
  92static PLIST_HEAD(swap_active_head);
  93
  94/*
  95 * all available (active, not full) swap_info_structs
  96 * protected with swap_avail_lock, ordered by priority.
  97 * This is used by folio_alloc_swap() instead of swap_active_head
  98 * because swap_active_head includes all swap_info_structs,
  99 * but folio_alloc_swap() doesn't need to look at full ones.
 100 * This uses its own lock instead of swap_lock because when a
 101 * swap_info_struct changes between not-full/full, it needs to
 102 * add/remove itself to/from this list, but the swap_info_struct->lock
 103 * is held and the locking order requires swap_lock to be taken
 104 * before any swap_info_struct->lock.
 105 */
 106static PLIST_HEAD(swap_avail_head);
 107static DEFINE_SPINLOCK(swap_avail_lock);
 108
 109struct swap_info_struct *swap_info[MAX_SWAPFILES];
 110
 111static struct kmem_cache *swap_table_cachep;
 112
 113static DEFINE_MUTEX(swapon_mutex);
 114
 115static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
 116/* Activity counter to indicate that a swapon or swapoff has occurred */
 117static atomic_t proc_poll_event = ATOMIC_INIT(0);
 118
 119atomic_t nr_rotate_swap = ATOMIC_INIT(0);
 120
 121struct percpu_swap_cluster {
 122	struct swap_info_struct *si[SWAP_NR_ORDERS];
 123	unsigned long offset[SWAP_NR_ORDERS];
 124	local_lock_t lock;
 125};
 126
 127static DEFINE_PER_CPU(struct percpu_swap_cluster, percpu_swap_cluster) = {
 128	.si = { NULL },
 129	.offset = { SWAP_ENTRY_INVALID },
 130	.lock = INIT_LOCAL_LOCK(),
 131};
 132
 133/* May return NULL on invalid type, caller must check for NULL return */
 134static struct swap_info_struct *swap_type_to_info(int type)
 135{
 136	if (type >= MAX_SWAPFILES)
 137		return NULL;
 138	return READ_ONCE(swap_info[type]); /* rcu_dereference() */
 139}
 140
 141/* May return NULL on invalid entry, caller must check for NULL return */
 142static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry)
 143{
 144	return swap_type_to_info(swp_type(entry));
 145}
 146
 147static inline unsigned char swap_count(unsigned char ent)
 148{
 149	return ent & ~SWAP_HAS_CACHE;	/* may include COUNT_CONTINUED flag */
 150}
 151
 152/*
 153 * Use the second highest bit of inuse_pages counter as the indicator
 154 * if one swap device is on the available plist, so the atomic can
 155 * still be updated arithmetically while having special data embedded.
 156 *
 157 * inuse_pages counter is the only thing indicating if a device should
 158 * be on avail_lists or not (except swapon / swapoff). By embedding the
 159 * off-list bit in the atomic counter, updates no longer need any lock
 160 * to check the list status.
 161 *
 162 * This bit will be set if the device is not on the plist and not
 163 * usable, will be cleared if the device is on the plist.
 164 */
 165#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2))
 166#define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT)
 167static long swap_usage_in_pages(struct swap_info_struct *si)
 168{
 169	return atomic_long_read(&si->inuse_pages) & SWAP_USAGE_COUNTER_MASK;
 170}
 171
 172/* Reclaim the swap entry anyway if possible */
 173#define TTRS_ANYWAY		0x1
 174/*
 175 * Reclaim the swap entry if there are no more mappings of the
 176 * corresponding page
 177 */
 178#define TTRS_UNMAPPED		0x2
 179/* Reclaim the swap entry if swap is getting full */
 180#define TTRS_FULL		0x4
 181
 182static bool swap_only_has_cache(struct swap_info_struct *si,
 183			      unsigned long offset, int nr_pages)
 184{
 185	unsigned char *map = si->swap_map + offset;
 186	unsigned char *map_end = map + nr_pages;
 187
 188	do {
 189		VM_BUG_ON(!(*map & SWAP_HAS_CACHE));
 190		if (*map != SWAP_HAS_CACHE)
 191			return false;
 192	} while (++map < map_end);
 193
 194	return true;
 195}
 196
 197static bool swap_is_last_map(struct swap_info_struct *si,
 198		unsigned long offset, int nr_pages, bool *has_cache)
 199{
 200	unsigned char *map = si->swap_map + offset;
 201	unsigned char *map_end = map + nr_pages;
 202	unsigned char count = *map;
 203
 204	if (swap_count(count) != 1 && swap_count(count) != SWAP_MAP_SHMEM)
 205		return false;
 206
 207	while (++map < map_end) {
 208		if (*map != count)
 209			return false;
 210	}
 211
 212	*has_cache = !!(count & SWAP_HAS_CACHE);
 213	return true;
 214}
 215
 216/*
 217 * returns number of pages in the folio that backs the swap entry. If positive,
 218 * the folio was reclaimed. If negative, the folio was not reclaimed. If 0, no
 219 * folio was associated with the swap entry.
 220 */
 221static int __try_to_reclaim_swap(struct swap_info_struct *si,
 222				 unsigned long offset, unsigned long flags)
 223{
 224	const swp_entry_t entry = swp_entry(si->type, offset);
 225	struct swap_cluster_info *ci;
 226	struct folio *folio;
 227	int ret, nr_pages;
 228	bool need_reclaim;
 229
 230again:
 231	folio = swap_cache_get_folio(entry);
 232	if (!folio)
 233		return 0;
 234
 235	nr_pages = folio_nr_pages(folio);
 236	ret = -nr_pages;
 237
 238	/*
 239	 * We hold a folio lock here. We have to use trylock for
 240	 * avoiding deadlock. This is a special case and you should
 241	 * use folio_free_swap() with explicit folio_lock() in usual
 242	 * operations.
 243	 */
 244	if (!folio_trylock(folio))
 245		goto out;
 246
 247	/*
 248	 * Offset could point to the middle of a large folio, or folio
 249	 * may no longer point to the expected offset before it's locked.
 250	 */
 251	if (!folio_matches_swap_entry(folio, entry)) {
 252		folio_unlock(folio);
 253		folio_put(folio);
 254		goto again;
 255	}
 256	offset = swp_offset(folio->swap);
 257
 258	need_reclaim = ((flags & TTRS_ANYWAY) ||
 259			((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) ||
 260			((flags & TTRS_FULL) && mem_cgroup_swap_full(folio)));
 261	if (!need_reclaim || !folio_swapcache_freeable(folio))
 262		goto out_unlock;
 263
 264	/*
 265	 * It's safe to delete the folio from swap cache only if the folio's
 266	 * swap_map is HAS_CACHE only, which means the slots have no page table
 267	 * reference or pending writeback, and can't be allocated to others.
 268	 */
 269	ci = swap_cluster_lock(si, offset);
 270	need_reclaim = swap_only_has_cache(si, offset, nr_pages);
 271	swap_cluster_unlock(ci);
 272	if (!need_reclaim)
 273		goto out_unlock;
 274
 275	swap_cache_del_folio(folio);
 276	folio_set_dirty(folio);
 277	ret = nr_pages;
 278out_unlock:
 279	folio_unlock(folio);
 280out:
 281	folio_put(folio);
 282	return ret;
 283}
 284
 285static inline struct swap_extent *first_se(struct swap_info_struct *sis)
 286{
 287	struct rb_node *rb = rb_first(&sis->swap_extent_root);
 288	return rb_entry(rb, struct swap_extent, rb_node);
 289}
 290
 291static inline struct swap_extent *next_se(struct swap_extent *se)
 292{
 293	struct rb_node *rb = rb_next(&se->rb_node);
 294	return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL;
 295}
 296
 297/*
 298 * swapon tell device that all the old swap contents can be discarded,
 299 * to allow the swap device to optimize its wear-levelling.
 300 */
 301static int discard_swap(struct swap_info_struct *si)
 302{
 303	struct swap_extent *se;
 304	sector_t start_block;
 305	sector_t nr_blocks;
 306	int err = 0;
 307
 308	/* Do not discard the swap header page! */
 309	se = first_se(si);
 310	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
 311	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
 312	if (nr_blocks) {
 313		err = blkdev_issue_discard(si->bdev, start_block,
 314				nr_blocks, GFP_KERNEL);
 315		if (err)
 316			return err;
 317		cond_resched();
 318	}
 319
 320	for (se = next_se(se); se; se = next_se(se)) {
 321		start_block = se->start_block << (PAGE_SHIFT - 9);
 322		nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
 323
 324		err = blkdev_issue_discard(si->bdev, start_block,
 325				nr_blocks, GFP_KERNEL);
 326		if (err)
 327			break;
 328
 329		cond_resched();
 330	}
 331	return err;		/* That will often be -EOPNOTSUPP */
 332}
 333
 334static struct swap_extent *
 335offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)
 336{
 337	struct swap_extent *se;
 338	struct rb_node *rb;
 339
 340	rb = sis->swap_extent_root.rb_node;
 341	while (rb) {
 342		se = rb_entry(rb, struct swap_extent, rb_node);
 343		if (offset < se->start_page)
 344			rb = rb->rb_left;
 345		else if (offset >= se->start_page + se->nr_pages)
 346			rb = rb->rb_right;
 347		else
 348			return se;
 349	}
 350	/* It *must* be present */
 351	BUG();
 352}
 353
 354sector_t swap_folio_sector(struct folio *folio)
 355{
 356	struct swap_info_struct *sis = __swap_entry_to_info(folio->swap);
 357	struct swap_extent *se;
 358	sector_t sector;
 359	pgoff_t offset;
 360
 361	offset = swp_offset(folio->swap);
 362	se = offset_to_swap_extent(sis, offset);
 363	sector = se->start_block + (offset - se->start_page);
 364	return sector << (PAGE_SHIFT - 9);
 365}
 366
 367/*
 368 * swap allocation tell device that a cluster of swap can now be discarded,
 369 * to allow the swap device to optimize its wear-levelling.
 370 */
 371static void discard_swap_cluster(struct swap_info_struct *si,
 372				 pgoff_t start_page, pgoff_t nr_pages)
 373{
 374	struct swap_extent *se = offset_to_swap_extent(si, start_page);
 375
 376	while (nr_pages) {
 377		pgoff_t offset = start_page - se->start_page;
 378		sector_t start_block = se->start_block + offset;
 379		sector_t nr_blocks = se->nr_pages - offset;
 380
 381		if (nr_blocks > nr_pages)
 382			nr_blocks = nr_pages;
 383		start_page += nr_blocks;
 384		nr_pages -= nr_blocks;
 385
 386		start_block <<= PAGE_SHIFT - 9;
 387		nr_blocks <<= PAGE_SHIFT - 9;
 388		if (blkdev_issue_discard(si->bdev, start_block,
 389					nr_blocks, GFP_NOIO))
 390			break;
 391
 392		se = next_se(se);
 393	}
 394}
 395
 396#define LATENCY_LIMIT		256
 397
 398static inline bool cluster_is_empty(struct swap_cluster_info *info)
 399{
 400	return info->count == 0;
 401}
 402
 403static inline bool cluster_is_discard(struct swap_cluster_info *info)
 404{
 405	return info->flags == CLUSTER_FLAG_DISCARD;
 406}
 407
 408static inline bool cluster_table_is_alloced(struct swap_cluster_info *ci)
 409{
 410	return rcu_dereference_protected(ci->table, lockdep_is_held(&ci->lock));
 411}
 412
 413static inline bool cluster_is_usable(struct swap_cluster_info *ci, int order)
 414{
 415	if (unlikely(ci->flags > CLUSTER_FLAG_USABLE))
 416		return false;
 417	if (!cluster_table_is_alloced(ci))
 418		return false;
 419	if (!order)
 420		return true;
 421	return cluster_is_empty(ci) || order == ci->order;
 422}
 423
 424static inline unsigned int cluster_index(struct swap_info_struct *si,
 425					 struct swap_cluster_info *ci)
 426{
 427	return ci - si->cluster_info;
 428}
 429
 430static inline unsigned int cluster_offset(struct swap_info_struct *si,
 431					  struct swap_cluster_info *ci)
 432{
 433	return cluster_index(si, ci) * SWAPFILE_CLUSTER;
 434}
 435
 436static struct swap_table *swap_table_alloc(gfp_t gfp)
 437{
 438	struct folio *folio;
 439
 440	if (!SWP_TABLE_USE_PAGE)
 441		return kmem_cache_zalloc(swap_table_cachep, gfp);
 442
 443	folio = folio_alloc(gfp | __GFP_ZERO, 0);
 444	if (folio)
 445		return folio_address(folio);
 446	return NULL;
 447}
 448
 449static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
 450{
 451	struct folio *folio;
 452
 453	folio = page_folio(container_of(head, struct page, rcu_head));
 454	folio_put(folio);
 455}
 456
 457static void swap_table_free(struct swap_table *table)
 458{
 459	if (!SWP_TABLE_USE_PAGE) {
 460		kmem_cache_free(swap_table_cachep, table);
 461		return;
 462	}
 463
 464	call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
 465		 swap_table_free_folio_rcu_cb);
 466}
 467
 468static void swap_cluster_free_table(struct swap_cluster_info *ci)
 469{
 470	unsigned int ci_off;
 471	struct swap_table *table;
 472
 473	/* Only empty cluster's table is allow to be freed  */
 474	lockdep_assert_held(&ci->lock);
 475	VM_WARN_ON_ONCE(!cluster_is_empty(ci));
 476	for (ci_off = 0; ci_off < SWAPFILE_CLUSTER; ci_off++)
 477		VM_WARN_ON_ONCE(!swp_tb_is_null(__swap_table_get(ci, ci_off)));
 478	table = (void *)rcu_dereference_protected(ci->table, true);
 479	rcu_assign_pointer(ci->table, NULL);
 480
 481	swap_table_free(table);
 482}
 483
 484/*
 485 * Allocate swap table for one cluster. Attempt an atomic allocation first,
 486 * then fallback to sleeping allocation.
 487 */
 488static struct swap_cluster_info *
 489swap_cluster_alloc_table(struct swap_info_struct *si,
 490			 struct swap_cluster_info *ci)
 491{
 492	struct swap_table *table;
 493
 494	/*
 495	 * Only cluster isolation from the allocator does table allocation.
 496	 * Swap allocator uses percpu clusters and holds the local lock.
 497	 */
 498	lockdep_assert_held(&ci->lock);
 499	lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock);
 500
 501	/* The cluster must be free and was just isolated from the free list. */
 502	VM_WARN_ON_ONCE(ci->flags || !cluster_is_empty(ci));
 503
 504	table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
 505	if (table) {
 506		rcu_assign_pointer(ci->table, table);
 507		return ci;
 508	}
 509
 510	/*
 511	 * Try a sleep allocation. Each isolated free cluster may cause
 512	 * a sleep allocation, but there is a limited number of them, so
 513	 * the potential recursive allocation is limited.
 514	 */
 515	spin_unlock(&ci->lock);
 516	if (!(si->flags & SWP_SOLIDSTATE))
 517		spin_unlock(&si->global_cluster_lock);
 518	local_unlock(&percpu_swap_cluster.lock);
 519
 520	table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | GFP_KERNEL);
 521
 522	/*
 523	 * Back to atomic context. We might have migrated to a new CPU with a
 524	 * usable percpu cluster. But just keep using the isolated cluster to
 525	 * make things easier. Migration indicates a slight change of workload
 526	 * so using a new free cluster might not be a bad idea, and the worst
 527	 * could happen with ignoring the percpu cluster is fragmentation,
 528	 * which is acceptable since this fallback and race is rare.
 529	 */
 530	local_lock(&percpu_swap_cluster.lock);
 531	if (!(si->flags & SWP_SOLIDSTATE))
 532		spin_lock(&si->global_cluster_lock);
 533	spin_lock(&ci->lock);
 534
 535	/* Nothing except this helper should touch a dangling empty cluster. */
 536	if (WARN_ON_ONCE(cluster_table_is_alloced(ci))) {
 537		if (table)
 538			swap_table_free(table);
 539		return ci;
 540	}
 541
 542	if (!table) {
 543		move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
 544		spin_unlock(&ci->lock);
 545		return NULL;
 546	}
 547
 548	rcu_assign_pointer(ci->table, table);
 549	return ci;
 550}
 551
 552static void move_cluster(struct swap_info_struct *si,
 553			 struct swap_cluster_info *ci, struct list_head *list,
 554			 enum swap_cluster_flags new_flags)
 555{
 556	VM_WARN_ON(ci->flags == new_flags);
 557
 558	BUILD_BUG_ON(1 << sizeof(ci->flags) * BITS_PER_BYTE < CLUSTER_FLAG_MAX);
 559	lockdep_assert_held(&ci->lock);
 560
 561	spin_lock(&si->lock);
 562	if (ci->flags == CLUSTER_FLAG_NONE)
 563		list_add_tail(&ci->list, list);
 564	else
 565		list_move_tail(&ci->list, list);
 566	spin_unlock(&si->lock);
 567	ci->flags = new_flags;
 568}
 569
 570/* Add a cluster to discard list and schedule it to do discard */
 571static void swap_cluster_schedule_discard(struct swap_info_struct *si,
 572		struct swap_cluster_info *ci)
 573{
 574	VM_BUG_ON(ci->flags == CLUSTER_FLAG_FREE);
 575	move_cluster(si, ci, &si->discard_clusters, CLUSTER_FLAG_DISCARD);
 576	schedule_work(&si->discard_work);
 577}
 578
 579static void __free_cluster(struct swap_info_struct *si, struct swap_cluster_info *ci)
 580{
 581	swap_cluster_free_table(ci);
 582	move_cluster(si, ci, &si->free_clusters, CLUSTER_FLAG_FREE);
 583	ci->order = 0;
 584}
 585
 586/*
 587 * Isolate and lock the first cluster that is not contented on a list,
 588 * clean its flag before taken off-list. Cluster flag must be in sync
 589 * with list status, so cluster updaters can always know the cluster
 590 * list status without touching si lock.
 591 *
 592 * Note it's possible that all clusters on a list are contented so
 593 * this returns NULL for an non-empty list.
 594 */
 595static struct swap_cluster_info *isolate_lock_cluster(
 596		struct swap_info_struct *si, struct list_head *list)
 597{
 598	struct swap_cluster_info *ci, *found = NULL;
 599
 600	spin_lock(&si->lock);
 601	list_for_each_entry(ci, list, list) {
 602		if (!spin_trylock(&ci->lock))
 603			continue;
 604
 605		/* We may only isolate and clear flags of following lists */
 606		VM_BUG_ON(!ci->flags);
 607		VM_BUG_ON(ci->flags > CLUSTER_FLAG_USABLE &&
 608			  ci->flags != CLUSTER_FLAG_FULL);
 609
 610		list_del(&ci->list);
 611		ci->flags = CLUSTER_FLAG_NONE;
 612		found = ci;
 613		break;
 614	}
 615	spin_unlock(&si->lock);
 616
 617	if (found && !cluster_table_is_alloced(found)) {
 618		/* Only an empty free cluster's swap table can be freed. */
 619		VM_WARN_ON_ONCE(list != &si->free_clusters);
 620		VM_WARN_ON_ONCE(!cluster_is_empty(found));
 621		return swap_cluster_alloc_table(si, found);
 622	}
 623
 624	return found;
 625}
 626
 627/*
 628 * Doing discard actually. After a cluster discard is finished, the cluster
 629 * will be added to free cluster list. Discard cluster is a bit special as
 630 * they don't participate in allocation or reclaim, so clusters marked as
 631 * CLUSTER_FLAG_DISCARD must remain off-list or on discard list.
 632 */
 633static bool swap_do_scheduled_discard(struct swap_info_struct *si)
 634{
 635	struct swap_cluster_info *ci;
 636	bool ret = false;
 637	unsigned int idx;
 638
 639	spin_lock(&si->lock);
 640	while (!list_empty(&si->discard_clusters)) {
 641		ci = list_first_entry(&si->discard_clusters, struct swap_cluster_info, list);
 642		/*
 643		 * Delete the cluster from list to prepare for discard, but keep
 644		 * the CLUSTER_FLAG_DISCARD flag, percpu_swap_cluster could be
 645		 * pointing to it, or ran into by relocate_cluster.
 646		 */
 647		list_del(&ci->list);
 648		idx = cluster_index(si, ci);
 649		spin_unlock(&si->lock);
 650		discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
 651				SWAPFILE_CLUSTER);
 652
 653		spin_lock(&ci->lock);
 654		/*
 655		 * Discard is done, clear its flags as it's off-list, then
 656		 * return the cluster to allocation list.
 657		 */
 658		ci->flags = CLUSTER_FLAG_NONE;
 659		__free_cluster(si, ci);
 660		spin_unlock(&ci->lock);
 661		ret = true;
 662		spin_lock(&si->lock);
 663	}
 664	spin_unlock(&si->lock);
 665	return ret;
 666}
 667
 668static void swap_discard_work(struct work_struct *work)
 669{
 670	struct swap_info_struct *si;
 671
 672	si = container_of(work, struct swap_info_struct, discard_work);
 673
 674	swap_do_scheduled_discard(si);
 675}
 676
 677static void swap_users_ref_free(struct percpu_ref *ref)
 678{
 679	struct swap_info_struct *si;
 680
 681	si = container_of(ref, struct swap_info_struct, users);
 682	complete(&si->comp);
 683}
 684
 685/*
 686 * Must be called after freeing if ci->count == 0, moves the cluster to free
 687 * or discard list.
 688 */
 689static void free_cluster(struct swap_info_struct *si, struct swap_cluster_info *ci)
 690{
 691	VM_BUG_ON(ci->count != 0);
 692	VM_BUG_ON(ci->flags == CLUSTER_FLAG_FREE);
 693	lockdep_assert_held(&ci->lock);
 694
 695	/*
 696	 * If the swap is discardable, prepare discard the cluster
 697	 * instead of free it immediately. The cluster will be freed
 698	 * after discard.
 699	 */
 700	if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
 701	    (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
 702		swap_cluster_schedule_discard(si, ci);
 703		return;
 704	}
 705
 706	__free_cluster(si, ci);
 707}
 708
 709/*
 710 * Must be called after freeing if ci->count != 0, moves the cluster to
 711 * nonfull list.
 712 */
 713static void partial_free_cluster(struct swap_info_struct *si,
 714				 struct swap_cluster_info *ci)
 715{
 716	VM_BUG_ON(!ci->count || ci->count == SWAPFILE_CLUSTER);
 717	lockdep_assert_held(&ci->lock);
 718
 719	if (ci->flags != CLUSTER_FLAG_NONFULL)
 720		move_cluster(si, ci, &si->nonfull_clusters[ci->order],
 721			     CLUSTER_FLAG_NONFULL);
 722}
 723
 724/*
 725 * Must be called after allocation, moves the cluster to full or frag list.
 726 * Note: allocation doesn't acquire si lock, and may drop the ci lock for
 727 * reclaim, so the cluster could be any where when called.
 728 */
 729static void relocate_cluster(struct swap_info_struct *si,
 730			     struct swap_cluster_info *ci)
 731{
 732	lockdep_assert_held(&ci->lock);
 733
 734	/* Discard cluster must remain off-list or on discard list */
 735	if (cluster_is_discard(ci))
 736		return;
 737
 738	if (!ci->count) {
 739		if (ci->flags != CLUSTER_FLAG_FREE)
 740			free_cluster(si, ci);
 741	} else if (ci->count != SWAPFILE_CLUSTER) {
 742		if (ci->flags != CLUSTER_FLAG_FRAG)
 743			move_cluster(si, ci, &si->frag_clusters[ci->order],
 744				     CLUSTER_FLAG_FRAG);
 745	} else {
 746		if (ci->flags != CLUSTER_FLAG_FULL)
 747			move_cluster(si, ci, &si->full_clusters,
 748				     CLUSTER_FLAG_FULL);
 749	}
 750}
 751
 752/*
 753 * The cluster corresponding to @offset will be accounted as having one bad
 754 * slot. The cluster will not be added to the free cluster list, and its
 755 * usage counter will be increased by 1. Only used for initialization.
 756 */
 757static int swap_cluster_setup_bad_slot(struct swap_cluster_info *cluster_info,
 758				       unsigned long offset)
 759{
 760	unsigned long idx = offset / SWAPFILE_CLUSTER;
 761	struct swap_table *table;
 762	struct swap_cluster_info *ci;
 763
 764	ci = cluster_info + idx;
 765	if (!ci->table) {
 766		table = swap_table_alloc(GFP_KERNEL);
 767		if (!table)
 768			return -ENOMEM;
 769		rcu_assign_pointer(ci->table, table);
 770	}
 771
 772	ci->count++;
 773
 774	WARN_ON(ci->count > SWAPFILE_CLUSTER);
 775	WARN_ON(ci->flags);
 776
 777	return 0;
 778}
 779
 780static bool cluster_reclaim_range(struct swap_info_struct *si,
 781				  struct swap_cluster_info *ci,
 782				  unsigned long start, unsigned long end)
 783{
 784	unsigned char *map = si->swap_map;
 785	unsigned long offset = start;
 786	int nr_reclaim;
 787
 788	spin_unlock(&ci->lock);
 789	do {
 790		switch (READ_ONCE(map[offset])) {
 791		case 0:
 792			offset++;
 793			break;
 794		case SWAP_HAS_CACHE:
 795			nr_reclaim = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
 796			if (nr_reclaim > 0)
 797				offset += nr_reclaim;
 798			else
 799				goto out;
 800			break;
 801		default:
 802			goto out;
 803		}
 804	} while (offset < end);
 805out:
 806	spin_lock(&ci->lock);
 807	/*
 808	 * Recheck the range no matter reclaim succeeded or not, the slot
 809	 * could have been be freed while we are not holding the lock.
 810	 */
 811	for (offset = start; offset < end; offset++)
 812		if (READ_ONCE(map[offset]))
 813			return false;
 814
 815	return true;
 816}
 817
 818static bool cluster_scan_range(struct swap_info_struct *si,
 819			       struct swap_cluster_info *ci,
 820			       unsigned long start, unsigned int nr_pages,
 821			       bool *need_reclaim)
 822{
 823	unsigned long offset, end = start + nr_pages;
 824	unsigned char *map = si->swap_map;
 825
 826	if (cluster_is_empty(ci))
 827		return true;
 828
 829	for (offset = start; offset < end; offset++) {
 830		switch (READ_ONCE(map[offset])) {
 831		case 0:
 832			continue;
 833		case SWAP_HAS_CACHE:
 834			if (!vm_swap_full())
 835				return false;
 836			*need_reclaim = true;
 837			continue;
 838		default:
 839			return false;
 840		}
 841	}
 842
 843	return true;
 844}
 845
 846/*
 847 * Currently, the swap table is not used for count tracking, just
 848 * do a sanity check here to ensure nothing leaked, so the swap
 849 * table should be empty upon freeing.
 850 */
 851static void swap_cluster_assert_table_empty(struct swap_cluster_info *ci,
 852				unsigned int start, unsigned int nr)
 853{
 854	unsigned int ci_off = start % SWAPFILE_CLUSTER;
 855	unsigned int ci_end = ci_off + nr;
 856	unsigned long swp_tb;
 857
 858	if (IS_ENABLED(CONFIG_DEBUG_VM)) {
 859		do {
 860			swp_tb = __swap_table_get(ci, ci_off);
 861			VM_WARN_ON_ONCE(!swp_tb_is_null(swp_tb));
 862		} while (++ci_off < ci_end);
 863	}
 864}
 865
 866static bool cluster_alloc_range(struct swap_info_struct *si, struct swap_cluster_info *ci,
 867				unsigned int start, unsigned char usage,
 868				unsigned int order)
 869{
 870	unsigned int nr_pages = 1 << order;
 871
 872	lockdep_assert_held(&ci->lock);
 873
 874	if (!(si->flags & SWP_WRITEOK))
 875		return false;
 876
 877	/*
 878	 * The first allocation in a cluster makes the
 879	 * cluster exclusive to this order
 880	 */
 881	if (cluster_is_empty(ci))
 882		ci->order = order;
 883
 884	memset(si->swap_map + start, usage, nr_pages);
 885	swap_cluster_assert_table_empty(ci, start, nr_pages);
 886	swap_range_alloc(si, nr_pages);
 887	ci->count += nr_pages;
 888
 889	return true;
 890}
 891
 892/* Try use a new cluster for current CPU and allocate from it. */
 893static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
 894					    struct swap_cluster_info *ci,
 895					    unsigned long offset,
 896					    unsigned int order,
 897					    unsigned char usage)
 898{
 899	unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
 900	unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER);
 901	unsigned long end = min(start + SWAPFILE_CLUSTER, si->max);
 902	unsigned int nr_pages = 1 << order;
 903	bool need_reclaim, ret;
 904
 905	lockdep_assert_held(&ci->lock);
 906
 907	if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER)
 908		goto out;
 909
 910	for (end -= nr_pages; offset <= end; offset += nr_pages) {
 911		need_reclaim = false;
 912		if (!cluster_scan_range(si, ci, offset, nr_pages, &need_reclaim))
 913			continue;
 914		if (need_reclaim) {
 915			ret = cluster_reclaim_range(si, ci, offset, offset + nr_pages);
 916			/*
 917			 * Reclaim drops ci->lock and cluster could be used
 918			 * by another order. Not checking flag as off-list
 919			 * cluster has no flag set, and change of list
 920			 * won't cause fragmentation.
 921			 */
 922			if (!cluster_is_usable(ci, order))
 923				goto out;
 924			if (cluster_is_empty(ci))
 925				offset = start;
 926			/* Reclaim failed but cluster is usable, try next */
 927			if (!ret)
 928				continue;
 929		}
 930		if (!cluster_alloc_range(si, ci, offset, usage, order))
 931			break;
 932		found = offset;
 933		offset += nr_pages;
 934		if (ci->count < SWAPFILE_CLUSTER && offset <= end)
 935			next = offset;
 936		break;
 937	}
 938out:
 939	relocate_cluster(si, ci);
 940	swap_cluster_unlock(ci);
 941	if (si->flags & SWP_SOLIDSTATE) {
 942		this_cpu_write(percpu_swap_cluster.offset[order], next);
 943		this_cpu_write(percpu_swap_cluster.si[order], si);
 944	} else {
 945		si->global_cluster->next[order] = next;
 946	}
 947	return found;
 948}
 949
 950static unsigned int alloc_swap_scan_list(struct swap_info_struct *si,
 951					 struct list_head *list,
 952					 unsigned int order,
 953					 unsigned char usage,
 954					 bool scan_all)
 955{
 956	unsigned int found = SWAP_ENTRY_INVALID;
 957
 958	do {
 959		struct swap_cluster_info *ci = isolate_lock_cluster(si, list);
 960		unsigned long offset;
 961
 962		if (!ci)
 963			break;
 964		offset = cluster_offset(si, ci);
 965		found = alloc_swap_scan_cluster(si, ci, offset, order, usage);
 966		if (found)
 967			break;
 968	} while (scan_all);
 969
 970	return found;
 971}
 972
 973static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force)
 974{
 975	long to_scan = 1;
 976	unsigned long offset, end;
 977	struct swap_cluster_info *ci;
 978	unsigned char *map = si->swap_map;
 979	int nr_reclaim;
 980
 981	if (force)
 982		to_scan = swap_usage_in_pages(si) / SWAPFILE_CLUSTER;
 983
 984	while ((ci = isolate_lock_cluster(si, &si->full_clusters))) {
 985		offset = cluster_offset(si, ci);
 986		end = min(si->max, offset + SWAPFILE_CLUSTER);
 987		to_scan--;
 988
 989		while (offset < end) {
 990			if (READ_ONCE(map[offset]) == SWAP_HAS_CACHE) {
 991				spin_unlock(&ci->lock);
 992				nr_reclaim = __try_to_reclaim_swap(si, offset,
 993								   TTRS_ANYWAY);
 994				spin_lock(&ci->lock);
 995				if (nr_reclaim) {
 996					offset += abs(nr_reclaim);
 997					continue;
 998				}
 999			}
1000			offset++;
1001		}
1002
1003		/* in case no swap cache is reclaimed */
1004		if (ci->flags == CLUSTER_FLAG_NONE)
1005			relocate_cluster(si, ci);
1006
1007		swap_cluster_unlock(ci);
1008		if (to_scan <= 0)
1009			break;
1010	}
1011}
1012
1013static void swap_reclaim_work(struct work_struct *work)
1014{
1015	struct swap_info_struct *si;
1016
1017	si = container_of(work, struct swap_info_struct, reclaim_work);
1018
1019	swap_reclaim_full_clusters(si, true);
1020}
1021
1022/*
1023 * Try to allocate swap entries with specified order and try set a new
1024 * cluster for current CPU too.
1025 */
1026static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int order,
1027					      unsigned char usage)
1028{
1029	struct swap_cluster_info *ci;
1030	unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
1031
1032	/*
1033	 * Swapfile is not block device so unable
1034	 * to allocate large entries.
1035	 */
1036	if (order && !(si->flags & SWP_BLKDEV))
1037		return 0;
1038
1039	if (!(si->flags & SWP_SOLIDSTATE)) {
1040		/* Serialize HDD SWAP allocation for each device. */
1041		spin_lock(&si->global_cluster_lock);
1042		offset = si->global_cluster->next[order];
1043		if (offset == SWAP_ENTRY_INVALID)
1044			goto new_cluster;
1045
1046		ci = swap_cluster_lock(si, offset);
1047		/* Cluster could have been used by another order */
1048		if (cluster_is_usable(ci, order)) {
1049			if (cluster_is_empty(ci))
1050				offset = cluster_offset(si, ci);
1051			found = alloc_swap_scan_cluster(si, ci, offset,
1052							order, usage);
1053		} else {
1054			swap_cluster_unlock(ci);
1055		}
1056		if (found)
1057			goto done;
1058	}
1059
1060new_cluster:
1061	/*
1062	 * If the device need discard, prefer new cluster over nonfull
1063	 * to spread out the writes.
1064	 */
1065	if (si->flags & SWP_PAGE_DISCARD) {
1066		found = alloc_swap_scan_list(si, &si->free_clusters, order, usage,
1067					     false);
1068		if (found)
1069			goto done;
1070	}
1071
1072	if (order < PMD_ORDER) {
1073		found = alloc_swap_scan_list(si, &si->nonfull_clusters[order],
1074					     order, usage, true);
1075		if (found)
1076			goto done;
1077	}
1078
1079	if (!(si->flags & SWP_PAGE_DISCARD)) {
1080		found = alloc_swap_scan_list(si, &si->free_clusters, order, usage,
1081					     false);
1082		if (found)
1083			goto done;
1084	}
1085
1086	/* Try reclaim full clusters if free and nonfull lists are drained */
1087	if (vm_swap_full())
1088		swap_reclaim_full_clusters(si, false);
1089
1090	if (order < PMD_ORDER) {
1091		/*
1092		 * Scan only one fragment cluster is good enough. Order 0
1093		 * allocation will surely success, and large allocation
1094		 * failure is not critical. Scanning one cluster still
1095		 * keeps the list rotated and reclaimed (for HAS_CACHE).
1096		 */
1097		found = alloc_swap_scan_list(si, &si->frag_clusters[order], order,
1098					     usage, false);
1099		if (found)
1100			goto done;
1101	}
1102
1103	if (order)
1104		goto done;
1105
1106	/* Order 0 stealing from higher order */
1107	for (int o = 1; o < SWAP_NR_ORDERS; o++) {
1108		/*
1109		 * Clusters here have at least one usable slots and can't fail order 0
1110		 * allocation, but reclaim may drop si->lock and race with another user.
1111		 */
1112		found = alloc_swap_scan_list(si, &si->frag_clusters[o],
1113					     0, usage, true);
1114		if (found)
1115			goto done;
1116
1117		found = alloc_swap_scan_list(si, &si->nonfull_clusters[o],
1118					     0, usage, true);
1119		if (found)
1120			goto done;
1121	}
1122done:
1123	if (!(si->flags & SWP_SOLIDSTATE))
1124		spin_unlock(&si->global_cluster_lock);
1125
1126	return found;
1127}
1128
1129/* SWAP_USAGE_OFFLIST_BIT can only be set by this helper. */
1130static void del_from_avail_list(struct swap_info_struct *si, bool swapoff)
1131{
1132	unsigned long pages;
1133
1134	spin_lock(&swap_avail_lock);
1135
1136	if (swapoff) {
1137		/*
1138		 * Forcefully remove it. Clear the SWP_WRITEOK flags for
1139		 * swapoff here so it's synchronized by both si->lock and
1140		 * swap_avail_lock, to ensure the result can be seen by
1141		 * add_to_avail_list.
1142		 */
1143		lockdep_assert_held(&si->lock);
1144		si->flags &= ~SWP_WRITEOK;
1145		atomic_long_or(SWAP_USAGE_OFFLIST_BIT, &si->inuse_pages);
1146	} else {
1147		/*
1148		 * If not called by swapoff, take it off-list only if it's
1149		 * full and SWAP_USAGE_OFFLIST_BIT is not set (strictly
1150		 * si->inuse_pages == pages), any concurrent slot freeing,
1151		 * or device already removed from plist by someone else
1152		 * will make this return false.
1153		 */
1154		pages = si->pages;
1155		if (!atomic_long_try_cmpxchg(&si->inuse_pages, &pages,
1156					     pages | SWAP_USAGE_OFFLIST_BIT))
1157			goto skip;
1158	}
1159
1160	plist_del(&si->avail_list, &swap_avail_head);
1161
1162skip:
1163	spin_unlock(&swap_avail_lock);
1164}
1165
1166/* SWAP_USAGE_OFFLIST_BIT can only be cleared by this helper. */
1167static void add_to_avail_list(struct swap_info_struct *si, bool swapon)
1168{
1169	long val;
1170	unsigned long pages;
1171
1172	spin_lock(&swap_avail_lock);
1173
1174	/* Corresponding to SWP_WRITEOK clearing in del_from_avail_list */
1175	if (swapon) {
1176		lockdep_assert_held(&si->lock);
1177		si->flags |= SWP_WRITEOK;
1178	} else {
1179		if (!(READ_ONCE(si->flags) & SWP_WRITEOK))
1180			goto skip;
1181	}
1182
1183	if (!(atomic_long_read(&si->inuse_pages) & SWAP_USAGE_OFFLIST_BIT))
1184		goto skip;
1185
1186	val = atomic_long_fetch_and_relaxed(~SWAP_USAGE_OFFLIST_BIT, &si->inuse_pages);
1187
1188	/*
1189	 * When device is full and device is on the plist, only one updater will
1190	 * see (inuse_pages == si->pages) and will call del_from_avail_list. If
1191	 * that updater happen to be here, just skip adding.
1192	 */
1193	pages = si->pages;
1194	if (val == pages) {
1195		/* Just like the cmpxchg in del_from_avail_list */
1196		if (atomic_long_try_cmpxchg(&si->inuse_pages, &pages,
1197					    pages | SWAP_USAGE_OFFLIST_BIT))
1198			goto skip;
1199	}
1200
1201	plist_add(&si->avail_list, &swap_avail_head);
1202
1203skip:
1204	spin_unlock(&swap_avail_lock);
1205}
1206
1207/*
1208 * swap_usage_add / swap_usage_sub of each slot are serialized by ci->lock
1209 * within each cluster, so the total contribution to the global counter should
1210 * always be positive and cannot exceed the total number of usable slots.
1211 */
1212static bool swap_usage_add(struct swap_info_struct *si, unsigned int nr_entries)
1213{
1214	long val = atomic_long_add_return_relaxed(nr_entries, &si->inuse_pages);
1215
1216	/*
1217	 * If device is full, and SWAP_USAGE_OFFLIST_BIT is not set,
1218	 * remove it from the plist.
1219	 */
1220	if (unlikely(val == si->pages)) {
1221		del_from_avail_list(si, false);
1222		return true;
1223	}
1224
1225	return false;
1226}
1227
1228static void swap_usage_sub(struct swap_info_struct *si, unsigned int nr_entries)
1229{
1230	long val = atomic_long_sub_return_relaxed(nr_entries, &si->inuse_pages);
1231
1232	/*
1233	 * If device is not full, and SWAP_USAGE_OFFLIST_BIT is set,
1234	 * add it to the plist.
1235	 */
1236	if (unlikely(val & SWAP_USAGE_OFFLIST_BIT))
1237		add_to_avail_list(si, false);
1238}
1239
1240static void swap_range_alloc(struct swap_info_struct *si,
1241			     unsigned int nr_entries)
1242{
1243	if (swap_usage_add(si, nr_entries)) {
1244		if (vm_swap_full())
1245			schedule_work(&si->reclaim_work);
1246	}
1247	atomic_long_sub(nr_entries, &nr_swap_pages);
1248}
1249
1250static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
1251			    unsigned int nr_entries)
1252{
1253	unsigned long begin = offset;
1254	unsigned long end = offset + nr_entries - 1;
1255	void (*swap_slot_free_notify)(struct block_device *, unsigned long);
1256	unsigned int i;
1257
1258	/*
1259	 * Use atomic clear_bit operations only on zeromap instead of non-atomic
1260	 * bitmap_clear to prevent adjacent bits corruption due to simultaneous writes.
1261	 */
1262	for (i = 0; i < nr_entries; i++) {
1263		clear_bit(offset + i, si->zeromap);
1264		zswap_invalidate(swp_entry(si->type, offset + i));
1265	}
1266
1267	if (si->flags & SWP_BLKDEV)
1268		swap_slot_free_notify =
1269			si->bdev->bd_disk->fops->swap_slot_free_notify;
1270	else
1271		swap_slot_free_notify = NULL;
1272	while (offset <= end) {
1273		arch_swap_invalidate_page(si->type, offset);
1274		if (swap_slot_free_notify)
1275			swap_slot_free_notify(si->bdev, offset);
1276		offset++;
1277	}
1278	__swap_cache_clear_shadow(swp_entry(si->type, begin), nr_entries);
1279
1280	/*
1281	 * Make sure that try_to_unuse() observes si->inuse_pages reaching 0
1282	 * only after the above cleanups are done.
1283	 */
1284	smp_wmb();
1285	atomic_long_add(nr_entries, &nr_swap_pages);
1286	swap_usage_sub(si, nr_entries);
1287}
1288
1289static bool get_swap_device_info(struct swap_info_struct *si)
1290{
1291	if (!percpu_ref_tryget_live(&si->users))
1292		return false;
1293	/*
1294	 * Guarantee the si->users are checked before accessing other
1295	 * fields of swap_info_struct, and si->flags (SWP_WRITEOK) is
1296	 * up to dated.
1297	 *
1298	 * Paired with the spin_unlock() after setup_swap_info() in
1299	 * enable_swap_info(), and smp_wmb() in swapoff.
1300	 */
1301	smp_rmb();
1302	return true;
1303}
1304
1305/*
1306 * Fast path try to get swap entries with specified order from current
1307 * CPU's swap entry pool (a cluster).
1308 */
1309static bool swap_alloc_fast(swp_entry_t *entry,
1310			    int order)
1311{
1312	struct swap_cluster_info *ci;
1313	struct swap_info_struct *si;
1314	unsigned int offset, found = SWAP_ENTRY_INVALID;
1315
1316	/*
1317	 * Once allocated, swap_info_struct will never be completely freed,
1318	 * so checking it's liveness by get_swap_device_info is enough.
1319	 */
1320	si = this_cpu_read(percpu_swap_cluster.si[order]);
1321	offset = this_cpu_read(percpu_swap_cluster.offset[order]);
1322	if (!si || !offset || !get_swap_device_info(si))
1323		return false;
1324
1325	ci = swap_cluster_lock(si, offset);
1326	if (cluster_is_usable(ci, order)) {
1327		if (cluster_is_empty(ci))
1328			offset = cluster_offset(si, ci);
1329		found = alloc_swap_scan_cluster(si, ci, offset, order, SWAP_HAS_CACHE);
1330		if (found)
1331			*entry = swp_entry(si->type, found);
1332	} else {
1333		swap_cluster_unlock(ci);
1334	}
1335
1336	put_swap_device(si);
1337	return !!found;
1338}
1339
1340/* Rotate the device and switch to a new cluster */
1341static void swap_alloc_slow(swp_entry_t *entry,
1342			    int order)
1343{
1344	unsigned long offset;
1345	struct swap_info_struct *si, *next;
1346
1347	spin_lock(&swap_avail_lock);
1348start_over:
1349	plist_for_each_entry_safe(si, next, &swap_avail_head, avail_list) {
1350		/* Rotate the device and switch to a new cluster */
1351		plist_requeue(&si->avail_list, &swap_avail_head);
1352		spin_unlock(&swap_avail_lock);
1353		if (get_swap_device_info(si)) {
1354			offset = cluster_alloc_swap_entry(si, order, SWAP_HAS_CACHE);
1355			put_swap_device(si);
1356			if (offset) {
1357				*entry = swp_entry(si->type, offset);
1358				return;
1359			}
1360			if (order)
1361				return;
1362		}
1363
1364		spin_lock(&swap_avail_lock);
1365		/*
1366		 * if we got here, it's likely that si was almost full before,
1367		 * multiple callers probably all tried to get a page from the
1368		 * same si and it filled up before we could get one; or, the si
1369		 * filled up between us dropping swap_avail_lock.
1370		 * Since we dropped the swap_avail_lock, the swap_avail_list
1371		 * may have been modified; so if next is still in the
1372		 * swap_avail_head list then try it, otherwise start over if we
1373		 * have not gotten any slots.
1374		 */
1375		if (plist_node_empty(&next->avail_list))
1376			goto start_over;
1377	}
1378	spin_unlock(&swap_avail_lock);
1379}
1380
1381/*
1382 * Discard pending clusters in a synchronized way when under high pressure.
1383 * Return: true if any cluster is discarded.
1384 */
1385static bool swap_sync_discard(void)
1386{
1387	bool ret = false;
1388	struct swap_info_struct *si, *next;
1389
1390	spin_lock(&swap_lock);
1391start_over:
1392	plist_for_each_entry_safe(si, next, &swap_active_head, list) {
1393		spin_unlock(&swap_lock);
1394		if (get_swap_device_info(si)) {
1395			if (si->flags & SWP_PAGE_DISCARD)
1396				ret = swap_do_scheduled_discard(si);
1397			put_swap_device(si);
1398		}
1399		if (ret)
1400			return true;
1401
1402		spin_lock(&swap_lock);
1403		if (plist_node_empty(&next->list))
1404			goto start_over;
1405	}
1406	spin_unlock(&swap_lock);
1407
1408	return false;
1409}
1410
1411/**
1412 * folio_alloc_swap - allocate swap space for a folio
1413 * @folio: folio we want to move to swap
1414 *
1415 * Allocate swap space for the folio and add the folio to the
1416 * swap cache.
1417 *
1418 * Context: Caller needs to hold the folio lock.
1419 * Return: Whether the folio was added to the swap cache.
1420 */
1421int folio_alloc_swap(struct folio *folio)
1422{
1423	unsigned int order = folio_order(folio);
1424	unsigned int size = 1 << order;
1425	swp_entry_t entry = {};
1426
1427	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1428	VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio);
1429
1430	if (order) {
1431		/*
1432		 * Reject large allocation when THP_SWAP is disabled,
1433		 * the caller should split the folio and try again.
1434		 */
1435		if (!IS_ENABLED(CONFIG_THP_SWAP))
1436			return -EAGAIN;
1437
1438		/*
1439		 * Allocation size should never exceed cluster size
1440		 * (HPAGE_PMD_SIZE).
1441		 */
1442		if (size > SWAPFILE_CLUSTER) {
1443			VM_WARN_ON_ONCE(1);
1444			return -EINVAL;
1445		}
1446	}
1447
1448again:
1449	local_lock(&percpu_swap_cluster.lock);
1450	if (!swap_alloc_fast(&entry, order))
1451		swap_alloc_slow(&entry, order);
1452	local_unlock(&percpu_swap_cluster.lock);
1453
1454	if (unlikely(!order && !entry.val)) {
1455		if (swap_sync_discard())
1456			goto again;
1457	}
1458
1459	/* Need to call this even if allocation failed, for MEMCG_SWAP_FAIL. */
1460	if (mem_cgroup_try_charge_swap(folio, entry))
1461		goto out_free;
1462
1463	if (!entry.val)
1464		return -ENOMEM;
1465
1466	swap_cache_add_folio(folio, entry, NULL);
1467
1468	return 0;
1469
1470out_free:
1471	put_swap_folio(folio, entry);
1472	return -ENOMEM;
1473}
1474
1475static struct swap_info_struct *_swap_info_get(swp_entry_t entry)
1476{
1477	struct swap_info_struct *si;
1478	unsigned long offset;
1479
1480	if (!entry.val)
1481		goto out;
1482	si = swap_entry_to_info(entry);
1483	if (!si)
1484		goto bad_nofile;
1485	if (data_race(!(si->flags & SWP_USED)))
1486		goto bad_device;
1487	offset = swp_offset(entry);
1488	if (offset >= si->max)
1489		goto bad_offset;
1490	if (data_race(!si->swap_map[swp_offset(entry)]))
1491		goto bad_free;
1492	return si;
1493
1494bad_free:
1495	pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val);
1496	goto out;
1497bad_offset:
1498	pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1499	goto out;
1500bad_device:
1501	pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val);
1502	goto out;
1503bad_nofile:
1504	pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1505out:
1506	return NULL;
1507}
1508
1509static unsigned char swap_entry_put_locked(struct swap_info_struct *si,
1510					   struct swap_cluster_info *ci,
1511					   swp_entry_t entry,
1512					   unsigned char usage)
1513{
1514	unsigned long offset = swp_offset(entry);
1515	unsigned char count;
1516	unsigned char has_cache;
1517
1518	count = si->swap_map[offset];
1519
1520	has_cache = count & SWAP_HAS_CACHE;
1521	count &= ~SWAP_HAS_CACHE;
1522
1523	if (usage == SWAP_HAS_CACHE) {
1524		VM_BUG_ON(!has_cache);
1525		has_cache = 0;
1526	} else if (count == SWAP_MAP_SHMEM) {
1527		/*
1528		 * Or we could insist on shmem.c using a special
1529		 * swap_shmem_free() and free_shmem_swap_and_cache()...
1530		 */
1531		count = 0;
1532	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
1533		if (count == COUNT_CONTINUED) {
1534			if (swap_count_continued(si, offset, count))
1535				count = SWAP_MAP_MAX | COUNT_CONTINUED;
1536			else
1537				count = SWAP_MAP_MAX;
1538		} else
1539			count--;
1540	}
1541
1542	usage = count | has_cache;
1543	if (usage)
1544		WRITE_ONCE(si->swap_map[offset], usage);
1545	else
1546		swap_entries_free(si, ci, entry, 1);
1547
1548	return usage;
1549}
1550
1551/*
1552 * When we get a swap entry, if there aren't some other ways to
1553 * prevent swapoff, such as the folio in swap cache is locked, RCU
1554 * reader side is locked, etc., the swap entry may become invalid
1555 * because of swapoff.  Then, we need to enclose all swap related
1556 * functions with get_swap_device() and put_swap_device(), unless the
1557 * swap functions call get/put_swap_device() by themselves.
1558 *
1559 * RCU reader side lock (including any spinlock) is sufficient to
1560 * prevent swapoff, because synchronize_rcu() is called in swapoff()
1561 * before freeing data structures.
1562 *
1563 * Check whether swap entry is valid in the swap device.  If so,
1564 * return pointer to swap_info_struct, and keep the swap entry valid
1565 * via preventing the swap device from being swapoff, until
1566 * put_swap_device() is called.  Otherwise return NULL.
1567 *
1568 * Notice that swapoff or swapoff+swapon can still happen before the
1569 * percpu_ref_tryget_live() in get_swap_device() or after the
1570 * percpu_ref_put() in put_swap_device() if there isn't any other way
1571 * to prevent swapoff.  The caller must be prepared for that.  For
1572 * example, the following situation is possible.
1573 *
1574 *   CPU1				CPU2
1575 *   do_swap_page()
1576 *     ...				swapoff+swapon
1577 *     __read_swap_cache_async()
1578 *       swapcache_prepare()
1579 *         __swap_duplicate()
1580 *           // check swap_map
1581 *     // verify PTE not changed
1582 *
1583 * In __swap_duplicate(), the swap_map need to be checked before
1584 * changing partly because the specified swap entry may be for another
1585 * swap device which has been swapoff.  And in do_swap_page(), after
1586 * the page is read from the swap device, the PTE is verified not
1587 * changed with the page table locked to check whether the swap device
1588 * has been swapoff or swapoff+swapon.
1589 */
1590struct swap_info_struct *get_swap_device(swp_entry_t entry)
1591{
1592	struct swap_info_struct *si;
1593	unsigned long offset;
1594
1595	if (!entry.val)
1596		goto out;
1597	si = swap_entry_to_info(entry);
1598	if (!si)
1599		goto bad_nofile;
1600	if (!get_swap_device_info(si))
1601		goto out;
1602	offset = swp_offset(entry);
1603	if (offset >= si->max)
1604		goto put_out;
1605
1606	return si;
1607bad_nofile:
1608	pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1609out:
1610	return NULL;
1611put_out:
1612	pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1613	percpu_ref_put(&si->users);
1614	return NULL;
1615}
1616
1617static void swap_entries_put_cache(struct swap_info_struct *si,
1618				   swp_entry_t entry, int nr)
1619{
1620	unsigned long offset = swp_offset(entry);
1621	struct swap_cluster_info *ci;
1622
1623	ci = swap_cluster_lock(si, offset);
1624	if (swap_only_has_cache(si, offset, nr)) {
1625		swap_entries_free(si, ci, entry, nr);
1626	} else {
1627		for (int i = 0; i < nr; i++, entry.val++)
1628			swap_entry_put_locked(si, ci, entry, SWAP_HAS_CACHE);
1629	}
1630	swap_cluster_unlock(ci);
1631}
1632
1633static bool swap_entries_put_map(struct swap_info_struct *si,
1634				 swp_entry_t entry, int nr)
1635{
1636	unsigned long offset = swp_offset(entry);
1637	struct swap_cluster_info *ci;
1638	bool has_cache = false;
1639	unsigned char count;
1640	int i;
1641
1642	if (nr <= 1)
1643		goto fallback;
1644	count = swap_count(data_race(si->swap_map[offset]));
1645	if (count != 1 && count != SWAP_MAP_SHMEM)
1646		goto fallback;
1647
1648	ci = swap_cluster_lock(si, offset);
1649	if (!swap_is_last_map(si, offset, nr, &has_cache)) {
1650		goto locked_fallback;
1651	}
1652	if (!has_cache)
1653		swap_entries_free(si, ci, entry, nr);
1654	else
1655		for (i = 0; i < nr; i++)
1656			WRITE_ONCE(si->swap_map[offset + i], SWAP_HAS_CACHE);
1657	swap_cluster_unlock(ci);
1658
1659	return has_cache;
1660
1661fallback:
1662	ci = swap_cluster_lock(si, offset);
1663locked_fallback:
1664	for (i = 0; i < nr; i++, entry.val++) {
1665		count = swap_entry_put_locked(si, ci, entry, 1);
1666		if (count == SWAP_HAS_CACHE)
1667			has_cache = true;
1668	}
1669	swap_cluster_unlock(ci);
1670	return has_cache;
1671}
1672
1673/*
1674 * Only functions with "_nr" suffix are able to free entries spanning
1675 * cross multi clusters, so ensure the range is within a single cluster
1676 * when freeing entries with functions without "_nr" suffix.
1677 */
1678static bool swap_entries_put_map_nr(struct swap_info_struct *si,
1679				    swp_entry_t entry, int nr)
1680{
1681	int cluster_nr, cluster_rest;
1682	unsigned long offset = swp_offset(entry);
1683	bool has_cache = false;
1684
1685	cluster_rest = SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER;
1686	while (nr) {
1687		cluster_nr = min(nr, cluster_rest);
1688		has_cache |= swap_entries_put_map(si, entry, cluster_nr);
1689		cluster_rest = SWAPFILE_CLUSTER;
1690		nr -= cluster_nr;
1691		entry.val += cluster_nr;
1692	}
1693
1694	return has_cache;
1695}
1696
1697/*
1698 * Check if it's the last ref of swap entry in the freeing path.
1699 * Qualified value includes 1, SWAP_HAS_CACHE or SWAP_MAP_SHMEM.
1700 */
1701static inline bool __maybe_unused swap_is_last_ref(unsigned char count)
1702{
1703	return (count == SWAP_HAS_CACHE) || (count == 1) ||
1704	       (count == SWAP_MAP_SHMEM);
1705}
1706
1707/*
1708 * Drop the last ref of swap entries, caller have to ensure all entries
1709 * belong to the same cgroup and cluster.
1710 */
1711static void swap_entries_free(struct swap_info_struct *si,
1712			      struct swap_cluster_info *ci,
1713			      swp_entry_t entry, unsigned int nr_pages)
1714{
1715	unsigned long offset = swp_offset(entry);
1716	unsigned char *map = si->swap_map + offset;
1717	unsigned char *map_end = map + nr_pages;
1718
1719	/* It should never free entries across different clusters */
1720	VM_BUG_ON(ci != __swap_offset_to_cluster(si, offset + nr_pages - 1));
1721	VM_BUG_ON(cluster_is_empty(ci));
1722	VM_BUG_ON(ci->count < nr_pages);
1723
1724	ci->count -= nr_pages;
1725	do {
1726		VM_BUG_ON(!swap_is_last_ref(*map));
1727		*map = 0;
1728	} while (++map < map_end);
1729
1730	mem_cgroup_uncharge_swap(entry, nr_pages);
1731	swap_range_free(si, offset, nr_pages);
1732	swap_cluster_assert_table_empty(ci, offset, nr_pages);
1733
1734	if (!ci->count)
1735		free_cluster(si, ci);
1736	else
1737		partial_free_cluster(si, ci);
1738}
1739
1740/*
1741 * Caller has made sure that the swap device corresponding to entry
1742 * is still around or has not been recycled.
1743 */
1744void swap_free_nr(swp_entry_t entry, int nr_pages)
1745{
1746	int nr;
1747	struct swap_info_struct *sis;
1748	unsigned long offset = swp_offset(entry);
1749
1750	sis = _swap_info_get(entry);
1751	if (!sis)
1752		return;
1753
1754	while (nr_pages) {
1755		nr = min_t(int, nr_pages, SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER);
1756		swap_entries_put_map(sis, swp_entry(sis->type, offset), nr);
1757		offset += nr;
1758		nr_pages -= nr;
1759	}
1760}
1761
1762/*
1763 * Called after dropping swapcache to decrease refcnt to swap entries.
1764 */
1765void put_swap_folio(struct folio *folio, swp_entry_t entry)
1766{
1767	struct swap_info_struct *si;
1768	int size = 1 << swap_entry_order(folio_order(folio));
1769
1770	si = _swap_info_get(entry);
1771	if (!si)
1772		return;
1773
1774	swap_entries_put_cache(si, entry, size);
1775}
1776
1777int __swap_count(swp_entry_t entry)
1778{
1779	struct swap_info_struct *si = __swap_entry_to_info(entry);
1780	pgoff_t offset = swp_offset(entry);
1781
1782	return swap_count(si->swap_map[offset]);
1783}
1784
1785/*
1786 * How many references to @entry are currently swapped out?
1787 * This does not give an exact answer when swap count is continued,
1788 * but does include the high COUNT_CONTINUED flag to allow for that.
1789 */
1790bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry)
1791{
1792	pgoff_t offset = swp_offset(entry);
1793	struct swap_cluster_info *ci;
1794	int count;
1795
1796	ci = swap_cluster_lock(si, offset);
1797	count = swap_count(si->swap_map[offset]);
1798	swap_cluster_unlock(ci);
1799	return !!count;
1800}
1801
1802/*
1803 * How many references to @entry are currently swapped out?
1804 * This considers COUNT_CONTINUED so it returns exact answer.
1805 */
1806int swp_swapcount(swp_entry_t entry)
1807{
1808	int count, tmp_count, n;
1809	struct swap_info_struct *si;
1810	struct swap_cluster_info *ci;
1811	struct page *page;
1812	pgoff_t offset;
1813	unsigned char *map;
1814
1815	si = _swap_info_get(entry);
1816	if (!si)
1817		return 0;
1818
1819	offset = swp_offset(entry);
1820
1821	ci = swap_cluster_lock(si, offset);
1822
1823	count = swap_count(si->swap_map[offset]);
1824	if (!(count & COUNT_CONTINUED))
1825		goto out;
1826
1827	count &= ~COUNT_CONTINUED;
1828	n = SWAP_MAP_MAX + 1;
1829
1830	page = vmalloc_to_page(si->swap_map + offset);
1831	offset &= ~PAGE_MASK;
1832	VM_BUG_ON(page_private(page) != SWP_CONTINUED);
1833
1834	do {
1835		page = list_next_entry(page, lru);
1836		map = kmap_local_page(page);
1837		tmp_count = map[offset];
1838		kunmap_local(map);
1839
1840		count += (tmp_count & ~COUNT_CONTINUED) * n;
1841		n *= (SWAP_CONT_MAX + 1);
1842	} while (tmp_count & COUNT_CONTINUED);
1843out:
1844	swap_cluster_unlock(ci);
1845	return count;
1846}
1847
1848static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
1849					 swp_entry_t entry, int order)
1850{
1851	struct swap_cluster_info *ci;
1852	unsigned char *map = si->swap_map;
1853	unsigned int nr_pages = 1 << order;
1854	unsigned long roffset = swp_offset(entry);
1855	unsigned long offset = round_down(roffset, nr_pages);
1856	int i;
1857	bool ret = false;
1858
1859	ci = swap_cluster_lock(si, offset);
1860	if (nr_pages == 1) {
1861		if (swap_count(map[roffset]))
1862			ret = true;
1863		goto unlock_out;
1864	}
1865	for (i = 0; i < nr_pages; i++) {
1866		if (swap_count(map[offset + i])) {
1867			ret = true;
1868			break;
1869		}
1870	}
1871unlock_out:
1872	swap_cluster_unlock(ci);
1873	return ret;
1874}
1875
1876static bool folio_swapped(struct folio *folio)
1877{
1878	swp_entry_t entry = folio->swap;
1879	struct swap_info_struct *si = _swap_info_get(entry);
1880
1881	if (!si)
1882		return false;
1883
1884	if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio)))
1885		return swap_entry_swapped(si, entry);
1886
1887	return swap_page_trans_huge_swapped(si, entry, folio_order(folio));
1888}
1889
1890static bool folio_swapcache_freeable(struct folio *folio)
1891{
1892	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
1893
1894	if (!folio_test_swapcache(folio))
1895		return false;
1896	if (folio_test_writeback(folio))
1897		return false;
1898
1899	/*
1900	 * Once hibernation has begun to create its image of memory,
1901	 * there's a danger that one of the calls to folio_free_swap()
1902	 * - most probably a call from __try_to_reclaim_swap() while
1903	 * hibernation is allocating its own swap pages for the image,
1904	 * but conceivably even a call from memory reclaim - will free
1905	 * the swap from a folio which has already been recorded in the
1906	 * image as a clean swapcache folio, and then reuse its swap for
1907	 * another page of the image.  On waking from hibernation, the
1908	 * original folio might be freed under memory pressure, then
1909	 * later read back in from swap, now with the wrong data.
1910	 *
1911	 * Hibernation suspends storage while it is writing the image
1912	 * to disk so check that here.
1913	 */
1914	if (pm_suspended_storage())
1915		return false;
1916
1917	return true;
1918}
1919
1920/**
1921 * folio_free_swap() - Free the swap space used for this folio.
1922 * @folio: The folio to remove.
1923 *
1924 * If swap is getting full, or if there are no more mappings of this folio,
1925 * then call folio_free_swap to free its swap space.
1926 *
1927 * Return: true if we were able to release the swap space.
1928 */
1929bool folio_free_swap(struct folio *folio)
1930{
1931	if (!folio_swapcache_freeable(folio))
1932		return false;
1933	if (folio_swapped(folio))
1934		return false;
1935
1936	swap_cache_del_folio(folio);
1937	folio_set_dirty(folio);
1938	return true;
1939}
1940
1941/**
1942 * free_swap_and_cache_nr() - Release reference on range of swap entries and
1943 *                            reclaim their cache if no more references remain.
1944 * @entry: First entry of range.
1945 * @nr: Number of entries in range.
1946 *
1947 * For each swap entry in the contiguous range, release a reference. If any swap
1948 * entries become free, try to reclaim their underlying folios, if present. The
1949 * offset range is defined by [entry.offset, entry.offset + nr).
1950 */
1951void free_swap_and_cache_nr(swp_entry_t entry, int nr)
1952{
1953	const unsigned long start_offset = swp_offset(entry);
1954	const unsigned long end_offset = start_offset + nr;
1955	struct swap_info_struct *si;
1956	bool any_only_cache = false;
1957	unsigned long offset;
1958
1959	si = get_swap_device(entry);
1960	if (!si)
1961		return;
1962
1963	if (WARN_ON(end_offset > si->max))
1964		goto out;
1965
1966	/*
1967	 * First free all entries in the range.
1968	 */
1969	any_only_cache = swap_entries_put_map_nr(si, entry, nr);
1970
1971	/*
1972	 * Short-circuit the below loop if none of the entries had their
1973	 * reference drop to zero.
1974	 */
1975	if (!any_only_cache)
1976		goto out;
1977
1978	/*
1979	 * Now go back over the range trying to reclaim the swap cache.
1980	 */
1981	for (offset = start_offset; offset < end_offset; offset += nr) {
1982		nr = 1;
1983		if (READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) {
1984			/*
1985			 * Folios are always naturally aligned in swap so
1986			 * advance forward to the next boundary. Zero means no
1987			 * folio was found for the swap entry, so advance by 1
1988			 * in this case. Negative value means folio was found
1989			 * but could not be reclaimed. Here we can still advance
1990			 * to the next boundary.
1991			 */
1992			nr = __try_to_reclaim_swap(si, offset,
1993						   TTRS_UNMAPPED | TTRS_FULL);
1994			if (nr == 0)
1995				nr = 1;
1996			else if (nr < 0)
1997				nr = -nr;
1998			nr = ALIGN(offset + 1, nr) - offset;
1999		}
2000	}
2001
2002out:
2003	put_swap_device(si);
2004}
2005
2006#ifdef CONFIG_HIBERNATION
2007
2008swp_entry_t get_swap_page_of_type(int type)
2009{
2010	struct swap_info_struct *si = swap_type_to_info(type);
2011	unsigned long offset;
2012	swp_entry_t entry = {0};
2013
2014	if (!si)
2015		goto fail;
2016
2017	/* This is called for allocating swap entry, not cache */
2018	if (get_swap_device_info(si)) {
2019		if (si->flags & SWP_WRITEOK) {
2020			/*
2021			 * Grab the local lock to be complaint
2022			 * with swap table allocation.
2023			 */
2024			local_lock(&percpu_swap_cluster.lock);
2025			offset = cluster_alloc_swap_entry(si, 0, 1);
2026			local_unlock(&percpu_swap_cluster.lock);
2027			if (offset)
2028				entry = swp_entry(si->type, offset);
2029		}
2030		put_swap_device(si);
2031	}
2032fail:
2033	return entry;
2034}
2035
2036/*
2037 * Find the swap type that corresponds to given device (if any).
2038 *
2039 * @offset - number of the PAGE_SIZE-sized block of the device, starting
2040 * from 0, in which the swap header is expected to be located.
2041 *
2042 * This is needed for the suspend to disk (aka swsusp).
2043 */
2044int swap_type_of(dev_t device, sector_t offset)
2045{
2046	int type;
2047
2048	if (!device)
2049		return -1;
2050
2051	spin_lock(&swap_lock);
2052	for (type = 0; type < nr_swapfiles; type++) {
2053		struct swap_info_struct *sis = swap_info[type];
2054
2055		if (!(sis->flags & SWP_WRITEOK))
2056			continue;
2057
2058		if (device == sis->bdev->bd_dev) {
2059			struct swap_extent *se = first_se(sis);
2060
2061			if (se->start_block == offset) {
2062				spin_unlock(&swap_lock);
2063				return type;
2064			}
2065		}
2066	}
2067	spin_unlock(&swap_lock);
2068	return -ENODEV;
2069}
2070
2071int find_first_swap(dev_t *device)
2072{
2073	int type;
2074
2075	spin_lock(&swap_lock);
2076	for (type = 0; type < nr_swapfiles; type++) {
2077		struct swap_info_struct *sis = swap_info[type];
2078
2079		if (!(sis->flags & SWP_WRITEOK))
2080			continue;
2081		*device = sis->bdev->bd_dev;
2082		spin_unlock(&swap_lock);
2083		return type;
2084	}
2085	spin_unlock(&swap_lock);
2086	return -ENODEV;
2087}
2088
2089/*
2090 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
2091 * corresponding to given index in swap_info (swap type).
2092 */
2093sector_t swapdev_block(int type, pgoff_t offset)
2094{
2095	struct swap_info_struct *si = swap_type_to_info(type);
2096	struct swap_extent *se;
2097
2098	if (!si || !(si->flags & SWP_WRITEOK))
2099		return 0;
2100	se = offset_to_swap_extent(si, offset);
2101	return se->start_block + (offset - se->start_page);
2102}
2103
2104/*
2105 * Return either the total number of swap pages of given type, or the number
2106 * of free pages of that type (depending on @free)
2107 *
2108 * This is needed for software suspend
2109 */
2110unsigned int count_swap_pages(int type, int free)
2111{
2112	unsigned int n = 0;
2113
2114	spin_lock(&swap_lock);
2115	if ((unsigned int)type < nr_swapfiles) {
2116		struct swap_info_struct *sis = swap_info[type];
2117
2118		spin_lock(&sis->lock);
2119		if (sis->flags & SWP_WRITEOK) {
2120			n = sis->pages;
2121			if (free)
2122				n -= swap_usage_in_pages(sis);
2123		}
2124		spin_unlock(&sis->lock);
2125	}
2126	spin_unlock(&swap_lock);
2127	return n;
2128}
2129#endif /* CONFIG_HIBERNATION */
2130
2131static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
2132{
2133	return pte_same(pte_swp_clear_flags(pte), swp_pte);
2134}
2135
2136/*
2137 * No need to decide whether this PTE shares the swap entry with others,
2138 * just let do_wp_page work it out if a write is requested later - to
2139 * force COW, vm_page_prot omits write permission from any private vma.
2140 */
2141static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
2142		unsigned long addr, swp_entry_t entry, struct folio *folio)
2143{
2144	struct page *page;
2145	struct folio *swapcache;
2146	spinlock_t *ptl;
2147	pte_t *pte, new_pte, old_pte;
2148	bool hwpoisoned = false;
2149	int ret = 1;
2150
2151	/*
2152	 * If the folio is removed from swap cache by others, continue to
2153	 * unuse other PTEs. try_to_unuse may try again if we missed this one.
2154	 */
2155	if (!folio_matches_swap_entry(folio, entry))
2156		return 0;
2157
2158	swapcache = folio;
2159	folio = ksm_might_need_to_copy(folio, vma, addr);
2160	if (unlikely(!folio))
2161		return -ENOMEM;
2162	else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
2163		hwpoisoned = true;
2164		folio = swapcache;
2165	}
2166
2167	page = folio_file_page(folio, swp_offset(entry));
2168	if (PageHWPoison(page))
2169		hwpoisoned = true;
2170
2171	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
2172	if (unlikely(!pte || !pte_same_as_swp(ptep_get(pte),
2173						swp_entry_to_pte(entry)))) {
2174		ret = 0;
2175		goto out;
2176	}
2177
2178	old_pte = ptep_get(pte);
2179
2180	if (unlikely(hwpoisoned || !folio_test_uptodate(folio))) {
2181		swp_entry_t swp_entry;
2182
2183		dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
2184		if (hwpoisoned) {
2185			swp_entry = make_hwpoison_entry(page);
2186		} else {
2187			swp_entry = make_poisoned_swp_entry();
2188		}
2189		new_pte = swp_entry_to_pte(swp_entry);
2190		ret = 0;
2191		goto setpte;
2192	}
2193
2194	/*
2195	 * Some architectures may have to restore extra metadata to the page
2196	 * when reading from swap. This metadata may be indexed by swap entry
2197	 * so this must be called before swap_free().
2198	 */
2199	arch_swap_restore(folio_swap(entry, folio), folio);
2200
2201	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
2202	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
2203	folio_get(folio);
2204	if (folio == swapcache) {
2205		rmap_t rmap_flags = RMAP_NONE;
2206
2207		/*
2208		 * See do_swap_page(): writeback would be problematic.
2209		 * However, we do a folio_wait_writeback() just before this
2210		 * call and have the folio locked.
2211		 */
2212		VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);
2213		if (pte_swp_exclusive(old_pte))
2214			rmap_flags |= RMAP_EXCLUSIVE;
2215		/*
2216		 * We currently only expect small !anon folios, which are either
2217		 * fully exclusive or fully shared. If we ever get large folios
2218		 * here, we have to be careful.
2219		 */
2220		if (!folio_test_anon(folio)) {
2221			VM_WARN_ON_ONCE(folio_test_large(folio));
2222			VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
2223			folio_add_new_anon_rmap(folio, vma, addr, rmap_flags);
2224		} else {
2225			folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags);
2226		}
2227	} else { /* ksm created a completely new copy */
2228		folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
2229		folio_add_lru_vma(folio, vma);
2230	}
2231	new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot));
2232	if (pte_swp_soft_dirty(old_pte))
2233		new_pte = pte_mksoft_dirty(new_pte);
2234	if (pte_swp_uffd_wp(old_pte))
2235		new_pte = pte_mkuffd_wp(new_pte);
2236setpte:
2237	set_pte_at(vma->vm_mm, addr, pte, new_pte);
2238	swap_free(entry);
2239out:
2240	if (pte)
2241		pte_unmap_unlock(pte, ptl);
2242	if (folio != swapcache) {
2243		folio_unlock(folio);
2244		folio_put(folio);
2245	}
2246	return ret;
2247}
2248
2249static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
2250			unsigned long addr, unsigned long end,
2251			unsigned int type)
2252{
2253	pte_t *pte = NULL;
2254	struct swap_info_struct *si;
2255
2256	si = swap_info[type];
2257	do {
2258		struct folio *folio;
2259		unsigned long offset;
2260		unsigned char swp_count;
2261		softleaf_t entry;
2262		int ret;
2263		pte_t ptent;
2264
2265		if (!pte++) {
2266			pte = pte_offset_map(pmd, addr);
2267			if (!pte)
2268				break;
2269		}
2270
2271		ptent = ptep_get_lockless(pte);
2272		entry = softleaf_from_pte(ptent);
2273
2274		if (!softleaf_is_swap(entry))
2275			continue;
2276		if (swp_type(entry) != type)
2277			continue;
2278
2279		offset = swp_offset(entry);
2280		pte_unmap(pte);
2281		pte = NULL;
2282
2283		folio = swap_cache_get_folio(entry);
2284		if (!folio) {
2285			struct vm_fault vmf = {
2286				.vma = vma,
2287				.address = addr,
2288				.real_address = addr,
2289				.pmd = pmd,
2290			};
2291
2292			folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
2293						&vmf);
2294		}
2295		if (!folio) {
2296			swp_count = READ_ONCE(si->swap_map[offset]);
2297			if (swp_count == 0 || swp_count == SWAP_MAP_BAD)
2298				continue;
2299			return -ENOMEM;
2300		}
2301
2302		folio_lock(folio);
2303		folio_wait_writeback(folio);
2304		ret = unuse_pte(vma, pmd, addr, entry, folio);
2305		if (ret < 0) {
2306			folio_unlock(folio);
2307			folio_put(folio);
2308			return ret;
2309		}
2310
2311		folio_free_swap(folio);
2312		folio_unlock(folio);
2313		folio_put(folio);
2314	} while (addr += PAGE_SIZE, addr != end);
2315
2316	if (pte)
2317		pte_unmap(pte);
2318	return 0;
2319}
2320
2321static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
2322				unsigned long addr, unsigned long end,
2323				unsigned int type)
2324{
2325	pmd_t *pmd;
2326	unsigned long next;
2327	int ret;
2328
2329	pmd = pmd_offset(pud, addr);
2330	do {
2331		cond_resched();
2332		next = pmd_addr_end(addr, end);
2333		ret = unuse_pte_range(vma, pmd, addr, next, type);
2334		if (ret)
2335			return ret;
2336	} while (pmd++, addr = next, addr != end);
2337	return 0;
2338}
2339
2340static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
2341				unsigned long addr, unsigned long end,
2342				unsigned int type)
2343{
2344	pud_t *pud;
2345	unsigned long next;
2346	int ret;
2347
2348	pud = pud_offset(p4d, addr);
2349	do {
2350		next = pud_addr_end(addr, end);
2351		if (pud_none_or_clear_bad(pud))
2352			continue;
2353		ret = unuse_pmd_range(vma, pud, addr, next, type);
2354		if (ret)
2355			return ret;
2356	} while (pud++, addr = next, addr != end);
2357	return 0;
2358}
2359
2360static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
2361				unsigned long addr, unsigned long end,
2362				unsigned int type)
2363{
2364	p4d_t *p4d;
2365	unsigned long next;
2366	int ret;
2367
2368	p4d = p4d_offset(pgd, addr);
2369	do {
2370		next = p4d_addr_end(addr, end);
2371		if (p4d_none_or_clear_bad(p4d))
2372			continue;
2373		ret = unuse_pud_range(vma, p4d, addr, next, type);
2374		if (ret)
2375			return ret;
2376	} while (p4d++, addr = next, addr != end);
2377	return 0;
2378}
2379
2380static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
2381{
2382	pgd_t *pgd;
2383	unsigned long addr, end, next;
2384	int ret;
2385
2386	addr = vma->vm_start;
2387	end = vma->vm_end;
2388
2389	pgd = pgd_offset(vma->vm_mm, addr);
2390	do {
2391		next = pgd_addr_end(addr, end);
2392		if (pgd_none_or_clear_bad(pgd))
2393			continue;
2394		ret = unuse_p4d_range(vma, pgd, addr, next, type);
2395		if (ret)
2396			return ret;
2397	} while (pgd++, addr = next, addr != end);
2398	return 0;
2399}
2400
2401static int unuse_mm(struct mm_struct *mm, unsigned int type)
2402{
2403	struct vm_area_struct *vma;
2404	int ret = 0;
2405	VMA_ITERATOR(vmi, mm, 0);
2406
2407	mmap_read_lock(mm);
2408	if (check_stable_address_space(mm))
2409		goto unlock;
2410	for_each_vma(vmi, vma) {
2411		if (vma->anon_vma && !is_vm_hugetlb_page(vma)) {
2412			ret = unuse_vma(vma, type);
2413			if (ret)
2414				break;
2415		}
2416
2417		cond_resched();
2418	}
2419unlock:
2420	mmap_read_unlock(mm);
2421	return ret;
2422}
2423
2424/*
2425 * Scan swap_map from current position to next entry still in use.
2426 * Return 0 if there are no inuse entries after prev till end of
2427 * the map.
2428 */
2429static unsigned int find_next_to_unuse(struct swap_info_struct *si,
2430					unsigned int prev)
2431{
2432	unsigned int i;
2433	unsigned char count;
2434
2435	/*
2436	 * No need for swap_lock here: we're just looking
2437	 * for whether an entry is in use, not modifying it; false
2438	 * hits are okay, and sys_swapoff() has already prevented new
2439	 * allocations from this area (while holding swap_lock).
2440	 */
2441	for (i = prev + 1; i < si->max; i++) {
2442		count = READ_ONCE(si->swap_map[i]);
2443		if (count && swap_count(count) != SWAP_MAP_BAD)
2444			break;
2445		if ((i % LATENCY_LIMIT) == 0)
2446			cond_resched();
2447	}
2448
2449	if (i == si->max)
2450		i = 0;
2451
2452	return i;
2453}
2454
2455static int try_to_unuse(unsigned int type)
2456{
2457	struct mm_struct *prev_mm;
2458	struct mm_struct *mm;
2459	struct list_head *p;
2460	int retval = 0;
2461	struct swap_info_struct *si = swap_info[type];
2462	struct folio *folio;
2463	swp_entry_t entry;
2464	unsigned int i;
2465
2466	if (!swap_usage_in_pages(si))
2467		goto success;
2468
2469retry:
2470	retval = shmem_unuse(type);
2471	if (retval)
2472		return retval;
2473
2474	prev_mm = &init_mm;
2475	mmget(prev_mm);
2476
2477	spin_lock(&mmlist_lock);
2478	p = &init_mm.mmlist;
2479	while (swap_usage_in_pages(si) &&
2480	       !signal_pending(current) &&
2481	       (p = p->next) != &init_mm.mmlist) {
2482
2483		mm = list_entry(p, struct mm_struct, mmlist);
2484		if (!mmget_not_zero(mm))
2485			continue;
2486		spin_unlock(&mmlist_lock);
2487		mmput(prev_mm);
2488		prev_mm = mm;
2489		retval = unuse_mm(mm, type);
2490		if (retval) {
2491			mmput(prev_mm);
2492			return retval;
2493		}
2494
2495		/*
2496		 * Make sure that we aren't completely killing
2497		 * interactive performance.
2498		 */
2499		cond_resched();
2500		spin_lock(&mmlist_lock);
2501	}
2502	spin_unlock(&mmlist_lock);
2503
2504	mmput(prev_mm);
2505
2506	i = 0;
2507	while (swap_usage_in_pages(si) &&
2508	       !signal_pending(current) &&
2509	       (i = find_next_to_unuse(si, i)) != 0) {
2510
2511		entry = swp_entry(type, i);
2512		folio = swap_cache_get_folio(entry);
2513		if (!folio)
2514			continue;
2515
2516		/*
2517		 * It is conceivable that a racing task removed this folio from
2518		 * swap cache just before we acquired the page lock. The folio
2519		 * might even be back in swap cache on another swap area. But
2520		 * that is okay, folio_free_swap() only removes stale folios.
2521		 */
2522		folio_lock(folio);
2523		folio_wait_writeback(folio);
2524		folio_free_swap(folio);
2525		folio_unlock(folio);
2526		folio_put(folio);
2527	}
2528
2529	/*
2530	 * Lets check again to see if there are still swap entries in the map.
2531	 * If yes, we would need to do retry the unuse logic again.
2532	 * Under global memory pressure, swap entries can be reinserted back
2533	 * into process space after the mmlist loop above passes over them.
2534	 *
2535	 * Limit the number of retries? No: when mmget_not_zero()
2536	 * above fails, that mm is likely to be freeing swap from
2537	 * exit_mmap(), which proceeds at its own independent pace;
2538	 * and even shmem_writeout() could have been preempted after
2539	 * folio_alloc_swap(), temporarily hiding that swap.  It's easy
2540	 * and robust (though cpu-intensive) just to keep retrying.
2541	 */
2542	if (swap_usage_in_pages(si)) {
2543		if (!signal_pending(current))
2544			goto retry;
2545		return -EINTR;
2546	}
2547
2548success:
2549	/*
2550	 * Make sure that further cleanups after try_to_unuse() returns happen
2551	 * after swap_range_free() reduces si->inuse_pages to 0.
2552	 */
2553	smp_mb();
2554	return 0;
2555}
2556
2557/*
2558 * After a successful try_to_unuse, if no swap is now in use, we know
2559 * we can empty the mmlist.  swap_lock must be held on entry and exit.
2560 * Note that mmlist_lock nests inside swap_lock, and an mm must be
2561 * added to the mmlist just after page_duplicate - before would be racy.
2562 */
2563static void drain_mmlist(void)
2564{
2565	struct list_head *p, *next;
2566	unsigned int type;
2567
2568	for (type = 0; type < nr_swapfiles; type++)
2569		if (swap_usage_in_pages(swap_info[type]))
2570			return;
2571	spin_lock(&mmlist_lock);
2572	list_for_each_safe(p, next, &init_mm.mmlist)
2573		list_del_init(p);
2574	spin_unlock(&mmlist_lock);
2575}
2576
2577/*
2578 * Free all of a swapdev's extent information
2579 */
2580static void destroy_swap_extents(struct swap_info_struct *sis)
2581{
2582	while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) {
2583		struct rb_node *rb = sis->swap_extent_root.rb_node;
2584		struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node);
2585
2586		rb_erase(rb, &sis->swap_extent_root);
2587		kfree(se);
2588	}
2589
2590	if (sis->flags & SWP_ACTIVATED) {
2591		struct file *swap_file = sis->swap_file;
2592		struct address_space *mapping = swap_file->f_mapping;
2593
2594		sis->flags &= ~SWP_ACTIVATED;
2595		if (mapping->a_ops->swap_deactivate)
2596			mapping->a_ops->swap_deactivate(swap_file);
2597	}
2598}
2599
2600/*
2601 * Add a block range (and the corresponding page range) into this swapdev's
2602 * extent tree.
2603 *
2604 * This function rather assumes that it is called in ascending page order.
2605 */
2606int
2607add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
2608		unsigned long nr_pages, sector_t start_block)
2609{
2610	struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL;
2611	struct swap_extent *se;
2612	struct swap_extent *new_se;
2613
2614	/*
2615	 * place the new node at the right most since the
2616	 * function is called in ascending page order.
2617	 */
2618	while (*link) {
2619		parent = *link;
2620		link = &parent->rb_right;
2621	}
2622
2623	if (parent) {
2624		se = rb_entry(parent, struct swap_extent, rb_node);
2625		BUG_ON(se->start_page + se->nr_pages != start_page);
2626		if (se->start_block + se->nr_pages == start_block) {
2627			/* Merge it */
2628			se->nr_pages += nr_pages;
2629			return 0;
2630		}
2631	}
2632
2633	/* No merge, insert a new extent. */
2634	new_se = kmalloc(sizeof(*se), GFP_KERNEL);
2635	if (new_se == NULL)
2636		return -ENOMEM;
2637	new_se->start_page = start_page;
2638	new_se->nr_pages = nr_pages;
2639	new_se->start_block = start_block;
2640
2641	rb_link_node(&new_se->rb_node, parent, link);
2642	rb_insert_color(&new_se->rb_node, &sis->swap_extent_root);
2643	return 1;
2644}
2645EXPORT_SYMBOL_GPL(add_swap_extent);
2646
2647/*
2648 * A `swap extent' is a simple thing which maps a contiguous range of pages
2649 * onto a contiguous range of disk blocks.  A rbtree of swap extents is
2650 * built at swapon time and is then used at swap_writepage/swap_read_folio
2651 * time for locating where on disk a page belongs.
2652 *
2653 * If the swapfile is an S_ISBLK block device, a single extent is installed.
2654 * This is done so that the main operating code can treat S_ISBLK and S_ISREG
2655 * swap files identically.
2656 *
2657 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2658 * extent rbtree operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
2659 * swapfiles are handled *identically* after swapon time.
2660 *
2661 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
2662 * and will parse them into a rbtree, in PAGE_SIZE chunks.  If some stray
2663 * blocks are found which do not fall within the PAGE_SIZE alignment
2664 * requirements, they are simply tossed out - we will never use those blocks
2665 * for swapping.
2666 *
2667 * For all swap devices we set S_SWAPFILE across the life of the swapon.  This
2668 * prevents users from writing to the swap device, which will corrupt memory.
2669 *
2670 * The amount of disk space which a single swap extent represents varies.
2671 * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
2672 * extents in the rbtree. - akpm.
2673 */
2674static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
2675{
2676	struct file *swap_file = sis->swap_file;
2677	struct address_space *mapping = swap_file->f_mapping;
2678	struct inode *inode = mapping->host;
2679	int ret;
2680
2681	if (S_ISBLK(inode->i_mode)) {
2682		ret = add_swap_extent(sis, 0, sis->max, 0);
2683		*span = sis->pages;
2684		return ret;
2685	}
2686
2687	if (mapping->a_ops->swap_activate) {
2688		ret = mapping->a_ops->swap_activate(sis, swap_file, span);
2689		if (ret < 0)
2690			return ret;
2691		sis->flags |= SWP_ACTIVATED;
2692		if ((sis->flags & SWP_FS_OPS) &&
2693		    sio_pool_init() != 0) {
2694			destroy_swap_extents(sis);
2695			return -ENOMEM;
2696		}
2697		return ret;
2698	}
2699
2700	return generic_swapfile_activate(sis, swap_file, span);
2701}
2702
2703static void setup_swap_info(struct swap_info_struct *si, int prio,
2704			    unsigned char *swap_map,
2705			    struct swap_cluster_info *cluster_info,
2706			    unsigned long *zeromap)
2707{
2708	si->prio = prio;
2709	/*
2710	 * the plist prio is negated because plist ordering is
2711	 * low-to-high, while swap ordering is high-to-low
2712	 */
2713	si->list.prio = -si->prio;
2714	si->avail_list.prio = -si->prio;
2715	si->swap_map = swap_map;
2716	si->cluster_info = cluster_info;
2717	si->zeromap = zeromap;
2718}
2719
2720static void _enable_swap_info(struct swap_info_struct *si)
2721{
2722	atomic_long_add(si->pages, &nr_swap_pages);
2723	total_swap_pages += si->pages;
2724
2725	assert_spin_locked(&swap_lock);
2726
2727	plist_add(&si->list, &swap_active_head);
2728
2729	/* Add back to available list */
2730	add_to_avail_list(si, true);
2731}
2732
2733static void enable_swap_info(struct swap_info_struct *si, int prio,
2734				unsigned char *swap_map,
2735				struct swap_cluster_info *cluster_info,
2736				unsigned long *zeromap)
2737{
2738	spin_lock(&swap_lock);
2739	spin_lock(&si->lock);
2740	setup_swap_info(si, prio, swap_map, cluster_info, zeromap);
2741	spin_unlock(&si->lock);
2742	spin_unlock(&swap_lock);
2743	/*
2744	 * Finished initializing swap device, now it's safe to reference it.
2745	 */
2746	percpu_ref_resurrect(&si->users);
2747	spin_lock(&swap_lock);
2748	spin_lock(&si->lock);
2749	_enable_swap_info(si);
2750	spin_unlock(&si->lock);
2751	spin_unlock(&swap_lock);
2752}
2753
2754static void reinsert_swap_info(struct swap_info_struct *si)
2755{
2756	spin_lock(&swap_lock);
2757	spin_lock(&si->lock);
2758	setup_swap_info(si, si->prio, si->swap_map, si->cluster_info, si->zeromap);
2759	_enable_swap_info(si);
2760	spin_unlock(&si->lock);
2761	spin_unlock(&swap_lock);
2762}
2763
2764/*
2765 * Called after clearing SWP_WRITEOK, ensures cluster_alloc_range
2766 * see the updated flags, so there will be no more allocations.
2767 */
2768static void wait_for_allocation(struct swap_info_struct *si)
2769{
2770	unsigned long offset;
2771	unsigned long end = ALIGN(si->max, SWAPFILE_CLUSTER);
2772	struct swap_cluster_info *ci;
2773
2774	BUG_ON(si->flags & SWP_WRITEOK);
2775
2776	for (offset = 0; offset < end; offset += SWAPFILE_CLUSTER) {
2777		ci = swap_cluster_lock(si, offset);
2778		swap_cluster_unlock(ci);
2779	}
2780}
2781
2782static void free_cluster_info(struct swap_cluster_info *cluster_info,
2783			      unsigned long maxpages)
2784{
2785	struct swap_cluster_info *ci;
2786	int i, nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2787
2788	if (!cluster_info)
2789		return;
2790	for (i = 0; i < nr_clusters; i++) {
2791		ci = cluster_info + i;
2792		/* Cluster with bad marks count will have a remaining table */
2793		spin_lock(&ci->lock);
2794		if (rcu_dereference_protected(ci->table, true)) {
2795			ci->count = 0;
2796			swap_cluster_free_table(ci);
2797		}
2798		spin_unlock(&ci->lock);
2799	}
2800	kvfree(cluster_info);
2801}
2802
2803/*
2804 * Called after swap device's reference count is dead, so
2805 * neither scan nor allocation will use it.
2806 */
2807static void flush_percpu_swap_cluster(struct swap_info_struct *si)
2808{
2809	int cpu, i;
2810	struct swap_info_struct **pcp_si;
2811
2812	for_each_possible_cpu(cpu) {
2813		pcp_si = per_cpu_ptr(percpu_swap_cluster.si, cpu);
2814		/*
2815		 * Invalidate the percpu swap cluster cache, si->users
2816		 * is dead, so no new user will point to it, just flush
2817		 * any existing user.
2818		 */
2819		for (i = 0; i < SWAP_NR_ORDERS; i++)
2820			cmpxchg(&pcp_si[i], si, NULL);
2821	}
2822}
2823
2824
2825SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
2826{
2827	struct swap_info_struct *p = NULL;
2828	unsigned char *swap_map;
2829	unsigned long *zeromap;
2830	struct swap_cluster_info *cluster_info;
2831	struct file *swap_file, *victim;
2832	struct address_space *mapping;
2833	struct inode *inode;
2834	struct filename *pathname;
2835	unsigned int maxpages;
2836	int err, found = 0;
2837
2838	if (!capable(CAP_SYS_ADMIN))
2839		return -EPERM;
2840
2841	BUG_ON(!current->mm);
2842
2843	pathname = getname(specialfile);
2844	if (IS_ERR(pathname))
2845		return PTR_ERR(pathname);
2846
2847	victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
2848	err = PTR_ERR(victim);
2849	if (IS_ERR(victim))
2850		goto out;
2851
2852	mapping = victim->f_mapping;
2853	spin_lock(&swap_lock);
2854	plist_for_each_entry(p, &swap_active_head, list) {
2855		if (p->flags & SWP_WRITEOK) {
2856			if (p->swap_file->f_mapping == mapping) {
2857				found = 1;
2858				break;
2859			}
2860		}
2861	}
2862	if (!found) {
2863		err = -EINVAL;
2864		spin_unlock(&swap_lock);
2865		goto out_dput;
2866	}
2867	if (!security_vm_enough_memory_mm(current->mm, p->pages))
2868		vm_unacct_memory(p->pages);
2869	else {
2870		err = -ENOMEM;
2871		spin_unlock(&swap_lock);
2872		goto out_dput;
2873	}
2874	spin_lock(&p->lock);
2875	del_from_avail_list(p, true);
2876	plist_del(&p->list, &swap_active_head);
2877	atomic_long_sub(p->pages, &nr_swap_pages);
2878	total_swap_pages -= p->pages;
2879	spin_unlock(&p->lock);
2880	spin_unlock(&swap_lock);
2881
2882	wait_for_allocation(p);
2883
2884	set_current_oom_origin();
2885	err = try_to_unuse(p->type);
2886	clear_current_oom_origin();
2887
2888	if (err) {
2889		/* re-insert swap space back into swap_list */
2890		reinsert_swap_info(p);
2891		goto out_dput;
2892	}
2893
2894	/*
2895	 * Wait for swap operations protected by get/put_swap_device()
2896	 * to complete.  Because of synchronize_rcu() here, all swap
2897	 * operations protected by RCU reader side lock (including any
2898	 * spinlock) will be waited too.  This makes it easy to
2899	 * prevent folio_test_swapcache() and the following swap cache
2900	 * operations from racing with swapoff.
2901	 */
2902	percpu_ref_kill(&p->users);
2903	synchronize_rcu();
2904	wait_for_completion(&p->comp);
2905
2906	flush_work(&p->discard_work);
2907	flush_work(&p->reclaim_work);
2908	flush_percpu_swap_cluster(p);
2909
2910	destroy_swap_extents(p);
2911	if (p->flags & SWP_CONTINUED)
2912		free_swap_count_continuations(p);
2913
2914	if (!(p->flags & SWP_SOLIDSTATE))
2915		atomic_dec(&nr_rotate_swap);
2916
2917	mutex_lock(&swapon_mutex);
2918	spin_lock(&swap_lock);
2919	spin_lock(&p->lock);
2920	drain_mmlist();
2921
2922	swap_file = p->swap_file;
2923	p->swap_file = NULL;
2924	swap_map = p->swap_map;
2925	p->swap_map = NULL;
2926	zeromap = p->zeromap;
2927	p->zeromap = NULL;
2928	maxpages = p->max;
2929	cluster_info = p->cluster_info;
2930	p->max = 0;
2931	p->cluster_info = NULL;
2932	spin_unlock(&p->lock);
2933	spin_unlock(&swap_lock);
2934	arch_swap_invalidate_area(p->type);
2935	zswap_swapoff(p->type);
2936	mutex_unlock(&swapon_mutex);
2937	kfree(p->global_cluster);
2938	p->global_cluster = NULL;
2939	vfree(swap_map);
2940	kvfree(zeromap);
2941	free_cluster_info(cluster_info, maxpages);
2942	/* Destroy swap account information */
2943	swap_cgroup_swapoff(p->type);
2944
2945	inode = mapping->host;
2946
2947	inode_lock(inode);
2948	inode->i_flags &= ~S_SWAPFILE;
2949	inode_unlock(inode);
2950	filp_close(swap_file, NULL);
2951
2952	/*
2953	 * Clear the SWP_USED flag after all resources are freed so that swapon
2954	 * can reuse this swap_info in alloc_swap_info() safely.  It is ok to
2955	 * not hold p->lock after we cleared its SWP_WRITEOK.
2956	 */
2957	spin_lock(&swap_lock);
2958	p->flags = 0;
2959	spin_unlock(&swap_lock);
2960
2961	err = 0;
2962	atomic_inc(&proc_poll_event);
2963	wake_up_interruptible(&proc_poll_wait);
2964
2965out_dput:
2966	filp_close(victim, NULL);
2967out:
2968	putname(pathname);
2969	return err;
2970}
2971
2972#ifdef CONFIG_PROC_FS
2973static __poll_t swaps_poll(struct file *file, poll_table *wait)
2974{
2975	struct seq_file *seq = file->private_data;
2976
2977	poll_wait(file, &proc_poll_wait, wait);
2978
2979	if (seq->poll_event != atomic_read(&proc_poll_event)) {
2980		seq->poll_event = atomic_read(&proc_poll_event);
2981		return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI;
2982	}
2983
2984	return EPOLLIN | EPOLLRDNORM;
2985}
2986
2987/* iterator */
2988static void *swap_start(struct seq_file *swap, loff_t *pos)
2989{
2990	struct swap_info_struct *si;
2991	int type;
2992	loff_t l = *pos;
2993
2994	mutex_lock(&swapon_mutex);
2995
2996	if (!l)
2997		return SEQ_START_TOKEN;
2998
2999	for (type = 0; (si = swap_type_to_info(type)); type++) {
3000		if (!(si->flags & SWP_USED) || !si->swap_map)
3001			continue;
3002		if (!--l)
3003			return si;
3004	}
3005
3006	return NULL;
3007}
3008
3009static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
3010{
3011	struct swap_info_struct *si = v;
3012	int type;
3013
3014	if (v == SEQ_START_TOKEN)
3015		type = 0;
3016	else
3017		type = si->type + 1;
3018
3019	++(*pos);
3020	for (; (si = swap_type_to_info(type)); type++) {
3021		if (!(si->flags & SWP_USED) || !si->swap_map)
3022			continue;
3023		return si;
3024	}
3025
3026	return NULL;
3027}
3028
3029static void swap_stop(struct seq_file *swap, void *v)
3030{
3031	mutex_unlock(&swapon_mutex);
3032}
3033
3034static int swap_show(struct seq_file *swap, void *v)
3035{
3036	struct swap_info_struct *si = v;
3037	struct file *file;
3038	int len;
3039	unsigned long bytes, inuse;
3040
3041	if (si == SEQ_START_TOKEN) {
3042		seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");
3043		return 0;
3044	}
3045
3046	bytes = K(si->pages);
3047	inuse = K(swap_usage_in_pages(si));
3048
3049	file = si->swap_file;
3050	len = seq_file_path(swap, file, " \t\n\\");
3051	seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n",
3052			len < 40 ? 40 - len : 1, " ",
3053			S_ISBLK(file_inode(file)->i_mode) ?
3054				"partition" : "file\t",
3055			bytes, bytes < 10000000 ? "\t" : "",
3056			inuse, inuse < 10000000 ? "\t" : "",
3057			si->prio);
3058	return 0;
3059}
3060
3061static const struct seq_operations swaps_op = {
3062	.start =	swap_start,
3063	.next =		swap_next,
3064	.stop =		swap_stop,
3065	.show =		swap_show
3066};
3067
3068static int swaps_open(struct inode *inode, struct file *file)
3069{
3070	struct seq_file *seq;
3071	int ret;
3072
3073	ret = seq_open(file, &swaps_op);
3074	if (ret)
3075		return ret;
3076
3077	seq = file->private_data;
3078	seq->poll_event = atomic_read(&proc_poll_event);
3079	return 0;
3080}
3081
3082static const struct proc_ops swaps_proc_ops = {
3083	.proc_flags	= PROC_ENTRY_PERMANENT,
3084	.proc_open	= swaps_open,
3085	.proc_read	= seq_read,
3086	.proc_lseek	= seq_lseek,
3087	.proc_release	= seq_release,
3088	.proc_poll	= swaps_poll,
3089};
3090
3091static int __init procswaps_init(void)
3092{
3093	proc_create("swaps", 0, NULL, &swaps_proc_ops);
3094	return 0;
3095}
3096__initcall(procswaps_init);
3097#endif /* CONFIG_PROC_FS */
3098
3099#ifdef MAX_SWAPFILES_CHECK
3100static int __init max_swapfiles_check(void)
3101{
3102	MAX_SWAPFILES_CHECK();
3103	return 0;
3104}
3105late_initcall(max_swapfiles_check);
3106#endif
3107
3108static struct swap_info_struct *alloc_swap_info(void)
3109{
3110	struct swap_info_struct *p;
3111	struct swap_info_struct *defer = NULL;
3112	unsigned int type;
3113
3114	p = kvzalloc(sizeof(struct swap_info_struct), GFP_KERNEL);
3115	if (!p)
3116		return ERR_PTR(-ENOMEM);
3117
3118	if (percpu_ref_init(&p->users, swap_users_ref_free,
3119			    PERCPU_REF_INIT_DEAD, GFP_KERNEL)) {
3120		kvfree(p);
3121		return ERR_PTR(-ENOMEM);
3122	}
3123
3124	spin_lock(&swap_lock);
3125	for (type = 0; type < nr_swapfiles; type++) {
3126		if (!(swap_info[type]->flags & SWP_USED))
3127			break;
3128	}
3129	if (type >= MAX_SWAPFILES) {
3130		spin_unlock(&swap_lock);
3131		percpu_ref_exit(&p->users);
3132		kvfree(p);
3133		return ERR_PTR(-EPERM);
3134	}
3135	if (type >= nr_swapfiles) {
3136		p->type = type;
3137		/*
3138		 * Publish the swap_info_struct after initializing it.
3139		 * Note that kvzalloc() above zeroes all its fields.
3140		 */
3141		smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */
3142		nr_swapfiles++;
3143	} else {
3144		defer = p;
3145		p = swap_info[type];
3146		/*
3147		 * Do not memset this entry: a racing procfs swap_next()
3148		 * would be relying on p->type to remain valid.
3149		 */
3150	}
3151	p->swap_extent_root = RB_ROOT;
3152	plist_node_init(&p->list, 0);
3153	plist_node_init(&p->avail_list, 0);
3154	p->flags = SWP_USED;
3155	spin_unlock(&swap_lock);
3156	if (defer) {
3157		percpu_ref_exit(&defer->users);
3158		kvfree(defer);
3159	}
3160	spin_lock_init(&p->lock);
3161	spin_lock_init(&p->cont_lock);
3162	atomic_long_set(&p->inuse_pages, SWAP_USAGE_OFFLIST_BIT);
3163	init_completion(&p->comp);
3164
3165	return p;
3166}
3167
3168static int claim_swapfile(struct swap_info_struct *si, struct inode *inode)
3169{
3170	if (S_ISBLK(inode->i_mode)) {
3171		si->bdev = I_BDEV(inode);
3172		/*
3173		 * Zoned block devices contain zones that have a sequential
3174		 * write only restriction.  Hence zoned block devices are not
3175		 * suitable for swapping.  Disallow them here.
3176		 */
3177		if (bdev_is_zoned(si->bdev))
3178			return -EINVAL;
3179		si->flags |= SWP_BLKDEV;
3180	} else if (S_ISREG(inode->i_mode)) {
3181		si->bdev = inode->i_sb->s_bdev;
3182	}
3183
3184	return 0;
3185}
3186
3187
3188/*
3189 * Find out how many pages are allowed for a single swap device. There
3190 * are two limiting factors:
3191 * 1) the number of bits for the swap offset in the swp_entry_t type, and
3192 * 2) the number of bits in the swap pte, as defined by the different
3193 * architectures.
3194 *
3195 * In order to find the largest possible bit mask, a swap entry with
3196 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
3197 * decoded to a swp_entry_t again, and finally the swap offset is
3198 * extracted.
3199 *
3200 * This will mask all the bits from the initial ~0UL mask that can't
3201 * be encoded in either the swp_entry_t or the architecture definition
3202 * of a swap pte.
3203 */
3204unsigned long generic_max_swapfile_size(void)
3205{
3206	swp_entry_t entry = swp_entry(0, ~0UL);
3207	const pte_t pte = softleaf_to_pte(entry);
3208
3209	/*
3210	 * Since the PTE can be an invalid softleaf entry (e.g. the none PTE),
3211	 * we need to do this manually.
3212	 */
3213	entry = __pte_to_swp_entry(pte);
3214	entry = swp_entry(__swp_type(entry), __swp_offset(entry));
3215
3216	return swp_offset(entry) + 1;
3217}
3218
3219/* Can be overridden by an architecture for additional checks. */
3220__weak unsigned long arch_max_swapfile_size(void)
3221{
3222	return generic_max_swapfile_size();
3223}
3224
3225static unsigned long read_swap_header(struct swap_info_struct *si,
3226					union swap_header *swap_header,
3227					struct inode *inode)
3228{
3229	int i;
3230	unsigned long maxpages;
3231	unsigned long swapfilepages;
3232	unsigned long last_page;
3233
3234	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
3235		pr_err("Unable to find swap-space signature\n");
3236		return 0;
3237	}
3238
3239	/* swap partition endianness hack... */
3240	if (swab32(swap_header->info.version) == 1) {
3241		swab32s(&swap_header->info.version);
3242		swab32s(&swap_header->info.last_page);
3243		swab32s(&swap_header->info.nr_badpages);
3244		if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3245			return 0;
3246		for (i = 0; i < swap_header->info.nr_badpages; i++)
3247			swab32s(&swap_header->info.badpages[i]);
3248	}
3249	/* Check the swap header's sub-version */
3250	if (swap_header->info.version != 1) {
3251		pr_warn("Unable to handle swap header version %d\n",
3252			swap_header->info.version);
3253		return 0;
3254	}
3255
3256	maxpages = swapfile_maximum_size;
3257	last_page = swap_header->info.last_page;
3258	if (!last_page) {
3259		pr_warn("Empty swap-file\n");
3260		return 0;
3261	}
3262	if (last_page > maxpages) {
3263		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
3264			K(maxpages), K(last_page));
3265	}
3266	if (maxpages > last_page) {
3267		maxpages = last_page + 1;
3268		/* p->max is an unsigned int: don't overflow it */
3269		if ((unsigned int)maxpages == 0)
3270			maxpages = UINT_MAX;
3271	}
3272
3273	if (!maxpages)
3274		return 0;
3275	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
3276	if (swapfilepages && maxpages > swapfilepages) {
3277		pr_warn("Swap area shorter than signature indicates\n");
3278		return 0;
3279	}
3280	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
3281		return 0;
3282	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
3283		return 0;
3284
3285	return maxpages;
3286}
3287
3288static int setup_swap_map(struct swap_info_struct *si,
3289			  union swap_header *swap_header,
3290			  unsigned char *swap_map,
3291			  unsigned long maxpages)
3292{
3293	unsigned long i;
3294
3295	swap_map[0] = SWAP_MAP_BAD; /* omit header page */
3296	for (i = 0; i < swap_header->info.nr_badpages; i++) {
3297		unsigned int page_nr = swap_header->info.badpages[i];
3298		if (page_nr == 0 || page_nr > swap_header->info.last_page)
3299			return -EINVAL;
3300		if (page_nr < maxpages) {
3301			swap_map[page_nr] = SWAP_MAP_BAD;
3302			si->pages--;
3303		}
3304	}
3305
3306	if (!si->pages) {
3307		pr_warn("Empty swap-file\n");
3308		return -EINVAL;
3309	}
3310
3311	return 0;
3312}
3313
3314static struct swap_cluster_info *setup_clusters(struct swap_info_struct *si,
3315						union swap_header *swap_header,
3316						unsigned long maxpages)
3317{
3318	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3319	struct swap_cluster_info *cluster_info;
3320	int err = -ENOMEM;
3321	unsigned long i;
3322
3323	cluster_info = kvcalloc(nr_clusters, sizeof(*cluster_info), GFP_KERNEL);
3324	if (!cluster_info)
3325		goto err;
3326
3327	for (i = 0; i < nr_clusters; i++)
3328		spin_lock_init(&cluster_info[i].lock);
3329
3330	if (!(si->flags & SWP_SOLIDSTATE)) {
3331		si->global_cluster = kmalloc(sizeof(*si->global_cluster),
3332				     GFP_KERNEL);
3333		if (!si->global_cluster)
3334			goto err;
3335		for (i = 0; i < SWAP_NR_ORDERS; i++)
3336			si->global_cluster->next[i] = SWAP_ENTRY_INVALID;
3337		spin_lock_init(&si->global_cluster_lock);
3338	}
3339
3340	/*
3341	 * Mark unusable pages as unavailable. The clusters aren't
3342	 * marked free yet, so no list operations are involved yet.
3343	 *
3344	 * See setup_swap_map(): header page, bad pages,
3345	 * and the EOF part of the last cluster.
3346	 */
3347	err = swap_cluster_setup_bad_slot(cluster_info, 0);
3348	if (err)
3349		goto err;
3350	for (i = 0; i < swap_header->info.nr_badpages; i++) {
3351		unsigned int page_nr = swap_header->info.badpages[i];
3352
3353		if (page_nr >= maxpages)
3354			continue;
3355		err = swap_cluster_setup_bad_slot(cluster_info, page_nr);
3356		if (err)
3357			goto err;
3358	}
3359	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) {
3360		err = swap_cluster_setup_bad_slot(cluster_info, i);
3361		if (err)
3362			goto err;
3363	}
3364
3365	INIT_LIST_HEAD(&si->free_clusters);
3366	INIT_LIST_HEAD(&si->full_clusters);
3367	INIT_LIST_HEAD(&si->discard_clusters);
3368
3369	for (i = 0; i < SWAP_NR_ORDERS; i++) {
3370		INIT_LIST_HEAD(&si->nonfull_clusters[i]);
3371		INIT_LIST_HEAD(&si->frag_clusters[i]);
3372	}
3373
3374	for (i = 0; i < nr_clusters; i++) {
3375		struct swap_cluster_info *ci = &cluster_info[i];
3376
3377		if (ci->count) {
3378			ci->flags = CLUSTER_FLAG_NONFULL;
3379			list_add_tail(&ci->list, &si->nonfull_clusters[0]);
3380		} else {
3381			ci->flags = CLUSTER_FLAG_FREE;
3382			list_add_tail(&ci->list, &si->free_clusters);
3383		}
3384	}
3385
3386	return cluster_info;
3387err:
3388	free_cluster_info(cluster_info, maxpages);
3389	return ERR_PTR(err);
3390}
3391
3392SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
3393{
3394	struct swap_info_struct *si;
3395	struct filename *name;
3396	struct file *swap_file = NULL;
3397	struct address_space *mapping;
3398	struct dentry *dentry;
3399	int prio;
3400	int error;
3401	union swap_header *swap_header;
3402	int nr_extents;
3403	sector_t span;
3404	unsigned long maxpages;
3405	unsigned char *swap_map = NULL;
3406	unsigned long *zeromap = NULL;
3407	struct swap_cluster_info *cluster_info = NULL;
3408	struct folio *folio = NULL;
3409	struct inode *inode = NULL;
3410	bool inced_nr_rotate_swap = false;
3411
3412	if (swap_flags & ~SWAP_FLAGS_VALID)
3413		return -EINVAL;
3414
3415	if (!capable(CAP_SYS_ADMIN))
3416		return -EPERM;
3417
3418	si = alloc_swap_info();
3419	if (IS_ERR(si))
3420		return PTR_ERR(si);
3421
3422	INIT_WORK(&si->discard_work, swap_discard_work);
3423	INIT_WORK(&si->reclaim_work, swap_reclaim_work);
3424
3425	name = getname(specialfile);
3426	if (IS_ERR(name)) {
3427		error = PTR_ERR(name);
3428		name = NULL;
3429		goto bad_swap;
3430	}
3431	swap_file = file_open_name(name, O_RDWR | O_LARGEFILE | O_EXCL, 0);
3432	if (IS_ERR(swap_file)) {
3433		error = PTR_ERR(swap_file);
3434		swap_file = NULL;
3435		goto bad_swap;
3436	}
3437
3438	si->swap_file = swap_file;
3439	mapping = swap_file->f_mapping;
3440	dentry = swap_file->f_path.dentry;
3441	inode = mapping->host;
3442
3443	error = claim_swapfile(si, inode);
3444	if (unlikely(error))
3445		goto bad_swap;
3446
3447	inode_lock(inode);
3448	if (d_unlinked(dentry) || cant_mount(dentry)) {
3449		error = -ENOENT;
3450		goto bad_swap_unlock_inode;
3451	}
3452	if (IS_SWAPFILE(inode)) {
3453		error = -EBUSY;
3454		goto bad_swap_unlock_inode;
3455	}
3456
3457	/*
3458	 * The swap subsystem needs a major overhaul to support this.
3459	 * It doesn't work yet so just disable it for now.
3460	 */
3461	if (mapping_min_folio_order(mapping) > 0) {
3462		error = -EINVAL;
3463		goto bad_swap_unlock_inode;
3464	}
3465
3466	/*
3467	 * Read the swap header.
3468	 */
3469	if (!mapping->a_ops->read_folio) {
3470		error = -EINVAL;
3471		goto bad_swap_unlock_inode;
3472	}
3473	folio = read_mapping_folio(mapping, 0, swap_file);
3474	if (IS_ERR(folio)) {
3475		error = PTR_ERR(folio);
3476		goto bad_swap_unlock_inode;
3477	}
3478	swap_header = kmap_local_folio(folio, 0);
3479
3480	maxpages = read_swap_header(si, swap_header, inode);
3481	if (unlikely(!maxpages)) {
3482		error = -EINVAL;
3483		goto bad_swap_unlock_inode;
3484	}
3485
3486	si->max = maxpages;
3487	si->pages = maxpages - 1;
3488	nr_extents = setup_swap_extents(si, &span);
3489	if (nr_extents < 0) {
3490		error = nr_extents;
3491		goto bad_swap_unlock_inode;
3492	}
3493	if (si->pages != si->max - 1) {
3494		pr_err("swap:%u != (max:%u - 1)\n", si->pages, si->max);
3495		error = -EINVAL;
3496		goto bad_swap_unlock_inode;
3497	}
3498
3499	maxpages = si->max;
3500
3501	/* OK, set up the swap map and apply the bad block list */
3502	swap_map = vzalloc(maxpages);
3503	if (!swap_map) {
3504		error = -ENOMEM;
3505		goto bad_swap_unlock_inode;
3506	}
3507
3508	error = swap_cgroup_swapon(si->type, maxpages);
3509	if (error)
3510		goto bad_swap_unlock_inode;
3511
3512	error = setup_swap_map(si, swap_header, swap_map, maxpages);
3513	if (error)
3514		goto bad_swap_unlock_inode;
3515
3516	/*
3517	 * Use kvmalloc_array instead of bitmap_zalloc as the allocation order might
3518	 * be above MAX_PAGE_ORDER incase of a large swap file.
3519	 */
3520	zeromap = kvmalloc_array(BITS_TO_LONGS(maxpages), sizeof(long),
3521				    GFP_KERNEL | __GFP_ZERO);
3522	if (!zeromap) {
3523		error = -ENOMEM;
3524		goto bad_swap_unlock_inode;
3525	}
3526
3527	if (si->bdev && bdev_stable_writes(si->bdev))
3528		si->flags |= SWP_STABLE_WRITES;
3529
3530	if (si->bdev && bdev_synchronous(si->bdev))
3531		si->flags |= SWP_SYNCHRONOUS_IO;
3532
3533	if (si->bdev && bdev_nonrot(si->bdev)) {
3534		si->flags |= SWP_SOLIDSTATE;
3535	} else {
3536		atomic_inc(&nr_rotate_swap);
3537		inced_nr_rotate_swap = true;
3538	}
3539
3540	cluster_info = setup_clusters(si, swap_header, maxpages);
3541	if (IS_ERR(cluster_info)) {
3542		error = PTR_ERR(cluster_info);
3543		cluster_info = NULL;
3544		goto bad_swap_unlock_inode;
3545	}
3546
3547	if ((swap_flags & SWAP_FLAG_DISCARD) &&
3548	    si->bdev && bdev_max_discard_sectors(si->bdev)) {
3549		/*
3550		 * When discard is enabled for swap with no particular
3551		 * policy flagged, we set all swap discard flags here in
3552		 * order to sustain backward compatibility with older
3553		 * swapon(8) releases.
3554		 */
3555		si->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
3556			     SWP_PAGE_DISCARD);
3557
3558		/*
3559		 * By flagging sys_swapon, a sysadmin can tell us to
3560		 * either do single-time area discards only, or to just
3561		 * perform discards for released swap page-clusters.
3562		 * Now it's time to adjust the p->flags accordingly.
3563		 */
3564		if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
3565			si->flags &= ~SWP_PAGE_DISCARD;
3566		else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
3567			si->flags &= ~SWP_AREA_DISCARD;
3568
3569		/* issue a swapon-time discard if it's still required */
3570		if (si->flags & SWP_AREA_DISCARD) {
3571			int err = discard_swap(si);
3572			if (unlikely(err))
3573				pr_err("swapon: discard_swap(%p): %d\n",
3574					si, err);
3575		}
3576	}
3577
3578	error = zswap_swapon(si->type, maxpages);
3579	if (error)
3580		goto bad_swap_unlock_inode;
3581
3582	/*
3583	 * Flush any pending IO and dirty mappings before we start using this
3584	 * swap device.
3585	 */
3586	inode->i_flags |= S_SWAPFILE;
3587	error = inode_drain_writes(inode);
3588	if (error) {
3589		inode->i_flags &= ~S_SWAPFILE;
3590		goto free_swap_zswap;
3591	}
3592
3593	mutex_lock(&swapon_mutex);
3594	prio = DEF_SWAP_PRIO;
3595	if (swap_flags & SWAP_FLAG_PREFER)
3596		prio = swap_flags & SWAP_FLAG_PRIO_MASK;
3597	enable_swap_info(si, prio, swap_map, cluster_info, zeromap);
3598
3599	pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s\n",
3600		K(si->pages), name->name, si->prio, nr_extents,
3601		K((unsigned long long)span),
3602		(si->flags & SWP_SOLIDSTATE) ? "SS" : "",
3603		(si->flags & SWP_DISCARDABLE) ? "D" : "",
3604		(si->flags & SWP_AREA_DISCARD) ? "s" : "",
3605		(si->flags & SWP_PAGE_DISCARD) ? "c" : "");
3606
3607	mutex_unlock(&swapon_mutex);
3608	atomic_inc(&proc_poll_event);
3609	wake_up_interruptible(&proc_poll_wait);
3610
3611	error = 0;
3612	goto out;
3613free_swap_zswap:
3614	zswap_swapoff(si->type);
3615bad_swap_unlock_inode:
3616	inode_unlock(inode);
3617bad_swap:
3618	kfree(si->global_cluster);
3619	si->global_cluster = NULL;
3620	inode = NULL;
3621	destroy_swap_extents(si);
3622	swap_cgroup_swapoff(si->type);
3623	spin_lock(&swap_lock);
3624	si->swap_file = NULL;
3625	si->flags = 0;
3626	spin_unlock(&swap_lock);
3627	vfree(swap_map);
3628	kvfree(zeromap);
3629	if (cluster_info)
3630		free_cluster_info(cluster_info, maxpages);
3631	if (inced_nr_rotate_swap)
3632		atomic_dec(&nr_rotate_swap);
3633	if (swap_file)
3634		filp_close(swap_file, NULL);
3635out:
3636	if (!IS_ERR_OR_NULL(folio))
3637		folio_release_kmap(folio, swap_header);
3638	if (name)
3639		putname(name);
3640	if (inode)
3641		inode_unlock(inode);
3642	return error;
3643}
3644
3645void si_swapinfo(struct sysinfo *val)
3646{
3647	unsigned int type;
3648	unsigned long nr_to_be_unused = 0;
3649
3650	spin_lock(&swap_lock);
3651	for (type = 0; type < nr_swapfiles; type++) {
3652		struct swap_info_struct *si = swap_info[type];
3653
3654		if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
3655			nr_to_be_unused += swap_usage_in_pages(si);
3656	}
3657	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
3658	val->totalswap = total_swap_pages + nr_to_be_unused;
3659	spin_unlock(&swap_lock);
3660}
3661
3662/*
3663 * Verify that nr swap entries are valid and increment their swap map counts.
3664 *
3665 * Returns error code in following case.
3666 * - success -> 0
3667 * - swp_entry is invalid -> EINVAL
3668 * - swap-cache reference is requested but there is already one. -> EEXIST
3669 * - swap-cache reference is requested but the entry is not used. -> ENOENT
3670 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
3671 */
3672static int __swap_duplicate(swp_entry_t entry, unsigned char usage, int nr)
3673{
3674	struct swap_info_struct *si;
3675	struct swap_cluster_info *ci;
3676	unsigned long offset;
3677	unsigned char count;
3678	unsigned char has_cache;
3679	int err, i;
3680
3681	si = swap_entry_to_info(entry);
3682	if (WARN_ON_ONCE(!si)) {
3683		pr_err("%s%08lx\n", Bad_file, entry.val);
3684		return -EINVAL;
3685	}
3686
3687	offset = swp_offset(entry);
3688	VM_WARN_ON(nr > SWAPFILE_CLUSTER - offset % SWAPFILE_CLUSTER);
3689	VM_WARN_ON(usage == 1 && nr > 1);
3690	ci = swap_cluster_lock(si, offset);
3691
3692	err = 0;
3693	for (i = 0; i < nr; i++) {
3694		count = si->swap_map[offset + i];
3695
3696		/*
3697		 * swapin_readahead() doesn't check if a swap entry is valid, so the
3698		 * swap entry could be SWAP_MAP_BAD. Check here with lock held.
3699		 */
3700		if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
3701			err = -ENOENT;
3702			goto unlock_out;
3703		}
3704
3705		has_cache = count & SWAP_HAS_CACHE;
3706		count &= ~SWAP_HAS_CACHE;
3707
3708		if (!count && !has_cache) {
3709			err = -ENOENT;
3710		} else if (usage == SWAP_HAS_CACHE) {
3711			if (has_cache)
3712				err = -EEXIST;
3713		} else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) {
3714			err = -EINVAL;
3715		}
3716
3717		if (err)
3718			goto unlock_out;
3719	}
3720
3721	for (i = 0; i < nr; i++) {
3722		count = si->swap_map[offset + i];
3723		has_cache = count & SWAP_HAS_CACHE;
3724		count &= ~SWAP_HAS_CACHE;
3725
3726		if (usage == SWAP_HAS_CACHE)
3727			has_cache = SWAP_HAS_CACHE;
3728		else if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
3729			count += usage;
3730		else if (swap_count_continued(si, offset + i, count))
3731			count = COUNT_CONTINUED;
3732		else {
3733			/*
3734			 * Don't need to rollback changes, because if
3735			 * usage == 1, there must be nr == 1.
3736			 */
3737			err = -ENOMEM;
3738			goto unlock_out;
3739		}
3740
3741		WRITE_ONCE(si->swap_map[offset + i], count | has_cache);
3742	}
3743
3744unlock_out:
3745	swap_cluster_unlock(ci);
3746	return err;
3747}
3748
3749/*
3750 * Help swapoff by noting that swap entry belongs to shmem/tmpfs
3751 * (in which case its reference count is never incremented).
3752 */
3753void swap_shmem_alloc(swp_entry_t entry, int nr)
3754{
3755	__swap_duplicate(entry, SWAP_MAP_SHMEM, nr);
3756}
3757
3758/*
3759 * Increase reference count of swap entry by 1.
3760 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
3761 * but could not be atomically allocated.  Returns 0, just as if it succeeded,
3762 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
3763 * might occur if a page table entry has got corrupted.
3764 */
3765int swap_duplicate(swp_entry_t entry)
3766{
3767	int err = 0;
3768
3769	while (!err && __swap_duplicate(entry, 1, 1) == -ENOMEM)
3770		err = add_swap_count_continuation(entry, GFP_ATOMIC);
3771	return err;
3772}
3773
3774/*
3775 * @entry: first swap entry from which we allocate nr swap cache.
3776 *
3777 * Called when allocating swap cache for existing swap entries,
3778 * This can return error codes. Returns 0 at success.
3779 * -EEXIST means there is a swap cache.
3780 * Note: return code is different from swap_duplicate().
3781 */
3782int swapcache_prepare(swp_entry_t entry, int nr)
3783{
3784	return __swap_duplicate(entry, SWAP_HAS_CACHE, nr);
3785}
3786
3787/*
3788 * Caller should ensure entries belong to the same folio so
3789 * the entries won't span cross cluster boundary.
3790 */
3791void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr)
3792{
3793	swap_entries_put_cache(si, entry, nr);
3794}
3795
3796/*
3797 * add_swap_count_continuation - called when a swap count is duplicated
3798 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
3799 * page of the original vmalloc'ed swap_map, to hold the continuation count
3800 * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
3801 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
3802 *
3803 * These continuation pages are seldom referenced: the common paths all work
3804 * on the original swap_map, only referring to a continuation page when the
3805 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
3806 *
3807 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
3808 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
3809 * can be called after dropping locks.
3810 */
3811int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
3812{
3813	struct swap_info_struct *si;
3814	struct swap_cluster_info *ci;
3815	struct page *head;
3816	struct page *page;
3817	struct page *list_page;
3818	pgoff_t offset;
3819	unsigned char count;
3820	int ret = 0;
3821
3822	/*
3823	 * When debugging, it's easier to use __GFP_ZERO here; but it's better
3824	 * for latency not to zero a page while GFP_ATOMIC and holding locks.
3825	 */
3826	page = alloc_page(gfp_mask | __GFP_HIGHMEM);
3827
3828	si = get_swap_device(entry);
3829	if (!si) {
3830		/*
3831		 * An acceptable race has occurred since the failing
3832		 * __swap_duplicate(): the swap device may be swapoff
3833		 */
3834		goto outer;
3835	}
3836
3837	offset = swp_offset(entry);
3838
3839	ci = swap_cluster_lock(si, offset);
3840
3841	count = swap_count(si->swap_map[offset]);
3842
3843	if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
3844		/*
3845		 * The higher the swap count, the more likely it is that tasks
3846		 * will race to add swap count continuation: we need to avoid
3847		 * over-provisioning.
3848		 */
3849		goto out;
3850	}
3851
3852	if (!page) {
3853		ret = -ENOMEM;
3854		goto out;
3855	}
3856
3857	head = vmalloc_to_page(si->swap_map + offset);
3858	offset &= ~PAGE_MASK;
3859
3860	spin_lock(&si->cont_lock);
3861	/*
3862	 * Page allocation does not initialize the page's lru field,
3863	 * but it does always reset its private field.
3864	 */
3865	if (!page_private(head)) {
3866		BUG_ON(count & COUNT_CONTINUED);
3867		INIT_LIST_HEAD(&head->lru);
3868		set_page_private(head, SWP_CONTINUED);
3869		si->flags |= SWP_CONTINUED;
3870	}
3871
3872	list_for_each_entry(list_page, &head->lru, lru) {
3873		unsigned char *map;
3874
3875		/*
3876		 * If the previous map said no continuation, but we've found
3877		 * a continuation page, free our allocation and use this one.
3878		 */
3879		if (!(count & COUNT_CONTINUED))
3880			goto out_unlock_cont;
3881
3882		map = kmap_local_page(list_page) + offset;
3883		count = *map;
3884		kunmap_local(map);
3885
3886		/*
3887		 * If this continuation count now has some space in it,
3888		 * free our allocation and use this one.
3889		 */
3890		if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
3891			goto out_unlock_cont;
3892	}
3893
3894	list_add_tail(&page->lru, &head->lru);
3895	page = NULL;			/* now it's attached, don't free it */
3896out_unlock_cont:
3897	spin_unlock(&si->cont_lock);
3898out:
3899	swap_cluster_unlock(ci);
3900	put_swap_device(si);
3901outer:
3902	if (page)
3903		__free_page(page);
3904	return ret;
3905}
3906
3907/*
3908 * swap_count_continued - when the original swap_map count is incremented
3909 * from SWAP_MAP_MAX, check if there is already a continuation page to carry
3910 * into, carry if so, or else fail until a new continuation page is allocated;
3911 * when the original swap_map count is decremented from 0 with continuation,
3912 * borrow from the continuation and report whether it still holds more.
3913 * Called while __swap_duplicate() or caller of swap_entry_put_locked()
3914 * holds cluster lock.
3915 */
3916static bool swap_count_continued(struct swap_info_struct *si,
3917				 pgoff_t offset, unsigned char count)
3918{
3919	struct page *head;
3920	struct page *page;
3921	unsigned char *map;
3922	bool ret;
3923
3924	head = vmalloc_to_page(si->swap_map + offset);
3925	if (page_private(head) != SWP_CONTINUED) {
3926		BUG_ON(count & COUNT_CONTINUED);
3927		return false;		/* need to add count continuation */
3928	}
3929
3930	spin_lock(&si->cont_lock);
3931	offset &= ~PAGE_MASK;
3932	page = list_next_entry(head, lru);
3933	map = kmap_local_page(page) + offset;
3934
3935	if (count == SWAP_MAP_MAX)	/* initial increment from swap_map */
3936		goto init_map;		/* jump over SWAP_CONT_MAX checks */
3937
3938	if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
3939		/*
3940		 * Think of how you add 1 to 999
3941		 */
3942		while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
3943			kunmap_local(map);
3944			page = list_next_entry(page, lru);
3945			BUG_ON(page == head);
3946			map = kmap_local_page(page) + offset;
3947		}
3948		if (*map == SWAP_CONT_MAX) {
3949			kunmap_local(map);
3950			page = list_next_entry(page, lru);
3951			if (page == head) {
3952				ret = false;	/* add count continuation */
3953				goto out;
3954			}
3955			map = kmap_local_page(page) + offset;
3956init_map:		*map = 0;		/* we didn't zero the page */
3957		}
3958		*map += 1;
3959		kunmap_local(map);
3960		while ((page = list_prev_entry(page, lru)) != head) {
3961			map = kmap_local_page(page) + offset;
3962			*map = COUNT_CONTINUED;
3963			kunmap_local(map);
3964		}
3965		ret = true;			/* incremented */
3966
3967	} else {				/* decrementing */
3968		/*
3969		 * Think of how you subtract 1 from 1000
3970		 */
3971		BUG_ON(count != COUNT_CONTINUED);
3972		while (*map == COUNT_CONTINUED) {
3973			kunmap_local(map);
3974			page = list_next_entry(page, lru);
3975			BUG_ON(page == head);
3976			map = kmap_local_page(page) + offset;
3977		}
3978		BUG_ON(*map == 0);
3979		*map -= 1;
3980		if (*map == 0)
3981			count = 0;
3982		kunmap_local(map);
3983		while ((page = list_prev_entry(page, lru)) != head) {
3984			map = kmap_local_page(page) + offset;
3985			*map = SWAP_CONT_MAX | count;
3986			count = COUNT_CONTINUED;
3987			kunmap_local(map);
3988		}
3989		ret = count == COUNT_CONTINUED;
3990	}
3991out:
3992	spin_unlock(&si->cont_lock);
3993	return ret;
3994}
3995
3996/*
3997 * free_swap_count_continuations - swapoff free all the continuation pages
3998 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
3999 */
4000static void free_swap_count_continuations(struct swap_info_struct *si)
4001{
4002	pgoff_t offset;
4003
4004	for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
4005		struct page *head;
4006		head = vmalloc_to_page(si->swap_map + offset);
4007		if (page_private(head)) {
4008			struct page *page, *next;
4009
4010			list_for_each_entry_safe(page, next, &head->lru, lru) {
4011				list_del(&page->lru);
4012				__free_page(page);
4013			}
4014		}
4015	}
4016}
4017
4018#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
4019static bool __has_usable_swap(void)
4020{
4021	return !plist_head_empty(&swap_active_head);
4022}
4023
4024void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp)
4025{
4026	struct swap_info_struct *si;
4027
4028	if (!(gfp & __GFP_IO))
4029		return;
4030
4031	if (!__has_usable_swap())
4032		return;
4033
4034	if (!blk_cgroup_congested())
4035		return;
4036
4037	/*
4038	 * We've already scheduled a throttle, avoid taking the global swap
4039	 * lock.
4040	 */
4041	if (current->throttle_disk)
4042		return;
4043
4044	spin_lock(&swap_avail_lock);
4045	plist_for_each_entry(si, &swap_avail_head, avail_list) {
4046		if (si->bdev) {
4047			blkcg_schedule_throttle(si->bdev->bd_disk, true);
4048			break;
4049		}
4050	}
4051	spin_unlock(&swap_avail_lock);
4052}
4053#endif
4054
4055static int __init swapfile_init(void)
4056{
4057	swapfile_maximum_size = arch_max_swapfile_size();
4058
4059	/*
4060	 * Once a cluster is freed, it's swap table content is read
4061	 * only, and all swap cache readers (swap_cache_*) verifies
4062	 * the content before use. So it's safe to use RCU slab here.
4063	 */
4064	if (!SWP_TABLE_USE_PAGE)
4065		swap_table_cachep = kmem_cache_create("swap_table",
4066				    sizeof(struct swap_table),
4067				    0, SLAB_PANIC | SLAB_TYPESAFE_BY_RCU, NULL);
4068
4069#ifdef CONFIG_MIGRATION
4070	if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
4071		swap_migration_ad_supported = true;
4072#endif	/* CONFIG_MIGRATION */
4073
4074	return 0;
4075}
4076subsys_initcall(swapfile_init);