snapshot.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * linux/kernel/power/snapshot.c
   4 *
   5 * This file provides system snapshot/restore functionality for swsusp.
   6 *
   7 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
   8 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
   9 */
  10
  11#define pr_fmt(fmt) "PM: hibernation: " fmt
  12
  13#include <linux/version.h>
  14#include <linux/module.h>
  15#include <linux/mm.h>
  16#include <linux/suspend.h>
  17#include <linux/delay.h>
  18#include <linux/bitops.h>
  19#include <linux/spinlock.h>
  20#include <linux/kernel.h>
  21#include <linux/pm.h>
  22#include <linux/device.h>
  23#include <linux/init.h>
  24#include <linux/memblock.h>
  25#include <linux/nmi.h>
  26#include <linux/syscalls.h>
  27#include <linux/console.h>
  28#include <linux/highmem.h>
  29#include <linux/list.h>
  30#include <linux/slab.h>
  31#include <linux/compiler.h>
  32#include <linux/ktime.h>
  33#include <linux/set_memory.h>
  34
  35#include <linux/uaccess.h>
  36#include <asm/mmu_context.h>
  37#include <asm/tlbflush.h>
  38#include <asm/io.h>
  39
  40#include "power.h"
  41
  42#if defined(CONFIG_STRICT_KERNEL_RWX) && defined(CONFIG_ARCH_HAS_SET_MEMORY)
  43static bool hibernate_restore_protection;
  44static bool hibernate_restore_protection_active;
  45
  46void enable_restore_image_protection(void)
  47{
  48	hibernate_restore_protection = true;
  49}
  50
  51static inline void hibernate_restore_protection_begin(void)
  52{
  53	hibernate_restore_protection_active = hibernate_restore_protection;
  54}
  55
  56static inline void hibernate_restore_protection_end(void)
  57{
  58	hibernate_restore_protection_active = false;
  59}
  60
  61static inline int __must_check hibernate_restore_protect_page(void *page_address)
  62{
  63	if (hibernate_restore_protection_active)
  64		return set_memory_ro((unsigned long)page_address, 1);
  65	return 0;
  66}
  67
  68static inline int hibernate_restore_unprotect_page(void *page_address)
  69{
  70	if (hibernate_restore_protection_active)
  71		return set_memory_rw((unsigned long)page_address, 1);
  72	return 0;
  73}
  74#else
  75static inline void hibernate_restore_protection_begin(void) {}
  76static inline void hibernate_restore_protection_end(void) {}
  77static inline int __must_check hibernate_restore_protect_page(void *page_address) {return 0; }
  78static inline int hibernate_restore_unprotect_page(void *page_address) {return 0; }
  79#endif /* CONFIG_STRICT_KERNEL_RWX  && CONFIG_ARCH_HAS_SET_MEMORY */
  80
  81
  82/*
  83 * The calls to set_direct_map_*() should not fail because remapping a page
  84 * here means that we only update protection bits in an existing PTE.
  85 * It is still worth to have a warning here if something changes and this
  86 * will no longer be the case.
  87 */
  88static inline void hibernate_map_page(struct page *page)
  89{
  90	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {
  91		int ret = set_direct_map_default_noflush(page);
  92
  93		if (ret)
  94			pr_warn_once("Failed to remap page\n");
  95	} else {
  96		debug_pagealloc_map_pages(page, 1);
  97	}
  98}
  99
 100static inline void hibernate_unmap_page(struct page *page)
 101{
 102	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {
 103		unsigned long addr = (unsigned long)page_address(page);
 104		int ret  = set_direct_map_invalid_noflush(page);
 105
 106		if (ret)
 107			pr_warn_once("Failed to remap page\n");
 108
 109		flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
 110	} else {
 111		debug_pagealloc_unmap_pages(page, 1);
 112	}
 113}
 114
 115static int swsusp_page_is_free(struct page *);
 116static void swsusp_set_page_forbidden(struct page *);
 117static void swsusp_unset_page_forbidden(struct page *);
 118
 119/*
 120 * Number of bytes to reserve for memory allocations made by device drivers
 121 * from their ->freeze() and ->freeze_noirq() callbacks so that they don't
 122 * cause image creation to fail (tunable via /sys/power/reserved_size).
 123 */
 124unsigned long reserved_size;
 125
 126void __init hibernate_reserved_size_init(void)
 127{
 128	reserved_size = SPARE_PAGES * PAGE_SIZE;
 129}
 130
 131/*
 132 * Preferred image size in bytes (tunable via /sys/power/image_size).
 133 * When it is set to N, swsusp will do its best to ensure the image
 134 * size will not exceed N bytes, but if that is impossible, it will
 135 * try to create the smallest image possible.
 136 */
 137unsigned long image_size;
 138
 139void __init hibernate_image_size_init(void)
 140{
 141	image_size = ((totalram_pages() * 2) / 5) * PAGE_SIZE;
 142}
 143
 144/*
 145 * List of PBEs needed for restoring the pages that were allocated before
 146 * the suspend and included in the suspend image, but have also been
 147 * allocated by the "resume" kernel, so their contents cannot be written
 148 * directly to their "original" page frames.
 149 */
 150struct pbe *restore_pblist;
 151
 152/* struct linked_page is used to build chains of pages */
 153
 154#define LINKED_PAGE_DATA_SIZE	(PAGE_SIZE - sizeof(void *))
 155
 156struct linked_page {
 157	struct linked_page *next;
 158	char data[LINKED_PAGE_DATA_SIZE];
 159} __packed;
 160
 161/*
 162 * List of "safe" pages (ie. pages that were not used by the image kernel
 163 * before hibernation) that may be used as temporary storage for image kernel
 164 * memory contents.
 165 */
 166static struct linked_page *safe_pages_list;
 167
 168/* Pointer to an auxiliary buffer (1 page) */
 169static void *buffer;
 170
 171#define PG_ANY		0
 172#define PG_SAFE		1
 173#define PG_UNSAFE_CLEAR	1
 174#define PG_UNSAFE_KEEP	0
 175
 176static unsigned int allocated_unsafe_pages;
 177
 178/**
 179 * get_image_page - Allocate a page for a hibernation image.
 180 * @gfp_mask: GFP mask for the allocation.
 181 * @safe_needed: Get pages that were not used before hibernation (restore only)
 182 *
 183 * During image restoration, for storing the PBE list and the image data, we can
 184 * only use memory pages that do not conflict with the pages used before
 185 * hibernation.  The "unsafe" pages have PageNosaveFree set and we count them
 186 * using allocated_unsafe_pages.
 187 *
 188 * Each allocated image page is marked as PageNosave and PageNosaveFree so that
 189 * swsusp_free() can release it.
 190 */
 191static void *get_image_page(gfp_t gfp_mask, int safe_needed)
 192{
 193	void *res;
 194
 195	res = (void *)get_zeroed_page(gfp_mask);
 196	if (safe_needed)
 197		while (res && swsusp_page_is_free(virt_to_page(res))) {
 198			/* The page is unsafe, mark it for swsusp_free() */
 199			swsusp_set_page_forbidden(virt_to_page(res));
 200			allocated_unsafe_pages++;
 201			res = (void *)get_zeroed_page(gfp_mask);
 202		}
 203	if (res) {
 204		swsusp_set_page_forbidden(virt_to_page(res));
 205		swsusp_set_page_free(virt_to_page(res));
 206	}
 207	return res;
 208}
 209
 210static void *__get_safe_page(gfp_t gfp_mask)
 211{
 212	if (safe_pages_list) {
 213		void *ret = safe_pages_list;
 214
 215		safe_pages_list = safe_pages_list->next;
 216		memset(ret, 0, PAGE_SIZE);
 217		return ret;
 218	}
 219	return get_image_page(gfp_mask, PG_SAFE);
 220}
 221
 222unsigned long get_safe_page(gfp_t gfp_mask)
 223{
 224	return (unsigned long)__get_safe_page(gfp_mask);
 225}
 226
 227static struct page *alloc_image_page(gfp_t gfp_mask)
 228{
 229	struct page *page;
 230
 231	page = alloc_page(gfp_mask);
 232	if (page) {
 233		swsusp_set_page_forbidden(page);
 234		swsusp_set_page_free(page);
 235	}
 236	return page;
 237}
 238
 239static void recycle_safe_page(void *page_address)
 240{
 241	struct linked_page *lp = page_address;
 242
 243	lp->next = safe_pages_list;
 244	safe_pages_list = lp;
 245}
 246
 247/**
 248 * free_image_page - Free a page allocated for hibernation image.
 249 * @addr: Address of the page to free.
 250 * @clear_nosave_free: If set, clear the PageNosaveFree bit for the page.
 251 *
 252 * The page to free should have been allocated by get_image_page() (page flags
 253 * set by it are affected).
 254 */
 255static inline void free_image_page(void *addr, int clear_nosave_free)
 256{
 257	struct page *page;
 258
 259	BUG_ON(!virt_addr_valid(addr));
 260
 261	page = virt_to_page(addr);
 262
 263	swsusp_unset_page_forbidden(page);
 264	if (clear_nosave_free)
 265		swsusp_unset_page_free(page);
 266
 267	__free_page(page);
 268}
 269
 270static inline void free_list_of_pages(struct linked_page *list,
 271				      int clear_page_nosave)
 272{
 273	while (list) {
 274		struct linked_page *lp = list->next;
 275
 276		free_image_page(list, clear_page_nosave);
 277		list = lp;
 278	}
 279}
 280
 281/*
 282 * struct chain_allocator is used for allocating small objects out of
 283 * a linked list of pages called 'the chain'.
 284 *
 285 * The chain grows each time when there is no room for a new object in
 286 * the current page.  The allocated objects cannot be freed individually.
 287 * It is only possible to free them all at once, by freeing the entire
 288 * chain.
 289 *
 290 * NOTE: The chain allocator may be inefficient if the allocated objects
 291 * are not much smaller than PAGE_SIZE.
 292 */
 293struct chain_allocator {
 294	struct linked_page *chain;	/* the chain */
 295	unsigned int used_space;	/* total size of objects allocated out
 296					   of the current page */
 297	gfp_t gfp_mask;		/* mask for allocating pages */
 298	int safe_needed;	/* if set, only "safe" pages are allocated */
 299};
 300
 301static void chain_init(struct chain_allocator *ca, gfp_t gfp_mask,
 302		       int safe_needed)
 303{
 304	ca->chain = NULL;
 305	ca->used_space = LINKED_PAGE_DATA_SIZE;
 306	ca->gfp_mask = gfp_mask;
 307	ca->safe_needed = safe_needed;
 308}
 309
 310static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
 311{
 312	void *ret;
 313
 314	if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
 315		struct linked_page *lp;
 316
 317		lp = ca->safe_needed ? __get_safe_page(ca->gfp_mask) :
 318					get_image_page(ca->gfp_mask, PG_ANY);
 319		if (!lp)
 320			return NULL;
 321
 322		lp->next = ca->chain;
 323		ca->chain = lp;
 324		ca->used_space = 0;
 325	}
 326	ret = ca->chain->data + ca->used_space;
 327	ca->used_space += size;
 328	return ret;
 329}
 330
 331/*
 332 * Data types related to memory bitmaps.
 333 *
 334 * Memory bitmap is a structure consisting of many linked lists of
 335 * objects.  The main list's elements are of type struct zone_bitmap
 336 * and each of them corresponds to one zone.  For each zone bitmap
 337 * object there is a list of objects of type struct bm_block that
 338 * represent each blocks of bitmap in which information is stored.
 339 *
 340 * struct memory_bitmap contains a pointer to the main list of zone
 341 * bitmap objects, a struct bm_position used for browsing the bitmap,
 342 * and a pointer to the list of pages used for allocating all of the
 343 * zone bitmap objects and bitmap block objects.
 344 *
 345 * NOTE: It has to be possible to lay out the bitmap in memory
 346 * using only allocations of order 0.  Additionally, the bitmap is
 347 * designed to work with arbitrary number of zones (this is over the
 348 * top for now, but let's avoid making unnecessary assumptions ;-).
 349 *
 350 * struct zone_bitmap contains a pointer to a list of bitmap block
 351 * objects and a pointer to the bitmap block object that has been
 352 * most recently used for setting bits.  Additionally, it contains the
 353 * PFNs that correspond to the start and end of the represented zone.
 354 *
 355 * struct bm_block contains a pointer to the memory page in which
 356 * information is stored (in the form of a block of bitmap)
 357 * It also contains the pfns that correspond to the start and end of
 358 * the represented memory area.
 359 *
 360 * The memory bitmap is organized as a radix tree to guarantee fast random
 361 * access to the bits. There is one radix tree for each zone (as returned
 362 * from create_mem_extents).
 363 *
 364 * One radix tree is represented by one struct mem_zone_bm_rtree. There are
 365 * two linked lists for the nodes of the tree, one for the inner nodes and
 366 * one for the leaf nodes. The linked leaf nodes are used for fast linear
 367 * access of the memory bitmap.
 368 *
 369 * The struct rtree_node represents one node of the radix tree.
 370 */
 371
 372#define BM_END_OF_MAP	(~0UL)
 373
 374#define BM_BITS_PER_BLOCK	(PAGE_SIZE * BITS_PER_BYTE)
 375#define BM_BLOCK_SHIFT		(PAGE_SHIFT + 3)
 376#define BM_BLOCK_MASK		((1UL << BM_BLOCK_SHIFT) - 1)
 377
 378/*
 379 * struct rtree_node is a wrapper struct to link the nodes
 380 * of the rtree together for easy linear iteration over
 381 * bits and easy freeing
 382 */
 383struct rtree_node {
 384	struct list_head list;
 385	unsigned long *data;
 386};
 387
 388/*
 389 * struct mem_zone_bm_rtree represents a bitmap used for one
 390 * populated memory zone.
 391 */
 392struct mem_zone_bm_rtree {
 393	struct list_head list;		/* Link Zones together         */
 394	struct list_head nodes;		/* Radix Tree inner nodes      */
 395	struct list_head leaves;	/* Radix Tree leaves           */
 396	unsigned long start_pfn;	/* Zone start page frame       */
 397	unsigned long end_pfn;		/* Zone end page frame + 1     */
 398	struct rtree_node *rtree;	/* Radix Tree Root             */
 399	int levels;			/* Number of Radix Tree Levels */
 400	unsigned int blocks;		/* Number of Bitmap Blocks     */
 401};
 402
 403/* struct bm_position is used for browsing memory bitmaps */
 404
 405struct bm_position {
 406	struct mem_zone_bm_rtree *zone;
 407	struct rtree_node *node;
 408	unsigned long node_pfn;
 409	unsigned long cur_pfn;
 410	int node_bit;
 411};
 412
 413struct memory_bitmap {
 414	struct list_head zones;
 415	struct linked_page *p_list;	/* list of pages used to store zone
 416					   bitmap objects and bitmap block
 417					   objects */
 418	struct bm_position cur;	/* most recently used bit position */
 419};
 420
 421/* Functions that operate on memory bitmaps */
 422
 423#define BM_ENTRIES_PER_LEVEL	(PAGE_SIZE / sizeof(unsigned long))
 424#if BITS_PER_LONG == 32
 425#define BM_RTREE_LEVEL_SHIFT	(PAGE_SHIFT - 2)
 426#else
 427#define BM_RTREE_LEVEL_SHIFT	(PAGE_SHIFT - 3)
 428#endif
 429#define BM_RTREE_LEVEL_MASK	((1UL << BM_RTREE_LEVEL_SHIFT) - 1)
 430
 431/**
 432 * alloc_rtree_node - Allocate a new node and add it to the radix tree.
 433 * @gfp_mask: GFP mask for the allocation.
 434 * @safe_needed: Get pages not used before hibernation (restore only)
 435 * @ca: Pointer to a linked list of pages ("a chain") to allocate from
 436 * @list: Radix Tree node to add.
 437 *
 438 * This function is used to allocate inner nodes as well as the
 439 * leave nodes of the radix tree. It also adds the node to the
 440 * corresponding linked list passed in by the *list parameter.
 441 */
 442static struct rtree_node *alloc_rtree_node(gfp_t gfp_mask, int safe_needed,
 443					   struct chain_allocator *ca,
 444					   struct list_head *list)
 445{
 446	struct rtree_node *node;
 447
 448	node = chain_alloc(ca, sizeof(struct rtree_node));
 449	if (!node)
 450		return NULL;
 451
 452	node->data = get_image_page(gfp_mask, safe_needed);
 453	if (!node->data)
 454		return NULL;
 455
 456	list_add_tail(&node->list, list);
 457
 458	return node;
 459}
 460
 461/**
 462 * add_rtree_block - Add a new leave node to the radix tree.
 463 *
 464 * The leave nodes need to be allocated in order to keep the leaves
 465 * linked list in order. This is guaranteed by the zone->blocks
 466 * counter.
 467 */
 468static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,
 469			   int safe_needed, struct chain_allocator *ca)
 470{
 471	struct rtree_node *node, *block, **dst;
 472	unsigned int levels_needed, block_nr;
 473	int i;
 474
 475	block_nr = zone->blocks;
 476	levels_needed = 0;
 477
 478	/* How many levels do we need for this block nr? */
 479	while (block_nr) {
 480		levels_needed += 1;
 481		block_nr >>= BM_RTREE_LEVEL_SHIFT;
 482	}
 483
 484	/* Make sure the rtree has enough levels */
 485	for (i = zone->levels; i < levels_needed; i++) {
 486		node = alloc_rtree_node(gfp_mask, safe_needed, ca,
 487					&zone->nodes);
 488		if (!node)
 489			return -ENOMEM;
 490
 491		node->data[0] = (unsigned long)zone->rtree;
 492		zone->rtree = node;
 493		zone->levels += 1;
 494	}
 495
 496	/* Allocate new block */
 497	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);
 498	if (!block)
 499		return -ENOMEM;
 500
 501	/* Now walk the rtree to insert the block */
 502	node = zone->rtree;
 503	dst = &zone->rtree;
 504	block_nr = zone->blocks;
 505	for (i = zone->levels; i > 0; i--) {
 506		int index;
 507
 508		if (!node) {
 509			node = alloc_rtree_node(gfp_mask, safe_needed, ca,
 510						&zone->nodes);
 511			if (!node)
 512				return -ENOMEM;
 513			*dst = node;
 514		}
 515
 516		index = block_nr >> ((i - 1) * BM_RTREE_LEVEL_SHIFT);
 517		index &= BM_RTREE_LEVEL_MASK;
 518		dst = (struct rtree_node **)&((*dst)->data[index]);
 519		node = *dst;
 520	}
 521
 522	zone->blocks += 1;
 523	*dst = block;
 524
 525	return 0;
 526}
 527
 528static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
 529			       int clear_nosave_free);
 530
 531/**
 532 * create_zone_bm_rtree - Create a radix tree for one zone.
 533 *
 534 * Allocated the mem_zone_bm_rtree structure and initializes it.
 535 * This function also allocated and builds the radix tree for the
 536 * zone.
 537 */
 538static struct mem_zone_bm_rtree *create_zone_bm_rtree(gfp_t gfp_mask,
 539						      int safe_needed,
 540						      struct chain_allocator *ca,
 541						      unsigned long start,
 542						      unsigned long end)
 543{
 544	struct mem_zone_bm_rtree *zone;
 545	unsigned int i, nr_blocks;
 546	unsigned long pages;
 547
 548	pages = end - start;
 549	zone  = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));
 550	if (!zone)
 551		return NULL;
 552
 553	INIT_LIST_HEAD(&zone->nodes);
 554	INIT_LIST_HEAD(&zone->leaves);
 555	zone->start_pfn = start;
 556	zone->end_pfn = end;
 557	nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
 558
 559	for (i = 0; i < nr_blocks; i++) {
 560		if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {
 561			free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);
 562			return NULL;
 563		}
 564	}
 565
 566	return zone;
 567}
 568
 569/**
 570 * free_zone_bm_rtree - Free the memory of the radix tree.
 571 *
 572 * Free all node pages of the radix tree. The mem_zone_bm_rtree
 573 * structure itself is not freed here nor are the rtree_node
 574 * structs.
 575 */
 576static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
 577			       int clear_nosave_free)
 578{
 579	struct rtree_node *node;
 580
 581	list_for_each_entry(node, &zone->nodes, list)
 582		free_image_page(node->data, clear_nosave_free);
 583
 584	list_for_each_entry(node, &zone->leaves, list)
 585		free_image_page(node->data, clear_nosave_free);
 586}
 587
 588static void memory_bm_position_reset(struct memory_bitmap *bm)
 589{
 590	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,
 591				  list);
 592	bm->cur.node = list_entry(bm->cur.zone->leaves.next,
 593				  struct rtree_node, list);
 594	bm->cur.node_pfn = 0;
 595	bm->cur.cur_pfn = BM_END_OF_MAP;
 596	bm->cur.node_bit = 0;
 597}
 598
 599static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
 600
 601struct mem_extent {
 602	struct list_head hook;
 603	unsigned long start;
 604	unsigned long end;
 605};
 606
 607/**
 608 * free_mem_extents - Free a list of memory extents.
 609 * @list: List of extents to free.
 610 */
 611static void free_mem_extents(struct list_head *list)
 612{
 613	struct mem_extent *ext, *aux;
 614
 615	list_for_each_entry_safe(ext, aux, list, hook) {
 616		list_del(&ext->hook);
 617		kfree(ext);
 618	}
 619}
 620
 621/**
 622 * create_mem_extents - Create a list of memory extents.
 623 * @list: List to put the extents into.
 624 * @gfp_mask: Mask to use for memory allocations.
 625 *
 626 * The extents represent contiguous ranges of PFNs.
 627 */
 628static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
 629{
 630	struct zone *zone;
 631
 632	INIT_LIST_HEAD(list);
 633
 634	for_each_populated_zone(zone) {
 635		unsigned long zone_start, zone_end;
 636		struct mem_extent *ext, *cur, *aux;
 637
 638		zone_start = zone->zone_start_pfn;
 639		zone_end = zone_end_pfn(zone);
 640
 641		list_for_each_entry(ext, list, hook)
 642			if (zone_start <= ext->end)
 643				break;
 644
 645		if (&ext->hook == list || zone_end < ext->start) {
 646			/* New extent is necessary */
 647			struct mem_extent *new_ext;
 648
 649			new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask);
 650			if (!new_ext) {
 651				free_mem_extents(list);
 652				return -ENOMEM;
 653			}
 654			new_ext->start = zone_start;
 655			new_ext->end = zone_end;
 656			list_add_tail(&new_ext->hook, &ext->hook);
 657			continue;
 658		}
 659
 660		/* Merge this zone's range of PFNs with the existing one */
 661		if (zone_start < ext->start)
 662			ext->start = zone_start;
 663		if (zone_end > ext->end)
 664			ext->end = zone_end;
 665
 666		/* More merging may be possible */
 667		cur = ext;
 668		list_for_each_entry_safe_continue(cur, aux, list, hook) {
 669			if (zone_end < cur->start)
 670				break;
 671			if (zone_end < cur->end)
 672				ext->end = cur->end;
 673			list_del(&cur->hook);
 674			kfree(cur);
 675		}
 676	}
 677
 678	return 0;
 679}
 680
 681/**
 682 * memory_bm_create - Allocate memory for a memory bitmap.
 683 */
 684static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask,
 685			    int safe_needed)
 686{
 687	struct chain_allocator ca;
 688	struct list_head mem_extents;
 689	struct mem_extent *ext;
 690	int error;
 691
 692	chain_init(&ca, gfp_mask, safe_needed);
 693	INIT_LIST_HEAD(&bm->zones);
 694
 695	error = create_mem_extents(&mem_extents, gfp_mask);
 696	if (error)
 697		return error;
 698
 699	list_for_each_entry(ext, &mem_extents, hook) {
 700		struct mem_zone_bm_rtree *zone;
 701
 702		zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,
 703					    ext->start, ext->end);
 704		if (!zone) {
 705			error = -ENOMEM;
 706			goto Error;
 707		}
 708		list_add_tail(&zone->list, &bm->zones);
 709	}
 710
 711	bm->p_list = ca.chain;
 712	memory_bm_position_reset(bm);
 713 Exit:
 714	free_mem_extents(&mem_extents);
 715	return error;
 716
 717 Error:
 718	bm->p_list = ca.chain;
 719	memory_bm_free(bm, PG_UNSAFE_CLEAR);
 720	goto Exit;
 721}
 722
 723/**
 724 * memory_bm_free - Free memory occupied by the memory bitmap.
 725 * @bm: Memory bitmap.
 726 */
 727static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
 728{
 729	struct mem_zone_bm_rtree *zone;
 730
 731	list_for_each_entry(zone, &bm->zones, list)
 732		free_zone_bm_rtree(zone, clear_nosave_free);
 733
 734	free_list_of_pages(bm->p_list, clear_nosave_free);
 735
 736	INIT_LIST_HEAD(&bm->zones);
 737}
 738
 739/**
 740 * memory_bm_find_bit - Find the bit for a given PFN in a memory bitmap.
 741 *
 742 * Find the bit in memory bitmap @bm that corresponds to the given PFN.
 743 * The cur.zone, cur.block and cur.node_pfn members of @bm are updated.
 744 *
 745 * Walk the radix tree to find the page containing the bit that represents @pfn
 746 * and return the position of the bit in @addr and @bit_nr.
 747 */
 748static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
 749			      void **addr, unsigned int *bit_nr)
 750{
 751	struct mem_zone_bm_rtree *curr, *zone;
 752	struct rtree_node *node;
 753	int i, block_nr;
 754
 755	zone = bm->cur.zone;
 756
 757	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)
 758		goto zone_found;
 759
 760	zone = NULL;
 761
 762	/* Find the right zone */
 763	list_for_each_entry(curr, &bm->zones, list) {
 764		if (pfn >= curr->start_pfn && pfn < curr->end_pfn) {
 765			zone = curr;
 766			break;
 767		}
 768	}
 769
 770	if (!zone)
 771		return -EFAULT;
 772
 773zone_found:
 774	/*
 775	 * We have found the zone. Now walk the radix tree to find the leaf node
 776	 * for our PFN.
 777	 */
 778
 779	/*
 780	 * If the zone we wish to scan is the current zone and the
 781	 * pfn falls into the current node then we do not need to walk
 782	 * the tree.
 783	 */
 784	node = bm->cur.node;
 785	if (zone == bm->cur.zone &&
 786	    ((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)
 787		goto node_found;
 788
 789	node      = zone->rtree;
 790	block_nr  = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;
 791
 792	for (i = zone->levels; i > 0; i--) {
 793		int index;
 794
 795		index = block_nr >> ((i - 1) * BM_RTREE_LEVEL_SHIFT);
 796		index &= BM_RTREE_LEVEL_MASK;
 797		BUG_ON(node->data[index] == 0);
 798		node = (struct rtree_node *)node->data[index];
 799	}
 800
 801node_found:
 802	/* Update last position */
 803	bm->cur.zone = zone;
 804	bm->cur.node = node;
 805	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;
 806	bm->cur.cur_pfn = pfn;
 807
 808	/* Set return values */
 809	*addr = node->data;
 810	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;
 811
 812	return 0;
 813}
 814
 815static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
 816{
 817	void *addr;
 818	unsigned int bit;
 819	int error;
 820
 821	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 822	BUG_ON(error);
 823	set_bit(bit, addr);
 824}
 825
 826static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
 827{
 828	void *addr;
 829	unsigned int bit;
 830	int error;
 831
 832	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 833	if (!error)
 834		set_bit(bit, addr);
 835
 836	return error;
 837}
 838
 839static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
 840{
 841	void *addr;
 842	unsigned int bit;
 843	int error;
 844
 845	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 846	BUG_ON(error);
 847	clear_bit(bit, addr);
 848}
 849
 850static void memory_bm_clear_current(struct memory_bitmap *bm)
 851{
 852	int bit;
 853
 854	bit = max(bm->cur.node_bit - 1, 0);
 855	clear_bit(bit, bm->cur.node->data);
 856}
 857
 858static unsigned long memory_bm_get_current(struct memory_bitmap *bm)
 859{
 860	return bm->cur.cur_pfn;
 861}
 862
 863static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
 864{
 865	void *addr;
 866	unsigned int bit;
 867	int error;
 868
 869	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
 870	BUG_ON(error);
 871	return test_bit(bit, addr);
 872}
 873
 874static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
 875{
 876	void *addr;
 877	unsigned int bit;
 878
 879	return !memory_bm_find_bit(bm, pfn, &addr, &bit);
 880}
 881
 882/*
 883 * rtree_next_node - Jump to the next leaf node.
 884 *
 885 * Set the position to the beginning of the next node in the
 886 * memory bitmap. This is either the next node in the current
 887 * zone's radix tree or the first node in the radix tree of the
 888 * next zone.
 889 *
 890 * Return true if there is a next node, false otherwise.
 891 */
 892static bool rtree_next_node(struct memory_bitmap *bm)
 893{
 894	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {
 895		bm->cur.node = list_entry(bm->cur.node->list.next,
 896					  struct rtree_node, list);
 897		bm->cur.node_pfn += BM_BITS_PER_BLOCK;
 898		bm->cur.node_bit  = 0;
 899		touch_softlockup_watchdog();
 900		return true;
 901	}
 902
 903	/* No more nodes, goto next zone */
 904	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {
 905		bm->cur.zone = list_entry(bm->cur.zone->list.next,
 906				  struct mem_zone_bm_rtree, list);
 907		bm->cur.node = list_entry(bm->cur.zone->leaves.next,
 908					  struct rtree_node, list);
 909		bm->cur.node_pfn = 0;
 910		bm->cur.node_bit = 0;
 911		return true;
 912	}
 913
 914	/* No more zones */
 915	return false;
 916}
 917
 918/**
 919 * memory_bm_next_pfn - Find the next set bit in a memory bitmap.
 920 * @bm: Memory bitmap.
 921 *
 922 * Starting from the last returned position this function searches for the next
 923 * set bit in @bm and returns the PFN represented by it.  If no more bits are
 924 * set, BM_END_OF_MAP is returned.
 925 *
 926 * It is required to run memory_bm_position_reset() before the first call to
 927 * this function for the given memory bitmap.
 928 */
 929static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
 930{
 931	unsigned long bits, pfn, pages;
 932	int bit;
 933
 934	do {
 935		pages	  = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;
 936		bits      = min(pages - bm->cur.node_pfn, BM_BITS_PER_BLOCK);
 937		bit	  = find_next_bit(bm->cur.node->data, bits,
 938					  bm->cur.node_bit);
 939		if (bit < bits) {
 940			pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;
 941			bm->cur.node_bit = bit + 1;
 942			bm->cur.cur_pfn = pfn;
 943			return pfn;
 944		}
 945	} while (rtree_next_node(bm));
 946
 947	bm->cur.cur_pfn = BM_END_OF_MAP;
 948	return BM_END_OF_MAP;
 949}
 950
 951/*
 952 * This structure represents a range of page frames the contents of which
 953 * should not be saved during hibernation.
 954 */
 955struct nosave_region {
 956	struct list_head list;
 957	unsigned long start_pfn;
 958	unsigned long end_pfn;
 959};
 960
 961static LIST_HEAD(nosave_regions);
 962
 963static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)
 964{
 965	struct rtree_node *node;
 966
 967	list_for_each_entry(node, &zone->nodes, list)
 968		recycle_safe_page(node->data);
 969
 970	list_for_each_entry(node, &zone->leaves, list)
 971		recycle_safe_page(node->data);
 972}
 973
 974static void memory_bm_recycle(struct memory_bitmap *bm)
 975{
 976	struct mem_zone_bm_rtree *zone;
 977	struct linked_page *p_list;
 978
 979	list_for_each_entry(zone, &bm->zones, list)
 980		recycle_zone_bm_rtree(zone);
 981
 982	p_list = bm->p_list;
 983	while (p_list) {
 984		struct linked_page *lp = p_list;
 985
 986		p_list = lp->next;
 987		recycle_safe_page(lp);
 988	}
 989}
 990
 991/**
 992 * register_nosave_region - Register a region of unsaveable memory.
 993 *
 994 * Register a range of page frames the contents of which should not be saved
 995 * during hibernation (to be used in the early initialization code).
 996 */
 997void __init register_nosave_region(unsigned long start_pfn, unsigned long end_pfn)
 998{
 999	struct nosave_region *region;
1000
1001	if (start_pfn >= end_pfn)
1002		return;
1003
1004	if (!list_empty(&nosave_regions)) {
1005		/* Try to extend the previous region (they should be sorted) */
1006		region = list_entry(nosave_regions.prev,
1007					struct nosave_region, list);
1008		if (region->end_pfn == start_pfn) {
1009			region->end_pfn = end_pfn;
1010			goto Report;
1011		}
1012	}
1013	/* This allocation cannot fail */
1014	region = memblock_alloc_or_panic(sizeof(struct nosave_region),
1015				SMP_CACHE_BYTES);
1016	region->start_pfn = start_pfn;
1017	region->end_pfn = end_pfn;
1018	list_add_tail(&region->list, &nosave_regions);
1019 Report:
1020	pr_info("Registered nosave memory: [mem %#010llx-%#010llx]\n",
1021		(unsigned long long) start_pfn << PAGE_SHIFT,
1022		((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1023}
1024
1025/*
1026 * Set bits in this map correspond to the page frames the contents of which
1027 * should not be saved during the suspend.
1028 */
1029static struct memory_bitmap *forbidden_pages_map;
1030
1031/* Set bits in this map correspond to free page frames. */
1032static struct memory_bitmap *free_pages_map;
1033
1034/*
1035 * Each page frame allocated for creating the image is marked by setting the
1036 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
1037 */
1038
1039void swsusp_set_page_free(struct page *page)
1040{
1041	if (free_pages_map)
1042		memory_bm_set_bit(free_pages_map, page_to_pfn(page));
1043}
1044
1045static int swsusp_page_is_free(struct page *page)
1046{
1047	return free_pages_map ?
1048		memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
1049}
1050
1051void swsusp_unset_page_free(struct page *page)
1052{
1053	if (free_pages_map)
1054		memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
1055}
1056
1057static void swsusp_set_page_forbidden(struct page *page)
1058{
1059	if (forbidden_pages_map)
1060		memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
1061}
1062
1063int swsusp_page_is_forbidden(struct page *page)
1064{
1065	return forbidden_pages_map ?
1066		memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
1067}
1068
1069static void swsusp_unset_page_forbidden(struct page *page)
1070{
1071	if (forbidden_pages_map)
1072		memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
1073}
1074
1075/**
1076 * mark_nosave_pages - Mark pages that should not be saved.
1077 * @bm: Memory bitmap.
1078 *
1079 * Set the bits in @bm that correspond to the page frames the contents of which
1080 * should not be saved.
1081 */
1082static void mark_nosave_pages(struct memory_bitmap *bm)
1083{
1084	struct nosave_region *region;
1085
1086	if (list_empty(&nosave_regions))
1087		return;
1088
1089	list_for_each_entry(region, &nosave_regions, list) {
1090		unsigned long pfn;
1091
1092		pr_debug("Marking nosave pages: [mem %#010llx-%#010llx]\n",
1093			 (unsigned long long) region->start_pfn << PAGE_SHIFT,
1094			 ((unsigned long long) region->end_pfn << PAGE_SHIFT)
1095				- 1);
1096
1097		for_each_valid_pfn(pfn, region->start_pfn, region->end_pfn) {
1098			/*
1099			 * It is safe to ignore the result of
1100			 * mem_bm_set_bit_check() here, since we won't
1101			 * touch the PFNs for which the error is
1102			 * returned anyway.
1103			 */
1104			mem_bm_set_bit_check(bm, pfn);
1105		}
1106	}
1107}
1108
1109/**
1110 * create_basic_memory_bitmaps - Create bitmaps to hold basic page information.
1111 *
1112 * Create bitmaps needed for marking page frames that should not be saved and
1113 * free page frames.  The forbidden_pages_map and free_pages_map pointers are
1114 * only modified if everything goes well, because we don't want the bits to be
1115 * touched before both bitmaps are set up.
1116 */
1117int create_basic_memory_bitmaps(void)
1118{
1119	struct memory_bitmap *bm1, *bm2;
1120	int error;
1121
1122	if (forbidden_pages_map && free_pages_map)
1123		return 0;
1124	else
1125		BUG_ON(forbidden_pages_map || free_pages_map);
1126
1127	bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
1128	if (!bm1)
1129		return -ENOMEM;
1130
1131	error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
1132	if (error)
1133		goto Free_first_object;
1134
1135	bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
1136	if (!bm2)
1137		goto Free_first_bitmap;
1138
1139	error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
1140	if (error)
1141		goto Free_second_object;
1142
1143	forbidden_pages_map = bm1;
1144	free_pages_map = bm2;
1145	mark_nosave_pages(forbidden_pages_map);
1146
1147	pr_debug("Basic memory bitmaps created\n");
1148
1149	return 0;
1150
1151 Free_second_object:
1152	kfree(bm2);
1153 Free_first_bitmap:
1154	memory_bm_free(bm1, PG_UNSAFE_CLEAR);
1155 Free_first_object:
1156	kfree(bm1);
1157	return -ENOMEM;
1158}
1159
1160/**
1161 * free_basic_memory_bitmaps - Free memory bitmaps holding basic information.
1162 *
1163 * Free memory bitmaps allocated by create_basic_memory_bitmaps().  The
1164 * auxiliary pointers are necessary so that the bitmaps themselves are not
1165 * referred to while they are being freed.
1166 */
1167void free_basic_memory_bitmaps(void)
1168{
1169	struct memory_bitmap *bm1, *bm2;
1170
1171	if (WARN_ON(!(forbidden_pages_map && free_pages_map)))
1172		return;
1173
1174	bm1 = forbidden_pages_map;
1175	bm2 = free_pages_map;
1176	forbidden_pages_map = NULL;
1177	free_pages_map = NULL;
1178	memory_bm_free(bm1, PG_UNSAFE_CLEAR);
1179	kfree(bm1);
1180	memory_bm_free(bm2, PG_UNSAFE_CLEAR);
1181	kfree(bm2);
1182
1183	pr_debug("Basic memory bitmaps freed\n");
1184}
1185
1186static void clear_or_poison_free_page(struct page *page)
1187{
1188	if (page_poisoning_enabled_static())
1189		__kernel_poison_pages(page, 1);
1190	else if (want_init_on_free())
1191		clear_highpage(page);
1192}
1193
1194void clear_or_poison_free_pages(void)
1195{
1196	struct memory_bitmap *bm = free_pages_map;
1197	unsigned long pfn;
1198
1199	if (WARN_ON(!(free_pages_map)))
1200		return;
1201
1202	if (page_poisoning_enabled() || want_init_on_free()) {
1203		memory_bm_position_reset(bm);
1204		pfn = memory_bm_next_pfn(bm);
1205		while (pfn != BM_END_OF_MAP) {
1206			if (pfn_valid(pfn))
1207				clear_or_poison_free_page(pfn_to_page(pfn));
1208
1209			pfn = memory_bm_next_pfn(bm);
1210		}
1211		memory_bm_position_reset(bm);
1212		pr_info("free pages cleared after restore\n");
1213	}
1214}
1215
1216/**
1217 * snapshot_additional_pages - Estimate the number of extra pages needed.
1218 * @zone: Memory zone to carry out the computation for.
1219 *
1220 * Estimate the number of additional pages needed for setting up a hibernation
1221 * image data structures for @zone (usually, the returned value is greater than
1222 * the exact number).
1223 */
1224unsigned int snapshot_additional_pages(struct zone *zone)
1225{
1226	unsigned int rtree, nodes;
1227
1228	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
1229	rtree += DIV_ROUND_UP(rtree * sizeof(struct rtree_node),
1230			      LINKED_PAGE_DATA_SIZE);
1231	while (nodes > 1) {
1232		nodes = DIV_ROUND_UP(nodes, BM_ENTRIES_PER_LEVEL);
1233		rtree += nodes;
1234	}
1235
1236	return 2 * rtree;
1237}
1238
1239/*
1240 * Touch the watchdog for every WD_PAGE_COUNT pages.
1241 */
1242#define WD_PAGE_COUNT	(128*1024)
1243
1244static void mark_free_pages(struct zone *zone)
1245{
1246	unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
1247	unsigned long flags;
1248	unsigned int order, t;
1249	struct page *page;
1250
1251	if (zone_is_empty(zone))
1252		return;
1253
1254	spin_lock_irqsave(&zone->lock, flags);
1255
1256	max_zone_pfn = zone_end_pfn(zone);
1257	for_each_valid_pfn(pfn, zone->zone_start_pfn, max_zone_pfn) {
1258		page = pfn_to_page(pfn);
1259
1260		if (!--page_count) {
1261			touch_nmi_watchdog();
1262			page_count = WD_PAGE_COUNT;
1263		}
1264
1265		if (page_zone(page) != zone)
1266			continue;
1267
1268		if (!swsusp_page_is_forbidden(page))
1269			swsusp_unset_page_free(page);
1270	}
1271
1272	for_each_migratetype_order(order, t) {
1273		list_for_each_entry(page,
1274				&zone->free_area[order].free_list[t], buddy_list) {
1275			unsigned long i;
1276
1277			pfn = page_to_pfn(page);
1278			for (i = 0; i < (1UL << order); i++) {
1279				if (!--page_count) {
1280					touch_nmi_watchdog();
1281					page_count = WD_PAGE_COUNT;
1282				}
1283				swsusp_set_page_free(pfn_to_page(pfn + i));
1284			}
1285		}
1286	}
1287	spin_unlock_irqrestore(&zone->lock, flags);
1288}
1289
1290#ifdef CONFIG_HIGHMEM
1291/**
1292 * count_free_highmem_pages - Compute the total number of free highmem pages.
1293 *
1294 * The returned number is system-wide.
1295 */
1296static unsigned int count_free_highmem_pages(void)
1297{
1298	struct zone *zone;
1299	unsigned int cnt = 0;
1300
1301	for_each_populated_zone(zone)
1302		if (is_highmem(zone))
1303			cnt += zone_page_state(zone, NR_FREE_PAGES);
1304
1305	return cnt;
1306}
1307
1308/**
1309 * saveable_highmem_page - Check if a highmem page is saveable.
1310 *
1311 * Determine whether a highmem page should be included in a hibernation image.
1312 *
1313 * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
1314 * and it isn't part of a free chunk of pages.
1315 */
1316static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)
1317{
1318	struct page *page;
1319
1320	if (!pfn_valid(pfn))
1321		return NULL;
1322
1323	page = pfn_to_online_page(pfn);
1324	if (!page || page_zone(page) != zone)
1325		return NULL;
1326
1327	BUG_ON(!PageHighMem(page));
1328
1329	if (swsusp_page_is_forbidden(page) ||  swsusp_page_is_free(page))
1330		return NULL;
1331
1332	if (PageReserved(page) || PageOffline(page))
1333		return NULL;
1334
1335	if (page_is_guard(page))
1336		return NULL;
1337
1338	return page;
1339}
1340
1341/**
1342 * count_highmem_pages - Compute the total number of saveable highmem pages.
1343 */
1344static unsigned int count_highmem_pages(void)
1345{
1346	struct zone *zone;
1347	unsigned int n = 0;
1348
1349	for_each_populated_zone(zone) {
1350		unsigned long pfn, max_zone_pfn;
1351
1352		if (!is_highmem(zone))
1353			continue;
1354
1355		mark_free_pages(zone);
1356		max_zone_pfn = zone_end_pfn(zone);
1357		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1358			if (saveable_highmem_page(zone, pfn))
1359				n++;
1360	}
1361	return n;
1362}
1363#endif /* CONFIG_HIGHMEM */
1364
1365/**
1366 * saveable_page - Check if the given page is saveable.
1367 *
1368 * Determine whether a non-highmem page should be included in a hibernation
1369 * image.
1370 *
1371 * We should save the page if it isn't Nosave, and is not in the range
1372 * of pages statically defined as 'unsaveable', and it isn't part of
1373 * a free chunk of pages.
1374 */
1375static struct page *saveable_page(struct zone *zone, unsigned long pfn)
1376{
1377	struct page *page;
1378
1379	if (!pfn_valid(pfn))
1380		return NULL;
1381
1382	page = pfn_to_online_page(pfn);
1383	if (!page || page_zone(page) != zone)
1384		return NULL;
1385
1386	BUG_ON(PageHighMem(page));
1387
1388	if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
1389		return NULL;
1390
1391	if (PageOffline(page))
1392		return NULL;
1393
1394	if (PageReserved(page)
1395	    && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
1396		return NULL;
1397
1398	if (page_is_guard(page))
1399		return NULL;
1400
1401	return page;
1402}
1403
1404/**
1405 * count_data_pages - Compute the total number of saveable non-highmem pages.
1406 */
1407static unsigned int count_data_pages(void)
1408{
1409	struct zone *zone;
1410	unsigned long pfn, max_zone_pfn;
1411	unsigned int n = 0;
1412
1413	for_each_populated_zone(zone) {
1414		if (is_highmem(zone))
1415			continue;
1416
1417		mark_free_pages(zone);
1418		max_zone_pfn = zone_end_pfn(zone);
1419		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1420			if (saveable_page(zone, pfn))
1421				n++;
1422	}
1423	return n;
1424}
1425
1426/*
1427 * This is needed, because copy_page and memcpy are not usable for copying
1428 * task structs. Returns true if the page was filled with only zeros,
1429 * otherwise false.
1430 */
1431static inline bool do_copy_page(long *dst, long *src)
1432{
1433	long z = 0;
1434	int n;
1435
1436	for (n = PAGE_SIZE / sizeof(long); n; n--) {
1437		z |= *src;
1438		*dst++ = *src++;
1439	}
1440	return !z;
1441}
1442
1443/**
1444 * safe_copy_page - Copy a page in a safe way.
1445 *
1446 * Check if the page we are going to copy is marked as present in the kernel
1447 * page tables. This always is the case if CONFIG_DEBUG_PAGEALLOC or
1448 * CONFIG_ARCH_HAS_SET_DIRECT_MAP is not set. In that case kernel_page_present()
1449 * always returns 'true'. Returns true if the page was entirely composed of
1450 * zeros, otherwise it will return false.
1451 */
1452static bool safe_copy_page(void *dst, struct page *s_page)
1453{
1454	bool zeros_only;
1455
1456	if (kernel_page_present(s_page)) {
1457		zeros_only = do_copy_page(dst, page_address(s_page));
1458	} else {
1459		hibernate_map_page(s_page);
1460		zeros_only = do_copy_page(dst, page_address(s_page));
1461		hibernate_unmap_page(s_page);
1462	}
1463	return zeros_only;
1464}
1465
1466#ifdef CONFIG_HIGHMEM
1467static inline struct page *page_is_saveable(struct zone *zone, unsigned long pfn)
1468{
1469	return is_highmem(zone) ?
1470		saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
1471}
1472
1473static bool copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1474{
1475	struct page *s_page, *d_page;
1476	void *src, *dst;
1477	bool zeros_only;
1478
1479	s_page = pfn_to_page(src_pfn);
1480	d_page = pfn_to_page(dst_pfn);
1481	if (PageHighMem(s_page)) {
1482		src = kmap_local_page(s_page);
1483		dst = kmap_local_page(d_page);
1484		zeros_only = do_copy_page(dst, src);
1485		kunmap_local(dst);
1486		kunmap_local(src);
1487	} else {
1488		if (PageHighMem(d_page)) {
1489			/*
1490			 * The page pointed to by src may contain some kernel
1491			 * data modified by kmap_atomic()
1492			 */
1493			zeros_only = safe_copy_page(buffer, s_page);
1494			dst = kmap_local_page(d_page);
1495			copy_page(dst, buffer);
1496			kunmap_local(dst);
1497		} else {
1498			zeros_only = safe_copy_page(page_address(d_page), s_page);
1499		}
1500	}
1501	return zeros_only;
1502}
1503#else
1504#define page_is_saveable(zone, pfn)	saveable_page(zone, pfn)
1505
1506static inline int copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1507{
1508	return safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1509				pfn_to_page(src_pfn));
1510}
1511#endif /* CONFIG_HIGHMEM */
1512
1513/*
1514 * Copy data pages will copy all pages into pages pulled from the copy_bm.
1515 * If a page was entirely filled with zeros it will be marked in the zero_bm.
1516 *
1517 * Returns the number of pages copied.
1518 */
1519static unsigned long copy_data_pages(struct memory_bitmap *copy_bm,
1520			    struct memory_bitmap *orig_bm,
1521			    struct memory_bitmap *zero_bm)
1522{
1523	unsigned long copied_pages = 0;
1524	struct zone *zone;
1525	unsigned long pfn, copy_pfn;
1526
1527	for_each_populated_zone(zone) {
1528		unsigned long max_zone_pfn;
1529
1530		mark_free_pages(zone);
1531		max_zone_pfn = zone_end_pfn(zone);
1532		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1533			if (page_is_saveable(zone, pfn))
1534				memory_bm_set_bit(orig_bm, pfn);
1535	}
1536	memory_bm_position_reset(orig_bm);
1537	memory_bm_position_reset(copy_bm);
1538	copy_pfn = memory_bm_next_pfn(copy_bm);
1539	for (;;) {
1540		pfn = memory_bm_next_pfn(orig_bm);
1541		if (unlikely(pfn == BM_END_OF_MAP))
1542			break;
1543		if (copy_data_page(copy_pfn, pfn)) {
1544			memory_bm_set_bit(zero_bm, pfn);
1545			/* Use this copy_pfn for a page that is not full of zeros */
1546			continue;
1547		}
1548		copied_pages++;
1549		copy_pfn = memory_bm_next_pfn(copy_bm);
1550	}
1551	return copied_pages;
1552}
1553
1554/* Total number of image pages */
1555static unsigned int nr_copy_pages;
1556/* Number of pages needed for saving the original pfns of the image pages */
1557static unsigned int nr_meta_pages;
1558/* Number of zero pages */
1559static unsigned int nr_zero_pages;
1560
1561/*
1562 * Numbers of normal and highmem page frames allocated for hibernation image
1563 * before suspending devices.
1564 */
1565static unsigned int alloc_normal, alloc_highmem;
1566/*
1567 * Memory bitmap used for marking saveable pages (during hibernation) or
1568 * hibernation image pages (during restore)
1569 */
1570static struct memory_bitmap orig_bm;
1571/*
1572 * Memory bitmap used during hibernation for marking allocated page frames that
1573 * will contain copies of saveable pages.  During restore it is initially used
1574 * for marking hibernation image pages, but then the set bits from it are
1575 * duplicated in @orig_bm and it is released.  On highmem systems it is next
1576 * used for marking "safe" highmem pages, but it has to be reinitialized for
1577 * this purpose.
1578 */
1579static struct memory_bitmap copy_bm;
1580
1581/* Memory bitmap which tracks which saveable pages were zero filled. */
1582static struct memory_bitmap zero_bm;
1583
1584/**
1585 * swsusp_free - Free pages allocated for hibernation image.
1586 *
1587 * Image pages are allocated before snapshot creation, so they need to be
1588 * released after resume.
1589 */
1590void swsusp_free(void)
1591{
1592	unsigned long fb_pfn, fr_pfn;
1593
1594	if (!forbidden_pages_map || !free_pages_map)
1595		goto out;
1596
1597	memory_bm_position_reset(forbidden_pages_map);
1598	memory_bm_position_reset(free_pages_map);
1599
1600loop:
1601	fr_pfn = memory_bm_next_pfn(free_pages_map);
1602	fb_pfn = memory_bm_next_pfn(forbidden_pages_map);
1603
1604	/*
1605	 * Find the next bit set in both bitmaps. This is guaranteed to
1606	 * terminate when fb_pfn == fr_pfn == BM_END_OF_MAP.
1607	 */
1608	do {
1609		if (fb_pfn < fr_pfn)
1610			fb_pfn = memory_bm_next_pfn(forbidden_pages_map);
1611		if (fr_pfn < fb_pfn)
1612			fr_pfn = memory_bm_next_pfn(free_pages_map);
1613	} while (fb_pfn != fr_pfn);
1614
1615	if (fr_pfn != BM_END_OF_MAP && pfn_valid(fr_pfn)) {
1616		struct page *page = pfn_to_page(fr_pfn);
1617
1618		memory_bm_clear_current(forbidden_pages_map);
1619		memory_bm_clear_current(free_pages_map);
1620		hibernate_restore_unprotect_page(page_address(page));
1621		__free_page(page);
1622		goto loop;
1623	}
1624
1625out:
1626	nr_copy_pages = 0;
1627	nr_meta_pages = 0;
1628	nr_zero_pages = 0;
1629	restore_pblist = NULL;
1630	buffer = NULL;
1631	alloc_normal = 0;
1632	alloc_highmem = 0;
1633	hibernate_restore_protection_end();
1634}
1635
1636/* Helper functions used for the shrinking of memory. */
1637
1638#define GFP_IMAGE	(GFP_KERNEL | __GFP_NOWARN)
1639
1640/**
1641 * preallocate_image_pages - Allocate a number of pages for hibernation image.
1642 * @nr_pages: Number of page frames to allocate.
1643 * @mask: GFP flags to use for the allocation.
1644 *
1645 * Return value: Number of page frames actually allocated
1646 */
1647static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1648{
1649	unsigned long nr_alloc = 0;
1650
1651	while (nr_pages > 0) {
1652		struct page *page;
1653
1654		page = alloc_image_page(mask);
1655		if (!page)
1656			break;
1657		memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1658		if (PageHighMem(page))
1659			alloc_highmem++;
1660		else
1661			alloc_normal++;
1662		nr_pages--;
1663		nr_alloc++;
1664	}
1665
1666	return nr_alloc;
1667}
1668
1669static unsigned long preallocate_image_memory(unsigned long nr_pages,
1670					      unsigned long avail_normal)
1671{
1672	unsigned long alloc;
1673
1674	if (avail_normal <= alloc_normal)
1675		return 0;
1676
1677	alloc = avail_normal - alloc_normal;
1678	if (nr_pages < alloc)
1679		alloc = nr_pages;
1680
1681	return preallocate_image_pages(alloc, GFP_IMAGE);
1682}
1683
1684#ifdef CONFIG_HIGHMEM
1685static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1686{
1687	return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
1688}
1689
1690/**
1691 *  __fraction - Compute (an approximation of) x * (multiplier / base).
1692 */
1693static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1694{
1695	return div64_u64(x * multiplier, base);
1696}
1697
1698static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1699						  unsigned long highmem,
1700						  unsigned long total)
1701{
1702	unsigned long alloc = __fraction(nr_pages, highmem, total);
1703
1704	return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
1705}
1706#else /* CONFIG_HIGHMEM */
1707static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1708{
1709	return 0;
1710}
1711
1712static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1713							 unsigned long highmem,
1714							 unsigned long total)
1715{
1716	return 0;
1717}
1718#endif /* CONFIG_HIGHMEM */
1719
1720/**
1721 * free_unnecessary_pages - Release preallocated pages not needed for the image.
1722 */
1723static unsigned long free_unnecessary_pages(void)
1724{
1725	unsigned long save, to_free_normal, to_free_highmem, free;
1726
1727	save = count_data_pages();
1728	if (alloc_normal >= save) {
1729		to_free_normal = alloc_normal - save;
1730		save = 0;
1731	} else {
1732		to_free_normal = 0;
1733		save -= alloc_normal;
1734	}
1735	save += count_highmem_pages();
1736	if (alloc_highmem >= save) {
1737		to_free_highmem = alloc_highmem - save;
1738	} else {
1739		to_free_highmem = 0;
1740		save -= alloc_highmem;
1741		if (to_free_normal > save)
1742			to_free_normal -= save;
1743		else
1744			to_free_normal = 0;
1745	}
1746	free = to_free_normal + to_free_highmem;
1747
1748	memory_bm_position_reset(&copy_bm);
1749
1750	while (to_free_normal > 0 || to_free_highmem > 0) {
1751		unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1752		struct page *page = pfn_to_page(pfn);
1753
1754		if (PageHighMem(page)) {
1755			if (!to_free_highmem)
1756				continue;
1757			to_free_highmem--;
1758			alloc_highmem--;
1759		} else {
1760			if (!to_free_normal)
1761				continue;
1762			to_free_normal--;
1763			alloc_normal--;
1764		}
1765		memory_bm_clear_bit(&copy_bm, pfn);
1766		swsusp_unset_page_forbidden(page);
1767		swsusp_unset_page_free(page);
1768		__free_page(page);
1769	}
1770
1771	return free;
1772}
1773
1774/**
1775 * minimum_image_size - Estimate the minimum acceptable size of an image.
1776 * @saveable: Number of saveable pages in the system.
1777 *
1778 * We want to avoid attempting to free too much memory too hard, so estimate the
1779 * minimum acceptable size of a hibernation image to use as the lower limit for
1780 * preallocating memory.
1781 *
1782 * We assume that the minimum image size should be proportional to
1783 *
1784 * [number of saveable pages] - [number of pages that can be freed in theory]
1785 *
1786 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1787 * and (3) inactive anonymous pages, (4) active and (5) inactive file pages.
1788 */
1789static unsigned long minimum_image_size(unsigned long saveable)
1790{
1791	unsigned long size;
1792
1793	size = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B)
1794		+ global_node_page_state(NR_ACTIVE_ANON)
1795		+ global_node_page_state(NR_INACTIVE_ANON)
1796		+ global_node_page_state(NR_ACTIVE_FILE)
1797		+ global_node_page_state(NR_INACTIVE_FILE);
1798
1799	return saveable <= size ? 0 : saveable - size;
1800}
1801
1802/**
1803 * hibernate_preallocate_memory - Preallocate memory for hibernation image.
1804 *
1805 * To create a hibernation image it is necessary to make a copy of every page
1806 * frame in use.  We also need a number of page frames to be free during
1807 * hibernation for allocations made while saving the image and for device
1808 * drivers, in case they need to allocate memory from their hibernation
1809 * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1810 * estimate) and reserved_size divided by PAGE_SIZE (which is tunable through
1811 * /sys/power/reserved_size, respectively).  To make this happen, we compute the
1812 * total number of available page frames and allocate at least
1813 *
1814 * ([page frames total] - PAGES_FOR_IO - [metadata pages]) / 2
1815 *  - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1816 *
1817 * of them, which corresponds to the maximum size of a hibernation image.
1818 *
1819 * If image_size is set below the number following from the above formula,
1820 * the preallocation of memory is continued until the total number of saveable
1821 * pages in the system is below the requested image size or the minimum
1822 * acceptable image size returned by minimum_image_size(), whichever is greater.
1823 */
1824int hibernate_preallocate_memory(void)
1825{
1826	struct zone *zone;
1827	unsigned long saveable, size, max_size, count, highmem, pages = 0;
1828	unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1829	ktime_t start, stop;
1830	int error;
1831
1832	pr_info("Preallocating image memory\n");
1833	start = ktime_get();
1834
1835	error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1836	if (error) {
1837		pr_err("Cannot allocate original bitmap\n");
1838		goto err_out;
1839	}
1840
1841	error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1842	if (error) {
1843		pr_err("Cannot allocate copy bitmap\n");
1844		goto err_out;
1845	}
1846
1847	error = memory_bm_create(&zero_bm, GFP_IMAGE, PG_ANY);
1848	if (error) {
1849		pr_err("Cannot allocate zero bitmap\n");
1850		goto err_out;
1851	}
1852
1853	alloc_normal = 0;
1854	alloc_highmem = 0;
1855	nr_zero_pages = 0;
1856
1857	/* Count the number of saveable data pages. */
1858	save_highmem = count_highmem_pages();
1859	saveable = count_data_pages();
1860
1861	/*
1862	 * Compute the total number of page frames we can use (count) and the
1863	 * number of pages needed for image metadata (size).
1864	 */
1865	count = saveable;
1866	saveable += save_highmem;
1867	highmem = save_highmem;
1868	size = 0;
1869	for_each_populated_zone(zone) {
1870		size += snapshot_additional_pages(zone);
1871		if (is_highmem(zone))
1872			highmem += zone_page_state(zone, NR_FREE_PAGES);
1873		else
1874			count += zone_page_state(zone, NR_FREE_PAGES);
1875	}
1876	avail_normal = count;
1877	count += highmem;
1878	count -= totalreserve_pages;
1879
1880	/* Compute the maximum number of saveable pages to leave in memory. */
1881	max_size = (count - (size + PAGES_FOR_IO)) / 2
1882			- 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
1883	/* Compute the desired number of image pages specified by image_size. */
1884	size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1885	if (size > max_size)
1886		size = max_size;
1887	/*
1888	 * If the desired number of image pages is at least as large as the
1889	 * current number of saveable pages in memory, allocate page frames for
1890	 * the image and we're done.
1891	 */
1892	if (size >= saveable) {
1893		pages = preallocate_image_highmem(save_highmem);
1894		pages += preallocate_image_memory(saveable - pages, avail_normal);
1895		goto out;
1896	}
1897
1898	/* Estimate the minimum size of the image. */
1899	pages = minimum_image_size(saveable);
1900	/*
1901	 * To avoid excessive pressure on the normal zone, leave room in it to
1902	 * accommodate an image of the minimum size (unless it's already too
1903	 * small, in which case don't preallocate pages from it at all).
1904	 */
1905	if (avail_normal > pages)
1906		avail_normal -= pages;
1907	else
1908		avail_normal = 0;
1909	if (size < pages)
1910		size = min_t(unsigned long, pages, max_size);
1911
1912	/*
1913	 * Let the memory management subsystem know that we're going to need a
1914	 * large number of page frames to allocate and make it free some memory.
1915	 * NOTE: If this is not done, performance will be hurt badly in some
1916	 * test cases.
1917	 */
1918	shrink_all_memory(saveable - size);
1919
1920	/*
1921	 * The number of saveable pages in memory was too high, so apply some
1922	 * pressure to decrease it.  First, make room for the largest possible
1923	 * image and fail if that doesn't work.  Next, try to decrease the size
1924	 * of the image as much as indicated by 'size' using allocations from
1925	 * highmem and non-highmem zones separately.
1926	 */
1927	pages_highmem = preallocate_image_highmem(highmem / 2);
1928	alloc = count - max_size;
1929	if (alloc > pages_highmem)
1930		alloc -= pages_highmem;
1931	else
1932		alloc = 0;
1933	pages = preallocate_image_memory(alloc, avail_normal);
1934	if (pages < alloc) {
1935		/* We have exhausted non-highmem pages, try highmem. */
1936		alloc -= pages;
1937		pages += pages_highmem;
1938		pages_highmem = preallocate_image_highmem(alloc);
1939		if (pages_highmem < alloc) {
1940			pr_err("Image allocation is %lu pages short\n",
1941				alloc - pages_highmem);
1942			goto err_out;
1943		}
1944		pages += pages_highmem;
1945		/*
1946		 * size is the desired number of saveable pages to leave in
1947		 * memory, so try to preallocate (all memory - size) pages.
1948		 */
1949		alloc = (count - pages) - size;
1950		pages += preallocate_image_highmem(alloc);
1951	} else {
1952		/*
1953		 * There are approximately max_size saveable pages at this point
1954		 * and we want to reduce this number down to size.
1955		 */
1956		alloc = max_size - size;
1957		size = preallocate_highmem_fraction(alloc, highmem, count);
1958		pages_highmem += size;
1959		alloc -= size;
1960		size = preallocate_image_memory(alloc, avail_normal);
1961		pages_highmem += preallocate_image_highmem(alloc - size);
1962		pages += pages_highmem + size;
1963	}
1964
1965	/*
1966	 * We only need as many page frames for the image as there are saveable
1967	 * pages in memory, but we have allocated more.  Release the excessive
1968	 * ones now.
1969	 */
1970	pages -= free_unnecessary_pages();
1971
1972 out:
1973	stop = ktime_get();
1974	pr_info("Allocated %lu pages for snapshot\n", pages);
1975	swsusp_show_speed(start, stop, pages, "Allocated");
1976
1977	return 0;
1978
1979 err_out:
1980	swsusp_free();
1981	return -ENOMEM;
1982}
1983
1984#ifdef CONFIG_HIGHMEM
1985/**
1986 * count_pages_for_highmem - Count non-highmem pages needed for copying highmem.
1987 *
1988 * Compute the number of non-highmem pages that will be necessary for creating
1989 * copies of highmem pages.
1990 */
1991static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1992{
1993	unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1994
1995	if (free_highmem >= nr_highmem)
1996		nr_highmem = 0;
1997	else
1998		nr_highmem -= free_highmem;
1999
2000	return nr_highmem;
2001}
2002#else
2003static unsigned int count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
2004#endif /* CONFIG_HIGHMEM */
2005
2006/**
2007 * enough_free_mem - Check if there is enough free memory for the image.
2008 */
2009static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
2010{
2011	struct zone *zone;
2012	unsigned int free = alloc_normal;
2013
2014	for_each_populated_zone(zone)
2015		if (!is_highmem(zone))
2016			free += zone_page_state(zone, NR_FREE_PAGES);
2017
2018	nr_pages += count_pages_for_highmem(nr_highmem);
2019	pr_debug("Normal pages needed: %u + %u, available pages: %u\n",
2020		 nr_pages, PAGES_FOR_IO, free);
2021
2022	return free > nr_pages + PAGES_FOR_IO;
2023}
2024
2025#ifdef CONFIG_HIGHMEM
2026/**
2027 * get_highmem_buffer - Allocate a buffer for highmem pages.
2028 *
2029 * If there are some highmem pages in the hibernation image, we may need a
2030 * buffer to copy them and/or load their data.
2031 */
2032static inline int get_highmem_buffer(int safe_needed)
2033{
2034	buffer = get_image_page(GFP_ATOMIC, safe_needed);
2035	return buffer ? 0 : -ENOMEM;
2036}
2037
2038/**
2039 * alloc_highmem_pages - Allocate some highmem pages for the image.
2040 *
2041 * Try to allocate as many pages as needed, but if the number of free highmem
2042 * pages is less than that, allocate them all.
2043 */
2044static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,
2045					       unsigned int nr_highmem)
2046{
2047	unsigned int to_alloc = count_free_highmem_pages();
2048
2049	if (to_alloc > nr_highmem)
2050		to_alloc = nr_highmem;
2051
2052	nr_highmem -= to_alloc;
2053	while (to_alloc-- > 0) {
2054		struct page *page;
2055
2056		page = alloc_image_page(__GFP_HIGHMEM|__GFP_KSWAPD_RECLAIM);
2057		memory_bm_set_bit(bm, page_to_pfn(page));
2058	}
2059	return nr_highmem;
2060}
2061#else
2062static inline int get_highmem_buffer(int safe_needed) { return 0; }
2063
2064static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,
2065					       unsigned int n) { return 0; }
2066#endif /* CONFIG_HIGHMEM */
2067
2068/**
2069 * swsusp_alloc - Allocate memory for hibernation image.
2070 *
2071 * We first try to allocate as many highmem pages as there are
2072 * saveable highmem pages in the system.  If that fails, we allocate
2073 * non-highmem pages for the copies of the remaining highmem ones.
2074 *
2075 * In this approach it is likely that the copies of highmem pages will
2076 * also be located in the high memory, because of the way in which
2077 * copy_data_pages() works.
2078 */
2079static int swsusp_alloc(struct memory_bitmap *copy_bm,
2080			unsigned int nr_pages, unsigned int nr_highmem)
2081{
2082	if (nr_highmem > 0) {
2083		if (get_highmem_buffer(PG_ANY))
2084			goto err_out;
2085		if (nr_highmem > alloc_highmem) {
2086			nr_highmem -= alloc_highmem;
2087			nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
2088		}
2089	}
2090	if (nr_pages > alloc_normal) {
2091		nr_pages -= alloc_normal;
2092		while (nr_pages-- > 0) {
2093			struct page *page;
2094
2095			page = alloc_image_page(GFP_ATOMIC);
2096			if (!page)
2097				goto err_out;
2098			memory_bm_set_bit(copy_bm, page_to_pfn(page));
2099		}
2100	}
2101
2102	return 0;
2103
2104 err_out:
2105	swsusp_free();
2106	return -ENOMEM;
2107}
2108
2109asmlinkage __visible int swsusp_save(void)
2110{
2111	unsigned int nr_pages, nr_highmem;
2112
2113	pm_deferred_pr_dbg("Creating image\n");
2114
2115	drain_local_pages(NULL);
2116	nr_pages = count_data_pages();
2117	nr_highmem = count_highmem_pages();
2118	pm_deferred_pr_dbg("Need to copy %u pages\n", nr_pages + nr_highmem);
2119
2120	if (!enough_free_mem(nr_pages, nr_highmem)) {
2121		pm_deferred_pr_dbg("Not enough free memory for image creation\n");
2122		return -ENOMEM;
2123	}
2124
2125	if (swsusp_alloc(&copy_bm, nr_pages, nr_highmem))
2126		return -ENOMEM;
2127
2128	/*
2129	 * During allocating of suspend pagedir, new cold pages may appear.
2130	 * Kill them.
2131	 */
2132	drain_local_pages(NULL);
2133	nr_copy_pages = copy_data_pages(&copy_bm, &orig_bm, &zero_bm);
2134
2135	/*
2136	 * End of critical section. From now on, we can write to memory,
2137	 * but we should not touch disk. This specially means we must _not_
2138	 * touch swap space! Except we must write out our image of course.
2139	 */
2140	nr_pages += nr_highmem;
2141	/* We don't actually copy the zero pages */
2142	nr_zero_pages = nr_pages - nr_copy_pages;
2143	nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
2144
2145	pm_deferred_pr_dbg("Image created (%d pages copied, %d zero pages)\n",
2146			   nr_copy_pages, nr_zero_pages);
2147
2148	return 0;
2149}
2150
2151#ifndef CONFIG_ARCH_HIBERNATION_HEADER
2152static int init_header_complete(struct swsusp_info *info)
2153{
2154	memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
2155	info->version_code = LINUX_VERSION_CODE;
2156	return 0;
2157}
2158
2159static const char *check_image_kernel(struct swsusp_info *info)
2160{
2161	if (info->version_code != LINUX_VERSION_CODE)
2162		return "kernel version";
2163	if (strcmp(info->uts.sysname, init_utsname()->sysname))
2164		return "system type";
2165	if (strcmp(info->uts.release, init_utsname()->release))
2166		return "kernel release";
2167	if (strcmp(info->uts.version, init_utsname()->version))
2168		return "version";
2169	if (strcmp(info->uts.machine, init_utsname()->machine))
2170		return "machine";
2171	return NULL;
2172}
2173#endif /* CONFIG_ARCH_HIBERNATION_HEADER */
2174
2175unsigned long snapshot_get_image_size(void)
2176{
2177	return nr_copy_pages + nr_meta_pages + 1;
2178}
2179
2180static int init_header(struct swsusp_info *info)
2181{
2182	memset(info, 0, sizeof(struct swsusp_info));
2183	info->num_physpages = get_num_physpages();
2184	info->image_pages = nr_copy_pages;
2185	info->pages = snapshot_get_image_size();
2186	info->size = info->pages;
2187	info->size <<= PAGE_SHIFT;
2188	return init_header_complete(info);
2189}
2190
2191#define ENCODED_PFN_ZERO_FLAG ((unsigned long)1 << (BITS_PER_LONG - 1))
2192#define ENCODED_PFN_MASK (~ENCODED_PFN_ZERO_FLAG)
2193
2194/**
2195 * pack_pfns - Prepare PFNs for saving.
2196 * @bm: Memory bitmap.
2197 * @buf: Memory buffer to store the PFNs in.
2198 * @zero_bm: Memory bitmap containing PFNs of zero pages.
2199 *
2200 * PFNs corresponding to set bits in @bm are stored in the area of memory
2201 * pointed to by @buf (1 page at a time). Pages which were filled with only
2202 * zeros will have the highest bit set in the packed format to distinguish
2203 * them from PFNs which will be contained in the image file.
2204 */
2205static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm,
2206		struct memory_bitmap *zero_bm)
2207{
2208	int j;
2209
2210	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
2211		buf[j] = memory_bm_next_pfn(bm);
2212		if (unlikely(buf[j] == BM_END_OF_MAP))
2213			break;
2214		if (memory_bm_test_bit(zero_bm, buf[j]))
2215			buf[j] |= ENCODED_PFN_ZERO_FLAG;
2216	}
2217}
2218
2219/**
2220 * snapshot_read_next - Get the address to read the next image page from.
2221 * @handle: Snapshot handle to be used for the reading.
2222 *
2223 * On the first call, @handle should point to a zeroed snapshot_handle
2224 * structure.  The structure gets populated then and a pointer to it should be
2225 * passed to this function every next time.
2226 *
2227 * On success, the function returns a positive number.  Then, the caller
2228 * is allowed to read up to the returned number of bytes from the memory
2229 * location computed by the data_of() macro.
2230 *
2231 * The function returns 0 to indicate the end of the data stream condition,
2232 * and negative numbers are returned on errors.  If that happens, the structure
2233 * pointed to by @handle is not updated and should not be used any more.
2234 */
2235int snapshot_read_next(struct snapshot_handle *handle)
2236{
2237	if (handle->cur > nr_meta_pages + nr_copy_pages)
2238		return 0;
2239
2240	if (!buffer) {
2241		/* This makes the buffer be freed by swsusp_free() */
2242		buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2243		if (!buffer)
2244			return -ENOMEM;
2245	}
2246	if (!handle->cur) {
2247		int error;
2248
2249		error = init_header((struct swsusp_info *)buffer);
2250		if (error)
2251			return error;
2252		handle->buffer = buffer;
2253		memory_bm_position_reset(&orig_bm);
2254		memory_bm_position_reset(&copy_bm);
2255	} else if (handle->cur <= nr_meta_pages) {
2256		clear_page(buffer);
2257		pack_pfns(buffer, &orig_bm, &zero_bm);
2258	} else {
2259		struct page *page;
2260
2261		page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
2262		if (PageHighMem(page)) {
2263			/*
2264			 * Highmem pages are copied to the buffer,
2265			 * because we can't return with a kmapped
2266			 * highmem page (we may not be called again).
2267			 */
2268			void *kaddr;
2269
2270			kaddr = kmap_local_page(page);
2271			copy_page(buffer, kaddr);
2272			kunmap_local(kaddr);
2273			handle->buffer = buffer;
2274		} else {
2275			handle->buffer = page_address(page);
2276		}
2277	}
2278	handle->cur++;
2279	return PAGE_SIZE;
2280}
2281
2282static void duplicate_memory_bitmap(struct memory_bitmap *dst,
2283				    struct memory_bitmap *src)
2284{
2285	unsigned long pfn;
2286
2287	memory_bm_position_reset(src);
2288	pfn = memory_bm_next_pfn(src);
2289	while (pfn != BM_END_OF_MAP) {
2290		memory_bm_set_bit(dst, pfn);
2291		pfn = memory_bm_next_pfn(src);
2292	}
2293}
2294
2295/**
2296 * mark_unsafe_pages - Mark pages that were used before hibernation.
2297 *
2298 * Mark the pages that cannot be used for storing the image during restoration,
2299 * because they conflict with the pages that had been used before hibernation.
2300 */
2301static void mark_unsafe_pages(struct memory_bitmap *bm)
2302{
2303	unsigned long pfn;
2304
2305	/* Clear the "free"/"unsafe" bit for all PFNs */
2306	memory_bm_position_reset(free_pages_map);
2307	pfn = memory_bm_next_pfn(free_pages_map);
2308	while (pfn != BM_END_OF_MAP) {
2309		memory_bm_clear_current(free_pages_map);
2310		pfn = memory_bm_next_pfn(free_pages_map);
2311	}
2312
2313	/* Mark pages that correspond to the "original" PFNs as "unsafe" */
2314	duplicate_memory_bitmap(free_pages_map, bm);
2315
2316	allocated_unsafe_pages = 0;
2317}
2318
2319static int check_header(struct swsusp_info *info)
2320{
2321	const char *reason;
2322
2323	reason = check_image_kernel(info);
2324	if (!reason && info->num_physpages != get_num_physpages())
2325		reason = "memory size";
2326	if (reason) {
2327		pr_err("Image mismatch: %s\n", reason);
2328		return -EPERM;
2329	}
2330	return 0;
2331}
2332
2333/**
2334 * load_header - Check the image header and copy the data from it.
2335 */
2336static int load_header(struct swsusp_info *info)
2337{
2338	int error;
2339
2340	restore_pblist = NULL;
2341	error = check_header(info);
2342	if (!error) {
2343		nr_copy_pages = info->image_pages;
2344		nr_meta_pages = info->pages - info->image_pages - 1;
2345	}
2346	return error;
2347}
2348
2349/**
2350 * unpack_orig_pfns - Set bits corresponding to given PFNs in a memory bitmap.
2351 * @bm: Memory bitmap.
2352 * @buf: Area of memory containing the PFNs.
2353 * @zero_bm: Memory bitmap with the zero PFNs marked.
2354 *
2355 * For each element of the array pointed to by @buf (1 page at a time), set the
2356 * corresponding bit in @bm. If the page was originally populated with only
2357 * zeros then a corresponding bit will also be set in @zero_bm.
2358 */
2359static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm,
2360		struct memory_bitmap *zero_bm)
2361{
2362	unsigned long decoded_pfn;
2363	bool zero;
2364	int j;
2365
2366	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
2367		if (unlikely(buf[j] == BM_END_OF_MAP))
2368			break;
2369
2370		zero = !!(buf[j] & ENCODED_PFN_ZERO_FLAG);
2371		decoded_pfn = buf[j] & ENCODED_PFN_MASK;
2372		if (pfn_valid(decoded_pfn) && memory_bm_pfn_present(bm, decoded_pfn)) {
2373			memory_bm_set_bit(bm, decoded_pfn);
2374			if (zero) {
2375				memory_bm_set_bit(zero_bm, decoded_pfn);
2376				nr_zero_pages++;
2377			}
2378		} else {
2379			if (!pfn_valid(decoded_pfn))
2380				pr_err(FW_BUG "Memory map mismatch at 0x%llx after hibernation\n",
2381				       (unsigned long long)PFN_PHYS(decoded_pfn));
2382			return -EFAULT;
2383		}
2384	}
2385
2386	return 0;
2387}
2388
2389#ifdef CONFIG_HIGHMEM
2390/*
2391 * struct highmem_pbe is used for creating the list of highmem pages that
2392 * should be restored atomically during the resume from disk, because the page
2393 * frames they have occupied before the suspend are in use.
2394 */
2395struct highmem_pbe {
2396	struct page *copy_page;	/* data is here now */
2397	struct page *orig_page;	/* data was here before the suspend */
2398	struct highmem_pbe *next;
2399};
2400
2401/*
2402 * List of highmem PBEs needed for restoring the highmem pages that were
2403 * allocated before the suspend and included in the suspend image, but have
2404 * also been allocated by the "resume" kernel, so their contents cannot be
2405 * written directly to their "original" page frames.
2406 */
2407static struct highmem_pbe *highmem_pblist;
2408
2409/**
2410 * count_highmem_image_pages - Compute the number of highmem pages in the image.
2411 * @bm: Memory bitmap.
2412 *
2413 * The bits in @bm that correspond to image pages are assumed to be set.
2414 */
2415static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
2416{
2417	unsigned long pfn;
2418	unsigned int cnt = 0;
2419
2420	memory_bm_position_reset(bm);
2421	pfn = memory_bm_next_pfn(bm);
2422	while (pfn != BM_END_OF_MAP) {
2423		if (PageHighMem(pfn_to_page(pfn)))
2424			cnt++;
2425
2426		pfn = memory_bm_next_pfn(bm);
2427	}
2428	return cnt;
2429}
2430
2431static unsigned int safe_highmem_pages;
2432
2433static struct memory_bitmap *safe_highmem_bm;
2434
2435/**
2436 * prepare_highmem_image - Allocate memory for loading highmem data from image.
2437 * @bm: Pointer to an uninitialized memory bitmap structure.
2438 * @nr_highmem_p: Pointer to the number of highmem image pages.
2439 *
2440 * Try to allocate as many highmem pages as there are highmem image pages
2441 * (@nr_highmem_p points to the variable containing the number of highmem image
2442 * pages).  The pages that are "safe" (ie. will not be overwritten when the
2443 * hibernation image is restored entirely) have the corresponding bits set in
2444 * @bm (it must be uninitialized).
2445 *
2446 * NOTE: This function should not be called if there are no highmem image pages.
2447 */
2448static int prepare_highmem_image(struct memory_bitmap *bm,
2449				 unsigned int *nr_highmem_p)
2450{
2451	unsigned int to_alloc;
2452
2453	if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
2454		return -ENOMEM;
2455
2456	if (get_highmem_buffer(PG_SAFE))
2457		return -ENOMEM;
2458
2459	to_alloc = count_free_highmem_pages();
2460	if (to_alloc > *nr_highmem_p)
2461		to_alloc = *nr_highmem_p;
2462	else
2463		*nr_highmem_p = to_alloc;
2464
2465	safe_highmem_pages = 0;
2466	while (to_alloc-- > 0) {
2467		struct page *page;
2468
2469		page = alloc_page(__GFP_HIGHMEM);
2470		if (!swsusp_page_is_free(page)) {
2471			/* The page is "safe", set its bit the bitmap */
2472			memory_bm_set_bit(bm, page_to_pfn(page));
2473			safe_highmem_pages++;
2474		}
2475		/* Mark the page as allocated */
2476		swsusp_set_page_forbidden(page);
2477		swsusp_set_page_free(page);
2478	}
2479	memory_bm_position_reset(bm);
2480	safe_highmem_bm = bm;
2481	return 0;
2482}
2483
2484static struct page *last_highmem_page;
2485
2486/**
2487 * get_highmem_page_buffer - Prepare a buffer to store a highmem image page.
2488 *
2489 * For a given highmem image page get a buffer that suspend_write_next() should
2490 * return to its caller to write to.
2491 *
2492 * If the page is to be saved to its "original" page frame or a copy of
2493 * the page is to be made in the highmem, @buffer is returned.  Otherwise,
2494 * the copy of the page is to be made in normal memory, so the address of
2495 * the copy is returned.
2496 *
2497 * If @buffer is returned, the caller of suspend_write_next() will write
2498 * the page's contents to @buffer, so they will have to be copied to the
2499 * right location on the next call to suspend_write_next() and it is done
2500 * with the help of copy_last_highmem_page().  For this purpose, if
2501 * @buffer is returned, @last_highmem_page is set to the page to which
2502 * the data will have to be copied from @buffer.
2503 */
2504static void *get_highmem_page_buffer(struct page *page,
2505				     struct chain_allocator *ca)
2506{
2507	struct highmem_pbe *pbe;
2508	void *kaddr;
2509
2510	if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
2511		/*
2512		 * We have allocated the "original" page frame and we can
2513		 * use it directly to store the loaded page.
2514		 */
2515		last_highmem_page = page;
2516		return buffer;
2517	}
2518	/*
2519	 * The "original" page frame has not been allocated and we have to
2520	 * use a "safe" page frame to store the loaded page.
2521	 */
2522	pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
2523	if (!pbe) {
2524		swsusp_free();
2525		return ERR_PTR(-ENOMEM);
2526	}
2527	pbe->orig_page = page;
2528	if (safe_highmem_pages > 0) {
2529		struct page *tmp;
2530
2531		/* Copy of the page will be stored in high memory */
2532		kaddr = buffer;
2533		tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
2534		safe_highmem_pages--;
2535		last_highmem_page = tmp;
2536		pbe->copy_page = tmp;
2537	} else {
2538		/* Copy of the page will be stored in normal memory */
2539		kaddr = __get_safe_page(ca->gfp_mask);
2540		if (!kaddr)
2541			return ERR_PTR(-ENOMEM);
2542		pbe->copy_page = virt_to_page(kaddr);
2543	}
2544	pbe->next = highmem_pblist;
2545	highmem_pblist = pbe;
2546	return kaddr;
2547}
2548
2549/**
2550 * copy_last_highmem_page - Copy most the most recent highmem image page.
2551 *
2552 * Copy the contents of a highmem image from @buffer, where the caller of
2553 * snapshot_write_next() has stored them, to the right location represented by
2554 * @last_highmem_page .
2555 */
2556static void copy_last_highmem_page(void)
2557{
2558	if (last_highmem_page) {
2559		void *dst;
2560
2561		dst = kmap_local_page(last_highmem_page);
2562		copy_page(dst, buffer);
2563		kunmap_local(dst);
2564		last_highmem_page = NULL;
2565	}
2566}
2567
2568static inline int last_highmem_page_copied(void)
2569{
2570	return !last_highmem_page;
2571}
2572
2573static inline void free_highmem_data(void)
2574{
2575	if (safe_highmem_bm)
2576		memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
2577
2578	if (buffer)
2579		free_image_page(buffer, PG_UNSAFE_CLEAR);
2580}
2581#else
2582static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2583
2584static inline int prepare_highmem_image(struct memory_bitmap *bm,
2585					unsigned int *nr_highmem_p) { return 0; }
2586
2587static inline void *get_highmem_page_buffer(struct page *page,
2588					    struct chain_allocator *ca)
2589{
2590	return ERR_PTR(-EINVAL);
2591}
2592
2593static inline void copy_last_highmem_page(void) {}
2594static inline int last_highmem_page_copied(void) { return 1; }
2595static inline void free_highmem_data(void) {}
2596#endif /* CONFIG_HIGHMEM */
2597
2598#define PBES_PER_LINKED_PAGE	(LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2599
2600/**
2601 * prepare_image - Make room for loading hibernation image.
2602 * @new_bm: Uninitialized memory bitmap structure.
2603 * @bm: Memory bitmap with unsafe pages marked.
2604 * @zero_bm: Memory bitmap containing the zero pages.
2605 *
2606 * Use @bm to mark the pages that will be overwritten in the process of
2607 * restoring the system memory state from the suspend image ("unsafe" pages)
2608 * and allocate memory for the image.
2609 *
2610 * The idea is to allocate a new memory bitmap first and then allocate
2611 * as many pages as needed for image data, but without specifying what those
2612 * pages will be used for just yet.  Instead, we mark them all as allocated and
2613 * create a lists of "safe" pages to be used later.  On systems with high
2614 * memory a list of "safe" highmem pages is created too.
2615 *
2616 * Because it was not known which pages were unsafe when @zero_bm was created,
2617 * make a copy of it and recreate it within safe pages.
2618 */
2619static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm,
2620		struct memory_bitmap *zero_bm)
2621{
2622	unsigned int nr_pages, nr_highmem;
2623	struct memory_bitmap tmp;
2624	struct linked_page *lp;
2625	int error;
2626
2627	/* If there is no highmem, the buffer will not be necessary */
2628	free_image_page(buffer, PG_UNSAFE_CLEAR);
2629	buffer = NULL;
2630
2631	nr_highmem = count_highmem_image_pages(bm);
2632	mark_unsafe_pages(bm);
2633
2634	error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2635	if (error)
2636		goto Free;
2637
2638	duplicate_memory_bitmap(new_bm, bm);
2639	memory_bm_free(bm, PG_UNSAFE_KEEP);
2640
2641	/* Make a copy of zero_bm so it can be created in safe pages */
2642	error = memory_bm_create(&tmp, GFP_ATOMIC, PG_SAFE);
2643	if (error)
2644		goto Free;
2645
2646	duplicate_memory_bitmap(&tmp, zero_bm);
2647	memory_bm_free(zero_bm, PG_UNSAFE_KEEP);
2648
2649	/* Recreate zero_bm in safe pages */
2650	error = memory_bm_create(zero_bm, GFP_ATOMIC, PG_SAFE);
2651	if (error)
2652		goto Free;
2653
2654	duplicate_memory_bitmap(zero_bm, &tmp);
2655	memory_bm_free(&tmp, PG_UNSAFE_CLEAR);
2656	/* At this point zero_bm is in safe pages and it can be used for restoring. */
2657
2658	if (nr_highmem > 0) {
2659		error = prepare_highmem_image(bm, &nr_highmem);
2660		if (error)
2661			goto Free;
2662	}
2663	/*
2664	 * Reserve some safe pages for potential later use.
2665	 *
2666	 * NOTE: This way we make sure there will be enough safe pages for the
2667	 * chain_alloc() in get_buffer().  It is a bit wasteful, but
2668	 * nr_copy_pages cannot be greater than 50% of the memory anyway.
2669	 *
2670	 * nr_copy_pages cannot be less than allocated_unsafe_pages too.
2671	 */
2672	nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;
2673	nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2674	while (nr_pages > 0) {
2675		lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2676		if (!lp) {
2677			error = -ENOMEM;
2678			goto Free;
2679		}
2680		lp->next = safe_pages_list;
2681		safe_pages_list = lp;
2682		nr_pages--;
2683	}
2684	/* Preallocate memory for the image */
2685	nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;
2686	while (nr_pages > 0) {
2687		lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2688		if (!lp) {
2689			error = -ENOMEM;
2690			goto Free;
2691		}
2692		if (!swsusp_page_is_free(virt_to_page(lp))) {
2693			/* The page is "safe", add it to the list */
2694			lp->next = safe_pages_list;
2695			safe_pages_list = lp;
2696		}
2697		/* Mark the page as allocated */
2698		swsusp_set_page_forbidden(virt_to_page(lp));
2699		swsusp_set_page_free(virt_to_page(lp));
2700		nr_pages--;
2701	}
2702	return 0;
2703
2704 Free:
2705	swsusp_free();
2706	return error;
2707}
2708
2709/**
2710 * get_buffer - Get the address to store the next image data page.
2711 *
2712 * Get the address that snapshot_write_next() should return to its caller to
2713 * write to.
2714 */
2715static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
2716{
2717	struct pbe *pbe;
2718	struct page *page;
2719	unsigned long pfn = memory_bm_next_pfn(bm);
2720
2721	if (pfn == BM_END_OF_MAP)
2722		return ERR_PTR(-EFAULT);
2723
2724	page = pfn_to_page(pfn);
2725	if (PageHighMem(page))
2726		return get_highmem_page_buffer(page, ca);
2727
2728	if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2729		/*
2730		 * We have allocated the "original" page frame and we can
2731		 * use it directly to store the loaded page.
2732		 */
2733		return page_address(page);
2734
2735	/*
2736	 * The "original" page frame has not been allocated and we have to
2737	 * use a "safe" page frame to store the loaded page.
2738	 */
2739	pbe = chain_alloc(ca, sizeof(struct pbe));
2740	if (!pbe) {
2741		swsusp_free();
2742		return ERR_PTR(-ENOMEM);
2743	}
2744	pbe->orig_address = page_address(page);
2745	pbe->address = __get_safe_page(ca->gfp_mask);
2746	if (!pbe->address)
2747		return ERR_PTR(-ENOMEM);
2748	pbe->next = restore_pblist;
2749	restore_pblist = pbe;
2750	return pbe->address;
2751}
2752
2753/**
2754 * snapshot_write_next - Get the address to store the next image page.
2755 * @handle: Snapshot handle structure to guide the writing.
2756 *
2757 * On the first call, @handle should point to a zeroed snapshot_handle
2758 * structure.  The structure gets populated then and a pointer to it should be
2759 * passed to this function every next time.
2760 *
2761 * On success, the function returns a positive number.  Then, the caller
2762 * is allowed to write up to the returned number of bytes to the memory
2763 * location computed by the data_of() macro.
2764 *
2765 * The function returns 0 to indicate the "end of file" condition.  Negative
2766 * numbers are returned on errors, in which cases the structure pointed to by
2767 * @handle is not updated and should not be used any more.
2768 */
2769int snapshot_write_next(struct snapshot_handle *handle)
2770{
2771	static struct chain_allocator ca;
2772	int error;
2773
2774next:
2775	/* Check if we have already loaded the entire image */
2776	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages)
2777		return 0;
2778
2779	if (!handle->cur) {
2780		if (!buffer)
2781			/* This makes the buffer be freed by swsusp_free() */
2782			buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2783
2784		if (!buffer)
2785			return -ENOMEM;
2786
2787		handle->buffer = buffer;
2788	} else if (handle->cur == 1) {
2789		error = load_header(buffer);
2790		if (error)
2791			return error;
2792
2793		safe_pages_list = NULL;
2794
2795		error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2796		if (error)
2797			return error;
2798
2799		error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY);
2800		if (error)
2801			return error;
2802
2803		nr_zero_pages = 0;
2804
2805		hibernate_restore_protection_begin();
2806	} else if (handle->cur <= nr_meta_pages + 1) {
2807		error = unpack_orig_pfns(buffer, &copy_bm, &zero_bm);
2808		if (error)
2809			return error;
2810
2811		if (handle->cur == nr_meta_pages + 1) {
2812			error = prepare_image(&orig_bm, &copy_bm, &zero_bm);
2813			if (error)
2814				return error;
2815
2816			chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2817			memory_bm_position_reset(&orig_bm);
2818			memory_bm_position_reset(&zero_bm);
2819			restore_pblist = NULL;
2820			handle->buffer = get_buffer(&orig_bm, &ca);
2821			if (IS_ERR(handle->buffer))
2822				return PTR_ERR(handle->buffer);
2823		}
2824	} else {
2825		copy_last_highmem_page();
2826		error = hibernate_restore_protect_page(handle->buffer);
2827		if (error)
2828			return error;
2829		handle->buffer = get_buffer(&orig_bm, &ca);
2830		if (IS_ERR(handle->buffer))
2831			return PTR_ERR(handle->buffer);
2832	}
2833	handle->sync_read = (handle->buffer == buffer);
2834	handle->cur++;
2835
2836	/* Zero pages were not included in the image, memset it and move on. */
2837	if (handle->cur > nr_meta_pages + 1 &&
2838	    memory_bm_test_bit(&zero_bm, memory_bm_get_current(&orig_bm))) {
2839		memset(handle->buffer, 0, PAGE_SIZE);
2840		goto next;
2841	}
2842
2843	return PAGE_SIZE;
2844}
2845
2846/**
2847 * snapshot_write_finalize - Complete the loading of a hibernation image.
2848 *
2849 * Must be called after the last call to snapshot_write_next() in case the last
2850 * page in the image happens to be a highmem page and its contents should be
2851 * stored in highmem.  Additionally, it recycles bitmap memory that's not
2852 * necessary any more.
2853 */
2854int snapshot_write_finalize(struct snapshot_handle *handle)
2855{
2856	int error;
2857
2858	/*
2859	 * Call snapshot_write_next() to drain any trailing zero pages,
2860	 * but make sure we're in the data page region first.
2861	 * This function can return PAGE_SIZE if the kernel was expecting
2862	 * another copy page. Return -ENODATA in that situation.
2863	 */
2864	if (handle->cur > nr_meta_pages + 1) {
2865		error = snapshot_write_next(handle);
2866		if (error)
2867			return error > 0 ? -ENODATA : error;
2868	}
2869	copy_last_highmem_page();
2870	error = hibernate_restore_protect_page(handle->buffer);
2871	/* Do that only if we have loaded the image entirely */
2872	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) {
2873		memory_bm_recycle(&orig_bm);
2874		free_highmem_data();
2875	}
2876	return error;
2877}
2878
2879int snapshot_image_loaded(struct snapshot_handle *handle)
2880{
2881	return !(!nr_copy_pages || !last_highmem_page_copied() ||
2882			handle->cur <= nr_meta_pages + nr_copy_pages + nr_zero_pages);
2883}
2884
2885#ifdef CONFIG_HIGHMEM
2886/* Assumes that @buf is ready and points to a "safe" page */
2887static inline void swap_two_pages_data(struct page *p1, struct page *p2,
2888				       void *buf)
2889{
2890	void *kaddr1, *kaddr2;
2891
2892	kaddr1 = kmap_local_page(p1);
2893	kaddr2 = kmap_local_page(p2);
2894	copy_page(buf, kaddr1);
2895	copy_page(kaddr1, kaddr2);
2896	copy_page(kaddr2, buf);
2897	kunmap_local(kaddr2);
2898	kunmap_local(kaddr1);
2899}
2900
2901/**
2902 * restore_highmem - Put highmem image pages into their original locations.
2903 *
2904 * For each highmem page that was in use before hibernation and is included in
2905 * the image, and also has been allocated by the "restore" kernel, swap its
2906 * current contents with the previous (ie. "before hibernation") ones.
2907 *
2908 * If the restore eventually fails, we can call this function once again and
2909 * restore the highmem state as seen by the restore kernel.
2910 */
2911int restore_highmem(void)
2912{
2913	struct highmem_pbe *pbe = highmem_pblist;
2914	void *buf;
2915
2916	if (!pbe)
2917		return 0;
2918
2919	buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2920	if (!buf)
2921		return -ENOMEM;
2922
2923	while (pbe) {
2924		swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2925		pbe = pbe->next;
2926	}
2927	free_image_page(buf, PG_UNSAFE_CLEAR);
2928	return 0;
2929}
2930#endif /* CONFIG_HIGHMEM */