swap.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * linux/kernel/power/swap.c
   4 *
   5 * This file provides functions for reading the suspend image from
   6 * and writing it to a swap partition.
   7 *
   8 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
   9 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
  10 * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com>
  11 */
  12
  13#define pr_fmt(fmt) "PM: " fmt
  14
  15#include <crypto/acompress.h>
  16#include <linux/module.h>
  17#include <linux/file.h>
  18#include <linux/delay.h>
  19#include <linux/bitops.h>
  20#include <linux/device.h>
  21#include <linux/bio.h>
  22#include <linux/blkdev.h>
  23#include <linux/swap.h>
  24#include <linux/swapops.h>
  25#include <linux/pm.h>
  26#include <linux/slab.h>
  27#include <linux/vmalloc.h>
  28#include <linux/cpumask.h>
  29#include <linux/atomic.h>
  30#include <linux/kthread.h>
  31#include <linux/crc32.h>
  32#include <linux/ktime.h>
  33
  34#include "power.h"
  35
  36#define HIBERNATE_SIG	"S1SUSPEND"
  37
  38u32 swsusp_hardware_signature;
  39
  40/*
  41 * When reading an {un,}compressed image, we may restore pages in place,
  42 * in which case some architectures need these pages cleaning before they
  43 * can be executed. We don't know which pages these may be, so clean the lot.
  44 */
  45static bool clean_pages_on_read;
  46static bool clean_pages_on_decompress;
  47
  48/*
  49 * The swap map is a data structure used for keeping track of each page
  50 * written to a swap partition.  It consists of many swap_map_page structures
  51 * that contain each an array of MAP_PAGE_ENTRIES swap entries.  These
  52 * structures are stored on the swap and linked together with the help of the
  53 * .next_swap member.
  54 *
  55 * The swap map is created during suspend.  The swap map pages are allocated and
  56 * populated one at a time, so we only need one memory page to set up the entire
  57 * structure.
  58 *
  59 * During resume we pick up all swap_map_page structures into a list.
  60 */
  61#define MAP_PAGE_ENTRIES	(PAGE_SIZE / sizeof(sector_t) - 1)
  62
  63/*
  64 * Number of free pages that are not high.
  65 */
  66static inline unsigned long low_free_pages(void)
  67{
  68	return nr_free_pages() - nr_free_highpages();
  69}
  70
  71/*
  72 * Number of pages required to be kept free while writing the image. Always
  73 * half of all available low pages before the writing starts.
  74 */
  75static inline unsigned long reqd_free_pages(void)
  76{
  77	return low_free_pages() / 2;
  78}
  79
  80struct swap_map_page {
  81	sector_t entries[MAP_PAGE_ENTRIES];
  82	sector_t next_swap;
  83};
  84
  85struct swap_map_page_list {
  86	struct swap_map_page *map;
  87	struct swap_map_page_list *next;
  88};
  89
  90/*
  91 * The swap_map_handle structure is used for handling swap in a file-alike way.
  92 */
  93struct swap_map_handle {
  94	struct swap_map_page *cur;
  95	struct swap_map_page_list *maps;
  96	sector_t cur_swap;
  97	sector_t first_sector;
  98	unsigned int k;
  99	unsigned long reqd_free_pages;
 100	u32 crc32;
 101};
 102
 103struct swsusp_header {
 104	char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) -
 105	              sizeof(u32) - sizeof(u32)];
 106	u32	hw_sig;
 107	u32	crc32;
 108	sector_t image;
 109	unsigned int flags;	/* Flags to pass to the "boot" kernel */
 110	char	orig_sig[10];
 111	char	sig[10];
 112} __packed;
 113
 114static struct swsusp_header *swsusp_header;
 115
 116/*
 117 * The following functions are used for tracing the allocated swap pages, so
 118 * that they can be freed in case of an error.
 119 */
 120struct swsusp_extent {
 121	struct rb_node node;
 122	unsigned long start;
 123	unsigned long end;
 124};
 125
 126static struct rb_root swsusp_extents = RB_ROOT;
 127
 128static int swsusp_extents_insert(unsigned long swap_offset)
 129{
 130	struct rb_node **new = &(swsusp_extents.rb_node);
 131	struct rb_node *parent = NULL;
 132	struct swsusp_extent *ext;
 133
 134	/* Figure out where to put the new node */
 135	while (*new) {
 136		ext = rb_entry(*new, struct swsusp_extent, node);
 137		parent = *new;
 138		if (swap_offset < ext->start) {
 139			/* Try to merge */
 140			if (swap_offset == ext->start - 1) {
 141				ext->start--;
 142				return 0;
 143			}
 144			new = &((*new)->rb_left);
 145		} else if (swap_offset > ext->end) {
 146			/* Try to merge */
 147			if (swap_offset == ext->end + 1) {
 148				ext->end++;
 149				return 0;
 150			}
 151			new = &((*new)->rb_right);
 152		} else {
 153			/* It already is in the tree */
 154			return -EINVAL;
 155		}
 156	}
 157	/* Add the new node and rebalance the tree. */
 158	ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
 159	if (!ext)
 160		return -ENOMEM;
 161
 162	ext->start = swap_offset;
 163	ext->end = swap_offset;
 164	rb_link_node(&ext->node, parent, new);
 165	rb_insert_color(&ext->node, &swsusp_extents);
 166	return 0;
 167}
 168
 169sector_t alloc_swapdev_block(int swap)
 170{
 171	unsigned long offset;
 172
 173	/*
 174	 * Allocate a swap page and register that it has been allocated, so that
 175	 * it can be freed in case of an error.
 176	 */
 177	offset = swp_offset(get_swap_page_of_type(swap));
 178	if (offset) {
 179		if (swsusp_extents_insert(offset))
 180			swap_free(swp_entry(swap, offset));
 181		else
 182			return swapdev_block(swap, offset);
 183	}
 184	return 0;
 185}
 186
 187void free_all_swap_pages(int swap)
 188{
 189	struct rb_node *node;
 190
 191	/*
 192	 * Free swap pages allocated for saving image data.  It also frees the
 193	 * extents used to register which swap entries had been allocated.
 194	 */
 195	while ((node = swsusp_extents.rb_node)) {
 196		struct swsusp_extent *ext;
 197
 198		ext = rb_entry(node, struct swsusp_extent, node);
 199		rb_erase(node, &swsusp_extents);
 200		swap_free_nr(swp_entry(swap, ext->start),
 201			     ext->end - ext->start + 1);
 202
 203		kfree(ext);
 204	}
 205}
 206
 207int swsusp_swap_in_use(void)
 208{
 209	return (swsusp_extents.rb_node != NULL);
 210}
 211
 212/*
 213 * General things
 214 */
 215
 216static unsigned short root_swap = 0xffff;
 217static struct file *hib_resume_bdev_file;
 218
 219struct hib_bio_batch {
 220	atomic_t		count;
 221	wait_queue_head_t	wait;
 222	blk_status_t		error;
 223	struct blk_plug		plug;
 224};
 225
 226static void hib_init_batch(struct hib_bio_batch *hb)
 227{
 228	atomic_set(&hb->count, 0);
 229	init_waitqueue_head(&hb->wait);
 230	hb->error = BLK_STS_OK;
 231	blk_start_plug(&hb->plug);
 232}
 233
 234static void hib_finish_batch(struct hib_bio_batch *hb)
 235{
 236	blk_finish_plug(&hb->plug);
 237}
 238
 239static void hib_end_io(struct bio *bio)
 240{
 241	struct hib_bio_batch *hb = bio->bi_private;
 242	struct page *page = bio_first_page_all(bio);
 243
 244	if (bio->bi_status) {
 245		pr_alert("Read-error on swap-device (%u:%u:%Lu)\n",
 246			 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
 247			 (unsigned long long)bio->bi_iter.bi_sector);
 248	}
 249
 250	if (bio_data_dir(bio) == WRITE)
 251		put_page(page);
 252	else if (clean_pages_on_read)
 253		flush_icache_range((unsigned long)page_address(page),
 254				   (unsigned long)page_address(page) + PAGE_SIZE);
 255
 256	if (bio->bi_status && !hb->error)
 257		hb->error = bio->bi_status;
 258	if (atomic_dec_and_test(&hb->count))
 259		wake_up(&hb->wait);
 260
 261	bio_put(bio);
 262}
 263
 264static int hib_submit_io_sync(blk_opf_t opf, pgoff_t page_off, void *addr)
 265{
 266	return bdev_rw_virt(file_bdev(hib_resume_bdev_file),
 267			page_off * (PAGE_SIZE >> 9), addr, PAGE_SIZE, opf);
 268}
 269
 270static int hib_submit_io_async(blk_opf_t opf, pgoff_t page_off, void *addr,
 271			 struct hib_bio_batch *hb)
 272{
 273	struct bio *bio;
 274
 275	bio = bio_alloc(file_bdev(hib_resume_bdev_file), 1, opf,
 276			GFP_NOIO | __GFP_HIGH);
 277	bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
 278	bio_add_virt_nofail(bio, addr, PAGE_SIZE);
 279	bio->bi_end_io = hib_end_io;
 280	bio->bi_private = hb;
 281	atomic_inc(&hb->count);
 282	submit_bio(bio);
 283	return 0;
 284}
 285
 286static int hib_wait_io(struct hib_bio_batch *hb)
 287{
 288	/*
 289	 * We are relying on the behavior of blk_plug that a thread with
 290	 * a plug will flush the plug list before sleeping.
 291	 */
 292	wait_event(hb->wait, atomic_read(&hb->count) == 0);
 293	return blk_status_to_errno(hb->error);
 294}
 295
 296/*
 297 * Saving part
 298 */
 299
 300static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
 301{
 302	int error;
 303
 304	hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header);
 305	if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
 306	    !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
 307		memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
 308		memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
 309		swsusp_header->image = handle->first_sector;
 310		if (swsusp_hardware_signature) {
 311			swsusp_header->hw_sig = swsusp_hardware_signature;
 312			flags |= SF_HW_SIG;
 313		}
 314		swsusp_header->flags = flags;
 315		if (flags & SF_CRC32_MODE)
 316			swsusp_header->crc32 = handle->crc32;
 317		error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC,
 318				      swsusp_resume_block, swsusp_header);
 319	} else {
 320		pr_err("Swap header not found!\n");
 321		error = -ENODEV;
 322	}
 323	return error;
 324}
 325
 326/*
 327 * Hold the swsusp_header flag. This is used in software_resume() in
 328 * 'kernel/power/hibernate' to check if the image is compressed and query
 329 * for the compression algorithm support(if so).
 330 */
 331unsigned int swsusp_header_flags;
 332
 333static int swsusp_swap_check(void)
 334{
 335	int res;
 336
 337	/*
 338	 * Check if the resume device is a swap device and get its index (if so).
 339	 * This is called before saving the image.
 340	 */
 341	if (swsusp_resume_device)
 342		res = swap_type_of(swsusp_resume_device, swsusp_resume_block);
 343	else
 344		res = find_first_swap(&swsusp_resume_device);
 345	if (res < 0)
 346		return res;
 347	root_swap = res;
 348
 349	hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device,
 350			BLK_OPEN_WRITE, NULL, NULL);
 351	if (IS_ERR(hib_resume_bdev_file))
 352		return PTR_ERR(hib_resume_bdev_file);
 353
 354	return 0;
 355}
 356
 357static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb)
 358{
 359	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
 360	void *src;
 361	int ret;
 362
 363	if (!offset)
 364		return -ENOSPC;
 365
 366	if (!hb)
 367		goto sync_io;
 368
 369	src = (void *)__get_free_page(gfp);
 370	if (!src) {
 371		ret = hib_wait_io(hb); /* Free pages */
 372		if (ret)
 373			return ret;
 374		src = (void *)__get_free_page(gfp);
 375		if (WARN_ON_ONCE(!src))
 376			goto sync_io;
 377	}
 378
 379	copy_page(src, buf);
 380	return hib_submit_io_async(REQ_OP_WRITE | REQ_SYNC, offset, src, hb);
 381sync_io:
 382	return hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, offset, buf);
 383}
 384
 385static void release_swap_writer(struct swap_map_handle *handle)
 386{
 387	if (handle->cur)
 388		free_page((unsigned long)handle->cur);
 389	handle->cur = NULL;
 390}
 391
 392static int get_swap_writer(struct swap_map_handle *handle)
 393{
 394	int ret;
 395
 396	ret = swsusp_swap_check();
 397	if (ret) {
 398		if (ret != -ENOSPC)
 399			pr_err("Cannot find swap device, try swapon -a\n");
 400		return ret;
 401	}
 402	handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
 403	if (!handle->cur) {
 404		ret = -ENOMEM;
 405		goto err_close;
 406	}
 407	handle->cur_swap = alloc_swapdev_block(root_swap);
 408	if (!handle->cur_swap) {
 409		ret = -ENOSPC;
 410		goto err_rel;
 411	}
 412	handle->k = 0;
 413	handle->reqd_free_pages = reqd_free_pages();
 414	handle->first_sector = handle->cur_swap;
 415	return 0;
 416err_rel:
 417	release_swap_writer(handle);
 418err_close:
 419	swsusp_close();
 420	return ret;
 421}
 422
 423static int swap_write_page(struct swap_map_handle *handle, void *buf,
 424		struct hib_bio_batch *hb)
 425{
 426	int error;
 427	sector_t offset;
 428
 429	if (!handle->cur)
 430		return -EINVAL;
 431	offset = alloc_swapdev_block(root_swap);
 432	error = write_page(buf, offset, hb);
 433	if (error)
 434		return error;
 435	handle->cur->entries[handle->k++] = offset;
 436	if (handle->k >= MAP_PAGE_ENTRIES) {
 437		offset = alloc_swapdev_block(root_swap);
 438		if (!offset)
 439			return -ENOSPC;
 440		handle->cur->next_swap = offset;
 441		error = write_page(handle->cur, handle->cur_swap, hb);
 442		if (error)
 443			goto out;
 444		clear_page(handle->cur);
 445		handle->cur_swap = offset;
 446		handle->k = 0;
 447
 448		if (hb && low_free_pages() <= handle->reqd_free_pages) {
 449			error = hib_wait_io(hb);
 450			if (error)
 451				goto out;
 452			/*
 453			 * Recalculate the number of required free pages, to
 454			 * make sure we never take more than half.
 455			 */
 456			handle->reqd_free_pages = reqd_free_pages();
 457		}
 458	}
 459 out:
 460	return error;
 461}
 462
 463static int flush_swap_writer(struct swap_map_handle *handle)
 464{
 465	if (handle->cur && handle->cur_swap)
 466		return write_page(handle->cur, handle->cur_swap, NULL);
 467	else
 468		return -EINVAL;
 469}
 470
 471static int swap_writer_finish(struct swap_map_handle *handle,
 472		unsigned int flags, int error)
 473{
 474	if (!error) {
 475		pr_info("S");
 476		error = mark_swapfiles(handle, flags);
 477		pr_cont("|\n");
 478		flush_swap_writer(handle);
 479	}
 480
 481	if (error)
 482		free_all_swap_pages(root_swap);
 483	release_swap_writer(handle);
 484	swsusp_close();
 485
 486	return error;
 487}
 488
 489/*
 490 * Bytes we need for compressed data in worst case. We assume(limitation)
 491 * this is the worst of all the compression algorithms.
 492 */
 493#define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2)
 494
 495/* We need to remember how much compressed data we need to read. */
 496#define CMP_HEADER	sizeof(size_t)
 497
 498/* Number of pages/bytes we'll compress at one time. */
 499#define UNC_PAGES	32
 500#define UNC_SIZE	(UNC_PAGES * PAGE_SIZE)
 501
 502/* Number of pages we need for compressed data (worst case). */
 503#define CMP_PAGES	DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \
 504				CMP_HEADER, PAGE_SIZE)
 505#define CMP_SIZE	(CMP_PAGES * PAGE_SIZE)
 506
 507/* Default number of threads for compression/decompression. */
 508#define CMP_THREADS    3
 509static unsigned int hibernate_compression_threads = CMP_THREADS;
 510
 511/* Minimum/maximum number of pages for read buffering. */
 512#define CMP_MIN_RD_PAGES	1024
 513#define CMP_MAX_RD_PAGES	8192
 514
 515static int save_image(struct swap_map_handle *handle,
 516                      struct snapshot_handle *snapshot,
 517                      unsigned int nr_to_write)
 518{
 519	unsigned int m;
 520	int ret;
 521	int nr_pages;
 522	int err2;
 523	struct hib_bio_batch hb;
 524	ktime_t start;
 525	ktime_t stop;
 526
 527	hib_init_batch(&hb);
 528
 529	pr_info("Saving image data pages (%u pages)...\n",
 530		nr_to_write);
 531	m = nr_to_write / 10;
 532	if (!m)
 533		m = 1;
 534	nr_pages = 0;
 535	start = ktime_get();
 536	while (1) {
 537		ret = snapshot_read_next(snapshot);
 538		if (ret <= 0)
 539			break;
 540		ret = swap_write_page(handle, data_of(*snapshot), &hb);
 541		if (ret)
 542			break;
 543		if (!(nr_pages % m))
 544			pr_info("Image saving progress: %3d%%\n",
 545				nr_pages / m * 10);
 546		nr_pages++;
 547	}
 548	err2 = hib_wait_io(&hb);
 549	hib_finish_batch(&hb);
 550	stop = ktime_get();
 551	if (!ret)
 552		ret = err2;
 553	if (!ret)
 554		pr_info("Image saving done\n");
 555	swsusp_show_speed(start, stop, nr_to_write, "Wrote");
 556	return ret;
 557}
 558
 559/*
 560 * Structure used for CRC32.
 561 */
 562struct crc_data {
 563	struct task_struct *thr;                  /* thread */
 564	atomic_t ready;                           /* ready to start flag */
 565	atomic_t stop;                            /* ready to stop flag */
 566	unsigned run_threads;                     /* nr current threads */
 567	wait_queue_head_t go;                     /* start crc update */
 568	wait_queue_head_t done;                   /* crc update done */
 569	u32 *crc32;                               /* points to handle's crc32 */
 570	size_t **unc_len;			  /* uncompressed lengths */
 571	unsigned char **unc;			  /* uncompressed data */
 572};
 573
 574static struct crc_data *alloc_crc_data(int nr_threads)
 575{
 576	struct crc_data *crc;
 577
 578	crc = kzalloc(sizeof(*crc), GFP_KERNEL);
 579	if (!crc)
 580		return NULL;
 581
 582	crc->unc = kcalloc(nr_threads, sizeof(*crc->unc), GFP_KERNEL);
 583	if (!crc->unc)
 584		goto err_free_crc;
 585
 586	crc->unc_len = kcalloc(nr_threads, sizeof(*crc->unc_len), GFP_KERNEL);
 587	if (!crc->unc_len)
 588		goto err_free_unc;
 589
 590	return crc;
 591
 592err_free_unc:
 593	kfree(crc->unc);
 594err_free_crc:
 595	kfree(crc);
 596	return NULL;
 597}
 598
 599static void free_crc_data(struct crc_data *crc)
 600{
 601	if (!crc)
 602		return;
 603
 604	if (crc->thr)
 605		kthread_stop(crc->thr);
 606
 607	kfree(crc->unc_len);
 608	kfree(crc->unc);
 609	kfree(crc);
 610}
 611
 612static int crc32_threadfn(void *data)
 613{
 614	struct crc_data *d = data;
 615	unsigned i;
 616
 617	while (1) {
 618		wait_event(d->go, atomic_read_acquire(&d->ready) ||
 619		                  kthread_should_stop());
 620		if (kthread_should_stop()) {
 621			d->thr = NULL;
 622			atomic_set_release(&d->stop, 1);
 623			wake_up(&d->done);
 624			break;
 625		}
 626		atomic_set(&d->ready, 0);
 627
 628		for (i = 0; i < d->run_threads; i++)
 629			*d->crc32 = crc32_le(*d->crc32,
 630			                     d->unc[i], *d->unc_len[i]);
 631		atomic_set_release(&d->stop, 1);
 632		wake_up(&d->done);
 633	}
 634	return 0;
 635}
 636
 637/*
 638 * Structure used for data compression.
 639 */
 640struct cmp_data {
 641	struct task_struct *thr;                  /* thread */
 642	struct crypto_acomp *cc;		  /* crypto compressor */
 643	struct acomp_req *cr;			  /* crypto request */
 644	atomic_t ready;                           /* ready to start flag */
 645	atomic_t stop;                            /* ready to stop flag */
 646	int ret;                                  /* return code */
 647	wait_queue_head_t go;                     /* start compression */
 648	wait_queue_head_t done;                   /* compression done */
 649	size_t unc_len;                           /* uncompressed length */
 650	size_t cmp_len;                           /* compressed length */
 651	unsigned char unc[UNC_SIZE];              /* uncompressed buffer */
 652	unsigned char cmp[CMP_SIZE];              /* compressed buffer */
 653};
 654
 655/* Indicates the image size after compression */
 656static atomic64_t compressed_size = ATOMIC_INIT(0);
 657
 658static int compress_threadfn(void *data)
 659{
 660	struct cmp_data *d = data;
 661
 662	while (1) {
 663		wait_event(d->go, atomic_read_acquire(&d->ready) ||
 664		                  kthread_should_stop());
 665		if (kthread_should_stop()) {
 666			d->thr = NULL;
 667			d->ret = -1;
 668			atomic_set_release(&d->stop, 1);
 669			wake_up(&d->done);
 670			break;
 671		}
 672		atomic_set(&d->ready, 0);
 673
 674		acomp_request_set_callback(d->cr, CRYPTO_TFM_REQ_MAY_SLEEP,
 675					   NULL, NULL);
 676		acomp_request_set_src_nondma(d->cr, d->unc, d->unc_len);
 677		acomp_request_set_dst_nondma(d->cr, d->cmp + CMP_HEADER,
 678					     CMP_SIZE - CMP_HEADER);
 679		d->ret = crypto_acomp_compress(d->cr);
 680		d->cmp_len = d->cr->dlen;
 681
 682		atomic64_add(d->cmp_len, &compressed_size);
 683		atomic_set_release(&d->stop, 1);
 684		wake_up(&d->done);
 685	}
 686	return 0;
 687}
 688
 689static int save_compressed_image(struct swap_map_handle *handle,
 690				 struct snapshot_handle *snapshot,
 691				 unsigned int nr_to_write)
 692{
 693	unsigned int m;
 694	int ret = 0;
 695	int nr_pages;
 696	int err2;
 697	struct hib_bio_batch hb;
 698	ktime_t start;
 699	ktime_t stop;
 700	size_t off;
 701	unsigned int thr, run_threads, nr_threads;
 702	unsigned char *page = NULL;
 703	struct cmp_data *data = NULL;
 704	struct crc_data *crc = NULL;
 705
 706	hib_init_batch(&hb);
 707
 708	atomic64_set(&compressed_size, 0);
 709
 710	/*
 711	 * We'll limit the number of threads for compression to limit memory
 712	 * footprint.
 713	 */
 714	nr_threads = num_online_cpus() - 1;
 715	nr_threads = clamp_val(nr_threads, 1, hibernate_compression_threads);
 716
 717	page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH);
 718	if (!page) {
 719		pr_err("Failed to allocate %s page\n", hib_comp_algo);
 720		ret = -ENOMEM;
 721		goto out_clean;
 722	}
 723
 724	data = vcalloc(nr_threads, sizeof(*data));
 725	if (!data) {
 726		pr_err("Failed to allocate %s data\n", hib_comp_algo);
 727		ret = -ENOMEM;
 728		goto out_clean;
 729	}
 730
 731	crc = alloc_crc_data(nr_threads);
 732	if (!crc) {
 733		pr_err("Failed to allocate crc\n");
 734		ret = -ENOMEM;
 735		goto out_clean;
 736	}
 737
 738	/*
 739	 * Start the compression threads.
 740	 */
 741	for (thr = 0; thr < nr_threads; thr++) {
 742		init_waitqueue_head(&data[thr].go);
 743		init_waitqueue_head(&data[thr].done);
 744
 745		data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC);
 746		if (IS_ERR_OR_NULL(data[thr].cc)) {
 747			pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc));
 748			ret = -EFAULT;
 749			goto out_clean;
 750		}
 751
 752		data[thr].cr = acomp_request_alloc(data[thr].cc);
 753		if (!data[thr].cr) {
 754			pr_err("Could not allocate comp request\n");
 755			ret = -ENOMEM;
 756			goto out_clean;
 757		}
 758
 759		data[thr].thr = kthread_run(compress_threadfn,
 760		                            &data[thr],
 761		                            "image_compress/%u", thr);
 762		if (IS_ERR(data[thr].thr)) {
 763			data[thr].thr = NULL;
 764			pr_err("Cannot start compression threads\n");
 765			ret = -ENOMEM;
 766			goto out_clean;
 767		}
 768	}
 769
 770	/*
 771	 * Start the CRC32 thread.
 772	 */
 773	init_waitqueue_head(&crc->go);
 774	init_waitqueue_head(&crc->done);
 775
 776	handle->crc32 = 0;
 777	crc->crc32 = &handle->crc32;
 778	for (thr = 0; thr < nr_threads; thr++) {
 779		crc->unc[thr] = data[thr].unc;
 780		crc->unc_len[thr] = &data[thr].unc_len;
 781	}
 782
 783	crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
 784	if (IS_ERR(crc->thr)) {
 785		crc->thr = NULL;
 786		pr_err("Cannot start CRC32 thread\n");
 787		ret = -ENOMEM;
 788		goto out_clean;
 789	}
 790
 791	/*
 792	 * Adjust the number of required free pages after all allocations have
 793	 * been done. We don't want to run out of pages when writing.
 794	 */
 795	handle->reqd_free_pages = reqd_free_pages();
 796
 797	pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo);
 798	pr_info("Compressing and saving image data (%u pages)...\n",
 799		nr_to_write);
 800	m = nr_to_write / 10;
 801	if (!m)
 802		m = 1;
 803	nr_pages = 0;
 804	start = ktime_get();
 805	for (;;) {
 806		for (thr = 0; thr < nr_threads; thr++) {
 807			for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) {
 808				ret = snapshot_read_next(snapshot);
 809				if (ret < 0)
 810					goto out_finish;
 811
 812				if (!ret)
 813					break;
 814
 815				memcpy(data[thr].unc + off,
 816				       data_of(*snapshot), PAGE_SIZE);
 817
 818				if (!(nr_pages % m))
 819					pr_info("Image saving progress: %3d%%\n",
 820						nr_pages / m * 10);
 821				nr_pages++;
 822			}
 823			if (!off)
 824				break;
 825
 826			data[thr].unc_len = off;
 827
 828			atomic_set_release(&data[thr].ready, 1);
 829			wake_up(&data[thr].go);
 830		}
 831
 832		if (!thr)
 833			break;
 834
 835		crc->run_threads = thr;
 836		atomic_set_release(&crc->ready, 1);
 837		wake_up(&crc->go);
 838
 839		for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
 840			wait_event(data[thr].done,
 841				atomic_read_acquire(&data[thr].stop));
 842			atomic_set(&data[thr].stop, 0);
 843
 844			ret = data[thr].ret;
 845
 846			if (ret < 0) {
 847				pr_err("%s compression failed\n", hib_comp_algo);
 848				goto out_finish;
 849			}
 850
 851			if (unlikely(!data[thr].cmp_len ||
 852			             data[thr].cmp_len >
 853				     bytes_worst_compress(data[thr].unc_len))) {
 854				pr_err("Invalid %s compressed length\n", hib_comp_algo);
 855				ret = -1;
 856				goto out_finish;
 857			}
 858
 859			*(size_t *)data[thr].cmp = data[thr].cmp_len;
 860
 861			/*
 862			 * Given we are writing one page at a time to disk, we
 863			 * copy that much from the buffer, although the last
 864			 * bit will likely be smaller than full page. This is
 865			 * OK - we saved the length of the compressed data, so
 866			 * any garbage at the end will be discarded when we
 867			 * read it.
 868			 */
 869			for (off = 0;
 870			     off < CMP_HEADER + data[thr].cmp_len;
 871			     off += PAGE_SIZE) {
 872				memcpy(page, data[thr].cmp + off, PAGE_SIZE);
 873
 874				ret = swap_write_page(handle, page, &hb);
 875				if (ret)
 876					goto out_finish;
 877			}
 878		}
 879
 880		wait_event(crc->done, atomic_read_acquire(&crc->stop));
 881		atomic_set(&crc->stop, 0);
 882	}
 883
 884out_finish:
 885	err2 = hib_wait_io(&hb);
 886	stop = ktime_get();
 887	if (!ret)
 888		ret = err2;
 889	if (!ret) {
 890		swsusp_show_speed(start, stop, nr_to_write, "Wrote");
 891		pr_info("Image size after compression: %lld kbytes\n",
 892			(atomic64_read(&compressed_size) / 1024));
 893		pr_info("Image saving done\n");
 894	} else {
 895		pr_err("Image saving failed: %d\n", ret);
 896	}
 897
 898out_clean:
 899	hib_finish_batch(&hb);
 900	free_crc_data(crc);
 901	if (data) {
 902		for (thr = 0; thr < nr_threads; thr++) {
 903			if (data[thr].thr)
 904				kthread_stop(data[thr].thr);
 905			if (data[thr].cr)
 906				acomp_request_free(data[thr].cr);
 907
 908			if (!IS_ERR_OR_NULL(data[thr].cc))
 909				crypto_free_acomp(data[thr].cc);
 910		}
 911		vfree(data);
 912	}
 913	if (page)
 914		free_page((unsigned long)page);
 915
 916	return ret;
 917}
 918
 919static int enough_swap(unsigned int nr_pages)
 920{
 921	unsigned int free_swap = count_swap_pages(root_swap, 1);
 922	unsigned int required;
 923
 924	pr_debug("Free swap pages: %u\n", free_swap);
 925
 926	required = PAGES_FOR_IO + nr_pages;
 927	return free_swap > required;
 928}
 929
 930/**
 931 * swsusp_write - Write entire image and metadata.
 932 * @flags: flags to pass to the "boot" kernel in the image header
 933 *
 934 * It is important _NOT_ to umount filesystems at this point. We want them
 935 * synced (in case something goes wrong) but we DO not want to mark filesystem
 936 * clean: it is not. (And it does not matter, if we resume correctly, we'll mark
 937 * system clean, anyway.)
 938 *
 939 * Return: 0 on success, negative error code on failure.
 940 */
 941int swsusp_write(unsigned int flags)
 942{
 943	struct swap_map_handle handle;
 944	struct snapshot_handle snapshot;
 945	struct swsusp_info *header;
 946	unsigned long pages;
 947	int error;
 948
 949	pages = snapshot_get_image_size();
 950	error = get_swap_writer(&handle);
 951	if (error) {
 952		pr_err("Cannot get swap writer\n");
 953		return error;
 954	}
 955	if (flags & SF_NOCOMPRESS_MODE) {
 956		if (!enough_swap(pages)) {
 957			pr_err("Not enough free swap\n");
 958			error = -ENOSPC;
 959			goto out_finish;
 960		}
 961	}
 962	memset(&snapshot, 0, sizeof(struct snapshot_handle));
 963	error = snapshot_read_next(&snapshot);
 964	if (error < (int)PAGE_SIZE) {
 965		if (error >= 0)
 966			error = -EFAULT;
 967
 968		goto out_finish;
 969	}
 970	header = (struct swsusp_info *)data_of(snapshot);
 971	error = swap_write_page(&handle, header, NULL);
 972	if (!error) {
 973		error = (flags & SF_NOCOMPRESS_MODE) ?
 974			save_image(&handle, &snapshot, pages - 1) :
 975			save_compressed_image(&handle, &snapshot, pages - 1);
 976	}
 977out_finish:
 978	error = swap_writer_finish(&handle, flags, error);
 979	return error;
 980}
 981
 982/*
 983 * The following functions allow us to read data using a swap map in a file-like
 984 * way.
 985 */
 986
 987static void release_swap_reader(struct swap_map_handle *handle)
 988{
 989	struct swap_map_page_list *tmp;
 990
 991	while (handle->maps) {
 992		if (handle->maps->map)
 993			free_page((unsigned long)handle->maps->map);
 994		tmp = handle->maps;
 995		handle->maps = handle->maps->next;
 996		kfree(tmp);
 997	}
 998	handle->cur = NULL;
 999}
1000
1001static int get_swap_reader(struct swap_map_handle *handle,
1002		unsigned int *flags_p)
1003{
1004	int error;
1005	struct swap_map_page_list *tmp, *last;
1006	sector_t offset;
1007
1008	*flags_p = swsusp_header->flags;
1009
1010	if (!swsusp_header->image) /* how can this happen? */
1011		return -EINVAL;
1012
1013	handle->cur = NULL;
1014	last = handle->maps = NULL;
1015	offset = swsusp_header->image;
1016	while (offset) {
1017		tmp = kzalloc(sizeof(*handle->maps), GFP_KERNEL);
1018		if (!tmp) {
1019			release_swap_reader(handle);
1020			return -ENOMEM;
1021		}
1022		if (!handle->maps)
1023			handle->maps = tmp;
1024		if (last)
1025			last->next = tmp;
1026		last = tmp;
1027
1028		tmp->map = (struct swap_map_page *)
1029			   __get_free_page(GFP_NOIO | __GFP_HIGH);
1030		if (!tmp->map) {
1031			release_swap_reader(handle);
1032			return -ENOMEM;
1033		}
1034
1035		error = hib_submit_io_sync(REQ_OP_READ, offset, tmp->map);
1036		if (error) {
1037			release_swap_reader(handle);
1038			return error;
1039		}
1040		offset = tmp->map->next_swap;
1041	}
1042	handle->k = 0;
1043	handle->cur = handle->maps->map;
1044	return 0;
1045}
1046
1047static int swap_read_page(struct swap_map_handle *handle, void *buf,
1048		struct hib_bio_batch *hb)
1049{
1050	sector_t offset;
1051	int error;
1052	struct swap_map_page_list *tmp;
1053
1054	if (!handle->cur)
1055		return -EINVAL;
1056	offset = handle->cur->entries[handle->k];
1057	if (!offset)
1058		return -EFAULT;
1059	if (hb)
1060		error = hib_submit_io_async(REQ_OP_READ, offset, buf, hb);
1061	else
1062		error = hib_submit_io_sync(REQ_OP_READ, offset, buf);
1063	if (error)
1064		return error;
1065	if (++handle->k >= MAP_PAGE_ENTRIES) {
1066		handle->k = 0;
1067		free_page((unsigned long)handle->maps->map);
1068		tmp = handle->maps;
1069		handle->maps = handle->maps->next;
1070		kfree(tmp);
1071		if (!handle->maps)
1072			release_swap_reader(handle);
1073		else
1074			handle->cur = handle->maps->map;
1075	}
1076	return error;
1077}
1078
1079static int swap_reader_finish(struct swap_map_handle *handle)
1080{
1081	release_swap_reader(handle);
1082
1083	return 0;
1084}
1085
1086static int load_image(struct swap_map_handle *handle,
1087                      struct snapshot_handle *snapshot,
1088                      unsigned int nr_to_read)
1089{
1090	unsigned int m;
1091	int ret = 0;
1092	ktime_t start;
1093	ktime_t stop;
1094	struct hib_bio_batch hb;
1095	int err2;
1096	unsigned nr_pages;
1097
1098	hib_init_batch(&hb);
1099
1100	clean_pages_on_read = true;
1101	pr_info("Loading image data pages (%u pages)...\n", nr_to_read);
1102	m = nr_to_read / 10;
1103	if (!m)
1104		m = 1;
1105	nr_pages = 0;
1106	start = ktime_get();
1107	for ( ; ; ) {
1108		ret = snapshot_write_next(snapshot);
1109		if (ret <= 0)
1110			break;
1111		ret = swap_read_page(handle, data_of(*snapshot), &hb);
1112		if (ret)
1113			break;
1114		if (snapshot->sync_read)
1115			ret = hib_wait_io(&hb);
1116		if (ret)
1117			break;
1118		if (!(nr_pages % m))
1119			pr_info("Image loading progress: %3d%%\n",
1120				nr_pages / m * 10);
1121		nr_pages++;
1122	}
1123	err2 = hib_wait_io(&hb);
1124	hib_finish_batch(&hb);
1125	stop = ktime_get();
1126	if (!ret)
1127		ret = err2;
1128	if (!ret) {
1129		pr_info("Image loading done\n");
1130		ret = snapshot_write_finalize(snapshot);
1131		if (!ret && !snapshot_image_loaded(snapshot))
1132			ret = -ENODATA;
1133	}
1134	swsusp_show_speed(start, stop, nr_to_read, "Read");
1135	return ret;
1136}
1137
1138/*
1139 * Structure used for data decompression.
1140 */
1141struct dec_data {
1142	struct task_struct *thr;                  /* thread */
1143	struct crypto_acomp *cc;		  /* crypto compressor */
1144	struct acomp_req *cr;			  /* crypto request */
1145	atomic_t ready;                           /* ready to start flag */
1146	atomic_t stop;                            /* ready to stop flag */
1147	int ret;                                  /* return code */
1148	wait_queue_head_t go;                     /* start decompression */
1149	wait_queue_head_t done;                   /* decompression done */
1150	size_t unc_len;                           /* uncompressed length */
1151	size_t cmp_len;                           /* compressed length */
1152	unsigned char unc[UNC_SIZE];              /* uncompressed buffer */
1153	unsigned char cmp[CMP_SIZE];              /* compressed buffer */
1154};
1155
1156static int decompress_threadfn(void *data)
1157{
1158	struct dec_data *d = data;
1159
1160	while (1) {
1161		wait_event(d->go, atomic_read_acquire(&d->ready) ||
1162		                  kthread_should_stop());
1163		if (kthread_should_stop()) {
1164			d->thr = NULL;
1165			d->ret = -1;
1166			atomic_set_release(&d->stop, 1);
1167			wake_up(&d->done);
1168			break;
1169		}
1170		atomic_set(&d->ready, 0);
1171
1172		acomp_request_set_callback(d->cr, CRYPTO_TFM_REQ_MAY_SLEEP,
1173					   NULL, NULL);
1174		acomp_request_set_src_nondma(d->cr, d->cmp + CMP_HEADER,
1175					     d->cmp_len);
1176		acomp_request_set_dst_nondma(d->cr, d->unc, UNC_SIZE);
1177		d->ret = crypto_acomp_decompress(d->cr);
1178		d->unc_len = d->cr->dlen;
1179
1180		if (clean_pages_on_decompress)
1181			flush_icache_range((unsigned long)d->unc,
1182					   (unsigned long)d->unc + d->unc_len);
1183
1184		atomic_set_release(&d->stop, 1);
1185		wake_up(&d->done);
1186	}
1187	return 0;
1188}
1189
1190static int load_compressed_image(struct swap_map_handle *handle,
1191				 struct snapshot_handle *snapshot,
1192				 unsigned int nr_to_read)
1193{
1194	unsigned int m;
1195	int ret = 0;
1196	int eof = 0;
1197	struct hib_bio_batch hb;
1198	ktime_t start;
1199	ktime_t stop;
1200	unsigned nr_pages;
1201	size_t off;
1202	unsigned i, thr, run_threads, nr_threads;
1203	unsigned ring = 0, pg = 0, ring_size = 0,
1204	         have = 0, want, need, asked = 0;
1205	unsigned long read_pages = 0;
1206	unsigned char **page = NULL;
1207	struct dec_data *data = NULL;
1208	struct crc_data *crc = NULL;
1209
1210	hib_init_batch(&hb);
1211
1212	/*
1213	 * We'll limit the number of threads for decompression to limit memory
1214	 * footprint.
1215	 */
1216	nr_threads = num_online_cpus() - 1;
1217	nr_threads = clamp_val(nr_threads, 1, hibernate_compression_threads);
1218
1219	page = vmalloc_array(CMP_MAX_RD_PAGES, sizeof(*page));
1220	if (!page) {
1221		pr_err("Failed to allocate %s page\n", hib_comp_algo);
1222		ret = -ENOMEM;
1223		goto out_clean;
1224	}
1225
1226	data = vcalloc(nr_threads, sizeof(*data));
1227	if (!data) {
1228		pr_err("Failed to allocate %s data\n", hib_comp_algo);
1229		ret = -ENOMEM;
1230		goto out_clean;
1231	}
1232
1233	crc = alloc_crc_data(nr_threads);
1234	if (!crc) {
1235		pr_err("Failed to allocate crc\n");
1236		ret = -ENOMEM;
1237		goto out_clean;
1238	}
1239
1240	clean_pages_on_decompress = true;
1241
1242	/*
1243	 * Start the decompression threads.
1244	 */
1245	for (thr = 0; thr < nr_threads; thr++) {
1246		init_waitqueue_head(&data[thr].go);
1247		init_waitqueue_head(&data[thr].done);
1248
1249		data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC);
1250		if (IS_ERR_OR_NULL(data[thr].cc)) {
1251			pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc));
1252			ret = -EFAULT;
1253			goto out_clean;
1254		}
1255
1256		data[thr].cr = acomp_request_alloc(data[thr].cc);
1257		if (!data[thr].cr) {
1258			pr_err("Could not allocate comp request\n");
1259			ret = -ENOMEM;
1260			goto out_clean;
1261		}
1262
1263		data[thr].thr = kthread_run(decompress_threadfn,
1264		                            &data[thr],
1265		                            "image_decompress/%u", thr);
1266		if (IS_ERR(data[thr].thr)) {
1267			data[thr].thr = NULL;
1268			pr_err("Cannot start decompression threads\n");
1269			ret = -ENOMEM;
1270			goto out_clean;
1271		}
1272	}
1273
1274	/*
1275	 * Start the CRC32 thread.
1276	 */
1277	init_waitqueue_head(&crc->go);
1278	init_waitqueue_head(&crc->done);
1279
1280	handle->crc32 = 0;
1281	crc->crc32 = &handle->crc32;
1282	for (thr = 0; thr < nr_threads; thr++) {
1283		crc->unc[thr] = data[thr].unc;
1284		crc->unc_len[thr] = &data[thr].unc_len;
1285	}
1286
1287	crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
1288	if (IS_ERR(crc->thr)) {
1289		crc->thr = NULL;
1290		pr_err("Cannot start CRC32 thread\n");
1291		ret = -ENOMEM;
1292		goto out_clean;
1293	}
1294
1295	/*
1296	 * Set the number of pages for read buffering.
1297	 * This is complete guesswork, because we'll only know the real
1298	 * picture once prepare_image() is called, which is much later on
1299	 * during the image load phase. We'll assume the worst case and
1300	 * say that none of the image pages are from high memory.
1301	 */
1302	if (low_free_pages() > snapshot_get_image_size())
1303		read_pages = (low_free_pages() - snapshot_get_image_size()) / 2;
1304	read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES);
1305
1306	for (i = 0; i < read_pages; i++) {
1307		page[i] = (void *)__get_free_page(i < CMP_PAGES ?
1308						  GFP_NOIO | __GFP_HIGH :
1309						  GFP_NOIO | __GFP_NOWARN |
1310						  __GFP_NORETRY);
1311
1312		if (!page[i]) {
1313			if (i < CMP_PAGES) {
1314				ring_size = i;
1315				pr_err("Failed to allocate %s pages\n", hib_comp_algo);
1316				ret = -ENOMEM;
1317				goto out_clean;
1318			} else {
1319				break;
1320			}
1321		}
1322	}
1323	want = ring_size = i;
1324
1325	pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo);
1326	pr_info("Loading and decompressing image data (%u pages)...\n",
1327		nr_to_read);
1328	m = nr_to_read / 10;
1329	if (!m)
1330		m = 1;
1331	nr_pages = 0;
1332	start = ktime_get();
1333
1334	ret = snapshot_write_next(snapshot);
1335	if (ret <= 0)
1336		goto out_finish;
1337
1338	for(;;) {
1339		for (i = 0; !eof && i < want; i++) {
1340			ret = swap_read_page(handle, page[ring], &hb);
1341			if (ret) {
1342				/*
1343				 * On real read error, finish. On end of data,
1344				 * set EOF flag and just exit the read loop.
1345				 */
1346				if (handle->cur &&
1347				    handle->cur->entries[handle->k]) {
1348					goto out_finish;
1349				} else {
1350					eof = 1;
1351					break;
1352				}
1353			}
1354			if (++ring >= ring_size)
1355				ring = 0;
1356		}
1357		asked += i;
1358		want -= i;
1359
1360		/*
1361		 * We are out of data, wait for some more.
1362		 */
1363		if (!have) {
1364			if (!asked)
1365				break;
1366
1367			ret = hib_wait_io(&hb);
1368			if (ret)
1369				goto out_finish;
1370			have += asked;
1371			asked = 0;
1372			if (eof)
1373				eof = 2;
1374		}
1375
1376		if (crc->run_threads) {
1377			wait_event(crc->done, atomic_read_acquire(&crc->stop));
1378			atomic_set(&crc->stop, 0);
1379			crc->run_threads = 0;
1380		}
1381
1382		for (thr = 0; have && thr < nr_threads; thr++) {
1383			data[thr].cmp_len = *(size_t *)page[pg];
1384			if (unlikely(!data[thr].cmp_len ||
1385			             data[thr].cmp_len >
1386					bytes_worst_compress(UNC_SIZE))) {
1387				pr_err("Invalid %s compressed length\n", hib_comp_algo);
1388				ret = -1;
1389				goto out_finish;
1390			}
1391
1392			need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER,
1393			                    PAGE_SIZE);
1394			if (need > have) {
1395				if (eof > 1) {
1396					ret = -1;
1397					goto out_finish;
1398				}
1399				break;
1400			}
1401
1402			for (off = 0;
1403			     off < CMP_HEADER + data[thr].cmp_len;
1404			     off += PAGE_SIZE) {
1405				memcpy(data[thr].cmp + off,
1406				       page[pg], PAGE_SIZE);
1407				have--;
1408				want++;
1409				if (++pg >= ring_size)
1410					pg = 0;
1411			}
1412
1413			atomic_set_release(&data[thr].ready, 1);
1414			wake_up(&data[thr].go);
1415		}
1416
1417		/*
1418		 * Wait for more data while we are decompressing.
1419		 */
1420		if (have < CMP_PAGES && asked) {
1421			ret = hib_wait_io(&hb);
1422			if (ret)
1423				goto out_finish;
1424			have += asked;
1425			asked = 0;
1426			if (eof)
1427				eof = 2;
1428		}
1429
1430		for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
1431			wait_event(data[thr].done,
1432				atomic_read_acquire(&data[thr].stop));
1433			atomic_set(&data[thr].stop, 0);
1434
1435			ret = data[thr].ret;
1436
1437			if (ret < 0) {
1438				pr_err("%s decompression failed\n", hib_comp_algo);
1439				goto out_finish;
1440			}
1441
1442			if (unlikely(!data[thr].unc_len ||
1443				data[thr].unc_len > UNC_SIZE ||
1444				data[thr].unc_len & (PAGE_SIZE - 1))) {
1445				pr_err("Invalid %s uncompressed length\n", hib_comp_algo);
1446				ret = -1;
1447				goto out_finish;
1448			}
1449
1450			for (off = 0;
1451			     off < data[thr].unc_len; off += PAGE_SIZE) {
1452				memcpy(data_of(*snapshot),
1453				       data[thr].unc + off, PAGE_SIZE);
1454
1455				if (!(nr_pages % m))
1456					pr_info("Image loading progress: %3d%%\n",
1457						nr_pages / m * 10);
1458				nr_pages++;
1459
1460				ret = snapshot_write_next(snapshot);
1461				if (ret <= 0) {
1462					crc->run_threads = thr + 1;
1463					atomic_set_release(&crc->ready, 1);
1464					wake_up(&crc->go);
1465					goto out_finish;
1466				}
1467			}
1468		}
1469
1470		crc->run_threads = thr;
1471		atomic_set_release(&crc->ready, 1);
1472		wake_up(&crc->go);
1473	}
1474
1475out_finish:
1476	if (crc->run_threads) {
1477		wait_event(crc->done, atomic_read_acquire(&crc->stop));
1478		atomic_set(&crc->stop, 0);
1479	}
1480	stop = ktime_get();
1481	if (!ret) {
1482		pr_info("Image loading done\n");
1483		ret = snapshot_write_finalize(snapshot);
1484		if (!ret && !snapshot_image_loaded(snapshot))
1485			ret = -ENODATA;
1486		if (!ret) {
1487			if (swsusp_header->flags & SF_CRC32_MODE) {
1488				if(handle->crc32 != swsusp_header->crc32) {
1489					pr_err("Invalid image CRC32!\n");
1490					ret = -ENODATA;
1491				}
1492			}
1493		}
1494	}
1495	swsusp_show_speed(start, stop, nr_to_read, "Read");
1496out_clean:
1497	hib_finish_batch(&hb);
1498	for (i = 0; i < ring_size; i++)
1499		free_page((unsigned long)page[i]);
1500	free_crc_data(crc);
1501	if (data) {
1502		for (thr = 0; thr < nr_threads; thr++) {
1503			if (data[thr].thr)
1504				kthread_stop(data[thr].thr);
1505			if (data[thr].cr)
1506				acomp_request_free(data[thr].cr);
1507
1508			if (!IS_ERR_OR_NULL(data[thr].cc))
1509				crypto_free_acomp(data[thr].cc);
1510		}
1511		vfree(data);
1512	}
1513	vfree(page);
1514
1515	return ret;
1516}
1517
1518/**
1519 *	swsusp_read - read the hibernation image.
1520 *	@flags_p: flags passed by the "frozen" kernel in the image header should
1521 *		  be written into this memory location
1522 *
1523 *	Return: 0 on success, negative error code on failure.
1524 */
1525int swsusp_read(unsigned int *flags_p)
1526{
1527	int error;
1528	struct swap_map_handle handle;
1529	struct snapshot_handle snapshot;
1530	struct swsusp_info *header;
1531
1532	memset(&snapshot, 0, sizeof(struct snapshot_handle));
1533	error = snapshot_write_next(&snapshot);
1534	if (error < (int)PAGE_SIZE)
1535		return error < 0 ? error : -EFAULT;
1536	header = (struct swsusp_info *)data_of(snapshot);
1537	error = get_swap_reader(&handle, flags_p);
1538	if (error)
1539		goto end;
1540	if (!error)
1541		error = swap_read_page(&handle, header, NULL);
1542	if (!error) {
1543		error = (*flags_p & SF_NOCOMPRESS_MODE) ?
1544			load_image(&handle, &snapshot, header->pages - 1) :
1545			load_compressed_image(&handle, &snapshot, header->pages - 1);
1546	}
1547	swap_reader_finish(&handle);
1548end:
1549	if (!error)
1550		pr_debug("Image successfully loaded\n");
1551	else
1552		pr_debug("Error %d resuming\n", error);
1553	return error;
1554}
1555
1556static void *swsusp_holder;
1557
1558/**
1559 * swsusp_check - Open the resume device and check for the swsusp signature.
1560 * @exclusive: Open the resume device exclusively.
1561 *
1562 * Return: 0 if a valid image is found, negative error code otherwise.
1563 */
1564int swsusp_check(bool exclusive)
1565{
1566	void *holder = exclusive ? &swsusp_holder : NULL;
1567	int error;
1568
1569	hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device,
1570				BLK_OPEN_READ, holder, NULL);
1571	if (!IS_ERR(hib_resume_bdev_file)) {
1572		clear_page(swsusp_header);
1573		error = hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block,
1574					swsusp_header);
1575		if (error)
1576			goto put;
1577
1578		if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
1579			memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
1580			swsusp_header_flags = swsusp_header->flags;
1581			/* Reset swap signature now */
1582			error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC,
1583						swsusp_resume_block,
1584						swsusp_header);
1585		} else {
1586			error = -EINVAL;
1587		}
1588		if (!error && swsusp_header->flags & SF_HW_SIG &&
1589		    swsusp_header->hw_sig != swsusp_hardware_signature) {
1590			pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n",
1591				swsusp_header->hw_sig, swsusp_hardware_signature);
1592			error = -EINVAL;
1593		}
1594
1595put:
1596		if (error)
1597			bdev_fput(hib_resume_bdev_file);
1598		else
1599			pr_debug("Image signature found, resuming\n");
1600	} else {
1601		error = PTR_ERR(hib_resume_bdev_file);
1602	}
1603
1604	if (error)
1605		pr_debug("Image not found (code %d)\n", error);
1606
1607	return error;
1608}
1609
1610/**
1611 * swsusp_close - close resume device.
1612 */
1613void swsusp_close(void)
1614{
1615	if (IS_ERR(hib_resume_bdev_file)) {
1616		pr_debug("Image device not initialised\n");
1617		return;
1618	}
1619
1620	fput(hib_resume_bdev_file);
1621}
1622
1623/**
1624 * swsusp_unmark - Unmark swsusp signature in the resume device
1625 *
1626 * Return: 0 on success, negative error code on failure.
1627 */
1628#ifdef CONFIG_SUSPEND
1629int swsusp_unmark(void)
1630{
1631	int error;
1632
1633	hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header);
1634	if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) {
1635		memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
1636		error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC,
1637					swsusp_resume_block,
1638					swsusp_header);
1639	} else {
1640		pr_err("Cannot find swsusp signature!\n");
1641		error = -ENODEV;
1642	}
1643
1644	/*
1645	 * We just returned from suspend, we don't need the image any more.
1646	 */
1647	free_all_swap_pages(root_swap);
1648
1649	return error;
1650}
1651#endif
1652
1653static ssize_t hibernate_compression_threads_show(struct kobject *kobj,
1654				struct kobj_attribute *attr, char *buf)
1655{
1656	return sysfs_emit(buf, "%d\n", hibernate_compression_threads);
1657}
1658
1659static ssize_t hibernate_compression_threads_store(struct kobject *kobj,
1660				struct kobj_attribute *attr,
1661				const char *buf, size_t n)
1662{
1663	unsigned long val;
1664
1665	if (kstrtoul(buf, 0, &val))
1666		return -EINVAL;
1667
1668	if (val < 1)
1669		return -EINVAL;
1670
1671	hibernate_compression_threads = val;
1672	return n;
1673}
1674power_attr(hibernate_compression_threads);
1675
1676static struct attribute *g[] = {
1677	&hibernate_compression_threads_attr.attr,
1678	NULL,
1679};
1680
1681static const struct attribute_group attr_group = {
1682	.attrs = g,
1683};
1684
1685static int __init swsusp_header_init(void)
1686{
1687	int error;
1688
1689	error = sysfs_create_group(power_kobj, &attr_group);
1690	if (error)
1691		return -ENOMEM;
1692
1693	swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
1694	if (!swsusp_header)
1695		panic("Could not allocate memory for swsusp_header\n");
1696	return 0;
1697}
1698
1699core_initcall(swsusp_header_init);
1700
1701static int __init hibernate_compression_threads_setup(char *str)
1702{
1703	int rc = kstrtouint(str, 0, &hibernate_compression_threads);
1704
1705	if (rc)
1706		return rc;
1707
1708	if (hibernate_compression_threads < 1)
1709		hibernate_compression_threads = CMP_THREADS;
1710
1711	return 1;
1712
1713}
1714
1715__setup("hibernate_compression_threads=", hibernate_compression_threads_setup);