mremap_test.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright 2020 Google LLC
   4 */
   5#define _GNU_SOURCE
   6
   7#include <errno.h>
   8#include <fcntl.h>
   9#include <linux/userfaultfd.h>
  10#include <stdlib.h>
  11#include <stdio.h>
  12#include <string.h>
  13#include <sys/ioctl.h>
  14#include <sys/mman.h>
  15#include <syscall.h>
  16#include <time.h>
  17#include <stdbool.h>
  18
  19#include "kselftest.h"
  20
  21#define EXPECT_SUCCESS 0
  22#define EXPECT_FAILURE 1
  23#define NON_OVERLAPPING 0
  24#define OVERLAPPING 1
  25#define NS_PER_SEC 1000000000ULL
  26#define VALIDATION_DEFAULT_THRESHOLD 4	/* 4MB */
  27#define VALIDATION_NO_THRESHOLD 0	/* Verify the entire region */
  28
  29#ifndef MIN
  30#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
  31#define MAX(X, Y) ((X) > (Y) ? (X) : (Y))
  32#endif
  33#define SIZE_MB(m) ((size_t)m * (1024 * 1024))
  34#define SIZE_KB(k) ((size_t)k * 1024)
  35
  36struct config {
  37	unsigned long long src_alignment;
  38	unsigned long long dest_alignment;
  39	unsigned long long region_size;
  40	int overlapping;
  41	unsigned int dest_preamble_size;
  42};
  43
  44struct test {
  45	const char *name;
  46	struct config config;
  47	int expect_failure;
  48};
  49
  50enum {
  51	_1KB = 1ULL << 10,	/* 1KB -> not page aligned */
  52	_4KB = 4ULL << 10,
  53	_8KB = 8ULL << 10,
  54	_1MB = 1ULL << 20,
  55	_2MB = 2ULL << 20,
  56	_4MB = 4ULL << 20,
  57	_5MB = 5ULL << 20,
  58	_1GB = 1ULL << 30,
  59	_2GB = 2ULL << 30,
  60	PMD = _2MB,
  61	PUD = _1GB,
  62};
  63
  64#define PTE page_size
  65
  66#define MAKE_TEST(source_align, destination_align, size,	\
  67		  overlaps, should_fail, test_name)		\
  68(struct test){							\
  69	.name = test_name,					\
  70	.config = {						\
  71		.src_alignment = source_align,			\
  72		.dest_alignment = destination_align,		\
  73		.region_size = size,				\
  74		.overlapping = overlaps,			\
  75	},							\
  76	.expect_failure = should_fail				\
  77}
  78
  79/* compute square root using binary search */
  80static unsigned long get_sqrt(unsigned long val)
  81{
  82	unsigned long low = 1;
  83
  84	/* assuming rand_size is less than 1TB */
  85	unsigned long high = (1UL << 20);
  86
  87	while (low <= high) {
  88		unsigned long mid = low + (high - low) / 2;
  89		unsigned long temp = mid * mid;
  90
  91		if (temp == val)
  92			return mid;
  93		if (temp < val)
  94			low = mid + 1;
  95		high = mid - 1;
  96	}
  97	return low;
  98}
  99
 100/*
 101 * Returns false if the requested remap region overlaps with an
 102 * existing mapping (e.g text, stack) else returns true.
 103 */
 104static bool is_remap_region_valid(void *addr, unsigned long long size)
 105{
 106	void *remap_addr = NULL;
 107	bool ret = true;
 108
 109	/* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
 110	remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
 111					 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
 112					 -1, 0);
 113
 114	if (remap_addr == MAP_FAILED) {
 115		if (errno == EEXIST)
 116			ret = false;
 117	} else {
 118		munmap(remap_addr, size);
 119	}
 120
 121	return ret;
 122}
 123
 124/* Returns mmap_min_addr sysctl tunable from procfs */
 125static unsigned long long get_mmap_min_addr(void)
 126{
 127	FILE *fp;
 128	int n_matched;
 129	static unsigned long long addr;
 130
 131	if (addr)
 132		return addr;
 133
 134	fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
 135	if (fp == NULL) {
 136		ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
 137			strerror(errno));
 138		exit(KSFT_SKIP);
 139	}
 140
 141	n_matched = fscanf(fp, "%llu", &addr);
 142	if (n_matched != 1) {
 143		ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
 144			strerror(errno));
 145		fclose(fp);
 146		exit(KSFT_SKIP);
 147	}
 148
 149	fclose(fp);
 150	return addr;
 151}
 152
 153/*
 154 * Using /proc/self/maps, assert that the specified address range is contained
 155 * within a single mapping.
 156 */
 157static bool is_range_mapped(FILE *maps_fp, unsigned long start,
 158			    unsigned long end)
 159{
 160	char *line = NULL;
 161	size_t len = 0;
 162	bool success = false;
 163	unsigned long first_val, second_val;
 164
 165	rewind(maps_fp);
 166
 167	while (getline(&line, &len, maps_fp) != -1) {
 168		if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) {
 169			ksft_exit_fail_msg("cannot parse /proc/self/maps\n");
 170			break;
 171		}
 172
 173		if (first_val <= start && second_val >= end) {
 174			success = true;
 175			fflush(maps_fp);
 176			break;
 177		}
 178	}
 179
 180	return success;
 181}
 182
 183/* Check if [ptr, ptr + size) mapped in /proc/self/maps. */
 184static bool is_ptr_mapped(FILE *maps_fp, void *ptr, unsigned long size)
 185{
 186	unsigned long start = (unsigned long)ptr;
 187	unsigned long end = start + size;
 188
 189	return is_range_mapped(maps_fp, start, end);
 190}
 191
 192/*
 193 * Returns the start address of the mapping on success, else returns
 194 * NULL on failure.
 195 */
 196static void *get_source_mapping(struct config c)
 197{
 198	unsigned long long addr = 0ULL;
 199	void *src_addr = NULL;
 200	unsigned long long mmap_min_addr;
 201
 202	mmap_min_addr = get_mmap_min_addr();
 203	/*
 204	 * For some tests, we need to not have any mappings below the
 205	 * source mapping. Add some headroom to mmap_min_addr for this.
 206	 */
 207	mmap_min_addr += 10 * _4MB;
 208
 209retry:
 210	addr += c.src_alignment;
 211	if (addr < mmap_min_addr)
 212		goto retry;
 213
 214	src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
 215					MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
 216					-1, 0);
 217	if (src_addr == MAP_FAILED) {
 218		if (errno == EPERM || errno == EEXIST)
 219			goto retry;
 220		goto error;
 221	}
 222	/*
 223	 * Check that the address is aligned to the specified alignment.
 224	 * Addresses which have alignments that are multiples of that
 225	 * specified are not considered valid. For instance, 1GB address is
 226	 * 2MB-aligned, however it will not be considered valid for a
 227	 * requested alignment of 2MB. This is done to reduce coincidental
 228	 * alignment in the tests.
 229	 */
 230	if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
 231			!((unsigned long long) src_addr & c.src_alignment)) {
 232		munmap(src_addr, c.region_size);
 233		goto retry;
 234	}
 235
 236	if (!src_addr)
 237		goto error;
 238
 239	return src_addr;
 240error:
 241	ksft_print_msg("Failed to map source region: %s\n",
 242			strerror(errno));
 243	return NULL;
 244}
 245
 246/*
 247 * This test validates that merge is called when expanding a mapping.
 248 * Mapping containing three pages is created, middle page is unmapped
 249 * and then the mapping containing the first page is expanded so that
 250 * it fills the created hole. The two parts should merge creating
 251 * single mapping with three pages.
 252 */
 253static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size)
 254{
 255	char *test_name = "mremap expand merge";
 256	bool success = false;
 257	char *remap, *start;
 258
 259	start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
 260		     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 261
 262	if (start == MAP_FAILED) {
 263		ksft_print_msg("mmap failed: %s\n", strerror(errno));
 264		goto out;
 265	}
 266
 267	munmap(start + page_size, page_size);
 268	remap = mremap(start, page_size, 2 * page_size, 0);
 269	if (remap == MAP_FAILED) {
 270		ksft_print_msg("mremap failed: %s\n", strerror(errno));
 271		munmap(start, page_size);
 272		munmap(start + 2 * page_size, page_size);
 273		goto out;
 274	}
 275
 276	success = is_range_mapped(maps_fp, (unsigned long)start,
 277				  (unsigned long)(start + 3 * page_size));
 278	munmap(start, 3 * page_size);
 279
 280out:
 281	if (success)
 282		ksft_test_result_pass("%s\n", test_name);
 283	else
 284		ksft_test_result_fail("%s\n", test_name);
 285}
 286
 287/*
 288 * Similar to mremap_expand_merge() except instead of removing the middle page,
 289 * we remove the last then attempt to remap offset from the second page. This
 290 * should result in the mapping being restored to its former state.
 291 */
 292static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size)
 293{
 294
 295	char *test_name = "mremap expand merge offset";
 296	bool success = false;
 297	char *remap, *start;
 298
 299	start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
 300		     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 301
 302	if (start == MAP_FAILED) {
 303		ksft_print_msg("mmap failed: %s\n", strerror(errno));
 304		goto out;
 305	}
 306
 307	/* Unmap final page to ensure we have space to expand. */
 308	munmap(start + 2 * page_size, page_size);
 309	remap = mremap(start + page_size, page_size, 2 * page_size, 0);
 310	if (remap == MAP_FAILED) {
 311		ksft_print_msg("mremap failed: %s\n", strerror(errno));
 312		munmap(start, 2 * page_size);
 313		goto out;
 314	}
 315
 316	success = is_range_mapped(maps_fp, (unsigned long)start,
 317				  (unsigned long)(start + 3 * page_size));
 318	munmap(start, 3 * page_size);
 319
 320out:
 321	if (success)
 322		ksft_test_result_pass("%s\n", test_name);
 323	else
 324		ksft_test_result_fail("%s\n", test_name);
 325}
 326
 327/*
 328 * Verify that an mremap within a range does not cause corruption
 329 * of unrelated part of range.
 330 *
 331 * Consider the following range which is 2MB aligned and is
 332 * a part of a larger 20MB range which is not shown. Each
 333 * character is 256KB below making the source and destination
 334 * 2MB each. The lower case letters are moved (s to d) and the
 335 * upper case letters are not moved. The below test verifies
 336 * that the upper case S letters are not corrupted by the
 337 * adjacent mremap.
 338 *
 339 * |DDDDddddSSSSssss|
 340 */
 341static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr)
 342{
 343	char *test_name = "mremap mremap move within range";
 344	void *src, *dest;
 345	unsigned int i, success = 1;
 346
 347	size_t size = SIZE_MB(20);
 348	void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 349			 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 350	if (ptr == MAP_FAILED) {
 351		perror("mmap");
 352		success = 0;
 353		goto out;
 354	}
 355	memset(ptr, 0, size);
 356
 357	src = ptr + SIZE_MB(6);
 358	src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1));
 359
 360	/* Set byte pattern for source block. */
 361	memcpy(src, rand_addr, SIZE_MB(2));
 362
 363	dest = src - SIZE_MB(2);
 364
 365	void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1),
 366						   MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1));
 367	if (new_ptr == MAP_FAILED) {
 368		perror("mremap");
 369		success = 0;
 370		goto out;
 371	}
 372
 373	/* Verify byte pattern after remapping */
 374	srand(pattern_seed);
 375	for (i = 0; i < SIZE_MB(1); i++) {
 376		char c = (char) rand();
 377
 378		if (((char *)src)[i] != c) {
 379			ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n",
 380				       i);
 381			ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
 382					((char *) src)[i] & 0xff);
 383			success = 0;
 384		}
 385	}
 386
 387out:
 388	if (munmap(ptr, size) == -1)
 389		perror("munmap");
 390
 391	if (success)
 392		ksft_test_result_pass("%s\n", test_name);
 393	else
 394		ksft_test_result_fail("%s\n", test_name);
 395}
 396
 397static bool is_multiple_vma_range_ok(unsigned int pattern_seed,
 398				     char *ptr, unsigned long page_size)
 399{
 400	int i;
 401
 402	srand(pattern_seed);
 403	for (i = 0; i <= 10; i += 2) {
 404		int j;
 405		char *buf = &ptr[i * page_size];
 406		size_t size = i == 4 ? 2 * page_size : page_size;
 407
 408		for (j = 0; j < size; j++) {
 409			char chr = rand();
 410
 411			if (chr != buf[j]) {
 412				ksft_print_msg("page %d offset %d corrupted, expected %d got %d\n",
 413					       i, j, chr, buf[j]);
 414				return false;
 415			}
 416		}
 417	}
 418
 419	return true;
 420}
 421
 422static void mremap_move_multiple_vmas(unsigned int pattern_seed,
 423				      unsigned long page_size,
 424				      bool dont_unmap)
 425{
 426	int mremap_flags = MREMAP_FIXED | MREMAP_MAYMOVE;
 427	char *test_name = "mremap move multiple vmas";
 428	const size_t size = 11 * page_size;
 429	bool success = true;
 430	char *ptr, *tgt_ptr;
 431	int i;
 432
 433	if (dont_unmap)
 434		mremap_flags |= MREMAP_DONTUNMAP;
 435
 436	ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 437		   MAP_PRIVATE | MAP_ANON, -1, 0);
 438	if (ptr == MAP_FAILED) {
 439		perror("mmap");
 440		success = false;
 441		goto out;
 442	}
 443
 444	tgt_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE,
 445		       MAP_PRIVATE | MAP_ANON, -1, 0);
 446	if (tgt_ptr == MAP_FAILED) {
 447		perror("mmap");
 448		success = false;
 449		goto out;
 450	}
 451	if (munmap(tgt_ptr, 2 * size)) {
 452		perror("munmap");
 453		success = false;
 454		goto out_unmap;
 455	}
 456
 457	/*
 458	 * Unmap so we end up with:
 459	 *
 460	 *  0   2   4 5 6   8   10 offset in buffer
 461	 * |*| |*| |*****| |*| |*|
 462	 * |*| |*| |*****| |*| |*|
 463	 *  0   1   2 3 4   5   6  pattern offset
 464	 */
 465	for (i = 1; i < 10; i += 2) {
 466		if (i == 5)
 467			continue;
 468
 469		if (munmap(&ptr[i * page_size], page_size)) {
 470			perror("munmap");
 471			success = false;
 472			goto out_unmap;
 473		}
 474	}
 475
 476	srand(pattern_seed);
 477
 478	/* Set up random patterns. */
 479	for (i = 0; i <= 10; i += 2) {
 480		int j;
 481		size_t size = i == 4 ? 2 * page_size : page_size;
 482		char *buf = &ptr[i * page_size];
 483
 484		for (j = 0; j < size; j++)
 485			buf[j] = rand();
 486	}
 487
 488	/* First, just move the whole thing. */
 489	if (mremap(ptr, size, size, mremap_flags, tgt_ptr) == MAP_FAILED) {
 490		perror("mremap");
 491		success = false;
 492		goto out_unmap;
 493	}
 494	/* Check move was ok. */
 495	if (!is_multiple_vma_range_ok(pattern_seed, tgt_ptr, page_size)) {
 496		success = false;
 497		goto out_unmap;
 498	}
 499
 500	/* Move next to itself. */
 501	if (mremap(tgt_ptr, size, size, mremap_flags,
 502		   &tgt_ptr[size]) == MAP_FAILED) {
 503		perror("mremap");
 504		success = false;
 505		goto out_unmap;
 506	}
 507	/* Check that the move is ok. */
 508	if (!is_multiple_vma_range_ok(pattern_seed, &tgt_ptr[size], page_size)) {
 509		success = false;
 510		goto out_unmap;
 511	}
 512
 513	/* Map a range to overwrite. */
 514	if (mmap(tgt_ptr, size, PROT_NONE,
 515		 MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0) == MAP_FAILED) {
 516		perror("mmap tgt");
 517		success = false;
 518		goto out_unmap;
 519	}
 520	/* Move and overwrite. */
 521	if (mremap(&tgt_ptr[size], size, size,
 522		   mremap_flags, tgt_ptr) == MAP_FAILED) {
 523		perror("mremap");
 524		success = false;
 525		goto out_unmap;
 526	}
 527	/* Check that the move is ok. */
 528	if (!is_multiple_vma_range_ok(pattern_seed, tgt_ptr, page_size)) {
 529		success = false;
 530		goto out_unmap;
 531	}
 532
 533out_unmap:
 534	if (munmap(tgt_ptr, 2 * size))
 535		perror("munmap tgt");
 536	if (munmap(ptr, size))
 537		perror("munmap src");
 538
 539out:
 540	if (success)
 541		ksft_test_result_pass("%s%s\n", test_name,
 542				      dont_unmap ? " [dontunnmap]" : "");
 543	else
 544		ksft_test_result_fail("%s%s\n", test_name,
 545				      dont_unmap ? " [dontunnmap]" : "");
 546}
 547
 548static void mremap_shrink_multiple_vmas(unsigned long page_size,
 549					bool inplace)
 550{
 551	char *test_name = "mremap shrink multiple vmas";
 552	const size_t size = 10 * page_size;
 553	bool success = true;
 554	char *ptr, *tgt_ptr;
 555	void *res;
 556	int i;
 557
 558	ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 559		   MAP_PRIVATE | MAP_ANON, -1, 0);
 560	if (ptr == MAP_FAILED) {
 561		perror("mmap");
 562		success = false;
 563		goto out;
 564	}
 565
 566	tgt_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 567		       MAP_PRIVATE | MAP_ANON, -1, 0);
 568	if (tgt_ptr == MAP_FAILED) {
 569		perror("mmap");
 570		success = false;
 571		goto out;
 572	}
 573	if (munmap(tgt_ptr, size)) {
 574		perror("munmap");
 575		success = false;
 576		goto out_unmap;
 577	}
 578
 579	/*
 580	 * Unmap so we end up with:
 581	 *
 582	 *  0   2   4   6   8   10 offset in buffer
 583	 * |*| |*| |*| |*| |*| |*|
 584	 * |*| |*| |*| |*| |*| |*|
 585	 */
 586	for (i = 1; i < 10; i += 2) {
 587		if (munmap(&ptr[i * page_size], page_size)) {
 588			perror("munmap");
 589			success = false;
 590			goto out_unmap;
 591		}
 592	}
 593
 594	/*
 595	 * Shrink in-place across multiple VMAs and gaps so we end up with:
 596	 *
 597	 *  0
 598	 * |*|
 599	 * |*|
 600	 */
 601	if (inplace)
 602		res = mremap(ptr, size, page_size, 0);
 603	else
 604		res = mremap(ptr, size, page_size, MREMAP_MAYMOVE | MREMAP_FIXED,
 605			     tgt_ptr);
 606
 607	if (res == MAP_FAILED) {
 608		perror("mremap");
 609		success = false;
 610		goto out_unmap;
 611	}
 612
 613out_unmap:
 614	if (munmap(tgt_ptr, size))
 615		perror("munmap tgt");
 616	if (munmap(ptr, size))
 617		perror("munmap src");
 618out:
 619	if (success)
 620		ksft_test_result_pass("%s%s\n", test_name,
 621				      inplace ? " [inplace]" : "");
 622	else
 623		ksft_test_result_fail("%s%s\n", test_name,
 624				      inplace ? " [inplace]" : "");
 625}
 626
 627static void mremap_move_multiple_vmas_split(unsigned int pattern_seed,
 628					    unsigned long page_size,
 629					    bool dont_unmap)
 630{
 631	char *test_name = "mremap move multiple vmas split";
 632	int mremap_flags = MREMAP_FIXED | MREMAP_MAYMOVE;
 633	const size_t size = 10 * page_size;
 634	bool success = true;
 635	char *ptr, *tgt_ptr;
 636	int i;
 637
 638	if (dont_unmap)
 639		mremap_flags |= MREMAP_DONTUNMAP;
 640
 641	ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 642		   MAP_PRIVATE | MAP_ANON, -1, 0);
 643	if (ptr == MAP_FAILED) {
 644		perror("mmap");
 645		success = false;
 646		goto out;
 647	}
 648
 649	tgt_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 650		       MAP_PRIVATE | MAP_ANON, -1, 0);
 651	if (tgt_ptr == MAP_FAILED) {
 652		perror("mmap");
 653		success = false;
 654		goto out;
 655	}
 656	if (munmap(tgt_ptr, size)) {
 657		perror("munmap");
 658		success = false;
 659		goto out_unmap;
 660	}
 661
 662	/*
 663	 * Unmap so we end up with:
 664	 *
 665	 *  0 1 2 3 4 5 6 7 8 9 10 offset in buffer
 666	 * |**********| |*******|
 667	 * |**********| |*******|
 668	 *  0 1 2 3 4   5 6 7 8 9  pattern offset
 669	 */
 670	if (munmap(&ptr[5 * page_size], page_size)) {
 671		perror("munmap");
 672		success = false;
 673		goto out_unmap;
 674	}
 675
 676	/* Set up random patterns. */
 677	srand(pattern_seed);
 678	for (i = 0; i < 10; i++) {
 679		int j;
 680		char *buf = &ptr[i * page_size];
 681
 682		if (i == 5)
 683			continue;
 684
 685		for (j = 0; j < page_size; j++)
 686			buf[j] = rand();
 687	}
 688
 689	/*
 690	 * Move the below:
 691	 *
 692	 *      <------------->
 693	 *  0 1 2 3 4 5 6 7 8 9 10 offset in buffer
 694	 * |**********| |*******|
 695	 * |**********| |*******|
 696	 *  0 1 2 3 4   5 6 7 8 9  pattern offset
 697	 *
 698	 * Into:
 699	 *
 700	 * 0 1 2 3 4 5 6 7 offset in buffer
 701	 * |*****| |*****|
 702	 * |*****| |*****|
 703	 * 2 3 4   5 6 7   pattern offset
 704	 */
 705	if (mremap(&ptr[2 * page_size], size - 3 * page_size, size - 3 * page_size,
 706		   mremap_flags, tgt_ptr) == MAP_FAILED) {
 707		perror("mremap");
 708		success = false;
 709		goto out_unmap;
 710	}
 711
 712	/* Offset into random pattern. */
 713	srand(pattern_seed);
 714	for (i = 0; i < 2 * page_size; i++)
 715		rand();
 716
 717	/* Check pattern. */
 718	for (i = 0; i < 7; i++) {
 719		int j;
 720		char *buf = &tgt_ptr[i * page_size];
 721
 722		if (i == 3)
 723			continue;
 724
 725		for (j = 0; j < page_size; j++) {
 726			char chr = rand();
 727
 728			if (chr != buf[j]) {
 729				ksft_print_msg("page %d offset %d corrupted, expected %d got %d\n",
 730					       i, j, chr, buf[j]);
 731				goto out_unmap;
 732			}
 733		}
 734	}
 735
 736out_unmap:
 737	if (munmap(tgt_ptr, size))
 738		perror("munmap tgt");
 739	if (munmap(ptr, size))
 740		perror("munmap src");
 741out:
 742	if (success)
 743		ksft_test_result_pass("%s%s\n", test_name,
 744				      dont_unmap ? " [dontunnmap]" : "");
 745	else
 746		ksft_test_result_fail("%s%s\n", test_name,
 747				      dont_unmap ? " [dontunnmap]" : "");
 748}
 749
 750#ifdef __NR_userfaultfd
 751static void mremap_move_multi_invalid_vmas(FILE *maps_fp,
 752		unsigned long page_size)
 753{
 754	char *test_name = "mremap move multiple invalid vmas";
 755	const size_t size = 10 * page_size;
 756	bool success = true;
 757	char *ptr, *tgt_ptr;
 758	int uffd, err, i;
 759	void *res;
 760	struct uffdio_api api = {
 761		.api = UFFD_API,
 762		.features = UFFD_EVENT_PAGEFAULT,
 763	};
 764
 765	uffd = syscall(__NR_userfaultfd, O_NONBLOCK);
 766	if (uffd == -1) {
 767		err = errno;
 768		perror("userfaultfd");
 769		if (err == EPERM) {
 770			ksft_test_result_skip("%s - missing uffd", test_name);
 771			return;
 772		}
 773		success = false;
 774		goto out;
 775	}
 776	if (ioctl(uffd, UFFDIO_API, &api)) {
 777		perror("ioctl UFFDIO_API");
 778		success = false;
 779		goto out_close_uffd;
 780	}
 781
 782	ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
 783		   MAP_PRIVATE | MAP_ANON, -1, 0);
 784	if (ptr == MAP_FAILED) {
 785		perror("mmap");
 786		success = false;
 787		goto out_close_uffd;
 788	}
 789
 790	tgt_ptr = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0);
 791	if (tgt_ptr == MAP_FAILED) {
 792		perror("mmap");
 793		success = false;
 794		goto out_close_uffd;
 795	}
 796	if (munmap(tgt_ptr, size)) {
 797		perror("munmap");
 798		success = false;
 799		goto out_unmap;
 800	}
 801
 802	/*
 803	 * Unmap so we end up with:
 804	 *
 805	 *  0   2   4   6   8   10 offset in buffer
 806	 * |*| |*| |*| |*| |*|
 807	 * |*| |*| |*| |*| |*|
 808	 *
 809	 * Additionally, register each with UFFD.
 810	 */
 811	for (i = 0; i < 10; i += 2) {
 812		void *unmap_ptr = &ptr[(i + 1) * page_size];
 813		unsigned long start = (unsigned long)&ptr[i * page_size];
 814		struct uffdio_register reg = {
 815			.range = {
 816				.start = start,
 817				.len = page_size,
 818			},
 819			.mode = UFFDIO_REGISTER_MODE_MISSING,
 820		};
 821
 822		if (ioctl(uffd, UFFDIO_REGISTER, &reg) == -1) {
 823			perror("ioctl UFFDIO_REGISTER");
 824			success = false;
 825			goto out_unmap;
 826		}
 827		if (munmap(unmap_ptr, page_size)) {
 828			perror("munmap");
 829			success = false;
 830			goto out_unmap;
 831		}
 832	}
 833
 834	/*
 835	 * Now try to move the entire range which is invalid for multi VMA move.
 836	 *
 837	 * This will fail, and no VMA should be moved, as we check this ahead of
 838	 * time.
 839	 */
 840	res = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr);
 841	err = errno;
 842	if (res != MAP_FAILED) {
 843		fprintf(stderr, "mremap() succeeded for multi VMA uffd armed\n");
 844		success = false;
 845		goto out_unmap;
 846	}
 847	if (err != EFAULT) {
 848		errno = err;
 849		perror("mremap() unexpected error");
 850		success = false;
 851		goto out_unmap;
 852	}
 853	if (is_ptr_mapped(maps_fp, tgt_ptr, page_size)) {
 854		fprintf(stderr,
 855			"Invalid uffd-armed VMA at start of multi range moved\n");
 856		success = false;
 857		goto out_unmap;
 858	}
 859
 860	/*
 861	 * Now try to move a single VMA, this should succeed as not multi VMA
 862	 * move.
 863	 */
 864	res = mremap(ptr, page_size, page_size,
 865		     MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr);
 866	if (res == MAP_FAILED) {
 867		perror("mremap single invalid-multi VMA");
 868		success = false;
 869		goto out_unmap;
 870	}
 871
 872	/*
 873	 * Unmap the VMA, and remap a non-uffd registered (therefore, multi VMA
 874	 * move valid) VMA at the start of ptr range.
 875	 */
 876	if (munmap(tgt_ptr, page_size)) {
 877		perror("munmap");
 878		success = false;
 879		goto out_unmap;
 880	}
 881	res = mmap(ptr, page_size, PROT_READ | PROT_WRITE,
 882		   MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
 883	if (res == MAP_FAILED) {
 884		perror("mmap");
 885		success = false;
 886		goto out_unmap;
 887	}
 888
 889	/*
 890	 * Now try to move the entire range, we should succeed in moving the
 891	 * first VMA, but no others, and report a failure.
 892	 */
 893	res = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr);
 894	err = errno;
 895	if (res != MAP_FAILED) {
 896		fprintf(stderr, "mremap() succeeded for multi VMA uffd armed\n");
 897		success = false;
 898		goto out_unmap;
 899	}
 900	if (err != EFAULT) {
 901		errno = err;
 902		perror("mremap() unexpected error");
 903		success = false;
 904		goto out_unmap;
 905	}
 906	if (!is_ptr_mapped(maps_fp, tgt_ptr, page_size)) {
 907		fprintf(stderr, "Valid VMA not moved\n");
 908		success = false;
 909		goto out_unmap;
 910	}
 911
 912	/*
 913	 * Unmap the VMA, and map valid VMA at start of ptr range, and replace
 914	 * all existing multi-move invalid VMAs, except the last, with valid
 915	 * multi-move VMAs.
 916	 */
 917	if (munmap(tgt_ptr, page_size)) {
 918		perror("munmap");
 919		success = false;
 920		goto out_unmap;
 921	}
 922	if (munmap(ptr, size - 2 * page_size)) {
 923		perror("munmap");
 924		success = false;
 925		goto out_unmap;
 926	}
 927	for (i = 0; i < 8; i += 2) {
 928		res = mmap(&ptr[i * page_size], page_size,
 929			   PROT_READ | PROT_WRITE,
 930			   MAP_PRIVATE | MAP_ANON | MAP_FIXED, -1, 0);
 931		if (res == MAP_FAILED) {
 932			perror("mmap");
 933			success = false;
 934			goto out_unmap;
 935		}
 936	}
 937
 938	/*
 939	 * Now try to move the entire range, we should succeed in moving all but
 940	 * the last VMA, and report a failure.
 941	 */
 942	res = mremap(ptr, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, tgt_ptr);
 943	err = errno;
 944	if (res != MAP_FAILED) {
 945		fprintf(stderr, "mremap() succeeded for multi VMA uffd armed\n");
 946		success = false;
 947		goto out_unmap;
 948	}
 949	if (err != EFAULT) {
 950		errno = err;
 951		perror("mremap() unexpected error");
 952		success = false;
 953		goto out_unmap;
 954	}
 955
 956	for (i = 0; i < 10; i += 2) {
 957		bool is_mapped = is_ptr_mapped(maps_fp,
 958				&tgt_ptr[i * page_size], page_size);
 959
 960		if (i < 8 && !is_mapped) {
 961			fprintf(stderr, "Valid VMA not moved at %d\n", i);
 962			success = false;
 963			goto out_unmap;
 964		} else if (i == 8 && is_mapped) {
 965			fprintf(stderr, "Invalid VMA moved at %d\n", i);
 966			success = false;
 967			goto out_unmap;
 968		}
 969	}
 970
 971out_unmap:
 972	if (munmap(tgt_ptr, size))
 973		perror("munmap tgt");
 974	if (munmap(ptr, size))
 975		perror("munmap src");
 976out_close_uffd:
 977	close(uffd);
 978out:
 979	if (success)
 980		ksft_test_result_pass("%s\n", test_name);
 981	else
 982		ksft_test_result_fail("%s\n", test_name);
 983}
 984#else
 985static void mremap_move_multi_invalid_vmas(FILE *maps_fp, unsigned long page_size)
 986{
 987	char *test_name = "mremap move multiple invalid vmas";
 988
 989	ksft_test_result_skip("%s - missing uffd", test_name);
 990}
 991#endif /* __NR_userfaultfd */
 992
 993/* Returns the time taken for the remap on success else returns -1. */
 994static long long remap_region(struct config c, unsigned int threshold_mb,
 995			      char *rand_addr)
 996{
 997	void *addr, *tmp_addr, *src_addr, *dest_addr, *dest_preamble_addr = NULL;
 998	unsigned long long t, d;
 999	struct timespec t_start = {0, 0}, t_end = {0, 0};
1000	long long  start_ns, end_ns, align_mask, ret, offset;
1001	unsigned long long threshold;
1002	unsigned long num_chunks;
1003
1004	if (threshold_mb == VALIDATION_NO_THRESHOLD)
1005		threshold = c.region_size;
1006	else
1007		threshold = MIN(threshold_mb * _1MB, c.region_size);
1008
1009	src_addr = get_source_mapping(c);
1010	if (!src_addr) {
1011		ret = -1;
1012		goto out;
1013	}
1014
1015	/* Set byte pattern for source block. */
1016	memcpy(src_addr, rand_addr, threshold);
1017
1018	/* Mask to zero out lower bits of address for alignment */
1019	align_mask = ~(c.dest_alignment - 1);
1020	/* Offset of destination address from the end of the source region */
1021	offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
1022	addr = (void *) (((unsigned long long) src_addr + c.region_size
1023			  + offset) & align_mask);
1024
1025	/* Remap after the destination block preamble. */
1026	addr += c.dest_preamble_size;
1027
1028	/* See comment in get_source_mapping() */
1029	if (!((unsigned long long) addr & c.dest_alignment))
1030		addr = (void *) ((unsigned long long) addr | c.dest_alignment);
1031
1032	/* Don't destroy existing mappings unless expected to overlap */
1033	while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
1034		/* Check for unsigned overflow */
1035		tmp_addr = addr + c.dest_alignment;
1036		if (tmp_addr < addr) {
1037			ksft_print_msg("Couldn't find a valid region to remap to\n");
1038			ret = -1;
1039			goto clean_up_src;
1040		}
1041		addr += c.dest_alignment;
1042	}
1043
1044	if (c.dest_preamble_size) {
1045		dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size,
1046					  PROT_READ | PROT_WRITE,
1047					  MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
1048							-1, 0);
1049		if (dest_preamble_addr == MAP_FAILED) {
1050			ksft_print_msg("Failed to map dest preamble region: %s\n",
1051					strerror(errno));
1052			ret = -1;
1053			goto clean_up_src;
1054		}
1055
1056		/* Set byte pattern for the dest preamble block. */
1057		memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size);
1058	}
1059
1060	clock_gettime(CLOCK_MONOTONIC, &t_start);
1061	dest_addr = mremap(src_addr, c.region_size, c.region_size,
1062					  MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
1063	clock_gettime(CLOCK_MONOTONIC, &t_end);
1064
1065	if (dest_addr == MAP_FAILED) {
1066		ksft_print_msg("mremap failed: %s\n", strerror(errno));
1067		ret = -1;
1068		goto clean_up_dest_preamble;
1069	}
1070
1071	/*
1072	 * Verify byte pattern after remapping. Employ an algorithm with a
1073	 * square root time complexity in threshold: divide the range into
1074	 * chunks, if memcmp() returns non-zero, only then perform an
1075	 * iteration in that chunk to find the mismatch index.
1076	 */
1077	num_chunks = get_sqrt(threshold);
1078	for (unsigned long i = 0; i < num_chunks; ++i) {
1079		size_t chunk_size = threshold / num_chunks;
1080		unsigned long shift = i * chunk_size;
1081
1082		if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size))
1083			continue;
1084
1085		/* brute force iteration only over mismatch segment */
1086		for (t = shift; t < shift + chunk_size; ++t) {
1087			if (((char *) dest_addr)[t] != rand_addr[t]) {
1088				ksft_print_msg("Data after remap doesn't match at offset %llu\n",
1089						t);
1090				ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff,
1091						((char *) dest_addr)[t] & 0xff);
1092				ret = -1;
1093				goto clean_up_dest;
1094			}
1095		}
1096	}
1097
1098	/*
1099	 * if threshold is not divisible by num_chunks, then check the
1100	 * last chunk
1101	 */
1102	for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) {
1103		if (((char *) dest_addr)[t] != rand_addr[t]) {
1104			ksft_print_msg("Data after remap doesn't match at offset %llu\n",
1105					t);
1106			ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff,
1107					((char *) dest_addr)[t] & 0xff);
1108			ret = -1;
1109			goto clean_up_dest;
1110		}
1111	}
1112
1113	/* Verify the dest preamble byte pattern after remapping */
1114	if (!c.dest_preamble_size)
1115		goto no_preamble;
1116
1117	num_chunks = get_sqrt(c.dest_preamble_size);
1118
1119	for (unsigned long i = 0; i < num_chunks; ++i) {
1120		size_t chunk_size = c.dest_preamble_size / num_chunks;
1121		unsigned long shift = i * chunk_size;
1122
1123		if (!memcmp(dest_preamble_addr + shift, rand_addr + shift,
1124			    chunk_size))
1125			continue;
1126
1127		/* brute force iteration only over mismatched segment */
1128		for (d = shift; d < shift + chunk_size; ++d) {
1129			if (((char *) dest_preamble_addr)[d] != rand_addr[d]) {
1130				ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n",
1131						d);
1132				ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff,
1133						((char *) dest_preamble_addr)[d] & 0xff);
1134				ret = -1;
1135				goto clean_up_dest;
1136			}
1137		}
1138	}
1139
1140	for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) {
1141		if (((char *) dest_preamble_addr)[d] != rand_addr[d]) {
1142			ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n",
1143					d);
1144			ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff,
1145					((char *) dest_preamble_addr)[d] & 0xff);
1146			ret = -1;
1147			goto clean_up_dest;
1148		}
1149	}
1150
1151no_preamble:
1152	start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
1153	end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
1154	ret = end_ns - start_ns;
1155
1156/*
1157 * Since the destination address is specified using MREMAP_FIXED, subsequent
1158 * mremap will unmap any previous mapping at the address range specified by
1159 * dest_addr and region_size. This significantly affects the remap time of
1160 * subsequent tests. So we clean up mappings after each test.
1161 */
1162clean_up_dest:
1163	munmap(dest_addr, c.region_size);
1164clean_up_dest_preamble:
1165	if (c.dest_preamble_size && dest_preamble_addr)
1166		munmap(dest_preamble_addr, c.dest_preamble_size);
1167clean_up_src:
1168	munmap(src_addr, c.region_size);
1169out:
1170	return ret;
1171}
1172
1173/*
1174 * Verify that an mremap aligning down does not destroy
1175 * the beginning of the mapping just because the aligned
1176 * down address landed on a mapping that maybe does not exist.
1177 */
1178static void mremap_move_1mb_from_start(unsigned int pattern_seed,
1179				       char *rand_addr)
1180{
1181	char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src";
1182	void *src = NULL, *dest = NULL;
1183	unsigned int i, success = 1;
1184
1185	/* Config to reuse get_source_mapping() to do an aligned mmap. */
1186	struct config c = {
1187		.src_alignment = SIZE_MB(1) + SIZE_KB(256),
1188		.region_size = SIZE_MB(6)
1189	};
1190
1191	src = get_source_mapping(c);
1192	if (!src) {
1193		success = 0;
1194		goto out;
1195	}
1196
1197	c.src_alignment = SIZE_MB(1) + SIZE_KB(256);
1198	dest = get_source_mapping(c);
1199	if (!dest) {
1200		success = 0;
1201		goto out;
1202	}
1203
1204	/* Set byte pattern for source block. */
1205	memcpy(src, rand_addr, SIZE_MB(2));
1206
1207	/*
1208	 * Unmap the beginning of dest so that the aligned address
1209	 * falls on no mapping.
1210	 */
1211	munmap(dest, SIZE_MB(1));
1212
1213	void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1),
1214						   MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1));
1215	if (new_ptr == MAP_FAILED) {
1216		perror("mremap");
1217		success = 0;
1218		goto out;
1219	}
1220
1221	/* Verify byte pattern after remapping */
1222	srand(pattern_seed);
1223	for (i = 0; i < SIZE_MB(1); i++) {
1224		char c = (char) rand();
1225
1226		if (((char *)src)[i] != c) {
1227			ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n",
1228				       i);
1229			ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
1230					((char *) src)[i] & 0xff);
1231			success = 0;
1232		}
1233	}
1234
1235out:
1236	if (src && munmap(src, c.region_size) == -1)
1237		perror("munmap src");
1238
1239	if (dest && munmap(dest, c.region_size) == -1)
1240		perror("munmap dest");
1241
1242	if (success)
1243		ksft_test_result_pass("%s\n", test_name);
1244	else
1245		ksft_test_result_fail("%s\n", test_name);
1246}
1247
1248static void run_mremap_test_case(struct test test_case, int *failures,
1249				 unsigned int threshold_mb,
1250				 char *rand_addr)
1251{
1252	long long remap_time = remap_region(test_case.config, threshold_mb,
1253					    rand_addr);
1254
1255	if (remap_time < 0) {
1256		if (test_case.expect_failure)
1257			ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
1258					      test_case.name);
1259		else {
1260			ksft_test_result_fail("%s\n", test_case.name);
1261			*failures += 1;
1262		}
1263	} else {
1264		/*
1265		 * Comparing mremap time is only applicable if entire region
1266		 * was faulted in.
1267		 */
1268		if (threshold_mb == VALIDATION_NO_THRESHOLD ||
1269		    test_case.config.region_size <= threshold_mb * _1MB)
1270			ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
1271					      test_case.name, remap_time);
1272		else
1273			ksft_test_result_pass("%s\n", test_case.name);
1274	}
1275}
1276
1277static void usage(const char *cmd)
1278{
1279	fprintf(stderr,
1280		"Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
1281		"-t\t only validate threshold_mb of the remapped region\n"
1282		"  \t if 0 is supplied no threshold is used; all tests\n"
1283		"  \t are run and remapped regions validated fully.\n"
1284		"  \t The default threshold used is 4MB.\n"
1285		"-p\t provide a seed to generate the random pattern for\n"
1286		"  \t validating the remapped region.\n", cmd);
1287}
1288
1289static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
1290		      unsigned int *pattern_seed)
1291{
1292	const char *optstr = "t:p:";
1293	int opt;
1294
1295	while ((opt = getopt(argc, argv, optstr)) != -1) {
1296		switch (opt) {
1297		case 't':
1298			*threshold_mb = atoi(optarg);
1299			break;
1300		case 'p':
1301			*pattern_seed = atoi(optarg);
1302			break;
1303		default:
1304			usage(argv[0]);
1305			return -1;
1306		}
1307	}
1308
1309	if (optind < argc) {
1310		usage(argv[0]);
1311		return -1;
1312	}
1313
1314	return 0;
1315}
1316
1317#define MAX_TEST 15
1318#define MAX_PERF_TEST 3
1319int main(int argc, char **argv)
1320{
1321	int failures = 0;
1322	unsigned int i;
1323	int run_perf_tests;
1324	unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
1325
1326	/* hard-coded test configs */
1327	size_t max_test_variable_region_size = _2GB;
1328	size_t max_test_constant_region_size = _2MB;
1329	size_t dest_preamble_size = 10 * _4MB;
1330
1331	unsigned int pattern_seed;
1332	char *rand_addr;
1333	size_t rand_size;
1334	int num_expand_tests = 2;
1335	int num_misc_tests = 9;
1336	struct test test_cases[MAX_TEST] = {};
1337	struct test perf_test_cases[MAX_PERF_TEST];
1338	int page_size;
1339	time_t t;
1340	FILE *maps_fp;
1341
1342	pattern_seed = (unsigned int) time(&t);
1343
1344	if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
1345		exit(EXIT_FAILURE);
1346
1347	ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
1348		       threshold_mb, pattern_seed);
1349
1350	/*
1351	 * set preallocated random array according to test configs; see the
1352	 * functions for the logic of setting the size
1353	 */
1354	if (!threshold_mb)
1355		rand_size = MAX(max_test_variable_region_size,
1356				max_test_constant_region_size);
1357	else
1358		rand_size = MAX(MIN(threshold_mb * _1MB,
1359				    max_test_variable_region_size),
1360				max_test_constant_region_size);
1361	rand_size = MAX(dest_preamble_size, rand_size);
1362
1363	rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE,
1364				 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1365	if (rand_addr == MAP_FAILED) {
1366		perror("mmap");
1367		ksft_exit_fail_msg("cannot mmap rand_addr\n");
1368	}
1369
1370	/* fill stream of random bytes */
1371	srand(pattern_seed);
1372	for (unsigned long i = 0; i < rand_size; ++i)
1373		rand_addr[i] = (char) rand();
1374
1375	page_size = sysconf(_SC_PAGESIZE);
1376
1377	/* Expected mremap failures */
1378	test_cases[0] =	MAKE_TEST(page_size, page_size, page_size,
1379				  OVERLAPPING, EXPECT_FAILURE,
1380				  "mremap - Source and Destination Regions Overlapping");
1381
1382	test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
1383				  NON_OVERLAPPING, EXPECT_FAILURE,
1384				  "mremap - Destination Address Misaligned (1KB-aligned)");
1385	test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
1386				  NON_OVERLAPPING, EXPECT_FAILURE,
1387				  "mremap - Source Address Misaligned (1KB-aligned)");
1388
1389	/* Src addr PTE aligned */
1390	test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
1391				  NON_OVERLAPPING, EXPECT_SUCCESS,
1392				  "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
1393
1394	/* Src addr 1MB aligned */
1395	test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1396				  "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
1397	test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1398				  "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
1399
1400	/* Src addr PMD aligned */
1401	test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1402				  "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
1403	test_cases[7] =	MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1404				  "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
1405	test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1406				  "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
1407
1408	/* Src addr PUD aligned */
1409	test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1410				  "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
1411	test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1412				   "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
1413	test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1414				   "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
1415	test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1416				   "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
1417
1418	/* Src and Dest addr 1MB aligned. 5MB mremap. */
1419	test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1420				  "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
1421
1422	/* Src and Dest addr 1MB aligned. 5MB mremap. */
1423	test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS,
1424				  "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble");
1425	test_cases[14].config.dest_preamble_size = 10 * _4MB;
1426
1427	perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1428					"1GB mremap - Source PTE-aligned, Destination PTE-aligned");
1429	/*
1430	 * mremap 1GB region - Page table level aligned time
1431	 * comparison.
1432	 */
1433	perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1434				       "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
1435	perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
1436				       "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
1437
1438	run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
1439				(threshold_mb * _1MB >= _1GB);
1440
1441	ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
1442		      ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests);
1443
1444	for (i = 0; i < ARRAY_SIZE(test_cases); i++)
1445		run_mremap_test_case(test_cases[i], &failures, threshold_mb,
1446				     rand_addr);
1447
1448	maps_fp = fopen("/proc/self/maps", "r");
1449
1450	if (maps_fp == NULL) {
1451		munmap(rand_addr, rand_size);
1452		ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno));
1453	}
1454
1455	mremap_expand_merge(maps_fp, page_size);
1456	mremap_expand_merge_offset(maps_fp, page_size);
1457
1458	mremap_move_within_range(pattern_seed, rand_addr);
1459	mremap_move_1mb_from_start(pattern_seed, rand_addr);
1460	mremap_shrink_multiple_vmas(page_size, /* inplace= */true);
1461	mremap_shrink_multiple_vmas(page_size, /* inplace= */false);
1462	mremap_move_multiple_vmas(pattern_seed, page_size, /* dontunmap= */ false);
1463	mremap_move_multiple_vmas(pattern_seed, page_size, /* dontunmap= */ true);
1464	mremap_move_multiple_vmas_split(pattern_seed, page_size, /* dontunmap= */ false);
1465	mremap_move_multiple_vmas_split(pattern_seed, page_size, /* dontunmap= */ true);
1466	mremap_move_multi_invalid_vmas(maps_fp, page_size);
1467
1468	fclose(maps_fp);
1469
1470	if (run_perf_tests) {
1471		ksft_print_msg("\n%s\n",
1472		 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
1473		for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
1474			run_mremap_test_case(perf_test_cases[i], &failures,
1475					     threshold_mb,
1476					     rand_addr);
1477	}
1478
1479	munmap(rand_addr, rand_size);
1480
1481	if (failures > 0)
1482		ksft_exit_fail();
1483	else
1484		ksft_exit_pass();
1485}