hmm-tests.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * HMM stands for Heterogeneous Memory Management, it is a helper layer inside
   4 * the linux kernel to help device drivers mirror a process address space in
   5 * the device. This allows the device to use the same address space which
   6 * makes communication and data exchange a lot easier.
   7 *
   8 * This framework's sole purpose is to exercise various code paths inside
   9 * the kernel to make sure that HMM performs as expected and to flush out any
  10 * bugs.
  11 */
  12
  13#include "kselftest_harness.h"
  14
  15#include <errno.h>
  16#include <fcntl.h>
  17#include <stdio.h>
  18#include <stdlib.h>
  19#include <stdint.h>
  20#include <unistd.h>
  21#include <strings.h>
  22#include <time.h>
  23#include <pthread.h>
  24#include <sys/types.h>
  25#include <sys/stat.h>
  26#include <sys/mman.h>
  27#include <sys/ioctl.h>
  28#include <sys/time.h>
  29
  30
  31/*
  32 * This is a private UAPI to the kernel test module so it isn't exported
  33 * in the usual include/uapi/... directory.
  34 */
  35#include <lib/test_hmm_uapi.h>
  36#include <mm/gup_test.h>
  37#include <mm/vm_util.h>
  38
  39struct hmm_buffer {
  40	void		*ptr;
  41	void		*mirror;
  42	unsigned long	size;
  43	int		fd;
  44	uint64_t	cpages;
  45	uint64_t	faults;
  46};
  47
  48enum {
  49	HMM_PRIVATE_DEVICE_ONE,
  50	HMM_PRIVATE_DEVICE_TWO,
  51	HMM_COHERENCE_DEVICE_ONE,
  52	HMM_COHERENCE_DEVICE_TWO,
  53};
  54
  55#define ONEKB		(1 << 10)
  56#define ONEMEG		(1 << 20)
  57#define TWOMEG		(1 << 21)
  58#define HMM_BUFFER_SIZE (1024 << 12)
  59#define HMM_PATH_MAX    64
  60#define NTIMES		10
  61
  62#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
  63/* Just the flags we need, copied from mm.h: */
  64
  65#ifndef FOLL_WRITE
  66#define FOLL_WRITE	0x01	/* check pte is writable */
  67#endif
  68
  69#ifndef FOLL_LONGTERM
  70#define FOLL_LONGTERM   0x100 /* mapping lifetime is indefinite */
  71#endif
  72FIXTURE(hmm)
  73{
  74	int		fd;
  75	unsigned int	page_size;
  76	unsigned int	page_shift;
  77};
  78
  79FIXTURE_VARIANT(hmm)
  80{
  81	int     device_number;
  82};
  83
  84FIXTURE_VARIANT_ADD(hmm, hmm_device_private)
  85{
  86	.device_number = HMM_PRIVATE_DEVICE_ONE,
  87};
  88
  89FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)
  90{
  91	.device_number = HMM_COHERENCE_DEVICE_ONE,
  92};
  93
  94FIXTURE(hmm2)
  95{
  96	int		fd0;
  97	int		fd1;
  98	unsigned int	page_size;
  99	unsigned int	page_shift;
 100};
 101
 102FIXTURE_VARIANT(hmm2)
 103{
 104	int     device_number0;
 105	int     device_number1;
 106};
 107
 108FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)
 109{
 110	.device_number0 = HMM_PRIVATE_DEVICE_ONE,
 111	.device_number1 = HMM_PRIVATE_DEVICE_TWO,
 112};
 113
 114FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)
 115{
 116	.device_number0 = HMM_COHERENCE_DEVICE_ONE,
 117	.device_number1 = HMM_COHERENCE_DEVICE_TWO,
 118};
 119
 120static int hmm_open(int unit)
 121{
 122	char pathname[HMM_PATH_MAX];
 123	int fd;
 124
 125	snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit);
 126	fd = open(pathname, O_RDWR, 0);
 127	if (fd < 0)
 128		fprintf(stderr, "could not open hmm dmirror driver (%s)\n",
 129			pathname);
 130	return fd;
 131}
 132
 133static bool hmm_is_coherent_type(int dev_num)
 134{
 135	return (dev_num >= HMM_COHERENCE_DEVICE_ONE);
 136}
 137
 138FIXTURE_SETUP(hmm)
 139{
 140	self->page_size = sysconf(_SC_PAGE_SIZE);
 141	self->page_shift = ffs(self->page_size) - 1;
 142
 143	self->fd = hmm_open(variant->device_number);
 144	if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))
 145		SKIP(return, "DEVICE_COHERENT not available");
 146	ASSERT_GE(self->fd, 0);
 147}
 148
 149FIXTURE_SETUP(hmm2)
 150{
 151	self->page_size = sysconf(_SC_PAGE_SIZE);
 152	self->page_shift = ffs(self->page_size) - 1;
 153
 154	self->fd0 = hmm_open(variant->device_number0);
 155	if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))
 156		SKIP(return, "DEVICE_COHERENT not available");
 157	ASSERT_GE(self->fd0, 0);
 158	self->fd1 = hmm_open(variant->device_number1);
 159	ASSERT_GE(self->fd1, 0);
 160}
 161
 162FIXTURE_TEARDOWN(hmm)
 163{
 164	int ret = close(self->fd);
 165
 166	ASSERT_EQ(ret, 0);
 167	self->fd = -1;
 168}
 169
 170FIXTURE_TEARDOWN(hmm2)
 171{
 172	int ret = close(self->fd0);
 173
 174	ASSERT_EQ(ret, 0);
 175	self->fd0 = -1;
 176
 177	ret = close(self->fd1);
 178	ASSERT_EQ(ret, 0);
 179	self->fd1 = -1;
 180}
 181
 182static int hmm_dmirror_cmd(int fd,
 183			   unsigned long request,
 184			   struct hmm_buffer *buffer,
 185			   unsigned long npages)
 186{
 187	struct hmm_dmirror_cmd cmd;
 188	int ret;
 189
 190	/* Simulate a device reading system memory. */
 191	cmd.addr = (__u64)buffer->ptr;
 192	cmd.ptr = (__u64)buffer->mirror;
 193	cmd.npages = npages;
 194
 195	for (;;) {
 196		ret = ioctl(fd, request, &cmd);
 197		if (ret == 0)
 198			break;
 199		if (errno == EINTR)
 200			continue;
 201		return -errno;
 202	}
 203	buffer->cpages = cmd.cpages;
 204	buffer->faults = cmd.faults;
 205
 206	return 0;
 207}
 208
 209static void hmm_buffer_free(struct hmm_buffer *buffer)
 210{
 211	if (buffer == NULL)
 212		return;
 213
 214	if (buffer->ptr) {
 215		munmap(buffer->ptr, buffer->size);
 216		buffer->ptr = NULL;
 217	}
 218	free(buffer->mirror);
 219	free(buffer);
 220}
 221
 222/*
 223 * Create a temporary file that will be deleted on close.
 224 */
 225static int hmm_create_file(unsigned long size)
 226{
 227	char path[HMM_PATH_MAX];
 228	int fd;
 229
 230	strcpy(path, "/tmp");
 231	fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600);
 232	if (fd >= 0) {
 233		int r;
 234
 235		do {
 236			r = ftruncate(fd, size);
 237		} while (r == -1 && errno == EINTR);
 238		if (!r)
 239			return fd;
 240		close(fd);
 241	}
 242	return -1;
 243}
 244
 245/*
 246 * Return a random unsigned number.
 247 */
 248static unsigned int hmm_random(void)
 249{
 250	static int fd = -1;
 251	unsigned int r;
 252
 253	if (fd < 0) {
 254		fd = open("/dev/urandom", O_RDONLY);
 255		if (fd < 0) {
 256			fprintf(stderr, "%s:%d failed to open /dev/urandom\n",
 257					__FILE__, __LINE__);
 258			return ~0U;
 259		}
 260	}
 261	read(fd, &r, sizeof(r));
 262	return r;
 263}
 264
 265static void hmm_nanosleep(unsigned int n)
 266{
 267	struct timespec t;
 268
 269	t.tv_sec = 0;
 270	t.tv_nsec = n;
 271	nanosleep(&t, NULL);
 272}
 273
 274static int hmm_migrate_sys_to_dev(int fd,
 275				   struct hmm_buffer *buffer,
 276				   unsigned long npages)
 277{
 278	return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);
 279}
 280
 281static int hmm_migrate_dev_to_sys(int fd,
 282				   struct hmm_buffer *buffer,
 283				   unsigned long npages)
 284{
 285	return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);
 286}
 287
 288/*
 289 * Simple NULL test of device open/close.
 290 */
 291TEST_F(hmm, open_close)
 292{
 293}
 294
 295/*
 296 * Read private anonymous memory.
 297 */
 298TEST_F(hmm, anon_read)
 299{
 300	struct hmm_buffer *buffer;
 301	unsigned long npages;
 302	unsigned long size;
 303	unsigned long i;
 304	int *ptr;
 305	int ret;
 306	int val;
 307
 308	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 309	ASSERT_NE(npages, 0);
 310	size = npages << self->page_shift;
 311
 312	buffer = malloc(sizeof(*buffer));
 313	ASSERT_NE(buffer, NULL);
 314
 315	buffer->fd = -1;
 316	buffer->size = size;
 317	buffer->mirror = malloc(size);
 318	ASSERT_NE(buffer->mirror, NULL);
 319
 320	buffer->ptr = mmap(NULL, size,
 321			   PROT_READ | PROT_WRITE,
 322			   MAP_PRIVATE | MAP_ANONYMOUS,
 323			   buffer->fd, 0);
 324	ASSERT_NE(buffer->ptr, MAP_FAILED);
 325
 326	/*
 327	 * Initialize buffer in system memory but leave the first two pages
 328	 * zero (pte_none and pfn_zero).
 329	 */
 330	i = 2 * self->page_size / sizeof(*ptr);
 331	for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 332		ptr[i] = i;
 333
 334	/* Set buffer permission to read-only. */
 335	ret = mprotect(buffer->ptr, size, PROT_READ);
 336	ASSERT_EQ(ret, 0);
 337
 338	/* Populate the CPU page table with a special zero page. */
 339	val = *(int *)(buffer->ptr + self->page_size);
 340	ASSERT_EQ(val, 0);
 341
 342	/* Simulate a device reading system memory. */
 343	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 344	ASSERT_EQ(ret, 0);
 345	ASSERT_EQ(buffer->cpages, npages);
 346	ASSERT_EQ(buffer->faults, 1);
 347
 348	/* Check what the device read. */
 349	ptr = buffer->mirror;
 350	for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i)
 351		ASSERT_EQ(ptr[i], 0);
 352	for (; i < size / sizeof(*ptr); ++i)
 353		ASSERT_EQ(ptr[i], i);
 354
 355	hmm_buffer_free(buffer);
 356}
 357
 358/*
 359 * Read private anonymous memory which has been protected with
 360 * mprotect() PROT_NONE.
 361 */
 362TEST_F(hmm, anon_read_prot)
 363{
 364	struct hmm_buffer *buffer;
 365	unsigned long npages;
 366	unsigned long size;
 367	unsigned long i;
 368	int *ptr;
 369	int ret;
 370
 371	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 372	ASSERT_NE(npages, 0);
 373	size = npages << self->page_shift;
 374
 375	buffer = malloc(sizeof(*buffer));
 376	ASSERT_NE(buffer, NULL);
 377
 378	buffer->fd = -1;
 379	buffer->size = size;
 380	buffer->mirror = malloc(size);
 381	ASSERT_NE(buffer->mirror, NULL);
 382
 383	buffer->ptr = mmap(NULL, size,
 384			   PROT_READ | PROT_WRITE,
 385			   MAP_PRIVATE | MAP_ANONYMOUS,
 386			   buffer->fd, 0);
 387	ASSERT_NE(buffer->ptr, MAP_FAILED);
 388
 389	/* Initialize buffer in system memory. */
 390	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 391		ptr[i] = i;
 392
 393	/* Initialize mirror buffer so we can verify it isn't written. */
 394	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 395		ptr[i] = -i;
 396
 397	/* Protect buffer from reading. */
 398	ret = mprotect(buffer->ptr, size, PROT_NONE);
 399	ASSERT_EQ(ret, 0);
 400
 401	/* Simulate a device reading system memory. */
 402	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 403	ASSERT_EQ(ret, -EFAULT);
 404
 405	/* Allow CPU to read the buffer so we can check it. */
 406	ret = mprotect(buffer->ptr, size, PROT_READ);
 407	ASSERT_EQ(ret, 0);
 408	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 409		ASSERT_EQ(ptr[i], i);
 410
 411	/* Check what the device read. */
 412	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 413		ASSERT_EQ(ptr[i], -i);
 414
 415	hmm_buffer_free(buffer);
 416}
 417
 418/*
 419 * Write private anonymous memory.
 420 */
 421TEST_F(hmm, anon_write)
 422{
 423	struct hmm_buffer *buffer;
 424	unsigned long npages;
 425	unsigned long size;
 426	unsigned long i;
 427	int *ptr;
 428	int ret;
 429
 430	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 431	ASSERT_NE(npages, 0);
 432	size = npages << self->page_shift;
 433
 434	buffer = malloc(sizeof(*buffer));
 435	ASSERT_NE(buffer, NULL);
 436
 437	buffer->fd = -1;
 438	buffer->size = size;
 439	buffer->mirror = malloc(size);
 440	ASSERT_NE(buffer->mirror, NULL);
 441
 442	buffer->ptr = mmap(NULL, size,
 443			   PROT_READ | PROT_WRITE,
 444			   MAP_PRIVATE | MAP_ANONYMOUS,
 445			   buffer->fd, 0);
 446	ASSERT_NE(buffer->ptr, MAP_FAILED);
 447
 448	/* Initialize data that the device will write to buffer->ptr. */
 449	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 450		ptr[i] = i;
 451
 452	/* Simulate a device writing system memory. */
 453	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 454	ASSERT_EQ(ret, 0);
 455	ASSERT_EQ(buffer->cpages, npages);
 456	ASSERT_EQ(buffer->faults, 1);
 457
 458	/* Check what the device wrote. */
 459	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 460		ASSERT_EQ(ptr[i], i);
 461
 462	hmm_buffer_free(buffer);
 463}
 464
 465/*
 466 * Write private anonymous memory which has been protected with
 467 * mprotect() PROT_READ.
 468 */
 469TEST_F(hmm, anon_write_prot)
 470{
 471	struct hmm_buffer *buffer;
 472	unsigned long npages;
 473	unsigned long size;
 474	unsigned long i;
 475	int *ptr;
 476	int ret;
 477
 478	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 479	ASSERT_NE(npages, 0);
 480	size = npages << self->page_shift;
 481
 482	buffer = malloc(sizeof(*buffer));
 483	ASSERT_NE(buffer, NULL);
 484
 485	buffer->fd = -1;
 486	buffer->size = size;
 487	buffer->mirror = malloc(size);
 488	ASSERT_NE(buffer->mirror, NULL);
 489
 490	buffer->ptr = mmap(NULL, size,
 491			   PROT_READ,
 492			   MAP_PRIVATE | MAP_ANONYMOUS,
 493			   buffer->fd, 0);
 494	ASSERT_NE(buffer->ptr, MAP_FAILED);
 495
 496	/* Simulate a device reading a zero page of memory. */
 497	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);
 498	ASSERT_EQ(ret, 0);
 499	ASSERT_EQ(buffer->cpages, 1);
 500	ASSERT_EQ(buffer->faults, 1);
 501
 502	/* Initialize data that the device will write to buffer->ptr. */
 503	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 504		ptr[i] = i;
 505
 506	/* Simulate a device writing system memory. */
 507	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 508	ASSERT_EQ(ret, -EPERM);
 509
 510	/* Check what the device wrote. */
 511	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 512		ASSERT_EQ(ptr[i], 0);
 513
 514	/* Now allow writing and see that the zero page is replaced. */
 515	ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);
 516	ASSERT_EQ(ret, 0);
 517
 518	/* Simulate a device writing system memory. */
 519	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 520	ASSERT_EQ(ret, 0);
 521	ASSERT_EQ(buffer->cpages, npages);
 522	ASSERT_EQ(buffer->faults, 1);
 523
 524	/* Check what the device wrote. */
 525	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 526		ASSERT_EQ(ptr[i], i);
 527
 528	hmm_buffer_free(buffer);
 529}
 530
 531/*
 532 * Check that a device writing an anonymous private mapping
 533 * will copy-on-write if a child process inherits the mapping.
 534 *
 535 * Also verifies after fork() memory the device can be read by child.
 536 */
 537TEST_F(hmm, anon_write_child)
 538{
 539	struct hmm_buffer *buffer;
 540	unsigned long npages;
 541	unsigned long size;
 542	unsigned long i;
 543	void *old_ptr;
 544	void *map;
 545	int *ptr;
 546	pid_t pid;
 547	int child_fd;
 548	int ret, use_thp, migrate;
 549
 550	for (migrate = 0; migrate < 2; ++migrate) {
 551		for (use_thp = 0; use_thp < 2; ++use_thp) {
 552			npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE,
 553				       self->page_size) >> self->page_shift;
 554			ASSERT_NE(npages, 0);
 555			size = npages << self->page_shift;
 556
 557			buffer = malloc(sizeof(*buffer));
 558			ASSERT_NE(buffer, NULL);
 559
 560			buffer->fd = -1;
 561			buffer->size = size * 2;
 562			buffer->mirror = malloc(size);
 563			ASSERT_NE(buffer->mirror, NULL);
 564
 565			buffer->ptr = mmap(NULL, size * 2,
 566					   PROT_READ | PROT_WRITE,
 567					   MAP_PRIVATE | MAP_ANONYMOUS,
 568					   buffer->fd, 0);
 569			ASSERT_NE(buffer->ptr, MAP_FAILED);
 570
 571			old_ptr = buffer->ptr;
 572			if (use_thp) {
 573				map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 574				ret = madvise(map, size, MADV_HUGEPAGE);
 575				ASSERT_EQ(ret, 0);
 576				buffer->ptr = map;
 577			}
 578
 579			/* Initialize buffer->ptr so we can tell if it is written. */
 580			for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 581				ptr[i] = i;
 582
 583			/* Initialize data that the device will write to buffer->ptr. */
 584			for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 585				ptr[i] = -i;
 586
 587			if (migrate) {
 588				ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
 589				ASSERT_EQ(ret, 0);
 590				ASSERT_EQ(buffer->cpages, npages);
 591
 592			}
 593
 594			pid = fork();
 595			if (pid == -1)
 596				ASSERT_EQ(pid, 0);
 597			if (pid != 0) {
 598				waitpid(pid, &ret, 0);
 599				ASSERT_EQ(WIFEXITED(ret), 1);
 600
 601				/* Check that the parent's buffer did not change. */
 602				for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 603					ASSERT_EQ(ptr[i], i);
 604
 605				buffer->ptr = old_ptr;
 606				hmm_buffer_free(buffer);
 607				continue;
 608			}
 609
 610			/* Check that we see the parent's values. */
 611			for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 612				ASSERT_EQ(ptr[i], i);
 613			if (!migrate) {
 614				for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 615					ASSERT_EQ(ptr[i], -i);
 616			}
 617
 618			/* The child process needs its own mirror to its own mm. */
 619			child_fd = hmm_open(0);
 620			ASSERT_GE(child_fd, 0);
 621
 622			/* Simulate a device writing system memory. */
 623			ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
 624			ASSERT_EQ(ret, 0);
 625			ASSERT_EQ(buffer->cpages, npages);
 626			ASSERT_EQ(buffer->faults, 1);
 627
 628			/* Check what the device wrote. */
 629			if (!migrate) {
 630				for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 631					ASSERT_EQ(ptr[i], -i);
 632			}
 633
 634			close(child_fd);
 635			exit(0);
 636		}
 637	}
 638}
 639
 640/*
 641 * Check that a device writing an anonymous shared mapping
 642 * will not copy-on-write if a child process inherits the mapping.
 643 */
 644TEST_F(hmm, anon_write_child_shared)
 645{
 646	struct hmm_buffer *buffer;
 647	unsigned long npages;
 648	unsigned long size;
 649	unsigned long i;
 650	int *ptr;
 651	pid_t pid;
 652	int child_fd;
 653	int ret;
 654
 655	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 656	ASSERT_NE(npages, 0);
 657	size = npages << self->page_shift;
 658
 659	buffer = malloc(sizeof(*buffer));
 660	ASSERT_NE(buffer, NULL);
 661
 662	buffer->fd = -1;
 663	buffer->size = size;
 664	buffer->mirror = malloc(size);
 665	ASSERT_NE(buffer->mirror, NULL);
 666
 667	buffer->ptr = mmap(NULL, size,
 668			   PROT_READ | PROT_WRITE,
 669			   MAP_SHARED | MAP_ANONYMOUS,
 670			   buffer->fd, 0);
 671	ASSERT_NE(buffer->ptr, MAP_FAILED);
 672
 673	/* Initialize buffer->ptr so we can tell if it is written. */
 674	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 675		ptr[i] = i;
 676
 677	/* Initialize data that the device will write to buffer->ptr. */
 678	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 679		ptr[i] = -i;
 680
 681	pid = fork();
 682	if (pid == -1)
 683		ASSERT_EQ(pid, 0);
 684	if (pid != 0) {
 685		waitpid(pid, &ret, 0);
 686		ASSERT_EQ(WIFEXITED(ret), 1);
 687
 688		/* Check that the parent's buffer did change. */
 689		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 690			ASSERT_EQ(ptr[i], -i);
 691		return;
 692	}
 693
 694	/* Check that we see the parent's values. */
 695	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 696		ASSERT_EQ(ptr[i], i);
 697	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 698		ASSERT_EQ(ptr[i], -i);
 699
 700	/* The child process needs its own mirror to its own mm. */
 701	child_fd = hmm_open(0);
 702	ASSERT_GE(child_fd, 0);
 703
 704	/* Simulate a device writing system memory. */
 705	ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
 706	ASSERT_EQ(ret, 0);
 707	ASSERT_EQ(buffer->cpages, npages);
 708	ASSERT_EQ(buffer->faults, 1);
 709
 710	/* Check what the device wrote. */
 711	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 712		ASSERT_EQ(ptr[i], -i);
 713
 714	close(child_fd);
 715	exit(0);
 716}
 717
 718/*
 719 * Write private anonymous huge page.
 720 */
 721TEST_F(hmm, anon_write_huge)
 722{
 723	struct hmm_buffer *buffer;
 724	unsigned long npages;
 725	unsigned long size;
 726	unsigned long i;
 727	void *old_ptr;
 728	void *map;
 729	int *ptr;
 730	int ret;
 731
 732	size = 2 * read_pmd_pagesize();
 733
 734	buffer = malloc(sizeof(*buffer));
 735	ASSERT_NE(buffer, NULL);
 736
 737	buffer->fd = -1;
 738	buffer->size = size;
 739	buffer->mirror = malloc(size);
 740	ASSERT_NE(buffer->mirror, NULL);
 741
 742	buffer->ptr = mmap(NULL, size,
 743			   PROT_READ | PROT_WRITE,
 744			   MAP_PRIVATE | MAP_ANONYMOUS,
 745			   buffer->fd, 0);
 746	ASSERT_NE(buffer->ptr, MAP_FAILED);
 747
 748	size /= 2;
 749	npages = size >> self->page_shift;
 750	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
 751	ret = madvise(map, size, MADV_HUGEPAGE);
 752	ASSERT_EQ(ret, 0);
 753	old_ptr = buffer->ptr;
 754	buffer->ptr = map;
 755
 756	/* Initialize data that the device will write to buffer->ptr. */
 757	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 758		ptr[i] = i;
 759
 760	/* Simulate a device writing system memory. */
 761	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 762	ASSERT_EQ(ret, 0);
 763	ASSERT_EQ(buffer->cpages, npages);
 764	ASSERT_EQ(buffer->faults, 1);
 765
 766	/* Check what the device wrote. */
 767	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 768		ASSERT_EQ(ptr[i], i);
 769
 770	buffer->ptr = old_ptr;
 771	hmm_buffer_free(buffer);
 772}
 773
 774/*
 775 * Write huge TLBFS page.
 776 */
 777TEST_F(hmm, anon_write_hugetlbfs)
 778{
 779	struct hmm_buffer *buffer;
 780	unsigned long npages;
 781	unsigned long size;
 782	unsigned long default_hsize = default_huge_page_size();
 783	unsigned long i;
 784	int *ptr;
 785	int ret;
 786
 787	if (!default_hsize)
 788		SKIP(return, "Huge page size could not be determined");
 789
 790	size = ALIGN(TWOMEG, default_hsize);
 791	npages = size >> self->page_shift;
 792
 793	buffer = malloc(sizeof(*buffer));
 794	ASSERT_NE(buffer, NULL);
 795
 796	buffer->ptr = mmap(NULL, size,
 797				   PROT_READ | PROT_WRITE,
 798				   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
 799				   -1, 0);
 800	if (buffer->ptr == MAP_FAILED) {
 801		free(buffer);
 802		SKIP(return, "Huge page could not be allocated");
 803	}
 804
 805	buffer->fd = -1;
 806	buffer->size = size;
 807	buffer->mirror = malloc(size);
 808	ASSERT_NE(buffer->mirror, NULL);
 809
 810	/* Initialize data that the device will write to buffer->ptr. */
 811	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 812		ptr[i] = i;
 813
 814	/* Simulate a device writing system memory. */
 815	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 816	ASSERT_EQ(ret, 0);
 817	ASSERT_EQ(buffer->cpages, npages);
 818	ASSERT_EQ(buffer->faults, 1);
 819
 820	/* Check what the device wrote. */
 821	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 822		ASSERT_EQ(ptr[i], i);
 823
 824	munmap(buffer->ptr, buffer->size);
 825	buffer->ptr = NULL;
 826	hmm_buffer_free(buffer);
 827}
 828
 829/*
 830 * Read mmap'ed file memory.
 831 */
 832TEST_F(hmm, file_read)
 833{
 834	struct hmm_buffer *buffer;
 835	unsigned long npages;
 836	unsigned long size;
 837	unsigned long i;
 838	int *ptr;
 839	int ret;
 840	int fd;
 841	ssize_t len;
 842
 843	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 844	ASSERT_NE(npages, 0);
 845	size = npages << self->page_shift;
 846
 847	fd = hmm_create_file(size);
 848	ASSERT_GE(fd, 0);
 849
 850	buffer = malloc(sizeof(*buffer));
 851	ASSERT_NE(buffer, NULL);
 852
 853	buffer->fd = fd;
 854	buffer->size = size;
 855	buffer->mirror = malloc(size);
 856	ASSERT_NE(buffer->mirror, NULL);
 857
 858	/* Write initial contents of the file. */
 859	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 860		ptr[i] = i;
 861	len = pwrite(fd, buffer->mirror, size, 0);
 862	ASSERT_EQ(len, size);
 863	memset(buffer->mirror, 0, size);
 864
 865	buffer->ptr = mmap(NULL, size,
 866			   PROT_READ,
 867			   MAP_SHARED,
 868			   buffer->fd, 0);
 869	ASSERT_NE(buffer->ptr, MAP_FAILED);
 870
 871	/* Simulate a device reading system memory. */
 872	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
 873	ASSERT_EQ(ret, 0);
 874	ASSERT_EQ(buffer->cpages, npages);
 875	ASSERT_EQ(buffer->faults, 1);
 876
 877	/* Check what the device read. */
 878	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 879		ASSERT_EQ(ptr[i], i);
 880
 881	hmm_buffer_free(buffer);
 882}
 883
 884/*
 885 * Write mmap'ed file memory.
 886 */
 887TEST_F(hmm, file_write)
 888{
 889	struct hmm_buffer *buffer;
 890	unsigned long npages;
 891	unsigned long size;
 892	unsigned long i;
 893	int *ptr;
 894	int ret;
 895	int fd;
 896	ssize_t len;
 897
 898	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 899	ASSERT_NE(npages, 0);
 900	size = npages << self->page_shift;
 901
 902	fd = hmm_create_file(size);
 903	ASSERT_GE(fd, 0);
 904
 905	buffer = malloc(sizeof(*buffer));
 906	ASSERT_NE(buffer, NULL);
 907
 908	buffer->fd = fd;
 909	buffer->size = size;
 910	buffer->mirror = malloc(size);
 911	ASSERT_NE(buffer->mirror, NULL);
 912
 913	buffer->ptr = mmap(NULL, size,
 914			   PROT_READ | PROT_WRITE,
 915			   MAP_SHARED,
 916			   buffer->fd, 0);
 917	ASSERT_NE(buffer->ptr, MAP_FAILED);
 918
 919	/* Initialize data that the device will write to buffer->ptr. */
 920	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 921		ptr[i] = i;
 922
 923	/* Simulate a device writing system memory. */
 924	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
 925	ASSERT_EQ(ret, 0);
 926	ASSERT_EQ(buffer->cpages, npages);
 927	ASSERT_EQ(buffer->faults, 1);
 928
 929	/* Check what the device wrote. */
 930	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 931		ASSERT_EQ(ptr[i], i);
 932
 933	/* Check that the device also wrote the file. */
 934	len = pread(fd, buffer->mirror, size, 0);
 935	ASSERT_EQ(len, size);
 936	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 937		ASSERT_EQ(ptr[i], i);
 938
 939	hmm_buffer_free(buffer);
 940}
 941
 942/*
 943 * Migrate anonymous memory to device private memory.
 944 */
 945TEST_F(hmm, migrate)
 946{
 947	struct hmm_buffer *buffer;
 948	unsigned long npages;
 949	unsigned long size;
 950	unsigned long i;
 951	int *ptr;
 952	int ret;
 953
 954	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
 955	ASSERT_NE(npages, 0);
 956	size = npages << self->page_shift;
 957
 958	buffer = malloc(sizeof(*buffer));
 959	ASSERT_NE(buffer, NULL);
 960
 961	buffer->fd = -1;
 962	buffer->size = size;
 963	buffer->mirror = malloc(size);
 964	ASSERT_NE(buffer->mirror, NULL);
 965
 966	buffer->ptr = mmap(NULL, size,
 967			   PROT_READ | PROT_WRITE,
 968			   MAP_PRIVATE | MAP_ANONYMOUS,
 969			   buffer->fd, 0);
 970	ASSERT_NE(buffer->ptr, MAP_FAILED);
 971
 972	/* Initialize buffer in system memory. */
 973	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
 974		ptr[i] = i;
 975
 976	/* Migrate memory to device. */
 977	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
 978	ASSERT_EQ(ret, 0);
 979	ASSERT_EQ(buffer->cpages, npages);
 980
 981	/* Check what the device read. */
 982	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
 983		ASSERT_EQ(ptr[i], i);
 984
 985	hmm_buffer_free(buffer);
 986}
 987
 988/*
 989 * Migrate anonymous memory to device private memory and fault some of it back
 990 * to system memory, then try migrating the resulting mix of system and device
 991 * private memory to the device.
 992 */
 993TEST_F(hmm, migrate_fault)
 994{
 995	struct hmm_buffer *buffer;
 996	unsigned long npages;
 997	unsigned long size;
 998	unsigned long i;
 999	int *ptr;
1000	int ret;
1001
1002	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1003	ASSERT_NE(npages, 0);
1004	size = npages << self->page_shift;
1005
1006	buffer = malloc(sizeof(*buffer));
1007	ASSERT_NE(buffer, NULL);
1008
1009	buffer->fd = -1;
1010	buffer->size = size;
1011	buffer->mirror = malloc(size);
1012	ASSERT_NE(buffer->mirror, NULL);
1013
1014	buffer->ptr = mmap(NULL, size,
1015			   PROT_READ | PROT_WRITE,
1016			   MAP_PRIVATE | MAP_ANONYMOUS,
1017			   buffer->fd, 0);
1018	ASSERT_NE(buffer->ptr, MAP_FAILED);
1019
1020	/* Initialize buffer in system memory. */
1021	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1022		ptr[i] = i;
1023
1024	/* Migrate memory to device. */
1025	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1026	ASSERT_EQ(ret, 0);
1027	ASSERT_EQ(buffer->cpages, npages);
1028
1029	/* Check what the device read. */
1030	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1031		ASSERT_EQ(ptr[i], i);
1032
1033	/* Fault half the pages back to system memory and check them. */
1034	for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1035		ASSERT_EQ(ptr[i], i);
1036
1037	/* Migrate memory to the device again. */
1038	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1039	ASSERT_EQ(ret, 0);
1040	ASSERT_EQ(buffer->cpages, npages);
1041
1042	/* Check what the device read. */
1043	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1044		ASSERT_EQ(ptr[i], i);
1045
1046	hmm_buffer_free(buffer);
1047}
1048
1049TEST_F(hmm, migrate_release)
1050{
1051	struct hmm_buffer *buffer;
1052	unsigned long npages;
1053	unsigned long size;
1054	unsigned long i;
1055	int *ptr;
1056	int ret;
1057
1058	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1059	ASSERT_NE(npages, 0);
1060	size = npages << self->page_shift;
1061
1062	buffer = malloc(sizeof(*buffer));
1063	ASSERT_NE(buffer, NULL);
1064
1065	buffer->fd = -1;
1066	buffer->size = size;
1067	buffer->mirror = malloc(size);
1068	ASSERT_NE(buffer->mirror, NULL);
1069
1070	buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1071			   MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
1072	ASSERT_NE(buffer->ptr, MAP_FAILED);
1073
1074	/* Initialize buffer in system memory. */
1075	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1076		ptr[i] = i;
1077
1078	/* Migrate memory to device. */
1079	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1080	ASSERT_EQ(ret, 0);
1081	ASSERT_EQ(buffer->cpages, npages);
1082
1083	/* Check what the device read. */
1084	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1085		ASSERT_EQ(ptr[i], i);
1086
1087	/* Release device memory. */
1088	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages);
1089	ASSERT_EQ(ret, 0);
1090
1091	/* Fault pages back to system memory and check them. */
1092	for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1093		ASSERT_EQ(ptr[i], i);
1094
1095	hmm_buffer_free(buffer);
1096}
1097
1098/*
1099 * Migrate anonymous shared memory to device private memory.
1100 */
1101TEST_F(hmm, migrate_shared)
1102{
1103	struct hmm_buffer *buffer;
1104	unsigned long npages;
1105	unsigned long size;
1106	int ret;
1107
1108	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1109	ASSERT_NE(npages, 0);
1110	size = npages << self->page_shift;
1111
1112	buffer = malloc(sizeof(*buffer));
1113	ASSERT_NE(buffer, NULL);
1114
1115	buffer->fd = -1;
1116	buffer->size = size;
1117	buffer->mirror = malloc(size);
1118	ASSERT_NE(buffer->mirror, NULL);
1119
1120	buffer->ptr = mmap(NULL, size,
1121			   PROT_READ | PROT_WRITE,
1122			   MAP_SHARED | MAP_ANONYMOUS,
1123			   buffer->fd, 0);
1124	ASSERT_NE(buffer->ptr, MAP_FAILED);
1125
1126	/* Migrate memory to device. */
1127	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1128	ASSERT_EQ(ret, -ENOENT);
1129
1130	hmm_buffer_free(buffer);
1131}
1132
1133/*
1134 * Try to migrate various memory types to device private memory.
1135 */
1136TEST_F(hmm2, migrate_mixed)
1137{
1138	struct hmm_buffer *buffer;
1139	unsigned long npages;
1140	unsigned long size;
1141	int *ptr;
1142	unsigned char *p;
1143	int ret;
1144	int val;
1145
1146	npages = 6;
1147	size = npages << self->page_shift;
1148
1149	buffer = malloc(sizeof(*buffer));
1150	ASSERT_NE(buffer, NULL);
1151
1152	buffer->fd = -1;
1153	buffer->size = size;
1154	buffer->mirror = malloc(size);
1155	ASSERT_NE(buffer->mirror, NULL);
1156
1157	/* Reserve a range of addresses. */
1158	buffer->ptr = mmap(NULL, size,
1159			   PROT_NONE,
1160			   MAP_PRIVATE | MAP_ANONYMOUS,
1161			   buffer->fd, 0);
1162	ASSERT_NE(buffer->ptr, MAP_FAILED);
1163	p = buffer->ptr;
1164
1165	/* Migrating a protected area should be an error. */
1166	ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1167	ASSERT_EQ(ret, -EINVAL);
1168
1169	/* Punch a hole after the first page address. */
1170	ret = munmap(buffer->ptr + self->page_size, self->page_size);
1171	ASSERT_EQ(ret, 0);
1172
1173	/* We expect an error if the vma doesn't cover the range. */
1174	ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3);
1175	ASSERT_EQ(ret, -EINVAL);
1176
1177	/* Page 2 will be a read-only zero page. */
1178	ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1179				PROT_READ);
1180	ASSERT_EQ(ret, 0);
1181	ptr = (int *)(buffer->ptr + 2 * self->page_size);
1182	val = *ptr + 3;
1183	ASSERT_EQ(val, 3);
1184
1185	/* Page 3 will be read-only. */
1186	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1187				PROT_READ | PROT_WRITE);
1188	ASSERT_EQ(ret, 0);
1189	ptr = (int *)(buffer->ptr + 3 * self->page_size);
1190	*ptr = val;
1191	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1192				PROT_READ);
1193	ASSERT_EQ(ret, 0);
1194
1195	/* Page 4-5 will be read-write. */
1196	ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size,
1197				PROT_READ | PROT_WRITE);
1198	ASSERT_EQ(ret, 0);
1199	ptr = (int *)(buffer->ptr + 4 * self->page_size);
1200	*ptr = val;
1201	ptr = (int *)(buffer->ptr + 5 * self->page_size);
1202	*ptr = val;
1203
1204	/* Now try to migrate pages 2-5 to device 1. */
1205	buffer->ptr = p + 2 * self->page_size;
1206	ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4);
1207	ASSERT_EQ(ret, 0);
1208	ASSERT_EQ(buffer->cpages, 4);
1209
1210	/* Page 5 won't be migrated to device 0 because it's on device 1. */
1211	buffer->ptr = p + 5 * self->page_size;
1212	ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1213	ASSERT_EQ(ret, -ENOENT);
1214	buffer->ptr = p;
1215
1216	buffer->ptr = p;
1217	hmm_buffer_free(buffer);
1218}
1219
1220/*
1221 * Migrate anonymous memory to device memory and back to system memory
1222 * multiple times. In case of private zone configuration, this is done
1223 * through fault pages accessed by CPU. In case of coherent zone configuration,
1224 * the pages from the device should be explicitly migrated back to system memory.
1225 * The reason is Coherent device zone has coherent access by CPU, therefore
1226 * it will not generate any page fault.
1227 */
1228TEST_F(hmm, migrate_multiple)
1229{
1230	struct hmm_buffer *buffer;
1231	unsigned long npages;
1232	unsigned long size;
1233	unsigned long i;
1234	unsigned long c;
1235	int *ptr;
1236	int ret;
1237
1238	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1239	ASSERT_NE(npages, 0);
1240	size = npages << self->page_shift;
1241
1242	for (c = 0; c < NTIMES; c++) {
1243		buffer = malloc(sizeof(*buffer));
1244		ASSERT_NE(buffer, NULL);
1245
1246		buffer->fd = -1;
1247		buffer->size = size;
1248		buffer->mirror = malloc(size);
1249		ASSERT_NE(buffer->mirror, NULL);
1250
1251		buffer->ptr = mmap(NULL, size,
1252				   PROT_READ | PROT_WRITE,
1253				   MAP_PRIVATE | MAP_ANONYMOUS,
1254				   buffer->fd, 0);
1255		ASSERT_NE(buffer->ptr, MAP_FAILED);
1256
1257		/* Initialize buffer in system memory. */
1258		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1259			ptr[i] = i;
1260
1261		/* Migrate memory to device. */
1262		ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1263		ASSERT_EQ(ret, 0);
1264		ASSERT_EQ(buffer->cpages, npages);
1265
1266		/* Check what the device read. */
1267		for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1268			ASSERT_EQ(ptr[i], i);
1269
1270		/* Migrate back to system memory and check them. */
1271		if (hmm_is_coherent_type(variant->device_number)) {
1272			ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
1273			ASSERT_EQ(ret, 0);
1274			ASSERT_EQ(buffer->cpages, npages);
1275		}
1276
1277		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1278			ASSERT_EQ(ptr[i], i);
1279
1280		hmm_buffer_free(buffer);
1281	}
1282}
1283
1284/*
1285 * Read anonymous memory multiple times.
1286 */
1287TEST_F(hmm, anon_read_multiple)
1288{
1289	struct hmm_buffer *buffer;
1290	unsigned long npages;
1291	unsigned long size;
1292	unsigned long i;
1293	unsigned long c;
1294	int *ptr;
1295	int ret;
1296
1297	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1298	ASSERT_NE(npages, 0);
1299	size = npages << self->page_shift;
1300
1301	for (c = 0; c < NTIMES; c++) {
1302		buffer = malloc(sizeof(*buffer));
1303		ASSERT_NE(buffer, NULL);
1304
1305		buffer->fd = -1;
1306		buffer->size = size;
1307		buffer->mirror = malloc(size);
1308		ASSERT_NE(buffer->mirror, NULL);
1309
1310		buffer->ptr = mmap(NULL, size,
1311				   PROT_READ | PROT_WRITE,
1312				   MAP_PRIVATE | MAP_ANONYMOUS,
1313				   buffer->fd, 0);
1314		ASSERT_NE(buffer->ptr, MAP_FAILED);
1315
1316		/* Initialize buffer in system memory. */
1317		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1318			ptr[i] = i + c;
1319
1320		/* Simulate a device reading system memory. */
1321		ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1322				      npages);
1323		ASSERT_EQ(ret, 0);
1324		ASSERT_EQ(buffer->cpages, npages);
1325		ASSERT_EQ(buffer->faults, 1);
1326
1327		/* Check what the device read. */
1328		for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1329			ASSERT_EQ(ptr[i], i + c);
1330
1331		hmm_buffer_free(buffer);
1332	}
1333}
1334
1335void *unmap_buffer(void *p)
1336{
1337	struct hmm_buffer *buffer = p;
1338
1339	/* Delay for a bit and then unmap buffer while it is being read. */
1340	hmm_nanosleep(hmm_random() % 32000);
1341	munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);
1342	buffer->ptr = NULL;
1343
1344	return NULL;
1345}
1346
1347/*
1348 * Try reading anonymous memory while it is being unmapped.
1349 */
1350TEST_F(hmm, anon_teardown)
1351{
1352	unsigned long npages;
1353	unsigned long size;
1354	unsigned long c;
1355	void *ret;
1356
1357	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1358	ASSERT_NE(npages, 0);
1359	size = npages << self->page_shift;
1360
1361	for (c = 0; c < NTIMES; ++c) {
1362		pthread_t thread;
1363		struct hmm_buffer *buffer;
1364		unsigned long i;
1365		int *ptr;
1366		int rc;
1367
1368		buffer = malloc(sizeof(*buffer));
1369		ASSERT_NE(buffer, NULL);
1370
1371		buffer->fd = -1;
1372		buffer->size = size;
1373		buffer->mirror = malloc(size);
1374		ASSERT_NE(buffer->mirror, NULL);
1375
1376		buffer->ptr = mmap(NULL, size,
1377				   PROT_READ | PROT_WRITE,
1378				   MAP_PRIVATE | MAP_ANONYMOUS,
1379				   buffer->fd, 0);
1380		ASSERT_NE(buffer->ptr, MAP_FAILED);
1381
1382		/* Initialize buffer in system memory. */
1383		for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1384			ptr[i] = i + c;
1385
1386		rc = pthread_create(&thread, NULL, unmap_buffer, buffer);
1387		ASSERT_EQ(rc, 0);
1388
1389		/* Simulate a device reading system memory. */
1390		rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1391				     npages);
1392		if (rc == 0) {
1393			ASSERT_EQ(buffer->cpages, npages);
1394			ASSERT_EQ(buffer->faults, 1);
1395
1396			/* Check what the device read. */
1397			for (i = 0, ptr = buffer->mirror;
1398			     i < size / sizeof(*ptr);
1399			     ++i)
1400				ASSERT_EQ(ptr[i], i + c);
1401		}
1402
1403		pthread_join(thread, &ret);
1404		hmm_buffer_free(buffer);
1405	}
1406}
1407
1408/*
1409 * Test memory snapshot without faulting in pages accessed by the device.
1410 */
1411TEST_F(hmm, mixedmap)
1412{
1413	struct hmm_buffer *buffer;
1414	unsigned long npages;
1415	unsigned long size;
1416	unsigned char *m;
1417	int ret;
1418
1419	npages = 1;
1420	size = npages << self->page_shift;
1421
1422	buffer = malloc(sizeof(*buffer));
1423	ASSERT_NE(buffer, NULL);
1424
1425	buffer->fd = -1;
1426	buffer->size = size;
1427	buffer->mirror = malloc(npages);
1428	ASSERT_NE(buffer->mirror, NULL);
1429
1430
1431	/* Reserve a range of addresses. */
1432	buffer->ptr = mmap(NULL, size,
1433			   PROT_READ | PROT_WRITE,
1434			   MAP_PRIVATE,
1435			   self->fd, 0);
1436	ASSERT_NE(buffer->ptr, MAP_FAILED);
1437
1438	/* Simulate a device snapshotting CPU pagetables. */
1439	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1440	ASSERT_EQ(ret, 0);
1441	ASSERT_EQ(buffer->cpages, npages);
1442
1443	/* Check what the device saw. */
1444	m = buffer->mirror;
1445	ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);
1446
1447	hmm_buffer_free(buffer);
1448}
1449
1450/*
1451 * Test memory snapshot without faulting in pages accessed by the device.
1452 */
1453TEST_F(hmm2, snapshot)
1454{
1455	struct hmm_buffer *buffer;
1456	unsigned long npages;
1457	unsigned long size;
1458	int *ptr;
1459	unsigned char *p;
1460	unsigned char *m;
1461	int ret;
1462	int val;
1463
1464	npages = 7;
1465	size = npages << self->page_shift;
1466
1467	buffer = malloc(sizeof(*buffer));
1468	ASSERT_NE(buffer, NULL);
1469
1470	buffer->fd = -1;
1471	buffer->size = size;
1472	buffer->mirror = malloc(npages);
1473	ASSERT_NE(buffer->mirror, NULL);
1474
1475	/* Reserve a range of addresses. */
1476	buffer->ptr = mmap(NULL, size,
1477			   PROT_NONE,
1478			   MAP_PRIVATE | MAP_ANONYMOUS,
1479			   buffer->fd, 0);
1480	ASSERT_NE(buffer->ptr, MAP_FAILED);
1481	p = buffer->ptr;
1482
1483	/* Punch a hole after the first page address. */
1484	ret = munmap(buffer->ptr + self->page_size, self->page_size);
1485	ASSERT_EQ(ret, 0);
1486
1487	/* Page 2 will be read-only zero page. */
1488	ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1489				PROT_READ);
1490	ASSERT_EQ(ret, 0);
1491	ptr = (int *)(buffer->ptr + 2 * self->page_size);
1492	val = *ptr + 3;
1493	ASSERT_EQ(val, 3);
1494
1495	/* Page 3 will be read-only. */
1496	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1497				PROT_READ | PROT_WRITE);
1498	ASSERT_EQ(ret, 0);
1499	ptr = (int *)(buffer->ptr + 3 * self->page_size);
1500	*ptr = val;
1501	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1502				PROT_READ);
1503	ASSERT_EQ(ret, 0);
1504
1505	/* Page 4-6 will be read-write. */
1506	ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size,
1507				PROT_READ | PROT_WRITE);
1508	ASSERT_EQ(ret, 0);
1509	ptr = (int *)(buffer->ptr + 4 * self->page_size);
1510	*ptr = val;
1511
1512	/* Page 5 will be migrated to device 0. */
1513	buffer->ptr = p + 5 * self->page_size;
1514	ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1);
1515	ASSERT_EQ(ret, 0);
1516	ASSERT_EQ(buffer->cpages, 1);
1517
1518	/* Page 6 will be migrated to device 1. */
1519	buffer->ptr = p + 6 * self->page_size;
1520	ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1);
1521	ASSERT_EQ(ret, 0);
1522	ASSERT_EQ(buffer->cpages, 1);
1523
1524	/* Simulate a device snapshotting CPU pagetables. */
1525	buffer->ptr = p;
1526	ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1527	ASSERT_EQ(ret, 0);
1528	ASSERT_EQ(buffer->cpages, npages);
1529
1530	/* Check what the device saw. */
1531	m = buffer->mirror;
1532	ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR);
1533	ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR);
1534	ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);
1535	ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);
1536	ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);
1537	if (!hmm_is_coherent_type(variant->device_number0)) {
1538		ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
1539				HMM_DMIRROR_PROT_WRITE);
1540		ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
1541	} else {
1542		ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |
1543				HMM_DMIRROR_PROT_WRITE);
1544		ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |
1545				HMM_DMIRROR_PROT_WRITE);
1546	}
1547
1548	hmm_buffer_free(buffer);
1549}
1550
1551/*
1552 * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
1553 * should be mapped by a large page table entry.
1554 */
1555TEST_F(hmm, compound)
1556{
1557	struct hmm_buffer *buffer;
1558	unsigned long npages;
1559	unsigned long size;
1560	unsigned long default_hsize = default_huge_page_size();
1561	int *ptr;
1562	unsigned char *m;
1563	int ret;
1564	unsigned long i;
1565
1566	/* Skip test if we can't allocate a hugetlbfs page. */
1567
1568	if (!default_hsize)
1569		SKIP(return, "Huge page size could not be determined");
1570
1571	size = ALIGN(TWOMEG, default_hsize);
1572	npages = size >> self->page_shift;
1573
1574	buffer = malloc(sizeof(*buffer));
1575	ASSERT_NE(buffer, NULL);
1576
1577	buffer->ptr = mmap(NULL, size,
1578				   PROT_READ | PROT_WRITE,
1579				   MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
1580				   -1, 0);
1581	if (buffer->ptr == MAP_FAILED) {
1582		free(buffer);
1583		return;
1584	}
1585
1586	buffer->size = size;
1587	buffer->mirror = malloc(npages);
1588	ASSERT_NE(buffer->mirror, NULL);
1589
1590	/* Initialize the pages the device will snapshot in buffer->ptr. */
1591	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1592		ptr[i] = i;
1593
1594	/* Simulate a device snapshotting CPU pagetables. */
1595	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1596	ASSERT_EQ(ret, 0);
1597	ASSERT_EQ(buffer->cpages, npages);
1598
1599	/* Check what the device saw. */
1600	m = buffer->mirror;
1601	for (i = 0; i < npages; ++i)
1602		ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |
1603				HMM_DMIRROR_PROT_PMD);
1604
1605	/* Make the region read-only. */
1606	ret = mprotect(buffer->ptr, size, PROT_READ);
1607	ASSERT_EQ(ret, 0);
1608
1609	/* Simulate a device snapshotting CPU pagetables. */
1610	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1611	ASSERT_EQ(ret, 0);
1612	ASSERT_EQ(buffer->cpages, npages);
1613
1614	/* Check what the device saw. */
1615	m = buffer->mirror;
1616	for (i = 0; i < npages; ++i)
1617		ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
1618				HMM_DMIRROR_PROT_PMD);
1619
1620	munmap(buffer->ptr, buffer->size);
1621	buffer->ptr = NULL;
1622	hmm_buffer_free(buffer);
1623}
1624
1625/*
1626 * Test two devices reading the same memory (double mapped).
1627 */
1628TEST_F(hmm2, double_map)
1629{
1630	struct hmm_buffer *buffer;
1631	unsigned long npages;
1632	unsigned long size;
1633	unsigned long i;
1634	int *ptr;
1635	int ret;
1636
1637	npages = 6;
1638	size = npages << self->page_shift;
1639
1640	buffer = malloc(sizeof(*buffer));
1641	ASSERT_NE(buffer, NULL);
1642
1643	buffer->fd = -1;
1644	buffer->size = size;
1645	buffer->mirror = malloc(size);
1646	ASSERT_NE(buffer->mirror, NULL);
1647
1648	/* Reserve a range of addresses. */
1649	buffer->ptr = mmap(NULL, size,
1650			   PROT_READ | PROT_WRITE,
1651			   MAP_PRIVATE | MAP_ANONYMOUS,
1652			   buffer->fd, 0);
1653	ASSERT_NE(buffer->ptr, MAP_FAILED);
1654
1655	/* Initialize buffer in system memory. */
1656	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1657		ptr[i] = i;
1658
1659	/* Make region read-only. */
1660	ret = mprotect(buffer->ptr, size, PROT_READ);
1661	ASSERT_EQ(ret, 0);
1662
1663	/* Simulate device 0 reading system memory. */
1664	ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1665	ASSERT_EQ(ret, 0);
1666	ASSERT_EQ(buffer->cpages, npages);
1667	ASSERT_EQ(buffer->faults, 1);
1668
1669	/* Check what the device read. */
1670	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1671		ASSERT_EQ(ptr[i], i);
1672
1673	/* Simulate device 1 reading system memory. */
1674	ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages);
1675	ASSERT_EQ(ret, 0);
1676	ASSERT_EQ(buffer->cpages, npages);
1677	ASSERT_EQ(buffer->faults, 1);
1678
1679	/* Check what the device read. */
1680	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1681		ASSERT_EQ(ptr[i], i);
1682
1683	/* Migrate pages to device 1 and try to read from device 0. */
1684	ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages);
1685	ASSERT_EQ(ret, 0);
1686	ASSERT_EQ(buffer->cpages, npages);
1687
1688	ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1689	ASSERT_EQ(ret, 0);
1690	ASSERT_EQ(buffer->cpages, npages);
1691	ASSERT_EQ(buffer->faults, 1);
1692
1693	/* Check what device 0 read. */
1694	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1695		ASSERT_EQ(ptr[i], i);
1696
1697	hmm_buffer_free(buffer);
1698}
1699
1700/*
1701 * Basic check of exclusive faulting.
1702 */
1703TEST_F(hmm, exclusive)
1704{
1705	struct hmm_buffer *buffer;
1706	unsigned long npages;
1707	unsigned long size;
1708	unsigned long i;
1709	int *ptr;
1710	int ret;
1711
1712	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1713	ASSERT_NE(npages, 0);
1714	size = npages << self->page_shift;
1715
1716	buffer = malloc(sizeof(*buffer));
1717	ASSERT_NE(buffer, NULL);
1718
1719	buffer->fd = -1;
1720	buffer->size = size;
1721	buffer->mirror = malloc(size);
1722	ASSERT_NE(buffer->mirror, NULL);
1723
1724	buffer->ptr = mmap(NULL, size,
1725			   PROT_READ | PROT_WRITE,
1726			   MAP_PRIVATE | MAP_ANONYMOUS,
1727			   buffer->fd, 0);
1728	ASSERT_NE(buffer->ptr, MAP_FAILED);
1729
1730	/* Initialize buffer in system memory. */
1731	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1732		ptr[i] = i;
1733
1734	/* Map memory exclusively for device access. */
1735	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1736	ASSERT_EQ(ret, 0);
1737	ASSERT_EQ(buffer->cpages, npages);
1738
1739	/* Check what the device read. */
1740	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1741		ASSERT_EQ(ptr[i], i);
1742
1743	/* Fault pages back to system memory and check them. */
1744	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1745		ASSERT_EQ(ptr[i]++, i);
1746
1747	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1748		ASSERT_EQ(ptr[i], i+1);
1749
1750	/* Check atomic access revoked */
1751	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages);
1752	ASSERT_EQ(ret, 0);
1753
1754	hmm_buffer_free(buffer);
1755}
1756
1757TEST_F(hmm, exclusive_mprotect)
1758{
1759	struct hmm_buffer *buffer;
1760	unsigned long npages;
1761	unsigned long size;
1762	unsigned long i;
1763	int *ptr;
1764	int ret;
1765
1766	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1767	ASSERT_NE(npages, 0);
1768	size = npages << self->page_shift;
1769
1770	buffer = malloc(sizeof(*buffer));
1771	ASSERT_NE(buffer, NULL);
1772
1773	buffer->fd = -1;
1774	buffer->size = size;
1775	buffer->mirror = malloc(size);
1776	ASSERT_NE(buffer->mirror, NULL);
1777
1778	buffer->ptr = mmap(NULL, size,
1779			   PROT_READ | PROT_WRITE,
1780			   MAP_PRIVATE | MAP_ANONYMOUS,
1781			   buffer->fd, 0);
1782	ASSERT_NE(buffer->ptr, MAP_FAILED);
1783
1784	/* Initialize buffer in system memory. */
1785	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1786		ptr[i] = i;
1787
1788	/* Map memory exclusively for device access. */
1789	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1790	ASSERT_EQ(ret, 0);
1791	ASSERT_EQ(buffer->cpages, npages);
1792
1793	/* Check what the device read. */
1794	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1795		ASSERT_EQ(ptr[i], i);
1796
1797	ret = mprotect(buffer->ptr, size, PROT_READ);
1798	ASSERT_EQ(ret, 0);
1799
1800	/* Simulate a device writing system memory. */
1801	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
1802	ASSERT_EQ(ret, -EPERM);
1803
1804	hmm_buffer_free(buffer);
1805}
1806
1807/*
1808 * Check copy-on-write works.
1809 */
1810TEST_F(hmm, exclusive_cow)
1811{
1812	struct hmm_buffer *buffer;
1813	unsigned long npages;
1814	unsigned long size;
1815	unsigned long i;
1816	int *ptr;
1817	int ret;
1818
1819	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1820	ASSERT_NE(npages, 0);
1821	size = npages << self->page_shift;
1822
1823	buffer = malloc(sizeof(*buffer));
1824	ASSERT_NE(buffer, NULL);
1825
1826	buffer->fd = -1;
1827	buffer->size = size;
1828	buffer->mirror = malloc(size);
1829	ASSERT_NE(buffer->mirror, NULL);
1830
1831	buffer->ptr = mmap(NULL, size,
1832			   PROT_READ | PROT_WRITE,
1833			   MAP_PRIVATE | MAP_ANONYMOUS,
1834			   buffer->fd, 0);
1835	ASSERT_NE(buffer->ptr, MAP_FAILED);
1836
1837	/* Initialize buffer in system memory. */
1838	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1839		ptr[i] = i;
1840
1841	/* Map memory exclusively for device access. */
1842	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1843	ASSERT_EQ(ret, 0);
1844	ASSERT_EQ(buffer->cpages, npages);
1845
1846	fork();
1847
1848	/* Fault pages back to system memory and check them. */
1849	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1850		ASSERT_EQ(ptr[i]++, i);
1851
1852	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1853		ASSERT_EQ(ptr[i], i+1);
1854
1855	hmm_buffer_free(buffer);
1856}
1857
1858static int gup_test_exec(int gup_fd, unsigned long addr, int cmd,
1859			 int npages, int size, int flags)
1860{
1861	struct gup_test gup = {
1862		.nr_pages_per_call	= npages,
1863		.addr			= addr,
1864		.gup_flags		= FOLL_WRITE | flags,
1865		.size			= size,
1866	};
1867
1868	if (ioctl(gup_fd, cmd, &gup)) {
1869		perror("ioctl on error\n");
1870		return errno;
1871	}
1872
1873	return 0;
1874}
1875
1876/*
1877 * Test get user device pages through gup_test. Setting PIN_LONGTERM flag.
1878 * This should trigger a migration back to system memory for both, private
1879 * and coherent type pages.
1880 * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added
1881 * to your configuration before you run it.
1882 */
1883TEST_F(hmm, hmm_gup_test)
1884{
1885	struct hmm_buffer *buffer;
1886	int gup_fd;
1887	unsigned long npages;
1888	unsigned long size;
1889	unsigned long i;
1890	int *ptr;
1891	int ret;
1892	unsigned char *m;
1893
1894	gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
1895	if (gup_fd == -1)
1896		SKIP(return, "Skipping test, could not find gup_test driver");
1897
1898	npages = 4;
1899	size = npages << self->page_shift;
1900
1901	buffer = malloc(sizeof(*buffer));
1902	ASSERT_NE(buffer, NULL);
1903
1904	buffer->fd = -1;
1905	buffer->size = size;
1906	buffer->mirror = malloc(size);
1907	ASSERT_NE(buffer->mirror, NULL);
1908
1909	buffer->ptr = mmap(NULL, size,
1910			   PROT_READ | PROT_WRITE,
1911			   MAP_PRIVATE | MAP_ANONYMOUS,
1912			   buffer->fd, 0);
1913	ASSERT_NE(buffer->ptr, MAP_FAILED);
1914
1915	/* Initialize buffer in system memory. */
1916	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1917		ptr[i] = i;
1918
1919	/* Migrate memory to device. */
1920	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
1921	ASSERT_EQ(ret, 0);
1922	ASSERT_EQ(buffer->cpages, npages);
1923	/* Check what the device read. */
1924	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1925		ASSERT_EQ(ptr[i], i);
1926
1927	ASSERT_EQ(gup_test_exec(gup_fd,
1928				(unsigned long)buffer->ptr,
1929				GUP_BASIC_TEST, 1, self->page_size, 0), 0);
1930	ASSERT_EQ(gup_test_exec(gup_fd,
1931				(unsigned long)buffer->ptr + 1 * self->page_size,
1932				GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0);
1933	ASSERT_EQ(gup_test_exec(gup_fd,
1934				(unsigned long)buffer->ptr + 2 * self->page_size,
1935				PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0);
1936	ASSERT_EQ(gup_test_exec(gup_fd,
1937				(unsigned long)buffer->ptr + 3 * self->page_size,
1938				PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0);
1939
1940	/* Take snapshot to CPU pagetables */
1941	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1942	ASSERT_EQ(ret, 0);
1943	ASSERT_EQ(buffer->cpages, npages);
1944	m = buffer->mirror;
1945	if (hmm_is_coherent_type(variant->device_number)) {
1946		ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);
1947		ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);
1948	} else {
1949		ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);
1950		ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);
1951	}
1952	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);
1953	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]);
1954	/*
1955	 * Check again the content on the pages. Make sure there's no
1956	 * corrupted data.
1957	 */
1958	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1959		ASSERT_EQ(ptr[i], i);
1960
1961	close(gup_fd);
1962	hmm_buffer_free(buffer);
1963}
1964
1965/*
1966 * Test copy-on-write in device pages.
1967 * In case of writing to COW private page(s), a page fault will migrate pages
1968 * back to system memory first. Then, these pages will be duplicated. In case
1969 * of COW device coherent type, pages are duplicated directly from device
1970 * memory.
1971 */
1972TEST_F(hmm, hmm_cow_in_device)
1973{
1974	struct hmm_buffer *buffer;
1975	unsigned long npages;
1976	unsigned long size;
1977	unsigned long i;
1978	int *ptr;
1979	int ret;
1980	unsigned char *m;
1981	pid_t pid;
1982	int status;
1983
1984	npages = 4;
1985	size = npages << self->page_shift;
1986
1987	buffer = malloc(sizeof(*buffer));
1988	ASSERT_NE(buffer, NULL);
1989
1990	buffer->fd = -1;
1991	buffer->size = size;
1992	buffer->mirror = malloc(size);
1993	ASSERT_NE(buffer->mirror, NULL);
1994
1995	buffer->ptr = mmap(NULL, size,
1996			   PROT_READ | PROT_WRITE,
1997			   MAP_PRIVATE | MAP_ANONYMOUS,
1998			   buffer->fd, 0);
1999	ASSERT_NE(buffer->ptr, MAP_FAILED);
2000
2001	/* Initialize buffer in system memory. */
2002	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2003		ptr[i] = i;
2004
2005	/* Migrate memory to device. */
2006
2007	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2008	ASSERT_EQ(ret, 0);
2009	ASSERT_EQ(buffer->cpages, npages);
2010
2011	pid = fork();
2012	if (pid == -1)
2013		ASSERT_EQ(pid, 0);
2014	if (!pid) {
2015		/* Child process waits for SIGTERM from the parent. */
2016		while (1) {
2017		}
2018		/* Should not reach this */
2019	}
2020	/* Parent process writes to COW pages(s) and gets a
2021	 * new copy in system. In case of device private pages,
2022	 * this write causes a migration to system mem first.
2023	 */
2024	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2025		ptr[i] = i;
2026
2027	/* Terminate child and wait */
2028	EXPECT_EQ(0, kill(pid, SIGTERM));
2029	EXPECT_EQ(pid, waitpid(pid, &status, 0));
2030	EXPECT_NE(0, WIFSIGNALED(status));
2031	EXPECT_EQ(SIGTERM, WTERMSIG(status));
2032
2033	/* Take snapshot to CPU pagetables */
2034	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
2035	ASSERT_EQ(ret, 0);
2036	ASSERT_EQ(buffer->cpages, npages);
2037	m = buffer->mirror;
2038	for (i = 0; i < npages; i++)
2039		ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]);
2040
2041	hmm_buffer_free(buffer);
2042}
2043
2044/*
2045 * Migrate private anonymous huge empty page.
2046 */
2047TEST_F(hmm, migrate_anon_huge_empty)
2048{
2049	struct hmm_buffer *buffer;
2050	unsigned long npages;
2051	unsigned long size;
2052	unsigned long i;
2053	void *old_ptr;
2054	void *map;
2055	int *ptr;
2056	int ret;
2057
2058	size = read_pmd_pagesize();
2059
2060	buffer = malloc(sizeof(*buffer));
2061	ASSERT_NE(buffer, NULL);
2062
2063	buffer->fd = -1;
2064	buffer->size = 2 * size;
2065	buffer->mirror = malloc(size);
2066	ASSERT_NE(buffer->mirror, NULL);
2067	memset(buffer->mirror, 0xFF, size);
2068
2069	buffer->ptr = mmap(NULL, 2 * size,
2070			   PROT_READ,
2071			   MAP_PRIVATE | MAP_ANONYMOUS,
2072			   buffer->fd, 0);
2073	ASSERT_NE(buffer->ptr, MAP_FAILED);
2074
2075	npages = size >> self->page_shift;
2076	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2077	ret = madvise(map, size, MADV_HUGEPAGE);
2078	ASSERT_EQ(ret, 0);
2079	old_ptr = buffer->ptr;
2080	buffer->ptr = map;
2081
2082	/* Migrate memory to device. */
2083	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2084	ASSERT_EQ(ret, 0);
2085	ASSERT_EQ(buffer->cpages, npages);
2086
2087	/* Check what the device read. */
2088	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2089		ASSERT_EQ(ptr[i], 0);
2090
2091	buffer->ptr = old_ptr;
2092	hmm_buffer_free(buffer);
2093}
2094
2095/*
2096 * Migrate private anonymous huge zero page.
2097 */
2098TEST_F(hmm, migrate_anon_huge_zero)
2099{
2100	struct hmm_buffer *buffer;
2101	unsigned long npages;
2102	unsigned long size;
2103	unsigned long i;
2104	void *old_ptr;
2105	void *map;
2106	int *ptr;
2107	int ret;
2108	int val;
2109
2110	size = read_pmd_pagesize();
2111
2112	buffer = malloc(sizeof(*buffer));
2113	ASSERT_NE(buffer, NULL);
2114
2115	buffer->fd = -1;
2116	buffer->size = 2 * size;
2117	buffer->mirror = malloc(size);
2118	ASSERT_NE(buffer->mirror, NULL);
2119	memset(buffer->mirror, 0xFF, size);
2120
2121	buffer->ptr = mmap(NULL, 2 * size,
2122			   PROT_READ,
2123			   MAP_PRIVATE | MAP_ANONYMOUS,
2124			   buffer->fd, 0);
2125	ASSERT_NE(buffer->ptr, MAP_FAILED);
2126
2127	npages = size >> self->page_shift;
2128	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2129	ret = madvise(map, size, MADV_HUGEPAGE);
2130	ASSERT_EQ(ret, 0);
2131	old_ptr = buffer->ptr;
2132	buffer->ptr = map;
2133
2134	/* Initialize a read-only zero huge page. */
2135	val = *(int *)buffer->ptr;
2136	ASSERT_EQ(val, 0);
2137
2138	/* Migrate memory to device. */
2139	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2140	ASSERT_EQ(ret, 0);
2141	ASSERT_EQ(buffer->cpages, npages);
2142
2143	/* Check what the device read. */
2144	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2145		ASSERT_EQ(ptr[i], 0);
2146
2147	/* Fault pages back to system memory and check them. */
2148	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) {
2149		ASSERT_EQ(ptr[i], 0);
2150		/* If it asserts once, it probably will 500,000 times */
2151		if (ptr[i] != 0)
2152			break;
2153	}
2154
2155	buffer->ptr = old_ptr;
2156	hmm_buffer_free(buffer);
2157}
2158
2159/*
2160 * Migrate private anonymous huge page and free.
2161 */
2162TEST_F(hmm, migrate_anon_huge_free)
2163{
2164	struct hmm_buffer *buffer;
2165	unsigned long npages;
2166	unsigned long size;
2167	unsigned long i;
2168	void *old_ptr;
2169	void *map;
2170	int *ptr;
2171	int ret;
2172
2173	size = read_pmd_pagesize();
2174
2175	buffer = malloc(sizeof(*buffer));
2176	ASSERT_NE(buffer, NULL);
2177
2178	buffer->fd = -1;
2179	buffer->size = 2 * size;
2180	buffer->mirror = malloc(size);
2181	ASSERT_NE(buffer->mirror, NULL);
2182	memset(buffer->mirror, 0xFF, size);
2183
2184	buffer->ptr = mmap(NULL, 2 * size,
2185			   PROT_READ | PROT_WRITE,
2186			   MAP_PRIVATE | MAP_ANONYMOUS,
2187			   buffer->fd, 0);
2188	ASSERT_NE(buffer->ptr, MAP_FAILED);
2189
2190	npages = size >> self->page_shift;
2191	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2192	ret = madvise(map, size, MADV_HUGEPAGE);
2193	ASSERT_EQ(ret, 0);
2194	old_ptr = buffer->ptr;
2195	buffer->ptr = map;
2196
2197	/* Initialize buffer in system memory. */
2198	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2199		ptr[i] = i;
2200
2201	/* Migrate memory to device. */
2202	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2203	ASSERT_EQ(ret, 0);
2204	ASSERT_EQ(buffer->cpages, npages);
2205
2206	/* Check what the device read. */
2207	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2208		ASSERT_EQ(ptr[i], i);
2209
2210	/* Try freeing it. */
2211	ret = madvise(map, size, MADV_FREE);
2212	ASSERT_EQ(ret, 0);
2213
2214	buffer->ptr = old_ptr;
2215	hmm_buffer_free(buffer);
2216}
2217
2218/*
2219 * Migrate private anonymous huge page and fault back to sysmem.
2220 */
2221TEST_F(hmm, migrate_anon_huge_fault)
2222{
2223	struct hmm_buffer *buffer;
2224	unsigned long npages;
2225	unsigned long size;
2226	unsigned long i;
2227	void *old_ptr;
2228	void *map;
2229	int *ptr;
2230	int ret;
2231
2232	size = read_pmd_pagesize();
2233
2234	buffer = malloc(sizeof(*buffer));
2235	ASSERT_NE(buffer, NULL);
2236
2237	buffer->fd = -1;
2238	buffer->size = 2 * size;
2239	buffer->mirror = malloc(size);
2240	ASSERT_NE(buffer->mirror, NULL);
2241	memset(buffer->mirror, 0xFF, size);
2242
2243	buffer->ptr = mmap(NULL, 2 * size,
2244			   PROT_READ | PROT_WRITE,
2245			   MAP_PRIVATE | MAP_ANONYMOUS,
2246			   buffer->fd, 0);
2247	ASSERT_NE(buffer->ptr, MAP_FAILED);
2248
2249	npages = size >> self->page_shift;
2250	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2251	ret = madvise(map, size, MADV_HUGEPAGE);
2252	ASSERT_EQ(ret, 0);
2253	old_ptr = buffer->ptr;
2254	buffer->ptr = map;
2255
2256	/* Initialize buffer in system memory. */
2257	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2258		ptr[i] = i;
2259
2260	/* Migrate memory to device. */
2261	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2262	ASSERT_EQ(ret, 0);
2263	ASSERT_EQ(buffer->cpages, npages);
2264
2265	/* Check what the device read. */
2266	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2267		ASSERT_EQ(ptr[i], i);
2268
2269	/* Fault pages back to system memory and check them. */
2270	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2271		ASSERT_EQ(ptr[i], i);
2272
2273	buffer->ptr = old_ptr;
2274	hmm_buffer_free(buffer);
2275}
2276
2277/*
2278 * Migrate memory and fault back to sysmem after partially unmapping.
2279 */
2280TEST_F(hmm, migrate_partial_unmap_fault)
2281{
2282	struct hmm_buffer *buffer;
2283	unsigned long npages;
2284	unsigned long size = read_pmd_pagesize();
2285	unsigned long i;
2286	void *old_ptr;
2287	void *map;
2288	int *ptr;
2289	int ret, j, use_thp;
2290	int offsets[] = { 0, 512 * ONEKB, ONEMEG };
2291
2292	for (use_thp = 0; use_thp < 2; ++use_thp) {
2293		for (j = 0; j < ARRAY_SIZE(offsets); ++j) {
2294			buffer = malloc(sizeof(*buffer));
2295			ASSERT_NE(buffer, NULL);
2296
2297			buffer->fd = -1;
2298			buffer->size = 2 * size;
2299			buffer->mirror = malloc(size);
2300			ASSERT_NE(buffer->mirror, NULL);
2301			memset(buffer->mirror, 0xFF, size);
2302
2303			buffer->ptr = mmap(NULL, 2 * size,
2304					   PROT_READ | PROT_WRITE,
2305					   MAP_PRIVATE | MAP_ANONYMOUS,
2306					   buffer->fd, 0);
2307			ASSERT_NE(buffer->ptr, MAP_FAILED);
2308
2309			npages = size >> self->page_shift;
2310			map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2311			if (use_thp)
2312				ret = madvise(map, size, MADV_HUGEPAGE);
2313			else
2314				ret = madvise(map, size, MADV_NOHUGEPAGE);
2315			ASSERT_EQ(ret, 0);
2316			old_ptr = buffer->ptr;
2317			buffer->ptr = map;
2318
2319			/* Initialize buffer in system memory. */
2320			for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2321				ptr[i] = i;
2322
2323			/* Migrate memory to device. */
2324			ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2325			ASSERT_EQ(ret, 0);
2326			ASSERT_EQ(buffer->cpages, npages);
2327
2328			/* Check what the device read. */
2329			for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2330				ASSERT_EQ(ptr[i], i);
2331
2332			munmap(buffer->ptr + offsets[j], ONEMEG);
2333
2334			/* Fault pages back to system memory and check them. */
2335			for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2336				if (i * sizeof(int) < offsets[j] ||
2337				    i * sizeof(int) >= offsets[j] + ONEMEG)
2338					ASSERT_EQ(ptr[i], i);
2339
2340			buffer->ptr = old_ptr;
2341			hmm_buffer_free(buffer);
2342		}
2343	}
2344}
2345
2346TEST_F(hmm, migrate_remap_fault)
2347{
2348	struct hmm_buffer *buffer;
2349	unsigned long npages;
2350	unsigned long size = read_pmd_pagesize();
2351	unsigned long i;
2352	void *old_ptr, *new_ptr = NULL;
2353	void *map;
2354	int *ptr;
2355	int ret, j, use_thp, dont_unmap, before;
2356	int offsets[] = { 0, 512 * ONEKB, ONEMEG };
2357
2358	for (before = 0; before < 2; ++before) {
2359		for (dont_unmap = 0; dont_unmap < 2; ++dont_unmap) {
2360			for (use_thp = 0; use_thp < 2; ++use_thp) {
2361				for (j = 0; j < ARRAY_SIZE(offsets); ++j) {
2362					int flags = MREMAP_MAYMOVE | MREMAP_FIXED;
2363
2364					if (dont_unmap)
2365						flags |= MREMAP_DONTUNMAP;
2366
2367					buffer = malloc(sizeof(*buffer));
2368					ASSERT_NE(buffer, NULL);
2369
2370					buffer->fd = -1;
2371					buffer->size = 8 * size;
2372					buffer->mirror = malloc(size);
2373					ASSERT_NE(buffer->mirror, NULL);
2374					memset(buffer->mirror, 0xFF, size);
2375
2376					buffer->ptr = mmap(NULL, buffer->size,
2377							   PROT_READ | PROT_WRITE,
2378							   MAP_PRIVATE | MAP_ANONYMOUS,
2379							   buffer->fd, 0);
2380					ASSERT_NE(buffer->ptr, MAP_FAILED);
2381
2382					npages = size >> self->page_shift;
2383					map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
2384					if (use_thp)
2385						ret = madvise(map, size, MADV_HUGEPAGE);
2386					else
2387						ret = madvise(map, size, MADV_NOHUGEPAGE);
2388					ASSERT_EQ(ret, 0);
2389					old_ptr = buffer->ptr;
2390					munmap(map + size, size * 2);
2391					buffer->ptr = map;
2392
2393					/* Initialize buffer in system memory. */
2394					for (i = 0, ptr = buffer->ptr;
2395					     i < size / sizeof(*ptr); ++i)
2396						ptr[i] = i;
2397
2398					if (before) {
2399						new_ptr = mremap((void *)map, size, size, flags,
2400								 map + size + offsets[j]);
2401						ASSERT_NE(new_ptr, MAP_FAILED);
2402						buffer->ptr = new_ptr;
2403					}
2404
2405					/* Migrate memory to device. */
2406					ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2407					ASSERT_EQ(ret, 0);
2408					ASSERT_EQ(buffer->cpages, npages);
2409
2410					/* Check what the device read. */
2411					for (i = 0, ptr = buffer->mirror;
2412					     i < size / sizeof(*ptr); ++i)
2413						ASSERT_EQ(ptr[i], i);
2414
2415					if (!before) {
2416						new_ptr = mremap((void *)map, size, size, flags,
2417								 map + size + offsets[j]);
2418						ASSERT_NE(new_ptr, MAP_FAILED);
2419						buffer->ptr = new_ptr;
2420					}
2421
2422					/* Fault pages back to system memory and check them. */
2423					for (i = 0, ptr = buffer->ptr;
2424					     i < size / sizeof(*ptr); ++i)
2425						ASSERT_EQ(ptr[i], i);
2426
2427					munmap(new_ptr, size);
2428					buffer->ptr = old_ptr;
2429					hmm_buffer_free(buffer);
2430				}
2431			}
2432		}
2433	}
2434}
2435
2436/*
2437 * Migrate private anonymous huge page with allocation errors.
2438 */
2439TEST_F(hmm, migrate_anon_huge_err)
2440{
2441	struct hmm_buffer *buffer;
2442	unsigned long npages;
2443	unsigned long size;
2444	unsigned long i;
2445	void *old_ptr;
2446	void *map;
2447	int *ptr;
2448	int ret;
2449
2450	size = read_pmd_pagesize();
2451
2452	buffer = malloc(sizeof(*buffer));
2453	ASSERT_NE(buffer, NULL);
2454
2455	buffer->fd = -1;
2456	buffer->size = 2 * size;
2457	buffer->mirror = malloc(2 * size);
2458	ASSERT_NE(buffer->mirror, NULL);
2459	memset(buffer->mirror, 0xFF, 2 * size);
2460
2461	old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE,
2462			MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2463	ASSERT_NE(old_ptr, MAP_FAILED);
2464
2465	npages = size >> self->page_shift;
2466	map = (void *)ALIGN((uintptr_t)old_ptr, size);
2467	ret = madvise(map, size, MADV_HUGEPAGE);
2468	ASSERT_EQ(ret, 0);
2469	buffer->ptr = map;
2470
2471	/* Initialize buffer in system memory. */
2472	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2473		ptr[i] = i;
2474
2475	/* Migrate memory to device but force a THP allocation error. */
2476	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2477			      HMM_DMIRROR_FLAG_FAIL_ALLOC);
2478	ASSERT_EQ(ret, 0);
2479	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2480	ASSERT_EQ(ret, 0);
2481	ASSERT_EQ(buffer->cpages, npages);
2482
2483	/* Check what the device read. */
2484	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) {
2485		ASSERT_EQ(ptr[i], i);
2486		if (ptr[i] != i)
2487			break;
2488	}
2489
2490	/* Try faulting back a single (PAGE_SIZE) page. */
2491	ptr = buffer->ptr;
2492	ASSERT_EQ(ptr[2048], 2048);
2493
2494	/* unmap and remap the region to reset things. */
2495	ret = munmap(old_ptr, 2 * size);
2496	ASSERT_EQ(ret, 0);
2497	old_ptr = mmap(NULL, 2 * size, PROT_READ | PROT_WRITE,
2498			MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2499	ASSERT_NE(old_ptr, MAP_FAILED);
2500	map = (void *)ALIGN((uintptr_t)old_ptr, size);
2501	ret = madvise(map, size, MADV_HUGEPAGE);
2502	ASSERT_EQ(ret, 0);
2503	buffer->ptr = map;
2504
2505	/* Initialize buffer in system memory. */
2506	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2507		ptr[i] = i;
2508
2509	/* Migrate THP to device. */
2510	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2511	ASSERT_EQ(ret, 0);
2512	ASSERT_EQ(buffer->cpages, npages);
2513
2514	/*
2515	 * Force an allocation error when faulting back a THP resident in the
2516	 * device.
2517	 */
2518	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2519			      HMM_DMIRROR_FLAG_FAIL_ALLOC);
2520	ASSERT_EQ(ret, 0);
2521
2522	ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages);
2523	ASSERT_EQ(ret, 0);
2524	ptr = buffer->ptr;
2525	ASSERT_EQ(ptr[2048], 2048);
2526
2527	buffer->ptr = old_ptr;
2528	hmm_buffer_free(buffer);
2529}
2530
2531/*
2532 * Migrate private anonymous huge zero page with allocation errors.
2533 */
2534TEST_F(hmm, migrate_anon_huge_zero_err)
2535{
2536	struct hmm_buffer *buffer;
2537	unsigned long npages;
2538	unsigned long size;
2539	unsigned long i;
2540	void *old_ptr;
2541	void *map;
2542	int *ptr;
2543	int ret;
2544
2545	size = read_pmd_pagesize();
2546
2547	buffer = malloc(sizeof(*buffer));
2548	ASSERT_NE(buffer, NULL);
2549
2550	buffer->fd = -1;
2551	buffer->size = 2 * size;
2552	buffer->mirror = malloc(2 * size);
2553	ASSERT_NE(buffer->mirror, NULL);
2554	memset(buffer->mirror, 0xFF, 2 * size);
2555
2556	old_ptr = mmap(NULL, 2 * size, PROT_READ,
2557			MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2558	ASSERT_NE(old_ptr, MAP_FAILED);
2559
2560	npages = size >> self->page_shift;
2561	map = (void *)ALIGN((uintptr_t)old_ptr, size);
2562	ret = madvise(map, size, MADV_HUGEPAGE);
2563	ASSERT_EQ(ret, 0);
2564	buffer->ptr = map;
2565
2566	/* Migrate memory to device but force a THP allocation error. */
2567	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2568			      HMM_DMIRROR_FLAG_FAIL_ALLOC);
2569	ASSERT_EQ(ret, 0);
2570	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2571	ASSERT_EQ(ret, 0);
2572	ASSERT_EQ(buffer->cpages, npages);
2573
2574	/* Check what the device read. */
2575	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
2576		ASSERT_EQ(ptr[i], 0);
2577
2578	/* Try faulting back a single (PAGE_SIZE) page. */
2579	ptr = buffer->ptr;
2580	ASSERT_EQ(ptr[2048], 0);
2581
2582	/* unmap and remap the region to reset things. */
2583	ret = munmap(old_ptr, 2 * size);
2584	ASSERT_EQ(ret, 0);
2585	old_ptr = mmap(NULL, 2 * size, PROT_READ,
2586			MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
2587	ASSERT_NE(old_ptr, MAP_FAILED);
2588	map = (void *)ALIGN((uintptr_t)old_ptr, size);
2589	ret = madvise(map, size, MADV_HUGEPAGE);
2590	ASSERT_EQ(ret, 0);
2591	buffer->ptr = map;
2592
2593	/* Initialize buffer in system memory (zero THP page). */
2594	ret = ptr[0];
2595	ASSERT_EQ(ret, 0);
2596
2597	/* Migrate memory to device but force a THP allocation error. */
2598	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_FLAGS, buffer,
2599			      HMM_DMIRROR_FLAG_FAIL_ALLOC);
2600	ASSERT_EQ(ret, 0);
2601	ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages);
2602	ASSERT_EQ(ret, 0);
2603	ASSERT_EQ(buffer->cpages, npages);
2604
2605	/* Fault the device memory back and check it. */
2606	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2607		ASSERT_EQ(ptr[i], 0);
2608
2609	buffer->ptr = old_ptr;
2610	hmm_buffer_free(buffer);
2611}
2612
2613struct benchmark_results {
2614	double sys_to_dev_time;
2615	double dev_to_sys_time;
2616	double throughput_s2d;
2617	double throughput_d2s;
2618};
2619
2620static double get_time_ms(void)
2621{
2622	struct timeval tv;
2623
2624	gettimeofday(&tv, NULL);
2625	return (tv.tv_sec * 1000.0) + (tv.tv_usec / 1000.0);
2626}
2627
2628static inline struct hmm_buffer *hmm_buffer_alloc(unsigned long size)
2629{
2630	struct hmm_buffer *buffer;
2631
2632	buffer = malloc(sizeof(*buffer));
2633
2634	buffer->fd = -1;
2635	buffer->size = size;
2636	buffer->mirror = malloc(size);
2637	memset(buffer->mirror, 0xFF, size);
2638	return buffer;
2639}
2640
2641static void print_benchmark_results(const char *test_name, size_t buffer_size,
2642				     struct benchmark_results *thp,
2643				     struct benchmark_results *regular)
2644{
2645	double s2d_improvement = ((regular->sys_to_dev_time - thp->sys_to_dev_time) /
2646				 regular->sys_to_dev_time) * 100.0;
2647	double d2s_improvement = ((regular->dev_to_sys_time - thp->dev_to_sys_time) /
2648				 regular->dev_to_sys_time) * 100.0;
2649	double throughput_s2d_improvement = ((thp->throughput_s2d - regular->throughput_s2d) /
2650					    regular->throughput_s2d) * 100.0;
2651	double throughput_d2s_improvement = ((thp->throughput_d2s - regular->throughput_d2s) /
2652					    regular->throughput_d2s) * 100.0;
2653
2654	printf("\n=== %s (%.1f MB) ===\n", test_name, buffer_size / (1024.0 * 1024.0));
2655	printf("                     | With THP        | Without THP     | Improvement\n");
2656	printf("---------------------------------------------------------------------\n");
2657	printf("Sys->Dev Migration   | %.3f ms        | %.3f ms        | %.1f%%\n",
2658	       thp->sys_to_dev_time, regular->sys_to_dev_time, s2d_improvement);
2659	printf("Dev->Sys Migration   | %.3f ms        | %.3f ms        | %.1f%%\n",
2660	       thp->dev_to_sys_time, regular->dev_to_sys_time, d2s_improvement);
2661	printf("S->D Throughput      | %.2f GB/s      | %.2f GB/s      | %.1f%%\n",
2662	       thp->throughput_s2d, regular->throughput_s2d, throughput_s2d_improvement);
2663	printf("D->S Throughput      | %.2f GB/s      | %.2f GB/s      | %.1f%%\n",
2664	       thp->throughput_d2s, regular->throughput_d2s, throughput_d2s_improvement);
2665}
2666
2667/*
2668 * Run a single migration benchmark
2669 * fd: file descriptor for hmm device
2670 * use_thp: whether to use THP
2671 * buffer_size: size of buffer to allocate
2672 * iterations: number of iterations
2673 * results: where to store results
2674 */
2675static inline int run_migration_benchmark(int fd, int use_thp, size_t buffer_size,
2676					   int iterations, struct benchmark_results *results)
2677{
2678	struct hmm_buffer *buffer;
2679	unsigned long npages = buffer_size / sysconf(_SC_PAGESIZE);
2680	double start, end;
2681	double s2d_total = 0, d2s_total = 0;
2682	int ret, i;
2683	int *ptr;
2684
2685	buffer = hmm_buffer_alloc(buffer_size);
2686
2687	/* Map memory */
2688	buffer->ptr = mmap(NULL, buffer_size, PROT_READ | PROT_WRITE,
2689			  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
2690
2691	if (!buffer->ptr)
2692		return -1;
2693
2694	/* Apply THP hint if requested */
2695	if (use_thp)
2696		ret = madvise(buffer->ptr, buffer_size, MADV_HUGEPAGE);
2697	else
2698		ret = madvise(buffer->ptr, buffer_size, MADV_NOHUGEPAGE);
2699
2700	if (ret)
2701		return ret;
2702
2703	/* Initialize memory to make sure pages are allocated */
2704	ptr = (int *)buffer->ptr;
2705	for (i = 0; i < buffer_size / sizeof(int); i++)
2706		ptr[i] = i & 0xFF;
2707
2708	/* Warmup iteration */
2709	ret = hmm_migrate_sys_to_dev(fd, buffer, npages);
2710	if (ret)
2711		return ret;
2712
2713	ret = hmm_migrate_dev_to_sys(fd, buffer, npages);
2714	if (ret)
2715		return ret;
2716
2717	/* Benchmark iterations */
2718	for (i = 0; i < iterations; i++) {
2719		/* System to device migration */
2720		start = get_time_ms();
2721
2722		ret = hmm_migrate_sys_to_dev(fd, buffer, npages);
2723		if (ret)
2724			return ret;
2725
2726		end = get_time_ms();
2727		s2d_total += (end - start);
2728
2729		/* Device to system migration */
2730		start = get_time_ms();
2731
2732		ret = hmm_migrate_dev_to_sys(fd, buffer, npages);
2733		if (ret)
2734			return ret;
2735
2736		end = get_time_ms();
2737		d2s_total += (end - start);
2738	}
2739
2740	/* Calculate average times and throughput */
2741	results->sys_to_dev_time = s2d_total / iterations;
2742	results->dev_to_sys_time = d2s_total / iterations;
2743	results->throughput_s2d = (buffer_size / (1024.0 * 1024.0 * 1024.0)) /
2744				 (results->sys_to_dev_time / 1000.0);
2745	results->throughput_d2s = (buffer_size / (1024.0 * 1024.0 * 1024.0)) /
2746				 (results->dev_to_sys_time / 1000.0);
2747
2748	/* Cleanup */
2749	hmm_buffer_free(buffer);
2750	return 0;
2751}
2752
2753/*
2754 * Benchmark THP migration with different buffer sizes
2755 */
2756TEST_F_TIMEOUT(hmm, benchmark_thp_migration, 120)
2757{
2758	struct benchmark_results thp_results, regular_results;
2759	size_t thp_size = 2 * 1024 * 1024; /* 2MB - typical THP size */
2760	int iterations = 5;
2761
2762	printf("\nHMM THP Migration Benchmark\n");
2763	printf("---------------------------\n");
2764	printf("System page size: %ld bytes\n", sysconf(_SC_PAGESIZE));
2765
2766	/* Test different buffer sizes */
2767	size_t test_sizes[] = {
2768		thp_size / 4,      /* 512KB - smaller than THP */
2769		thp_size / 2,      /* 1MB - half THP */
2770		thp_size,          /* 2MB - single THP */
2771		thp_size * 2,      /* 4MB - two THPs */
2772		thp_size * 4,      /* 8MB - four THPs */
2773		thp_size * 8,       /* 16MB - eight THPs */
2774		thp_size * 128,       /* 256MB - one twenty eight THPs */
2775	};
2776
2777	static const char *const test_names[] = {
2778		"Small Buffer (512KB)",
2779		"Half THP Size (1MB)",
2780		"Single THP Size (2MB)",
2781		"Two THP Size (4MB)",
2782		"Four THP Size (8MB)",
2783		"Eight THP Size (16MB)",
2784		"One twenty eight THP Size (256MB)"
2785	};
2786
2787	int num_tests = ARRAY_SIZE(test_sizes);
2788
2789	/* Run all tests */
2790	for (int i = 0; i < num_tests; i++) {
2791		/* Test with THP */
2792		ASSERT_EQ(run_migration_benchmark(self->fd, 1, test_sizes[i],
2793					iterations, &thp_results), 0);
2794
2795		/* Test without THP */
2796		ASSERT_EQ(run_migration_benchmark(self->fd, 0, test_sizes[i],
2797					iterations, &regular_results), 0);
2798
2799		/* Print results */
2800		print_benchmark_results(test_names[i], test_sizes[i],
2801					&thp_results, &regular_results);
2802	}
2803}
2804TEST_HARNESS_MAIN