protection_keys.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Tests Memory Protection Keys (see Documentation/core-api/protection-keys.rst)
   4 *
   5 * There are examples in here of:
   6 *  * how to set protection keys on memory
   7 *  * how to set/clear bits in pkey registers (the rights register)
   8 *  * how to handle SEGV_PKUERR signals and extract pkey-relevant
   9 *    information from the siginfo
  10 *
  11 * Things to add:
  12 *	make sure KSM and KSM COW breaking works
  13 *	prefault pages in at malloc, or not
  14 *	protect MPX bounds tables with protection keys?
  15 *	make sure VMA splitting/merging is working correctly
  16 *	OOMs can destroy mm->mmap (see exit_mmap()), so make sure it is immune to pkeys
  17 *	look for pkey "leaks" where it is still set on a VMA but "freed" back to the kernel
  18 *	do a plain mprotect() to a mprotect_pkey() area and make sure the pkey sticks
  19 *
  20 * Compile like this:
  21 *	gcc -mxsave      -o protection_keys    -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
  22 *	gcc -mxsave -m32 -o protection_keys_32 -O2 -g -std=gnu99 -pthread -Wall protection_keys.c -lrt -ldl -lm
  23 */
  24#define _GNU_SOURCE
  25#define __SANE_USERSPACE_TYPES__
  26#include <errno.h>
  27#include <linux/elf.h>
  28#include <linux/futex.h>
  29#include <time.h>
  30#include <sys/time.h>
  31#include <sys/syscall.h>
  32#include <string.h>
  33#include <stdio.h>
  34#include <stdint.h>
  35#include <stdbool.h>
  36#include <signal.h>
  37#include <assert.h>
  38#include <stdlib.h>
  39#include <ucontext.h>
  40#include <sys/mman.h>
  41#include <sys/types.h>
  42#include <sys/wait.h>
  43#include <sys/stat.h>
  44#include <fcntl.h>
  45#include <unistd.h>
  46#include <sys/ptrace.h>
  47#include <setjmp.h>
  48
  49#include "pkey-helpers.h"
  50
  51int iteration_nr = 1;
  52int test_nr;
  53
  54u64 shadow_pkey_reg;
  55int dprint_in_signal;
  56
  57noinline int read_ptr(int *ptr)
  58{
  59	/* Keep GCC from optimizing this away somehow */
  60	barrier();
  61	return *ptr;
  62}
  63
  64static void cat_into_file(char *str, char *file)
  65{
  66	int fd = open(file, O_RDWR);
  67	int ret;
  68
  69	dprintf2("%s(): writing '%s' to '%s'\n", __func__, str, file);
  70	/*
  71	 * these need to be raw because they are called under
  72	 * pkey_assert()
  73	 */
  74	if (fd < 0) {
  75		fprintf(stderr, "error opening '%s'\n", str);
  76		perror("error: ");
  77		exit(__LINE__);
  78	}
  79
  80	ret = write(fd, str, strlen(str));
  81	if (ret != strlen(str)) {
  82		perror("write to file failed");
  83		fprintf(stderr, "filename: '%s' str: '%s'\n", file, str);
  84		exit(__LINE__);
  85	}
  86	close(fd);
  87}
  88
  89#if CONTROL_TRACING > 0
  90static int warned_tracing;
  91static int tracing_root_ok(void)
  92{
  93	if (geteuid() != 0) {
  94		if (!warned_tracing)
  95			fprintf(stderr, "WARNING: not run as root, "
  96					"can not do tracing control\n");
  97		warned_tracing = 1;
  98		return 0;
  99	}
 100	return 1;
 101}
 102#endif
 103
 104static void tracing_on(void)
 105{
 106#if CONTROL_TRACING > 0
 107#define TRACEDIR "/sys/kernel/tracing"
 108	char pidstr[32];
 109
 110	if (!tracing_root_ok())
 111		return;
 112
 113	sprintf(pidstr, "%d", getpid());
 114	cat_into_file("0", TRACEDIR "/tracing_on");
 115	cat_into_file("\n", TRACEDIR "/trace");
 116	if (1) {
 117		cat_into_file("function_graph", TRACEDIR "/current_tracer");
 118		cat_into_file("1", TRACEDIR "/options/funcgraph-proc");
 119	} else {
 120		cat_into_file("nop", TRACEDIR "/current_tracer");
 121	}
 122	cat_into_file(pidstr, TRACEDIR "/set_ftrace_pid");
 123	cat_into_file("1", TRACEDIR "/tracing_on");
 124	dprintf1("enabled tracing\n");
 125#endif
 126}
 127
 128static void tracing_off(void)
 129{
 130#if CONTROL_TRACING > 0
 131	if (!tracing_root_ok())
 132		return;
 133	cat_into_file("0", "/sys/kernel/tracing/tracing_on");
 134#endif
 135}
 136
 137void abort_hooks(void)
 138{
 139	fprintf(stderr, "running %s()...\n", __func__);
 140	tracing_off();
 141#ifdef SLEEP_ON_ABORT
 142	sleep(SLEEP_ON_ABORT);
 143#endif
 144}
 145
 146/*
 147 * This attempts to have roughly a page of instructions followed by a few
 148 * instructions that do a write, and another page of instructions.  That
 149 * way, we are pretty sure that the write is in the second page of
 150 * instructions and has at least a page of padding behind it.
 151 *
 152 * *That* lets us be sure to madvise() away the write instruction, which
 153 * will then fault, which makes sure that the fault code handles
 154 * execute-only memory properly.
 155 */
 156#if defined(__powerpc64__) || defined(__aarch64__)
 157/* This way, both 4K and 64K alignment are maintained */
 158__attribute__((__aligned__(65536)))
 159#else
 160__attribute__((__aligned__(PAGE_SIZE)))
 161#endif
 162static void lots_o_noops_around_write(int *write_to_me)
 163{
 164	dprintf3("running %s()\n", __func__);
 165	__page_o_noops();
 166	/* Assume this happens in the second page of instructions: */
 167	*write_to_me = __LINE__;
 168	/* pad out by another page: */
 169	__page_o_noops();
 170	dprintf3("%s() done\n", __func__);
 171}
 172
 173static void dump_mem(void *dumpme, int len_bytes)
 174{
 175	char *c = (void *)dumpme;
 176	int i;
 177
 178	for (i = 0; i < len_bytes; i += sizeof(u64)) {
 179		u64 *ptr = (u64 *)(c + i);
 180		dprintf1("dump[%03d][@%p]: %016llx\n", i, ptr, *ptr);
 181	}
 182}
 183
 184static u32 hw_pkey_get(int pkey, unsigned long flags)
 185{
 186	u64 pkey_reg = __read_pkey_reg();
 187
 188	dprintf1("%s(pkey=%d, flags=%lx) = %x / %d\n",
 189			__func__, pkey, flags, 0, 0);
 190	dprintf2("%s() raw pkey_reg: %016llx\n", __func__, pkey_reg);
 191
 192	return (u32) get_pkey_bits(pkey_reg, pkey);
 193}
 194
 195static int hw_pkey_set(int pkey, unsigned long rights, unsigned long flags)
 196{
 197	u32 mask = (PKEY_DISABLE_ACCESS|PKEY_DISABLE_WRITE);
 198	u64 old_pkey_reg = __read_pkey_reg();
 199	u64 new_pkey_reg;
 200
 201	/* make sure that 'rights' only contains the bits we expect: */
 202	assert(!(rights & ~mask));
 203
 204	/* modify bits accordingly in old pkey_reg and assign it */
 205	new_pkey_reg = set_pkey_bits(old_pkey_reg, pkey, rights);
 206
 207	__write_pkey_reg(new_pkey_reg);
 208
 209	dprintf3("%s(pkey=%d, rights=%lx, flags=%lx) = %x"
 210		" pkey_reg now: %016llx old_pkey_reg: %016llx\n",
 211		__func__, pkey, rights, flags, 0, __read_pkey_reg(),
 212		old_pkey_reg);
 213	return 0;
 214}
 215
 216static void pkey_disable_set(int pkey, int flags)
 217{
 218	unsigned long syscall_flags = 0;
 219	int ret;
 220	int pkey_rights;
 221
 222	dprintf1("START->%s(%d, 0x%x)\n", __func__,
 223		pkey, flags);
 224	pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
 225
 226	pkey_rights = hw_pkey_get(pkey, syscall_flags);
 227
 228	dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
 229			pkey, pkey, pkey_rights);
 230
 231	pkey_assert(pkey_rights >= 0);
 232
 233	pkey_rights |= flags;
 234
 235	ret = hw_pkey_set(pkey, pkey_rights, syscall_flags);
 236	assert(!ret);
 237	/* pkey_reg and flags have the same format */
 238	shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, pkey, pkey_rights);
 239	dprintf1("%s(%d) shadow: 0x%016llx\n",
 240		__func__, pkey, shadow_pkey_reg);
 241
 242	pkey_assert(ret >= 0);
 243
 244	pkey_rights = hw_pkey_get(pkey, syscall_flags);
 245	dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
 246			pkey, pkey, pkey_rights);
 247
 248	dprintf1("%s(%d) pkey_reg: 0x%016llx\n",
 249		__func__, pkey, read_pkey_reg());
 250	dprintf1("END<---%s(%d, 0x%x)\n", __func__,
 251		pkey, flags);
 252}
 253
 254static void pkey_disable_clear(int pkey, int flags)
 255{
 256	unsigned long syscall_flags = 0;
 257	int ret;
 258	int pkey_rights = hw_pkey_get(pkey, syscall_flags);
 259
 260	pkey_assert(flags & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE));
 261
 262	dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
 263			pkey, pkey, pkey_rights);
 264	pkey_assert(pkey_rights >= 0);
 265
 266	pkey_rights &= ~flags;
 267
 268	ret = hw_pkey_set(pkey, pkey_rights, 0);
 269	shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, pkey, pkey_rights);
 270	pkey_assert(ret >= 0);
 271
 272	pkey_rights = hw_pkey_get(pkey, syscall_flags);
 273	dprintf1("%s(%d) hw_pkey_get(%d): %x\n", __func__,
 274			pkey, pkey, pkey_rights);
 275
 276	dprintf1("%s(%d) pkey_reg: 0x%016llx\n", __func__,
 277			pkey, read_pkey_reg());
 278}
 279
 280__maybe_unused static void pkey_write_allow(int pkey)
 281{
 282	pkey_disable_clear(pkey, PKEY_DISABLE_WRITE);
 283}
 284__maybe_unused static void pkey_write_deny(int pkey)
 285{
 286	pkey_disable_set(pkey, PKEY_DISABLE_WRITE);
 287}
 288__maybe_unused static void pkey_access_allow(int pkey)
 289{
 290	pkey_disable_clear(pkey, PKEY_DISABLE_ACCESS);
 291}
 292__maybe_unused static void pkey_access_deny(int pkey)
 293{
 294	pkey_disable_set(pkey, PKEY_DISABLE_ACCESS);
 295}
 296
 297static char *si_code_str(int si_code)
 298{
 299	if (si_code == SEGV_MAPERR)
 300		return "SEGV_MAPERR";
 301	if (si_code == SEGV_ACCERR)
 302		return "SEGV_ACCERR";
 303	if (si_code == SEGV_BNDERR)
 304		return "SEGV_BNDERR";
 305	if (si_code == SEGV_PKUERR)
 306		return "SEGV_PKUERR";
 307	return "UNKNOWN";
 308}
 309
 310static int pkey_faults;
 311static int last_si_pkey = -1;
 312static void signal_handler(int signum, siginfo_t *si, void *vucontext)
 313{
 314	ucontext_t *uctxt = vucontext;
 315	int trapno;
 316	unsigned long ip;
 317#ifdef MCONTEXT_FPREGS
 318	char *fpregs;
 319#endif
 320#if defined(__i386__) || defined(__x86_64__) /* arch */
 321	u32 *pkey_reg_ptr;
 322	int pkey_reg_offset;
 323#endif /* arch */
 324	u64 siginfo_pkey;
 325	u32 *si_pkey_ptr;
 326
 327	dprint_in_signal = 1;
 328	dprintf1(">>>>===============SIGSEGV============================\n");
 329	dprintf1("%s()::%d, pkey_reg: 0x%016llx shadow: %016llx\n",
 330			__func__, __LINE__,
 331			__read_pkey_reg(), shadow_pkey_reg);
 332
 333	trapno = MCONTEXT_TRAPNO(uctxt->uc_mcontext);
 334	ip = MCONTEXT_IP(uctxt->uc_mcontext);
 335#ifdef MCONTEXT_FPREGS
 336	fpregs = (char *) uctxt->uc_mcontext.fpregs;
 337#endif
 338
 339	dprintf2("%s() trapno: %d ip: 0x%016lx info->si_code: %s/%d\n",
 340			__func__, trapno, ip, si_code_str(si->si_code),
 341			si->si_code);
 342
 343#if defined(__i386__) || defined(__x86_64__) /* arch */
 344#ifdef __i386__
 345	/*
 346	 * 32-bit has some extra padding so that userspace can tell whether
 347	 * the XSTATE header is present in addition to the "legacy" FPU
 348	 * state.  We just assume that it is here.
 349	 */
 350	fpregs += 0x70;
 351#endif /* i386 */
 352	pkey_reg_offset = pkey_reg_xstate_offset();
 353	pkey_reg_ptr = (void *)(&fpregs[pkey_reg_offset]);
 354
 355	/*
 356	 * If we got a PKEY fault, we *HAVE* to have at least one bit set in
 357	 * here.
 358	 */
 359	dprintf1("pkey_reg_xstate_offset: %d\n", pkey_reg_xstate_offset());
 360	if (DEBUG_LEVEL > 4)
 361		dump_mem(pkey_reg_ptr - 128, 256);
 362	pkey_assert(*pkey_reg_ptr);
 363#endif /* arch */
 364
 365	dprintf1("siginfo: %p\n", si);
 366#ifdef MCONTEXT_FPREGS
 367	dprintf1(" fpregs: %p\n", fpregs);
 368#endif
 369
 370	if ((si->si_code == SEGV_MAPERR) ||
 371	    (si->si_code == SEGV_ACCERR) ||
 372	    (si->si_code == SEGV_BNDERR)) {
 373		printf("non-PK si_code, exiting...\n");
 374		exit(4);
 375	}
 376
 377	si_pkey_ptr = siginfo_get_pkey_ptr(si);
 378	dprintf1("si_pkey_ptr: %p\n", si_pkey_ptr);
 379	dump_mem((u8 *)si_pkey_ptr - 8, 24);
 380	siginfo_pkey = *si_pkey_ptr;
 381	pkey_assert(siginfo_pkey < NR_PKEYS);
 382	last_si_pkey = siginfo_pkey;
 383
 384	/*
 385	 * need __read_pkey_reg() version so we do not do shadow_pkey_reg
 386	 * checking
 387	 */
 388	dprintf1("signal pkey_reg from  pkey_reg: %016llx\n",
 389			__read_pkey_reg());
 390	dprintf1("pkey from siginfo: %016llx\n", siginfo_pkey);
 391#if defined(__i386__) || defined(__x86_64__) /* arch */
 392	dprintf1("signal pkey_reg from xsave: %08x\n", *pkey_reg_ptr);
 393	*(u64 *)pkey_reg_ptr = 0x00000000;
 394	dprintf1("WARNING: set PKEY_REG=0 to allow faulting instruction to continue\n");
 395#elif defined(__powerpc64__) /* arch */
 396	/* restore access and let the faulting instruction continue */
 397	pkey_access_allow(siginfo_pkey);
 398#elif defined(__aarch64__)
 399	aarch64_write_signal_pkey(uctxt, PKEY_REG_ALLOW_ALL);
 400#endif /* arch */
 401	pkey_faults++;
 402	dprintf1("<<<<==================================================\n");
 403	dprint_in_signal = 0;
 404}
 405
 406static void sig_chld(int x)
 407{
 408	dprint_in_signal = 1;
 409	dprintf2("[%d] SIGCHLD: %d\n", getpid(), x);
 410	dprint_in_signal = 0;
 411}
 412
 413static void setup_sigsegv_handler(void)
 414{
 415	int r, rs;
 416	struct sigaction newact;
 417	struct sigaction oldact;
 418
 419	/* #PF is mapped to sigsegv */
 420	int signum  = SIGSEGV;
 421
 422	newact.sa_handler = 0;
 423	newact.sa_sigaction = signal_handler;
 424
 425	/*sigset_t - signals to block while in the handler */
 426	/* get the old signal mask. */
 427	rs = sigprocmask(SIG_SETMASK, 0, &newact.sa_mask);
 428	pkey_assert(rs == 0);
 429
 430	/* call sa_sigaction, not sa_handler*/
 431	newact.sa_flags = SA_SIGINFO;
 432
 433	newact.sa_restorer = 0;  /* void(*)(), obsolete */
 434	r = sigaction(signum, &newact, &oldact);
 435	r = sigaction(SIGALRM, &newact, &oldact);
 436	pkey_assert(r == 0);
 437}
 438
 439static void setup_handlers(void)
 440{
 441	signal(SIGCHLD, &sig_chld);
 442	setup_sigsegv_handler();
 443}
 444
 445static pid_t fork_lazy_child(void)
 446{
 447	pid_t forkret;
 448
 449	forkret = fork();
 450	pkey_assert(forkret >= 0);
 451	dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
 452
 453	if (!forkret) {
 454		/* in the child */
 455		while (1) {
 456			dprintf1("child sleeping...\n");
 457			sleep(30);
 458		}
 459	}
 460	return forkret;
 461}
 462
 463static int alloc_pkey(void)
 464{
 465	int ret;
 466	unsigned long init_val = PKEY_UNRESTRICTED;
 467
 468	dprintf1("%s()::%d, pkey_reg: 0x%016llx shadow: %016llx\n",
 469			__func__, __LINE__, __read_pkey_reg(), shadow_pkey_reg);
 470	ret = sys_pkey_alloc(0, init_val);
 471	/*
 472	 * pkey_alloc() sets PKEY register, so we need to reflect it in
 473	 * shadow_pkey_reg:
 474	 */
 475	dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 476			" shadow: 0x%016llx\n",
 477			__func__, __LINE__, ret, __read_pkey_reg(),
 478			shadow_pkey_reg);
 479	if (ret > 0) {
 480		/* clear both the bits: */
 481		shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, ret,
 482						~PKEY_MASK);
 483		dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 484				" shadow: 0x%016llx\n",
 485				__func__,
 486				__LINE__, ret, __read_pkey_reg(),
 487				shadow_pkey_reg);
 488		/*
 489		 * move the new state in from init_val
 490		 * (remember, we cheated and init_val == pkey_reg format)
 491		 */
 492		shadow_pkey_reg = set_pkey_bits(shadow_pkey_reg, ret,
 493						init_val);
 494	}
 495	dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 496			" shadow: 0x%016llx\n",
 497			__func__, __LINE__, ret, __read_pkey_reg(),
 498			shadow_pkey_reg);
 499	dprintf1("%s()::%d errno: %d\n", __func__, __LINE__, errno);
 500	/* for shadow checking: */
 501	read_pkey_reg();
 502	dprintf4("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 503		 " shadow: 0x%016llx\n",
 504		__func__, __LINE__, ret, __read_pkey_reg(),
 505		shadow_pkey_reg);
 506	return ret;
 507}
 508
 509/*
 510 * I had a bug where pkey bits could be set by mprotect() but
 511 * not cleared.  This ensures we get lots of random bit sets
 512 * and clears on the vma and pte pkey bits.
 513 */
 514static int alloc_random_pkey(void)
 515{
 516	int max_nr_pkey_allocs;
 517	int ret;
 518	int i;
 519	int alloced_pkeys[NR_PKEYS];
 520	int nr_alloced = 0;
 521	int random_index;
 522	memset(alloced_pkeys, 0, sizeof(alloced_pkeys));
 523
 524	/* allocate every possible key and make a note of which ones we got */
 525	max_nr_pkey_allocs = NR_PKEYS;
 526	for (i = 0; i < max_nr_pkey_allocs; i++) {
 527		int new_pkey = alloc_pkey();
 528		if (new_pkey < 0)
 529			break;
 530		alloced_pkeys[nr_alloced++] = new_pkey;
 531	}
 532
 533	pkey_assert(nr_alloced > 0);
 534	/* select a random one out of the allocated ones */
 535	random_index = rand() % nr_alloced;
 536	ret = alloced_pkeys[random_index];
 537	/* now zero it out so we don't free it next */
 538	alloced_pkeys[random_index] = 0;
 539
 540	/* go through the allocated ones that we did not want and free them */
 541	for (i = 0; i < nr_alloced; i++) {
 542		int free_ret;
 543		if (!alloced_pkeys[i])
 544			continue;
 545		free_ret = sys_pkey_free(alloced_pkeys[i]);
 546		pkey_assert(!free_ret);
 547	}
 548	dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 549			 " shadow: 0x%016llx\n", __func__,
 550			__LINE__, ret, __read_pkey_reg(), shadow_pkey_reg);
 551	return ret;
 552}
 553
 554int mprotect_pkey(void *ptr, size_t size, unsigned long orig_prot,
 555		unsigned long pkey)
 556{
 557	int nr_iterations = random() % 100;
 558	int ret;
 559
 560	while (nr_iterations-- >= 0) {
 561		int rpkey = alloc_random_pkey();
 562		ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
 563		dprintf1("sys_mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
 564				ptr, size, orig_prot, pkey, ret);
 565
 566		dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 567			" shadow: 0x%016llx\n",
 568			__func__, __LINE__, ret, __read_pkey_reg(),
 569			shadow_pkey_reg);
 570		sys_pkey_free(rpkey);
 571		dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 572			" shadow: 0x%016llx\n",
 573			__func__, __LINE__, ret, __read_pkey_reg(),
 574			shadow_pkey_reg);
 575	}
 576	pkey_assert(pkey < NR_PKEYS);
 577
 578	ret = sys_mprotect_pkey(ptr, size, orig_prot, pkey);
 579	dprintf1("mprotect_pkey(%p, %zx, prot=0x%lx, pkey=%ld) ret: %d\n",
 580			ptr, size, orig_prot, pkey, ret);
 581	pkey_assert(!ret);
 582	dprintf1("%s()::%d, ret: %d pkey_reg: 0x%016llx"
 583			" shadow: 0x%016llx\n", __func__,
 584			__LINE__, ret, __read_pkey_reg(), shadow_pkey_reg);
 585	return ret;
 586}
 587
 588struct pkey_malloc_record {
 589	void *ptr;
 590	long size;
 591	int prot;
 592};
 593struct pkey_malloc_record *pkey_malloc_records;
 594struct pkey_malloc_record *pkey_last_malloc_record;
 595static long nr_pkey_malloc_records;
 596void record_pkey_malloc(void *ptr, long size, int prot)
 597{
 598	long i;
 599	struct pkey_malloc_record *rec = NULL;
 600
 601	for (i = 0; i < nr_pkey_malloc_records; i++) {
 602		rec = &pkey_malloc_records[i];
 603		/* find a free record */
 604		if (rec)
 605			break;
 606	}
 607	if (!rec) {
 608		/* every record is full */
 609		size_t old_nr_records = nr_pkey_malloc_records;
 610		size_t new_nr_records = (nr_pkey_malloc_records * 2 + 1);
 611		size_t new_size = new_nr_records * sizeof(struct pkey_malloc_record);
 612		dprintf2("new_nr_records: %zd\n", new_nr_records);
 613		dprintf2("new_size: %zd\n", new_size);
 614		pkey_malloc_records = realloc(pkey_malloc_records, new_size);
 615		pkey_assert(pkey_malloc_records != NULL);
 616		rec = &pkey_malloc_records[nr_pkey_malloc_records];
 617		/*
 618		 * realloc() does not initialize memory, so zero it from
 619		 * the first new record all the way to the end.
 620		 */
 621		for (i = 0; i < new_nr_records - old_nr_records; i++)
 622			memset(rec + i, 0, sizeof(*rec));
 623	}
 624	dprintf3("filling malloc record[%d/%p]: {%p, %ld}\n",
 625		(int)(rec - pkey_malloc_records), rec, ptr, size);
 626	rec->ptr = ptr;
 627	rec->size = size;
 628	rec->prot = prot;
 629	pkey_last_malloc_record = rec;
 630	nr_pkey_malloc_records++;
 631}
 632
 633static void free_pkey_malloc(void *ptr)
 634{
 635	long i;
 636	int ret;
 637	dprintf3("%s(%p)\n", __func__, ptr);
 638	for (i = 0; i < nr_pkey_malloc_records; i++) {
 639		struct pkey_malloc_record *rec = &pkey_malloc_records[i];
 640		dprintf4("looking for ptr %p at record[%ld/%p]: {%p, %ld}\n",
 641				ptr, i, rec, rec->ptr, rec->size);
 642		if ((ptr <  rec->ptr) ||
 643		    (ptr >= rec->ptr + rec->size))
 644			continue;
 645
 646		dprintf3("found ptr %p at record[%ld/%p]: {%p, %ld}\n",
 647				ptr, i, rec, rec->ptr, rec->size);
 648		nr_pkey_malloc_records--;
 649		ret = munmap(rec->ptr, rec->size);
 650		dprintf3("munmap ret: %d\n", ret);
 651		pkey_assert(!ret);
 652		dprintf3("clearing rec->ptr, rec: %p\n", rec);
 653		rec->ptr = NULL;
 654		dprintf3("done clearing rec->ptr, rec: %p\n", rec);
 655		return;
 656	}
 657	pkey_assert(false);
 658}
 659
 660static void *malloc_pkey_with_mprotect(long size, int prot, u16 pkey)
 661{
 662	void *ptr;
 663	int ret;
 664
 665	read_pkey_reg();
 666	dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
 667			size, prot, pkey);
 668	pkey_assert(pkey < NR_PKEYS);
 669	ptr = mmap(NULL, size, prot, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
 670	pkey_assert(ptr != (void *)-1);
 671	ret = mprotect_pkey((void *)ptr, PAGE_SIZE, prot, pkey);
 672	pkey_assert(!ret);
 673	record_pkey_malloc(ptr, size, prot);
 674	read_pkey_reg();
 675
 676	dprintf1("%s() for pkey %d @ %p\n", __func__, pkey, ptr);
 677	return ptr;
 678}
 679
 680static void *malloc_pkey_anon_huge(long size, int prot, u16 pkey)
 681{
 682	int ret;
 683	void *ptr;
 684
 685	dprintf1("doing %s(size=%ld, prot=0x%x, pkey=%d)\n", __func__,
 686			size, prot, pkey);
 687	/*
 688	 * Guarantee we can fit at least one huge page in the resulting
 689	 * allocation by allocating space for 2:
 690	 */
 691	size = ALIGN_UP(size, HPAGE_SIZE * 2);
 692	ptr = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
 693	pkey_assert(ptr != (void *)-1);
 694	record_pkey_malloc(ptr, size, prot);
 695	mprotect_pkey(ptr, size, prot, pkey);
 696
 697	dprintf1("unaligned ptr: %p\n", ptr);
 698	ptr = ALIGN_PTR_UP(ptr, HPAGE_SIZE);
 699	dprintf1("  aligned ptr: %p\n", ptr);
 700	ret = madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE);
 701	dprintf1("MADV_HUGEPAGE ret: %d\n", ret);
 702	ret = madvise(ptr, HPAGE_SIZE, MADV_WILLNEED);
 703	dprintf1("MADV_WILLNEED ret: %d\n", ret);
 704	memset(ptr, 0, HPAGE_SIZE);
 705
 706	dprintf1("mmap()'d thp for pkey %d @ %p\n", pkey, ptr);
 707	return ptr;
 708}
 709
 710static int hugetlb_setup_ok;
 711#define SYSFS_FMT_NR_HUGE_PAGES "/sys/kernel/mm/hugepages/hugepages-%ldkB/nr_hugepages"
 712#define GET_NR_HUGE_PAGES 10
 713static void setup_hugetlbfs(void)
 714{
 715	int err;
 716	int fd;
 717	char buf[256];
 718	long hpagesz_kb;
 719	long hpagesz_mb;
 720
 721	if (geteuid() != 0) {
 722		fprintf(stderr, "WARNING: not run as root, can not do hugetlb test\n");
 723		return;
 724	}
 725
 726	cat_into_file(__stringify(GET_NR_HUGE_PAGES), "/proc/sys/vm/nr_hugepages");
 727
 728	/*
 729	 * Now go make sure that we got the pages and that they
 730	 * are PMD-level pages. Someone might have made PUD-level
 731	 * pages the default.
 732	 */
 733	hpagesz_kb = HPAGE_SIZE / 1024;
 734	hpagesz_mb = hpagesz_kb / 1024;
 735	sprintf(buf, SYSFS_FMT_NR_HUGE_PAGES, hpagesz_kb);
 736	fd = open(buf, O_RDONLY);
 737	if (fd < 0) {
 738		fprintf(stderr, "opening sysfs %ldM hugetlb config: %s\n",
 739			hpagesz_mb, strerror(errno));
 740		return;
 741	}
 742
 743	/* -1 to guarantee leaving the trailing \0 */
 744	err = read(fd, buf, sizeof(buf)-1);
 745	close(fd);
 746	if (err <= 0) {
 747		fprintf(stderr, "reading sysfs %ldM hugetlb config: %s\n",
 748			hpagesz_mb, strerror(errno));
 749		return;
 750	}
 751
 752	if (atoi(buf) != GET_NR_HUGE_PAGES) {
 753		fprintf(stderr, "could not confirm %ldM pages, got: '%s' expected %d\n",
 754			hpagesz_mb, buf, GET_NR_HUGE_PAGES);
 755		return;
 756	}
 757
 758	hugetlb_setup_ok = 1;
 759}
 760
 761static void *malloc_pkey_hugetlb(long size, int prot, u16 pkey)
 762{
 763	void *ptr;
 764	int flags = MAP_ANONYMOUS|MAP_PRIVATE|MAP_HUGETLB;
 765
 766	if (!hugetlb_setup_ok)
 767		return PTR_ERR_ENOTSUP;
 768
 769	dprintf1("doing %s(%ld, %x, %x)\n", __func__, size, prot, pkey);
 770	size = ALIGN_UP(size, HPAGE_SIZE * 2);
 771	pkey_assert(pkey < NR_PKEYS);
 772	ptr = mmap(NULL, size, PROT_NONE, flags, -1, 0);
 773	pkey_assert(ptr != (void *)-1);
 774	mprotect_pkey(ptr, size, prot, pkey);
 775
 776	record_pkey_malloc(ptr, size, prot);
 777
 778	dprintf1("mmap()'d hugetlbfs for pkey %d @ %p\n", pkey, ptr);
 779	return ptr;
 780}
 781
 782static void *(*pkey_malloc[])(long size, int prot, u16 pkey) = {
 783
 784	malloc_pkey_with_mprotect,
 785	malloc_pkey_with_mprotect_subpage,
 786	malloc_pkey_anon_huge,
 787	malloc_pkey_hugetlb
 788};
 789
 790static void *malloc_pkey(long size, int prot, u16 pkey)
 791{
 792	void *ret;
 793	static int malloc_type;
 794	int nr_malloc_types = ARRAY_SIZE(pkey_malloc);
 795
 796	pkey_assert(pkey < NR_PKEYS);
 797
 798	while (1) {
 799		pkey_assert(malloc_type < nr_malloc_types);
 800
 801		ret = pkey_malloc[malloc_type](size, prot, pkey);
 802		pkey_assert(ret != (void *)-1);
 803
 804		malloc_type++;
 805		if (malloc_type >= nr_malloc_types)
 806			malloc_type = (random()%nr_malloc_types);
 807
 808		/* try again if the malloc_type we tried is unsupported */
 809		if (ret == PTR_ERR_ENOTSUP)
 810			continue;
 811
 812		break;
 813	}
 814
 815	dprintf3("%s(%ld, prot=%x, pkey=%x) returning: %p\n", __func__,
 816			size, prot, pkey, ret);
 817	return ret;
 818}
 819
 820static int last_pkey_faults;
 821#define UNKNOWN_PKEY -2
 822void expected_pkey_fault(int pkey)
 823{
 824	dprintf2("%s(): last_pkey_faults: %d pkey_faults: %d\n",
 825			__func__, last_pkey_faults, pkey_faults);
 826	dprintf2("%s(%d): last_si_pkey: %d\n", __func__, pkey, last_si_pkey);
 827	pkey_assert(last_pkey_faults + 1 == pkey_faults);
 828
 829       /*
 830	* For exec-only memory, we do not know the pkey in
 831	* advance, so skip this check.
 832	*/
 833	if (pkey != UNKNOWN_PKEY)
 834		pkey_assert(last_si_pkey == pkey);
 835
 836#if defined(__i386__) || defined(__x86_64__) /* arch */
 837	/*
 838	 * The signal handler shold have cleared out PKEY register to let the
 839	 * test program continue.  We now have to restore it.
 840	 */
 841	if (__read_pkey_reg() != 0)
 842#elif defined(__aarch64__)
 843	if (__read_pkey_reg() != PKEY_REG_ALLOW_ALL)
 844#else
 845	if (__read_pkey_reg() != shadow_pkey_reg)
 846#endif /* arch */
 847		pkey_assert(0);
 848
 849	__write_pkey_reg(shadow_pkey_reg);
 850	dprintf1("%s() set pkey_reg=%016llx to restore state after signal "
 851		       "nuked it\n", __func__, shadow_pkey_reg);
 852	last_pkey_faults = pkey_faults;
 853	last_si_pkey = -1;
 854}
 855
 856#define do_not_expect_pkey_fault(msg)	do {			\
 857	if (last_pkey_faults != pkey_faults)			\
 858		dprintf0("unexpected PKey fault: %s\n", msg);	\
 859	pkey_assert(last_pkey_faults == pkey_faults);		\
 860} while (0)
 861
 862static int test_fds[10] = { -1 };
 863static int nr_test_fds;
 864static void __save_test_fd(int fd)
 865{
 866	pkey_assert(fd >= 0);
 867	pkey_assert(nr_test_fds < ARRAY_SIZE(test_fds));
 868	test_fds[nr_test_fds] = fd;
 869	nr_test_fds++;
 870}
 871
 872static int get_test_read_fd(void)
 873{
 874	int test_fd = open("/etc/passwd", O_RDONLY);
 875	__save_test_fd(test_fd);
 876	return test_fd;
 877}
 878
 879static void close_test_fds(void)
 880{
 881	int i;
 882
 883	for (i = 0; i < nr_test_fds; i++) {
 884		if (test_fds[i] < 0)
 885			continue;
 886		close(test_fds[i]);
 887		test_fds[i] = -1;
 888	}
 889	nr_test_fds = 0;
 890}
 891
 892static void test_pkey_alloc_free_attach_pkey0(int *ptr, u16 pkey)
 893{
 894	int i, err;
 895	int max_nr_pkey_allocs;
 896	int alloced_pkeys[NR_PKEYS];
 897	int nr_alloced = 0;
 898	long size;
 899
 900	pkey_assert(pkey_last_malloc_record);
 901	size = pkey_last_malloc_record->size;
 902	/*
 903	 * This is a bit of a hack.  But mprotect() requires
 904	 * huge-page-aligned sizes when operating on hugetlbfs.
 905	 * So, make sure that we use something that's a multiple
 906	 * of a huge page when we can.
 907	 */
 908	if (size >= HPAGE_SIZE)
 909		size = HPAGE_SIZE;
 910
 911	/* allocate every possible key and make sure key-0 never got allocated */
 912	max_nr_pkey_allocs = NR_PKEYS;
 913	for (i = 0; i < max_nr_pkey_allocs; i++) {
 914		int new_pkey = alloc_pkey();
 915		pkey_assert(new_pkey != 0);
 916
 917		if (new_pkey < 0)
 918			break;
 919		alloced_pkeys[nr_alloced++] = new_pkey;
 920	}
 921	/* free all the allocated keys */
 922	for (i = 0; i < nr_alloced; i++) {
 923		int free_ret;
 924
 925		if (!alloced_pkeys[i])
 926			continue;
 927		free_ret = sys_pkey_free(alloced_pkeys[i]);
 928		pkey_assert(!free_ret);
 929	}
 930
 931	/* attach key-0 in various modes */
 932	err = sys_mprotect_pkey(ptr, size, PROT_READ, 0);
 933	pkey_assert(!err);
 934	err = sys_mprotect_pkey(ptr, size, PROT_WRITE, 0);
 935	pkey_assert(!err);
 936	err = sys_mprotect_pkey(ptr, size, PROT_EXEC, 0);
 937	pkey_assert(!err);
 938	err = sys_mprotect_pkey(ptr, size, PROT_READ|PROT_WRITE, 0);
 939	pkey_assert(!err);
 940	err = sys_mprotect_pkey(ptr, size, PROT_READ|PROT_WRITE|PROT_EXEC, 0);
 941	pkey_assert(!err);
 942}
 943
 944static void test_read_of_write_disabled_region(int *ptr, u16 pkey)
 945{
 946	int ptr_contents;
 947
 948	dprintf1("disabling write access to PKEY[1], doing read\n");
 949	pkey_write_deny(pkey);
 950	ptr_contents = read_ptr(ptr);
 951	dprintf1("*ptr: %d\n", ptr_contents);
 952	dprintf1("\n");
 953}
 954static void test_read_of_access_disabled_region(int *ptr, u16 pkey)
 955{
 956	int ptr_contents;
 957
 958	dprintf1("disabling access to PKEY[%02d], doing read @ %p\n", pkey, ptr);
 959	read_pkey_reg();
 960	pkey_access_deny(pkey);
 961	ptr_contents = read_ptr(ptr);
 962	dprintf1("*ptr: %d\n", ptr_contents);
 963	expected_pkey_fault(pkey);
 964}
 965
 966static void test_read_of_access_disabled_region_with_page_already_mapped(int *ptr,
 967		u16 pkey)
 968{
 969	int ptr_contents;
 970
 971	dprintf1("disabling access to PKEY[%02d], doing read @ %p\n",
 972				pkey, ptr);
 973	ptr_contents = read_ptr(ptr);
 974	dprintf1("reading ptr before disabling the read : %d\n",
 975			ptr_contents);
 976	read_pkey_reg();
 977	pkey_access_deny(pkey);
 978	ptr_contents = read_ptr(ptr);
 979	dprintf1("*ptr: %d\n", ptr_contents);
 980	expected_pkey_fault(pkey);
 981}
 982
 983static void test_write_of_write_disabled_region_with_page_already_mapped(int *ptr,
 984		u16 pkey)
 985{
 986	*ptr = __LINE__;
 987	dprintf1("disabling write access; after accessing the page, "
 988		"to PKEY[%02d], doing write\n", pkey);
 989	pkey_write_deny(pkey);
 990	*ptr = __LINE__;
 991	expected_pkey_fault(pkey);
 992}
 993
 994static void test_write_of_write_disabled_region(int *ptr, u16 pkey)
 995{
 996	dprintf1("disabling write access to PKEY[%02d], doing write\n", pkey);
 997	pkey_write_deny(pkey);
 998	*ptr = __LINE__;
 999	expected_pkey_fault(pkey);
1000}
1001static void test_write_of_access_disabled_region(int *ptr, u16 pkey)
1002{
1003	dprintf1("disabling access to PKEY[%02d], doing write\n", pkey);
1004	pkey_access_deny(pkey);
1005	*ptr = __LINE__;
1006	expected_pkey_fault(pkey);
1007}
1008
1009static void test_write_of_access_disabled_region_with_page_already_mapped(int *ptr,
1010			u16 pkey)
1011{
1012	*ptr = __LINE__;
1013	dprintf1("disabling access; after accessing the page, "
1014		" to PKEY[%02d], doing write\n", pkey);
1015	pkey_access_deny(pkey);
1016	*ptr = __LINE__;
1017	expected_pkey_fault(pkey);
1018}
1019
1020static void test_kernel_write_of_access_disabled_region(int *ptr, u16 pkey)
1021{
1022	int ret;
1023	int test_fd = get_test_read_fd();
1024
1025	dprintf1("disabling access to PKEY[%02d], "
1026		 "having kernel read() to buffer\n", pkey);
1027	pkey_access_deny(pkey);
1028	ret = read(test_fd, ptr, 1);
1029	dprintf1("read ret: %d\n", ret);
1030	pkey_assert(ret);
1031}
1032
1033static void test_kernel_write_of_write_disabled_region(int *ptr, u16 pkey)
1034{
1035	int ret;
1036	int test_fd = get_test_read_fd();
1037
1038	pkey_write_deny(pkey);
1039	ret = read(test_fd, ptr, 100);
1040	dprintf1("read ret: %d\n", ret);
1041	if (ret < 0 && (DEBUG_LEVEL > 0))
1042		perror("verbose read result (OK for this to be bad)");
1043	pkey_assert(ret);
1044}
1045
1046static void test_kernel_gup_of_access_disabled_region(int *ptr, u16 pkey)
1047{
1048	int pipe_ret, vmsplice_ret;
1049	struct iovec iov;
1050	int pipe_fds[2];
1051
1052	pipe_ret = pipe(pipe_fds);
1053
1054	pkey_assert(pipe_ret == 0);
1055	dprintf1("disabling access to PKEY[%02d], "
1056		 "having kernel vmsplice from buffer\n", pkey);
1057	pkey_access_deny(pkey);
1058	iov.iov_base = ptr;
1059	iov.iov_len = PAGE_SIZE;
1060	vmsplice_ret = vmsplice(pipe_fds[1], &iov, 1, SPLICE_F_GIFT);
1061	dprintf1("vmsplice() ret: %d\n", vmsplice_ret);
1062	pkey_assert(vmsplice_ret == -1);
1063
1064	close(pipe_fds[0]);
1065	close(pipe_fds[1]);
1066}
1067
1068static void test_kernel_gup_write_to_write_disabled_region(int *ptr, u16 pkey)
1069{
1070	int ignored = 0xdada;
1071	int futex_ret;
1072	int some_int = __LINE__;
1073
1074	dprintf1("disabling write to PKEY[%02d], "
1075		 "doing futex gunk in buffer\n", pkey);
1076	*ptr = some_int;
1077	pkey_write_deny(pkey);
1078	futex_ret = syscall(SYS_futex, ptr, FUTEX_WAIT, some_int-1, NULL,
1079			&ignored, ignored);
1080	if (DEBUG_LEVEL > 0)
1081		perror("futex");
1082	dprintf1("futex() ret: %d\n", futex_ret);
1083}
1084
1085/* Assumes that all pkeys other than 'pkey' are unallocated */
1086static void test_pkey_syscalls_on_non_allocated_pkey(int *ptr, u16 pkey)
1087{
1088	int err;
1089	int i;
1090
1091	/* Note: 0 is the default pkey, so don't mess with it */
1092	for (i = 1; i < NR_PKEYS; i++) {
1093		if (pkey == i)
1094			continue;
1095
1096		dprintf1("trying get/set/free to non-allocated pkey: %2d\n", i);
1097		err = sys_pkey_free(i);
1098		pkey_assert(err);
1099
1100		err = sys_pkey_free(i);
1101		pkey_assert(err);
1102
1103		err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, i);
1104		pkey_assert(err);
1105	}
1106}
1107
1108/* Assumes that all pkeys other than 'pkey' are unallocated */
1109static void test_pkey_syscalls_bad_args(int *ptr, u16 pkey)
1110{
1111	int err;
1112	int bad_pkey = NR_PKEYS+99;
1113
1114	/* pass a known-invalid pkey in: */
1115	err = sys_mprotect_pkey(ptr, PAGE_SIZE, PROT_READ, bad_pkey);
1116	pkey_assert(err);
1117}
1118
1119static void become_child(void)
1120{
1121	pid_t forkret;
1122
1123	forkret = fork();
1124	pkey_assert(forkret >= 0);
1125	dprintf3("[%d] fork() ret: %d\n", getpid(), forkret);
1126
1127	if (!forkret) {
1128		/* in the child */
1129		return;
1130	}
1131	exit(0);
1132}
1133
1134/* Assumes that all pkeys other than 'pkey' are unallocated */
1135static void test_pkey_alloc_exhaust(int *ptr, u16 pkey)
1136{
1137	int err;
1138	int allocated_pkeys[NR_PKEYS] = {0};
1139	int nr_allocated_pkeys = 0;
1140	int i;
1141
1142	for (i = 0; i < NR_PKEYS*3; i++) {
1143		int new_pkey;
1144		dprintf1("%s() alloc loop: %d\n", __func__, i);
1145		new_pkey = alloc_pkey();
1146		dprintf4("%s()::%d, err: %d pkey_reg: 0x%016llx"
1147				" shadow: 0x%016llx\n",
1148				__func__, __LINE__, err, __read_pkey_reg(),
1149				shadow_pkey_reg);
1150		read_pkey_reg(); /* for shadow checking */
1151		dprintf2("%s() errno: %d ENOSPC: %d\n", __func__, errno, ENOSPC);
1152		if ((new_pkey == -1) && (errno == ENOSPC)) {
1153			dprintf2("%s() failed to allocate pkey after %d tries\n",
1154				__func__, nr_allocated_pkeys);
1155		} else {
1156			/*
1157			 * Ensure the number of successes never
1158			 * exceeds the number of keys supported
1159			 * in the hardware.
1160			 */
1161			pkey_assert(nr_allocated_pkeys < NR_PKEYS);
1162			allocated_pkeys[nr_allocated_pkeys++] = new_pkey;
1163		}
1164
1165		/*
1166		 * Make sure that allocation state is properly
1167		 * preserved across fork().
1168		 */
1169		if (i == NR_PKEYS*2)
1170			become_child();
1171	}
1172
1173	dprintf3("%s()::%d\n", __func__, __LINE__);
1174
1175	/*
1176	 * On x86:
1177	 * There are 16 pkeys supported in hardware.  Three are
1178	 * allocated by the time we get here:
1179	 *   1. The default key (0)
1180	 *   2. One possibly consumed by an execute-only mapping.
1181	 *   3. One allocated by the test code and passed in via
1182	 *      'pkey' to this function.
1183	 * Ensure that we can allocate at least another 13 (16-3).
1184	 *
1185	 * On powerpc:
1186	 * There are either 5, 28, 29 or 32 pkeys supported in
1187	 * hardware depending on the page size (4K or 64K) and
1188	 * platform (powernv or powervm). Four are allocated by
1189	 * the time we get here. These include pkey-0, pkey-1,
1190	 * exec-only pkey and the one allocated by the test code.
1191	 * Ensure that we can allocate the remaining.
1192	 */
1193	pkey_assert(i >= (NR_PKEYS - get_arch_reserved_keys() - 1));
1194
1195	for (i = 0; i < nr_allocated_pkeys; i++) {
1196		err = sys_pkey_free(allocated_pkeys[i]);
1197		pkey_assert(!err);
1198		read_pkey_reg(); /* for shadow checking */
1199	}
1200}
1201
1202static void arch_force_pkey_reg_init(void)
1203{
1204#if defined(__i386__) || defined(__x86_64__) /* arch */
1205	u64 *buf;
1206
1207	/*
1208	 * All keys should be allocated and set to allow reads and
1209	 * writes, so the register should be all 0.  If not, just
1210	 * skip the test.
1211	 */
1212	if (read_pkey_reg())
1213		return;
1214
1215	/*
1216	 * Just allocate an absurd about of memory rather than
1217	 * doing the XSAVE size enumeration dance.
1218	 */
1219	buf = mmap(NULL, 1*MB, PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1220
1221	/* These __builtins require compiling with -mxsave */
1222
1223	/* XSAVE to build a valid buffer: */
1224	__builtin_ia32_xsave(buf, XSTATE_PKEY);
1225	/* Clear XSTATE_BV[PKRU]: */
1226	buf[XSTATE_BV_OFFSET/sizeof(u64)] &= ~XSTATE_PKEY;
1227	/* XRSTOR will likely get PKRU back to the init state: */
1228	__builtin_ia32_xrstor(buf, XSTATE_PKEY);
1229
1230	munmap(buf, 1*MB);
1231#endif
1232}
1233
1234
1235/*
1236 * This is mostly useless on ppc for now.  But it will not
1237 * hurt anything and should give some better coverage as
1238 * a long-running test that continually checks the pkey
1239 * register.
1240 */
1241static void test_pkey_init_state(int *ptr, u16 pkey)
1242{
1243	int err;
1244	int allocated_pkeys[NR_PKEYS] = {0};
1245	int nr_allocated_pkeys = 0;
1246	int i;
1247
1248	for (i = 0; i < NR_PKEYS; i++) {
1249		int new_pkey = alloc_pkey();
1250
1251		if (new_pkey < 0)
1252			continue;
1253		allocated_pkeys[nr_allocated_pkeys++] = new_pkey;
1254	}
1255
1256	dprintf3("%s()::%d\n", __func__, __LINE__);
1257
1258	arch_force_pkey_reg_init();
1259
1260	/*
1261	 * Loop for a bit, hoping to get exercise the kernel
1262	 * context switch code.
1263	 */
1264	for (i = 0; i < 1000000; i++)
1265		read_pkey_reg();
1266
1267	for (i = 0; i < nr_allocated_pkeys; i++) {
1268		err = sys_pkey_free(allocated_pkeys[i]);
1269		pkey_assert(!err);
1270		read_pkey_reg(); /* for shadow checking */
1271	}
1272}
1273
1274/*
1275 * pkey 0 is special.  It is allocated by default, so you do not
1276 * have to call pkey_alloc() to use it first.  Make sure that it
1277 * is usable.
1278 */
1279static void test_mprotect_with_pkey_0(int *ptr, u16 pkey)
1280{
1281	long size;
1282	int prot;
1283
1284	assert(pkey_last_malloc_record);
1285	size = pkey_last_malloc_record->size;
1286	/*
1287	 * This is a bit of a hack.  But mprotect() requires
1288	 * huge-page-aligned sizes when operating on hugetlbfs.
1289	 * So, make sure that we use something that's a multiple
1290	 * of a huge page when we can.
1291	 */
1292	if (size >= HPAGE_SIZE)
1293		size = HPAGE_SIZE;
1294	prot = pkey_last_malloc_record->prot;
1295
1296	/* Use pkey 0 */
1297	mprotect_pkey(ptr, size, prot, 0);
1298
1299	/* Make sure that we can set it back to the original pkey. */
1300	mprotect_pkey(ptr, size, prot, pkey);
1301}
1302
1303static void test_ptrace_of_child(int *ptr, u16 pkey)
1304{
1305	__always_unused int peek_result;
1306	pid_t child_pid;
1307	void *ignored = 0;
1308	long ret;
1309	int status;
1310	/*
1311	 * This is the "control" for our little expermient.  Make sure
1312	 * we can always access it when ptracing.
1313	 */
1314	int *plain_ptr_unaligned = malloc(HPAGE_SIZE);
1315	int *plain_ptr = ALIGN_PTR_UP(plain_ptr_unaligned, PAGE_SIZE);
1316
1317	/*
1318	 * Fork a child which is an exact copy of this process, of course.
1319	 * That means we can do all of our tests via ptrace() and then plain
1320	 * memory access and ensure they work differently.
1321	 */
1322	child_pid = fork_lazy_child();
1323	dprintf1("[%d] child pid: %d\n", getpid(), child_pid);
1324
1325	ret = ptrace(PTRACE_ATTACH, child_pid, ignored, ignored);
1326	if (ret)
1327		perror("attach");
1328	dprintf1("[%d] attach ret: %ld %d\n", getpid(), ret, __LINE__);
1329	pkey_assert(ret != -1);
1330	ret = waitpid(child_pid, &status, WUNTRACED);
1331	if ((ret != child_pid) || !(WIFSTOPPED(status))) {
1332		fprintf(stderr, "weird waitpid result %ld stat %x\n",
1333				ret, status);
1334		pkey_assert(0);
1335	}
1336	dprintf2("waitpid ret: %ld\n", ret);
1337	dprintf2("waitpid status: %d\n", status);
1338
1339	pkey_access_deny(pkey);
1340	pkey_write_deny(pkey);
1341
1342	/* Write access, untested for now:
1343	ret = ptrace(PTRACE_POKEDATA, child_pid, peek_at, data);
1344	pkey_assert(ret != -1);
1345	dprintf1("poke at %p: %ld\n", peek_at, ret);
1346	*/
1347
1348	/*
1349	 * Try to access the pkey-protected "ptr" via ptrace:
1350	 */
1351	ret = ptrace(PTRACE_PEEKDATA, child_pid, ptr, ignored);
1352	/* expect it to work, without an error: */
1353	pkey_assert(ret != -1);
1354	/* Now access from the current task, and expect an exception: */
1355	peek_result = read_ptr(ptr);
1356	expected_pkey_fault(pkey);
1357
1358	/*
1359	 * Try to access the NON-pkey-protected "plain_ptr" via ptrace:
1360	 */
1361	ret = ptrace(PTRACE_PEEKDATA, child_pid, plain_ptr, ignored);
1362	/* expect it to work, without an error: */
1363	pkey_assert(ret != -1);
1364	/* Now access from the current task, and expect NO exception: */
1365	peek_result = read_ptr(plain_ptr);
1366	do_not_expect_pkey_fault("read plain pointer after ptrace");
1367
1368	ret = ptrace(PTRACE_DETACH, child_pid, ignored, 0);
1369	pkey_assert(ret != -1);
1370
1371	ret = kill(child_pid, SIGKILL);
1372	pkey_assert(ret != -1);
1373
1374	wait(&status);
1375
1376	free(plain_ptr_unaligned);
1377}
1378
1379static void *get_pointer_to_instructions(void)
1380{
1381	void *p1;
1382
1383	p1 = ALIGN_PTR_UP(&lots_o_noops_around_write, PAGE_SIZE);
1384	dprintf3("&lots_o_noops: %p\n", &lots_o_noops_around_write);
1385	/* lots_o_noops_around_write should be page-aligned already */
1386	assert(p1 == &lots_o_noops_around_write);
1387
1388	/* Point 'p1' at the *second* page of the function: */
1389	p1 += PAGE_SIZE;
1390
1391	/*
1392	 * Try to ensure we fault this in on next touch to ensure
1393	 * we get an instruction fault as opposed to a data one
1394	 */
1395	madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1396
1397	return p1;
1398}
1399
1400static void test_executing_on_unreadable_memory(int *ptr, u16 pkey)
1401{
1402	void *p1;
1403	int scratch;
1404	int ptr_contents;
1405	int ret;
1406
1407	p1 = get_pointer_to_instructions();
1408	lots_o_noops_around_write(&scratch);
1409	ptr_contents = read_ptr(p1);
1410	dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1411
1412	ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC, (u64)pkey);
1413	pkey_assert(!ret);
1414	pkey_access_deny(pkey);
1415
1416	dprintf2("pkey_reg: %016llx\n", read_pkey_reg());
1417
1418	/*
1419	 * Make sure this is an *instruction* fault
1420	 */
1421	madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1422	lots_o_noops_around_write(&scratch);
1423	do_not_expect_pkey_fault("executing on PROT_EXEC memory");
1424	expect_fault_on_read_execonly_key(p1, pkey);
1425
1426	// Reset back to PROT_EXEC | PROT_READ for architectures that support
1427	// non-PKEY execute-only permissions.
1428	ret = mprotect_pkey(p1, PAGE_SIZE, PROT_EXEC | PROT_READ, (u64)pkey);
1429	pkey_assert(!ret);
1430}
1431
1432static void test_implicit_mprotect_exec_only_memory(int *ptr, u16 pkey)
1433{
1434	void *p1;
1435	int scratch;
1436	int ptr_contents;
1437	int ret;
1438
1439	dprintf1("%s() start\n", __func__);
1440
1441	p1 = get_pointer_to_instructions();
1442	lots_o_noops_around_write(&scratch);
1443	ptr_contents = read_ptr(p1);
1444	dprintf2("ptr (%p) contents@%d: %x\n", p1, __LINE__, ptr_contents);
1445
1446	/* Use a *normal* mprotect(), not mprotect_pkey(): */
1447	ret = mprotect(p1, PAGE_SIZE, PROT_EXEC);
1448	pkey_assert(!ret);
1449
1450	/*
1451	 * Reset the shadow, assuming that the above mprotect()
1452	 * correctly changed PKRU, but to an unknown value since
1453	 * the actual allocated pkey is unknown.
1454	 */
1455	shadow_pkey_reg = __read_pkey_reg();
1456
1457	dprintf2("pkey_reg: %016llx\n", read_pkey_reg());
1458
1459	/* Make sure this is an *instruction* fault */
1460	madvise(p1, PAGE_SIZE, MADV_DONTNEED);
1461	lots_o_noops_around_write(&scratch);
1462	do_not_expect_pkey_fault("executing on PROT_EXEC memory");
1463	expect_fault_on_read_execonly_key(p1, UNKNOWN_PKEY);
1464
1465	/*
1466	 * Put the memory back to non-PROT_EXEC.  Should clear the
1467	 * exec-only pkey off the VMA and allow it to be readable
1468	 * again.  Go to PROT_NONE first to check for a kernel bug
1469	 * that did not clear the pkey when doing PROT_NONE.
1470	 */
1471	ret = mprotect(p1, PAGE_SIZE, PROT_NONE);
1472	pkey_assert(!ret);
1473
1474	ret = mprotect(p1, PAGE_SIZE, PROT_READ|PROT_EXEC);
1475	pkey_assert(!ret);
1476	ptr_contents = read_ptr(p1);
1477	do_not_expect_pkey_fault("plain read on recently PROT_EXEC area");
1478}
1479
1480#if defined(__i386__) || defined(__x86_64__)
1481static void test_ptrace_modifies_pkru(int *ptr, u16 pkey)
1482{
1483	u32 new_pkru;
1484	pid_t child;
1485	int status, ret;
1486	int pkey_offset = pkey_reg_xstate_offset();
1487	size_t xsave_size = cpu_max_xsave_size();
1488	void *xsave;
1489	u32 *pkey_register;
1490	u64 *xstate_bv;
1491	struct iovec iov;
1492
1493	new_pkru = ~read_pkey_reg();
1494	/* Don't make PROT_EXEC mappings inaccessible */
1495	new_pkru &= ~3;
1496
1497	child = fork();
1498	pkey_assert(child >= 0);
1499	dprintf3("[%d] fork() ret: %d\n", getpid(), child);
1500	if (!child) {
1501		ptrace(PTRACE_TRACEME, 0, 0, 0);
1502		/* Stop and allow the tracer to modify PKRU directly */
1503		raise(SIGSTOP);
1504
1505		/*
1506		 * need __read_pkey_reg() version so we do not do shadow_pkey_reg
1507		 * checking
1508		 */
1509		if (__read_pkey_reg() != new_pkru)
1510			exit(1);
1511
1512		/* Stop and allow the tracer to clear XSTATE_BV for PKRU */
1513		raise(SIGSTOP);
1514
1515		if (__read_pkey_reg() != 0)
1516			exit(1);
1517
1518		/* Stop and allow the tracer to examine PKRU */
1519		raise(SIGSTOP);
1520
1521		exit(0);
1522	}
1523
1524	pkey_assert(child == waitpid(child, &status, 0));
1525	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1526	pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1527
1528	xsave = (void *)malloc(xsave_size);
1529	pkey_assert(xsave > 0);
1530
1531	/* Modify the PKRU register directly */
1532	iov.iov_base = xsave;
1533	iov.iov_len = xsave_size;
1534	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1535	pkey_assert(ret == 0);
1536
1537	pkey_register = (u32 *)(xsave + pkey_offset);
1538	pkey_assert(*pkey_register == read_pkey_reg());
1539
1540	*pkey_register = new_pkru;
1541
1542	ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1543	pkey_assert(ret == 0);
1544
1545	/* Test that the modification is visible in ptrace before any execution */
1546	memset(xsave, 0xCC, xsave_size);
1547	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1548	pkey_assert(ret == 0);
1549	pkey_assert(*pkey_register == new_pkru);
1550
1551	/* Execute the tracee */
1552	ret = ptrace(PTRACE_CONT, child, 0, 0);
1553	pkey_assert(ret == 0);
1554
1555	/* Test that the tracee saw the PKRU value change */
1556	pkey_assert(child == waitpid(child, &status, 0));
1557	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1558	pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1559
1560	/* Test that the modification is visible in ptrace after execution */
1561	memset(xsave, 0xCC, xsave_size);
1562	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1563	pkey_assert(ret == 0);
1564	pkey_assert(*pkey_register == new_pkru);
1565
1566	/* Clear the PKRU bit from XSTATE_BV */
1567	xstate_bv = (u64 *)(xsave + 512);
1568	*xstate_bv &= ~(1 << 9);
1569
1570	ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1571	pkey_assert(ret == 0);
1572
1573	/* Test that the modification is visible in ptrace before any execution */
1574	memset(xsave, 0xCC, xsave_size);
1575	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1576	pkey_assert(ret == 0);
1577	pkey_assert(*pkey_register == 0);
1578
1579	ret = ptrace(PTRACE_CONT, child, 0, 0);
1580	pkey_assert(ret == 0);
1581
1582	/* Test that the tracee saw the PKRU value go to 0 */
1583	pkey_assert(child == waitpid(child, &status, 0));
1584	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1585	pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1586
1587	/* Test that the modification is visible in ptrace after execution */
1588	memset(xsave, 0xCC, xsave_size);
1589	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_X86_XSTATE, &iov);
1590	pkey_assert(ret == 0);
1591	pkey_assert(*pkey_register == 0);
1592
1593	ret = ptrace(PTRACE_CONT, child, 0, 0);
1594	pkey_assert(ret == 0);
1595	pkey_assert(child == waitpid(child, &status, 0));
1596	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1597	pkey_assert(WIFEXITED(status));
1598	pkey_assert(WEXITSTATUS(status) == 0);
1599	free(xsave);
1600}
1601#endif
1602
1603#if defined(__aarch64__)
1604static void test_ptrace_modifies_pkru(int *ptr, u16 pkey)
1605{
1606	pid_t child;
1607	int status, ret;
1608	struct iovec iov;
1609	u64 trace_pkey;
1610	/* Just a random pkey value.. */
1611	u64 new_pkey = (POE_X << PKEY_BITS_PER_PKEY * 2) |
1612			(POE_NONE << PKEY_BITS_PER_PKEY) |
1613			POE_RWX;
1614
1615	child = fork();
1616	pkey_assert(child >= 0);
1617	dprintf3("[%d] fork() ret: %d\n", getpid(), child);
1618	if (!child) {
1619		ptrace(PTRACE_TRACEME, 0, 0, 0);
1620
1621		/* Stop and allow the tracer to modify PKRU directly */
1622		raise(SIGSTOP);
1623
1624		/*
1625		 * need __read_pkey_reg() version so we do not do shadow_pkey_reg
1626		 * checking
1627		 */
1628		if (__read_pkey_reg() != new_pkey)
1629			exit(1);
1630
1631		raise(SIGSTOP);
1632
1633		exit(0);
1634	}
1635
1636	pkey_assert(child == waitpid(child, &status, 0));
1637	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1638	pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1639
1640	iov.iov_base = &trace_pkey;
1641	iov.iov_len = 8;
1642	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_ARM_POE, &iov);
1643	pkey_assert(ret == 0);
1644	pkey_assert(trace_pkey == read_pkey_reg());
1645
1646	trace_pkey = new_pkey;
1647
1648	ret = ptrace(PTRACE_SETREGSET, child, (void *)NT_ARM_POE, &iov);
1649	pkey_assert(ret == 0);
1650
1651	/* Test that the modification is visible in ptrace before any execution */
1652	memset(&trace_pkey, 0, sizeof(trace_pkey));
1653	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_ARM_POE, &iov);
1654	pkey_assert(ret == 0);
1655	pkey_assert(trace_pkey == new_pkey);
1656
1657	/* Execute the tracee */
1658	ret = ptrace(PTRACE_CONT, child, 0, 0);
1659	pkey_assert(ret == 0);
1660
1661	/* Test that the tracee saw the PKRU value change */
1662	pkey_assert(child == waitpid(child, &status, 0));
1663	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1664	pkey_assert(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP);
1665
1666	/* Test that the modification is visible in ptrace after execution */
1667	memset(&trace_pkey, 0, sizeof(trace_pkey));
1668	ret = ptrace(PTRACE_GETREGSET, child, (void *)NT_ARM_POE, &iov);
1669	pkey_assert(ret == 0);
1670	pkey_assert(trace_pkey == new_pkey);
1671
1672	ret = ptrace(PTRACE_CONT, child, 0, 0);
1673	pkey_assert(ret == 0);
1674	pkey_assert(child == waitpid(child, &status, 0));
1675	dprintf3("[%d] waitpid(%d) status: %x\n", getpid(), child, status);
1676	pkey_assert(WIFEXITED(status));
1677	pkey_assert(WEXITSTATUS(status) == 0);
1678}
1679#endif
1680
1681static void test_mprotect_pkey_on_unsupported_cpu(int *ptr, u16 pkey)
1682{
1683	int size = PAGE_SIZE;
1684	int sret;
1685
1686	if (cpu_has_pkeys()) {
1687		dprintf1("SKIP: %s: no CPU support\n", __func__);
1688		return;
1689	}
1690
1691	sret = syscall(__NR_pkey_mprotect, ptr, size, PROT_READ, pkey);
1692	pkey_assert(sret < 0);
1693}
1694
1695static void (*pkey_tests[])(int *ptr, u16 pkey) = {
1696	test_read_of_write_disabled_region,
1697	test_read_of_access_disabled_region,
1698	test_read_of_access_disabled_region_with_page_already_mapped,
1699	test_write_of_write_disabled_region,
1700	test_write_of_write_disabled_region_with_page_already_mapped,
1701	test_write_of_access_disabled_region,
1702	test_write_of_access_disabled_region_with_page_already_mapped,
1703	test_kernel_write_of_access_disabled_region,
1704	test_kernel_write_of_write_disabled_region,
1705	test_kernel_gup_of_access_disabled_region,
1706	test_kernel_gup_write_to_write_disabled_region,
1707	test_executing_on_unreadable_memory,
1708	test_implicit_mprotect_exec_only_memory,
1709	test_mprotect_with_pkey_0,
1710	test_ptrace_of_child,
1711	test_pkey_init_state,
1712	test_pkey_syscalls_on_non_allocated_pkey,
1713	test_pkey_syscalls_bad_args,
1714	test_pkey_alloc_exhaust,
1715	test_pkey_alloc_free_attach_pkey0,
1716#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__)
1717	test_ptrace_modifies_pkru,
1718#endif
1719};
1720
1721static void run_tests_once(void)
1722{
1723	int *ptr;
1724	int prot = PROT_READ|PROT_WRITE;
1725
1726	for (test_nr = 0; test_nr < ARRAY_SIZE(pkey_tests); test_nr++) {
1727		int pkey;
1728		int orig_pkey_faults = pkey_faults;
1729
1730		dprintf1("======================\n");
1731		dprintf1("test %d preparing...\n", test_nr);
1732
1733		tracing_on();
1734		pkey = alloc_random_pkey();
1735		dprintf1("test %d starting with pkey: %d\n", test_nr, pkey);
1736		ptr = malloc_pkey(PAGE_SIZE, prot, pkey);
1737		dprintf1("test %d starting...\n", test_nr);
1738		pkey_tests[test_nr](ptr, pkey);
1739		dprintf1("freeing test memory: %p\n", ptr);
1740		free_pkey_malloc(ptr);
1741		sys_pkey_free(pkey);
1742
1743		dprintf1("pkey_faults: %d\n", pkey_faults);
1744		dprintf1("orig_pkey_faults: %d\n", orig_pkey_faults);
1745
1746		tracing_off();
1747		close_test_fds();
1748
1749		printf("test %2d PASSED (iteration %d)\n", test_nr, iteration_nr);
1750		dprintf1("======================\n\n");
1751	}
1752	iteration_nr++;
1753}
1754
1755static void pkey_setup_shadow(void)
1756{
1757	shadow_pkey_reg = __read_pkey_reg();
1758}
1759
1760int main(void)
1761{
1762	int nr_iterations = 22;
1763	int pkeys_supported = is_pkeys_supported();
1764
1765	srand((unsigned int)time(NULL));
1766
1767	setup_handlers();
1768
1769	printf("has pkeys: %d\n", pkeys_supported);
1770
1771	if (!pkeys_supported) {
1772		int size = PAGE_SIZE;
1773		int *ptr;
1774
1775		printf("running PKEY tests for unsupported CPU/OS\n");
1776
1777		ptr  = mmap(NULL, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
1778		assert(ptr != (void *)-1);
1779		test_mprotect_pkey_on_unsupported_cpu(ptr, 1);
1780		exit(0);
1781	}
1782
1783	pkey_setup_shadow();
1784	printf("startup pkey_reg: %016llx\n", read_pkey_reg());
1785	setup_hugetlbfs();
1786
1787	while (nr_iterations-- > 0)
1788		run_tests_once();
1789
1790	printf("done (all tests OK)\n");
1791	return 0;
1792}