rseq_test.c

  1// SPDX-License-Identifier: GPL-2.0-only
  2
  3/*
  4 * Include rseq.c without _GNU_SOURCE defined, before including any headers, so
  5 * that rseq.c is compiled with its configuration, not KVM selftests' config.
  6 */
  7#undef _GNU_SOURCE
  8#include "../rseq/rseq.c"
  9#define _GNU_SOURCE
 10
 11#include <errno.h>
 12#include <fcntl.h>
 13#include <pthread.h>
 14#include <sched.h>
 15#include <stdio.h>
 16#include <stdlib.h>
 17#include <string.h>
 18#include <signal.h>
 19#include <syscall.h>
 20#include <sys/ioctl.h>
 21#include <sys/sysinfo.h>
 22#include <asm/barrier.h>
 23#include <linux/atomic.h>
 24#include <linux/rseq.h>
 25#include <linux/unistd.h>
 26
 27#include "kvm_util.h"
 28#include "processor.h"
 29#include "test_util.h"
 30#include "ucall_common.h"
 31
 32/*
 33 * Any bug related to task migration is likely to be timing-dependent; perform
 34 * a large number of migrations to reduce the odds of a false negative.
 35 */
 36#define NR_TASK_MIGRATIONS 100000
 37
 38static pthread_t migration_thread;
 39static cpu_set_t possible_mask;
 40static int min_cpu, max_cpu;
 41static bool done;
 42
 43static atomic_t seq_cnt;
 44
 45static void guest_code(void)
 46{
 47	for (;;)
 48		GUEST_SYNC(0);
 49}
 50
 51static int next_cpu(int cpu)
 52{
 53	/*
 54	 * Advance to the next CPU, skipping those that weren't in the original
 55	 * affinity set.  Sadly, there is no CPU_SET_FOR_EACH, and cpu_set_t's
 56	 * data storage is considered as opaque.  Note, if this task is pinned
 57	 * to a small set of discontigous CPUs, e.g. 2 and 1023, this loop will
 58	 * burn a lot cycles and the test will take longer than normal to
 59	 * complete.
 60	 */
 61	do {
 62		cpu++;
 63		if (cpu > max_cpu) {
 64			cpu = min_cpu;
 65			TEST_ASSERT(CPU_ISSET(cpu, &possible_mask),
 66				    "Min CPU = %d must always be usable", cpu);
 67			break;
 68		}
 69	} while (!CPU_ISSET(cpu, &possible_mask));
 70
 71	return cpu;
 72}
 73
 74static void *migration_worker(void *__rseq_tid)
 75{
 76	pid_t rseq_tid = (pid_t)(unsigned long)__rseq_tid;
 77	cpu_set_t allowed_mask;
 78	int r, i, cpu;
 79
 80	CPU_ZERO(&allowed_mask);
 81
 82	for (i = 0, cpu = min_cpu; i < NR_TASK_MIGRATIONS; i++, cpu = next_cpu(cpu)) {
 83		CPU_SET(cpu, &allowed_mask);
 84
 85		/*
 86		 * Bump the sequence count twice to allow the reader to detect
 87		 * that a migration may have occurred in between rseq and sched
 88		 * CPU ID reads.  An odd sequence count indicates a migration
 89		 * is in-progress, while a completely different count indicates
 90		 * a migration occurred since the count was last read.
 91		 */
 92		atomic_inc(&seq_cnt);
 93
 94		/*
 95		 * Ensure the odd count is visible while getcpu() isn't
 96		 * stable, i.e. while changing affinity is in-progress.
 97		 */
 98		smp_wmb();
 99		r = sched_setaffinity(rseq_tid, sizeof(allowed_mask), &allowed_mask);
100		TEST_ASSERT(!r, "sched_setaffinity failed, errno = %d (%s)",
101			    errno, strerror(errno));
102		smp_wmb();
103		atomic_inc(&seq_cnt);
104
105		CPU_CLR(cpu, &allowed_mask);
106
107		/*
108		 * Wait 1-10us before proceeding to the next iteration and more
109		 * specifically, before bumping seq_cnt again.  A delay is
110		 * needed on three fronts:
111		 *
112		 *  1. To allow sched_setaffinity() to prompt migration before
113		 *     ioctl(KVM_RUN) enters the guest so that TIF_NOTIFY_RESUME
114		 *     (or TIF_NEED_RESCHED, which indirectly leads to handling
115		 *     NOTIFY_RESUME) is handled in KVM context.
116		 *
117		 *     If NOTIFY_RESUME/NEED_RESCHED is set after KVM enters
118		 *     the guest, the guest will trigger a IO/MMIO exit all the
119		 *     way to userspace and the TIF flags will be handled by
120		 *     the generic "exit to userspace" logic, not by KVM.  The
121		 *     exit to userspace is necessary to give the test a chance
122		 *     to check the rseq CPU ID (see #2).
123		 *
124		 *     Alternatively, guest_code() could include an instruction
125		 *     to trigger an exit that is handled by KVM, but any such
126		 *     exit requires architecture specific code.
127		 *
128		 *  2. To let ioctl(KVM_RUN) make its way back to the test
129		 *     before the next round of migration.  The test's check on
130		 *     the rseq CPU ID must wait for migration to complete in
131		 *     order to avoid false positive, thus any kernel rseq bug
132		 *     will be missed if the next migration starts before the
133		 *     check completes.
134		 *
135		 *  3. To ensure the read-side makes efficient forward progress,
136		 *     e.g. if getcpu() involves a syscall. Stalling the read-side
137		 *     means the test will spend more time waiting for getcpu()
138		 *     to stabilize and less time trying to hit the timing-dependent
139		 *     bug.
140		 *
141		 * Because any bug in this area is likely to be timing-dependent,
142		 * run with a range of delays at 1us intervals from 1us to 10us
143		 * as a best effort to avoid tuning the test to the point where
144		 * it can hit _only_ the original bug and not detect future
145		 * regressions.
146		 *
147		 * The original bug can reproduce with a delay up to ~500us on
148		 * x86-64, but starts to require more iterations to reproduce
149		 * as the delay creeps above ~10us, and the average runtime of
150		 * each iteration obviously increases as well.  Cap the delay
151		 * at 10us to keep test runtime reasonable while minimizing
152		 * potential coverage loss.
153		 *
154		 * The lower bound for reproducing the bug is likely below 1us,
155		 * e.g. failures occur on x86-64 with nanosleep(0), but at that
156		 * point the overhead of the syscall likely dominates the delay.
157		 * Use usleep() for simplicity and to avoid unnecessary kernel
158		 * dependencies.
159		 */
160		usleep((i % 10) + 1);
161	}
162	done = true;
163	return NULL;
164}
165
166static void calc_min_max_cpu(void)
167{
168	int i, cnt, nproc;
169
170	TEST_REQUIRE(CPU_COUNT(&possible_mask) >= 2);
171
172	/*
173	 * CPU_SET doesn't provide a FOR_EACH helper, get the min/max CPU that
174	 * this task is affined to in order to reduce the time spent querying
175	 * unusable CPUs, e.g. if this task is pinned to a small percentage of
176	 * total CPUs.
177	 */
178	nproc = get_nprocs_conf();
179	min_cpu = -1;
180	max_cpu = -1;
181	cnt = 0;
182
183	for (i = 0; i < nproc; i++) {
184		if (!CPU_ISSET(i, &possible_mask))
185			continue;
186		if (min_cpu == -1)
187			min_cpu = i;
188		max_cpu = i;
189		cnt++;
190	}
191
192	__TEST_REQUIRE(cnt >= 2,
193		       "Only one usable CPU, task migration not possible");
194}
195
196static void help(const char *name)
197{
198	puts("");
199	printf("usage: %s [-h] [-u] [-l latency]\n", name);
200	printf(" -u: Don't sanity check the number of successful KVM_RUNs\n");
201	printf(" -l: Set /dev/cpu_dma_latency to suppress deep sleep states\n");
202	puts("");
203	exit(0);
204}
205
206int main(int argc, char *argv[])
207{
208	int r, i, snapshot, opt, fd = -1, latency = -1;
209	bool skip_sanity_check = false;
210	struct kvm_vm *vm;
211	struct kvm_vcpu *vcpu;
212	u32 cpu, rseq_cpu;
213
214	while ((opt = getopt(argc, argv, "hl:u")) != -1) {
215		switch (opt) {
216		case 'u':
217			skip_sanity_check = true;
218			break;
219		case 'l':
220			latency = atoi_paranoid(optarg);
221			break;
222		case 'h':
223		default:
224			help(argv[0]);
225			break;
226		}
227	}
228
229	r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask);
230	TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)", errno,
231		    strerror(errno));
232
233	calc_min_max_cpu();
234
235	r = rseq_register_current_thread();
236	TEST_ASSERT(!r, "rseq_register_current_thread failed, errno = %d (%s)",
237		    errno, strerror(errno));
238
239	/*
240	 * Create and run a dummy VM that immediately exits to userspace via
241	 * GUEST_SYNC, while concurrently migrating the process by setting its
242	 * CPU affinity.
243	 */
244	vm = vm_create_with_one_vcpu(&vcpu, guest_code);
245
246	pthread_create(&migration_thread, NULL, migration_worker,
247		       (void *)(unsigned long)syscall(SYS_gettid));
248
249	if (latency >= 0) {
250		/*
251		 * Writes to cpu_dma_latency persist only while the file is
252		 * open, i.e. it allows userspace to provide guaranteed latency
253		 * while running a workload.  Keep the file open until the test
254		 * completes, otherwise writing cpu_dma_latency is meaningless.
255		 */
256		fd = open("/dev/cpu_dma_latency", O_RDWR);
257		TEST_ASSERT(fd >= 0, __KVM_SYSCALL_ERROR("open() /dev/cpu_dma_latency", fd));
258
259		r = write(fd, &latency, 4);
260		TEST_ASSERT(r >= 1, "Error setting /dev/cpu_dma_latency");
261	}
262
263	for (i = 0; !done; i++) {
264		vcpu_run(vcpu);
265		TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC,
266			    "Guest failed?");
267
268		/*
269		 * Verify rseq's CPU matches sched's CPU.  Ensure migration
270		 * doesn't occur between getcpu() and reading the rseq cpu_id
271		 * by rereading both if the sequence count changes, or if the
272		 * count is odd (migration in-progress).
273		 */
274		do {
275			/*
276			 * Drop bit 0 to force a mismatch if the count is odd,
277			 * i.e. if a migration is in-progress.
278			 */
279			snapshot = atomic_read(&seq_cnt) & ~1;
280
281			/*
282			 * Ensure calling getcpu() and reading rseq.cpu_id complete
283			 * in a single "no migration" window, i.e. are not reordered
284			 * across the seq_cnt reads.
285			 */
286			smp_rmb();
287			r = sys_getcpu(&cpu, NULL);
288			TEST_ASSERT(!r, "getcpu failed, errno = %d (%s)",
289				    errno, strerror(errno));
290			rseq_cpu = rseq_current_cpu_raw();
291			smp_rmb();
292		} while (snapshot != atomic_read(&seq_cnt));
293
294		TEST_ASSERT(rseq_cpu == cpu,
295			    "rseq CPU = %d, sched CPU = %d", rseq_cpu, cpu);
296	}
297
298	if (fd > 0)
299		close(fd);
300
301	/*
302	 * Sanity check that the test was able to enter the guest a reasonable
303	 * number of times, e.g. didn't get stalled too often/long waiting for
304	 * getcpu() to stabilize.  A 2:1 migration:KVM_RUN ratio is a fairly
305	 * conservative ratio on x86-64, which can do _more_ KVM_RUNs than
306	 * migrations given the 1us+ delay in the migration task.
307	 *
308	 * Another reason why it may have small migration:KVM_RUN ratio is that,
309	 * on systems with large low power mode wakeup latency, it may happen
310	 * quite often that the scheduler is not able to wake up the target CPU
311	 * before the vCPU thread is scheduled to another CPU.
312	 */
313	TEST_ASSERT(skip_sanity_check || i > (NR_TASK_MIGRATIONS / 2),
314		    "Only performed %d KVM_RUNs, task stalled too much?\n\n"
315		    "  Try disabling deep sleep states to reduce CPU wakeup latency,\n"
316		    "  e.g. via cpuidle.off=1 or via -l <latency>, or run with -u to\n"
317		    "  disable this sanity check.", i);
318
319	pthread_join(migration_thread, NULL);
320
321	kvm_vm_free(vm);
322
323	rseq_unregister_current_thread();
324
325	return 0;
326}