kcsan_test.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * KCSAN test with various race scenarious to test runtime behaviour. Since the
   4 * interface with which KCSAN's reports are obtained is via the console, this is
   5 * the output we should verify. For each test case checks the presence (or
   6 * absence) of generated reports. Relies on 'console' tracepoint to capture
   7 * reports as they appear in the kernel log.
   8 *
   9 * Makes use of KUnit for test organization, and the Torture framework for test
  10 * thread control.
  11 *
  12 * Copyright (C) 2020, Google LLC.
  13 * Author: Marco Elver <elver@google.com>
  14 */
  15
  16#define pr_fmt(fmt) "kcsan_test: " fmt
  17
  18#include <kunit/test.h>
  19#include <linux/atomic.h>
  20#include <linux/bitops.h>
  21#include <linux/jiffies.h>
  22#include <linux/kcsan-checks.h>
  23#include <linux/kernel.h>
  24#include <linux/mutex.h>
  25#include <linux/sched.h>
  26#include <linux/seqlock.h>
  27#include <linux/spinlock.h>
  28#include <linux/string.h>
  29#include <linux/timer.h>
  30#include <linux/torture.h>
  31#include <linux/tracepoint.h>
  32#include <linux/types.h>
  33#include <trace/events/printk.h>
  34
  35#define KCSAN_TEST_REQUIRES(test, cond) do {			\
  36	if (!(cond))						\
  37		kunit_skip((test), "Test requires: " #cond);	\
  38} while (0)
  39
  40#ifdef CONFIG_CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE
  41#define __KCSAN_ACCESS_RW(alt) (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE)
  42#else
  43#define __KCSAN_ACCESS_RW(alt) (alt)
  44#endif
  45
  46/* Points to current test-case memory access "kernels". */
  47static void (*access_kernels[2])(void);
  48
  49static struct task_struct **threads; /* Lists of threads. */
  50static unsigned long end_time;       /* End time of test. */
  51
  52/* Report as observed from console. */
  53static struct {
  54	spinlock_t lock;
  55	int nlines;
  56	char lines[3][512];
  57} observed = {
  58	.lock = __SPIN_LOCK_UNLOCKED(observed.lock),
  59};
  60
  61/* Setup test checking loop. */
  62static __no_kcsan inline void
  63begin_test_checks(void (*func1)(void), void (*func2)(void))
  64{
  65	kcsan_disable_current();
  66
  67	/*
  68	 * Require at least as long as KCSAN_REPORT_ONCE_IN_MS, to ensure at
  69	 * least one race is reported.
  70	 */
  71	end_time = jiffies + msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS + 500);
  72
  73	/* Signal start; release potential initialization of shared data. */
  74	smp_store_release(&access_kernels[0], func1);
  75	smp_store_release(&access_kernels[1], func2);
  76}
  77
  78/* End test checking loop. */
  79static __no_kcsan inline bool
  80end_test_checks(bool stop)
  81{
  82	if (!stop && time_before(jiffies, end_time)) {
  83		/* Continue checking */
  84		might_sleep();
  85		return false;
  86	}
  87
  88	kcsan_enable_current();
  89	return true;
  90}
  91
  92/*
  93 * Probe for console output: checks if a race was reported, and obtains observed
  94 * lines of interest.
  95 */
  96__no_kcsan
  97static void probe_console(void *ignore, const char *buf, size_t len)
  98{
  99	unsigned long flags;
 100	int nlines;
 101
 102	/*
 103	 * Note that KCSAN reports under a global lock, so we do not risk the
 104	 * possibility of having multiple reports interleaved. If that were the
 105	 * case, we'd expect tests to fail.
 106	 */
 107
 108	spin_lock_irqsave(&observed.lock, flags);
 109	nlines = observed.nlines;
 110
 111	if (strnstr(buf, "BUG: KCSAN: ", len) && strnstr(buf, "test_", len)) {
 112		/*
 113		 * KCSAN report and related to the test.
 114		 *
 115		 * The provided @buf is not NUL-terminated; copy no more than
 116		 * @len bytes and let strscpy() add the missing NUL-terminator.
 117		 */
 118		strscpy(observed.lines[0], buf, min(len + 1, sizeof(observed.lines[0])));
 119		nlines = 1;
 120	} else if ((nlines == 1 || nlines == 2) && strnstr(buf, "bytes by", len)) {
 121		strscpy(observed.lines[nlines++], buf, min(len + 1, sizeof(observed.lines[0])));
 122
 123		if (strnstr(buf, "race at unknown origin", len)) {
 124			if (WARN_ON(nlines != 2))
 125				goto out;
 126
 127			/* No second line of interest. */
 128			strscpy(observed.lines[nlines++], "<none>");
 129		}
 130	}
 131
 132out:
 133	WRITE_ONCE(observed.nlines, nlines); /* Publish new nlines. */
 134	spin_unlock_irqrestore(&observed.lock, flags);
 135}
 136
 137/* Check if a report related to the test exists. */
 138__no_kcsan
 139static bool report_available(void)
 140{
 141	return READ_ONCE(observed.nlines) == ARRAY_SIZE(observed.lines);
 142}
 143
 144/* Report information we expect in a report. */
 145struct expect_report {
 146	/* Access information of both accesses. */
 147	struct {
 148		void *fn;    /* Function pointer to expected function of top frame. */
 149		void *addr;  /* Address of access; unchecked if NULL. */
 150		size_t size; /* Size of access; unchecked if @addr is NULL. */
 151		int type;    /* Access type, see KCSAN_ACCESS definitions. */
 152	} access[2];
 153};
 154
 155/* Check observed report matches information in @r. */
 156__no_kcsan
 157static bool __report_matches(const struct expect_report *r)
 158{
 159	const bool is_assert = (r->access[0].type | r->access[1].type) & KCSAN_ACCESS_ASSERT;
 160	bool ret = false;
 161	unsigned long flags;
 162	typeof(*observed.lines) *expect;
 163	const char *end;
 164	char *cur;
 165	int i;
 166
 167	/* Doubled-checked locking. */
 168	if (!report_available())
 169		return false;
 170
 171	expect = kmalloc(sizeof(observed.lines), GFP_KERNEL);
 172	if (WARN_ON(!expect))
 173		return false;
 174
 175	/* Generate expected report contents. */
 176
 177	/* Title */
 178	cur = expect[0];
 179	end = &expect[0][sizeof(expect[0]) - 1];
 180	cur += scnprintf(cur, end - cur, "BUG: KCSAN: %s in ",
 181			 is_assert ? "assert: race" : "data-race");
 182	if (r->access[1].fn) {
 183		char tmp[2][64];
 184		int cmp;
 185
 186		/* Expect lexographically sorted function names in title. */
 187		scnprintf(tmp[0], sizeof(tmp[0]), "%pS", r->access[0].fn);
 188		scnprintf(tmp[1], sizeof(tmp[1]), "%pS", r->access[1].fn);
 189		cmp = strcmp(tmp[0], tmp[1]);
 190		cur += scnprintf(cur, end - cur, "%ps / %ps",
 191				 cmp < 0 ? r->access[0].fn : r->access[1].fn,
 192				 cmp < 0 ? r->access[1].fn : r->access[0].fn);
 193	} else {
 194		scnprintf(cur, end - cur, "%pS", r->access[0].fn);
 195		/* The exact offset won't match, remove it. */
 196		cur = strchr(expect[0], '+');
 197		if (cur)
 198			*cur = '\0';
 199	}
 200
 201	/* Access 1 */
 202	cur = expect[1];
 203	end = &expect[1][sizeof(expect[1]) - 1];
 204	if (!r->access[1].fn)
 205		cur += scnprintf(cur, end - cur, "race at unknown origin, with ");
 206
 207	/* Access 1 & 2 */
 208	for (i = 0; i < 2; ++i) {
 209		const int ty = r->access[i].type;
 210		const char *const access_type =
 211			(ty & KCSAN_ACCESS_ASSERT) ?
 212				      ((ty & KCSAN_ACCESS_WRITE) ?
 213					       "assert no accesses" :
 214					       "assert no writes") :
 215				      ((ty & KCSAN_ACCESS_WRITE) ?
 216					       ((ty & KCSAN_ACCESS_COMPOUND) ?
 217							"read-write" :
 218							"write") :
 219					       "read");
 220		const bool is_atomic = (ty & KCSAN_ACCESS_ATOMIC);
 221		const bool is_scoped = (ty & KCSAN_ACCESS_SCOPED);
 222		const char *const access_type_aux =
 223				(is_atomic && is_scoped)	? " (marked, reordered)"
 224				: (is_atomic			? " (marked)"
 225				   : (is_scoped			? " (reordered)" : ""));
 226
 227		if (i == 1) {
 228			/* Access 2 */
 229			cur = expect[2];
 230			end = &expect[2][sizeof(expect[2]) - 1];
 231
 232			if (!r->access[1].fn) {
 233				/* Dummy string if no second access is available. */
 234				strscpy(expect[2], "<none>");
 235				break;
 236			}
 237		}
 238
 239		cur += scnprintf(cur, end - cur, "%s%s to ", access_type,
 240				 access_type_aux);
 241
 242		if (r->access[i].addr) /* Address is optional. */
 243			cur += scnprintf(cur, end - cur, "0x%px of %zu bytes",
 244					 r->access[i].addr, r->access[i].size);
 245	}
 246
 247	spin_lock_irqsave(&observed.lock, flags);
 248	if (!report_available())
 249		goto out; /* A new report is being captured. */
 250
 251	/* Finally match expected output to what we actually observed. */
 252	ret = strstr(observed.lines[0], expect[0]) &&
 253	      /* Access info may appear in any order. */
 254	      ((strstr(observed.lines[1], expect[1]) &&
 255		strstr(observed.lines[2], expect[2])) ||
 256	       (strstr(observed.lines[1], expect[2]) &&
 257		strstr(observed.lines[2], expect[1])));
 258out:
 259	spin_unlock_irqrestore(&observed.lock, flags);
 260	kfree(expect);
 261	return ret;
 262}
 263
 264static __always_inline const struct expect_report *
 265__report_set_scoped(struct expect_report *r, int accesses)
 266{
 267	BUILD_BUG_ON(accesses > 3);
 268
 269	if (accesses & 1)
 270		r->access[0].type |= KCSAN_ACCESS_SCOPED;
 271	else
 272		r->access[0].type &= ~KCSAN_ACCESS_SCOPED;
 273
 274	if (accesses & 2)
 275		r->access[1].type |= KCSAN_ACCESS_SCOPED;
 276	else
 277		r->access[1].type &= ~KCSAN_ACCESS_SCOPED;
 278
 279	return r;
 280}
 281
 282__no_kcsan
 283static bool report_matches_any_reordered(struct expect_report *r)
 284{
 285	return __report_matches(__report_set_scoped(r, 0)) ||
 286	       __report_matches(__report_set_scoped(r, 1)) ||
 287	       __report_matches(__report_set_scoped(r, 2)) ||
 288	       __report_matches(__report_set_scoped(r, 3));
 289}
 290
 291#ifdef CONFIG_KCSAN_WEAK_MEMORY
 292/* Due to reordering accesses, any access may appear as "(reordered)". */
 293#define report_matches report_matches_any_reordered
 294#else
 295#define report_matches __report_matches
 296#endif
 297
 298/* ===== Test kernels ===== */
 299
 300static long test_sink;
 301static long test_var;
 302/* @test_array should be large enough to fall into multiple watchpoint slots. */
 303static long test_array[3 * PAGE_SIZE / sizeof(long)];
 304static struct {
 305	long val[8];
 306} test_struct;
 307static long __data_racy test_data_racy;
 308static DEFINE_SEQLOCK(test_seqlock);
 309static DEFINE_SPINLOCK(test_spinlock);
 310static DEFINE_MUTEX(test_mutex);
 311
 312/*
 313 * Helper to avoid compiler optimizing out reads, and to generate source values
 314 * for writes.
 315 */
 316__no_kcsan
 317static noinline void sink_value(long v) { WRITE_ONCE(test_sink, v); }
 318
 319/*
 320 * Generates a delay and some accesses that enter the runtime but do not produce
 321 * data races.
 322 */
 323static noinline void test_delay(int iter)
 324{
 325	while (iter--)
 326		sink_value(READ_ONCE(test_sink));
 327}
 328
 329static noinline void test_kernel_read(void) { sink_value(test_var); }
 330
 331static noinline void test_kernel_write(void)
 332{
 333	test_var = READ_ONCE_NOCHECK(test_sink) + 1;
 334}
 335
 336static noinline void test_kernel_write_nochange(void) { test_var = 42; }
 337
 338/* Suffixed by value-change exception filter. */
 339static noinline void test_kernel_write_nochange_rcu(void) { test_var = 42; }
 340
 341static noinline void test_kernel_read_atomic(void)
 342{
 343	sink_value(READ_ONCE(test_var));
 344}
 345
 346static noinline void test_kernel_write_atomic(void)
 347{
 348	WRITE_ONCE(test_var, READ_ONCE_NOCHECK(test_sink) + 1);
 349}
 350
 351static noinline void test_kernel_atomic_rmw(void)
 352{
 353	/* Use builtin, so we can set up the "bad" atomic/non-atomic scenario. */
 354	__atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED);
 355}
 356
 357__no_kcsan
 358static noinline void test_kernel_write_uninstrumented(void) { test_var++; }
 359
 360static noinline void test_kernel_data_race(void) { data_race(test_var++); }
 361
 362static noinline void test_kernel_data_racy_qualifier(void) { test_data_racy++; }
 363
 364static noinline void test_kernel_assert_writer(void)
 365{
 366	ASSERT_EXCLUSIVE_WRITER(test_var);
 367}
 368
 369static noinline void test_kernel_assert_access(void)
 370{
 371	ASSERT_EXCLUSIVE_ACCESS(test_var);
 372}
 373
 374#define TEST_CHANGE_BITS 0xff00ff00
 375
 376static noinline void test_kernel_change_bits(void)
 377{
 378	if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) {
 379		/*
 380		 * Avoid race of unknown origin for this test, just pretend they
 381		 * are atomic.
 382		 */
 383		kcsan_nestable_atomic_begin();
 384		test_var ^= TEST_CHANGE_BITS;
 385		kcsan_nestable_atomic_end();
 386	} else
 387		WRITE_ONCE(test_var, READ_ONCE(test_var) ^ TEST_CHANGE_BITS);
 388}
 389
 390static noinline void test_kernel_assert_bits_change(void)
 391{
 392	ASSERT_EXCLUSIVE_BITS(test_var, TEST_CHANGE_BITS);
 393}
 394
 395static noinline void test_kernel_assert_bits_nochange(void)
 396{
 397	ASSERT_EXCLUSIVE_BITS(test_var, ~TEST_CHANGE_BITS);
 398}
 399
 400/*
 401 * Scoped assertions do trigger anywhere in scope. However, the report should
 402 * still only point at the start of the scope.
 403 */
 404static noinline void test_enter_scope(void)
 405{
 406	int x = 0;
 407
 408	/* Unrelated accesses to scoped assert. */
 409	READ_ONCE(test_sink);
 410	kcsan_check_read(&x, sizeof(x));
 411}
 412
 413static noinline void test_kernel_assert_writer_scoped(void)
 414{
 415	ASSERT_EXCLUSIVE_WRITER_SCOPED(test_var);
 416	test_enter_scope();
 417}
 418
 419static noinline void test_kernel_assert_access_scoped(void)
 420{
 421	ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_var);
 422	test_enter_scope();
 423}
 424
 425static noinline void test_kernel_rmw_array(void)
 426{
 427	int i;
 428
 429	for (i = 0; i < ARRAY_SIZE(test_array); ++i)
 430		test_array[i]++;
 431}
 432
 433static noinline void test_kernel_write_struct(void)
 434{
 435	kcsan_check_write(&test_struct, sizeof(test_struct));
 436	kcsan_disable_current();
 437	test_struct.val[3]++; /* induce value change */
 438	kcsan_enable_current();
 439}
 440
 441static noinline void test_kernel_write_struct_part(void)
 442{
 443	test_struct.val[3] = 42;
 444}
 445
 446static noinline void test_kernel_read_struct_zero_size(void)
 447{
 448	kcsan_check_read(&test_struct.val[3], 0);
 449}
 450
 451static noinline void test_kernel_jiffies_reader(void)
 452{
 453	sink_value((long)jiffies);
 454}
 455
 456static noinline void test_kernel_seqlock_reader(void)
 457{
 458	unsigned int seq;
 459
 460	do {
 461		seq = read_seqbegin(&test_seqlock);
 462		sink_value(test_var);
 463	} while (read_seqretry(&test_seqlock, seq));
 464}
 465
 466static noinline void test_kernel_seqlock_writer(void)
 467{
 468	unsigned long flags;
 469
 470	write_seqlock_irqsave(&test_seqlock, flags);
 471	test_var++;
 472	write_sequnlock_irqrestore(&test_seqlock, flags);
 473}
 474
 475static noinline void test_kernel_atomic_builtins(void)
 476{
 477	/*
 478	 * Generate concurrent accesses, expecting no reports, ensuring KCSAN
 479	 * treats builtin atomics as actually atomic.
 480	 */
 481	__atomic_load_n(&test_var, __ATOMIC_RELAXED);
 482}
 483
 484static noinline void test_kernel_xor_1bit(void)
 485{
 486	/* Do not report data races between the read-writes. */
 487	kcsan_nestable_atomic_begin();
 488	test_var ^= 0x10000;
 489	kcsan_nestable_atomic_end();
 490}
 491
 492#define TEST_KERNEL_LOCKED(name, acquire, release)		\
 493	static noinline void test_kernel_##name(void)		\
 494	{							\
 495		long *flag = &test_struct.val[0];		\
 496		long v = 0;					\
 497		if (!(acquire))					\
 498			return;					\
 499		while (v++ < 100) {				\
 500			test_var++;				\
 501			barrier();				\
 502		}						\
 503		release;					\
 504		test_delay(10);					\
 505	}
 506
 507TEST_KERNEL_LOCKED(with_memorder,
 508		   cmpxchg_acquire(flag, 0, 1) == 0,
 509		   smp_store_release(flag, 0));
 510TEST_KERNEL_LOCKED(wrong_memorder,
 511		   cmpxchg_relaxed(flag, 0, 1) == 0,
 512		   WRITE_ONCE(*flag, 0));
 513TEST_KERNEL_LOCKED(atomic_builtin_with_memorder,
 514		   __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED),
 515		   __atomic_store_n(flag, 0, __ATOMIC_RELEASE));
 516TEST_KERNEL_LOCKED(atomic_builtin_wrong_memorder,
 517		   __atomic_compare_exchange_n(flag, &v, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED),
 518		   __atomic_store_n(flag, 0, __ATOMIC_RELAXED));
 519
 520/* ===== Test cases ===== */
 521
 522/*
 523 * Tests that various barriers have the expected effect on internal state. Not
 524 * exhaustive on atomic_t operations. Unlike the selftest, also checks for
 525 * too-strict barrier instrumentation; these can be tolerated, because it does
 526 * not cause false positives, but at least we should be aware of such cases.
 527 */
 528static void test_barrier_nothreads(struct kunit *test)
 529{
 530#ifdef CONFIG_KCSAN_WEAK_MEMORY
 531	struct kcsan_scoped_access *reorder_access = &current->kcsan_ctx.reorder_access;
 532#else
 533	struct kcsan_scoped_access *reorder_access = NULL;
 534#endif
 535	arch_spinlock_t arch_spinlock = __ARCH_SPIN_LOCK_UNLOCKED;
 536	atomic_t dummy = ATOMIC_INIT(0);
 537
 538	KCSAN_TEST_REQUIRES(test, reorder_access != NULL);
 539	KCSAN_TEST_REQUIRES(test, IS_ENABLED(CONFIG_SMP));
 540
 541#define __KCSAN_EXPECT_BARRIER(access_type, barrier, order_before, name)			\
 542	do {											\
 543		reorder_access->type = (access_type) | KCSAN_ACCESS_SCOPED;			\
 544		reorder_access->size = sizeof(test_var);					\
 545		barrier;									\
 546		KUNIT_EXPECT_EQ_MSG(test, reorder_access->size,					\
 547				    order_before ? 0 : sizeof(test_var),			\
 548				    "improperly instrumented type=(" #access_type "): " name);	\
 549	} while (0)
 550#define KCSAN_EXPECT_READ_BARRIER(b, o)  __KCSAN_EXPECT_BARRIER(0, b, o, #b)
 551#define KCSAN_EXPECT_WRITE_BARRIER(b, o) __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_WRITE, b, o, #b)
 552#define KCSAN_EXPECT_RW_BARRIER(b, o)    __KCSAN_EXPECT_BARRIER(KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE, b, o, #b)
 553
 554	/*
 555	 * Lockdep initialization can strengthen certain locking operations due
 556	 * to calling into instrumented files; "warm up" our locks.
 557	 */
 558	spin_lock(&test_spinlock);
 559	spin_unlock(&test_spinlock);
 560	mutex_lock(&test_mutex);
 561	mutex_unlock(&test_mutex);
 562
 563	/* Force creating a valid entry in reorder_access first. */
 564	test_var = 0;
 565	while (test_var++ < 1000000 && reorder_access->size != sizeof(test_var))
 566		__kcsan_check_read(&test_var, sizeof(test_var));
 567	KUNIT_ASSERT_EQ(test, reorder_access->size, sizeof(test_var));
 568
 569	kcsan_nestable_atomic_begin(); /* No watchpoints in called functions. */
 570
 571	KCSAN_EXPECT_READ_BARRIER(mb(), true);
 572	KCSAN_EXPECT_READ_BARRIER(wmb(), false);
 573	KCSAN_EXPECT_READ_BARRIER(rmb(), true);
 574	KCSAN_EXPECT_READ_BARRIER(smp_mb(), true);
 575	KCSAN_EXPECT_READ_BARRIER(smp_wmb(), false);
 576	KCSAN_EXPECT_READ_BARRIER(smp_rmb(), true);
 577	KCSAN_EXPECT_READ_BARRIER(dma_wmb(), false);
 578	KCSAN_EXPECT_READ_BARRIER(dma_rmb(), true);
 579	KCSAN_EXPECT_READ_BARRIER(smp_mb__before_atomic(), true);
 580	KCSAN_EXPECT_READ_BARRIER(smp_mb__after_atomic(), true);
 581	KCSAN_EXPECT_READ_BARRIER(smp_mb__after_spinlock(), true);
 582	KCSAN_EXPECT_READ_BARRIER(smp_store_mb(test_var, 0), true);
 583	KCSAN_EXPECT_READ_BARRIER(smp_load_acquire(&test_var), false);
 584	KCSAN_EXPECT_READ_BARRIER(smp_store_release(&test_var, 0), true);
 585	KCSAN_EXPECT_READ_BARRIER(xchg(&test_var, 0), true);
 586	KCSAN_EXPECT_READ_BARRIER(xchg_release(&test_var, 0), true);
 587	KCSAN_EXPECT_READ_BARRIER(xchg_relaxed(&test_var, 0), false);
 588	KCSAN_EXPECT_READ_BARRIER(cmpxchg(&test_var, 0,  0), true);
 589	KCSAN_EXPECT_READ_BARRIER(cmpxchg_release(&test_var, 0,  0), true);
 590	KCSAN_EXPECT_READ_BARRIER(cmpxchg_relaxed(&test_var, 0,  0), false);
 591	KCSAN_EXPECT_READ_BARRIER(atomic_read(&dummy), false);
 592	KCSAN_EXPECT_READ_BARRIER(atomic_read_acquire(&dummy), false);
 593	KCSAN_EXPECT_READ_BARRIER(atomic_set(&dummy, 0), false);
 594	KCSAN_EXPECT_READ_BARRIER(atomic_set_release(&dummy, 0), true);
 595	KCSAN_EXPECT_READ_BARRIER(atomic_add(1, &dummy), false);
 596	KCSAN_EXPECT_READ_BARRIER(atomic_add_return(1, &dummy), true);
 597	KCSAN_EXPECT_READ_BARRIER(atomic_add_return_acquire(1, &dummy), false);
 598	KCSAN_EXPECT_READ_BARRIER(atomic_add_return_release(1, &dummy), true);
 599	KCSAN_EXPECT_READ_BARRIER(atomic_add_return_relaxed(1, &dummy), false);
 600	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add(1, &dummy), true);
 601	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_acquire(1, &dummy), false);
 602	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_release(1, &dummy), true);
 603	KCSAN_EXPECT_READ_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false);
 604	KCSAN_EXPECT_READ_BARRIER(test_and_set_bit(0, &test_var), true);
 605	KCSAN_EXPECT_READ_BARRIER(test_and_clear_bit(0, &test_var), true);
 606	KCSAN_EXPECT_READ_BARRIER(test_and_change_bit(0, &test_var), true);
 607	KCSAN_EXPECT_READ_BARRIER(clear_bit_unlock(0, &test_var), true);
 608	KCSAN_EXPECT_READ_BARRIER(__clear_bit_unlock(0, &test_var), true);
 609	KCSAN_EXPECT_READ_BARRIER(arch_spin_lock(&arch_spinlock), false);
 610	KCSAN_EXPECT_READ_BARRIER(arch_spin_unlock(&arch_spinlock), true);
 611	KCSAN_EXPECT_READ_BARRIER(spin_lock(&test_spinlock), false);
 612	KCSAN_EXPECT_READ_BARRIER(spin_unlock(&test_spinlock), true);
 613	KCSAN_EXPECT_READ_BARRIER(mutex_lock(&test_mutex), false);
 614	KCSAN_EXPECT_READ_BARRIER(mutex_unlock(&test_mutex), true);
 615
 616	KCSAN_EXPECT_WRITE_BARRIER(mb(), true);
 617	KCSAN_EXPECT_WRITE_BARRIER(wmb(), true);
 618	KCSAN_EXPECT_WRITE_BARRIER(rmb(), false);
 619	KCSAN_EXPECT_WRITE_BARRIER(smp_mb(), true);
 620	KCSAN_EXPECT_WRITE_BARRIER(smp_wmb(), true);
 621	KCSAN_EXPECT_WRITE_BARRIER(smp_rmb(), false);
 622	KCSAN_EXPECT_WRITE_BARRIER(dma_wmb(), true);
 623	KCSAN_EXPECT_WRITE_BARRIER(dma_rmb(), false);
 624	KCSAN_EXPECT_WRITE_BARRIER(smp_mb__before_atomic(), true);
 625	KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_atomic(), true);
 626	KCSAN_EXPECT_WRITE_BARRIER(smp_mb__after_spinlock(), true);
 627	KCSAN_EXPECT_WRITE_BARRIER(smp_store_mb(test_var, 0), true);
 628	KCSAN_EXPECT_WRITE_BARRIER(smp_load_acquire(&test_var), false);
 629	KCSAN_EXPECT_WRITE_BARRIER(smp_store_release(&test_var, 0), true);
 630	KCSAN_EXPECT_WRITE_BARRIER(xchg(&test_var, 0), true);
 631	KCSAN_EXPECT_WRITE_BARRIER(xchg_release(&test_var, 0), true);
 632	KCSAN_EXPECT_WRITE_BARRIER(xchg_relaxed(&test_var, 0), false);
 633	KCSAN_EXPECT_WRITE_BARRIER(cmpxchg(&test_var, 0,  0), true);
 634	KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_release(&test_var, 0,  0), true);
 635	KCSAN_EXPECT_WRITE_BARRIER(cmpxchg_relaxed(&test_var, 0,  0), false);
 636	KCSAN_EXPECT_WRITE_BARRIER(atomic_read(&dummy), false);
 637	KCSAN_EXPECT_WRITE_BARRIER(atomic_read_acquire(&dummy), false);
 638	KCSAN_EXPECT_WRITE_BARRIER(atomic_set(&dummy, 0), false);
 639	KCSAN_EXPECT_WRITE_BARRIER(atomic_set_release(&dummy, 0), true);
 640	KCSAN_EXPECT_WRITE_BARRIER(atomic_add(1, &dummy), false);
 641	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return(1, &dummy), true);
 642	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_acquire(1, &dummy), false);
 643	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_release(1, &dummy), true);
 644	KCSAN_EXPECT_WRITE_BARRIER(atomic_add_return_relaxed(1, &dummy), false);
 645	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add(1, &dummy), true);
 646	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_acquire(1, &dummy), false);
 647	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_release(1, &dummy), true);
 648	KCSAN_EXPECT_WRITE_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false);
 649	KCSAN_EXPECT_WRITE_BARRIER(test_and_set_bit(0, &test_var), true);
 650	KCSAN_EXPECT_WRITE_BARRIER(test_and_clear_bit(0, &test_var), true);
 651	KCSAN_EXPECT_WRITE_BARRIER(test_and_change_bit(0, &test_var), true);
 652	KCSAN_EXPECT_WRITE_BARRIER(clear_bit_unlock(0, &test_var), true);
 653	KCSAN_EXPECT_WRITE_BARRIER(__clear_bit_unlock(0, &test_var), true);
 654	KCSAN_EXPECT_WRITE_BARRIER(arch_spin_lock(&arch_spinlock), false);
 655	KCSAN_EXPECT_WRITE_BARRIER(arch_spin_unlock(&arch_spinlock), true);
 656	KCSAN_EXPECT_WRITE_BARRIER(spin_lock(&test_spinlock), false);
 657	KCSAN_EXPECT_WRITE_BARRIER(spin_unlock(&test_spinlock), true);
 658	KCSAN_EXPECT_WRITE_BARRIER(mutex_lock(&test_mutex), false);
 659	KCSAN_EXPECT_WRITE_BARRIER(mutex_unlock(&test_mutex), true);
 660
 661	KCSAN_EXPECT_RW_BARRIER(mb(), true);
 662	KCSAN_EXPECT_RW_BARRIER(wmb(), true);
 663	KCSAN_EXPECT_RW_BARRIER(rmb(), true);
 664	KCSAN_EXPECT_RW_BARRIER(smp_mb(), true);
 665	KCSAN_EXPECT_RW_BARRIER(smp_wmb(), true);
 666	KCSAN_EXPECT_RW_BARRIER(smp_rmb(), true);
 667	KCSAN_EXPECT_RW_BARRIER(dma_wmb(), true);
 668	KCSAN_EXPECT_RW_BARRIER(dma_rmb(), true);
 669	KCSAN_EXPECT_RW_BARRIER(smp_mb__before_atomic(), true);
 670	KCSAN_EXPECT_RW_BARRIER(smp_mb__after_atomic(), true);
 671	KCSAN_EXPECT_RW_BARRIER(smp_mb__after_spinlock(), true);
 672	KCSAN_EXPECT_RW_BARRIER(smp_store_mb(test_var, 0), true);
 673	KCSAN_EXPECT_RW_BARRIER(smp_load_acquire(&test_var), false);
 674	KCSAN_EXPECT_RW_BARRIER(smp_store_release(&test_var, 0), true);
 675	KCSAN_EXPECT_RW_BARRIER(xchg(&test_var, 0), true);
 676	KCSAN_EXPECT_RW_BARRIER(xchg_release(&test_var, 0), true);
 677	KCSAN_EXPECT_RW_BARRIER(xchg_relaxed(&test_var, 0), false);
 678	KCSAN_EXPECT_RW_BARRIER(cmpxchg(&test_var, 0,  0), true);
 679	KCSAN_EXPECT_RW_BARRIER(cmpxchg_release(&test_var, 0,  0), true);
 680	KCSAN_EXPECT_RW_BARRIER(cmpxchg_relaxed(&test_var, 0,  0), false);
 681	KCSAN_EXPECT_RW_BARRIER(atomic_read(&dummy), false);
 682	KCSAN_EXPECT_RW_BARRIER(atomic_read_acquire(&dummy), false);
 683	KCSAN_EXPECT_RW_BARRIER(atomic_set(&dummy, 0), false);
 684	KCSAN_EXPECT_RW_BARRIER(atomic_set_release(&dummy, 0), true);
 685	KCSAN_EXPECT_RW_BARRIER(atomic_add(1, &dummy), false);
 686	KCSAN_EXPECT_RW_BARRIER(atomic_add_return(1, &dummy), true);
 687	KCSAN_EXPECT_RW_BARRIER(atomic_add_return_acquire(1, &dummy), false);
 688	KCSAN_EXPECT_RW_BARRIER(atomic_add_return_release(1, &dummy), true);
 689	KCSAN_EXPECT_RW_BARRIER(atomic_add_return_relaxed(1, &dummy), false);
 690	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add(1, &dummy), true);
 691	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_acquire(1, &dummy), false);
 692	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_release(1, &dummy), true);
 693	KCSAN_EXPECT_RW_BARRIER(atomic_fetch_add_relaxed(1, &dummy), false);
 694	KCSAN_EXPECT_RW_BARRIER(test_and_set_bit(0, &test_var), true);
 695	KCSAN_EXPECT_RW_BARRIER(test_and_clear_bit(0, &test_var), true);
 696	KCSAN_EXPECT_RW_BARRIER(test_and_change_bit(0, &test_var), true);
 697	KCSAN_EXPECT_RW_BARRIER(clear_bit_unlock(0, &test_var), true);
 698	KCSAN_EXPECT_RW_BARRIER(__clear_bit_unlock(0, &test_var), true);
 699	KCSAN_EXPECT_RW_BARRIER(arch_spin_lock(&arch_spinlock), false);
 700	KCSAN_EXPECT_RW_BARRIER(arch_spin_unlock(&arch_spinlock), true);
 701	KCSAN_EXPECT_RW_BARRIER(spin_lock(&test_spinlock), false);
 702	KCSAN_EXPECT_RW_BARRIER(spin_unlock(&test_spinlock), true);
 703	KCSAN_EXPECT_RW_BARRIER(mutex_lock(&test_mutex), false);
 704	KCSAN_EXPECT_RW_BARRIER(mutex_unlock(&test_mutex), true);
 705	KCSAN_EXPECT_READ_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true);
 706	KCSAN_EXPECT_WRITE_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true);
 707	KCSAN_EXPECT_RW_BARRIER(xor_unlock_is_negative_byte(1, &test_var), true);
 708	kcsan_nestable_atomic_end();
 709}
 710
 711/* Simple test with normal data race. */
 712__no_kcsan
 713static void test_basic(struct kunit *test)
 714{
 715	struct expect_report expect = {
 716		.access = {
 717			{ test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 718			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 719		},
 720	};
 721	struct expect_report never = {
 722		.access = {
 723			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 724			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 725		},
 726	};
 727	bool match_expect = false;
 728	bool match_never = false;
 729
 730	begin_test_checks(test_kernel_write, test_kernel_read);
 731	do {
 732		match_expect |= report_matches(&expect);
 733		match_never = report_matches(&never);
 734	} while (!end_test_checks(match_never));
 735	KUNIT_EXPECT_TRUE(test, match_expect);
 736	KUNIT_EXPECT_FALSE(test, match_never);
 737}
 738
 739/*
 740 * Stress KCSAN with lots of concurrent races on different addresses until
 741 * timeout.
 742 */
 743__no_kcsan
 744static void test_concurrent_races(struct kunit *test)
 745{
 746	struct expect_report expect = {
 747		.access = {
 748			/* NULL will match any address. */
 749			{ test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
 750			{ test_kernel_rmw_array, NULL, 0, __KCSAN_ACCESS_RW(0) },
 751		},
 752	};
 753	struct expect_report never = {
 754		.access = {
 755			{ test_kernel_rmw_array, NULL, 0, 0 },
 756			{ test_kernel_rmw_array, NULL, 0, 0 },
 757		},
 758	};
 759	bool match_expect = false;
 760	bool match_never = false;
 761
 762	begin_test_checks(test_kernel_rmw_array, test_kernel_rmw_array);
 763	do {
 764		match_expect |= report_matches(&expect);
 765		match_never |= report_matches(&never);
 766	} while (!end_test_checks(false));
 767	KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check matches exist. */
 768	KUNIT_EXPECT_FALSE(test, match_never);
 769}
 770
 771/* Test the KCSAN_REPORT_VALUE_CHANGE_ONLY option. */
 772__no_kcsan
 773static void test_novalue_change(struct kunit *test)
 774{
 775	struct expect_report expect_rw = {
 776		.access = {
 777			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 778			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 779		},
 780	};
 781	struct expect_report expect_ww = {
 782		.access = {
 783			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 784			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 785		},
 786	};
 787	bool match_expect = false;
 788
 789	test_kernel_write_nochange(); /* Reset value. */
 790	begin_test_checks(test_kernel_write_nochange, test_kernel_read);
 791	do {
 792		match_expect = report_matches(&expect_rw) || report_matches(&expect_ww);
 793	} while (!end_test_checks(match_expect));
 794	if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY))
 795		KUNIT_EXPECT_FALSE(test, match_expect);
 796	else
 797		KUNIT_EXPECT_TRUE(test, match_expect);
 798}
 799
 800/*
 801 * Test that the rules where the KCSAN_REPORT_VALUE_CHANGE_ONLY option should
 802 * never apply work.
 803 */
 804__no_kcsan
 805static void test_novalue_change_exception(struct kunit *test)
 806{
 807	struct expect_report expect_rw = {
 808		.access = {
 809			{ test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 810			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 811		},
 812	};
 813	struct expect_report expect_ww = {
 814		.access = {
 815			{ test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 816			{ test_kernel_write_nochange_rcu, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 817		},
 818	};
 819	bool match_expect = false;
 820
 821	test_kernel_write_nochange_rcu(); /* Reset value. */
 822	begin_test_checks(test_kernel_write_nochange_rcu, test_kernel_read);
 823	do {
 824		match_expect = report_matches(&expect_rw) || report_matches(&expect_ww);
 825	} while (!end_test_checks(match_expect));
 826	KUNIT_EXPECT_TRUE(test, match_expect);
 827}
 828
 829/* Test that data races of unknown origin are reported. */
 830__no_kcsan
 831static void test_unknown_origin(struct kunit *test)
 832{
 833	struct expect_report expect = {
 834		.access = {
 835			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 836			{ NULL },
 837		},
 838	};
 839	bool match_expect = false;
 840
 841	begin_test_checks(test_kernel_write_uninstrumented, test_kernel_read);
 842	do {
 843		match_expect = report_matches(&expect);
 844	} while (!end_test_checks(match_expect));
 845	if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
 846		KUNIT_EXPECT_TRUE(test, match_expect);
 847	else
 848		KUNIT_EXPECT_FALSE(test, match_expect);
 849}
 850
 851/* Test KCSAN_ASSUME_PLAIN_WRITES_ATOMIC if it is selected. */
 852__no_kcsan
 853static void test_write_write_assume_atomic(struct kunit *test)
 854{
 855	struct expect_report expect = {
 856		.access = {
 857			{ test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 858			{ test_kernel_write, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
 859		},
 860	};
 861	bool match_expect = false;
 862
 863	begin_test_checks(test_kernel_write, test_kernel_write);
 864	do {
 865		sink_value(READ_ONCE(test_var)); /* induce value-change */
 866		match_expect = report_matches(&expect);
 867	} while (!end_test_checks(match_expect));
 868	if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC))
 869		KUNIT_EXPECT_FALSE(test, match_expect);
 870	else
 871		KUNIT_EXPECT_TRUE(test, match_expect);
 872}
 873
 874/*
 875 * Test that data races with writes larger than word-size are always reported,
 876 * even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
 877 */
 878__no_kcsan
 879static void test_write_write_struct(struct kunit *test)
 880{
 881	struct expect_report expect = {
 882		.access = {
 883			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 884			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 885		},
 886	};
 887	bool match_expect = false;
 888
 889	begin_test_checks(test_kernel_write_struct, test_kernel_write_struct);
 890	do {
 891		match_expect = report_matches(&expect);
 892	} while (!end_test_checks(match_expect));
 893	KUNIT_EXPECT_TRUE(test, match_expect);
 894}
 895
 896/*
 897 * Test that data races where only one write is larger than word-size are always
 898 * reported, even if KCSAN_ASSUME_PLAIN_WRITES_ATOMIC is selected.
 899 */
 900__no_kcsan
 901static void test_write_write_struct_part(struct kunit *test)
 902{
 903	struct expect_report expect = {
 904		.access = {
 905			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 906			{ test_kernel_write_struct_part, &test_struct.val[3], sizeof(test_struct.val[3]), KCSAN_ACCESS_WRITE },
 907		},
 908	};
 909	bool match_expect = false;
 910
 911	begin_test_checks(test_kernel_write_struct, test_kernel_write_struct_part);
 912	do {
 913		match_expect = report_matches(&expect);
 914	} while (!end_test_checks(match_expect));
 915	KUNIT_EXPECT_TRUE(test, match_expect);
 916}
 917
 918/* Test that races with atomic accesses never result in reports. */
 919__no_kcsan
 920static void test_read_atomic_write_atomic(struct kunit *test)
 921{
 922	bool match_never = false;
 923
 924	begin_test_checks(test_kernel_read_atomic, test_kernel_write_atomic);
 925	do {
 926		match_never = report_available();
 927	} while (!end_test_checks(match_never));
 928	KUNIT_EXPECT_FALSE(test, match_never);
 929}
 930
 931/* Test that a race with an atomic and plain access result in reports. */
 932__no_kcsan
 933static void test_read_plain_atomic_write(struct kunit *test)
 934{
 935	struct expect_report expect = {
 936		.access = {
 937			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 938			{ test_kernel_write_atomic, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
 939		},
 940	};
 941	bool match_expect = false;
 942
 943	KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS));
 944
 945	begin_test_checks(test_kernel_read, test_kernel_write_atomic);
 946	do {
 947		match_expect = report_matches(&expect);
 948	} while (!end_test_checks(match_expect));
 949	KUNIT_EXPECT_TRUE(test, match_expect);
 950}
 951
 952/* Test that atomic RMWs generate correct report. */
 953__no_kcsan
 954static void test_read_plain_atomic_rmw(struct kunit *test)
 955{
 956	struct expect_report expect = {
 957		.access = {
 958			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
 959			{ test_kernel_atomic_rmw, &test_var, sizeof(test_var),
 960				KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC },
 961		},
 962	};
 963	bool match_expect = false;
 964
 965	KCSAN_TEST_REQUIRES(test, !IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS));
 966
 967	begin_test_checks(test_kernel_read, test_kernel_atomic_rmw);
 968	do {
 969		match_expect = report_matches(&expect);
 970	} while (!end_test_checks(match_expect));
 971	KUNIT_EXPECT_TRUE(test, match_expect);
 972}
 973
 974/* Zero-sized accesses should never cause data race reports. */
 975__no_kcsan
 976static void test_zero_size_access(struct kunit *test)
 977{
 978	struct expect_report expect = {
 979		.access = {
 980			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 981			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 982		},
 983	};
 984	struct expect_report never = {
 985		.access = {
 986			{ test_kernel_write_struct, &test_struct, sizeof(test_struct), KCSAN_ACCESS_WRITE },
 987			{ test_kernel_read_struct_zero_size, &test_struct.val[3], 0, 0 },
 988		},
 989	};
 990	bool match_expect = false;
 991	bool match_never = false;
 992
 993	begin_test_checks(test_kernel_write_struct, test_kernel_read_struct_zero_size);
 994	do {
 995		match_expect |= report_matches(&expect);
 996		match_never = report_matches(&never);
 997	} while (!end_test_checks(match_never));
 998	KUNIT_EXPECT_TRUE(test, match_expect); /* Sanity check. */
 999	KUNIT_EXPECT_FALSE(test, match_never);
1000}
1001
1002/* Test the data_race() macro. */
1003__no_kcsan
1004static void test_data_race(struct kunit *test)
1005{
1006	bool match_never = false;
1007
1008	begin_test_checks(test_kernel_data_race, test_kernel_data_race);
1009	do {
1010		match_never = report_available();
1011	} while (!end_test_checks(match_never));
1012	KUNIT_EXPECT_FALSE(test, match_never);
1013}
1014
1015/* Test the __data_racy type qualifier. */
1016__no_kcsan
1017static void test_data_racy_qualifier(struct kunit *test)
1018{
1019	bool match_never = false;
1020
1021	begin_test_checks(test_kernel_data_racy_qualifier, test_kernel_data_racy_qualifier);
1022	do {
1023		match_never = report_available();
1024	} while (!end_test_checks(match_never));
1025	KUNIT_EXPECT_FALSE(test, match_never);
1026}
1027
1028__no_kcsan
1029static void test_assert_exclusive_writer(struct kunit *test)
1030{
1031	struct expect_report expect = {
1032		.access = {
1033			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1034			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
1035		},
1036	};
1037	bool match_expect = false;
1038
1039	begin_test_checks(test_kernel_assert_writer, test_kernel_write_nochange);
1040	do {
1041		match_expect = report_matches(&expect);
1042	} while (!end_test_checks(match_expect));
1043	KUNIT_EXPECT_TRUE(test, match_expect);
1044}
1045
1046__no_kcsan
1047static void test_assert_exclusive_access(struct kunit *test)
1048{
1049	struct expect_report expect = {
1050		.access = {
1051			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1052			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1053		},
1054	};
1055	bool match_expect = false;
1056
1057	begin_test_checks(test_kernel_assert_access, test_kernel_read);
1058	do {
1059		match_expect = report_matches(&expect);
1060	} while (!end_test_checks(match_expect));
1061	KUNIT_EXPECT_TRUE(test, match_expect);
1062}
1063
1064__no_kcsan
1065static void test_assert_exclusive_access_writer(struct kunit *test)
1066{
1067	struct expect_report expect_access_writer = {
1068		.access = {
1069			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1070			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1071		},
1072	};
1073	struct expect_report expect_access_access = {
1074		.access = {
1075			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1076			{ test_kernel_assert_access, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE },
1077		},
1078	};
1079	struct expect_report never = {
1080		.access = {
1081			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1082			{ test_kernel_assert_writer, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1083		},
1084	};
1085	bool match_expect_access_writer = false;
1086	bool match_expect_access_access = false;
1087	bool match_never = false;
1088
1089	begin_test_checks(test_kernel_assert_access, test_kernel_assert_writer);
1090	do {
1091		match_expect_access_writer |= report_matches(&expect_access_writer);
1092		match_expect_access_access |= report_matches(&expect_access_access);
1093		match_never |= report_matches(&never);
1094	} while (!end_test_checks(match_never));
1095	KUNIT_EXPECT_TRUE(test, match_expect_access_writer);
1096	KUNIT_EXPECT_TRUE(test, match_expect_access_access);
1097	KUNIT_EXPECT_FALSE(test, match_never);
1098}
1099
1100__no_kcsan
1101static void test_assert_exclusive_bits_change(struct kunit *test)
1102{
1103	struct expect_report expect = {
1104		.access = {
1105			{ test_kernel_assert_bits_change, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT },
1106			{ test_kernel_change_bits, &test_var, sizeof(test_var),
1107				KCSAN_ACCESS_WRITE | (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) ? 0 : KCSAN_ACCESS_ATOMIC) },
1108		},
1109	};
1110	bool match_expect = false;
1111
1112	begin_test_checks(test_kernel_assert_bits_change, test_kernel_change_bits);
1113	do {
1114		match_expect = report_matches(&expect);
1115	} while (!end_test_checks(match_expect));
1116	KUNIT_EXPECT_TRUE(test, match_expect);
1117}
1118
1119__no_kcsan
1120static void test_assert_exclusive_bits_nochange(struct kunit *test)
1121{
1122	bool match_never = false;
1123
1124	begin_test_checks(test_kernel_assert_bits_nochange, test_kernel_change_bits);
1125	do {
1126		match_never = report_available();
1127	} while (!end_test_checks(match_never));
1128	KUNIT_EXPECT_FALSE(test, match_never);
1129}
1130
1131__no_kcsan
1132static void test_assert_exclusive_writer_scoped(struct kunit *test)
1133{
1134	struct expect_report expect_start = {
1135		.access = {
1136			{ test_kernel_assert_writer_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
1137			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
1138		},
1139	};
1140	struct expect_report expect_inscope = {
1141		.access = {
1142			{ test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_SCOPED },
1143			{ test_kernel_write_nochange, &test_var, sizeof(test_var), KCSAN_ACCESS_WRITE },
1144		},
1145	};
1146	bool match_expect_start = false;
1147	bool match_expect_inscope = false;
1148
1149	begin_test_checks(test_kernel_assert_writer_scoped, test_kernel_write_nochange);
1150	do {
1151		match_expect_start |= report_matches(&expect_start);
1152		match_expect_inscope |= report_matches(&expect_inscope);
1153	} while (!end_test_checks(match_expect_inscope));
1154	KUNIT_EXPECT_TRUE(test, match_expect_start);
1155	KUNIT_EXPECT_FALSE(test, match_expect_inscope);
1156}
1157
1158__no_kcsan
1159static void test_assert_exclusive_access_scoped(struct kunit *test)
1160{
1161	struct expect_report expect_start1 = {
1162		.access = {
1163			{ test_kernel_assert_access_scoped, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
1164			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1165		},
1166	};
1167	struct expect_report expect_start2 = {
1168		.access = { expect_start1.access[0], expect_start1.access[0] },
1169	};
1170	struct expect_report expect_inscope = {
1171		.access = {
1172			{ test_enter_scope, &test_var, sizeof(test_var), KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_SCOPED },
1173			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1174		},
1175	};
1176	bool match_expect_start = false;
1177	bool match_expect_inscope = false;
1178
1179	begin_test_checks(test_kernel_assert_access_scoped, test_kernel_read);
1180	end_time += msecs_to_jiffies(1000); /* This test requires a bit more time. */
1181	do {
1182		match_expect_start |= report_matches(&expect_start1) || report_matches(&expect_start2);
1183		match_expect_inscope |= report_matches(&expect_inscope);
1184	} while (!end_test_checks(match_expect_inscope));
1185	KUNIT_EXPECT_TRUE(test, match_expect_start);
1186	KUNIT_EXPECT_FALSE(test, match_expect_inscope);
1187}
1188
1189/*
1190 * jiffies is special (declared to be volatile) and its accesses are typically
1191 * not marked; this test ensures that the compiler nor KCSAN gets confused about
1192 * jiffies's declaration on different architectures.
1193 */
1194__no_kcsan
1195static void test_jiffies_noreport(struct kunit *test)
1196{
1197	bool match_never = false;
1198
1199	begin_test_checks(test_kernel_jiffies_reader, test_kernel_jiffies_reader);
1200	do {
1201		match_never = report_available();
1202	} while (!end_test_checks(match_never));
1203	KUNIT_EXPECT_FALSE(test, match_never);
1204}
1205
1206/* Test that racing accesses in seqlock critical sections are not reported. */
1207__no_kcsan
1208static void test_seqlock_noreport(struct kunit *test)
1209{
1210	bool match_never = false;
1211
1212	begin_test_checks(test_kernel_seqlock_reader, test_kernel_seqlock_writer);
1213	do {
1214		match_never = report_available();
1215	} while (!end_test_checks(match_never));
1216	KUNIT_EXPECT_FALSE(test, match_never);
1217}
1218
1219/*
1220 * Test atomic builtins work and required instrumentation functions exist. We
1221 * also test that KCSAN understands they're atomic by racing with them via
1222 * test_kernel_atomic_builtins(), and expect no reports.
1223 *
1224 * The atomic builtins _SHOULD NOT_ be used in normal kernel code!
1225 */
1226static void test_atomic_builtins(struct kunit *test)
1227{
1228	bool match_never = false;
1229
1230	begin_test_checks(test_kernel_atomic_builtins, test_kernel_atomic_builtins);
1231	do {
1232		long tmp;
1233
1234		kcsan_enable_current();
1235
1236		__atomic_store_n(&test_var, 42L, __ATOMIC_RELAXED);
1237		KUNIT_EXPECT_EQ(test, 42L, __atomic_load_n(&test_var, __ATOMIC_RELAXED));
1238
1239		KUNIT_EXPECT_EQ(test, 42L, __atomic_exchange_n(&test_var, 20, __ATOMIC_RELAXED));
1240		KUNIT_EXPECT_EQ(test, 20L, test_var);
1241
1242		tmp = 20L;
1243		KUNIT_EXPECT_TRUE(test, __atomic_compare_exchange_n(&test_var, &tmp, 30L,
1244								    0, __ATOMIC_RELAXED,
1245								    __ATOMIC_RELAXED));
1246		KUNIT_EXPECT_EQ(test, tmp, 20L);
1247		KUNIT_EXPECT_EQ(test, test_var, 30L);
1248		KUNIT_EXPECT_FALSE(test, __atomic_compare_exchange_n(&test_var, &tmp, 40L,
1249								     1, __ATOMIC_RELAXED,
1250								     __ATOMIC_RELAXED));
1251		KUNIT_EXPECT_EQ(test, tmp, 30L);
1252		KUNIT_EXPECT_EQ(test, test_var, 30L);
1253
1254		KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_add(&test_var, 1, __ATOMIC_RELAXED));
1255		KUNIT_EXPECT_EQ(test, 31L, __atomic_fetch_sub(&test_var, 1, __ATOMIC_RELAXED));
1256		KUNIT_EXPECT_EQ(test, 30L, __atomic_fetch_and(&test_var, 0xf, __ATOMIC_RELAXED));
1257		KUNIT_EXPECT_EQ(test, 14L, __atomic_fetch_xor(&test_var, 0xf, __ATOMIC_RELAXED));
1258		KUNIT_EXPECT_EQ(test, 1L, __atomic_fetch_or(&test_var, 0xf0, __ATOMIC_RELAXED));
1259		KUNIT_EXPECT_EQ(test, 241L, __atomic_fetch_nand(&test_var, 0xf, __ATOMIC_RELAXED));
1260		KUNIT_EXPECT_EQ(test, -2L, test_var);
1261
1262		__atomic_thread_fence(__ATOMIC_SEQ_CST);
1263		__atomic_signal_fence(__ATOMIC_SEQ_CST);
1264
1265		kcsan_disable_current();
1266
1267		match_never = report_available();
1268	} while (!end_test_checks(match_never));
1269	KUNIT_EXPECT_FALSE(test, match_never);
1270}
1271
1272__no_kcsan
1273static void test_1bit_value_change(struct kunit *test)
1274{
1275	struct expect_report expect = {
1276		.access = {
1277			{ test_kernel_read, &test_var, sizeof(test_var), 0 },
1278			{ test_kernel_xor_1bit, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1279		},
1280	};
1281	bool match = false;
1282
1283	begin_test_checks(test_kernel_read, test_kernel_xor_1bit);
1284	do {
1285		match = IS_ENABLED(CONFIG_KCSAN_PERMISSIVE)
1286				? report_available()
1287				: report_matches(&expect);
1288	} while (!end_test_checks(match));
1289	if (IS_ENABLED(CONFIG_KCSAN_PERMISSIVE))
1290		KUNIT_EXPECT_FALSE(test, match);
1291	else
1292		KUNIT_EXPECT_TRUE(test, match);
1293}
1294
1295__no_kcsan
1296static void test_correct_barrier(struct kunit *test)
1297{
1298	struct expect_report expect = {
1299		.access = {
1300			{ test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1301			{ test_kernel_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1302		},
1303	};
1304	bool match_expect = false;
1305
1306	test_struct.val[0] = 0; /* init unlocked */
1307	begin_test_checks(test_kernel_with_memorder, test_kernel_with_memorder);
1308	do {
1309		match_expect = report_matches_any_reordered(&expect);
1310	} while (!end_test_checks(match_expect));
1311	KUNIT_EXPECT_FALSE(test, match_expect);
1312}
1313
1314__no_kcsan
1315static void test_missing_barrier(struct kunit *test)
1316{
1317	struct expect_report expect = {
1318		.access = {
1319			{ test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1320			{ test_kernel_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1321		},
1322	};
1323	bool match_expect = false;
1324
1325	test_struct.val[0] = 0; /* init unlocked */
1326	begin_test_checks(test_kernel_wrong_memorder, test_kernel_wrong_memorder);
1327	do {
1328		match_expect = report_matches_any_reordered(&expect);
1329	} while (!end_test_checks(match_expect));
1330	if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
1331		KUNIT_EXPECT_TRUE(test, match_expect);
1332	else
1333		KUNIT_EXPECT_FALSE(test, match_expect);
1334}
1335
1336__no_kcsan
1337static void test_atomic_builtins_correct_barrier(struct kunit *test)
1338{
1339	struct expect_report expect = {
1340		.access = {
1341			{ test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1342			{ test_kernel_atomic_builtin_with_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1343		},
1344	};
1345	bool match_expect = false;
1346
1347	test_struct.val[0] = 0; /* init unlocked */
1348	begin_test_checks(test_kernel_atomic_builtin_with_memorder,
1349			  test_kernel_atomic_builtin_with_memorder);
1350	do {
1351		match_expect = report_matches_any_reordered(&expect);
1352	} while (!end_test_checks(match_expect));
1353	KUNIT_EXPECT_FALSE(test, match_expect);
1354}
1355
1356__no_kcsan
1357static void test_atomic_builtins_missing_barrier(struct kunit *test)
1358{
1359	struct expect_report expect = {
1360		.access = {
1361			{ test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(KCSAN_ACCESS_WRITE) },
1362			{ test_kernel_atomic_builtin_wrong_memorder, &test_var, sizeof(test_var), __KCSAN_ACCESS_RW(0) },
1363		},
1364	};
1365	bool match_expect = false;
1366
1367	test_struct.val[0] = 0; /* init unlocked */
1368	begin_test_checks(test_kernel_atomic_builtin_wrong_memorder,
1369			  test_kernel_atomic_builtin_wrong_memorder);
1370	do {
1371		match_expect = report_matches_any_reordered(&expect);
1372	} while (!end_test_checks(match_expect));
1373	if (IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
1374		KUNIT_EXPECT_TRUE(test, match_expect);
1375	else
1376		KUNIT_EXPECT_FALSE(test, match_expect);
1377}
1378
1379/*
1380 * Generate thread counts for all test cases. Values generated are in interval
1381 * [2, 5] followed by exponentially increasing thread counts from 8 to 32.
1382 *
1383 * The thread counts are chosen to cover potentially interesting boundaries and
1384 * corner cases (2 to 5), and then stress the system with larger counts.
1385 */
1386static const void *nthreads_gen_params(struct kunit *test, const void *prev, char *desc)
1387{
1388	long nthreads = (long)prev;
1389
1390	if (nthreads < 0 || nthreads >= 32)
1391		nthreads = 0; /* stop */
1392	else if (!nthreads)
1393		nthreads = 2; /* initial value */
1394	else if (nthreads < 5)
1395		nthreads++;
1396	else if (nthreads == 5)
1397		nthreads = 8;
1398	else
1399		nthreads *= 2;
1400
1401	if (!preempt_model_preemptible() ||
1402	    !IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER)) {
1403		/*
1404		 * Without any preemption, keep 2 CPUs free for other tasks, one
1405		 * of which is the main test case function checking for
1406		 * completion or failure.
1407		 */
1408		const long min_unused_cpus = preempt_model_none() ? 2 : 0;
1409		const long min_required_cpus = 2 + min_unused_cpus;
1410
1411		if (num_online_cpus() < min_required_cpus) {
1412			pr_err_once("Too few online CPUs (%u < %ld) for test\n",
1413				    num_online_cpus(), min_required_cpus);
1414			nthreads = 0;
1415		} else if (nthreads >= num_online_cpus() - min_unused_cpus) {
1416			/* Use negative value to indicate last param. */
1417			nthreads = -(num_online_cpus() - min_unused_cpus);
1418			pr_warn_once("Limiting number of threads to %ld (only %d online CPUs)\n",
1419				     -nthreads, num_online_cpus());
1420		}
1421	}
1422
1423	snprintf(desc, KUNIT_PARAM_DESC_SIZE, "threads=%ld", abs(nthreads));
1424	return (void *)nthreads;
1425}
1426
1427#define KCSAN_KUNIT_CASE(test_name) KUNIT_CASE_PARAM(test_name, nthreads_gen_params)
1428static struct kunit_case kcsan_test_cases[] = {
1429	KUNIT_CASE(test_barrier_nothreads),
1430	KCSAN_KUNIT_CASE(test_basic),
1431	KCSAN_KUNIT_CASE(test_concurrent_races),
1432	KCSAN_KUNIT_CASE(test_novalue_change),
1433	KCSAN_KUNIT_CASE(test_novalue_change_exception),
1434	KCSAN_KUNIT_CASE(test_unknown_origin),
1435	KCSAN_KUNIT_CASE(test_write_write_assume_atomic),
1436	KCSAN_KUNIT_CASE(test_write_write_struct),
1437	KCSAN_KUNIT_CASE(test_write_write_struct_part),
1438	KCSAN_KUNIT_CASE(test_read_atomic_write_atomic),
1439	KCSAN_KUNIT_CASE(test_read_plain_atomic_write),
1440	KCSAN_KUNIT_CASE(test_read_plain_atomic_rmw),
1441	KCSAN_KUNIT_CASE(test_zero_size_access),
1442	KCSAN_KUNIT_CASE(test_data_race),
1443	KCSAN_KUNIT_CASE(test_data_racy_qualifier),
1444	KCSAN_KUNIT_CASE(test_assert_exclusive_writer),
1445	KCSAN_KUNIT_CASE(test_assert_exclusive_access),
1446	KCSAN_KUNIT_CASE(test_assert_exclusive_access_writer),
1447	KCSAN_KUNIT_CASE(test_assert_exclusive_bits_change),
1448	KCSAN_KUNIT_CASE(test_assert_exclusive_bits_nochange),
1449	KCSAN_KUNIT_CASE(test_assert_exclusive_writer_scoped),
1450	KCSAN_KUNIT_CASE(test_assert_exclusive_access_scoped),
1451	KCSAN_KUNIT_CASE(test_jiffies_noreport),
1452	KCSAN_KUNIT_CASE(test_seqlock_noreport),
1453	KCSAN_KUNIT_CASE(test_atomic_builtins),
1454	KCSAN_KUNIT_CASE(test_1bit_value_change),
1455	KCSAN_KUNIT_CASE(test_correct_barrier),
1456	KCSAN_KUNIT_CASE(test_missing_barrier),
1457	KCSAN_KUNIT_CASE(test_atomic_builtins_correct_barrier),
1458	KCSAN_KUNIT_CASE(test_atomic_builtins_missing_barrier),
1459	{},
1460};
1461
1462/* ===== End test cases ===== */
1463
1464/* Concurrent accesses from interrupts. */
1465__no_kcsan
1466static void access_thread_timer(struct timer_list *timer)
1467{
1468	static atomic_t cnt = ATOMIC_INIT(0);
1469	unsigned int idx;
1470	void (*func)(void);
1471
1472	idx = (unsigned int)atomic_inc_return(&cnt) % ARRAY_SIZE(access_kernels);
1473	/* Acquire potential initialization. */
1474	func = smp_load_acquire(&access_kernels[idx]);
1475	if (func)
1476		func();
1477}
1478
1479/* The main loop for each thread. */
1480__no_kcsan
1481static int access_thread(void *arg)
1482{
1483	struct timer_list timer;
1484	unsigned int cnt = 0;
1485	unsigned int idx;
1486	void (*func)(void);
1487
1488	timer_setup_on_stack(&timer, access_thread_timer, 0);
1489	do {
1490		might_sleep();
1491
1492		if (!timer_pending(&timer))
1493			mod_timer(&timer, jiffies + 1);
1494		else {
1495			/* Iterate through all kernels. */
1496			idx = cnt++ % ARRAY_SIZE(access_kernels);
1497			/* Acquire potential initialization. */
1498			func = smp_load_acquire(&access_kernels[idx]);
1499			if (func)
1500				func();
1501		}
1502	} while (!torture_must_stop());
1503	timer_delete_sync(&timer);
1504	timer_destroy_on_stack(&timer);
1505
1506	torture_kthread_stopping("access_thread");
1507	return 0;
1508}
1509
1510__no_kcsan
1511static int test_init(struct kunit *test)
1512{
1513	unsigned long flags;
1514	int nthreads;
1515	int i;
1516
1517	spin_lock_irqsave(&observed.lock, flags);
1518	for (i = 0; i < ARRAY_SIZE(observed.lines); ++i)
1519		observed.lines[i][0] = '\0';
1520	observed.nlines = 0;
1521	spin_unlock_irqrestore(&observed.lock, flags);
1522
1523	if (strstr(test->name, "nothreads"))
1524		return 0;
1525
1526	if (!torture_init_begin((char *)test->name, 1))
1527		return -EBUSY;
1528
1529	if (WARN_ON(threads))
1530		goto err;
1531
1532	for (i = 0; i < ARRAY_SIZE(access_kernels); ++i) {
1533		if (WARN_ON(access_kernels[i]))
1534			goto err;
1535	}
1536
1537	nthreads = abs((long)test->param_value);
1538	if (WARN_ON(!nthreads))
1539		goto err;
1540
1541	threads = kcalloc(nthreads + 1, sizeof(struct task_struct *), GFP_KERNEL);
1542	if (WARN_ON(!threads))
1543		goto err;
1544
1545	threads[nthreads] = NULL;
1546	for (i = 0; i < nthreads; ++i) {
1547		if (torture_create_kthread(access_thread, NULL, threads[i]))
1548			goto err;
1549	}
1550
1551	torture_init_end();
1552
1553	return 0;
1554
1555err:
1556	kfree(threads);
1557	threads = NULL;
1558	torture_init_end();
1559	return -EINVAL;
1560}
1561
1562__no_kcsan
1563static void test_exit(struct kunit *test)
1564{
1565	struct task_struct **stop_thread;
1566	int i;
1567
1568	if (strstr(test->name, "nothreads"))
1569		return;
1570
1571	if (torture_cleanup_begin())
1572		return;
1573
1574	for (i = 0; i < ARRAY_SIZE(access_kernels); ++i)
1575		WRITE_ONCE(access_kernels[i], NULL);
1576
1577	if (threads) {
1578		for (stop_thread = threads; *stop_thread; stop_thread++)
1579			torture_stop_kthread(reader_thread, *stop_thread);
1580
1581		kfree(threads);
1582		threads = NULL;
1583	}
1584
1585	torture_cleanup_end();
1586}
1587
1588__no_kcsan
1589static void register_tracepoints(void)
1590{
1591	register_trace_console(probe_console, NULL);
1592}
1593
1594__no_kcsan
1595static void unregister_tracepoints(void)
1596{
1597	unregister_trace_console(probe_console, NULL);
1598}
1599
1600static int kcsan_suite_init(struct kunit_suite *suite)
1601{
1602	register_tracepoints();
1603	return 0;
1604}
1605
1606static void kcsan_suite_exit(struct kunit_suite *suite)
1607{
1608	unregister_tracepoints();
1609	tracepoint_synchronize_unregister();
1610}
1611
1612static struct kunit_suite kcsan_test_suite = {
1613	.name = "kcsan",
1614	.test_cases = kcsan_test_cases,
1615	.init = test_init,
1616	.exit = test_exit,
1617	.suite_init = kcsan_suite_init,
1618	.suite_exit = kcsan_suite_exit,
1619};
1620
1621kunit_test_suites(&kcsan_test_suite);
1622
1623MODULE_DESCRIPTION("KCSAN test suite");
1624MODULE_LICENSE("GPL v2");
1625MODULE_AUTHOR("Marco Elver <elver@google.com>");