rcutorture.c

   1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Read-Copy Update module-based torture test facility
   4 *
   5 * Copyright (C) IBM Corporation, 2005, 2006
   6 *
   7 * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8 *	  Josh Triplett <josh@joshtriplett.org>
   9 *
  10 * See also:  Documentation/RCU/torture.rst
  11 */
  12
  13#define pr_fmt(fmt) fmt
  14
  15#include <linux/types.h>
  16#include <linux/kernel.h>
  17#include <linux/init.h>
  18#include <linux/module.h>
  19#include <linux/kthread.h>
  20#include <linux/err.h>
  21#include <linux/spinlock.h>
  22#include <linux/smp.h>
  23#include <linux/rcupdate_wait.h>
  24#include <linux/rcu_notifier.h>
  25#include <linux/interrupt.h>
  26#include <linux/sched/signal.h>
  27#include <uapi/linux/sched/types.h>
  28#include <linux/atomic.h>
  29#include <linux/bitops.h>
  30#include <linux/completion.h>
  31#include <linux/moduleparam.h>
  32#include <linux/percpu.h>
  33#include <linux/notifier.h>
  34#include <linux/reboot.h>
  35#include <linux/freezer.h>
  36#include <linux/cpu.h>
  37#include <linux/delay.h>
  38#include <linux/stat.h>
  39#include <linux/srcu.h>
  40#include <linux/slab.h>
  41#include <linux/trace_clock.h>
  42#include <asm/byteorder.h>
  43#include <linux/torture.h>
  44#include <linux/vmalloc.h>
  45#include <linux/sched/debug.h>
  46#include <linux/sched/sysctl.h>
  47#include <linux/oom.h>
  48#include <linux/tick.h>
  49#include <linux/rcupdate_trace.h>
  50#include <linux/nmi.h>
  51
  52#include "rcu.h"
  53
  54MODULE_DESCRIPTION("Read-Copy Update module-based torture test facility");
  55MODULE_LICENSE("GPL");
  56MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
  57
  58// Bits for ->extendables field, extendables param, and related definitions.
  59#define RCUTORTURE_RDR_SHIFT_1	8	// Put SRCU index in upper bits.
  60#define RCUTORTURE_RDR_MASK_1	(0xff << RCUTORTURE_RDR_SHIFT_1)
  61#define RCUTORTURE_RDR_SHIFT_2	16	// Put SRCU index in upper bits.
  62#define RCUTORTURE_RDR_MASK_2	(0xff << RCUTORTURE_RDR_SHIFT_2)
  63#define RCUTORTURE_RDR_BH	0x01	// Extend readers by disabling bh.
  64#define RCUTORTURE_RDR_IRQ	0x02	//  ... disabling interrupts.
  65#define RCUTORTURE_RDR_PREEMPT	0x04	//  ... disabling preemption.
  66#define RCUTORTURE_RDR_RBH	0x08	//  ... rcu_read_lock_bh().
  67#define RCUTORTURE_RDR_SCHED	0x10	//  ... rcu_read_lock_sched().
  68#define RCUTORTURE_RDR_RCU_1	0x20	//  ... entering another RCU reader.
  69#define RCUTORTURE_RDR_RCU_2	0x40	//  ... entering another RCU reader.
  70#define RCUTORTURE_RDR_UPDOWN	0x80	//  ... up-read from task, down-read from timer.
  71					//	Note: Manual start, automatic end.
  72#define RCUTORTURE_RDR_NBITS	8	// Number of bits defined above.
  73#define RCUTORTURE_MAX_EXTEND	\
  74	(RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \
  75	 RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED)  // Intentionally omit RCUTORTURE_RDR_UPDOWN.
  76#define RCUTORTURE_RDR_ALLBITS	\
  77	(RCUTORTURE_MAX_EXTEND | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2 | \
  78	 RCUTORTURE_RDR_MASK_1 | RCUTORTURE_RDR_MASK_2)
  79#define RCUTORTURE_RDR_MAX_LOOPS 0x7	/* Maximum reader extensions. */
  80					/* Must be power of two minus one. */
  81#define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3)
  82
  83torture_param(int, extendables, RCUTORTURE_MAX_EXTEND,
  84	      "Extend readers by disabling bh (1), irqs (2), or preempt (4)");
  85torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable");
  86torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)");
  87torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)");
  88torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)");
  89torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait");
  90torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)");
  91torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()");
  92torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
  93torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives");
  94torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives");
  95torture_param(bool, gp_cond_exp_full, false,
  96		    "Use conditional/async full-stateexpedited GP wait primitives");
  97torture_param(int, gp_cond_wi, 16 * USEC_PER_SEC / HZ,
  98		   "Wait interval for normal conditional grace periods, us (default 16 jiffies)");
  99torture_param(int, gp_cond_wi_exp, 128,
 100		   "Wait interval for expedited conditional grace periods, us (default 128 us)");
 101torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
 102torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");
 103torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
 104torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");
 105torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives");
 106torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives");
 107torture_param(int, gp_poll_wi, 16 * USEC_PER_SEC / HZ,
 108		   "Wait interval for normal polled grace periods, us (default 16 jiffies)");
 109torture_param(int, gp_poll_wi_exp, 128,
 110		   "Wait interval for expedited polled grace periods, us (default 128 us)");
 111torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
 112torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
 113torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
 114torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing");
 115torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers");
 116torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads");
 117torture_param(int, nreaders, -1, "Number of RCU reader threads");
 118torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing");
 119torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
 120torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable");
 121torture_param(bool, gpwrap_lag, true, "Enable grace-period wrap lag testing");
 122torture_param(int, gpwrap_lag_gps, 8, "Value to set for set_gpwrap_lag during an active testing period.");
 123torture_param(int, gpwrap_lag_cycle_mins, 30, "Total cycle duration for gpwrap lag testing (in minutes)");
 124torture_param(int, gpwrap_lag_active_mins, 5, "Duration for which gpwrap lag is active within each cycle (in minutes)");
 125torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable");
 126torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)");
 127torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable");
 128torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)");
 129torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)");
 130torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable");
 131torture_param(int, reader_flavor, SRCU_READ_FLAVOR_NORMAL, "Reader flavors to use, one per bit.");
 132torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
 133torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
 134torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable.");
 135torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s).");
 136torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall.");
 137torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling.");
 138torture_param(int, stall_cpu_block, 0, "Sleep while stalling.");
 139torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one.");
 140torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s).");
 141torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
 142torture_param(int, stutter, 5, "Number of seconds to run/halt test");
 143torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
 144torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds.");
 145torture_param(int, test_boost_holdoff, 0, "Holdoff time from rcutorture start, seconds.");
 146torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds.");
 147torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable.");
 148torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs");
 149torture_param(int, test_srcu_lockdep, 0, "Test specified SRCU deadlock scenario.");
 150torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
 151
 152static char *torture_type = "rcu";
 153module_param(torture_type, charp, 0444);
 154MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)");
 155
 156static int nrealnocbers;
 157static int nrealreaders;
 158static int nrealfakewriters;
 159static struct task_struct *writer_task;
 160static struct task_struct **fakewriter_tasks;
 161static struct task_struct **reader_tasks;
 162static struct task_struct *updown_task;
 163static struct task_struct **nocb_tasks;
 164static struct task_struct *stats_task;
 165static struct task_struct *fqs_task;
 166static struct task_struct *boost_tasks[NR_CPUS];
 167static struct task_struct *stall_task;
 168static struct task_struct **fwd_prog_tasks;
 169static struct task_struct **barrier_cbs_tasks;
 170static struct task_struct *barrier_task;
 171static struct task_struct *read_exit_task;
 172static struct task_struct *preempt_task;
 173
 174#define RCU_TORTURE_PIPE_LEN 10
 175
 176// Mailbox-like structure to check RCU global memory ordering.
 177struct rcu_torture_reader_check {
 178	unsigned long rtc_myloops;
 179	int rtc_chkrdr;
 180	unsigned long rtc_chkloops;
 181	int rtc_ready;
 182	struct rcu_torture_reader_check *rtc_assigner;
 183} ____cacheline_internodealigned_in_smp;
 184
 185// Update-side data structure used to check RCU readers.
 186struct rcu_torture {
 187	struct rcu_head rtort_rcu;
 188	int rtort_pipe_count;
 189	struct list_head rtort_free;
 190	int rtort_mbtest;
 191	struct rcu_torture_reader_check *rtort_chkp;
 192};
 193
 194static LIST_HEAD(rcu_torture_freelist);
 195static struct rcu_torture __rcu *rcu_torture_current;
 196static unsigned long rcu_torture_current_version;
 197static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 198static DEFINE_SPINLOCK(rcu_torture_lock);
 199static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count);
 200static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch);
 201static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 202static struct rcu_torture_reader_check *rcu_torture_reader_mbchk;
 203static atomic_t n_rcu_torture_alloc;
 204static atomic_t n_rcu_torture_alloc_fail;
 205static atomic_t n_rcu_torture_free;
 206static atomic_t n_rcu_torture_mberror;
 207static atomic_t n_rcu_torture_mbchk_fail;
 208static atomic_t n_rcu_torture_mbchk_tries;
 209static atomic_t n_rcu_torture_error;
 210static long n_rcu_torture_barrier_error;
 211static long n_rcu_torture_boost_ktrerror;
 212static long n_rcu_torture_boost_failure;
 213static long n_rcu_torture_boosts;
 214static atomic_long_t n_rcu_torture_timers;
 215static long n_barrier_attempts;
 216static long n_barrier_successes; /* did rcu_barrier test succeed? */
 217static unsigned long n_read_exits;
 218static struct list_head rcu_torture_removed;
 219static unsigned long shutdown_jiffies;
 220static unsigned long start_gp_seq;
 221static atomic_long_t n_nocb_offload;
 222static atomic_long_t n_nocb_deoffload;
 223
 224static int rcu_torture_writer_state;
 225#define RTWS_FIXED_DELAY	0
 226#define RTWS_DELAY		1
 227#define RTWS_REPLACE		2
 228#define RTWS_DEF_FREE		3
 229#define RTWS_EXP_SYNC		4
 230#define RTWS_COND_GET		5
 231#define RTWS_COND_GET_FULL	6
 232#define RTWS_COND_GET_EXP	7
 233#define RTWS_COND_GET_EXP_FULL	8
 234#define RTWS_COND_SYNC		9
 235#define RTWS_COND_SYNC_FULL	10
 236#define RTWS_COND_SYNC_EXP	11
 237#define RTWS_COND_SYNC_EXP_FULL	12
 238#define RTWS_POLL_GET		13
 239#define RTWS_POLL_GET_FULL	14
 240#define RTWS_POLL_GET_EXP	15
 241#define RTWS_POLL_GET_EXP_FULL	16
 242#define RTWS_POLL_WAIT		17
 243#define RTWS_POLL_WAIT_FULL	18
 244#define RTWS_POLL_WAIT_EXP	19
 245#define RTWS_POLL_WAIT_EXP_FULL	20
 246#define RTWS_SYNC		21
 247#define RTWS_STUTTER		22
 248#define RTWS_STOPPING		23
 249static const char * const rcu_torture_writer_state_names[] = {
 250	"RTWS_FIXED_DELAY",
 251	"RTWS_DELAY",
 252	"RTWS_REPLACE",
 253	"RTWS_DEF_FREE",
 254	"RTWS_EXP_SYNC",
 255	"RTWS_COND_GET",
 256	"RTWS_COND_GET_FULL",
 257	"RTWS_COND_GET_EXP",
 258	"RTWS_COND_GET_EXP_FULL",
 259	"RTWS_COND_SYNC",
 260	"RTWS_COND_SYNC_FULL",
 261	"RTWS_COND_SYNC_EXP",
 262	"RTWS_COND_SYNC_EXP_FULL",
 263	"RTWS_POLL_GET",
 264	"RTWS_POLL_GET_FULL",
 265	"RTWS_POLL_GET_EXP",
 266	"RTWS_POLL_GET_EXP_FULL",
 267	"RTWS_POLL_WAIT",
 268	"RTWS_POLL_WAIT_FULL",
 269	"RTWS_POLL_WAIT_EXP",
 270	"RTWS_POLL_WAIT_EXP_FULL",
 271	"RTWS_SYNC",
 272	"RTWS_STUTTER",
 273	"RTWS_STOPPING",
 274};
 275
 276/* Record reader segment types and duration for first failing read. */
 277struct rt_read_seg {
 278	int rt_readstate;
 279	unsigned long rt_delay_jiffies;
 280	unsigned long rt_delay_ms;
 281	unsigned long rt_delay_us;
 282	bool rt_preempted;
 283	int rt_cpu;
 284	int rt_end_cpu;
 285	unsigned long long rt_gp_seq;
 286	unsigned long long rt_gp_seq_end;
 287	u64 rt_ts;
 288};
 289static int err_segs_recorded;
 290static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS];
 291static int rt_read_nsegs;
 292static int rt_read_preempted;
 293
 294static const char *rcu_torture_writer_state_getname(void)
 295{
 296	unsigned int i = READ_ONCE(rcu_torture_writer_state);
 297
 298	if (i >= ARRAY_SIZE(rcu_torture_writer_state_names))
 299		return "???";
 300	return rcu_torture_writer_state_names[i];
 301}
 302
 303#ifdef CONFIG_RCU_TRACE
 304static u64 notrace rcu_trace_clock_local(void)
 305{
 306	u64 ts = trace_clock_local();
 307
 308	(void)do_div(ts, NSEC_PER_USEC);
 309	return ts;
 310}
 311#else /* #ifdef CONFIG_RCU_TRACE */
 312static u64 notrace rcu_trace_clock_local(void)
 313{
 314	return 0ULL;
 315}
 316#endif /* #else #ifdef CONFIG_RCU_TRACE */
 317
 318/*
 319 * Stop aggressive CPU-hog tests a bit before the end of the test in order
 320 * to avoid interfering with test shutdown.
 321 */
 322static bool shutdown_time_arrived(void)
 323{
 324	return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ);
 325}
 326
 327static unsigned long boost_starttime;	/* jiffies of next boost test start. */
 328static DEFINE_MUTEX(boost_mutex);	/* protect setting boost_starttime */
 329					/*  and boost task create/destroy. */
 330static atomic_t barrier_cbs_count;	/* Barrier callbacks registered. */
 331static bool barrier_phase;		/* Test phase. */
 332static atomic_t barrier_cbs_invoked;	/* Barrier callbacks invoked. */
 333static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */
 334static DECLARE_WAIT_QUEUE_HEAD(barrier_wq);
 335
 336static atomic_t rcu_fwd_cb_nodelay;	/* Short rcu_torture_delay() delays. */
 337
 338/*
 339 * Allocate an element from the rcu_tortures pool.
 340 */
 341static struct rcu_torture *
 342rcu_torture_alloc(void)
 343{
 344	struct list_head *p;
 345
 346	spin_lock_bh(&rcu_torture_lock);
 347	if (list_empty(&rcu_torture_freelist)) {
 348		atomic_inc(&n_rcu_torture_alloc_fail);
 349		spin_unlock_bh(&rcu_torture_lock);
 350		return NULL;
 351	}
 352	atomic_inc(&n_rcu_torture_alloc);
 353	p = rcu_torture_freelist.next;
 354	list_del_init(p);
 355	spin_unlock_bh(&rcu_torture_lock);
 356	return container_of(p, struct rcu_torture, rtort_free);
 357}
 358
 359/*
 360 * Free an element to the rcu_tortures pool.
 361 */
 362static void
 363rcu_torture_free(struct rcu_torture *p)
 364{
 365	atomic_inc(&n_rcu_torture_free);
 366	spin_lock_bh(&rcu_torture_lock);
 367	list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 368	spin_unlock_bh(&rcu_torture_lock);
 369}
 370
 371/*
 372 * Operations vector for selecting different types of tests.
 373 */
 374
 375struct rcu_torture_ops {
 376	int ttype;
 377	void (*init)(void);
 378	void (*cleanup)(void);
 379	int (*readlock)(void);
 380	void (*read_delay)(struct torture_random_state *rrsp,
 381			   struct rt_read_seg *rtrsp);
 382	void (*readunlock)(int idx);
 383	int (*readlock_held)(void);   // lockdep.
 384	int (*readlock_nesting)(void); // actual nesting, if available, -1 if not.
 385	int (*down_read)(void);
 386	void (*up_read)(int idx);
 387	unsigned long (*get_gp_seq)(void);
 388	unsigned long (*gp_diff)(unsigned long new, unsigned long old);
 389	void (*deferred_free)(struct rcu_torture *p);
 390	void (*sync)(void);
 391	void (*exp_sync)(void);
 392	void (*exp_current)(void);
 393	unsigned long (*get_gp_state_exp)(void);
 394	unsigned long (*start_gp_poll_exp)(void);
 395	void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp);
 396	bool (*poll_gp_state_exp)(unsigned long oldstate);
 397	void (*cond_sync_exp)(unsigned long oldstate);
 398	void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp);
 399	unsigned long (*get_comp_state)(void);
 400	void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp);
 401	bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2);
 402	bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2);
 403	unsigned long (*get_gp_state)(void);
 404	void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);
 405	unsigned long (*start_gp_poll)(void);
 406	void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp);
 407	bool (*poll_gp_state)(unsigned long oldstate);
 408	bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);
 409	bool (*poll_need_2gp)(bool poll, bool poll_full);
 410	void (*cond_sync)(unsigned long oldstate);
 411	void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp);
 412	int poll_active;
 413	int poll_active_full;
 414	call_rcu_func_t call;
 415	void (*cb_barrier)(void);
 416	void (*fqs)(void);
 417	void (*stats)(void);
 418	void (*gp_kthread_dbg)(void);
 419	bool (*check_boost_failed)(unsigned long gp_state, int *cpup);
 420	int (*stall_dur)(void);
 421	void (*get_gp_data)(int *flags, unsigned long *gp_seq);
 422	void (*gp_slow_register)(atomic_t *rgssp);
 423	void (*gp_slow_unregister)(atomic_t *rgssp);
 424	bool (*reader_blocked)(void);
 425	unsigned long long (*gather_gp_seqs)(void);
 426	void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len);
 427	void (*set_gpwrap_lag)(unsigned long lag);
 428	int (*get_gpwrap_count)(int cpu);
 429	long cbflood_max;
 430	int irq_capable;
 431	int can_boost;
 432	int extendables;
 433	int slow_gps;
 434	int no_pi_lock;
 435	int debug_objects;
 436	int start_poll_irqsoff;
 437	int have_up_down;
 438	const char *name;
 439};
 440
 441static struct rcu_torture_ops *cur_ops;
 442
 443/*
 444 * Definitions for rcu torture testing.
 445 */
 446
 447static int torture_readlock_not_held(void)
 448{
 449	return rcu_read_lock_bh_held() || rcu_read_lock_sched_held();
 450}
 451
 452static int rcu_torture_read_lock(void)
 453{
 454	rcu_read_lock();
 455	return 0;
 456}
 457
 458static void
 459rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
 460{
 461	unsigned long started;
 462	unsigned long completed;
 463	const unsigned long shortdelay_us = 200;
 464	unsigned long longdelay_ms = 300;
 465	unsigned long long ts;
 466
 467	/* We want a short delay sometimes to make a reader delay the grace
 468	 * period, and we want a long delay occasionally to trigger
 469	 * force_quiescent_state. */
 470
 471	if (!atomic_read(&rcu_fwd_cb_nodelay) &&
 472	    !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
 473		started = cur_ops->get_gp_seq();
 474		ts = rcu_trace_clock_local();
 475		if ((preempt_count() & HARDIRQ_MASK) || softirq_count())
 476			longdelay_ms = 5; /* Avoid triggering BH limits. */
 477		mdelay(longdelay_ms);
 478		rtrsp->rt_delay_ms = longdelay_ms;
 479		completed = cur_ops->get_gp_seq();
 480		do_trace_rcu_torture_read(cur_ops->name, NULL, ts,
 481					  started, completed);
 482	}
 483	if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) {
 484		udelay(shortdelay_us);
 485		rtrsp->rt_delay_us = shortdelay_us;
 486	}
 487	if (!preempt_count() &&
 488	    !(torture_random(rrsp) % (nrealreaders * 500)))
 489		torture_preempt_schedule();  /* QS only if preemptible. */
 490}
 491
 492static void rcu_torture_read_unlock(int idx)
 493{
 494	rcu_read_unlock();
 495}
 496
 497static int rcu_torture_readlock_nesting(void)
 498{
 499	if (IS_ENABLED(CONFIG_PREEMPT_RCU))
 500		return rcu_preempt_depth();
 501	if (IS_ENABLED(CONFIG_PREEMPT_COUNT))
 502		return (preempt_count() & PREEMPT_MASK);
 503	return -1;
 504}
 505
 506/*
 507 * Update callback in the pipe.  This should be invoked after a grace period.
 508 */
 509static bool
 510rcu_torture_pipe_update_one(struct rcu_torture *rp)
 511{
 512	int i;
 513	struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp);
 514
 515	if (rtrcp) {
 516		WRITE_ONCE(rp->rtort_chkp, NULL);
 517		smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire().
 518	}
 519	i = rp->rtort_pipe_count;
 520	if (i > RCU_TORTURE_PIPE_LEN)
 521		i = RCU_TORTURE_PIPE_LEN;
 522	atomic_inc(&rcu_torture_wcount[i]);
 523	WRITE_ONCE(rp->rtort_pipe_count, i + 1);
 524	ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count);
 525	if (i + 1 >= RCU_TORTURE_PIPE_LEN) {
 526		rp->rtort_mbtest = 0;
 527		return true;
 528	}
 529	return false;
 530}
 531
 532/*
 533 * Update all callbacks in the pipe.  Suitable for synchronous grace-period
 534 * primitives.
 535 */
 536static void
 537rcu_torture_pipe_update(struct rcu_torture *old_rp)
 538{
 539	struct rcu_torture *rp;
 540	struct rcu_torture *rp1;
 541
 542	if (old_rp)
 543		list_add(&old_rp->rtort_free, &rcu_torture_removed);
 544	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
 545		if (rcu_torture_pipe_update_one(rp)) {
 546			list_del(&rp->rtort_free);
 547			rcu_torture_free(rp);
 548		}
 549	}
 550}
 551
 552static void
 553rcu_torture_cb(struct rcu_head *p)
 554{
 555	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 556
 557	if (torture_must_stop_irq()) {
 558		/* Test is ending, just drop callbacks on the floor. */
 559		/* The next initialization will pick up the pieces. */
 560		return;
 561	}
 562	if (rcu_torture_pipe_update_one(rp))
 563		rcu_torture_free(rp);
 564	else
 565		cur_ops->deferred_free(rp);
 566}
 567
 568static unsigned long rcu_no_completed(void)
 569{
 570	return 0;
 571}
 572
 573static void rcu_torture_deferred_free(struct rcu_torture *p)
 574{
 575	call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb);
 576}
 577
 578static void rcu_sync_torture_init(void)
 579{
 580	INIT_LIST_HEAD(&rcu_torture_removed);
 581}
 582
 583static bool rcu_poll_need_2gp(bool poll, bool poll_full)
 584{
 585	return poll;
 586}
 587
 588static struct rcu_torture_ops rcu_ops = {
 589	.ttype			= RCU_FLAVOR,
 590	.init			= rcu_sync_torture_init,
 591	.readlock		= rcu_torture_read_lock,
 592	.read_delay		= rcu_read_delay,
 593	.readunlock		= rcu_torture_read_unlock,
 594	.readlock_held		= torture_readlock_not_held,
 595	.readlock_nesting	= rcu_torture_readlock_nesting,
 596	.get_gp_seq		= rcu_get_gp_seq,
 597	.gp_diff		= rcu_seq_diff,
 598	.deferred_free		= rcu_torture_deferred_free,
 599	.sync			= synchronize_rcu,
 600	.exp_sync		= synchronize_rcu_expedited,
 601	.same_gp_state		= same_state_synchronize_rcu,
 602	.same_gp_state_full	= same_state_synchronize_rcu_full,
 603	.get_comp_state		= get_completed_synchronize_rcu,
 604	.get_comp_state_full	= get_completed_synchronize_rcu_full,
 605	.get_gp_state		= get_state_synchronize_rcu,
 606	.get_gp_state_full	= get_state_synchronize_rcu_full,
 607	.start_gp_poll		= start_poll_synchronize_rcu,
 608	.start_gp_poll_full	= start_poll_synchronize_rcu_full,
 609	.poll_gp_state		= poll_state_synchronize_rcu,
 610	.poll_gp_state_full	= poll_state_synchronize_rcu_full,
 611	.poll_need_2gp		= rcu_poll_need_2gp,
 612	.cond_sync		= cond_synchronize_rcu,
 613	.cond_sync_full		= cond_synchronize_rcu_full,
 614	.poll_active		= NUM_ACTIVE_RCU_POLL_OLDSTATE,
 615	.poll_active_full	= NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE,
 616	.get_gp_state_exp	= get_state_synchronize_rcu,
 617	.start_gp_poll_exp	= start_poll_synchronize_rcu_expedited,
 618	.start_gp_poll_exp_full	= start_poll_synchronize_rcu_expedited_full,
 619	.poll_gp_state_exp	= poll_state_synchronize_rcu,
 620	.cond_sync_exp		= cond_synchronize_rcu_expedited,
 621	.cond_sync_exp_full	= cond_synchronize_rcu_expedited_full,
 622	.call			= call_rcu_hurry,
 623	.cb_barrier		= rcu_barrier,
 624	.fqs			= rcu_force_quiescent_state,
 625	.gp_kthread_dbg		= show_rcu_gp_kthreads,
 626	.check_boost_failed	= rcu_check_boost_fail,
 627	.stall_dur		= rcu_jiffies_till_stall_check,
 628	.get_gp_data		= rcutorture_get_gp_data,
 629	.gp_slow_register	= rcu_gp_slow_register,
 630	.gp_slow_unregister	= rcu_gp_slow_unregister,
 631	.reader_blocked		= IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)
 632				  ? has_rcu_reader_blocked
 633				  : NULL,
 634	.gather_gp_seqs		= rcutorture_gather_gp_seqs,
 635	.format_gp_seqs		= rcutorture_format_gp_seqs,
 636	.set_gpwrap_lag		= rcu_set_gpwrap_lag,
 637	.get_gpwrap_count	= rcu_get_gpwrap_count,
 638	.irq_capable		= 1,
 639	.can_boost		= IS_ENABLED(CONFIG_RCU_BOOST),
 640	.extendables		= RCUTORTURE_MAX_EXTEND,
 641	.debug_objects		= 1,
 642	.start_poll_irqsoff	= 1,
 643	.name			= "rcu"
 644};
 645
 646/*
 647 * Don't even think about trying any of these in real life!!!
 648 * The names includes "busted", and they really means it!
 649 * The only purpose of these functions is to provide a buggy RCU
 650 * implementation to make sure that rcutorture correctly emits
 651 * buggy-RCU error messages.
 652 */
 653static void rcu_busted_torture_deferred_free(struct rcu_torture *p)
 654{
 655	/* This is a deliberate bug for testing purposes only! */
 656	rcu_torture_cb(&p->rtort_rcu);
 657}
 658
 659static void synchronize_rcu_busted(void)
 660{
 661	/* This is a deliberate bug for testing purposes only! */
 662}
 663
 664static void
 665call_rcu_busted(struct rcu_head *head, rcu_callback_t func)
 666{
 667	/* This is a deliberate bug for testing purposes only! */
 668	func(head);
 669}
 670
 671static struct rcu_torture_ops rcu_busted_ops = {
 672	.ttype		= INVALID_RCU_FLAVOR,
 673	.init		= rcu_sync_torture_init,
 674	.readlock	= rcu_torture_read_lock,
 675	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
 676	.readunlock	= rcu_torture_read_unlock,
 677	.readlock_held	= torture_readlock_not_held,
 678	.get_gp_seq	= rcu_no_completed,
 679	.deferred_free	= rcu_busted_torture_deferred_free,
 680	.sync		= synchronize_rcu_busted,
 681	.exp_sync	= synchronize_rcu_busted,
 682	.call		= call_rcu_busted,
 683	.gather_gp_seqs	= rcutorture_gather_gp_seqs,
 684	.format_gp_seqs	= rcutorture_format_gp_seqs,
 685	.irq_capable	= 1,
 686	.extendables	= RCUTORTURE_MAX_EXTEND,
 687	.name		= "busted"
 688};
 689
 690/*
 691 * Definitions for srcu torture testing.
 692 */
 693
 694DEFINE_STATIC_SRCU(srcu_ctl);
 695DEFINE_STATIC_SRCU_FAST(srcu_ctlf);
 696DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_ctlfud);
 697static struct srcu_struct srcu_ctld;
 698static struct srcu_struct *srcu_ctlp = &srcu_ctl;
 699static struct rcu_torture_ops srcud_ops;
 700
 701static void srcu_torture_init(void)
 702{
 703	rcu_sync_torture_init();
 704	if (!reader_flavor || (reader_flavor & SRCU_READ_FLAVOR_NORMAL))
 705		VERBOSE_TOROUT_STRING("srcu_torture_init normal SRCU");
 706	if (reader_flavor & SRCU_READ_FLAVOR_NMI)
 707		VERBOSE_TOROUT_STRING("srcu_torture_init NMI-safe SRCU");
 708	if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
 709		srcu_ctlp = &srcu_ctlf;
 710		VERBOSE_TOROUT_STRING("srcu_torture_init fast SRCU");
 711	}
 712	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
 713		srcu_ctlp = &srcu_ctlfud;
 714		VERBOSE_TOROUT_STRING("srcu_torture_init fast-up/down SRCU");
 715	}
 716}
 717
 718static void srcu_get_gp_data(int *flags, unsigned long *gp_seq)
 719{
 720	srcutorture_get_gp_data(srcu_ctlp, flags, gp_seq);
 721}
 722
 723static int srcu_torture_read_lock(void)
 724{
 725	int idx;
 726	struct srcu_ctr __percpu *scp;
 727	int ret = 0;
 728
 729	WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL);
 730
 731	if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) {
 732		idx = srcu_read_lock(srcu_ctlp);
 733		WARN_ON_ONCE(idx & ~0x1);
 734		ret += idx;
 735	}
 736	if (reader_flavor & SRCU_READ_FLAVOR_NMI) {
 737		idx = srcu_read_lock_nmisafe(srcu_ctlp);
 738		WARN_ON_ONCE(idx & ~0x1);
 739		ret += idx << 1;
 740	}
 741	if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
 742		scp = srcu_read_lock_fast(srcu_ctlp);
 743		idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
 744		WARN_ON_ONCE(idx & ~0x1);
 745		ret += idx << 2;
 746	}
 747	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
 748		scp = srcu_read_lock_fast_updown(srcu_ctlp);
 749		idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
 750		WARN_ON_ONCE(idx & ~0x1);
 751		ret += idx << 3;
 752	}
 753	return ret;
 754}
 755
 756static void
 757srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp)
 758{
 759	long delay;
 760	const long uspertick = 1000000 / HZ;
 761	const long longdelay = 10;
 762
 763	/* We want there to be long-running readers, but not all the time. */
 764
 765	delay = torture_random(rrsp) %
 766		(nrealreaders * 2 * longdelay * uspertick);
 767	if (!delay && in_task()) {
 768		schedule_timeout_interruptible(longdelay);
 769		rtrsp->rt_delay_jiffies = longdelay;
 770	} else {
 771		rcu_read_delay(rrsp, rtrsp);
 772	}
 773}
 774
 775static void srcu_torture_read_unlock(int idx)
 776{
 777	WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
 778	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
 779		srcu_read_unlock_fast_updown(srcu_ctlp,
 780					     __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));
 781	if (reader_flavor & SRCU_READ_FLAVOR_FAST)
 782		srcu_read_unlock_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x4) >> 2));
 783	if (reader_flavor & SRCU_READ_FLAVOR_NMI)
 784		srcu_read_unlock_nmisafe(srcu_ctlp, (idx & 0x2) >> 1);
 785	if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL))
 786		srcu_read_unlock(srcu_ctlp, idx & 0x1);
 787}
 788
 789static int torture_srcu_read_lock_held(void)
 790{
 791	return srcu_read_lock_held(srcu_ctlp);
 792}
 793
 794static bool srcu_torture_have_up_down(void)
 795{
 796	int rf = reader_flavor;
 797
 798	if (!rf)
 799		rf = SRCU_READ_FLAVOR_NORMAL;
 800	return !!(cur_ops->have_up_down & rf);
 801}
 802
 803static int srcu_torture_down_read(void)
 804{
 805	int idx;
 806	struct srcu_ctr __percpu *scp;
 807
 808	WARN_ON_ONCE(reader_flavor & ~SRCU_READ_FLAVOR_ALL);
 809	WARN_ON_ONCE(reader_flavor & (reader_flavor - 1));
 810
 811	if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) {
 812		idx = srcu_down_read(srcu_ctlp);
 813		WARN_ON_ONCE(idx & ~0x1);
 814		return idx;
 815	}
 816	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
 817		scp = srcu_down_read_fast(srcu_ctlp);
 818		idx = __srcu_ptr_to_ctr(srcu_ctlp, scp);
 819		WARN_ON_ONCE(idx & ~0x1);
 820		return idx << 3;
 821	}
 822	WARN_ON_ONCE(1);
 823	return 0;
 824}
 825
 826static void srcu_torture_up_read(int idx)
 827{
 828	WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));
 829	if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN)
 830		srcu_up_read_fast(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3));
 831	else if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) ||
 832		 !(reader_flavor & SRCU_READ_FLAVOR_ALL))
 833		srcu_up_read(srcu_ctlp, idx & 0x1);
 834	else
 835		WARN_ON_ONCE(1);
 836}
 837
 838static unsigned long srcu_torture_completed(void)
 839{
 840	return srcu_batches_completed(srcu_ctlp);
 841}
 842
 843static void srcu_torture_deferred_free(struct rcu_torture *rp)
 844{
 845	call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb);
 846}
 847
 848static void srcu_torture_synchronize(void)
 849{
 850	synchronize_srcu(srcu_ctlp);
 851}
 852
 853static unsigned long srcu_torture_get_gp_state(void)
 854{
 855	return get_state_synchronize_srcu(srcu_ctlp);
 856}
 857
 858static unsigned long srcu_torture_start_gp_poll(void)
 859{
 860	return start_poll_synchronize_srcu(srcu_ctlp);
 861}
 862
 863static bool srcu_torture_poll_gp_state(unsigned long oldstate)
 864{
 865	return poll_state_synchronize_srcu(srcu_ctlp, oldstate);
 866}
 867
 868static void srcu_torture_call(struct rcu_head *head,
 869			      rcu_callback_t func)
 870{
 871	call_srcu(srcu_ctlp, head, func);
 872}
 873
 874static void srcu_torture_barrier(void)
 875{
 876	srcu_barrier(srcu_ctlp);
 877}
 878
 879static void srcu_torture_stats(void)
 880{
 881	srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG);
 882}
 883
 884static void srcu_torture_synchronize_expedited(void)
 885{
 886	synchronize_srcu_expedited(srcu_ctlp);
 887}
 888
 889static void srcu_torture_expedite_current(void)
 890{
 891	srcu_expedite_current(srcu_ctlp);
 892}
 893
 894static struct rcu_torture_ops srcu_ops = {
 895	.ttype		= SRCU_FLAVOR,
 896	.init		= srcu_torture_init,
 897	.readlock	= srcu_torture_read_lock,
 898	.read_delay	= srcu_read_delay,
 899	.readunlock	= srcu_torture_read_unlock,
 900	.down_read	= srcu_torture_down_read,
 901	.up_read	= srcu_torture_up_read,
 902	.readlock_held	= torture_srcu_read_lock_held,
 903	.get_gp_seq	= srcu_torture_completed,
 904	.gp_diff	= rcu_seq_diff,
 905	.deferred_free	= srcu_torture_deferred_free,
 906	.sync		= srcu_torture_synchronize,
 907	.exp_sync	= srcu_torture_synchronize_expedited,
 908	.exp_current	= srcu_torture_expedite_current,
 909	.same_gp_state	= same_state_synchronize_srcu,
 910	.get_comp_state = get_completed_synchronize_srcu,
 911	.get_gp_state	= srcu_torture_get_gp_state,
 912	.start_gp_poll	= srcu_torture_start_gp_poll,
 913	.poll_gp_state	= srcu_torture_poll_gp_state,
 914	.poll_active	= NUM_ACTIVE_SRCU_POLL_OLDSTATE,
 915	.call		= srcu_torture_call,
 916	.cb_barrier	= srcu_torture_barrier,
 917	.stats		= srcu_torture_stats,
 918	.get_gp_data	= srcu_get_gp_data,
 919	.cbflood_max	= 50000,
 920	.irq_capable	= 1,
 921	.no_pi_lock	= IS_ENABLED(CONFIG_TINY_SRCU),
 922	.debug_objects	= 1,
 923	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)
 924				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
 925	.name		= "srcu"
 926};
 927
 928static void srcud_torture_init(void)
 929{
 930	rcu_sync_torture_init();
 931	if (!reader_flavor || (reader_flavor & SRCU_READ_FLAVOR_NORMAL)) {
 932		WARN_ON(init_srcu_struct(&srcu_ctld));
 933		VERBOSE_TOROUT_STRING("srcud_torture_init normal SRCU");
 934	} else if (reader_flavor & SRCU_READ_FLAVOR_NMI) {
 935		WARN_ON(init_srcu_struct(&srcu_ctld));
 936		VERBOSE_TOROUT_STRING("srcud_torture_init NMI-safe SRCU");
 937	} else if (reader_flavor & SRCU_READ_FLAVOR_FAST) {
 938		WARN_ON(init_srcu_struct_fast(&srcu_ctld));
 939		VERBOSE_TOROUT_STRING("srcud_torture_init fast SRCU");
 940	} else if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) {
 941		WARN_ON(init_srcu_struct_fast_updown(&srcu_ctld));
 942		VERBOSE_TOROUT_STRING("srcud_torture_init fast-up/down SRCU");
 943	} else {
 944		WARN_ON(init_srcu_struct(&srcu_ctld));
 945	}
 946	srcu_ctlp = &srcu_ctld;
 947}
 948
 949static void srcu_torture_cleanup(void)
 950{
 951	cleanup_srcu_struct(&srcu_ctld);
 952	srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */
 953}
 954
 955/* As above, but dynamically allocated. */
 956static struct rcu_torture_ops srcud_ops = {
 957	.ttype		= SRCU_FLAVOR,
 958	.init		= srcud_torture_init,
 959	.cleanup	= srcu_torture_cleanup,
 960	.readlock	= srcu_torture_read_lock,
 961	.read_delay	= srcu_read_delay,
 962	.readunlock	= srcu_torture_read_unlock,
 963	.readlock_held	= torture_srcu_read_lock_held,
 964	.down_read	= srcu_torture_down_read,
 965	.up_read	= srcu_torture_up_read,
 966	.get_gp_seq	= srcu_torture_completed,
 967	.gp_diff	= rcu_seq_diff,
 968	.deferred_free	= srcu_torture_deferred_free,
 969	.sync		= srcu_torture_synchronize,
 970	.exp_sync	= srcu_torture_synchronize_expedited,
 971	.exp_current	= srcu_torture_expedite_current,
 972	.same_gp_state	= same_state_synchronize_srcu,
 973	.get_comp_state = get_completed_synchronize_srcu,
 974	.get_gp_state	= srcu_torture_get_gp_state,
 975	.start_gp_poll	= srcu_torture_start_gp_poll,
 976	.poll_gp_state	= srcu_torture_poll_gp_state,
 977	.poll_active	= NUM_ACTIVE_SRCU_POLL_OLDSTATE,
 978	.call		= srcu_torture_call,
 979	.cb_barrier	= srcu_torture_barrier,
 980	.stats		= srcu_torture_stats,
 981	.get_gp_data	= srcu_get_gp_data,
 982	.cbflood_max	= 50000,
 983	.irq_capable	= 1,
 984	.no_pi_lock	= IS_ENABLED(CONFIG_TINY_SRCU),
 985	.debug_objects	= 1,
 986	.have_up_down	= IS_ENABLED(CONFIG_TINY_SRCU)
 987				? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN,
 988	.name		= "srcud"
 989};
 990
 991/* As above, but broken due to inappropriate reader extension. */
 992static struct rcu_torture_ops busted_srcud_ops = {
 993	.ttype		= SRCU_FLAVOR,
 994	.init		= srcu_torture_init,
 995	.cleanup	= srcu_torture_cleanup,
 996	.readlock	= srcu_torture_read_lock,
 997	.read_delay	= rcu_read_delay,
 998	.readunlock	= srcu_torture_read_unlock,
 999	.readlock_held	= torture_srcu_read_lock_held,
1000	.get_gp_seq	= srcu_torture_completed,
1001	.deferred_free	= srcu_torture_deferred_free,
1002	.sync		= srcu_torture_synchronize,
1003	.exp_sync	= srcu_torture_synchronize_expedited,
1004	.call		= srcu_torture_call,
1005	.cb_barrier	= srcu_torture_barrier,
1006	.stats		= srcu_torture_stats,
1007	.irq_capable	= 1,
1008	.no_pi_lock	= IS_ENABLED(CONFIG_TINY_SRCU),
1009	.extendables	= RCUTORTURE_MAX_EXTEND,
1010	.name		= "busted_srcud"
1011};
1012
1013/*
1014 * Definitions for trivial CONFIG_PREEMPT=n-only torture testing.
1015 * This implementation does not work well with CPU hotplug nor
1016 * with rcutorture's shuffling.
1017 */
1018
1019static void synchronize_rcu_trivial(void)
1020{
1021	int cpu;
1022
1023	for_each_online_cpu(cpu) {
1024		torture_sched_setaffinity(current->pid, cpumask_of(cpu), true);
1025		WARN_ON_ONCE(raw_smp_processor_id() != cpu);
1026	}
1027}
1028
1029static void rcu_sync_torture_init_trivial(void)
1030{
1031	rcu_sync_torture_init();
1032	// if (onoff_interval || shuffle_interval) {
1033	if (WARN_ONCE(onoff_interval || shuffle_interval, "%s: Non-zero onoff_interval (%d) or shuffle_interval (%d) breaks trivial RCU, resetting to zero", __func__, onoff_interval, shuffle_interval)) {
1034		onoff_interval = 0;
1035		shuffle_interval = 0;
1036	}
1037}
1038
1039static int rcu_torture_read_lock_trivial(void)
1040{
1041	preempt_disable();
1042	return 0;
1043}
1044
1045static void rcu_torture_read_unlock_trivial(int idx)
1046{
1047	preempt_enable();
1048}
1049
1050static struct rcu_torture_ops trivial_ops = {
1051	.ttype		= RCU_TRIVIAL_FLAVOR,
1052	.init		= rcu_sync_torture_init_trivial,
1053	.readlock	= rcu_torture_read_lock_trivial,
1054	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
1055	.readunlock	= rcu_torture_read_unlock_trivial,
1056	.readlock_held	= torture_readlock_not_held,
1057	.get_gp_seq	= rcu_no_completed,
1058	.sync		= synchronize_rcu_trivial,
1059	.exp_sync	= synchronize_rcu_trivial,
1060	.irq_capable	= 1,
1061	.name		= "trivial"
1062};
1063
1064#ifdef CONFIG_TASKS_RCU
1065
1066/*
1067 * Definitions for RCU-tasks torture testing.
1068 */
1069
1070static int tasks_torture_read_lock(void)
1071{
1072	return 0;
1073}
1074
1075static void tasks_torture_read_unlock(int idx)
1076{
1077}
1078
1079static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
1080{
1081	call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb);
1082}
1083
1084static void synchronize_rcu_mult_test(void)
1085{
1086	synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry);
1087}
1088
1089static struct rcu_torture_ops tasks_ops = {
1090	.ttype		= RCU_TASKS_FLAVOR,
1091	.init		= rcu_sync_torture_init,
1092	.readlock	= tasks_torture_read_lock,
1093	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
1094	.readunlock	= tasks_torture_read_unlock,
1095	.get_gp_seq	= rcu_no_completed,
1096	.deferred_free	= rcu_tasks_torture_deferred_free,
1097	.sync		= synchronize_rcu_tasks,
1098	.exp_sync	= synchronize_rcu_mult_test,
1099	.call		= call_rcu_tasks,
1100	.cb_barrier	= rcu_barrier_tasks,
1101	.gp_kthread_dbg	= show_rcu_tasks_classic_gp_kthread,
1102	.get_gp_data	= rcu_tasks_get_gp_data,
1103	.irq_capable	= 1,
1104	.slow_gps	= 1,
1105	.name		= "tasks"
1106};
1107
1108#define TASKS_OPS &tasks_ops,
1109
1110#else // #ifdef CONFIG_TASKS_RCU
1111
1112#define TASKS_OPS
1113
1114#endif // #else #ifdef CONFIG_TASKS_RCU
1115
1116
1117#ifdef CONFIG_TASKS_RUDE_RCU
1118
1119/*
1120 * Definitions for rude RCU-tasks torture testing.
1121 */
1122
1123static struct rcu_torture_ops tasks_rude_ops = {
1124	.ttype		= RCU_TASKS_RUDE_FLAVOR,
1125	.init		= rcu_sync_torture_init,
1126	.readlock	= rcu_torture_read_lock_trivial,
1127	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */
1128	.readunlock	= rcu_torture_read_unlock_trivial,
1129	.get_gp_seq	= rcu_no_completed,
1130	.sync		= synchronize_rcu_tasks_rude,
1131	.exp_sync	= synchronize_rcu_tasks_rude,
1132	.gp_kthread_dbg	= show_rcu_tasks_rude_gp_kthread,
1133	.get_gp_data	= rcu_tasks_rude_get_gp_data,
1134	.cbflood_max	= 50000,
1135	.irq_capable	= 1,
1136	.name		= "tasks-rude"
1137};
1138
1139#define TASKS_RUDE_OPS &tasks_rude_ops,
1140
1141#else // #ifdef CONFIG_TASKS_RUDE_RCU
1142
1143#define TASKS_RUDE_OPS
1144
1145#endif // #else #ifdef CONFIG_TASKS_RUDE_RCU
1146
1147
1148#ifdef CONFIG_TASKS_TRACE_RCU
1149
1150/*
1151 * Definitions for tracing RCU-tasks torture testing.
1152 */
1153
1154static int tasks_tracing_torture_read_lock(void)
1155{
1156	rcu_read_lock_trace();
1157	return 0;
1158}
1159
1160static void tasks_tracing_torture_read_unlock(int idx)
1161{
1162	rcu_read_unlock_trace();
1163}
1164
1165static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p)
1166{
1167	call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb);
1168}
1169
1170static struct rcu_torture_ops tasks_tracing_ops = {
1171	.ttype		= RCU_TASKS_TRACING_FLAVOR,
1172	.init		= rcu_sync_torture_init,
1173	.readlock	= tasks_tracing_torture_read_lock,
1174	.read_delay	= srcu_read_delay,  /* just reuse srcu's version. */
1175	.readunlock	= tasks_tracing_torture_read_unlock,
1176	.readlock_held	= rcu_read_lock_trace_held,
1177	.get_gp_seq	= rcu_no_completed,
1178	.deferred_free	= rcu_tasks_tracing_torture_deferred_free,
1179	.sync		= synchronize_rcu_tasks_trace,
1180	.exp_sync	= synchronize_rcu_tasks_trace,
1181	.call		= call_rcu_tasks_trace,
1182	.cb_barrier	= rcu_barrier_tasks_trace,
1183	.gp_kthread_dbg	= show_rcu_tasks_trace_gp_kthread,
1184	.get_gp_data    = rcu_tasks_trace_get_gp_data,
1185	.cbflood_max	= 50000,
1186	.irq_capable	= 1,
1187	.slow_gps	= 1,
1188	.name		= "tasks-tracing"
1189};
1190
1191#define TASKS_TRACING_OPS &tasks_tracing_ops,
1192
1193#else // #ifdef CONFIG_TASKS_TRACE_RCU
1194
1195#define TASKS_TRACING_OPS
1196
1197#endif // #else #ifdef CONFIG_TASKS_TRACE_RCU
1198
1199
1200static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old)
1201{
1202	if (!cur_ops->gp_diff)
1203		return new - old;
1204	return cur_ops->gp_diff(new, old);
1205}
1206
1207/*
1208 * RCU torture priority-boost testing.  Runs one real-time thread per
1209 * CPU for moderate bursts, repeatedly starting grace periods and waiting
1210 * for them to complete.  If a given grace period takes too long, we assume
1211 * that priority inversion has occurred.
1212 */
1213
1214static int old_rt_runtime = -1;
1215
1216static void rcu_torture_disable_rt_throttle(void)
1217{
1218	/*
1219	 * Disable RT throttling so that rcutorture's boost threads don't get
1220	 * throttled. Only possible if rcutorture is built-in otherwise the
1221	 * user should manually do this by setting the sched_rt_period_us and
1222	 * sched_rt_runtime sysctls.
1223	 */
1224	if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1)
1225		return;
1226
1227	old_rt_runtime = sysctl_sched_rt_runtime;
1228	sysctl_sched_rt_runtime = -1;
1229}
1230
1231static void rcu_torture_enable_rt_throttle(void)
1232{
1233	if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1)
1234		return;
1235
1236	sysctl_sched_rt_runtime = old_rt_runtime;
1237	old_rt_runtime = -1;
1238}
1239
1240static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start)
1241{
1242	int cpu;
1243	static int dbg_done;
1244	unsigned long end = jiffies;
1245	bool gp_done;
1246	unsigned long j;
1247	static unsigned long last_persist;
1248	unsigned long lp;
1249	unsigned long mininterval = test_boost_duration * HZ - HZ / 2;
1250
1251	if (end - *start > mininterval) {
1252		// Recheck after checking time to avoid false positives.
1253		smp_mb(); // Time check before grace-period check.
1254		if (cur_ops->poll_gp_state(gp_state))
1255			return false; // passed, though perhaps just barely
1256		if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) {
1257			// At most one persisted message per boost test.
1258			j = jiffies;
1259			lp = READ_ONCE(last_persist);
1260			if (time_after(j, lp + mininterval) &&
1261			    cmpxchg(&last_persist, lp, j) == lp) {
1262				if (cpu < 0)
1263					pr_info("Boost inversion persisted: QS from all CPUs\n");
1264				else
1265					pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu);
1266			}
1267			return false; // passed on a technicality
1268		}
1269		VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
1270		n_rcu_torture_boost_failure++;
1271		if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) {
1272			pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n",
1273				current->rt_priority, gp_state, end - *start);
1274			cur_ops->gp_kthread_dbg();
1275			// Recheck after print to flag grace period ending during splat.
1276			gp_done = cur_ops->poll_gp_state(gp_state);
1277			pr_info("Boost inversion: GP %lu %s.\n", gp_state,
1278				gp_done ? "ended already" : "still pending");
1279
1280		}
1281
1282		return true; // failed
1283	} else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) {
1284		*start = jiffies;
1285	}
1286
1287	return false; // passed
1288}
1289
1290static int rcu_torture_boost(void *arg)
1291{
1292	unsigned long endtime;
1293	unsigned long gp_state;
1294	unsigned long gp_state_time;
1295	unsigned long oldstarttime;
1296	unsigned long booststarttime = get_torture_init_jiffies() + test_boost_holdoff * HZ;
1297
1298	if (test_boost_holdoff <= 0 || time_after(jiffies, booststarttime)) {
1299		VERBOSE_TOROUT_STRING("rcu_torture_boost started");
1300	} else {
1301		VERBOSE_TOROUT_STRING("rcu_torture_boost started holdoff period");
1302		while (time_before(jiffies, booststarttime)) {
1303			schedule_timeout_idle(HZ);
1304			if (kthread_should_stop())
1305				goto cleanup;
1306		}
1307		VERBOSE_TOROUT_STRING("rcu_torture_boost finished holdoff period");
1308	}
1309
1310	/* Set real-time priority. */
1311	sched_set_fifo_low(current);
1312
1313	/* Each pass through the following loop does one boost-test cycle. */
1314	do {
1315		bool failed = false; // Test failed already in this test interval
1316		bool gp_initiated = false;
1317
1318		if (kthread_should_stop())
1319			goto checkwait;
1320
1321		/* Wait for the next test interval. */
1322		oldstarttime = READ_ONCE(boost_starttime);
1323		while (time_before(jiffies, oldstarttime)) {
1324			schedule_timeout_interruptible(oldstarttime - jiffies);
1325			if (stutter_wait("rcu_torture_boost"))
1326				sched_set_fifo_low(current);
1327			if (torture_must_stop())
1328				goto checkwait;
1329		}
1330
1331		// Do one boost-test interval.
1332		endtime = oldstarttime + test_boost_duration * HZ;
1333		while (time_before(jiffies, endtime)) {
1334			// Has current GP gone too long?
1335			if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1336				failed = rcu_torture_boost_failed(gp_state, &gp_state_time);
1337			// If we don't have a grace period in flight, start one.
1338			if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) {
1339				gp_state = cur_ops->start_gp_poll();
1340				gp_initiated = true;
1341				gp_state_time = jiffies;
1342			}
1343			if (stutter_wait("rcu_torture_boost")) {
1344				sched_set_fifo_low(current);
1345				// If the grace period already ended,
1346				// we don't know when that happened, so
1347				// start over.
1348				if (cur_ops->poll_gp_state(gp_state))
1349					gp_initiated = false;
1350			}
1351			if (torture_must_stop())
1352				goto checkwait;
1353		}
1354
1355		// In case the grace period extended beyond the end of the loop.
1356		if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state))
1357			rcu_torture_boost_failed(gp_state, &gp_state_time);
1358
1359		/*
1360		 * Set the start time of the next test interval.
1361		 * Yes, this is vulnerable to long delays, but such
1362		 * delays simply cause a false negative for the next
1363		 * interval.  Besides, we are running at RT priority,
1364		 * so delays should be relatively rare.
1365		 */
1366		while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) {
1367			if (mutex_trylock(&boost_mutex)) {
1368				if (oldstarttime == boost_starttime) {
1369					WRITE_ONCE(boost_starttime,
1370						   jiffies + test_boost_interval * HZ);
1371					n_rcu_torture_boosts++;
1372				}
1373				mutex_unlock(&boost_mutex);
1374				break;
1375			}
1376			schedule_timeout_uninterruptible(HZ / 20);
1377		}
1378
1379		/* Go do the stutter. */
1380checkwait:	if (stutter_wait("rcu_torture_boost"))
1381			sched_set_fifo_low(current);
1382	} while (!torture_must_stop());
1383
1384cleanup:
1385	/* Clean up and exit. */
1386	while (!kthread_should_stop()) {
1387		torture_shutdown_absorb("rcu_torture_boost");
1388		schedule_timeout_uninterruptible(HZ / 20);
1389	}
1390	torture_kthread_stopping("rcu_torture_boost");
1391	return 0;
1392}
1393
1394/*
1395 * RCU torture force-quiescent-state kthread.  Repeatedly induces
1396 * bursts of calls to force_quiescent_state(), increasing the probability
1397 * of occurrence of some important types of race conditions.
1398 */
1399static int
1400rcu_torture_fqs(void *arg)
1401{
1402	unsigned long fqs_resume_time;
1403	int fqs_burst_remaining;
1404	int oldnice = task_nice(current);
1405
1406	VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
1407	do {
1408		fqs_resume_time = jiffies + fqs_stutter * HZ;
1409		while (time_before(jiffies, fqs_resume_time) &&
1410		       !kthread_should_stop()) {
1411			schedule_timeout_interruptible(HZ / 20);
1412		}
1413		fqs_burst_remaining = fqs_duration;
1414		while (fqs_burst_remaining > 0 &&
1415		       !kthread_should_stop()) {
1416			cur_ops->fqs();
1417			udelay(fqs_holdoff);
1418			fqs_burst_remaining -= fqs_holdoff;
1419		}
1420		if (stutter_wait("rcu_torture_fqs"))
1421			sched_set_normal(current, oldnice);
1422	} while (!torture_must_stop());
1423	torture_kthread_stopping("rcu_torture_fqs");
1424	return 0;
1425}
1426
1427// Used by writers to randomly choose from the available grace-period primitives.
1428static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { };
1429static int nsynctypes;
1430
1431/*
1432 * Determine which grace-period primitives are available.
1433 */
1434static void rcu_torture_write_types(void)
1435{
1436	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full;
1437	bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp;
1438	bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
1439	bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;
1440
1441	/* Initialize synctype[] array.  If none set, take default. */
1442	if (!gp_cond1 &&
1443	    !gp_cond_exp1 &&
1444	    !gp_cond_full1 &&
1445	    !gp_cond_exp_full1 &&
1446	    !gp_exp1 &&
1447	    !gp_poll_exp1 &&
1448	    !gp_poll_exp_full1 &&
1449	    !gp_normal1 &&
1450	    !gp_poll1 &&
1451	    !gp_poll_full1 &&
1452	    !gp_sync1) {
1453		gp_cond1 = true;
1454		gp_cond_exp1 = true;
1455		gp_cond_full1 = true;
1456		gp_cond_exp_full1 = true;
1457		gp_exp1 = true;
1458		gp_poll_exp1 = true;
1459		gp_poll_exp_full1 = true;
1460		gp_normal1 = true;
1461		gp_poll1 = true;
1462		gp_poll_full1 = true;
1463		gp_sync1 = true;
1464	}
1465	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
1466		synctype[nsynctypes++] = RTWS_COND_GET;
1467		pr_info("%s: Testing conditional GPs.\n", __func__);
1468	} else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) {
1469		pr_alert("%s: gp_cond without primitives.\n", __func__);
1470	}
1471	if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) {
1472		synctype[nsynctypes++] = RTWS_COND_GET_EXP;
1473		pr_info("%s: Testing conditional expedited GPs.\n", __func__);
1474	} else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {
1475		pr_alert("%s: gp_cond_exp without primitives.\n", __func__);
1476	}
1477	if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {
1478		synctype[nsynctypes++] = RTWS_COND_GET_FULL;
1479		pr_info("%s: Testing conditional full-state GPs.\n", __func__);
1480	} else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {
1481		pr_alert("%s: gp_cond_full without primitives.\n", __func__);
1482	}
1483	if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {
1484		synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL;
1485		pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__);
1486	} else if (gp_cond_exp_full &&
1487		   (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {
1488		pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__);
1489	}
1490	if (gp_exp1 && cur_ops->exp_sync) {
1491		synctype[nsynctypes++] = RTWS_EXP_SYNC;
1492		pr_info("%s: Testing expedited GPs.\n", __func__);
1493	} else if (gp_exp && !cur_ops->exp_sync) {
1494		pr_alert("%s: gp_exp without primitives.\n", __func__);
1495	}
1496	if (gp_normal1 && cur_ops->deferred_free) {
1497		synctype[nsynctypes++] = RTWS_DEF_FREE;
1498		pr_info("%s: Testing asynchronous GPs.\n", __func__);
1499	} else if (gp_normal && !cur_ops->deferred_free) {
1500		pr_alert("%s: gp_normal without primitives.\n", __func__);
1501	}
1502	if (gp_poll1 && cur_ops->get_comp_state && cur_ops->same_gp_state &&
1503	    cur_ops->start_gp_poll && cur_ops->poll_gp_state) {
1504		synctype[nsynctypes++] = RTWS_POLL_GET;
1505		pr_info("%s: Testing polling GPs.\n", __func__);
1506	} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
1507		pr_alert("%s: gp_poll without primitives.\n", __func__);
1508	}
1509	if (gp_poll_full1 && cur_ops->get_comp_state_full && cur_ops->same_gp_state_full
1510	    && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
1511		synctype[nsynctypes++] = RTWS_POLL_GET_FULL;
1512		pr_info("%s: Testing polling full-state GPs.\n", __func__);
1513	} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
1514		pr_alert("%s: gp_poll_full without primitives.\n", __func__);
1515	}
1516	if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
1517		synctype[nsynctypes++] = RTWS_POLL_GET_EXP;
1518		pr_info("%s: Testing polling expedited GPs.\n", __func__);
1519	} else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {
1520		pr_alert("%s: gp_poll_exp without primitives.\n", __func__);
1521	}
1522	if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {
1523		synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL;
1524		pr_info("%s: Testing polling full-state expedited GPs.\n", __func__);
1525	} else if (gp_poll_exp_full &&
1526		   (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {
1527		pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__);
1528	}
1529	if (gp_sync1 && cur_ops->sync) {
1530		synctype[nsynctypes++] = RTWS_SYNC;
1531		pr_info("%s: Testing normal GPs.\n", __func__);
1532	} else if (gp_sync && !cur_ops->sync) {
1533		pr_alert("%s: gp_sync without primitives.\n", __func__);
1534	}
1535	pr_alert("%s: Testing %d update types.\n", __func__, nsynctypes);
1536	pr_info("%s: gp_cond_wi %d gp_cond_wi_exp %d gp_poll_wi %d gp_poll_wi_exp %d\n", __func__, gp_cond_wi, gp_cond_wi_exp, gp_poll_wi, gp_poll_wi_exp);
1537}
1538
1539/*
1540 * Do the specified rcu_torture_writer() synchronous grace period,
1541 * while also testing out the polled APIs.  Note well that the single-CPU
1542 * grace-period optimizations must be accounted for.
1543 */
1544static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void))
1545{
1546	unsigned long cookie;
1547	struct rcu_gp_oldstate cookie_full;
1548	bool dopoll;
1549	bool dopoll_full;
1550	unsigned long r = torture_random(trsp);
1551
1552	dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);
1553	dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);
1554	if (dopoll || dopoll_full)
1555		cpus_read_lock();
1556	if (dopoll)
1557		cookie = cur_ops->get_gp_state();
1558	if (dopoll_full)
1559		cur_ops->get_gp_state_full(&cookie_full);
1560	if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))
1561		sync();
1562	sync();
1563	WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),
1564		  "%s: Cookie check 3 failed %pS() online %*pbl.",
1565		  __func__, sync, cpumask_pr_args(cpu_online_mask));
1566	WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),
1567		  "%s: Cookie check 4 failed %pS() online %*pbl",
1568		  __func__, sync, cpumask_pr_args(cpu_online_mask));
1569	if (dopoll || dopoll_full)
1570		cpus_read_unlock();
1571}
1572
1573/*
1574 * RCU torture writer kthread.  Repeatedly substitutes a new structure
1575 * for that pointed to by rcu_torture_current, freeing the old structure
1576 * after a series of grace periods (the "pipeline").
1577 */
1578static int
1579rcu_torture_writer(void *arg)
1580{
1581	bool booting_still = false;
1582	bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();
1583	unsigned long cookie;
1584	struct rcu_gp_oldstate cookie_full;
1585	int expediting = 0;
1586	unsigned long gp_snap;
1587	unsigned long gp_snap1;
1588	struct rcu_gp_oldstate gp_snap_full;
1589	struct rcu_gp_oldstate gp_snap1_full;
1590	int i;
1591	int idx;
1592	unsigned long j;
1593	int oldnice = task_nice(current);
1594	struct rcu_gp_oldstate *rgo = NULL;
1595	int rgo_size = 0;
1596	struct rcu_torture *rp;
1597	struct rcu_torture *old_rp;
1598	static DEFINE_TORTURE_RANDOM(rand);
1599	unsigned long stallsdone = jiffies;
1600	bool stutter_waited;
1601	unsigned long *ulo = NULL;
1602	int ulo_size = 0;
1603
1604	// If a new stall test is added, this must be adjusted.
1605	if (stall_cpu_holdoff + stall_gp_kthread + stall_cpu)
1606		stallsdone += (stall_cpu_holdoff + stall_gp_kthread + stall_cpu + 60) *
1607			      HZ * (stall_cpu_repeat + 1);
1608	VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
1609	if (!can_expedite)
1610		pr_alert("%s" TORTURE_FLAG
1611			 " GP expediting controlled from boot/sysfs for %s.\n",
1612			 torture_type, cur_ops->name);
1613	if (WARN_ONCE(nsynctypes == 0,
1614		      "%s: No update-side primitives.\n", __func__)) {
1615		/*
1616		 * No updates primitives, so don't try updating.
1617		 * The resulting test won't be testing much, hence the
1618		 * above WARN_ONCE().
1619		 */
1620		rcu_torture_writer_state = RTWS_STOPPING;
1621		torture_kthread_stopping("rcu_torture_writer");
1622		return 0;
1623	}
1624	if (cur_ops->poll_active > 0) {
1625		ulo = kcalloc(cur_ops->poll_active, sizeof(*ulo), GFP_KERNEL);
1626		if (!WARN_ON(!ulo))
1627			ulo_size = cur_ops->poll_active;
1628	}
1629	if (cur_ops->poll_active_full > 0) {
1630		rgo = kcalloc(cur_ops->poll_active_full, sizeof(*rgo), GFP_KERNEL);
1631		if (!WARN_ON(!rgo))
1632			rgo_size = cur_ops->poll_active_full;
1633	}
1634
1635	// If the system is still booting, let it finish.
1636	j = jiffies;
1637	while (!torture_must_stop() && !rcu_inkernel_boot_has_ended()) {
1638		booting_still = true;
1639		schedule_timeout_interruptible(HZ);
1640	}
1641	if (booting_still)
1642		pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n",
1643			 torture_type, jiffies - j);
1644
1645	do {
1646		rcu_torture_writer_state = RTWS_FIXED_DELAY;
1647		torture_hrtimeout_us(500, 1000, &rand);
1648		rp = rcu_torture_alloc();
1649		if (rp == NULL)
1650			continue;
1651		rp->rtort_pipe_count = 0;
1652		ASSERT_EXCLUSIVE_WRITER(rp->rtort_pipe_count);
1653		rcu_torture_writer_state = RTWS_DELAY;
1654		udelay(torture_random(&rand) & 0x3ff);
1655		rcu_torture_writer_state = RTWS_REPLACE;
1656		old_rp = rcu_dereference_check(rcu_torture_current,
1657					       current == writer_task);
1658		rp->rtort_mbtest = 1;
1659		rcu_assign_pointer(rcu_torture_current, rp);
1660		smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
1661		if (old_rp) {
1662			i = old_rp->rtort_pipe_count;
1663			if (i > RCU_TORTURE_PIPE_LEN)
1664				i = RCU_TORTURE_PIPE_LEN;
1665			atomic_inc(&rcu_torture_wcount[i]);
1666			WRITE_ONCE(old_rp->rtort_pipe_count,
1667				   old_rp->rtort_pipe_count + 1);
1668			ASSERT_EXCLUSIVE_WRITER(old_rp->rtort_pipe_count);
1669
1670			// Make sure readers block polled grace periods.
1671			if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {
1672				idx = cur_ops->readlock();
1673				cookie = cur_ops->get_gp_state();
1674				WARN_ONCE(cur_ops->poll_gp_state(cookie),
1675					  "%s: Cookie check 1 failed %s(%d) %lu->%lu\n",
1676					  __func__,
1677					  rcu_torture_writer_state_getname(),
1678					  rcu_torture_writer_state,
1679					  cookie, cur_ops->get_gp_state());
1680				if (cur_ops->get_comp_state) {
1681					cookie = cur_ops->get_comp_state();
1682					WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
1683				}
1684				cur_ops->readunlock(idx);
1685			}
1686			if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {
1687				idx = cur_ops->readlock();
1688				cur_ops->get_gp_state_full(&cookie_full);
1689				WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
1690					  "%s: Cookie check 5 failed %s(%d) online %*pbl\n",
1691					  __func__,
1692					  rcu_torture_writer_state_getname(),
1693					  rcu_torture_writer_state,
1694					  cpumask_pr_args(cpu_online_mask));
1695				if (cur_ops->get_comp_state_full) {
1696					cur_ops->get_comp_state_full(&cookie_full);
1697					WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));
1698				}
1699				cur_ops->readunlock(idx);
1700			}
1701			switch (synctype[torture_random(&rand) % nsynctypes]) {
1702			case RTWS_DEF_FREE:
1703				rcu_torture_writer_state = RTWS_DEF_FREE;
1704				cur_ops->deferred_free(old_rp);
1705				break;
1706			case RTWS_EXP_SYNC:
1707				rcu_torture_writer_state = RTWS_EXP_SYNC;
1708				do_rtws_sync(&rand, cur_ops->exp_sync);
1709				rcu_torture_pipe_update(old_rp);
1710				break;
1711			case RTWS_COND_GET:
1712				rcu_torture_writer_state = RTWS_COND_GET;
1713				gp_snap = cur_ops->get_gp_state();
1714				torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi,
1715						     1000, &rand);
1716				rcu_torture_writer_state = RTWS_COND_SYNC;
1717				cur_ops->cond_sync(gp_snap);
1718				rcu_torture_pipe_update(old_rp);
1719				break;
1720			case RTWS_COND_GET_EXP:
1721				rcu_torture_writer_state = RTWS_COND_GET_EXP;
1722				gp_snap = cur_ops->get_gp_state_exp();
1723				torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp,
1724						     1000, &rand);
1725				rcu_torture_writer_state = RTWS_COND_SYNC_EXP;
1726				cur_ops->cond_sync_exp(gp_snap);
1727				rcu_torture_pipe_update(old_rp);
1728				break;
1729			case RTWS_COND_GET_FULL:
1730				rcu_torture_writer_state = RTWS_COND_GET_FULL;
1731				cur_ops->get_gp_state_full(&gp_snap_full);
1732				torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi,
1733						     1000, &rand);
1734				rcu_torture_writer_state = RTWS_COND_SYNC_FULL;
1735				cur_ops->cond_sync_full(&gp_snap_full);
1736				rcu_torture_pipe_update(old_rp);
1737				break;
1738			case RTWS_COND_GET_EXP_FULL:
1739				rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL;
1740				cur_ops->get_gp_state_full(&gp_snap_full);
1741				torture_hrtimeout_us(torture_random(&rand) % gp_cond_wi_exp,
1742						     1000, &rand);
1743				rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL;
1744				cur_ops->cond_sync_exp_full(&gp_snap_full);
1745				rcu_torture_pipe_update(old_rp);
1746				break;
1747			case RTWS_POLL_GET:
1748				rcu_torture_writer_state = RTWS_POLL_GET;
1749				for (i = 0; i < ulo_size; i++)
1750					ulo[i] = cur_ops->get_comp_state();
1751				gp_snap = cur_ops->start_gp_poll();
1752				rcu_torture_writer_state = RTWS_POLL_WAIT;
1753				if (cur_ops->exp_current && !(torture_random(&rand) & 0xff))
1754					cur_ops->exp_current();
1755				while (!cur_ops->poll_gp_state(gp_snap)) {
1756					gp_snap1 = cur_ops->get_gp_state();
1757					for (i = 0; i < ulo_size; i++)
1758						if (cur_ops->poll_gp_state(ulo[i]) ||
1759						    cur_ops->same_gp_state(ulo[i], gp_snap1)) {
1760							ulo[i] = gp_snap1;
1761							break;
1762						}
1763					WARN_ON_ONCE(ulo_size > 0 && i >= ulo_size);
1764					torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi,
1765							     1000, &rand);
1766				}
1767				rcu_torture_pipe_update(old_rp);
1768				break;
1769			case RTWS_POLL_GET_FULL:
1770				rcu_torture_writer_state = RTWS_POLL_GET_FULL;
1771				for (i = 0; i < rgo_size; i++)
1772					cur_ops->get_comp_state_full(&rgo[i]);
1773				cur_ops->start_gp_poll_full(&gp_snap_full);
1774				rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
1775				if (cur_ops->exp_current && !(torture_random(&rand) & 0xff))
1776					cur_ops->exp_current();
1777				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1778					cur_ops->get_gp_state_full(&gp_snap1_full);
1779					for (i = 0; i < rgo_size; i++)
1780						if (cur_ops->poll_gp_state_full(&rgo[i]) ||
1781						    cur_ops->same_gp_state_full(&rgo[i],
1782										&gp_snap1_full)) {
1783							rgo[i] = gp_snap1_full;
1784							break;
1785						}
1786					WARN_ON_ONCE(rgo_size > 0 && i >= rgo_size);
1787					torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi,
1788							     1000, &rand);
1789				}
1790				rcu_torture_pipe_update(old_rp);
1791				break;
1792			case RTWS_POLL_GET_EXP:
1793				rcu_torture_writer_state = RTWS_POLL_GET_EXP;
1794				gp_snap = cur_ops->start_gp_poll_exp();
1795				rcu_torture_writer_state = RTWS_POLL_WAIT_EXP;
1796				while (!cur_ops->poll_gp_state_exp(gp_snap))
1797					torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp,
1798							     1000, &rand);
1799				rcu_torture_pipe_update(old_rp);
1800				break;
1801			case RTWS_POLL_GET_EXP_FULL:
1802				rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL;
1803				cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1804				rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL;
1805				while (!cur_ops->poll_gp_state_full(&gp_snap_full))
1806					torture_hrtimeout_us(torture_random(&rand) % gp_poll_wi_exp,
1807							     1000, &rand);
1808				rcu_torture_pipe_update(old_rp);
1809				break;
1810			case RTWS_SYNC:
1811				rcu_torture_writer_state = RTWS_SYNC;
1812				do_rtws_sync(&rand, cur_ops->sync);
1813				rcu_torture_pipe_update(old_rp);
1814				break;
1815			default:
1816				WARN_ON_ONCE(1);
1817				break;
1818			}
1819		}
1820		WRITE_ONCE(rcu_torture_current_version,
1821			   rcu_torture_current_version + 1);
1822		/* Cycle through nesting levels of rcu_expedite_gp() calls. */
1823		if (can_expedite &&
1824		    !(torture_random(&rand) & 0xff & (!!expediting - 1))) {
1825			WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited());
1826			if (expediting >= 0)
1827				rcu_expedite_gp();
1828			else
1829				rcu_unexpedite_gp();
1830			if (++expediting > 3)
1831				expediting = -expediting;
1832		} else if (!can_expedite) { /* Disabled during boot, recheck. */
1833			can_expedite = !rcu_gp_is_expedited() &&
1834				       !rcu_gp_is_normal();
1835		}
1836		rcu_torture_writer_state = RTWS_STUTTER;
1837		stutter_waited = stutter_wait("rcu_torture_writer");
1838		if (stutter_waited &&
1839		    !atomic_read(&rcu_fwd_cb_nodelay) &&
1840		    !cur_ops->slow_gps &&
1841		    !torture_must_stop() &&
1842		    time_after(jiffies, stallsdone))
1843			for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)
1844				if (list_empty(&rcu_tortures[i].rtort_free) &&
1845				    rcu_access_pointer(rcu_torture_current) != &rcu_tortures[i]) {
1846					tracing_off();
1847					if (cur_ops->gp_kthread_dbg)
1848						cur_ops->gp_kthread_dbg();
1849					WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count);
1850					rcu_ftrace_dump(DUMP_ALL);
1851					break;
1852				}
1853		if (stutter_waited)
1854			sched_set_normal(current, oldnice);
1855	} while (!torture_must_stop());
1856	rcu_torture_current = NULL;  // Let stats task know that we are done.
1857	/* Reset expediting back to unexpedited. */
1858	if (expediting > 0)
1859		expediting = -expediting;
1860	while (can_expedite && expediting++ < 0)
1861		rcu_unexpedite_gp();
1862	WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited());
1863	if (!can_expedite)
1864		pr_alert("%s" TORTURE_FLAG
1865			 " Dynamic grace-period expediting was disabled.\n",
1866			 torture_type);
1867	kfree(ulo);
1868	kfree(rgo);
1869	rcu_torture_writer_state = RTWS_STOPPING;
1870	torture_kthread_stopping("rcu_torture_writer");
1871	return 0;
1872}
1873
1874/*
1875 * RCU torture fake writer kthread.  Repeatedly calls sync, with a random
1876 * delay between calls.
1877 */
1878static int
1879rcu_torture_fakewriter(void *arg)
1880{
1881	unsigned long gp_snap;
1882	struct rcu_gp_oldstate gp_snap_full;
1883	DEFINE_TORTURE_RANDOM(rand);
1884
1885	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1886	set_user_nice(current, MAX_NICE);
1887
1888	if (WARN_ONCE(nsynctypes == 0,
1889		      "%s: No update-side primitives.\n", __func__)) {
1890		/*
1891		 * No updates primitives, so don't try updating.
1892		 * The resulting test won't be testing much, hence the
1893		 * above WARN_ONCE().
1894		 */
1895		torture_kthread_stopping("rcu_torture_fakewriter");
1896		return 0;
1897	}
1898
1899	do {
1900		torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand);
1901		if (cur_ops->cb_barrier != NULL &&
1902		    torture_random(&rand) % (nrealfakewriters * 8) == 0) {
1903			cur_ops->cb_barrier();
1904		} else {
1905			switch (synctype[torture_random(&rand) % nsynctypes]) {
1906			case RTWS_DEF_FREE:
1907				break;
1908			case RTWS_EXP_SYNC:
1909				cur_ops->exp_sync();
1910				break;
1911			case RTWS_COND_GET:
1912				gp_snap = cur_ops->get_gp_state();
1913				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
1914				cur_ops->cond_sync(gp_snap);
1915				break;
1916			case RTWS_COND_GET_EXP:
1917				gp_snap = cur_ops->get_gp_state_exp();
1918				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
1919				cur_ops->cond_sync_exp(gp_snap);
1920				break;
1921			case RTWS_COND_GET_FULL:
1922				cur_ops->get_gp_state_full(&gp_snap_full);
1923				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
1924				cur_ops->cond_sync_full(&gp_snap_full);
1925				break;
1926			case RTWS_COND_GET_EXP_FULL:
1927				cur_ops->get_gp_state_full(&gp_snap_full);
1928				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
1929				cur_ops->cond_sync_exp_full(&gp_snap_full);
1930				break;
1931			case RTWS_POLL_GET:
1932				if (cur_ops->start_poll_irqsoff)
1933					local_irq_disable();
1934				gp_snap = cur_ops->start_gp_poll();
1935				if (cur_ops->start_poll_irqsoff)
1936					local_irq_enable();
1937				while (!cur_ops->poll_gp_state(gp_snap)) {
1938					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
1939								  &rand);
1940				}
1941				break;
1942			case RTWS_POLL_GET_FULL:
1943				if (cur_ops->start_poll_irqsoff)
1944					local_irq_disable();
1945				cur_ops->start_gp_poll_full(&gp_snap_full);
1946				if (cur_ops->start_poll_irqsoff)
1947					local_irq_enable();
1948				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1949					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
1950								  &rand);
1951				}
1952				break;
1953			case RTWS_POLL_GET_EXP:
1954				gp_snap = cur_ops->start_gp_poll_exp();
1955				while (!cur_ops->poll_gp_state_exp(gp_snap)) {
1956					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
1957								  &rand);
1958				}
1959				break;
1960			case RTWS_POLL_GET_EXP_FULL:
1961				cur_ops->start_gp_poll_exp_full(&gp_snap_full);
1962				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
1963					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
1964								  &rand);
1965				}
1966				break;
1967			case RTWS_SYNC:
1968				cur_ops->sync();
1969				break;
1970			default:
1971				WARN_ON_ONCE(1);
1972				break;
1973			}
1974		}
1975		stutter_wait("rcu_torture_fakewriter");
1976	} while (!torture_must_stop());
1977
1978	torture_kthread_stopping("rcu_torture_fakewriter");
1979	return 0;
1980}
1981
1982static void rcu_torture_timer_cb(struct rcu_head *rhp)
1983{
1984	kfree(rhp);
1985}
1986
1987// Set up and carry out testing of RCU's global memory ordering
1988static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp,
1989					struct torture_random_state *trsp)
1990{
1991	unsigned long loops;
1992	int noc = torture_num_online_cpus();
1993	int rdrchked;
1994	int rdrchker;
1995	struct rcu_torture_reader_check *rtrcp; // Me.
1996	struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking.
1997	struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked.
1998	struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me.
1999
2000	if (myid < 0)
2001		return; // Don't try this from timer handlers.
2002
2003	// Increment my counter.
2004	rtrcp = &rcu_torture_reader_mbchk[myid];
2005	WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1);
2006
2007	// Attempt to assign someone else some checking work.
2008	rdrchked = torture_random(trsp) % nrealreaders;
2009	rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked];
2010	rdrchker = torture_random(trsp) % nrealreaders;
2011	rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker];
2012	if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker &&
2013	    smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below.
2014	    !READ_ONCE(rtp->rtort_chkp) &&
2015	    !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below.
2016		rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops);
2017		WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0);
2018		rtrcp->rtc_chkrdr = rdrchked;
2019		WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends.
2020		if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) ||
2021		    cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp))
2022			(void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out.
2023	}
2024
2025	// If assigned some completed work, do it!
2026	rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner);
2027	if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready))
2028		return; // No work or work not yet ready.
2029	rdrchked = rtrcp_assigner->rtc_chkrdr;
2030	if (WARN_ON_ONCE(rdrchked < 0))
2031		return;
2032	rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked];
2033	loops = READ_ONCE(rtrcp_chked->rtc_myloops);
2034	atomic_inc(&n_rcu_torture_mbchk_tries);
2035	if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops))
2036		atomic_inc(&n_rcu_torture_mbchk_fail);
2037	rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2;
2038	rtrcp_assigner->rtc_ready = 0;
2039	smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work.
2040	smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign.
2041}
2042
2043// Verify the specified RCUTORTURE_RDR* state.
2044#define ROEC_ARGS "%s %s: Current %#x  To add %#x  To remove %#x  preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count()
2045static void rcutorture_one_extend_check(char *s, int curstate, int new, int old)
2046{
2047	int mask;
2048
2049	if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi())
2050		return;
2051
2052	WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq(), ROEC_ARGS);
2053	WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS);
2054
2055	// If CONFIG_PREEMPT_COUNT=n, further checks are unreliable.
2056	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
2057		return;
2058
2059	WARN_ONCE((curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) &&
2060		  !softirq_count(), ROEC_ARGS);
2061	WARN_ONCE((curstate & (RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED)) &&
2062		  !(preempt_count() & PREEMPT_MASK), ROEC_ARGS);
2063	WARN_ONCE(cur_ops->readlock_nesting &&
2064		  (curstate & (RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2)) &&
2065		  cur_ops->readlock_nesting() == 0, ROEC_ARGS);
2066
2067	// Interrupt handlers have all sorts of stuff disabled, so ignore
2068	// unintended disabling.
2069	if (in_serving_softirq() || in_hardirq())
2070		return;
2071
2072	WARN_ONCE(cur_ops->extendables &&
2073		  !(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) &&
2074		  softirq_count(), ROEC_ARGS);
2075
2076	/*
2077	 * non-preemptible RCU in a preemptible kernel uses preempt_disable()
2078	 * as rcu_read_lock().
2079	 */
2080	mask = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
2081	if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
2082		mask |= RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
2083
2084	WARN_ONCE(cur_ops->extendables && !(curstate & mask) &&
2085		  (preempt_count() & PREEMPT_MASK), ROEC_ARGS);
2086
2087	/*
2088	 * non-preemptible RCU in a preemptible kernel uses "preempt_count() &
2089	 * PREEMPT_MASK" as ->readlock_nesting().
2090	 */
2091	mask = RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
2092	if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
2093		mask |= RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
2094
2095	if (IS_ENABLED(CONFIG_PREEMPT_RT) && softirq_count())
2096		mask |= RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
2097
2098	WARN_ONCE(cur_ops->readlock_nesting && !(curstate & mask) &&
2099		  cur_ops->readlock_nesting() > 0, ROEC_ARGS);
2100}
2101
2102/*
2103 * Do one extension of an RCU read-side critical section using the
2104 * current reader state in readstate (set to zero for initial entry
2105 * to extended critical section), set the new state as specified by
2106 * newstate (set to zero for final exit from extended critical section),
2107 * and random-number-generator state in trsp.  If this is neither the
2108 * beginning or end of the critical section and if there was actually a
2109 * change, do a ->read_delay().
2110 */
2111static void rcutorture_one_extend(int *readstate, int newstate, struct torture_random_state *trsp,
2112				  struct rt_read_seg *rtrsp)
2113{
2114	bool first;
2115	unsigned long flags;
2116	int idxnew1 = -1;
2117	int idxnew2 = -1;
2118	int idxold1 = *readstate;
2119	int idxold2 = idxold1;
2120	int statesnew = ~*readstate & newstate;
2121	int statesold = *readstate & ~newstate;
2122
2123	first = idxold1 == 0;
2124	WARN_ON_ONCE(idxold2 < 0);
2125	WARN_ON_ONCE(idxold2 & ~(RCUTORTURE_RDR_ALLBITS | RCUTORTURE_RDR_UPDOWN));
2126	rcutorture_one_extend_check("before change", idxold1, statesnew, statesold);
2127	rtrsp->rt_readstate = newstate;
2128
2129	/* First, put new protection in place to avoid critical-section gap. */
2130	if (statesnew & RCUTORTURE_RDR_BH)
2131		local_bh_disable();
2132	if (statesnew & RCUTORTURE_RDR_RBH)
2133		rcu_read_lock_bh();
2134	if (statesnew & RCUTORTURE_RDR_IRQ)
2135		local_irq_disable();
2136	if (statesnew & RCUTORTURE_RDR_PREEMPT)
2137		preempt_disable();
2138	if (statesnew & RCUTORTURE_RDR_SCHED)
2139		rcu_read_lock_sched();
2140	if (statesnew & RCUTORTURE_RDR_RCU_1)
2141		idxnew1 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1;
2142	if (statesnew & RCUTORTURE_RDR_RCU_2)
2143		idxnew2 = (cur_ops->readlock() << RCUTORTURE_RDR_SHIFT_2) & RCUTORTURE_RDR_MASK_2;
2144
2145	// Complain unless both the old and the new protection is in place.
2146	rcutorture_one_extend_check("during change", idxold1 | statesnew, statesnew, statesold);
2147
2148	// Sample CPU under both sets of protections to reduce confusion.
2149	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) {
2150		int cpu = raw_smp_processor_id();
2151		rtrsp->rt_cpu = cpu;
2152		if (!first) {
2153			rtrsp[-1].rt_end_cpu = cpu;
2154			if (cur_ops->reader_blocked)
2155				rtrsp[-1].rt_preempted = cur_ops->reader_blocked();
2156		}
2157	}
2158	// Sample grace-period sequence number, as good a place as any.
2159	if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && cur_ops->gather_gp_seqs) {
2160		rtrsp->rt_gp_seq = cur_ops->gather_gp_seqs();
2161		rtrsp->rt_ts = ktime_get_mono_fast_ns();
2162		if (!first)
2163			rtrsp[-1].rt_gp_seq_end = rtrsp->rt_gp_seq;
2164	}
2165
2166	/*
2167	 * Next, remove old protection, in decreasing order of strength
2168	 * to avoid unlock paths that aren't safe in the stronger
2169	 * context. Namely: BH can not be enabled with disabled interrupts.
2170	 * Additionally PREEMPT_RT requires that BH is enabled in preemptible
2171	 * context.
2172	 */
2173	if (statesold & RCUTORTURE_RDR_IRQ)
2174		local_irq_enable();
2175	if (statesold & RCUTORTURE_RDR_PREEMPT)
2176		preempt_enable();
2177	if (statesold & RCUTORTURE_RDR_SCHED)
2178		rcu_read_unlock_sched();
2179	if (statesold & RCUTORTURE_RDR_BH)
2180		local_bh_enable();
2181	if (statesold & RCUTORTURE_RDR_RBH)
2182		rcu_read_unlock_bh();
2183	if (statesold & RCUTORTURE_RDR_RCU_2) {
2184		cur_ops->readunlock((idxold2 & RCUTORTURE_RDR_MASK_2) >> RCUTORTURE_RDR_SHIFT_2);
2185		WARN_ON_ONCE(idxnew2 != -1);
2186		idxold2 = 0;
2187	}
2188	if (statesold & RCUTORTURE_RDR_RCU_1) {
2189		bool lockit;
2190
2191		lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff);
2192		if (lockit)
2193			raw_spin_lock_irqsave(&current->pi_lock, flags);
2194		cur_ops->readunlock((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1);
2195		WARN_ON_ONCE(idxnew1 != -1);
2196		idxold1 = 0;
2197		if (lockit)
2198			raw_spin_unlock_irqrestore(&current->pi_lock, flags);
2199	}
2200	if (statesold & RCUTORTURE_RDR_UPDOWN) {
2201		cur_ops->up_read((idxold1 & RCUTORTURE_RDR_MASK_1) >> RCUTORTURE_RDR_SHIFT_1);
2202		WARN_ON_ONCE(idxnew1 != -1);
2203		idxold1 = 0;
2204	}
2205
2206	/* Delay if neither beginning nor end and there was a change. */
2207	if ((statesnew || statesold) && *readstate && newstate)
2208		cur_ops->read_delay(trsp, rtrsp);
2209
2210	/* Update the reader state. */
2211	if (idxnew1 == -1)
2212		idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1;
2213	WARN_ON_ONCE(idxnew1 < 0);
2214	if (idxnew2 == -1)
2215		idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2;
2216	WARN_ON_ONCE(idxnew2 < 0);
2217	*readstate = idxnew1 | idxnew2 | newstate;
2218	WARN_ON_ONCE(*readstate < 0);
2219	if (WARN_ON_ONCE(*readstate & ~RCUTORTURE_RDR_ALLBITS))
2220		pr_info("Unexpected readstate value of %#x\n", *readstate);
2221	rcutorture_one_extend_check("after change", *readstate, statesnew, statesold);
2222}
2223
2224/* Return the biggest extendables mask given current RCU and boot parameters. */
2225static int rcutorture_extend_mask_max(void)
2226{
2227	int mask;
2228
2229	WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND);
2230	mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables;
2231	mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2;
2232	return mask;
2233}
2234
2235/* Return a random protection state mask, but with at least one bit set. */
2236static int
2237rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp)
2238{
2239	int mask = rcutorture_extend_mask_max();
2240	unsigned long randmask1 = torture_random(trsp);
2241	unsigned long randmask2 = randmask1 >> 3;
2242	unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED;
2243	unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ;
2244	unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH;
2245
2246	WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1);  // Can't have reader idx bits.
2247	/* Mostly only one bit (need preemption!), sometimes lots of bits. */
2248	if (!(randmask1 & 0x7))
2249		mask = mask & randmask2;
2250	else
2251		mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS));
2252
2253	// Can't have nested RCU reader without outer RCU reader.
2254	if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) {
2255		if (oldmask & RCUTORTURE_RDR_RCU_1)
2256			mask &= ~RCUTORTURE_RDR_RCU_2;
2257		else
2258			mask |= RCUTORTURE_RDR_RCU_1;
2259	}
2260
2261	/*
2262	 * Can't enable bh w/irq disabled.
2263	 */
2264	if (mask & RCUTORTURE_RDR_IRQ)
2265		mask |= oldmask & bhs;
2266
2267	/*
2268	 * Ideally these sequences would be detected in debug builds
2269	 * (regardless of RT), but until then don't stop testing
2270	 * them on non-RT.
2271	 */
2272	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
2273		/* Can't modify BH in atomic context */
2274		if (oldmask & preempts_irq)
2275			mask &= ~bhs;
2276		if ((oldmask | mask) & preempts_irq)
2277			mask |= oldmask & bhs;
2278	}
2279
2280	return mask ?: RCUTORTURE_RDR_RCU_1;
2281}
2282
2283/*
2284 * Do a randomly selected number of extensions of an existing RCU read-side
2285 * critical section.
2286 */
2287static struct rt_read_seg *
2288rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct rt_read_seg *rtrsp)
2289{
2290	int i;
2291	int j;
2292	int mask = rcutorture_extend_mask_max();
2293
2294	WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */
2295	if (!((mask - 1) & mask))
2296		return rtrsp;  /* Current RCU reader not extendable. */
2297	/* Bias towards larger numbers of loops. */
2298	i = torture_random(trsp);
2299	i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1;
2300	for (j = 0; j < i; j++) {
2301		mask = rcutorture_extend_mask(*readstate, trsp);
2302		WARN_ON_ONCE(mask & RCUTORTURE_RDR_UPDOWN);
2303		rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]);
2304	}
2305	return &rtrsp[j];
2306}
2307
2308struct rcu_torture_one_read_state {
2309	bool checkpolling;
2310	unsigned long cookie;
2311	struct rcu_gp_oldstate cookie_full;
2312	unsigned long started;
2313	struct rcu_torture *p;
2314	int readstate;
2315	struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS];
2316	struct rt_read_seg *rtrsp;
2317	unsigned long long ts;
2318};
2319
2320static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp,
2321					    struct torture_random_state *trsp)
2322{
2323	memset(rtorsp, 0, sizeof(*rtorsp));
2324	rtorsp->checkpolling = !(torture_random(trsp) & 0xfff);
2325	rtorsp->rtrsp = &rtorsp->rtseg[0];
2326}
2327
2328/*
2329 * Set up the first segment of a series of overlapping read-side
2330 * critical sections.  The caller must have actually initiated the
2331 * outermost read-side critical section.
2332 */
2333static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp,
2334				       struct torture_random_state *trsp, long myid)
2335{
2336	if (rtorsp->checkpolling) {
2337		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2338			rtorsp->cookie = cur_ops->get_gp_state();
2339		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2340			cur_ops->get_gp_state_full(&rtorsp->cookie_full);
2341	}
2342	rtorsp->started = cur_ops->get_gp_seq();
2343	rtorsp->ts = rcu_trace_clock_local();
2344	rtorsp->p = rcu_dereference_check(rcu_torture_current,
2345					  !cur_ops->readlock_held || cur_ops->readlock_held() ||
2346					  (rtorsp->readstate & RCUTORTURE_RDR_UPDOWN));
2347	if (rtorsp->p == NULL) {
2348		/* Wait for rcu_torture_writer to get underway */
2349		rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp);
2350		return false;
2351	}
2352	if (rtorsp->p->rtort_mbtest == 0)
2353		atomic_inc(&n_rcu_torture_mberror);
2354	rcu_torture_reader_do_mbchk(myid, rtorsp->p, trsp);
2355	return true;
2356}
2357
2358/*
2359 * Complete the last segment of a series of overlapping read-side
2360 * critical sections and check for errors.
2361 */
2362static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp,
2363				     struct torture_random_state *trsp)
2364{
2365	int i;
2366	unsigned long completed;
2367	int pipe_count;
2368	bool preempted = false;
2369	struct rt_read_seg *rtrsp1;
2370
2371	preempt_disable();
2372	pipe_count = READ_ONCE(rtorsp->p->rtort_pipe_count);
2373	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
2374		// Should not happen in a correct RCU implementation,
2375		// happens quite often for torture_type=busted.
2376		pipe_count = RCU_TORTURE_PIPE_LEN;
2377	}
2378	completed = cur_ops->get_gp_seq();
2379	if (pipe_count > 1) {
2380		do_trace_rcu_torture_read(cur_ops->name, &rtorsp->p->rtort_rcu,
2381					  rtorsp->ts, rtorsp->started, completed);
2382		rcu_ftrace_dump(DUMP_ALL);
2383	}
2384	__this_cpu_inc(rcu_torture_count[pipe_count]);
2385	completed = rcutorture_seq_diff(completed, rtorsp->started);
2386	if (completed > RCU_TORTURE_PIPE_LEN) {
2387		/* Should not happen, but... */
2388		completed = RCU_TORTURE_PIPE_LEN;
2389	}
2390	__this_cpu_inc(rcu_torture_batch[completed]);
2391	preempt_enable();
2392	if (rtorsp->checkpolling) {
2393		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
2394			WARN_ONCE(cur_ops->poll_gp_state(rtorsp->cookie),
2395				  "%s: Cookie check 2 failed %s(%d) %lu->%lu\n",
2396				  __func__,
2397				  rcu_torture_writer_state_getname(),
2398				  rcu_torture_writer_state,
2399				  rtorsp->cookie, cur_ops->get_gp_state());
2400		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
2401			WARN_ONCE(cur_ops->poll_gp_state_full(&rtorsp->cookie_full),
2402				  "%s: Cookie check 6 failed %s(%d) online %*pbl\n",
2403				  __func__,
2404				  rcu_torture_writer_state_getname(),
2405				  rcu_torture_writer_state,
2406				  cpumask_pr_args(cpu_online_mask));
2407	}
2408	if (cur_ops->reader_blocked)
2409		preempted = cur_ops->reader_blocked();
2410	rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp);
2411	WARN_ON_ONCE(rtorsp->readstate);
2412	// This next splat is expected behavior if leakpointer, especially
2413	// for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels.
2414	WARN_ON_ONCE(leakpointer && READ_ONCE(rtorsp->p->rtort_pipe_count) > 1);
2415
2416	/* If error or close call, record the sequence of reader protections. */
2417	if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) {
2418		i = 0;
2419		for (rtrsp1 = &rtorsp->rtseg[0]; rtrsp1 < rtorsp->rtrsp; rtrsp1++)
2420			err_segs[i++] = *rtrsp1;
2421		rt_read_nsegs = i;
2422		rt_read_preempted = preempted;
2423	}
2424}
2425
2426/*
2427 * Do one read-side critical section, returning false if there was
2428 * no data to read.  Can be invoked both from process context and
2429 * from a timer handler.
2430 */
2431static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
2432{
2433	int newstate;
2434	struct rcu_torture_one_read_state rtors;
2435
2436	WARN_ON_ONCE(!rcu_is_watching());
2437	init_rcu_torture_one_read_state(&rtors, trsp);
2438	newstate = rcutorture_extend_mask(rtors.readstate, trsp);
2439	WARN_ON_ONCE(newstate & RCUTORTURE_RDR_UPDOWN);
2440	rcutorture_one_extend(&rtors.readstate, newstate, trsp, rtors.rtrsp++);
2441	if (!rcu_torture_one_read_start(&rtors, trsp, myid))
2442		return false;
2443	rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp);
2444	rcu_torture_one_read_end(&rtors, trsp);
2445	return true;
2446}
2447
2448static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand);
2449
2450/*
2451 * RCU torture reader from timer handler.  Dereferences rcu_torture_current,
2452 * incrementing the corresponding element of the pipeline array.  The
2453 * counter in the element should never be greater than 1, otherwise, the
2454 * RCU implementation is broken.
2455 */
2456static void rcu_torture_timer(struct timer_list *unused)
2457{
2458	atomic_long_inc(&n_rcu_torture_timers);
2459	(void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1);
2460
2461	/* Test call_rcu() invocation from interrupt handler. */
2462	if (cur_ops->call) {
2463		struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT);
2464
2465		if (rhp)
2466			cur_ops->call(rhp, rcu_torture_timer_cb);
2467	}
2468}
2469
2470/*
2471 * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
2472 * incrementing the corresponding element of the pipeline array.  The
2473 * counter in the element should never be greater than 1, otherwise, the
2474 * RCU implementation is broken.
2475 */
2476static int
2477rcu_torture_reader(void *arg)
2478{
2479	unsigned long lastsleep = jiffies;
2480	long myid = (long)arg;
2481	int mynumonline = myid;
2482	DEFINE_TORTURE_RANDOM(rand);
2483	struct timer_list t;
2484
2485	VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
2486	set_user_nice(current, MAX_NICE);
2487	if (irqreader && cur_ops->irq_capable)
2488		timer_setup_on_stack(&t, rcu_torture_timer, 0);
2489	tick_dep_set_task(current, TICK_DEP_BIT_RCU);  // CPU bound, so need tick.
2490	do {
2491		if (irqreader && cur_ops->irq_capable) {
2492			if (!timer_pending(&t))
2493				mod_timer(&t, jiffies + 1);
2494		}
2495		if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop())
2496			schedule_timeout_interruptible(HZ);
2497		if (time_after(jiffies, lastsleep) && !torture_must_stop()) {
2498			torture_hrtimeout_us(500, 1000, &rand);
2499			lastsleep = jiffies + 10;
2500		}
2501		while (!torture_must_stop() &&
2502		       (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended()))
2503			schedule_timeout_interruptible(HZ / 5);
2504		stutter_wait("rcu_torture_reader");
2505	} while (!torture_must_stop());
2506	if (irqreader && cur_ops->irq_capable) {
2507		timer_delete_sync(&t);
2508		timer_destroy_on_stack(&t);
2509	}
2510	tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
2511	torture_kthread_stopping("rcu_torture_reader");
2512	return 0;
2513}
2514
2515struct rcu_torture_one_read_state_updown {
2516	struct hrtimer rtorsu_hrt;
2517	bool rtorsu_inuse;
2518	ktime_t rtorsu_kt;
2519	int rtorsu_cpu;
2520	unsigned long rtorsu_j;
2521	unsigned long rtorsu_ndowns;
2522	unsigned long rtorsu_nups;
2523	unsigned long rtorsu_nmigrates;
2524	struct torture_random_state rtorsu_trs;
2525	struct rcu_torture_one_read_state rtorsu_rtors;
2526};
2527
2528static struct rcu_torture_one_read_state_updown *updownreaders;
2529static DEFINE_TORTURE_RANDOM(rcu_torture_updown_rand);
2530static int rcu_torture_updown(void *arg);
2531
2532static enum hrtimer_restart rcu_torture_updown_hrt(struct hrtimer *hrtp)
2533{
2534	int cpu = raw_smp_processor_id();
2535	struct rcu_torture_one_read_state_updown *rtorsup;
2536
2537	rtorsup = container_of(hrtp, struct rcu_torture_one_read_state_updown, rtorsu_hrt);
2538	rcu_torture_one_read_end(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs);
2539	WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders);
2540	WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1);
2541	WRITE_ONCE(rtorsup->rtorsu_nmigrates,
2542		   rtorsup->rtorsu_nmigrates + (cpu != rtorsup->rtorsu_cpu));
2543	smp_store_release(&rtorsup->rtorsu_inuse, false);
2544	return HRTIMER_NORESTART;
2545}
2546
2547static int rcu_torture_updown_init(void)
2548{
2549	int i;
2550	struct torture_random_state *rand = &rcu_torture_updown_rand;
2551	int ret;
2552
2553	if (n_up_down < 0)
2554		return 0;
2555	if (!srcu_torture_have_up_down()) {
2556		VERBOSE_TOROUT_STRING("rcu_torture_updown_init: Disabling up/down reader tests due to lack of primitives");
2557		return 0;
2558	}
2559	updownreaders = kcalloc(n_up_down, sizeof(*updownreaders), GFP_KERNEL);
2560	if (!updownreaders) {
2561		VERBOSE_TOROUT_STRING("rcu_torture_updown_init: Out of memory, disabling up/down reader tests");
2562		return -ENOMEM;
2563	}
2564	for (i = 0; i < n_up_down; i++) {
2565		init_rcu_torture_one_read_state(&updownreaders[i].rtorsu_rtors, rand);
2566		hrtimer_setup(&updownreaders[i].rtorsu_hrt, rcu_torture_updown_hrt, CLOCK_MONOTONIC,
2567			      HRTIMER_MODE_REL | HRTIMER_MODE_HARD);
2568		torture_random_init(&updownreaders[i].rtorsu_trs);
2569		init_rcu_torture_one_read_state(&updownreaders[i].rtorsu_rtors,
2570						&updownreaders[i].rtorsu_trs);
2571	}
2572	ret = torture_create_kthread(rcu_torture_updown, rand, updown_task);
2573	if (ret) {
2574		kfree(updownreaders);
2575		updownreaders = NULL;
2576	}
2577	return ret;
2578}
2579
2580static void rcu_torture_updown_cleanup(void)
2581{
2582	struct rcu_torture_one_read_state_updown *rtorsup;
2583
2584	for (rtorsup = updownreaders; rtorsup < &updownreaders[n_up_down]; rtorsup++) {
2585		if (!smp_load_acquire(&rtorsup->rtorsu_inuse))
2586			continue;
2587		if (hrtimer_cancel(&rtorsup->rtorsu_hrt) || WARN_ON_ONCE(rtorsup->rtorsu_inuse)) {
2588			rcu_torture_one_read_end(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs);
2589			WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders);
2590			WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1);
2591			smp_store_release(&rtorsup->rtorsu_inuse, false);
2592		}
2593
2594	}
2595	kfree(updownreaders);
2596	updownreaders = NULL;
2597}
2598
2599// Do one reader for rcu_torture_updown().
2600static void rcu_torture_updown_one(struct rcu_torture_one_read_state_updown *rtorsup)
2601{
2602	int idx;
2603	int rawidx;
2604	ktime_t t;
2605
2606	init_rcu_torture_one_read_state(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs);
2607	rawidx = cur_ops->down_read();
2608	WRITE_ONCE(rtorsup->rtorsu_ndowns, rtorsup->rtorsu_ndowns + 1);
2609	idx = (rawidx << RCUTORTURE_RDR_SHIFT_1) & RCUTORTURE_RDR_MASK_1;
2610	rtorsup->rtorsu_rtors.readstate = idx | RCUTORTURE_RDR_UPDOWN;
2611	rtorsup->rtorsu_rtors.rtrsp++;
2612	rtorsup->rtorsu_cpu = raw_smp_processor_id();
2613	if (!rcu_torture_one_read_start(&rtorsup->rtorsu_rtors, &rtorsup->rtorsu_trs, -1)) {
2614		WARN_ONCE(rtorsup->rtorsu_nups >= rtorsup->rtorsu_ndowns, "%s: Up without matching down #%zu.\n", __func__, rtorsup - updownreaders);
2615		WRITE_ONCE(rtorsup->rtorsu_nups, rtorsup->rtorsu_nups + 1);
2616		schedule_timeout_idle(HZ);
2617		return;
2618	}
2619	smp_store_release(&rtorsup->rtorsu_inuse, true);
2620	t = torture_random(&rtorsup->rtorsu_trs) & 0xfffff; // One per million.
2621	if (t < 10 * 1000)
2622		t = 200 * 1000 * 1000;
2623	hrtimer_start(&rtorsup->rtorsu_hrt, t, HRTIMER_MODE_REL | HRTIMER_MODE_HARD);
2624	smp_mb(); // Sample jiffies after posting hrtimer.
2625	rtorsup->rtorsu_j = jiffies;  // Not used by hrtimer handler.
2626	rtorsup->rtorsu_kt = t;
2627}
2628
2629/*
2630 * RCU torture up/down reader kthread, starting RCU readers in kthread
2631 * context and ending them in hrtimer handlers.  Otherwise similar to
2632 * rcu_torture_reader().
2633 */
2634static int
2635rcu_torture_updown(void *arg)
2636{
2637	unsigned long j;
2638	struct rcu_torture_one_read_state_updown *rtorsup;
2639
2640	VERBOSE_TOROUT_STRING("rcu_torture_updown task started");
2641	do {
2642		for (rtorsup = updownreaders; rtorsup < &updownreaders[n_up_down]; rtorsup++) {
2643			if (torture_must_stop())
2644				break;
2645			j = smp_load_acquire(&jiffies); // Time before ->rtorsu_inuse.
2646			if (smp_load_acquire(&rtorsup->rtorsu_inuse)) {
2647				WARN_ONCE(time_after(j, rtorsup->rtorsu_j + 1 + HZ * 10),
2648					  "hrtimer queued at jiffies %lu for %lld ns took %lu jiffies\n", rtorsup->rtorsu_j, rtorsup->rtorsu_kt, j - rtorsup->rtorsu_j);
2649				continue;
2650			}
2651			rcu_torture_updown_one(rtorsup);
2652		}
2653		torture_hrtimeout_ms(1, 1000, &rcu_torture_updown_rand);
2654		stutter_wait("rcu_torture_updown");
2655	} while (!torture_must_stop());
2656	rcu_torture_updown_cleanup();
2657	torture_kthread_stopping("rcu_torture_updown");
2658	return 0;
2659}
2660
2661/*
2662 * Randomly Toggle CPUs' callback-offload state.  This uses hrtimers to
2663 * increase race probabilities and fuzzes the interval between toggling.
2664 */
2665static int rcu_nocb_toggle(void *arg)
2666{
2667	int cpu;
2668	int maxcpu = -1;
2669	int oldnice = task_nice(current);
2670	long r;
2671	DEFINE_TORTURE_RANDOM(rand);
2672	ktime_t toggle_delay;
2673	unsigned long toggle_fuzz;
2674	ktime_t toggle_interval = ms_to_ktime(nocbs_toggle);
2675
2676	VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started");
2677	while (!rcu_inkernel_boot_has_ended())
2678		schedule_timeout_interruptible(HZ / 10);
2679	for_each_possible_cpu(cpu)
2680		maxcpu = cpu;
2681	WARN_ON(maxcpu < 0);
2682	if (toggle_interval > ULONG_MAX)
2683		toggle_fuzz = ULONG_MAX >> 3;
2684	else
2685		toggle_fuzz = toggle_interval >> 3;
2686	if (toggle_fuzz <= 0)
2687		toggle_fuzz = NSEC_PER_USEC;
2688	do {
2689		r = torture_random(&rand);
2690		cpu = (r >> 1) % (maxcpu + 1);
2691		if (r & 0x1) {
2692			rcu_nocb_cpu_offload(cpu);
2693			atomic_long_inc(&n_nocb_offload);
2694		} else {
2695			rcu_nocb_cpu_deoffload(cpu);
2696			atomic_long_inc(&n_nocb_deoffload);
2697		}
2698		toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval;
2699		set_current_state(TASK_INTERRUPTIBLE);
2700		schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL);
2701		if (stutter_wait("rcu_nocb_toggle"))
2702			sched_set_normal(current, oldnice);
2703	} while (!torture_must_stop());
2704	torture_kthread_stopping("rcu_nocb_toggle");
2705	return 0;
2706}
2707
2708/*
2709 * Print torture statistics.  Caller must ensure that there is only
2710 * one call to this function at a given time!!!  This is normally
2711 * accomplished by relying on the module system to only have one copy
2712 * of the module loaded, and then by giving the rcu_torture_stats
2713 * kthread full control (or the init/cleanup functions when rcu_torture_stats
2714 * thread is not running).
2715 */
2716static void
2717rcu_torture_stats_print(void)
2718{
2719	int cpu;
2720	int i;
2721	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
2722	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
2723	long n_gpwraps = 0;
2724	unsigned long ndowns = 0;
2725	unsigned long nunexpired = 0;
2726	unsigned long nmigrates = 0;
2727	unsigned long nups = 0;
2728	struct rcu_torture *rtcp;
2729	static unsigned long rtcv_snap = ULONG_MAX;
2730	static bool splatted;
2731	struct task_struct *wtp;
2732
2733	for_each_possible_cpu(cpu) {
2734		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2735			pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
2736			batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
2737		}
2738		if (cur_ops->get_gpwrap_count)
2739			n_gpwraps += cur_ops->get_gpwrap_count(cpu);
2740	}
2741	if (updownreaders) {
2742		for (i = 0; i < n_up_down; i++) {
2743			ndowns += READ_ONCE(updownreaders[i].rtorsu_ndowns);
2744			nups += READ_ONCE(updownreaders[i].rtorsu_nups);
2745			nunexpired += READ_ONCE(updownreaders[i].rtorsu_inuse);
2746			nmigrates += READ_ONCE(updownreaders[i].rtorsu_nmigrates);
2747		}
2748	}
2749	for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) {
2750		if (pipesummary[i] != 0)
2751			break;
2752	} // The value of variable "i" is used later, so don't clobber it!
2753
2754	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2755	rtcp = rcu_access_pointer(rcu_torture_current);
2756	pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
2757		rtcp,
2758		rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
2759		rcu_torture_current_version,
2760		list_empty(&rcu_torture_freelist),
2761		atomic_read(&n_rcu_torture_alloc),
2762		atomic_read(&n_rcu_torture_alloc_fail),
2763		atomic_read(&n_rcu_torture_free));
2764	pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld ",
2765		atomic_read(&n_rcu_torture_mberror),
2766		atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries),
2767		n_rcu_torture_barrier_error,
2768		n_rcu_torture_boost_ktrerror);
2769	pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
2770		n_rcu_torture_boost_failure,
2771		n_rcu_torture_boosts,
2772		atomic_long_read(&n_rcu_torture_timers));
2773	if (updownreaders)
2774		pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired,  nmigrates);
2775	torture_onoff_stats();
2776	pr_cont("barrier: %ld/%ld:%ld ",
2777		data_race(n_barrier_successes),
2778		data_race(n_barrier_attempts),
2779		data_race(n_rcu_torture_barrier_error));
2780	pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic.
2781	pr_cont("nocb-toggles: %ld:%ld ",
2782		atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload));
2783	pr_cont("gpwraps: %ld\n", n_gpwraps);
2784
2785	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2786	if (atomic_read(&n_rcu_torture_mberror) ||
2787	    atomic_read(&n_rcu_torture_mbchk_fail) ||
2788	    n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror ||
2789	    n_rcu_torture_boost_failure || i > 1) {
2790		pr_cont("%s", "!!! ");
2791		atomic_inc(&n_rcu_torture_error);
2792		WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror));
2793		WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail));
2794		WARN_ON_ONCE(n_rcu_torture_barrier_error);  // rcu_barrier()
2795		WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread
2796		WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?)
2797		WARN_ON_ONCE(i > 1); // Too-short grace period
2798	}
2799	pr_cont("Reader Pipe: ");
2800	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2801		pr_cont(" %ld", pipesummary[i]);
2802	pr_cont("\n");
2803
2804	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2805	pr_cont("Reader Batch: ");
2806	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
2807		pr_cont(" %ld", batchsummary[i]);
2808	pr_cont("\n");
2809
2810	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
2811	pr_cont("Free-Block Circulation: ");
2812	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
2813		pr_cont(" %d", atomic_read(&rcu_torture_wcount[i]));
2814	}
2815	pr_cont("\n");
2816
2817	if (cur_ops->stats)
2818		cur_ops->stats();
2819	if (rtcv_snap == rcu_torture_current_version &&
2820	    rcu_access_pointer(rcu_torture_current) &&
2821	    !rcu_stall_is_suppressed() &&
2822	    rcu_inkernel_boot_has_ended()) {
2823		int __maybe_unused flags = 0;
2824		unsigned long __maybe_unused gp_seq = 0;
2825
2826		if (cur_ops->get_gp_data)
2827			cur_ops->get_gp_data(&flags, &gp_seq);
2828		wtp = READ_ONCE(writer_task);
2829		pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n",
2830			 rcu_torture_writer_state_getname(),
2831			 rcu_torture_writer_state, gp_seq, flags,
2832			 wtp == NULL ? ~0U : wtp->__state,
2833			 wtp == NULL ? -1 : (int)task_cpu(wtp));
2834		if (!splatted && wtp) {
2835			sched_show_task(wtp);
2836			splatted = true;
2837		}
2838		if (cur_ops->gp_kthread_dbg)
2839			cur_ops->gp_kthread_dbg();
2840		rcu_ftrace_dump(DUMP_ALL);
2841	}
2842	rtcv_snap = rcu_torture_current_version;
2843}
2844
2845/*
2846 * Periodically prints torture statistics, if periodic statistics printing
2847 * was specified via the stat_interval module parameter.
2848 */
2849static int
2850rcu_torture_stats(void *arg)
2851{
2852	VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
2853	do {
2854		schedule_timeout_interruptible(stat_interval * HZ);
2855		rcu_torture_stats_print();
2856		torture_shutdown_absorb("rcu_torture_stats");
2857	} while (!torture_must_stop());
2858	torture_kthread_stopping("rcu_torture_stats");
2859	return 0;
2860}
2861
2862/* Test mem_dump_obj() and friends.  */
2863static void rcu_torture_mem_dump_obj(void)
2864{
2865	struct rcu_head *rhp;
2866	struct kmem_cache *kcp;
2867	static int z;
2868
2869	kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL);
2870	if (WARN_ON_ONCE(!kcp))
2871		return;
2872	rhp = kmem_cache_alloc(kcp, GFP_KERNEL);
2873	if (WARN_ON_ONCE(!rhp)) {
2874		kmem_cache_destroy(kcp);
2875		return;
2876	}
2877	pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z);
2878	pr_alert("mem_dump_obj(ZERO_SIZE_PTR):");
2879	mem_dump_obj(ZERO_SIZE_PTR);
2880	pr_alert("mem_dump_obj(NULL):");
2881	mem_dump_obj(NULL);
2882	pr_alert("mem_dump_obj(%px):", &rhp);
2883	mem_dump_obj(&rhp);
2884	pr_alert("mem_dump_obj(%px):", rhp);
2885	mem_dump_obj(rhp);
2886	pr_alert("mem_dump_obj(%px):", &rhp->func);
2887	mem_dump_obj(&rhp->func);
2888	pr_alert("mem_dump_obj(%px):", &z);
2889	mem_dump_obj(&z);
2890	kmem_cache_free(kcp, rhp);
2891	kmem_cache_destroy(kcp);
2892	rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
2893	if (WARN_ON_ONCE(!rhp))
2894		return;
2895	pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp);
2896	pr_alert("mem_dump_obj(kmalloc %px):", rhp);
2897	mem_dump_obj(rhp);
2898	pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func);
2899	mem_dump_obj(&rhp->func);
2900	kfree(rhp);
2901	rhp = vmalloc(4096);
2902	if (WARN_ON_ONCE(!rhp))
2903		return;
2904	pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp);
2905	pr_alert("mem_dump_obj(vmalloc %px):", rhp);
2906	mem_dump_obj(rhp);
2907	pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func);
2908	mem_dump_obj(&rhp->func);
2909	vfree(rhp);
2910}
2911
2912static void
2913rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag)
2914{
2915	pr_alert("%s" TORTURE_FLAG
2916		 "--- %s: nreaders=%d nfakewriters=%d "
2917		 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
2918		 "shuffle_interval=%d stutter=%d irqreader=%d "
2919		 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
2920		 "test_boost=%d/%d test_boost_interval=%d "
2921		 "test_boost_duration=%d test_boost_holdoff=%d shutdown_secs=%d "
2922		 "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "
2923		 "stall_cpu_block=%d stall_cpu_repeat=%d "
2924		 "n_barrier_cbs=%d "
2925		 "onoff_interval=%d onoff_holdoff=%d "
2926		 "read_exit_delay=%d read_exit_burst=%d "
2927		 "reader_flavor=%x "
2928		 "nocbs_nthreads=%d nocbs_toggle=%d "
2929		 "test_nmis=%d "
2930		 "preempt_duration=%d preempt_interval=%d n_up_down=%d\n",
2931		 torture_type, tag, nrealreaders, nrealfakewriters,
2932		 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
2933		 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
2934		 test_boost, cur_ops->can_boost,
2935		 test_boost_interval, test_boost_duration, test_boost_holdoff, shutdown_secs,
2936		 stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,
2937		 stall_cpu_block, stall_cpu_repeat,
2938		 n_barrier_cbs,
2939		 onoff_interval, onoff_holdoff,
2940		 read_exit_delay, read_exit_burst,
2941		 reader_flavor,
2942		 nocbs_nthreads, nocbs_toggle,
2943		 test_nmis,
2944		 preempt_duration, preempt_interval, n_up_down);
2945}
2946
2947static int rcutorture_booster_cleanup(unsigned int cpu)
2948{
2949	struct task_struct *t;
2950
2951	if (boost_tasks[cpu] == NULL)
2952		return 0;
2953	mutex_lock(&boost_mutex);
2954	t = boost_tasks[cpu];
2955	boost_tasks[cpu] = NULL;
2956	rcu_torture_enable_rt_throttle();
2957	mutex_unlock(&boost_mutex);
2958
2959	/* This must be outside of the mutex, otherwise deadlock! */
2960	torture_stop_kthread(rcu_torture_boost, t);
2961	return 0;
2962}
2963
2964static int rcutorture_booster_init(unsigned int cpu)
2965{
2966	int retval;
2967
2968	if (boost_tasks[cpu] != NULL)
2969		return 0;  /* Already created, nothing more to do. */
2970
2971	// Testing RCU priority boosting requires rcutorture do
2972	// some serious abuse.  Counter this by running ksoftirqd
2973	// at higher priority.
2974	if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) {
2975		struct sched_param sp;
2976		struct task_struct *t;
2977
2978		t = per_cpu(ksoftirqd, cpu);
2979		WARN_ON_ONCE(!t);
2980		sp.sched_priority = 2;
2981		sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
2982#ifdef CONFIG_IRQ_FORCED_THREADING
2983		if (force_irqthreads()) {
2984			t = per_cpu(ktimerd, cpu);
2985			WARN_ON_ONCE(!t);
2986			sp.sched_priority = 2;
2987			sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
2988		}
2989#endif
2990	}
2991
2992	/* Don't allow time recalculation while creating a new task. */
2993	mutex_lock(&boost_mutex);
2994	rcu_torture_disable_rt_throttle();
2995	VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
2996	boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL,
2997					      cpu, "rcu_torture_boost_%u");
2998	if (IS_ERR(boost_tasks[cpu])) {
2999		retval = PTR_ERR(boost_tasks[cpu]);
3000		VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
3001		n_rcu_torture_boost_ktrerror++;
3002		boost_tasks[cpu] = NULL;
3003		mutex_unlock(&boost_mutex);
3004		return retval;
3005	}
3006	mutex_unlock(&boost_mutex);
3007	return 0;
3008}
3009
3010static int rcu_torture_stall_nf(struct notifier_block *nb, unsigned long v, void *ptr)
3011{
3012	pr_info("%s: v=%lu, duration=%lu.\n", __func__, v, (unsigned long)ptr);
3013	return NOTIFY_OK;
3014}
3015
3016static struct notifier_block rcu_torture_stall_block = {
3017	.notifier_call = rcu_torture_stall_nf,
3018};
3019
3020/*
3021 * CPU-stall kthread.  It waits as specified by stall_cpu_holdoff, then
3022 * induces a CPU stall for the time specified by stall_cpu.  If a new
3023 * stall test is added, stallsdone in rcu_torture_writer() must be adjusted.
3024 */
3025static void rcu_torture_stall_one(int rep, int irqsoff)
3026{
3027	int idx;
3028	unsigned long stop_at;
3029
3030	if (stall_cpu_holdoff > 0) {
3031		VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
3032		schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
3033		VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
3034	}
3035	if (!kthread_should_stop() && stall_gp_kthread > 0) {
3036		VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall");
3037		rcu_gp_set_torture_wait(stall_gp_kthread * HZ);
3038		for (idx = 0; idx < stall_gp_kthread + 2; idx++) {
3039			if (kthread_should_stop())
3040				break;
3041			schedule_timeout_uninterruptible(HZ);
3042		}
3043	}
3044	if (!kthread_should_stop() && stall_cpu > 0) {
3045		VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall");
3046		stop_at = ktime_get_seconds() + stall_cpu;
3047		/* RCU CPU stall is expected behavior in following code. */
3048		idx = cur_ops->readlock();
3049		if (irqsoff)
3050			local_irq_disable();
3051		else if (!stall_cpu_block)
3052			preempt_disable();
3053		pr_alert("%s start stall episode %d on CPU %d.\n",
3054			  __func__, rep + 1, raw_smp_processor_id());
3055		while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), stop_at) &&
3056		       !kthread_should_stop())
3057			if (stall_cpu_block) {
3058#ifdef CONFIG_PREEMPTION
3059				preempt_schedule();
3060#else
3061				schedule_timeout_uninterruptible(HZ);
3062#endif
3063			} else if (stall_no_softlockup) {
3064				touch_softlockup_watchdog();
3065			}
3066		if (irqsoff)
3067			local_irq_enable();
3068		else if (!stall_cpu_block)
3069			preempt_enable();
3070		cur_ops->readunlock(idx);
3071	}
3072}
3073
3074/*
3075 * CPU-stall kthread.  Invokes rcu_torture_stall_one() once, and then as many
3076 * additional times as specified by the stall_cpu_repeat module parameter.
3077 * Note that stall_cpu_irqsoff is ignored on the second and subsequent
3078 * stall.
3079 */
3080static int rcu_torture_stall(void *args)
3081{
3082	int i;
3083	int repeat = stall_cpu_repeat;
3084	int ret;
3085
3086	VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
3087	if (repeat < 0) {
3088		repeat = 0;
3089		WARN_ON_ONCE(IS_BUILTIN(CONFIG_RCU_TORTURE_TEST));
3090	}
3091	if (rcu_cpu_stall_notifiers) {
3092		ret = rcu_stall_chain_notifier_register(&rcu_torture_stall_block);
3093		if (ret)
3094			pr_info("%s: rcu_stall_chain_notifier_register() returned %d, %sexpected.\n",
3095				__func__, ret, !IS_ENABLED(CONFIG_RCU_STALL_COMMON) ? "un" : "");
3096	}
3097	for (i = 0; i <= repeat; i++) {
3098		if (kthread_should_stop())
3099			break;
3100		rcu_torture_stall_one(i, i == 0 ? stall_cpu_irqsoff : 0);
3101	}
3102	pr_alert("%s end.\n", __func__);
3103	if (rcu_cpu_stall_notifiers && !ret) {
3104		ret = rcu_stall_chain_notifier_unregister(&rcu_torture_stall_block);
3105		if (ret)
3106			pr_info("%s: rcu_stall_chain_notifier_unregister() returned %d.\n", __func__, ret);
3107	}
3108	torture_shutdown_absorb("rcu_torture_stall");
3109	while (!kthread_should_stop())
3110		schedule_timeout_interruptible(10 * HZ);
3111	return 0;
3112}
3113
3114/* Spawn CPU-stall kthread, if stall_cpu specified. */
3115static int __init rcu_torture_stall_init(void)
3116{
3117	if (stall_cpu <= 0 && stall_gp_kthread <= 0)
3118		return 0;
3119	return torture_create_kthread(rcu_torture_stall, NULL, stall_task);
3120}
3121
3122/* State structure for forward-progress self-propagating RCU callback. */
3123struct fwd_cb_state {
3124	struct rcu_head rh;
3125	int stop;
3126};
3127
3128/*
3129 * Forward-progress self-propagating RCU callback function.  Because
3130 * callbacks run from softirq, this function is an implicit RCU read-side
3131 * critical section.
3132 */
3133static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp)
3134{
3135	struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh);
3136
3137	if (READ_ONCE(fcsp->stop)) {
3138		WRITE_ONCE(fcsp->stop, 2);
3139		return;
3140	}
3141	cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb);
3142}
3143
3144/* State for continuous-flood RCU callbacks. */
3145struct rcu_fwd_cb {
3146	struct rcu_head rh;
3147	struct rcu_fwd_cb *rfc_next;
3148	struct rcu_fwd *rfc_rfp;
3149	int rfc_gps;
3150};
3151
3152#define MAX_FWD_CB_JIFFIES	(8 * HZ) /* Maximum CB test duration. */
3153#define MIN_FWD_CB_LAUNDERS	3	/* This many CB invocations to count. */
3154#define MIN_FWD_CBS_LAUNDERED	100	/* Number of counted CBs. */
3155#define FWD_CBS_HIST_DIV	10	/* Histogram buckets/second. */
3156#define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV))
3157
3158struct rcu_launder_hist {
3159	long n_launders;
3160	unsigned long launder_gp_seq;
3161};
3162
3163struct rcu_fwd {
3164	spinlock_t rcu_fwd_lock;
3165	struct rcu_fwd_cb *rcu_fwd_cb_head;
3166	struct rcu_fwd_cb **rcu_fwd_cb_tail;
3167	long n_launders_cb;
3168	unsigned long rcu_fwd_startat;
3169	struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST];
3170	unsigned long rcu_launder_gp_seq_start;
3171	int rcu_fwd_id;
3172};
3173
3174static DEFINE_MUTEX(rcu_fwd_mutex);
3175static struct rcu_fwd *rcu_fwds;
3176static unsigned long rcu_fwd_seq;
3177static atomic_long_t rcu_fwd_max_cbs;
3178static bool rcu_fwd_emergency_stop;
3179
3180static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp)
3181{
3182	unsigned long gps;
3183	unsigned long gps_old;
3184	int i;
3185	int j;
3186
3187	for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--)
3188		if (rfp->n_launders_hist[i].n_launders > 0)
3189			break;
3190	pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):",
3191		 __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat);
3192	gps_old = rfp->rcu_launder_gp_seq_start;
3193	for (j = 0; j <= i; j++) {
3194		gps = rfp->n_launders_hist[j].launder_gp_seq;
3195		pr_cont(" %ds/%d: %ld:%ld",
3196			j + 1, FWD_CBS_HIST_DIV,
3197			rfp->n_launders_hist[j].n_launders,
3198			rcutorture_seq_diff(gps, gps_old));
3199		gps_old = gps;
3200	}
3201	pr_cont("\n");
3202}
3203
3204/* Callback function for continuous-flood RCU callbacks. */
3205static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp)
3206{
3207	unsigned long flags;
3208	int i;
3209	struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh);
3210	struct rcu_fwd_cb **rfcpp;
3211	struct rcu_fwd *rfp = rfcp->rfc_rfp;
3212
3213	rfcp->rfc_next = NULL;
3214	rfcp->rfc_gps++;
3215	spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
3216	rfcpp = rfp->rcu_fwd_cb_tail;
3217	rfp->rcu_fwd_cb_tail = &rfcp->rfc_next;
3218	smp_store_release(rfcpp, rfcp);
3219	WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1);
3220	i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV));
3221	if (i >= ARRAY_SIZE(rfp->n_launders_hist))
3222		i = ARRAY_SIZE(rfp->n_launders_hist) - 1;
3223	rfp->n_launders_hist[i].n_launders++;
3224	rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq();
3225	spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
3226}
3227
3228// Give the scheduler a chance, even on nohz_full CPUs.
3229static void rcu_torture_fwd_prog_cond_resched(unsigned long iter)
3230{
3231	if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) {
3232		// Real call_rcu() floods hit userspace, so emulate that.
3233		if (need_resched() || (iter & 0xfff))
3234			schedule();
3235		return;
3236	}
3237	// No userspace emulation: CB invocation throttles call_rcu()
3238	cond_resched();
3239}
3240
3241/*
3242 * Free all callbacks on the rcu_fwd_cb_head list, either because the
3243 * test is over or because we hit an OOM event.
3244 */
3245static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp)
3246{
3247	unsigned long flags;
3248	unsigned long freed = 0;
3249	struct rcu_fwd_cb *rfcp;
3250
3251	for (;;) {
3252		spin_lock_irqsave(&rfp->rcu_fwd_lock, flags);
3253		rfcp = rfp->rcu_fwd_cb_head;
3254		if (!rfcp) {
3255			spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
3256			break;
3257		}
3258		rfp->rcu_fwd_cb_head = rfcp->rfc_next;
3259		if (!rfp->rcu_fwd_cb_head)
3260			rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head;
3261		spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags);
3262		kfree(rfcp);
3263		freed++;
3264		rcu_torture_fwd_prog_cond_resched(freed);
3265		if (tick_nohz_full_enabled()) {
3266			local_irq_save(flags);
3267			rcu_momentary_eqs();
3268			local_irq_restore(flags);
3269		}
3270	}
3271	return freed;
3272}
3273
3274/* Carry out need_resched()/cond_resched() forward-progress testing. */
3275static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp,
3276				    int *tested, int *tested_tries)
3277{
3278	unsigned long cver;
3279	unsigned long dur;
3280	struct fwd_cb_state fcs;
3281	unsigned long gps;
3282	int idx;
3283	int sd;
3284	int sd4;
3285	bool selfpropcb = false;
3286	unsigned long stopat;
3287	static DEFINE_TORTURE_RANDOM(trs);
3288
3289	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3290	if (!cur_ops->sync)
3291		return; // Cannot do need_resched() forward progress testing without ->sync.
3292	if (cur_ops->call && cur_ops->cb_barrier) {
3293		init_rcu_head_on_stack(&fcs.rh);
3294		selfpropcb = true;
3295	}
3296
3297	/* Tight loop containing cond_resched(). */
3298	atomic_inc(&rcu_fwd_cb_nodelay);
3299	cur_ops->sync(); /* Later readers see above write. */
3300	if  (selfpropcb) {
3301		WRITE_ONCE(fcs.stop, 0);
3302		cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb);
3303	}
3304	cver = READ_ONCE(rcu_torture_current_version);
3305	gps = cur_ops->get_gp_seq();
3306	sd = cur_ops->stall_dur() + 1;
3307	sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div;
3308	dur = sd4 + torture_random(&trs) % (sd - sd4);
3309	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
3310	stopat = rfp->rcu_fwd_startat + dur;
3311	while (time_before(jiffies, stopat) &&
3312	       !shutdown_time_arrived() &&
3313	       !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
3314		idx = cur_ops->readlock();
3315		udelay(10);
3316		cur_ops->readunlock(idx);
3317		if (!fwd_progress_need_resched || need_resched())
3318			cond_resched();
3319	}
3320	(*tested_tries)++;
3321	if (!time_before(jiffies, stopat) &&
3322	    !shutdown_time_arrived() &&
3323	    !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
3324		(*tested)++;
3325		cver = READ_ONCE(rcu_torture_current_version) - cver;
3326		gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
3327		WARN_ON(!cver && gps < 2);
3328		pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__,
3329			 rfp->rcu_fwd_id, dur, cver, gps);
3330	}
3331	if (selfpropcb) {
3332		WRITE_ONCE(fcs.stop, 1);
3333		cur_ops->sync(); /* Wait for running CB to complete. */
3334		pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3335		cur_ops->cb_barrier(); /* Wait for queued callbacks. */
3336	}
3337
3338	if (selfpropcb) {
3339		WARN_ON(READ_ONCE(fcs.stop) != 2);
3340		destroy_rcu_head_on_stack(&fcs.rh);
3341	}
3342	schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */
3343	atomic_dec(&rcu_fwd_cb_nodelay);
3344}
3345
3346/* Carry out call_rcu() forward-progress testing. */
3347static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp)
3348{
3349	unsigned long cver;
3350	unsigned long flags;
3351	unsigned long gps;
3352	int i;
3353	long n_launders;
3354	long n_launders_cb_snap;
3355	long n_launders_sa;
3356	long n_max_cbs;
3357	long n_max_gps;
3358	struct rcu_fwd_cb *rfcp;
3359	struct rcu_fwd_cb *rfcpn;
3360	unsigned long stopat;
3361	unsigned long stoppedat;
3362
3363	pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3364	if (READ_ONCE(rcu_fwd_emergency_stop))
3365		return; /* Get out of the way quickly, no GP wait! */
3366	if (!cur_ops->call)
3367		return; /* Can't do call_rcu() fwd prog without ->call. */
3368
3369	/* Loop continuously posting RCU callbacks. */
3370	atomic_inc(&rcu_fwd_cb_nodelay);
3371	cur_ops->sync(); /* Later readers see above write. */
3372	WRITE_ONCE(rfp->rcu_fwd_startat, jiffies);
3373	stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES;
3374	n_launders = 0;
3375	rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread
3376	n_launders_sa = 0;
3377	n_max_cbs = 0;
3378	n_max_gps = 0;
3379	for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++)
3380		rfp->n_launders_hist[i].n_launders = 0;
3381	cver = READ_ONCE(rcu_torture_current_version);
3382	gps = cur_ops->get_gp_seq();
3383	rfp->rcu_launder_gp_seq_start = gps;
3384	tick_dep_set_task(current, TICK_DEP_BIT_RCU);  // CPU bound, so need tick.
3385	while (time_before(jiffies, stopat) &&
3386	       !shutdown_time_arrived() &&
3387	       !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) {
3388		rfcp = READ_ONCE(rfp->rcu_fwd_cb_head);
3389		rfcpn = NULL;
3390		if (rfcp)
3391			rfcpn = READ_ONCE(rfcp->rfc_next);
3392		if (rfcpn) {
3393			if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS &&
3394			    ++n_max_gps >= MIN_FWD_CBS_LAUNDERED)
3395				break;
3396			rfp->rcu_fwd_cb_head = rfcpn;
3397			n_launders++;
3398			n_launders_sa++;
3399		} else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) {
3400			rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL);
3401			if (WARN_ON_ONCE(!rfcp)) {
3402				schedule_timeout_interruptible(1);
3403				continue;
3404			}
3405			n_max_cbs++;
3406			n_launders_sa = 0;
3407			rfcp->rfc_gps = 0;
3408			rfcp->rfc_rfp = rfp;
3409		} else {
3410			rfcp = NULL;
3411		}
3412		if (rfcp)
3413			cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr);
3414		rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs);
3415		if (tick_nohz_full_enabled()) {
3416			local_irq_save(flags);
3417			rcu_momentary_eqs();
3418			local_irq_restore(flags);
3419		}
3420	}
3421	stoppedat = jiffies;
3422	n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb);
3423	cver = READ_ONCE(rcu_torture_current_version) - cver;
3424	gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps);
3425	pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id);
3426	cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */
3427	(void)rcu_torture_fwd_prog_cbfree(rfp);
3428
3429	if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) &&
3430	    !shutdown_time_arrived()) {
3431		if (WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED) && cur_ops->gp_kthread_dbg)
3432			cur_ops->gp_kthread_dbg();
3433		pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld #online %u\n",
3434			 __func__,
3435			 stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat,
3436			 n_launders + n_max_cbs - n_launders_cb_snap,
3437			 n_launders, n_launders_sa,
3438			 n_max_gps, n_max_cbs, cver, gps, num_online_cpus());
3439		atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs);
3440		mutex_lock(&rcu_fwd_mutex); // Serialize histograms.
3441		rcu_torture_fwd_cb_hist(rfp);
3442		mutex_unlock(&rcu_fwd_mutex);
3443	}
3444	schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */
3445	tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
3446	atomic_dec(&rcu_fwd_cb_nodelay);
3447}
3448
3449
3450/*
3451 * OOM notifier, but this only prints diagnostic information for the
3452 * current forward-progress test.
3453 */
3454static int rcutorture_oom_notify(struct notifier_block *self,
3455				 unsigned long notused, void *nfreed)
3456{
3457	int i;
3458	long ncbs;
3459	struct rcu_fwd *rfp;
3460
3461	mutex_lock(&rcu_fwd_mutex);
3462	rfp = rcu_fwds;
3463	if (!rfp) {
3464		mutex_unlock(&rcu_fwd_mutex);
3465		return NOTIFY_OK;
3466	}
3467	WARN(1, "%s invoked upon OOM during forward-progress testing.\n",
3468	     __func__);
3469	for (i = 0; i < fwd_progress; i++) {
3470		rcu_torture_fwd_cb_hist(&rfp[i]);
3471		rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2);
3472	}
3473	WRITE_ONCE(rcu_fwd_emergency_stop, true);
3474	smp_mb(); /* Emergency stop before free and wait to avoid hangs. */
3475	ncbs = 0;
3476	for (i = 0; i < fwd_progress; i++)
3477		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
3478	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
3479	cur_ops->cb_barrier();
3480	ncbs = 0;
3481	for (i = 0; i < fwd_progress; i++)
3482		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
3483	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
3484	cur_ops->cb_barrier();
3485	ncbs = 0;
3486	for (i = 0; i < fwd_progress; i++)
3487		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
3488	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
3489	smp_mb(); /* Frees before return to avoid redoing OOM. */
3490	(*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */
3491	pr_info("%s returning after OOM processing.\n", __func__);
3492	mutex_unlock(&rcu_fwd_mutex);
3493	return NOTIFY_OK;
3494}
3495
3496static struct notifier_block rcutorture_oom_nb = {
3497	.notifier_call = rcutorture_oom_notify
3498};
3499
3500/* Carry out grace-period forward-progress testing. */
3501static int rcu_torture_fwd_prog(void *args)
3502{
3503	bool firsttime = true;
3504	long max_cbs;
3505	int oldnice = task_nice(current);
3506	unsigned long oldseq = READ_ONCE(rcu_fwd_seq);
3507	struct rcu_fwd *rfp = args;
3508	int tested = 0;
3509	int tested_tries = 0;
3510
3511	VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started");
3512	while (!rcu_inkernel_boot_has_ended())
3513		schedule_timeout_interruptible(HZ / 10);
3514	rcu_bind_current_to_nocb();
3515	if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST))
3516		set_user_nice(current, MAX_NICE);
3517	do {
3518		if (!rfp->rcu_fwd_id) {
3519			schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
3520			WRITE_ONCE(rcu_fwd_emergency_stop, false);
3521			if (!firsttime) {
3522				max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0);
3523				pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs);
3524			}
3525			firsttime = false;
3526			WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1);
3527		} else {
3528			while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop())
3529				schedule_timeout_interruptible(HZ / 20);
3530			oldseq = READ_ONCE(rcu_fwd_seq);
3531		}
3532		pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id);
3533		if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)
3534			rcu_torture_fwd_prog_cr(rfp);
3535		if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) &&
3536		    (!IS_ENABLED(CONFIG_TINY_RCU) ||
3537		     (rcu_inkernel_boot_has_ended() &&
3538		      torture_num_online_cpus() > rfp->rcu_fwd_id)))
3539			rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
3540
3541		/* Avoid slow periods, better to test when busy. */
3542		if (stutter_wait("rcu_torture_fwd_prog"))
3543			sched_set_normal(current, oldnice);
3544	} while (!torture_must_stop());
3545	/* Short runs might not contain a valid forward-progress attempt. */
3546	if (!rfp->rcu_fwd_id) {
3547		WARN_ON(!tested && tested_tries >= 5);
3548		pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries);
3549	}
3550	torture_kthread_stopping("rcu_torture_fwd_prog");
3551	return 0;
3552}
3553
3554/* If forward-progress checking is requested and feasible, spawn the thread. */
3555static int __init rcu_torture_fwd_prog_init(void)
3556{
3557	int i;
3558	int ret = 0;
3559	struct rcu_fwd *rfp;
3560
3561	if (!fwd_progress)
3562		return 0; /* Not requested, so don't do it. */
3563	if (fwd_progress >= nr_cpu_ids) {
3564		VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n");
3565		fwd_progress = nr_cpu_ids;
3566	} else if (fwd_progress < 0) {
3567		fwd_progress = nr_cpu_ids;
3568	}
3569	if ((!cur_ops->sync && !cur_ops->call) ||
3570	    (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) ||
3571	    cur_ops == &rcu_busted_ops) {
3572		VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test");
3573		fwd_progress = 0;
3574		return 0;
3575	}
3576	if (stall_cpu > 0 || (preempt_duration > 0 && IS_ENABLED(CONFIG_RCU_NOCB_CPU))) {
3577		VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall and/or preemption testing");
3578		fwd_progress = 0;
3579		if (IS_MODULE(CONFIG_RCU_TORTURE_TEST))
3580			return -EINVAL; /* In module, can fail back to user. */
3581		WARN_ON(1); /* Make sure rcutorture scripting notices conflict. */
3582		return 0;
3583	}
3584	if (fwd_progress_holdoff <= 0)
3585		fwd_progress_holdoff = 1;
3586	if (fwd_progress_div <= 0)
3587		fwd_progress_div = 4;
3588	rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL);
3589	fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL);
3590	if (!rfp || !fwd_prog_tasks) {
3591		kfree(rfp);
3592		kfree(fwd_prog_tasks);
3593		fwd_prog_tasks = NULL;
3594		fwd_progress = 0;
3595		return -ENOMEM;
3596	}
3597	for (i = 0; i < fwd_progress; i++) {
3598		spin_lock_init(&rfp[i].rcu_fwd_lock);
3599		rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head;
3600		rfp[i].rcu_fwd_id = i;
3601	}
3602	mutex_lock(&rcu_fwd_mutex);
3603	rcu_fwds = rfp;
3604	mutex_unlock(&rcu_fwd_mutex);
3605	register_oom_notifier(&rcutorture_oom_nb);
3606	for (i = 0; i < fwd_progress; i++) {
3607		ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]);
3608		if (ret) {
3609			fwd_progress = i;
3610			return ret;
3611		}
3612	}
3613	return 0;
3614}
3615
3616static void rcu_torture_fwd_prog_cleanup(void)
3617{
3618	int i;
3619	struct rcu_fwd *rfp;
3620
3621	if (!rcu_fwds || !fwd_prog_tasks)
3622		return;
3623	for (i = 0; i < fwd_progress; i++)
3624		torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]);
3625	unregister_oom_notifier(&rcutorture_oom_nb);
3626	mutex_lock(&rcu_fwd_mutex);
3627	rfp = rcu_fwds;
3628	rcu_fwds = NULL;
3629	mutex_unlock(&rcu_fwd_mutex);
3630	kfree(rfp);
3631	kfree(fwd_prog_tasks);
3632	fwd_prog_tasks = NULL;
3633}
3634
3635/* Callback function for RCU barrier testing. */
3636static void rcu_torture_barrier_cbf(struct rcu_head *rcu)
3637{
3638	atomic_inc(&barrier_cbs_invoked);
3639}
3640
3641/* IPI handler to get callback posted on desired CPU, if online. */
3642static int rcu_torture_barrier1cb(void *rcu_void)
3643{
3644	struct rcu_head *rhp = rcu_void;
3645
3646	cur_ops->call(rhp, rcu_torture_barrier_cbf);
3647	return 0;
3648}
3649
3650/* kthread function to register callbacks used to test RCU barriers. */
3651static int rcu_torture_barrier_cbs(void *arg)
3652{
3653	long myid = (long)arg;
3654	bool lastphase = false;
3655	bool newphase;
3656	struct rcu_head rcu;
3657
3658	init_rcu_head_on_stack(&rcu);
3659	VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
3660	set_user_nice(current, MAX_NICE);
3661	do {
3662		wait_event(barrier_cbs_wq[myid],
3663			   (newphase =
3664			    smp_load_acquire(&barrier_phase)) != lastphase ||
3665			   torture_must_stop());
3666		lastphase = newphase;
3667		if (torture_must_stop())
3668			break;
3669		/*
3670		 * The above smp_load_acquire() ensures barrier_phase load
3671		 * is ordered before the following ->call().
3672		 */
3673		if (smp_call_on_cpu(myid, rcu_torture_barrier1cb, &rcu, 1))
3674			cur_ops->call(&rcu, rcu_torture_barrier_cbf);
3675
3676		if (atomic_dec_and_test(&barrier_cbs_count))
3677			wake_up(&barrier_wq);
3678	} while (!torture_must_stop());
3679	if (cur_ops->cb_barrier != NULL)
3680		cur_ops->cb_barrier();
3681	destroy_rcu_head_on_stack(&rcu);
3682	torture_kthread_stopping("rcu_torture_barrier_cbs");
3683	return 0;
3684}
3685
3686/* kthread function to drive and coordinate RCU barrier testing. */
3687static int rcu_torture_barrier(void *arg)
3688{
3689	int i;
3690
3691	VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
3692	do {
3693		atomic_set(&barrier_cbs_invoked, 0);
3694		atomic_set(&barrier_cbs_count, n_barrier_cbs);
3695		/* Ensure barrier_phase ordered after prior assignments. */
3696		smp_store_release(&barrier_phase, !barrier_phase);
3697		for (i = 0; i < n_barrier_cbs; i++)
3698			wake_up(&barrier_cbs_wq[i]);
3699		wait_event(barrier_wq,
3700			   atomic_read(&barrier_cbs_count) == 0 ||
3701			   torture_must_stop());
3702		if (torture_must_stop())
3703			break;
3704		n_barrier_attempts++;
3705		cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */
3706		if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) {
3707			n_rcu_torture_barrier_error++;
3708			pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
3709			       atomic_read(&barrier_cbs_invoked),
3710			       n_barrier_cbs);
3711			WARN_ON(1);
3712			// Wait manually for the remaining callbacks
3713			i = 0;
3714			do {
3715				if (WARN_ON(i++ > HZ))
3716					i = INT_MIN;
3717				schedule_timeout_interruptible(1);
3718				cur_ops->cb_barrier();
3719			} while (atomic_read(&barrier_cbs_invoked) !=
3720				 n_barrier_cbs &&
3721				 !torture_must_stop());
3722			smp_mb(); // Can't trust ordering if broken.
3723			if (!torture_must_stop())
3724				pr_err("Recovered: barrier_cbs_invoked = %d\n",
3725				       atomic_read(&barrier_cbs_invoked));
3726		} else {
3727			n_barrier_successes++;
3728		}
3729		schedule_timeout_interruptible(HZ / 10);
3730	} while (!torture_must_stop());
3731	torture_kthread_stopping("rcu_torture_barrier");
3732	return 0;
3733}
3734
3735/* Initialize RCU barrier testing. */
3736static int rcu_torture_barrier_init(void)
3737{
3738	int i;
3739	int ret;
3740
3741	if (n_barrier_cbs <= 0)
3742		return 0;
3743	if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) {
3744		pr_alert("%s" TORTURE_FLAG
3745			 " Call or barrier ops missing for %s,\n",
3746			 torture_type, cur_ops->name);
3747		pr_alert("%s" TORTURE_FLAG
3748			 " RCU barrier testing omitted from run.\n",
3749			 torture_type);
3750		return 0;
3751	}
3752	atomic_set(&barrier_cbs_count, 0);
3753	atomic_set(&barrier_cbs_invoked, 0);
3754	barrier_cbs_tasks =
3755		kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]),
3756			GFP_KERNEL);
3757	barrier_cbs_wq =
3758		kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL);
3759	if (barrier_cbs_tasks == NULL || !barrier_cbs_wq)
3760		return -ENOMEM;
3761	for (i = 0; i < n_barrier_cbs; i++) {
3762		init_waitqueue_head(&barrier_cbs_wq[i]);
3763		ret = torture_create_kthread(rcu_torture_barrier_cbs,
3764					     (void *)(long)i,
3765					     barrier_cbs_tasks[i]);
3766		if (ret)
3767			return ret;
3768	}
3769	return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task);
3770}
3771
3772/* Clean up after RCU barrier testing. */
3773static void rcu_torture_barrier_cleanup(void)
3774{
3775	int i;
3776
3777	torture_stop_kthread(rcu_torture_barrier, barrier_task);
3778	if (barrier_cbs_tasks != NULL) {
3779		for (i = 0; i < n_barrier_cbs; i++)
3780			torture_stop_kthread(rcu_torture_barrier_cbs,
3781					     barrier_cbs_tasks[i]);
3782		kfree(barrier_cbs_tasks);
3783		barrier_cbs_tasks = NULL;
3784	}
3785	if (barrier_cbs_wq != NULL) {
3786		kfree(barrier_cbs_wq);
3787		barrier_cbs_wq = NULL;
3788	}
3789}
3790
3791static bool rcu_torture_can_boost(void)
3792{
3793	static int boost_warn_once;
3794	int prio;
3795
3796	if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2)
3797		return false;
3798	if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)
3799		return false;
3800
3801	prio = rcu_get_gp_kthreads_prio();
3802	if (!prio)
3803		return false;
3804
3805	if (prio < 2) {
3806		if (boost_warn_once == 1)
3807			return false;
3808
3809		pr_alert("%s: WARN: RCU kthread priority too low to test boosting.  Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME);
3810		boost_warn_once = 1;
3811		return false;
3812	}
3813
3814	return true;
3815}
3816
3817static bool read_exit_child_stop;
3818static bool read_exit_child_stopped;
3819static wait_queue_head_t read_exit_wq;
3820
3821// Child kthread which just does an rcutorture reader and exits.
3822static int rcu_torture_read_exit_child(void *trsp_in)
3823{
3824	struct torture_random_state *trsp = trsp_in;
3825
3826	set_user_nice(current, MAX_NICE);
3827	// Minimize time between reading and exiting.
3828	while (!kthread_should_stop())
3829		schedule_timeout_uninterruptible(HZ / 20);
3830	(void)rcu_torture_one_read(trsp, -1);
3831	return 0;
3832}
3833
3834// Parent kthread which creates and destroys read-exit child kthreads.
3835static int rcu_torture_read_exit(void *unused)
3836{
3837	bool errexit = false;
3838	int i;
3839	struct task_struct *tsp;
3840	DEFINE_TORTURE_RANDOM(trs);
3841
3842	// Allocate and initialize.
3843	set_user_nice(current, MAX_NICE);
3844	VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test");
3845
3846	// Each pass through this loop does one read-exit episode.
3847	do {
3848		VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode");
3849		for (i = 0; i < read_exit_burst; i++) {
3850			if (READ_ONCE(read_exit_child_stop))
3851				break;
3852			stutter_wait("rcu_torture_read_exit");
3853			// Spawn child.
3854			tsp = kthread_run(rcu_torture_read_exit_child,
3855					  &trs, "%s", "rcu_torture_read_exit_child");
3856			if (IS_ERR(tsp)) {
3857				TOROUT_ERRSTRING("out of memory");
3858				errexit = true;
3859				break;
3860			}
3861			cond_resched();
3862			kthread_stop(tsp);
3863			n_read_exits++;
3864		}
3865		VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode");
3866		rcu_barrier(); // Wait for task_struct free, avoid OOM.
3867		i = 0;
3868		for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++)
3869			schedule_timeout_uninterruptible(HZ);
3870	} while (!errexit && !READ_ONCE(read_exit_child_stop));
3871
3872	// Clean up and exit.
3873	smp_store_release(&read_exit_child_stopped, true); // After reaping.
3874	smp_mb(); // Store before wakeup.
3875	wake_up(&read_exit_wq);
3876	while (!torture_must_stop())
3877		schedule_timeout_uninterruptible(HZ / 20);
3878	torture_kthread_stopping("rcu_torture_read_exit");
3879	return 0;
3880}
3881
3882static int rcu_torture_read_exit_init(void)
3883{
3884	if (read_exit_burst <= 0)
3885		return 0;
3886	init_waitqueue_head(&read_exit_wq);
3887	read_exit_child_stop = false;
3888	read_exit_child_stopped = false;
3889	return torture_create_kthread(rcu_torture_read_exit, NULL,
3890				      read_exit_task);
3891}
3892
3893static void rcu_torture_read_exit_cleanup(void)
3894{
3895	if (!read_exit_task)
3896		return;
3897	WRITE_ONCE(read_exit_child_stop, true);
3898	smp_mb(); // Above write before wait.
3899	wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped));
3900	torture_stop_kthread(rcutorture_read_exit, read_exit_task);
3901}
3902
3903static void rcutorture_test_nmis(int n)
3904{
3905#if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)
3906	int cpu;
3907	int dumpcpu;
3908	int i;
3909
3910	for (i = 0; i < n; i++) {
3911		preempt_disable();
3912		cpu = smp_processor_id();
3913		dumpcpu = cpu + 1;
3914		if (dumpcpu >= nr_cpu_ids)
3915			dumpcpu = 0;
3916		pr_alert("%s: CPU %d invoking dump_cpu_task(%d)\n", __func__, cpu, dumpcpu);
3917		dump_cpu_task(dumpcpu);
3918		preempt_enable();
3919		schedule_timeout_uninterruptible(15 * HZ);
3920	}
3921#else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)
3922	WARN_ONCE(n, "Non-zero rcutorture.test_nmis=%d permitted only when rcutorture is built in.\n", test_nmis);
3923#endif // #else // #if IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)
3924}
3925
3926// Randomly preempt online CPUs.
3927static int rcu_torture_preempt(void *unused)
3928{
3929	int cpu = -1;
3930	DEFINE_TORTURE_RANDOM(rand);
3931
3932	schedule_timeout_idle(stall_cpu_holdoff);
3933	do {
3934		// Wait for preempt_interval ms with up to 100us fuzz.
3935		torture_hrtimeout_ms(preempt_interval, 100, &rand);
3936		// Select online CPU.
3937		cpu = cpumask_next(cpu, cpu_online_mask);
3938		if (cpu >= nr_cpu_ids)
3939			cpu = cpumask_next(-1, cpu_online_mask);
3940		WARN_ON_ONCE(cpu >= nr_cpu_ids);
3941		// Move to that CPU, if can't do so, retry later.
3942		if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false))
3943			continue;
3944		// Preempt at high-ish priority, then reset to normal.
3945		sched_set_fifo(current);
3946		torture_sched_setaffinity(current->pid, cpu_present_mask, true);
3947		mdelay(preempt_duration);
3948		sched_set_normal(current, 0);
3949		stutter_wait("rcu_torture_preempt");
3950	} while (!torture_must_stop());
3951	torture_kthread_stopping("rcu_torture_preempt");
3952	return 0;
3953}
3954
3955static enum cpuhp_state rcutor_hp;
3956
3957static struct hrtimer gpwrap_lag_timer;
3958static bool gpwrap_lag_active;
3959
3960/* Timer handler for toggling RCU grace-period sequence overflow test lag value */
3961static enum hrtimer_restart rcu_gpwrap_lag_timer(struct hrtimer *timer)
3962{
3963	ktime_t next_delay;
3964
3965	if (gpwrap_lag_active) {
3966		pr_alert("rcu-torture: Disabling gpwrap lag (value=0)\n");
3967		cur_ops->set_gpwrap_lag(0);
3968		gpwrap_lag_active = false;
3969		next_delay = ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0);
3970	} else {
3971		pr_alert("rcu-torture: Enabling gpwrap lag (value=%d)\n", gpwrap_lag_gps);
3972		cur_ops->set_gpwrap_lag(gpwrap_lag_gps);
3973		gpwrap_lag_active = true;
3974		next_delay = ktime_set(gpwrap_lag_active_mins * 60, 0);
3975	}
3976
3977	if (torture_must_stop_irq())
3978		return HRTIMER_NORESTART;
3979
3980	hrtimer_forward_now(timer, next_delay);
3981	return HRTIMER_RESTART;
3982}
3983
3984static int rcu_gpwrap_lag_init(void)
3985{
3986	if (!gpwrap_lag)
3987		return 0;
3988
3989	if (gpwrap_lag_cycle_mins <= 0 || gpwrap_lag_active_mins <= 0) {
3990		pr_alert("rcu-torture: lag timing parameters must be positive\n");
3991		return -EINVAL;
3992	}
3993
3994	hrtimer_setup(&gpwrap_lag_timer, rcu_gpwrap_lag_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3995	gpwrap_lag_active = false;
3996	hrtimer_start(&gpwrap_lag_timer,
3997		      ktime_set((gpwrap_lag_cycle_mins - gpwrap_lag_active_mins) * 60, 0), HRTIMER_MODE_REL);
3998
3999	return 0;
4000}
4001
4002static void rcu_gpwrap_lag_cleanup(void)
4003{
4004	hrtimer_cancel(&gpwrap_lag_timer);
4005	cur_ops->set_gpwrap_lag(0);
4006	gpwrap_lag_active = false;
4007}
4008static void
4009rcu_torture_cleanup(void)
4010{
4011	int firsttime;
4012	int flags = 0;
4013	unsigned long gp_seq = 0;
4014	int i;
4015	int j;
4016
4017	if (torture_cleanup_begin()) {
4018		if (cur_ops->cb_barrier != NULL) {
4019			pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
4020			cur_ops->cb_barrier();
4021		}
4022		if (cur_ops->gp_slow_unregister)
4023			cur_ops->gp_slow_unregister(NULL);
4024		return;
4025	}
4026	if (!cur_ops) {
4027		torture_cleanup_end();
4028		return;
4029	}
4030
4031	rcutorture_test_nmis(test_nmis);
4032
4033	if (cur_ops->gp_kthread_dbg)
4034		cur_ops->gp_kthread_dbg();
4035	torture_stop_kthread(rcu_torture_preempt, preempt_task);
4036	rcu_torture_read_exit_cleanup();
4037	rcu_torture_barrier_cleanup();
4038	rcu_torture_fwd_prog_cleanup();
4039	torture_stop_kthread(rcu_torture_stall, stall_task);
4040	torture_stop_kthread(rcu_torture_writer, writer_task);
4041
4042	if (nocb_tasks) {
4043		for (i = 0; i < nrealnocbers; i++)
4044			torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]);
4045		kfree(nocb_tasks);
4046		nocb_tasks = NULL;
4047	}
4048
4049	if (updown_task) {
4050		torture_stop_kthread(rcu_torture_updown, updown_task);
4051		updown_task = NULL;
4052	}
4053	if (reader_tasks) {
4054		for (i = 0; i < nrealreaders; i++)
4055			torture_stop_kthread(rcu_torture_reader,
4056					     reader_tasks[i]);
4057		kfree(reader_tasks);
4058		reader_tasks = NULL;
4059	}
4060	kfree(rcu_torture_reader_mbchk);
4061	rcu_torture_reader_mbchk = NULL;
4062
4063	if (fakewriter_tasks) {
4064		for (i = 0; i < nrealfakewriters; i++)
4065			torture_stop_kthread(rcu_torture_fakewriter,
4066					     fakewriter_tasks[i]);
4067		kfree(fakewriter_tasks);
4068		fakewriter_tasks = NULL;
4069	}
4070
4071	if (cur_ops->get_gp_data)
4072		cur_ops->get_gp_data(&flags, &gp_seq);
4073	pr_alert("%s:  End-test grace-period state: g%ld f%#x total-gps=%ld\n",
4074		 cur_ops->name, (long)gp_seq, flags,
4075		 rcutorture_seq_diff(gp_seq, start_gp_seq));
4076	torture_stop_kthread(rcu_torture_stats, stats_task);
4077	torture_stop_kthread(rcu_torture_fqs, fqs_task);
4078	if (rcu_torture_can_boost() && rcutor_hp >= 0)
4079		cpuhp_remove_state(rcutor_hp);
4080
4081	/*
4082	 * Wait for all RCU callbacks to fire, then do torture-type-specific
4083	 * cleanup operations.
4084	 */
4085	if (cur_ops->cb_barrier != NULL) {
4086		pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier);
4087		cur_ops->cb_barrier();
4088	}
4089	if (cur_ops->cleanup != NULL)
4090		cur_ops->cleanup();
4091
4092	rcu_torture_mem_dump_obj();
4093
4094	rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
4095
4096	if (err_segs_recorded) {
4097		pr_alert("Failure/close-call rcutorture reader segments:\n");
4098		if (rt_read_nsegs == 0)
4099			pr_alert("\t: No segments recorded!!!\n");
4100		firsttime = 1;
4101		for (i = 0; i < rt_read_nsegs; i++) {
4102			if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP))
4103				pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL));
4104			else
4105				pr_alert("\t");
4106			pr_cont("%d: %#4x", i, err_segs[i].rt_readstate);
4107			if (err_segs[i].rt_delay_jiffies != 0) {
4108				pr_cont("%s%ldjiffies", firsttime ? "" : "+",
4109					err_segs[i].rt_delay_jiffies);
4110				firsttime = 0;
4111			}
4112			if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) {
4113				pr_cont(" CPU %2d", err_segs[i].rt_cpu);
4114				if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu)
4115					pr_cont("->%-2d", err_segs[i].rt_end_cpu);
4116				else
4117					pr_cont(" ...");
4118			}
4119			if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) &&
4120			    cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) {
4121				char buf1[20+1];
4122				char buf2[20+1];
4123				char sepchar = '-';
4124
4125				cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq,
4126							buf1, ARRAY_SIZE(buf1));
4127				cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end,
4128							buf2, ARRAY_SIZE(buf2));
4129				if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) {
4130					if (buf2[0]) {
4131						for (j = 0; buf2[j]; j++)
4132							buf2[j] = '.';
4133						if (j)
4134							buf2[j - 1] = ' ';
4135					}
4136					sepchar = ' ';
4137				}
4138				pr_cont(" %s%c%s", buf1, sepchar, buf2);
4139			}
4140			if (err_segs[i].rt_delay_ms != 0) {
4141				pr_cont(" %s%ldms", firsttime ? "" : "+",
4142					err_segs[i].rt_delay_ms);
4143				firsttime = 0;
4144			}
4145			if (err_segs[i].rt_delay_us != 0) {
4146				pr_cont(" %s%ldus", firsttime ? "" : "+",
4147					err_segs[i].rt_delay_us);
4148				firsttime = 0;
4149			}
4150			pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : "");
4151			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH)
4152				pr_cont(" BH");
4153			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ)
4154				pr_cont(" IRQ");
4155			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT)
4156				pr_cont(" PREEMPT");
4157			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH)
4158				pr_cont(" RBH");
4159			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED)
4160				pr_cont(" SCHED");
4161			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1)
4162				pr_cont(" RCU_1");
4163			if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2)
4164				pr_cont(" RCU_2");
4165			pr_cont("\n");
4166
4167		}
4168		if (rt_read_preempted)
4169			pr_alert("\tReader was preempted.\n");
4170	}
4171	if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error)
4172		rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
4173	else if (torture_onoff_failures())
4174		rcu_torture_print_module_parms(cur_ops,
4175					       "End of test: RCU_HOTPLUG");
4176	else
4177		rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
4178	torture_cleanup_end();
4179	if (cur_ops->gp_slow_unregister)
4180		cur_ops->gp_slow_unregister(NULL);
4181
4182	if (gpwrap_lag && cur_ops->set_gpwrap_lag)
4183		rcu_gpwrap_lag_cleanup();
4184}
4185
4186static void rcu_torture_leak_cb(struct rcu_head *rhp)
4187{
4188}
4189
4190static void rcu_torture_err_cb(struct rcu_head *rhp)
4191{
4192	/*
4193	 * This -might- happen due to race conditions, but is unlikely.
4194	 * The scenario that leads to this happening is that the
4195	 * first of the pair of duplicate callbacks is queued,
4196	 * someone else starts a grace period that includes that
4197	 * callback, then the second of the pair must wait for the
4198	 * next grace period.  Unlikely, but can happen.  If it
4199	 * does happen, the debug-objects subsystem won't have splatted.
4200	 */
4201	pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME);
4202}
4203
4204/*
4205 * Verify that double-free causes debug-objects to complain, but only
4206 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.  Otherwise, say that the test
4207 * cannot be carried out.
4208 */
4209static void rcu_test_debug_objects(void)
4210{
4211	struct rcu_head rh1;
4212	struct rcu_head rh2;
4213	int idx;
4214
4215	if (!IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD)) {
4216		pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_%s()\n",
4217					KBUILD_MODNAME, cur_ops->name);
4218		return;
4219	}
4220
4221	if (WARN_ON_ONCE(cur_ops->debug_objects &&
4222			(!cur_ops->call || !cur_ops->cb_barrier)))
4223		return;
4224
4225	struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL);
4226
4227	init_rcu_head_on_stack(&rh1);
4228	init_rcu_head_on_stack(&rh2);
4229	pr_alert("%s: WARN: Duplicate call_%s() test starting.\n", KBUILD_MODNAME, cur_ops->name);
4230
4231	/* Try to queue the rh2 pair of callbacks for the same grace period. */
4232	idx = cur_ops->readlock(); /* Make it impossible to finish a grace period. */
4233	cur_ops->call(&rh1, rcu_torture_leak_cb); /* Start grace period. */
4234	cur_ops->call(&rh2, rcu_torture_leak_cb);
4235	cur_ops->call(&rh2, rcu_torture_err_cb); /* Duplicate callback. */
4236	if (rhp) {
4237		cur_ops->call(rhp, rcu_torture_leak_cb);
4238		cur_ops->call(rhp, rcu_torture_err_cb); /* Another duplicate callback. */
4239	}
4240	cur_ops->readunlock(idx);
4241
4242	/* Wait for them all to get done so we can safely return. */
4243	cur_ops->cb_barrier();
4244	pr_alert("%s: WARN: Duplicate call_%s() test complete.\n", KBUILD_MODNAME, cur_ops->name);
4245	destroy_rcu_head_on_stack(&rh1);
4246	destroy_rcu_head_on_stack(&rh2);
4247	kfree(rhp);
4248}
4249
4250static void rcutorture_sync(void)
4251{
4252	static unsigned long n;
4253
4254	if (cur_ops->sync && !(++n & 0xfff))
4255		cur_ops->sync();
4256}
4257
4258static DEFINE_MUTEX(mut0);
4259static DEFINE_MUTEX(mut1);
4260static DEFINE_MUTEX(mut2);
4261static DEFINE_MUTEX(mut3);
4262static DEFINE_MUTEX(mut4);
4263static DEFINE_MUTEX(mut5);
4264static DEFINE_MUTEX(mut6);
4265static DEFINE_MUTEX(mut7);
4266static DEFINE_MUTEX(mut8);
4267static DEFINE_MUTEX(mut9);
4268
4269static DECLARE_RWSEM(rwsem0);
4270static DECLARE_RWSEM(rwsem1);
4271static DECLARE_RWSEM(rwsem2);
4272static DECLARE_RWSEM(rwsem3);
4273static DECLARE_RWSEM(rwsem4);
4274static DECLARE_RWSEM(rwsem5);
4275static DECLARE_RWSEM(rwsem6);
4276static DECLARE_RWSEM(rwsem7);
4277static DECLARE_RWSEM(rwsem8);
4278static DECLARE_RWSEM(rwsem9);
4279
4280DEFINE_STATIC_SRCU(srcu0);
4281DEFINE_STATIC_SRCU(srcu1);
4282DEFINE_STATIC_SRCU(srcu2);
4283DEFINE_STATIC_SRCU(srcu3);
4284DEFINE_STATIC_SRCU(srcu4);
4285DEFINE_STATIC_SRCU(srcu5);
4286DEFINE_STATIC_SRCU(srcu6);
4287DEFINE_STATIC_SRCU(srcu7);
4288DEFINE_STATIC_SRCU(srcu8);
4289DEFINE_STATIC_SRCU(srcu9);
4290
4291static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i,
4292			     int cyclelen, int deadlock)
4293{
4294	int j = i + 1;
4295
4296	if (j >= cyclelen)
4297		j = deadlock ? 0 : -1;
4298	if (j >= 0)
4299		pr_info("%s: %s(%d), %s(%d), %s(%d)\n", f, fl, i, fs, j, fu, i);
4300	else
4301		pr_info("%s: %s(%d), %s(%d)\n", f, fl, i, fu, i);
4302	return j;
4303}
4304
4305// Test lockdep on SRCU-based deadlock scenarios.
4306static void rcu_torture_init_srcu_lockdep(void)
4307{
4308	int cyclelen;
4309	int deadlock;
4310	bool err = false;
4311	int i;
4312	int j;
4313	int idx;
4314	struct mutex *muts[] = { &mut0, &mut1, &mut2, &mut3, &mut4,
4315				 &mut5, &mut6, &mut7, &mut8, &mut9 };
4316	struct rw_semaphore *rwsems[] = { &rwsem0, &rwsem1, &rwsem2, &rwsem3, &rwsem4,
4317					  &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 };
4318	struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4,
4319					&srcu5, &srcu6, &srcu7, &srcu8, &srcu9 };
4320	int testtype;
4321
4322	if (!test_srcu_lockdep)
4323		return;
4324
4325	deadlock = test_srcu_lockdep / 1000;
4326	testtype = (test_srcu_lockdep / 10) % 100;
4327	cyclelen = test_srcu_lockdep % 10;
4328	WARN_ON_ONCE(ARRAY_SIZE(muts) != ARRAY_SIZE(srcus));
4329	if (WARN_ONCE(deadlock != !!deadlock,
4330		      "%s: test_srcu_lockdep=%d and deadlock digit %d must be zero or one.\n",
4331		      __func__, test_srcu_lockdep, deadlock))
4332		err = true;
4333	if (WARN_ONCE(cyclelen <= 0,
4334		      "%s: test_srcu_lockdep=%d and cycle-length digit %d must be greater than zero.\n",
4335		      __func__, test_srcu_lockdep, cyclelen))
4336		err = true;
4337	if (err)
4338		goto err_out;
4339
4340	if (testtype == 0) {
4341		pr_info("%s: test_srcu_lockdep = %05d: SRCU %d-way %sdeadlock.\n",
4342			__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4343		if (deadlock && cyclelen == 1)
4344			pr_info("%s: Expect hang.\n", __func__);
4345		for (i = 0; i < cyclelen; i++) {
4346			j = srcu_lockdep_next(__func__, "srcu_read_lock", "synchronize_srcu",
4347					      "srcu_read_unlock", i, cyclelen, deadlock);
4348			idx = srcu_read_lock(srcus[i]);
4349			if (j >= 0)
4350				synchronize_srcu(srcus[j]);
4351			srcu_read_unlock(srcus[i], idx);
4352		}
4353		return;
4354	}
4355
4356	if (testtype == 1) {
4357		pr_info("%s: test_srcu_lockdep = %05d: SRCU/mutex %d-way %sdeadlock.\n",
4358			__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4359		for (i = 0; i < cyclelen; i++) {
4360			pr_info("%s: srcu_read_lock(%d), mutex_lock(%d), mutex_unlock(%d), srcu_read_unlock(%d)\n",
4361				__func__, i, i, i, i);
4362			idx = srcu_read_lock(srcus[i]);
4363			mutex_lock(muts[i]);
4364			mutex_unlock(muts[i]);
4365			srcu_read_unlock(srcus[i], idx);
4366
4367			j = srcu_lockdep_next(__func__, "mutex_lock", "synchronize_srcu",
4368					      "mutex_unlock", i, cyclelen, deadlock);
4369			mutex_lock(muts[i]);
4370			if (j >= 0)
4371				synchronize_srcu(srcus[j]);
4372			mutex_unlock(muts[i]);
4373		}
4374		return;
4375	}
4376
4377	if (testtype == 2) {
4378		pr_info("%s: test_srcu_lockdep = %05d: SRCU/rwsem %d-way %sdeadlock.\n",
4379			__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4380		for (i = 0; i < cyclelen; i++) {
4381			pr_info("%s: srcu_read_lock(%d), down_read(%d), up_read(%d), srcu_read_unlock(%d)\n",
4382				__func__, i, i, i, i);
4383			idx = srcu_read_lock(srcus[i]);
4384			down_read(rwsems[i]);
4385			up_read(rwsems[i]);
4386			srcu_read_unlock(srcus[i], idx);
4387
4388			j = srcu_lockdep_next(__func__, "down_write", "synchronize_srcu",
4389					      "up_write", i, cyclelen, deadlock);
4390			down_write(rwsems[i]);
4391			if (j >= 0)
4392				synchronize_srcu(srcus[j]);
4393			up_write(rwsems[i]);
4394		}
4395		return;
4396	}
4397
4398#ifdef CONFIG_TASKS_TRACE_RCU
4399	if (testtype == 3) {
4400		pr_info("%s: test_srcu_lockdep = %05d: SRCU and Tasks Trace RCU %d-way %sdeadlock.\n",
4401			__func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-");
4402		if (deadlock && cyclelen == 1)
4403			pr_info("%s: Expect hang.\n", __func__);
4404		for (i = 0; i < cyclelen; i++) {
4405			char *fl = i == 0 ? "rcu_read_lock_trace" : "srcu_read_lock";
4406			char *fs = i == cyclelen - 1 ? "synchronize_rcu_tasks_trace"
4407						     : "synchronize_srcu";
4408			char *fu = i == 0 ? "rcu_read_unlock_trace" : "srcu_read_unlock";
4409
4410			j = srcu_lockdep_next(__func__, fl, fs, fu, i, cyclelen, deadlock);
4411			if (i == 0)
4412				rcu_read_lock_trace();
4413			else
4414				idx = srcu_read_lock(srcus[i]);
4415			if (j >= 0) {
4416				if (i == cyclelen - 1)
4417					synchronize_rcu_tasks_trace();
4418				else
4419					synchronize_srcu(srcus[j]);
4420			}
4421			if (i == 0)
4422				rcu_read_unlock_trace();
4423			else
4424				srcu_read_unlock(srcus[i], idx);
4425		}
4426		return;
4427	}
4428#endif // #ifdef CONFIG_TASKS_TRACE_RCU
4429
4430err_out:
4431	pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep);
4432	pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__);
4433	pr_info("%s: D: Deadlock if nonzero.\n", __func__);
4434	pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__);
4435	pr_info("%s: L: Cycle length.\n", __func__);
4436	if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU))
4437		pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__);
4438}
4439
4440static int __init
4441rcu_torture_init(void)
4442{
4443	long i;
4444	int cpu;
4445	int firsterr = 0;
4446	int flags = 0;
4447	unsigned long gp_seq = 0;
4448	static struct rcu_torture_ops *torture_ops[] = {
4449		&rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops,
4450		TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS
4451		&trivial_ops,
4452	};
4453
4454	if (!torture_init_begin(torture_type, verbose))
4455		return -EBUSY;
4456
4457	/* Process args and tell the world that the torturer is on the job. */
4458	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
4459		cur_ops = torture_ops[i];
4460		if (strcmp(torture_type, cur_ops->name) == 0)
4461			break;
4462	}
4463	if (i == ARRAY_SIZE(torture_ops)) {
4464		pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
4465			 torture_type);
4466		pr_alert("rcu-torture types:");
4467		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
4468			pr_cont(" %s", torture_ops[i]->name);
4469		pr_cont("\n");
4470		firsterr = -EINVAL;
4471		cur_ops = NULL;
4472		goto unwind;
4473	}
4474	if (cur_ops->fqs == NULL && fqs_duration != 0) {
4475		pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
4476		fqs_duration = 0;
4477	}
4478	if (nocbs_nthreads != 0 && (cur_ops != &rcu_ops ||
4479				    !IS_ENABLED(CONFIG_RCU_NOCB_CPU))) {
4480		pr_alert("rcu-torture types: %s and CONFIG_RCU_NOCB_CPU=%d, nocb toggle disabled.\n",
4481			 cur_ops->name, IS_ENABLED(CONFIG_RCU_NOCB_CPU));
4482		nocbs_nthreads = 0;
4483	}
4484	if (cur_ops->init)
4485		cur_ops->init();
4486
4487	rcu_torture_init_srcu_lockdep();
4488
4489	if (nfakewriters >= 0) {
4490		nrealfakewriters = nfakewriters;
4491	} else {
4492		nrealfakewriters = num_online_cpus() - 2 - nfakewriters;
4493		if (nrealfakewriters <= 0)
4494			nrealfakewriters = 1;
4495	}
4496
4497	if (nreaders >= 0) {
4498		nrealreaders = nreaders;
4499	} else {
4500		nrealreaders = num_online_cpus() - 2 - nreaders;
4501		if (nrealreaders <= 0)
4502			nrealreaders = 1;
4503	}
4504	rcu_torture_print_module_parms(cur_ops, "Start of test");
4505	if (cur_ops->get_gp_data)
4506		cur_ops->get_gp_data(&flags, &gp_seq);
4507	start_gp_seq = gp_seq;
4508	pr_alert("%s:  Start-test grace-period state: g%ld f%#x\n",
4509		 cur_ops->name, (long)gp_seq, flags);
4510
4511	/* Set up the freelist. */
4512
4513	INIT_LIST_HEAD(&rcu_torture_freelist);
4514	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
4515		rcu_tortures[i].rtort_mbtest = 0;
4516		list_add_tail(&rcu_tortures[i].rtort_free,
4517			      &rcu_torture_freelist);
4518	}
4519
4520	/* Initialize the statistics so that each run gets its own numbers. */
4521
4522	rcu_torture_current = NULL;
4523	rcu_torture_current_version = 0;
4524	atomic_set(&n_rcu_torture_alloc, 0);
4525	atomic_set(&n_rcu_torture_alloc_fail, 0);
4526	atomic_set(&n_rcu_torture_free, 0);
4527	atomic_set(&n_rcu_torture_mberror, 0);
4528	atomic_set(&n_rcu_torture_mbchk_fail, 0);
4529	atomic_set(&n_rcu_torture_mbchk_tries, 0);
4530	atomic_set(&n_rcu_torture_error, 0);
4531	n_rcu_torture_barrier_error = 0;
4532	n_rcu_torture_boost_ktrerror = 0;
4533	n_rcu_torture_boost_failure = 0;
4534	n_rcu_torture_boosts = 0;
4535	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
4536		atomic_set(&rcu_torture_wcount[i], 0);
4537	for_each_possible_cpu(cpu) {
4538		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
4539			per_cpu(rcu_torture_count, cpu)[i] = 0;
4540			per_cpu(rcu_torture_batch, cpu)[i] = 0;
4541		}
4542	}
4543	err_segs_recorded = 0;
4544	rt_read_nsegs = 0;
4545
4546	/* Start up the kthreads. */
4547
4548	rcu_torture_write_types();
4549	if (nrealfakewriters > 0) {
4550		fakewriter_tasks = kcalloc(nrealfakewriters,
4551					   sizeof(fakewriter_tasks[0]),
4552					   GFP_KERNEL);
4553		if (fakewriter_tasks == NULL) {
4554			TOROUT_ERRSTRING("out of memory");
4555			firsterr = -ENOMEM;
4556			goto unwind;
4557		}
4558	}
4559	for (i = 0; i < nrealfakewriters; i++) {
4560		firsterr = torture_create_kthread(rcu_torture_fakewriter,
4561						  NULL, fakewriter_tasks[i]);
4562		if (torture_init_error(firsterr))
4563			goto unwind;
4564	}
4565	reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]),
4566			       GFP_KERNEL);
4567	rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk),
4568					   GFP_KERNEL);
4569	if (!reader_tasks || !rcu_torture_reader_mbchk) {
4570		TOROUT_ERRSTRING("out of memory");
4571		firsterr = -ENOMEM;
4572		goto unwind;
4573	}
4574	for (i = 0; i < nrealreaders; i++) {
4575		rcu_torture_reader_mbchk[i].rtc_chkrdr = -1;
4576		firsterr = torture_create_kthread(rcu_torture_reader, (void *)i,
4577						  reader_tasks[i]);
4578		if (torture_init_error(firsterr))
4579			goto unwind;
4580	}
4581
4582	firsterr = torture_create_kthread(rcu_torture_writer, NULL,
4583					  writer_task);
4584	if (torture_init_error(firsterr))
4585		goto unwind;
4586
4587	firsterr = rcu_torture_updown_init();
4588	if (torture_init_error(firsterr))
4589		goto unwind;
4590	nrealnocbers = nocbs_nthreads;
4591	if (WARN_ON(nrealnocbers < 0))
4592		nrealnocbers = 1;
4593	if (WARN_ON(nocbs_toggle < 0))
4594		nocbs_toggle = HZ;
4595	if (nrealnocbers > 0) {
4596		nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL);
4597		if (nocb_tasks == NULL) {
4598			TOROUT_ERRSTRING("out of memory");
4599			firsterr = -ENOMEM;
4600			goto unwind;
4601		}
4602	} else {
4603		nocb_tasks = NULL;
4604	}
4605	for (i = 0; i < nrealnocbers; i++) {
4606		firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]);
4607		if (torture_init_error(firsterr))
4608			goto unwind;
4609	}
4610	if (stat_interval > 0) {
4611		firsterr = torture_create_kthread(rcu_torture_stats, NULL,
4612						  stats_task);
4613		if (torture_init_error(firsterr))
4614			goto unwind;
4615	}
4616	if (test_no_idle_hz && shuffle_interval > 0) {
4617		firsterr = torture_shuffle_init(shuffle_interval * HZ);
4618		if (torture_init_error(firsterr))
4619			goto unwind;
4620	}
4621	if (stutter < 0)
4622		stutter = 0;
4623	if (stutter) {
4624		int t;
4625
4626		t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ;
4627		firsterr = torture_stutter_init(stutter * HZ, t);
4628		if (torture_init_error(firsterr))
4629			goto unwind;
4630	}
4631	if (fqs_duration < 0)
4632		fqs_duration = 0;
4633	if (fqs_holdoff < 0)
4634		fqs_holdoff = 0;
4635	if (fqs_duration && fqs_holdoff) {
4636		/* Create the fqs thread */
4637		firsterr = torture_create_kthread(rcu_torture_fqs, NULL,
4638						  fqs_task);
4639		if (torture_init_error(firsterr))
4640			goto unwind;
4641	}
4642	if (test_boost_interval < 1)
4643		test_boost_interval = 1;
4644	if (test_boost_duration < 2)
4645		test_boost_duration = 2;
4646	if (rcu_torture_can_boost()) {
4647
4648		boost_starttime = jiffies + test_boost_interval * HZ;
4649
4650		firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE",
4651					     rcutorture_booster_init,
4652					     rcutorture_booster_cleanup);
4653		rcutor_hp = firsterr;
4654		if (torture_init_error(firsterr))
4655			goto unwind;
4656	}
4657	shutdown_jiffies = jiffies + shutdown_secs * HZ;
4658	firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup);
4659	if (torture_init_error(firsterr))
4660		goto unwind;
4661	firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval,
4662				      rcutorture_sync);
4663	if (torture_init_error(firsterr))
4664		goto unwind;
4665	firsterr = rcu_torture_stall_init();
4666	if (torture_init_error(firsterr))
4667		goto unwind;
4668	firsterr = rcu_torture_fwd_prog_init();
4669	if (torture_init_error(firsterr))
4670		goto unwind;
4671	firsterr = rcu_torture_barrier_init();
4672	if (torture_init_error(firsterr))
4673		goto unwind;
4674	firsterr = rcu_torture_read_exit_init();
4675	if (torture_init_error(firsterr))
4676		goto unwind;
4677	if (preempt_duration > 0) {
4678		firsterr = torture_create_kthread(rcu_torture_preempt, NULL, preempt_task);
4679		if (torture_init_error(firsterr))
4680			goto unwind;
4681	}
4682	if (object_debug)
4683		rcu_test_debug_objects();
4684
4685	if (cur_ops->gp_slow_register && !WARN_ON_ONCE(!cur_ops->gp_slow_unregister))
4686		cur_ops->gp_slow_register(&rcu_fwd_cb_nodelay);
4687
4688	if (gpwrap_lag && cur_ops->set_gpwrap_lag) {
4689		firsterr = rcu_gpwrap_lag_init();
4690		if (torture_init_error(firsterr))
4691			goto unwind;
4692	}
4693
4694	torture_init_end();
4695	return 0;
4696
4697unwind:
4698	torture_init_end();
4699	rcu_torture_cleanup();
4700	if (shutdown_secs) {
4701		WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST));
4702		kernel_power_off();
4703	}
4704	return firsterr;
4705}
4706
4707module_init(rcu_torture_init);
4708module_exit(rcu_torture_cleanup);