FILE
srcutree.c
File preview with syntax highlighting.
linux-6.19.14/kernel/rcu/srcutree.c
1// SPDX-License-Identifier: GPL-2.0+ 2/* 3 * Sleepable Read-Copy Update mechanism for mutual exclusion. 4 * 5 * Copyright (C) IBM Corporation, 2006 6 * Copyright (C) Fujitsu, 2012 7 * 8 * Authors: Paul McKenney <paulmck@linux.ibm.com> 9 * Lai Jiangshan <laijs@cn.fujitsu.com> 10 * 11 * For detailed explanation of Read-Copy Update mechanism see - 12 * Documentation/RCU/ *.txt 13 * 14 */ 15 16#define pr_fmt(fmt) "rcu: " fmt 17 18#include <linux/export.h> 19#include <linux/mutex.h> 20#include <linux/percpu.h> 21#include <linux/preempt.h> 22#include <linux/rcupdate_wait.h> 23#include <linux/sched.h> 24#include <linux/smp.h> 25#include <linux/delay.h> 26#include <linux/module.h> 27#include <linux/slab.h> 28#include <linux/srcu.h> 29 30#include "rcu.h" 31#include "rcu_segcblist.h" 32 33/* Holdoff in nanoseconds for auto-expediting. */ 34#define DEFAULT_SRCU_EXP_HOLDOFF (25 * 1000) 35static ulong exp_holdoff = DEFAULT_SRCU_EXP_HOLDOFF; 36module_param(exp_holdoff, ulong, 0444); 37 38/* Overflow-check frequency. N bits roughly says every 2**N grace periods. */ 39static ulong counter_wrap_check = (ULONG_MAX >> 2); 40module_param(counter_wrap_check, ulong, 0444); 41 42/* 43 * Control conversion to SRCU_SIZE_BIG: 44 * 0: Don't convert at all. 45 * 1: Convert at init_srcu_struct() time. 46 * 2: Convert when rcutorture invokes srcu_torture_stats_print(). 47 * 3: Decide at boot time based on system shape (default). 48 * 0x1x: Convert when excessive contention encountered. 49 */ 50#define SRCU_SIZING_NONE 0 51#define SRCU_SIZING_INIT 1 52#define SRCU_SIZING_TORTURE 2 53#define SRCU_SIZING_AUTO 3 54#define SRCU_SIZING_CONTEND 0x10 55#define SRCU_SIZING_IS(x) ((convert_to_big & ~SRCU_SIZING_CONTEND) == x) 56#define SRCU_SIZING_IS_NONE() (SRCU_SIZING_IS(SRCU_SIZING_NONE)) 57#define SRCU_SIZING_IS_INIT() (SRCU_SIZING_IS(SRCU_SIZING_INIT)) 58#define SRCU_SIZING_IS_TORTURE() (SRCU_SIZING_IS(SRCU_SIZING_TORTURE)) 59#define SRCU_SIZING_IS_CONTEND() (convert_to_big & SRCU_SIZING_CONTEND) 60static int convert_to_big = SRCU_SIZING_AUTO; 61module_param(convert_to_big, int, 0444); 62 63/* Number of CPUs to trigger init_srcu_struct()-time transition to big. */ 64static int big_cpu_lim __read_mostly = 128; 65module_param(big_cpu_lim, int, 0444); 66 67/* Contention events per jiffy to initiate transition to big. */ 68static int small_contention_lim __read_mostly = 100; 69module_param(small_contention_lim, int, 0444); 70 71/* Early-boot callback-management, so early that no lock is required! */ 72static LIST_HEAD(srcu_boot_list); 73static bool __read_mostly srcu_init_done; 74 75static void srcu_invoke_callbacks(struct work_struct *work); 76static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay); 77static void process_srcu(struct work_struct *work); 78static void srcu_delay_timer(struct timer_list *t); 79 80/* Wrappers for lock acquisition and release, see raw_spin_lock_rcu_node(). */ 81#define spin_lock_rcu_node(p) \ 82do { \ 83 spin_lock(&ACCESS_PRIVATE(p, lock)); \ 84 smp_mb__after_unlock_lock(); \ 85} while (0) 86 87#define spin_unlock_rcu_node(p) spin_unlock(&ACCESS_PRIVATE(p, lock)) 88 89#define spin_lock_irq_rcu_node(p) \ 90do { \ 91 spin_lock_irq(&ACCESS_PRIVATE(p, lock)); \ 92 smp_mb__after_unlock_lock(); \ 93} while (0) 94 95#define spin_unlock_irq_rcu_node(p) \ 96 spin_unlock_irq(&ACCESS_PRIVATE(p, lock)) 97 98#define spin_lock_irqsave_rcu_node(p, flags) \ 99do { \ 100 spin_lock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \ 101 smp_mb__after_unlock_lock(); \ 102} while (0) 103 104#define spin_trylock_irqsave_rcu_node(p, flags) \ 105({ \ 106 bool ___locked = spin_trylock_irqsave(&ACCESS_PRIVATE(p, lock), flags); \ 107 \ 108 if (___locked) \ 109 smp_mb__after_unlock_lock(); \ 110 ___locked; \ 111}) 112 113#define spin_unlock_irqrestore_rcu_node(p, flags) \ 114 spin_unlock_irqrestore(&ACCESS_PRIVATE(p, lock), flags) \ 115 116/* 117 * Initialize SRCU per-CPU data. Note that statically allocated 118 * srcu_struct structures might already have srcu_read_lock() and 119 * srcu_read_unlock() running against them. So if the is_static 120 * parameter is set, don't initialize ->srcu_ctrs[].srcu_locks and 121 * ->srcu_ctrs[].srcu_unlocks. 122 */ 123static void init_srcu_struct_data(struct srcu_struct *ssp) 124{ 125 int cpu; 126 struct srcu_data *sdp; 127 128 /* 129 * Initialize the per-CPU srcu_data array, which feeds into the 130 * leaves of the srcu_node tree. 131 */ 132 for_each_possible_cpu(cpu) { 133 sdp = per_cpu_ptr(ssp->sda, cpu); 134 spin_lock_init(&ACCESS_PRIVATE(sdp, lock)); 135 rcu_segcblist_init(&sdp->srcu_cblist); 136 sdp->srcu_cblist_invoking = false; 137 sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq; 138 sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq; 139 sdp->srcu_barrier_head.next = &sdp->srcu_barrier_head; 140 sdp->mynode = NULL; 141 sdp->cpu = cpu; 142 INIT_WORK(&sdp->work, srcu_invoke_callbacks); 143 timer_setup(&sdp->delay_work, srcu_delay_timer, 0); 144 sdp->ssp = ssp; 145 } 146} 147 148/* Invalid seq state, used during snp node initialization */ 149#define SRCU_SNP_INIT_SEQ 0x2 150 151/* 152 * Check whether sequence number corresponding to snp node, 153 * is invalid. 154 */ 155static inline bool srcu_invl_snp_seq(unsigned long s) 156{ 157 return s == SRCU_SNP_INIT_SEQ; 158} 159 160/* 161 * Allocated and initialize SRCU combining tree. Returns @true if 162 * allocation succeeded and @false otherwise. 163 */ 164static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) 165{ 166 int cpu; 167 int i; 168 int level = 0; 169 int levelspread[RCU_NUM_LVLS]; 170 struct srcu_data *sdp; 171 struct srcu_node *snp; 172 struct srcu_node *snp_first; 173 174 /* Initialize geometry if it has not already been initialized. */ 175 rcu_init_geometry(); 176 ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags); 177 if (!ssp->srcu_sup->node) 178 return false; 179 180 /* Work out the overall tree geometry. */ 181 ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0]; 182 for (i = 1; i < rcu_num_lvls; i++) 183 ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1]; 184 rcu_init_levelspread(levelspread, num_rcu_lvl); 185 186 /* Each pass through this loop initializes one srcu_node structure. */ 187 srcu_for_each_node_breadth_first(ssp, snp) { 188 spin_lock_init(&ACCESS_PRIVATE(snp, lock)); 189 BUILD_BUG_ON(ARRAY_SIZE(snp->srcu_have_cbs) != 190 ARRAY_SIZE(snp->srcu_data_have_cbs)); 191 for (i = 0; i < ARRAY_SIZE(snp->srcu_have_cbs); i++) { 192 snp->srcu_have_cbs[i] = SRCU_SNP_INIT_SEQ; 193 snp->srcu_data_have_cbs[i] = 0; 194 } 195 snp->srcu_gp_seq_needed_exp = SRCU_SNP_INIT_SEQ; 196 snp->grplo = -1; 197 snp->grphi = -1; 198 if (snp == &ssp->srcu_sup->node[0]) { 199 /* Root node, special case. */ 200 snp->srcu_parent = NULL; 201 continue; 202 } 203 204 /* Non-root node. */ 205 if (snp == ssp->srcu_sup->level[level + 1]) 206 level++; 207 snp->srcu_parent = ssp->srcu_sup->level[level - 1] + 208 (snp - ssp->srcu_sup->level[level]) / 209 levelspread[level - 1]; 210 } 211 212 /* 213 * Initialize the per-CPU srcu_data array, which feeds into the 214 * leaves of the srcu_node tree. 215 */ 216 level = rcu_num_lvls - 1; 217 snp_first = ssp->srcu_sup->level[level]; 218 for_each_possible_cpu(cpu) { 219 sdp = per_cpu_ptr(ssp->sda, cpu); 220 sdp->mynode = &snp_first[cpu / levelspread[level]]; 221 for (snp = sdp->mynode; snp != NULL; snp = snp->srcu_parent) { 222 if (snp->grplo < 0) 223 snp->grplo = cpu; 224 snp->grphi = cpu; 225 } 226 sdp->grpmask = 1UL << (cpu - sdp->mynode->grplo); 227 } 228 smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER); 229 return true; 230} 231 232/* 233 * Initialize non-compile-time initialized fields, including the 234 * associated srcu_node and srcu_data structures. The is_static parameter 235 * tells us that ->sda has already been wired up to srcu_data. 236 */ 237static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) 238{ 239 if (!is_static) 240 ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL); 241 if (!ssp->srcu_sup) 242 return -ENOMEM; 243 if (!is_static) 244 spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); 245 ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL; 246 ssp->srcu_sup->node = NULL; 247 mutex_init(&ssp->srcu_sup->srcu_cb_mutex); 248 mutex_init(&ssp->srcu_sup->srcu_gp_mutex); 249 ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL; 250 ssp->srcu_sup->srcu_barrier_seq = 0; 251 mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); 252 atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0); 253 INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu); 254 ssp->srcu_sup->sda_is_static = is_static; 255 if (!is_static) { 256 ssp->sda = alloc_percpu(struct srcu_data); 257 ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0]; 258 } 259 if (!ssp->sda) 260 goto err_free_sup; 261 init_srcu_struct_data(ssp); 262 ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL; 263 ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); 264 if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { 265 if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) 266 goto err_free_sda; 267 WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG); 268 } 269 ssp->srcu_sup->srcu_ssp = ssp; 270 smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 271 SRCU_GP_SEQ_INITIAL_VAL); /* Init done. */ 272 return 0; 273 274err_free_sda: 275 if (!is_static) { 276 free_percpu(ssp->sda); 277 ssp->sda = NULL; 278 } 279err_free_sup: 280 if (!is_static) { 281 kfree(ssp->srcu_sup); 282 ssp->srcu_sup = NULL; 283 } 284 return -ENOMEM; 285} 286 287#ifdef CONFIG_DEBUG_LOCK_ALLOC 288 289static int 290__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) 291{ 292 /* Don't re-initialize a lock while it is held. */ 293 debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); 294 lockdep_init_map(&ssp->dep_map, name, key, 0); 295 return init_srcu_struct_fields(ssp, false); 296} 297 298int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) 299{ 300 ssp->srcu_reader_flavor = 0; 301 return __init_srcu_struct_common(ssp, name, key); 302} 303EXPORT_SYMBOL_GPL(__init_srcu_struct); 304 305int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) 306{ 307 ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; 308 return __init_srcu_struct_common(ssp, name, key); 309} 310EXPORT_SYMBOL_GPL(__init_srcu_struct_fast); 311 312int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, 313 struct lock_class_key *key) 314{ 315 ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; 316 return __init_srcu_struct_common(ssp, name, key); 317} 318EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); 319 320#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 321 322/** 323 * init_srcu_struct - initialize a sleep-RCU structure 324 * @ssp: structure to initialize. 325 * 326 * Use this in place of DEFINE_SRCU() and DEFINE_STATIC_SRCU() 327 * for non-static srcu_struct structures that are to be passed to 328 * srcu_read_lock(), srcu_read_lock_nmisafe(), and friends. It is necessary 329 * to invoke this on a given srcu_struct before passing that srcu_struct 330 * to any other function. Each srcu_struct represents a separate domain 331 * of SRCU protection. 332 */ 333int init_srcu_struct(struct srcu_struct *ssp) 334{ 335 ssp->srcu_reader_flavor = 0; 336 return init_srcu_struct_fields(ssp, false); 337} 338EXPORT_SYMBOL_GPL(init_srcu_struct); 339 340/** 341 * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure 342 * @ssp: structure to initialize. 343 * 344 * Use this in place of DEFINE_SRCU_FAST() and DEFINE_STATIC_SRCU_FAST() 345 * for non-static srcu_struct structures that are to be passed to 346 * srcu_read_lock_fast() and friends. It is necessary to invoke this on a 347 * given srcu_struct before passing that srcu_struct to any other function. 348 * Each srcu_struct represents a separate domain of SRCU protection. 349 */ 350int init_srcu_struct_fast(struct srcu_struct *ssp) 351{ 352 ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; 353 return init_srcu_struct_fields(ssp, false); 354} 355EXPORT_SYMBOL_GPL(init_srcu_struct_fast); 356 357/** 358 * init_srcu_struct_fast_updown - initialize a fast-reader up/down sleep-RCU structure 359 * @ssp: structure to initialize. 360 * 361 * Use this function in place of DEFINE_SRCU_FAST_UPDOWN() and 362 * DEFINE_STATIC_SRCU_FAST_UPDOWN() for non-static srcu_struct 363 * structures that are to be passed to srcu_read_lock_fast_updown(), 364 * srcu_down_read_fast(), and friends. It is necessary to invoke this on a 365 * given srcu_struct before passing that srcu_struct to any other function. 366 * Each srcu_struct represents a separate domain of SRCU protection. 367 */ 368int init_srcu_struct_fast_updown(struct srcu_struct *ssp) 369{ 370 ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; 371 return init_srcu_struct_fields(ssp, false); 372} 373EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown); 374 375#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ 376 377/* 378 * Initiate a transition to SRCU_SIZE_BIG with lock held. 379 */ 380static void __srcu_transition_to_big(struct srcu_struct *ssp) 381{ 382 lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); 383 smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC); 384} 385 386/* 387 * Initiate an idempotent transition to SRCU_SIZE_BIG. 388 */ 389static void srcu_transition_to_big(struct srcu_struct *ssp) 390{ 391 unsigned long flags; 392 393 /* Double-checked locking on ->srcu_size-state. */ 394 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) 395 return; 396 spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); 397 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) { 398 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); 399 return; 400 } 401 __srcu_transition_to_big(ssp); 402 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); 403} 404 405/* 406 * Check to see if the just-encountered contention event justifies 407 * a transition to SRCU_SIZE_BIG. 408 */ 409static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp) 410{ 411 unsigned long j; 412 413 if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state) 414 return; 415 j = jiffies; 416 if (ssp->srcu_sup->srcu_size_jiffies != j) { 417 ssp->srcu_sup->srcu_size_jiffies = j; 418 ssp->srcu_sup->srcu_n_lock_retries = 0; 419 } 420 if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim) 421 return; 422 __srcu_transition_to_big(ssp); 423} 424 425/* 426 * Acquire the specified srcu_data structure's ->lock, but check for 427 * excessive contention, which results in initiation of a transition 428 * to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module 429 * parameter permits this. 430 */ 431static void spin_lock_irqsave_sdp_contention(struct srcu_data *sdp, unsigned long *flags) 432{ 433 struct srcu_struct *ssp = sdp->ssp; 434 435 if (spin_trylock_irqsave_rcu_node(sdp, *flags)) 436 return; 437 spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); 438 spin_lock_irqsave_check_contention(ssp); 439 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags); 440 spin_lock_irqsave_rcu_node(sdp, *flags); 441} 442 443/* 444 * Acquire the specified srcu_struct structure's ->lock, but check for 445 * excessive contention, which results in initiation of a transition 446 * to SRCU_SIZE_BIG. But only if the srcutree.convert_to_big module 447 * parameter permits this. 448 */ 449static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags) 450{ 451 if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags)) 452 return; 453 spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags); 454 spin_lock_irqsave_check_contention(ssp); 455} 456 457/* 458 * First-use initialization of statically allocated srcu_struct 459 * structure. Wiring up the combining tree is more than can be 460 * done with compile-time initialization, so this check is added 461 * to each update-side SRCU primitive. Use ssp->lock, which -is- 462 * compile-time initialized, to resolve races involving multiple 463 * CPUs trying to garner first-use privileges. 464 */ 465static void check_init_srcu_struct(struct srcu_struct *ssp) 466{ 467 unsigned long flags; 468 469 /* The smp_load_acquire() pairs with the smp_store_release(). */ 470 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/ 471 return; /* Already initialized. */ 472 spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); 473 if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) { 474 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); 475 return; 476 } 477 init_srcu_struct_fields(ssp, true); 478 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); 479} 480 481/* 482 * Is the current or any upcoming grace period to be expedited? 483 */ 484static bool srcu_gp_is_expedited(struct srcu_struct *ssp) 485{ 486 struct srcu_usage *sup = ssp->srcu_sup; 487 488 return ULONG_CMP_LT(READ_ONCE(sup->srcu_gp_seq), READ_ONCE(sup->srcu_gp_seq_needed_exp)); 489} 490 491/* 492 * Computes approximate total of the readers' ->srcu_ctrs[].srcu_locks 493 * values for the rank of per-CPU counters specified by idx, and returns 494 * true if the caller did the proper barrier (gp), and if the count of 495 * the locks matches that of the unlocks passed in. 496 */ 497static bool srcu_readers_lock_idx(struct srcu_struct *ssp, int idx, bool gp, unsigned long unlocks) 498{ 499 int cpu; 500 unsigned long mask = 0; 501 unsigned long sum = 0; 502 503 for_each_possible_cpu(cpu) { 504 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); 505 506 sum += atomic_long_read(&sdp->srcu_ctrs[idx].srcu_locks); 507 if (IS_ENABLED(CONFIG_PROVE_RCU)) 508 mask = mask | READ_ONCE(sdp->srcu_reader_flavor); 509 } 510 WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && (mask & (mask - 1)), 511 "Mixed reader flavors for srcu_struct at %ps.\n", ssp); 512 if (mask & SRCU_READ_FLAVOR_SLOWGP && !gp) 513 return false; 514 return sum == unlocks; 515} 516 517/* 518 * Returns approximate total of the readers' ->srcu_ctrs[].srcu_unlocks 519 * values for the rank of per-CPU counters specified by idx. 520 */ 521static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx, unsigned long *rdm) 522{ 523 int cpu; 524 unsigned long mask = ssp->srcu_reader_flavor; 525 unsigned long sum = 0; 526 527 for_each_possible_cpu(cpu) { 528 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); 529 530 sum += atomic_long_read(&sdp->srcu_ctrs[idx].srcu_unlocks); 531 mask = mask | READ_ONCE(sdp->srcu_reader_flavor); 532 } 533 WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && (mask & (mask - 1)), 534 "Mixed reader flavors for srcu_struct at %ps.\n", ssp); 535 *rdm = mask; 536 return sum; 537} 538 539/* 540 * Return true if the number of pre-existing readers is determined to 541 * be zero. 542 */ 543static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) 544{ 545 bool did_gp; 546 unsigned long rdm; 547 unsigned long unlocks; 548 549 unlocks = srcu_readers_unlock_idx(ssp, idx, &rdm); 550 did_gp = !!(rdm & SRCU_READ_FLAVOR_SLOWGP); 551 552 /* 553 * Make sure that a lock is always counted if the corresponding 554 * unlock is counted. Needs to be a smp_mb() as the read side may 555 * contain a read from a variable that is written to before the 556 * synchronize_srcu() in the write side. In this case smp_mb()s 557 * A and B (or X and Y) act like the store buffering pattern. 558 * 559 * This smp_mb() also pairs with smp_mb() C (or, in the case of X, 560 * Z) to prevent accesses after the synchronize_srcu() from being 561 * executed before the grace period ends. 562 */ 563 if (!did_gp) 564 smp_mb(); /* A */ 565 else if (srcu_gp_is_expedited(ssp)) 566 synchronize_rcu_expedited(); /* X */ 567 else 568 synchronize_rcu(); /* X */ 569 570 /* 571 * If the locks are the same as the unlocks, then there must have 572 * been no readers on this index at some point in this function. 573 * But there might be more readers, as a task might have read 574 * the current ->srcu_ctrp but not yet have incremented its CPU's 575 * ->srcu_ctrs[idx].srcu_locks counter. In fact, it is possible 576 * that most of the tasks have been preempted between fetching 577 * ->srcu_ctrp and incrementing ->srcu_ctrs[idx].srcu_locks. And 578 * there could be almost (ULONG_MAX / sizeof(struct task_struct)) 579 * tasks in a system whose address space was fully populated 580 * with memory. Call this quantity Nt. 581 * 582 * So suppose that the updater is preempted at this 583 * point in the code for a long time. That now-preempted 584 * updater has already flipped ->srcu_ctrp (possibly during 585 * the preceding grace period), done an smp_mb() (again, 586 * possibly during the preceding grace period), and summed up 587 * the ->srcu_ctrs[idx].srcu_unlocks counters. How many times 588 * can a given one of the aforementioned Nt tasks increment the 589 * old ->srcu_ctrp value's ->srcu_ctrs[idx].srcu_locks counter, 590 * in the absence of nesting? 591 * 592 * It can clearly do so once, given that it has already fetched 593 * the old value of ->srcu_ctrp and is just about to use that 594 * value to index its increment of ->srcu_ctrs[idx].srcu_locks. 595 * But as soon as it leaves that SRCU read-side critical section, 596 * it will increment ->srcu_ctrs[idx].srcu_unlocks, which must 597 * follow the updater's above read from that same value. Thus, 598 as soon the reading task does an smp_mb() and a later fetch from 599 * ->srcu_ctrp, that task will be guaranteed to get the new index. 600 * Except that the increment of ->srcu_ctrs[idx].srcu_unlocks 601 * in __srcu_read_unlock() is after the smp_mb(), and the fetch 602 * from ->srcu_ctrp in __srcu_read_lock() is before the smp_mb(). 603 * Thus, that task might not see the new value of ->srcu_ctrp until 604 * the -second- __srcu_read_lock(), which in turn means that this 605 * task might well increment ->srcu_ctrs[idx].srcu_locks for the 606 * old value of ->srcu_ctrp twice, not just once. 607 * 608 * However, it is important to note that a given smp_mb() takes 609 * effect not just for the task executing it, but also for any 610 * later task running on that same CPU. 611 * 612 * That is, there can be almost Nt + Nc further increments 613 * of ->srcu_ctrs[idx].srcu_locks for the old index, where Nc 614 * is the number of CPUs. But this is OK because the size of 615 * the task_struct structure limits the value of Nt and current 616 * systems limit Nc to a few thousand. 617 * 618 * OK, but what about nesting? This does impose a limit on 619 * nesting of half of the size of the task_struct structure 620 * (measured in bytes), which should be sufficient. A late 2022 621 * TREE01 rcutorture run reported this size to be no less than 622 * 9408 bytes, allowing up to 4704 levels of nesting, which is 623 * comfortably beyond excessive. Especially on 64-bit systems, 624 * which are unlikely to be configured with an address space fully 625 * populated with memory, at least not anytime soon. 626 */ 627 return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); 628} 629 630/** 631 * srcu_readers_active - returns true if there are readers. and false 632 * otherwise 633 * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). 634 * 635 * Note that this is not an atomic primitive, and can therefore suffer 636 * severe errors when invoked on an active srcu_struct. That said, it 637 * can be useful as an error check at cleanup time. 638 */ 639static bool srcu_readers_active(struct srcu_struct *ssp) 640{ 641 int cpu; 642 unsigned long sum = 0; 643 644 for_each_possible_cpu(cpu) { 645 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); 646 647 sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); 648 sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); 649 sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); 650 sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); 651 } 652 return sum; 653} 654 655/* 656 * We use an adaptive strategy for synchronize_srcu() and especially for 657 * synchronize_srcu_expedited(). We spin for a fixed time period 658 * (defined below, boot time configurable) to allow SRCU readers to exit 659 * their read-side critical sections. If there are still some readers 660 * after one jiffy, we repeatedly block for one jiffy time periods. 661 * The blocking time is increased as the grace-period age increases, 662 * with max blocking time capped at 10 jiffies. 663 */ 664#define SRCU_DEFAULT_RETRY_CHECK_DELAY 5 665 666static ulong srcu_retry_check_delay = SRCU_DEFAULT_RETRY_CHECK_DELAY; 667module_param(srcu_retry_check_delay, ulong, 0444); 668 669#define SRCU_INTERVAL 1 // Base delay if no expedited GPs pending. 670#define SRCU_MAX_INTERVAL 10 // Maximum incremental delay from slow readers. 671 672#define SRCU_DEFAULT_MAX_NODELAY_PHASE_LO 3UL // Lowmark on default per-GP-phase 673 // no-delay instances. 674#define SRCU_DEFAULT_MAX_NODELAY_PHASE_HI 1000UL // Highmark on default per-GP-phase 675 // no-delay instances. 676 677#define SRCU_UL_CLAMP_LO(val, low) ((val) > (low) ? (val) : (low)) 678#define SRCU_UL_CLAMP_HI(val, high) ((val) < (high) ? (val) : (high)) 679#define SRCU_UL_CLAMP(val, low, high) SRCU_UL_CLAMP_HI(SRCU_UL_CLAMP_LO((val), (low)), (high)) 680// per-GP-phase no-delay instances adjusted to allow non-sleeping poll upto 681// one jiffies time duration. Mult by 2 is done to factor in the srcu_get_delay() 682// called from process_srcu(). 683#define SRCU_DEFAULT_MAX_NODELAY_PHASE_ADJUSTED \ 684 (2UL * USEC_PER_SEC / HZ / SRCU_DEFAULT_RETRY_CHECK_DELAY) 685 686// Maximum per-GP-phase consecutive no-delay instances. 687#define SRCU_DEFAULT_MAX_NODELAY_PHASE \ 688 SRCU_UL_CLAMP(SRCU_DEFAULT_MAX_NODELAY_PHASE_ADJUSTED, \ 689 SRCU_DEFAULT_MAX_NODELAY_PHASE_LO, \ 690 SRCU_DEFAULT_MAX_NODELAY_PHASE_HI) 691 692static ulong srcu_max_nodelay_phase = SRCU_DEFAULT_MAX_NODELAY_PHASE; 693module_param(srcu_max_nodelay_phase, ulong, 0444); 694 695// Maximum consecutive no-delay instances. 696#define SRCU_DEFAULT_MAX_NODELAY (SRCU_DEFAULT_MAX_NODELAY_PHASE > 100 ? \ 697 SRCU_DEFAULT_MAX_NODELAY_PHASE : 100) 698 699static ulong srcu_max_nodelay = SRCU_DEFAULT_MAX_NODELAY; 700module_param(srcu_max_nodelay, ulong, 0444); 701 702/* 703 * Return grace-period delay, zero if there are expedited grace 704 * periods pending, SRCU_INTERVAL otherwise. 705 */ 706static unsigned long srcu_get_delay(struct srcu_struct *ssp) 707{ 708 unsigned long gpstart; 709 unsigned long j; 710 unsigned long jbase = SRCU_INTERVAL; 711 struct srcu_usage *sup = ssp->srcu_sup; 712 713 lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); 714 if (srcu_gp_is_expedited(ssp)) 715 jbase = 0; 716 if (rcu_seq_state(READ_ONCE(sup->srcu_gp_seq))) { 717 j = jiffies - 1; 718 gpstart = READ_ONCE(sup->srcu_gp_start); 719 if (time_after(j, gpstart)) 720 jbase += j - gpstart; 721 if (!jbase) { 722 ASSERT_EXCLUSIVE_WRITER(sup->srcu_n_exp_nodelay); 723 WRITE_ONCE(sup->srcu_n_exp_nodelay, READ_ONCE(sup->srcu_n_exp_nodelay) + 1); 724 if (READ_ONCE(sup->srcu_n_exp_nodelay) > srcu_max_nodelay_phase) 725 jbase = 1; 726 } 727 } 728 return jbase > SRCU_MAX_INTERVAL ? SRCU_MAX_INTERVAL : jbase; 729} 730 731/** 732 * cleanup_srcu_struct - deconstruct a sleep-RCU structure 733 * @ssp: structure to clean up. 734 * 735 * Must invoke this after you are finished using a given srcu_struct that 736 * was initialized via init_srcu_struct(), else you leak memory. 737 */ 738void cleanup_srcu_struct(struct srcu_struct *ssp) 739{ 740 int cpu; 741 unsigned long delay; 742 struct srcu_usage *sup = ssp->srcu_sup; 743 744 spin_lock_irq_rcu_node(ssp->srcu_sup); 745 delay = srcu_get_delay(ssp); 746 spin_unlock_irq_rcu_node(ssp->srcu_sup); 747 if (WARN_ON(!delay)) 748 return; /* Just leak it! */ 749 if (WARN_ON(srcu_readers_active(ssp))) 750 return; /* Just leak it! */ 751 flush_delayed_work(&sup->work); 752 for_each_possible_cpu(cpu) { 753 struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); 754 755 timer_delete_sync(&sdp->delay_work); 756 flush_work(&sdp->work); 757 if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist))) 758 return; /* Forgot srcu_barrier(), so just leak it! */ 759 } 760 if (WARN_ON(rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)) != SRCU_STATE_IDLE) || 761 WARN_ON(rcu_seq_current(&sup->srcu_gp_seq) != sup->srcu_gp_seq_needed) || 762 WARN_ON(srcu_readers_active(ssp))) { 763 pr_info("%s: Active srcu_struct %p read state: %d gp state: %lu/%lu\n", 764 __func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)), 765 rcu_seq_current(&sup->srcu_gp_seq), sup->srcu_gp_seq_needed); 766 return; // Caller forgot to stop doing call_srcu()? 767 // Or caller invoked start_poll_synchronize_srcu() 768 // and then cleanup_srcu_struct() before that grace 769 // period ended? 770 } 771 kfree(sup->node); 772 sup->node = NULL; 773 sup->srcu_size_state = SRCU_SIZE_SMALL; 774 if (!sup->sda_is_static) { 775 free_percpu(ssp->sda); 776 ssp->sda = NULL; 777 kfree(sup); 778 ssp->srcu_sup = NULL; 779 } 780} 781EXPORT_SYMBOL_GPL(cleanup_srcu_struct); 782 783/* 784 * Check for consistent reader flavor. 785 */ 786void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor) 787{ 788 int old_read_flavor; 789 struct srcu_data *sdp; 790 791 /* NMI-unsafe use in NMI is a bad sign, as is multi-bit read_flavor values. */ 792 WARN_ON_ONCE((read_flavor != SRCU_READ_FLAVOR_NMI) && in_nmi()); 793 WARN_ON_ONCE(read_flavor & (read_flavor - 1)); 794 795 sdp = raw_cpu_ptr(ssp->sda); 796 old_read_flavor = READ_ONCE(sdp->srcu_reader_flavor); 797 WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor); 798 WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor && 799 old_read_flavor != ssp->srcu_reader_flavor); 800 WARN_ON_ONCE(read_flavor == SRCU_READ_FLAVOR_FAST && !ssp->srcu_reader_flavor); 801 if (!old_read_flavor) { 802 old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor); 803 if (!old_read_flavor) 804 return; 805 } 806 WARN_ONCE(old_read_flavor != read_flavor, "CPU %d old state %d new state %d\n", sdp->cpu, old_read_flavor, read_flavor); 807} 808EXPORT_SYMBOL_GPL(__srcu_check_read_flavor); 809 810/* 811 * Counts the new reader in the appropriate per-CPU element of the 812 * srcu_struct. 813 * Returns a guaranteed non-negative index that must be passed to the 814 * matching __srcu_read_unlock(). 815 */ 816int __srcu_read_lock(struct srcu_struct *ssp) 817{ 818 struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp); 819 820 this_cpu_inc(scp->srcu_locks.counter); 821 smp_mb(); /* B */ /* Avoid leaking the critical section. */ 822 return __srcu_ptr_to_ctr(ssp, scp); 823} 824EXPORT_SYMBOL_GPL(__srcu_read_lock); 825 826/* 827 * Removes the count for the old reader from the appropriate per-CPU 828 * element of the srcu_struct. Note that this may well be a different 829 * CPU than that which was incremented by the corresponding srcu_read_lock(). 830 */ 831void __srcu_read_unlock(struct srcu_struct *ssp, int idx) 832{ 833 smp_mb(); /* C */ /* Avoid leaking the critical section. */ 834 this_cpu_inc(__srcu_ctr_to_ptr(ssp, idx)->srcu_unlocks.counter); 835} 836EXPORT_SYMBOL_GPL(__srcu_read_unlock); 837 838#ifdef CONFIG_NEED_SRCU_NMI_SAFE 839 840/* 841 * Counts the new reader in the appropriate per-CPU element of the 842 * srcu_struct, but in an NMI-safe manner using RMW atomics. 843 * Returns an index that must be passed to the matching srcu_read_unlock(). 844 */ 845int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) 846{ 847 struct srcu_ctr __percpu *scpp = READ_ONCE(ssp->srcu_ctrp); 848 struct srcu_ctr *scp = raw_cpu_ptr(scpp); 849 850 atomic_long_inc(&scp->srcu_locks); 851 smp_mb__after_atomic(); /* B */ /* Avoid leaking the critical section. */ 852 return __srcu_ptr_to_ctr(ssp, scpp); 853} 854EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe); 855 856/* 857 * Removes the count for the old reader from the appropriate per-CPU 858 * element of the srcu_struct. Note that this may well be a different 859 * CPU than that which was incremented by the corresponding srcu_read_lock(). 860 */ 861void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) 862{ 863 smp_mb__before_atomic(); /* C */ /* Avoid leaking the critical section. */ 864 atomic_long_inc(&raw_cpu_ptr(__srcu_ctr_to_ptr(ssp, idx))->srcu_unlocks); 865} 866EXPORT_SYMBOL_GPL(__srcu_read_unlock_nmisafe); 867 868#endif // CONFIG_NEED_SRCU_NMI_SAFE 869 870/* 871 * Start an SRCU grace period. 872 */ 873static void srcu_gp_start(struct srcu_struct *ssp) 874{ 875 int state; 876 877 lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock)); 878 WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)); 879 WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies); 880 WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0); 881 smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */ 882 rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq); 883 state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq); 884 WARN_ON_ONCE(state != SRCU_STATE_SCAN1); 885} 886 887 888static void srcu_delay_timer(struct timer_list *t) 889{ 890 struct srcu_data *sdp = container_of(t, struct srcu_data, delay_work); 891 892 queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); 893} 894 895static void srcu_queue_delayed_work_on(struct srcu_data *sdp, 896 unsigned long delay) 897{ 898 if (!delay) { 899 queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); 900 return; 901 } 902 903 timer_reduce(&sdp->delay_work, jiffies + delay); 904} 905 906/* 907 * Schedule callback invocation for the specified srcu_data structure, 908 * if possible, on the corresponding CPU. 909 */ 910static void srcu_schedule_cbs_sdp(struct srcu_data *sdp, unsigned long delay) 911{ 912 srcu_queue_delayed_work_on(sdp, delay); 913} 914 915/* 916 * Schedule callback invocation for all srcu_data structures associated 917 * with the specified srcu_node structure that have callbacks for the 918 * just-completed grace period, the one corresponding to idx. If possible, 919 * schedule this invocation on the corresponding CPUs. 920 */ 921static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp, 922 unsigned long mask, unsigned long delay) 923{ 924 int cpu; 925 926 for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) { 927 if (!(mask & (1UL << (cpu - snp->grplo)))) 928 continue; 929 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay); 930 } 931} 932 933/* 934 * Note the end of an SRCU grace period. Initiates callback invocation 935 * and starts a new grace period if needed. 936 * 937 * The ->srcu_cb_mutex acquisition does not protect any data, but 938 * instead prevents more than one grace period from starting while we 939 * are initiating callback invocation. This allows the ->srcu_have_cbs[] 940 * array to have a finite number of elements. 941 */ 942static void srcu_gp_end(struct srcu_struct *ssp) 943{ 944 unsigned long cbdelay = 1; 945 bool cbs; 946 bool last_lvl; 947 int cpu; 948 unsigned long gpseq; 949 int idx; 950 unsigned long mask; 951 struct srcu_data *sdp; 952 unsigned long sgsne; 953 struct srcu_node *snp; 954 int ss_state; 955 struct srcu_usage *sup = ssp->srcu_sup; 956 957 /* Prevent more than one additional grace period. */ 958 mutex_lock(&sup->srcu_cb_mutex); 959 960 /* End the current grace period. */ 961 spin_lock_irq_rcu_node(sup); 962 idx = rcu_seq_state(sup->srcu_gp_seq); 963 WARN_ON_ONCE(idx != SRCU_STATE_SCAN2); 964 if (srcu_gp_is_expedited(ssp)) 965 cbdelay = 0; 966 967 WRITE_ONCE(sup->srcu_last_gp_end, ktime_get_mono_fast_ns()); 968 rcu_seq_end(&sup->srcu_gp_seq); 969 gpseq = rcu_seq_current(&sup->srcu_gp_seq); 970 if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq)) 971 WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq); 972 spin_unlock_irq_rcu_node(sup); 973 mutex_unlock(&sup->srcu_gp_mutex); 974 /* A new grace period can start at this point. But only one. */ 975 976 /* Initiate callback invocation as needed. */ 977 ss_state = smp_load_acquire(&sup->srcu_size_state); 978 if (ss_state < SRCU_SIZE_WAIT_BARRIER) { 979 srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()), 980 cbdelay); 981 } else { 982 idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs); 983 srcu_for_each_node_breadth_first(ssp, snp) { 984 spin_lock_irq_rcu_node(snp); 985 cbs = false; 986 last_lvl = snp >= sup->level[rcu_num_lvls - 1]; 987 if (last_lvl) 988 cbs = ss_state < SRCU_SIZE_BIG || snp->srcu_have_cbs[idx] == gpseq; 989 snp->srcu_have_cbs[idx] = gpseq; 990 rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1); 991 sgsne = snp->srcu_gp_seq_needed_exp; 992 if (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, gpseq)) 993 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq); 994 if (ss_state < SRCU_SIZE_BIG) 995 mask = ~0; 996 else 997 mask = snp->srcu_data_have_cbs[idx]; 998 snp->srcu_data_have_cbs[idx] = 0; 999 spin_unlock_irq_rcu_node(snp); 1000 if (cbs) 1001 srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay); 1002 } 1003 } 1004 1005 /* Occasionally prevent srcu_data counter wrap. */ 1006 if (!(gpseq & counter_wrap_check)) 1007 for_each_possible_cpu(cpu) { 1008 sdp = per_cpu_ptr(ssp->sda, cpu); 1009 spin_lock_irq_rcu_node(sdp); 1010 if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100)) 1011 sdp->srcu_gp_seq_needed = gpseq; 1012 if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100)) 1013 sdp->srcu_gp_seq_needed_exp = gpseq; 1014 spin_unlock_irq_rcu_node(sdp); 1015 } 1016 1017 /* Callback initiation done, allow grace periods after next. */ 1018 mutex_unlock(&sup->srcu_cb_mutex); 1019 1020 /* Start a new grace period if needed. */ 1021 spin_lock_irq_rcu_node(sup); 1022 gpseq = rcu_seq_current(&sup->srcu_gp_seq); 1023 if (!rcu_seq_state(gpseq) && 1024 ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) { 1025 srcu_gp_start(ssp); 1026 spin_unlock_irq_rcu_node(sup); 1027 srcu_reschedule(ssp, 0); 1028 } else { 1029 spin_unlock_irq_rcu_node(sup); 1030 } 1031 1032 /* Transition to big if needed. */ 1033 if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) { 1034 if (ss_state == SRCU_SIZE_ALLOC) 1035 init_srcu_struct_nodes(ssp, GFP_KERNEL); 1036 else 1037 smp_store_release(&sup->srcu_size_state, ss_state + 1); 1038 } 1039} 1040 1041/* 1042 * Funnel-locking scheme to scalably mediate many concurrent expedited 1043 * grace-period requests. This function is invoked for the first known 1044 * expedited request for a grace period that has already been requested, 1045 * but without expediting. To start a completely new grace period, 1046 * whether expedited or not, use srcu_funnel_gp_start() instead. 1047 */ 1048static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp, 1049 unsigned long s) 1050{ 1051 unsigned long flags; 1052 unsigned long sgsne; 1053 1054 if (snp) 1055 for (; snp != NULL; snp = snp->srcu_parent) { 1056 sgsne = READ_ONCE(snp->srcu_gp_seq_needed_exp); 1057 if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) || 1058 (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s))) 1059 return; 1060 spin_lock_irqsave_rcu_node(snp, flags); 1061 sgsne = snp->srcu_gp_seq_needed_exp; 1062 if (!srcu_invl_snp_seq(sgsne) && ULONG_CMP_GE(sgsne, s)) { 1063 spin_unlock_irqrestore_rcu_node(snp, flags); 1064 return; 1065 } 1066 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s); 1067 spin_unlock_irqrestore_rcu_node(snp, flags); 1068 } 1069 spin_lock_irqsave_ssp_contention(ssp, &flags); 1070 if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s)) 1071 WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s); 1072 spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); 1073} 1074 1075/* 1076 * Funnel-locking scheme to scalably mediate many concurrent grace-period 1077 * requests. The winner has to do the work of actually starting grace 1078 * period s. Losers must either ensure that their desired grace-period 1079 * number is recorded on at least their leaf srcu_node structure, or they 1080 * must take steps to invoke their own callbacks. 1081 * 1082 * Note that this function also does the work of srcu_funnel_exp_start(), 1083 * in some cases by directly invoking it. 1084 * 1085 * The srcu read lock should be hold around this function. And s is a seq snap 1086 * after holding that lock. 1087 */ 1088static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, 1089 unsigned long s, bool do_norm) 1090{ 1091 unsigned long flags; 1092 int idx = rcu_seq_ctr(s) % ARRAY_SIZE(sdp->mynode->srcu_have_cbs); 1093 unsigned long sgsne; 1094 struct srcu_node *snp; 1095 struct srcu_node *snp_leaf; 1096 unsigned long snp_seq; 1097 struct srcu_usage *sup = ssp->srcu_sup; 1098 1099 /* Ensure that snp node tree is fully initialized before traversing it */ 1100 if (smp_load_acquire(&sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) 1101 snp_leaf = NULL; 1102 else 1103 snp_leaf = sdp->mynode; 1104 1105 if (snp_leaf) 1106 /* Each pass through the loop does one level of the srcu_node tree. */ 1107 for (snp = snp_leaf; snp != NULL; snp = snp->srcu_parent) { 1108 if (WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && snp != snp_leaf) 1109 return; /* GP already done and CBs recorded. */ 1110 spin_lock_irqsave_rcu_node(snp, flags); 1111 snp_seq = snp->srcu_have_cbs[idx]; 1112 if (!srcu_invl_snp_seq(snp_seq) && ULONG_CMP_GE(snp_seq, s)) { 1113 if (snp == snp_leaf && snp_seq == s) 1114 snp->srcu_data_have_cbs[idx] |= sdp->grpmask; 1115 spin_unlock_irqrestore_rcu_node(snp, flags); 1116 if (snp == snp_leaf && snp_seq != s) { 1117 srcu_schedule_cbs_sdp(sdp, do_norm ? SRCU_INTERVAL : 0); 1118 return; 1119 } 1120 if (!do_norm) 1121 srcu_funnel_exp_start(ssp, snp, s); 1122 return; 1123 } 1124 snp->srcu_have_cbs[idx] = s; 1125 if (snp == snp_leaf) 1126 snp->srcu_data_have_cbs[idx] |= sdp->grpmask; 1127 sgsne = snp->srcu_gp_seq_needed_exp; 1128 if (!do_norm && (srcu_invl_snp_seq(sgsne) || ULONG_CMP_LT(sgsne, s))) 1129 WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s); 1130 spin_unlock_irqrestore_rcu_node(snp, flags); 1131 } 1132 1133 /* Top of tree, must ensure the grace period will be started. */ 1134 spin_lock_irqsave_ssp_contention(ssp, &flags); 1135 if (ULONG_CMP_LT(sup->srcu_gp_seq_needed, s)) { 1136 /* 1137 * Record need for grace period s. Pair with load 1138 * acquire setting up for initialization. 1139 */ 1140 smp_store_release(&sup->srcu_gp_seq_needed, s); /*^^^*/ 1141 } 1142 if (!do_norm && ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, s)) 1143 WRITE_ONCE(sup->srcu_gp_seq_needed_exp, s); 1144 1145 /* If grace period not already in progress, start it. */ 1146 if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && 1147 rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) { 1148 srcu_gp_start(ssp); 1149 1150 // And how can that list_add() in the "else" clause 1151 // possibly be safe for concurrent execution? Well, 1152 // it isn't. And it does not have to be. After all, it 1153 // can only be executed during early boot when there is only 1154 // the one boot CPU running with interrupts still disabled. 1155 if (likely(srcu_init_done)) 1156 queue_delayed_work(rcu_gp_wq, &sup->work, 1157 !!srcu_get_delay(ssp)); 1158 else if (list_empty(&sup->work.work.entry)) 1159 list_add(&sup->work.work.entry, &srcu_boot_list); 1160 } 1161 spin_unlock_irqrestore_rcu_node(sup, flags); 1162} 1163 1164/* 1165 * Wait until all readers counted by array index idx complete, but 1166 * loop an additional time if there is an expedited grace period pending. 1167 * The caller must ensure that ->srcu_ctrp is not changed while checking. 1168 */ 1169static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) 1170{ 1171 unsigned long curdelay; 1172 1173 spin_lock_irq_rcu_node(ssp->srcu_sup); 1174 curdelay = !srcu_get_delay(ssp); 1175 spin_unlock_irq_rcu_node(ssp->srcu_sup); 1176 1177 for (;;) { 1178 if (srcu_readers_active_idx_check(ssp, idx)) 1179 return true; 1180 if ((--trycount + curdelay) <= 0) 1181 return false; 1182 udelay(srcu_retry_check_delay); 1183 } 1184} 1185 1186/* 1187 * Increment the ->srcu_ctrp counter so that future SRCU readers will 1188 * use the other rank of the ->srcu_(un)lock_count[] arrays. This allows 1189 * us to wait for pre-existing readers in a starvation-free manner. 1190 */ 1191static void srcu_flip(struct srcu_struct *ssp) 1192{ 1193 /* 1194 * Because the flip of ->srcu_ctrp is executed only if the 1195 * preceding call to srcu_readers_active_idx_check() found that 1196 * the ->srcu_ctrs[].srcu_unlocks and ->srcu_ctrs[].srcu_locks sums 1197 * matched and because that summing uses atomic_long_read(), 1198 * there is ordering due to a control dependency between that 1199 * summing and the WRITE_ONCE() in this call to srcu_flip(). 1200 * This ordering ensures that if this updater saw a given reader's 1201 * increment from __srcu_read_lock(), that reader was using a value 1202 * of ->srcu_ctrp from before the previous call to srcu_flip(), 1203 * which should be quite rare. This ordering thus helps forward 1204 * progress because the grace period could otherwise be delayed 1205 * by additional calls to __srcu_read_lock() using that old (soon 1206 * to be new) value of ->srcu_ctrp. 1207 * 1208 * This sum-equality check and ordering also ensures that if 1209 * a given call to __srcu_read_lock() uses the new value of 1210 * ->srcu_ctrp, this updater's earlier scans cannot have seen 1211 * that reader's increments, which is all to the good, because 1212 * this grace period need not wait on that reader. After all, 1213 * if those earlier scans had seen that reader, there would have 1214 * been a sum mismatch and this code would not be reached. 1215 * 1216 * This means that the following smp_mb() is redundant, but 1217 * it stays until either (1) Compilers learn about this sort of 1218 * control dependency or (2) Some production workload running on 1219 * a production system is unduly delayed by this slowpath smp_mb(). 1220 * Except for _lite() readers, where it is inoperative, which 1221 * means that it is a good thing that it is redundant. 1222 */ 1223 smp_mb(); /* E */ /* Pairs with B and C. */ 1224 1225 WRITE_ONCE(ssp->srcu_ctrp, 1226 &ssp->sda->srcu_ctrs[!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0])]); 1227 1228 /* 1229 * Ensure that if the updater misses an __srcu_read_unlock() 1230 * increment, that task's __srcu_read_lock() following its next 1231 * __srcu_read_lock() or __srcu_read_unlock() will see the above 1232 * counter update. Note that both this memory barrier and the 1233 * one in srcu_readers_active_idx_check() provide the guarantee 1234 * for __srcu_read_lock(). 1235 * 1236 * Note that this is a performance optimization, in which we spend 1237 * an otherwise unnecessary smp_mb() in order to reduce the number 1238 * of full per-CPU-variable scans in srcu_readers_lock_idx() and 1239 * srcu_readers_unlock_idx(). But this performance optimization 1240 * is not so optimal for SRCU-fast, where we would be spending 1241 * not smp_mb(), but rather synchronize_rcu(). At the same time, 1242 * the overhead of the smp_mb() is in the noise, so there is no 1243 * point in omitting it in the SRCU-fast case. So the same code 1244 * is executed either way. 1245 */ 1246 smp_mb(); /* D */ /* Pairs with C. */ 1247} 1248 1249/* 1250 * If SRCU is likely idle, in other words, the next SRCU grace period 1251 * should be expedited, return true, otherwise return false. Except that 1252 * in the presence of _lite() readers, always return false. 1253 * 1254 * Note that it is OK for several current from-idle requests for a new 1255 * grace period from idle to specify expediting because they will all end 1256 * up requesting the same grace period anyhow. So no loss. 1257 * 1258 * Note also that if any CPU (including the current one) is still invoking 1259 * callbacks, this function will nevertheless say "idle". This is not 1260 * ideal, but the overhead of checking all CPUs' callback lists is even 1261 * less ideal, especially on large systems. Furthermore, the wakeup 1262 * can happen before the callback is fully removed, so we have no choice 1263 * but to accept this type of error. 1264 * 1265 * This function is also subject to counter-wrap errors, but let's face 1266 * it, if this function was preempted for enough time for the counters 1267 * to wrap, it really doesn't matter whether or not we expedite the grace 1268 * period. The extra overhead of a needlessly expedited grace period is 1269 * negligible when amortized over that time period, and the extra latency 1270 * of a needlessly non-expedited grace period is similarly negligible. 1271 */ 1272static bool srcu_should_expedite(struct srcu_struct *ssp) 1273{ 1274 unsigned long curseq; 1275 unsigned long flags; 1276 struct srcu_data *sdp; 1277 unsigned long t; 1278 unsigned long tlast; 1279 1280 check_init_srcu_struct(ssp); 1281 /* If _lite() readers, don't do unsolicited expediting. */ 1282 if (this_cpu_read(ssp->sda->srcu_reader_flavor) & SRCU_READ_FLAVOR_SLOWGP) 1283 return false; 1284 /* If the local srcu_data structure has callbacks, not idle. */ 1285 sdp = raw_cpu_ptr(ssp->sda); 1286 spin_lock_irqsave_rcu_node(sdp, flags); 1287 if (rcu_segcblist_pend_cbs(&sdp->srcu_cblist)) { 1288 spin_unlock_irqrestore_rcu_node(sdp, flags); 1289 return false; /* Callbacks already present, so not idle. */ 1290 } 1291 spin_unlock_irqrestore_rcu_node(sdp, flags); 1292 1293 /* 1294 * No local callbacks, so probabilistically probe global state. 1295 * Exact information would require acquiring locks, which would 1296 * kill scalability, hence the probabilistic nature of the probe. 1297 */ 1298 1299 /* First, see if enough time has passed since the last GP. */ 1300 t = ktime_get_mono_fast_ns(); 1301 tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end); 1302 if (exp_holdoff == 0 || 1303 time_in_range_open(t, tlast, tlast + exp_holdoff)) 1304 return false; /* Too soon after last GP. */ 1305 1306 /* Next, check for probable idleness. */ 1307 curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq); 1308 smp_mb(); /* Order ->srcu_gp_seq with ->srcu_gp_seq_needed. */ 1309 if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed))) 1310 return false; /* Grace period in progress, so not idle. */ 1311 smp_mb(); /* Order ->srcu_gp_seq with prior access. */ 1312 if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)) 1313 return false; /* GP # changed, so not idle. */ 1314 return true; /* With reasonable probability, idle! */ 1315} 1316 1317/* 1318 * SRCU callback function to leak a callback. 1319 */ 1320static void srcu_leak_callback(struct rcu_head *rhp) 1321{ 1322} 1323 1324/* 1325 * Start an SRCU grace period, and also queue the callback if non-NULL. 1326 */ 1327static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, 1328 struct rcu_head *rhp, bool do_norm) 1329{ 1330 unsigned long flags; 1331 int idx; 1332 bool needexp = false; 1333 bool needgp = false; 1334 unsigned long s; 1335 struct srcu_data *sdp; 1336 struct srcu_node *sdp_mynode; 1337 int ss_state; 1338 1339 check_init_srcu_struct(ssp); 1340 /* 1341 * While starting a new grace period, make sure we are in an 1342 * SRCU read-side critical section so that the grace-period 1343 * sequence number cannot wrap around in the meantime. 1344 */ 1345 idx = __srcu_read_lock_nmisafe(ssp); 1346 ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state); 1347 if (ss_state < SRCU_SIZE_WAIT_CALL) 1348 sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id()); 1349 else 1350 sdp = raw_cpu_ptr(ssp->sda); 1351 spin_lock_irqsave_sdp_contention(sdp, &flags); 1352 if (rhp) 1353 rcu_segcblist_enqueue(&sdp->srcu_cblist, rhp); 1354 /* 1355 * It's crucial to capture the snapshot 's' for acceleration before 1356 * reading the current gp_seq that is used for advancing. This is 1357 * essential because if the acceleration snapshot is taken after a 1358 * failed advancement attempt, there's a risk that a grace period may 1359 * conclude and a new one may start in the interim. If the snapshot is 1360 * captured after this sequence of events, the acceleration snapshot 's' 1361 * could be excessively advanced, leading to acceleration failure. 1362 * In such a scenario, an 'acceleration leak' can occur, where new 1363 * callbacks become indefinitely stuck in the RCU_NEXT_TAIL segment. 1364 * Also note that encountering advancing failures is a normal 1365 * occurrence when the grace period for RCU_WAIT_TAIL is in progress. 1366 * 1367 * To see this, consider the following events which occur if 1368 * rcu_seq_snap() were to be called after advance: 1369 * 1370 * 1) The RCU_WAIT_TAIL segment has callbacks (gp_num = X + 4) and the 1371 * RCU_NEXT_READY_TAIL also has callbacks (gp_num = X + 8). 1372 * 1373 * 2) The grace period for RCU_WAIT_TAIL is seen as started but not 1374 * completed so rcu_seq_current() returns X + SRCU_STATE_SCAN1. 1375 * 1376 * 3) This value is passed to rcu_segcblist_advance() which can't move 1377 * any segment forward and fails. 1378 * 1379 * 4) srcu_gp_start_if_needed() still proceeds with callback acceleration. 1380 * But then the call to rcu_seq_snap() observes the grace period for the 1381 * RCU_WAIT_TAIL segment as completed and the subsequent one for the 1382 * RCU_NEXT_READY_TAIL segment as started (ie: X + 4 + SRCU_STATE_SCAN1) 1383 * so it returns a snapshot of the next grace period, which is X + 12. 1384 * 1385 * 5) The value of X + 12 is passed to rcu_segcblist_accelerate() but the 1386 * freshly enqueued callback in RCU_NEXT_TAIL can't move to 1387 * RCU_NEXT_READY_TAIL which already has callbacks for a previous grace 1388 * period (gp_num = X + 8). So acceleration fails. 1389 */ 1390 s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq); 1391 if (rhp) { 1392 rcu_segcblist_advance(&sdp->srcu_cblist, 1393 rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); 1394 /* 1395 * Acceleration can never fail because the base current gp_seq 1396 * used for acceleration is <= the value of gp_seq used for 1397 * advancing. This means that RCU_NEXT_TAIL segment will 1398 * always be able to be emptied by the acceleration into the 1399 * RCU_NEXT_READY_TAIL or RCU_WAIT_TAIL segments. 1400 */ 1401 WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s)); 1402 } 1403 if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { 1404 sdp->srcu_gp_seq_needed = s; 1405 needgp = true; 1406 } 1407 if (!do_norm && ULONG_CMP_LT(sdp->srcu_gp_seq_needed_exp, s)) { 1408 sdp->srcu_gp_seq_needed_exp = s; 1409 needexp = true; 1410 } 1411 spin_unlock_irqrestore_rcu_node(sdp, flags); 1412 1413 /* Ensure that snp node tree is fully initialized before traversing it */ 1414 if (ss_state < SRCU_SIZE_WAIT_BARRIER) 1415 sdp_mynode = NULL; 1416 else 1417 sdp_mynode = sdp->mynode; 1418 1419 if (needgp) 1420 srcu_funnel_gp_start(ssp, sdp, s, do_norm); 1421 else if (needexp) 1422 srcu_funnel_exp_start(ssp, sdp_mynode, s); 1423 __srcu_read_unlock_nmisafe(ssp, idx); 1424 return s; 1425} 1426 1427/* 1428 * Enqueue an SRCU callback on the srcu_data structure associated with 1429 * the current CPU and the specified srcu_struct structure, initiating 1430 * grace-period processing if it is not already running. 1431 * 1432 * Note that all CPUs must agree that the grace period extended beyond 1433 * all pre-existing SRCU read-side critical section. On systems with 1434 * more than one CPU, this means that when "func()" is invoked, each CPU 1435 * is guaranteed to have executed a full memory barrier since the end of 1436 * its last corresponding SRCU read-side critical section whose beginning 1437 * preceded the call to call_srcu(). It also means that each CPU executing 1438 * an SRCU read-side critical section that continues beyond the start of 1439 * "func()" must have executed a memory barrier after the call_srcu() 1440 * but before the beginning of that SRCU read-side critical section. 1441 * Note that these guarantees include CPUs that are offline, idle, or 1442 * executing in user mode, as well as CPUs that are executing in the kernel. 1443 * 1444 * Furthermore, if CPU A invoked call_srcu() and CPU B invoked the 1445 * resulting SRCU callback function "func()", then both CPU A and CPU 1446 * B are guaranteed to execute a full memory barrier during the time 1447 * interval between the call to call_srcu() and the invocation of "func()". 1448 * This guarantee applies even if CPU A and CPU B are the same CPU (but 1449 * again only if the system has more than one CPU). 1450 * 1451 * Of course, these guarantees apply only for invocations of call_srcu(), 1452 * srcu_read_lock(), and srcu_read_unlock() that are all passed the same 1453 * srcu_struct structure. 1454 */ 1455static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, 1456 rcu_callback_t func, bool do_norm) 1457{ 1458 if (debug_rcu_head_queue(rhp)) { 1459 /* Probable double call_srcu(), so leak the callback. */ 1460 WRITE_ONCE(rhp->func, srcu_leak_callback); 1461 WARN_ONCE(1, "call_srcu(): Leaked duplicate callback\n"); 1462 return; 1463 } 1464 rhp->func = func; 1465 (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); 1466} 1467 1468/** 1469 * call_srcu() - Queue a callback for invocation after an SRCU grace period 1470 * @ssp: srcu_struct in queue the callback 1471 * @rhp: structure to be used for queueing the SRCU callback. 1472 * @func: function to be invoked after the SRCU grace period 1473 * 1474 * The callback function will be invoked some time after a full SRCU 1475 * grace period elapses, in other words after all pre-existing SRCU 1476 * read-side critical sections have completed. However, the callback 1477 * function might well execute concurrently with other SRCU read-side 1478 * critical sections that started after call_srcu() was invoked. SRCU 1479 * read-side critical sections are delimited by srcu_read_lock() and 1480 * srcu_read_unlock(), and may be nested. 1481 * 1482 * The callback will be invoked from process context, but with bh 1483 * disabled. The callback function must therefore be fast and must 1484 * not block. 1485 * 1486 * See the description of call_rcu() for more detailed information on 1487 * memory ordering guarantees. 1488 */ 1489void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, 1490 rcu_callback_t func) 1491{ 1492 __call_srcu(ssp, rhp, func, true); 1493} 1494EXPORT_SYMBOL_GPL(call_srcu); 1495 1496/* 1497 * Helper function for synchronize_srcu() and synchronize_srcu_expedited(). 1498 */ 1499static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) 1500{ 1501 struct rcu_synchronize rcu; 1502 1503 srcu_lock_sync(&ssp->dep_map); 1504 1505 RCU_LOCKDEP_WARN(lockdep_is_held(ssp) || 1506 lock_is_held(&rcu_bh_lock_map) || 1507 lock_is_held(&rcu_lock_map) || 1508 lock_is_held(&rcu_sched_lock_map), 1509 "Illegal synchronize_srcu() in same-type SRCU (or in RCU) read-side critical section"); 1510 1511 if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) 1512 return; 1513 might_sleep(); 1514 check_init_srcu_struct(ssp); 1515 init_completion(&rcu.completion); 1516 init_rcu_head_on_stack(&rcu.head); 1517 __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); 1518 wait_for_completion(&rcu.completion); 1519 destroy_rcu_head_on_stack(&rcu.head); 1520 1521 /* 1522 * Make sure that later code is ordered after the SRCU grace 1523 * period. This pairs with the spin_lock_irq_rcu_node() 1524 * in srcu_invoke_callbacks(). Unlike Tree RCU, this is needed 1525 * because the current CPU might have been totally uninvolved with 1526 * (and thus unordered against) that grace period. 1527 */ 1528 smp_mb(); 1529} 1530 1531/** 1532 * synchronize_srcu_expedited - Brute-force SRCU grace period 1533 * @ssp: srcu_struct with which to synchronize. 1534 * 1535 * Wait for an SRCU grace period to elapse, but be more aggressive about 1536 * spinning rather than blocking when waiting. 1537 * 1538 * Note that synchronize_srcu_expedited() has the same deadlock and 1539 * memory-ordering properties as does synchronize_srcu(). 1540 */ 1541void synchronize_srcu_expedited(struct srcu_struct *ssp) 1542{ 1543 __synchronize_srcu(ssp, rcu_gp_is_normal()); 1544} 1545EXPORT_SYMBOL_GPL(synchronize_srcu_expedited); 1546 1547/** 1548 * synchronize_srcu - wait for prior SRCU read-side critical-section completion 1549 * @ssp: srcu_struct with which to synchronize. 1550 * 1551 * Wait for the count to drain to zero of both indexes. To avoid the 1552 * possible starvation of synchronize_srcu(), it waits for the count of 1553 * the index=!(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]) to drain to zero 1554 * at first, and then flip the ->srcu_ctrp and wait for the count of the 1555 * other index. 1556 * 1557 * Can block; must be called from process context. 1558 * 1559 * Note that it is illegal to call synchronize_srcu() from the corresponding 1560 * SRCU read-side critical section; doing so will result in deadlock. 1561 * However, it is perfectly legal to call synchronize_srcu() on one 1562 * srcu_struct from some other srcu_struct's read-side critical section, 1563 * as long as the resulting graph of srcu_structs is acyclic. 1564 * 1565 * There are memory-ordering constraints implied by synchronize_srcu(). 1566 * On systems with more than one CPU, when synchronize_srcu() returns, 1567 * each CPU is guaranteed to have executed a full memory barrier since 1568 * the end of its last corresponding SRCU read-side critical section 1569 * whose beginning preceded the call to synchronize_srcu(). In addition, 1570 * each CPU having an SRCU read-side critical section that extends beyond 1571 * the return from synchronize_srcu() is guaranteed to have executed a 1572 * full memory barrier after the beginning of synchronize_srcu() and before 1573 * the beginning of that SRCU read-side critical section. Note that these 1574 * guarantees include CPUs that are offline, idle, or executing in user mode, 1575 * as well as CPUs that are executing in the kernel. 1576 * 1577 * Furthermore, if CPU A invoked synchronize_srcu(), which returned 1578 * to its caller on CPU B, then both CPU A and CPU B are guaranteed 1579 * to have executed a full memory barrier during the execution of 1580 * synchronize_srcu(). This guarantee applies even if CPU A and CPU B 1581 * are the same CPU, but again only if the system has more than one CPU. 1582 * 1583 * Of course, these memory-ordering guarantees apply only when 1584 * synchronize_srcu(), srcu_read_lock(), and srcu_read_unlock() are 1585 * passed the same srcu_struct structure. 1586 * 1587 * Implementation of these memory-ordering guarantees is similar to 1588 * that of synchronize_rcu(). 1589 * 1590 * If SRCU is likely idle as determined by srcu_should_expedite(), 1591 * expedite the first request. This semantic was provided by Classic SRCU, 1592 * and is relied upon by its users, so TREE SRCU must also provide it. 1593 * Note that detecting idleness is heuristic and subject to both false 1594 * positives and negatives. 1595 */ 1596void synchronize_srcu(struct srcu_struct *ssp) 1597{ 1598 if (srcu_should_expedite(ssp) || rcu_gp_is_expedited()) 1599 synchronize_srcu_expedited(ssp); 1600 else 1601 __synchronize_srcu(ssp, true); 1602} 1603EXPORT_SYMBOL_GPL(synchronize_srcu); 1604 1605/** 1606 * get_state_synchronize_srcu - Provide an end-of-grace-period cookie 1607 * @ssp: srcu_struct to provide cookie for. 1608 * 1609 * This function returns a cookie that can be passed to 1610 * poll_state_synchronize_srcu(), which will return true if a full grace 1611 * period has elapsed in the meantime. It is the caller's responsibility 1612 * to make sure that grace period happens, for example, by invoking 1613 * call_srcu() after return from get_state_synchronize_srcu(). 1614 */ 1615unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp) 1616{ 1617 // Any prior manipulation of SRCU-protected data must happen 1618 // before the load from ->srcu_gp_seq. 1619 smp_mb(); 1620 return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq); 1621} 1622EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); 1623 1624/** 1625 * start_poll_synchronize_srcu - Provide cookie and start grace period 1626 * @ssp: srcu_struct to provide cookie for. 1627 * 1628 * This function returns a cookie that can be passed to 1629 * poll_state_synchronize_srcu(), which will return true if a full grace 1630 * period has elapsed in the meantime. Unlike get_state_synchronize_srcu(), 1631 * this function also ensures that any needed SRCU grace period will be 1632 * started. This convenience does come at a cost in terms of CPU overhead. 1633 */ 1634unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) 1635{ 1636 return srcu_gp_start_if_needed(ssp, NULL, true); 1637} 1638EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); 1639 1640/** 1641 * poll_state_synchronize_srcu - Has cookie's grace period ended? 1642 * @ssp: srcu_struct to provide cookie for. 1643 * @cookie: Return value from get_state_synchronize_srcu() or start_poll_synchronize_srcu(). 1644 * 1645 * This function takes the cookie that was returned from either 1646 * get_state_synchronize_srcu() or start_poll_synchronize_srcu(), and 1647 * returns @true if an SRCU grace period elapsed since the time that the 1648 * cookie was created. 1649 * 1650 * Because cookies are finite in size, wrapping/overflow is possible. 1651 * This is more pronounced on 32-bit systems where cookies are 32 bits, 1652 * where in theory wrapping could happen in about 14 hours assuming 1653 * 25-microsecond expedited SRCU grace periods. However, a more likely 1654 * overflow lower bound is on the order of 24 days in the case of 1655 * one-millisecond SRCU grace periods. Of course, wrapping in a 64-bit 1656 * system requires geologic timespans, as in more than seven million years 1657 * even for expedited SRCU grace periods. 1658 * 1659 * Wrapping/overflow is much more of an issue for CONFIG_SMP=n systems 1660 * that also have CONFIG_PREEMPTION=n, which selects Tiny SRCU. This uses 1661 * a 16-bit cookie, which rcutorture routinely wraps in a matter of a 1662 * few minutes. If this proves to be a problem, this counter will be 1663 * expanded to the same size as for Tree SRCU. 1664 */ 1665bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie) 1666{ 1667 if (cookie != SRCU_GET_STATE_COMPLETED && 1668 !rcu_seq_done_exact(&ssp->srcu_sup->srcu_gp_seq, cookie)) 1669 return false; 1670 // Ensure that the end of the SRCU grace period happens before 1671 // any subsequent code that the caller might execute. 1672 smp_mb(); // ^^^ 1673 return true; 1674} 1675EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu); 1676 1677/* 1678 * Callback function for srcu_barrier() use. 1679 */ 1680static void srcu_barrier_cb(struct rcu_head *rhp) 1681{ 1682 struct srcu_data *sdp; 1683 struct srcu_struct *ssp; 1684 1685 rhp->next = rhp; // Mark the callback as having been invoked. 1686 sdp = container_of(rhp, struct srcu_data, srcu_barrier_head); 1687 ssp = sdp->ssp; 1688 if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt)) 1689 complete(&ssp->srcu_sup->srcu_barrier_completion); 1690} 1691 1692/* 1693 * Enqueue an srcu_barrier() callback on the specified srcu_data 1694 * structure's ->cblist. but only if that ->cblist already has at least one 1695 * callback enqueued. Note that if a CPU already has callbacks enqueue, 1696 * it must have already registered the need for a future grace period, 1697 * so all we need do is enqueue a callback that will use the same grace 1698 * period as the last callback already in the queue. 1699 */ 1700static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp) 1701{ 1702 spin_lock_irq_rcu_node(sdp); 1703 atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt); 1704 sdp->srcu_barrier_head.func = srcu_barrier_cb; 1705 debug_rcu_head_queue(&sdp->srcu_barrier_head); 1706 if (!rcu_segcblist_entrain(&sdp->srcu_cblist, 1707 &sdp->srcu_barrier_head)) { 1708 debug_rcu_head_unqueue(&sdp->srcu_barrier_head); 1709 atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt); 1710 } 1711 spin_unlock_irq_rcu_node(sdp); 1712} 1713 1714/** 1715 * srcu_barrier - Wait until all in-flight call_srcu() callbacks complete. 1716 * @ssp: srcu_struct on which to wait for in-flight callbacks. 1717 */ 1718void srcu_barrier(struct srcu_struct *ssp) 1719{ 1720 int cpu; 1721 int idx; 1722 unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq); 1723 1724 check_init_srcu_struct(ssp); 1725 mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex); 1726 if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) { 1727 smp_mb(); /* Force ordering following return. */ 1728 mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex); 1729 return; /* Someone else did our work for us. */ 1730 } 1731 rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq); 1732 init_completion(&ssp->srcu_sup->srcu_barrier_completion); 1733 1734 /* Initial count prevents reaching zero until all CBs are posted. */ 1735 atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1); 1736 1737 idx = __srcu_read_lock_nmisafe(ssp); 1738 if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER) 1739 srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, get_boot_cpu_id())); 1740 else 1741 for_each_possible_cpu(cpu) 1742 srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu)); 1743 __srcu_read_unlock_nmisafe(ssp, idx); 1744 1745 /* Remove the initial count, at which point reaching zero can happen. */ 1746 if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt)) 1747 complete(&ssp->srcu_sup->srcu_barrier_completion); 1748 wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion); 1749 1750 rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq); 1751 mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex); 1752} 1753EXPORT_SYMBOL_GPL(srcu_barrier); 1754 1755/* Callback for srcu_expedite_current() usage. */ 1756static void srcu_expedite_current_cb(struct rcu_head *rhp) 1757{ 1758 unsigned long flags; 1759 bool needcb = false; 1760 struct srcu_data *sdp = container_of(rhp, struct srcu_data, srcu_ec_head); 1761 1762 spin_lock_irqsave_sdp_contention(sdp, &flags); 1763 if (sdp->srcu_ec_state == SRCU_EC_IDLE) { 1764 WARN_ON_ONCE(1); 1765 } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) { 1766 sdp->srcu_ec_state = SRCU_EC_IDLE; 1767 } else { 1768 WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST); 1769 sdp->srcu_ec_state = SRCU_EC_PENDING; 1770 needcb = true; 1771 } 1772 spin_unlock_irqrestore_rcu_node(sdp, flags); 1773 // If needed, requeue ourselves as an expedited SRCU callback. 1774 if (needcb) 1775 __call_srcu(sdp->ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false); 1776} 1777 1778/** 1779 * srcu_expedite_current - Expedite the current SRCU grace period 1780 * @ssp: srcu_struct to expedite. 1781 * 1782 * Cause the current SRCU grace period to become expedited. The grace 1783 * period following the current one might also be expedited. If there is 1784 * no current grace period, one might be created. If the current grace 1785 * period is currently sleeping, that sleep will complete before expediting 1786 * will take effect. 1787 */ 1788void srcu_expedite_current(struct srcu_struct *ssp) 1789{ 1790 unsigned long flags; 1791 bool needcb = false; 1792 struct srcu_data *sdp; 1793 1794 migrate_disable(); 1795 sdp = this_cpu_ptr(ssp->sda); 1796 spin_lock_irqsave_sdp_contention(sdp, &flags); 1797 if (sdp->srcu_ec_state == SRCU_EC_IDLE) { 1798 sdp->srcu_ec_state = SRCU_EC_PENDING; 1799 needcb = true; 1800 } else if (sdp->srcu_ec_state == SRCU_EC_PENDING) { 1801 sdp->srcu_ec_state = SRCU_EC_REPOST; 1802 } else { 1803 WARN_ON_ONCE(sdp->srcu_ec_state != SRCU_EC_REPOST); 1804 } 1805 spin_unlock_irqrestore_rcu_node(sdp, flags); 1806 // If needed, queue an expedited SRCU callback. 1807 if (needcb) 1808 __call_srcu(ssp, &sdp->srcu_ec_head, srcu_expedite_current_cb, false); 1809 migrate_enable(); 1810} 1811EXPORT_SYMBOL_GPL(srcu_expedite_current); 1812 1813/** 1814 * srcu_batches_completed - return batches completed. 1815 * @ssp: srcu_struct on which to report batch completion. 1816 * 1817 * Report the number of batches, correlated with, but not necessarily 1818 * precisely the same as, the number of grace periods that have elapsed. 1819 */ 1820unsigned long srcu_batches_completed(struct srcu_struct *ssp) 1821{ 1822 return READ_ONCE(ssp->srcu_sup->srcu_gp_seq); 1823} 1824EXPORT_SYMBOL_GPL(srcu_batches_completed); 1825 1826/* 1827 * Core SRCU state machine. Push state bits of ->srcu_gp_seq 1828 * to SRCU_STATE_SCAN2, and invoke srcu_gp_end() when scan has 1829 * completed in that state. 1830 */ 1831static void srcu_advance_state(struct srcu_struct *ssp) 1832{ 1833 int idx; 1834 1835 mutex_lock(&ssp->srcu_sup->srcu_gp_mutex); 1836 1837 /* 1838 * Because readers might be delayed for an extended period after 1839 * fetching ->srcu_ctrp for their index, at any point in time there 1840 * might well be readers using both idx=0 and idx=1. We therefore 1841 * need to wait for readers to clear from both index values before 1842 * invoking a callback. 1843 * 1844 * The load-acquire ensures that we see the accesses performed 1845 * by the prior grace period. 1846 */ 1847 idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */ 1848 if (idx == SRCU_STATE_IDLE) { 1849 spin_lock_irq_rcu_node(ssp->srcu_sup); 1850 if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { 1851 WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)); 1852 spin_unlock_irq_rcu_node(ssp->srcu_sup); 1853 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); 1854 return; 1855 } 1856 idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)); 1857 if (idx == SRCU_STATE_IDLE) 1858 srcu_gp_start(ssp); 1859 spin_unlock_irq_rcu_node(ssp->srcu_sup); 1860 if (idx != SRCU_STATE_IDLE) { 1861 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); 1862 return; /* Someone else started the grace period. */ 1863 } 1864 } 1865 1866 if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) { 1867 idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]); 1868 if (!try_check_zero(ssp, idx, 1)) { 1869 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); 1870 return; /* readers present, retry later. */ 1871 } 1872 srcu_flip(ssp); 1873 spin_lock_irq_rcu_node(ssp->srcu_sup); 1874 rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2); 1875 ssp->srcu_sup->srcu_n_exp_nodelay = 0; 1876 spin_unlock_irq_rcu_node(ssp->srcu_sup); 1877 } 1878 1879 if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) { 1880 1881 /* 1882 * SRCU read-side critical sections are normally short, 1883 * so check at least twice in quick succession after a flip. 1884 */ 1885 idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]); 1886 if (!try_check_zero(ssp, idx, 2)) { 1887 mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); 1888 return; /* readers present, retry later. */ 1889 } 1890 ssp->srcu_sup->srcu_n_exp_nodelay = 0; 1891 srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */ 1892 } 1893} 1894 1895/* 1896 * Invoke a limited number of SRCU callbacks that have passed through 1897 * their grace period. If there are more to do, SRCU will reschedule 1898 * the workqueue. Note that needed memory barriers have been executed 1899 * in this task's context by srcu_readers_active_idx_check(). 1900 */ 1901static void srcu_invoke_callbacks(struct work_struct *work) 1902{ 1903 long len; 1904 bool more; 1905 struct rcu_cblist ready_cbs; 1906 struct rcu_head *rhp; 1907 struct srcu_data *sdp; 1908 struct srcu_struct *ssp; 1909 1910 sdp = container_of(work, struct srcu_data, work); 1911 1912 ssp = sdp->ssp; 1913 rcu_cblist_init(&ready_cbs); 1914 spin_lock_irq_rcu_node(sdp); 1915 WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL)); 1916 rcu_segcblist_advance(&sdp->srcu_cblist, 1917 rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); 1918 /* 1919 * Although this function is theoretically re-entrant, concurrent 1920 * callbacks invocation is disallowed to avoid executing an SRCU barrier 1921 * too early. 1922 */ 1923 if (sdp->srcu_cblist_invoking || 1924 !rcu_segcblist_ready_cbs(&sdp->srcu_cblist)) { 1925 spin_unlock_irq_rcu_node(sdp); 1926 return; /* Someone else on the job or nothing to do. */ 1927 } 1928 1929 /* We are on the job! Extract and invoke ready callbacks. */ 1930 sdp->srcu_cblist_invoking = true; 1931 rcu_segcblist_extract_done_cbs(&sdp->srcu_cblist, &ready_cbs); 1932 len = ready_cbs.len; 1933 spin_unlock_irq_rcu_node(sdp); 1934 rhp = rcu_cblist_dequeue(&ready_cbs); 1935 for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) { 1936 debug_rcu_head_unqueue(rhp); 1937 debug_rcu_head_callback(rhp); 1938 local_bh_disable(); 1939 rhp->func(rhp); 1940 local_bh_enable(); 1941 } 1942 WARN_ON_ONCE(ready_cbs.len); 1943 1944 /* 1945 * Update counts, accelerate new callbacks, and if needed, 1946 * schedule another round of callback invocation. 1947 */ 1948 spin_lock_irq_rcu_node(sdp); 1949 rcu_segcblist_add_len(&sdp->srcu_cblist, -len); 1950 sdp->srcu_cblist_invoking = false; 1951 more = rcu_segcblist_ready_cbs(&sdp->srcu_cblist); 1952 spin_unlock_irq_rcu_node(sdp); 1953 /* An SRCU barrier or callbacks from previous nesting work pending */ 1954 if (more) 1955 srcu_schedule_cbs_sdp(sdp, 0); 1956} 1957 1958/* 1959 * Finished one round of SRCU grace period. Start another if there are 1960 * more SRCU callbacks queued, otherwise put SRCU into not-running state. 1961 */ 1962static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) 1963{ 1964 bool pushgp = true; 1965 1966 spin_lock_irq_rcu_node(ssp->srcu_sup); 1967 if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { 1968 if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) { 1969 /* All requests fulfilled, time to go idle. */ 1970 pushgp = false; 1971 } 1972 } else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) { 1973 /* Outstanding request and no GP. Start one. */ 1974 srcu_gp_start(ssp); 1975 } 1976 spin_unlock_irq_rcu_node(ssp->srcu_sup); 1977 1978 if (pushgp) 1979 queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay); 1980} 1981 1982/* 1983 * This is the work-queue function that handles SRCU grace periods. 1984 */ 1985static void process_srcu(struct work_struct *work) 1986{ 1987 unsigned long curdelay; 1988 unsigned long j; 1989 struct srcu_struct *ssp; 1990 struct srcu_usage *sup; 1991 1992 sup = container_of(work, struct srcu_usage, work.work); 1993 ssp = sup->srcu_ssp; 1994 1995 srcu_advance_state(ssp); 1996 spin_lock_irq_rcu_node(ssp->srcu_sup); 1997 curdelay = srcu_get_delay(ssp); 1998 spin_unlock_irq_rcu_node(ssp->srcu_sup); 1999 if (curdelay) { 2000 WRITE_ONCE(sup->reschedule_count, 0); 2001 } else { 2002 j = jiffies; 2003 if (READ_ONCE(sup->reschedule_jiffies) == j) { 2004 ASSERT_EXCLUSIVE_WRITER(sup->reschedule_count); 2005 WRITE_ONCE(sup->reschedule_count, READ_ONCE(sup->reschedule_count) + 1); 2006 if (READ_ONCE(sup->reschedule_count) > srcu_max_nodelay) 2007 curdelay = 1; 2008 } else { 2009 WRITE_ONCE(sup->reschedule_count, 1); 2010 WRITE_ONCE(sup->reschedule_jiffies, j); 2011 } 2012 } 2013 srcu_reschedule(ssp, curdelay); 2014} 2015 2016void srcutorture_get_gp_data(struct srcu_struct *ssp, int *flags, 2017 unsigned long *gp_seq) 2018{ 2019 *flags = 0; 2020 *gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq); 2021} 2022EXPORT_SYMBOL_GPL(srcutorture_get_gp_data); 2023 2024static const char * const srcu_size_state_name[] = { 2025 "SRCU_SIZE_SMALL", 2026 "SRCU_SIZE_ALLOC", 2027 "SRCU_SIZE_WAIT_BARRIER", 2028 "SRCU_SIZE_WAIT_CALL", 2029 "SRCU_SIZE_WAIT_CBS1", 2030 "SRCU_SIZE_WAIT_CBS2", 2031 "SRCU_SIZE_WAIT_CBS3", 2032 "SRCU_SIZE_WAIT_CBS4", 2033 "SRCU_SIZE_BIG", 2034 "SRCU_SIZE_???", 2035}; 2036 2037void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) 2038{ 2039 int cpu; 2040 int idx; 2041 unsigned long s0 = 0, s1 = 0; 2042 int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state); 2043 int ss_state_idx = ss_state; 2044 2045 idx = ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]; 2046 if (ss_state < 0 || ss_state >= ARRAY_SIZE(srcu_size_state_name)) 2047 ss_state_idx = ARRAY_SIZE(srcu_size_state_name) - 1; 2048 pr_alert("%s%s Tree SRCU g%ld state %d (%s)", 2049 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state, 2050 srcu_size_state_name[ss_state_idx]); 2051 if (!ssp->sda) { 2052 // Called after cleanup_srcu_struct(), perhaps. 2053 pr_cont(" No per-CPU srcu_data structures (->sda == NULL).\n"); 2054 } else { 2055 pr_cont(" per-CPU(idx=%d):", idx); 2056 for_each_possible_cpu(cpu) { 2057 unsigned long l0, l1; 2058 unsigned long u0, u1; 2059 long c0, c1; 2060 struct srcu_data *sdp; 2061 2062 sdp = per_cpu_ptr(ssp->sda, cpu); 2063 u0 = data_race(atomic_long_read(&sdp->srcu_ctrs[!idx].srcu_unlocks)); 2064 u1 = data_race(atomic_long_read(&sdp->srcu_ctrs[idx].srcu_unlocks)); 2065 2066 /* 2067 * Make sure that a lock is always counted if the corresponding 2068 * unlock is counted. 2069 */ 2070 smp_rmb(); 2071 2072 l0 = data_race(atomic_long_read(&sdp->srcu_ctrs[!idx].srcu_locks)); 2073 l1 = data_race(atomic_long_read(&sdp->srcu_ctrs[idx].srcu_locks)); 2074 2075 c0 = l0 - u0; 2076 c1 = l1 - u1; 2077 pr_cont(" %d(%ld,%ld %c)", 2078 cpu, c0, c1, 2079 "C."[rcu_segcblist_empty(&sdp->srcu_cblist)]); 2080 s0 += c0; 2081 s1 += c1; 2082 } 2083 pr_cont(" T(%ld,%ld)\n", s0, s1); 2084 } 2085 if (SRCU_SIZING_IS_TORTURE()) 2086 srcu_transition_to_big(ssp); 2087} 2088EXPORT_SYMBOL_GPL(srcu_torture_stats_print); 2089 2090static int __init srcu_bootup_announce(void) 2091{ 2092 pr_info("Hierarchical SRCU implementation.\n"); 2093 if (exp_holdoff != DEFAULT_SRCU_EXP_HOLDOFF) 2094 pr_info("\tNon-default auto-expedite holdoff of %lu ns.\n", exp_holdoff); 2095 if (srcu_retry_check_delay != SRCU_DEFAULT_RETRY_CHECK_DELAY) 2096 pr_info("\tNon-default retry check delay of %lu us.\n", srcu_retry_check_delay); 2097 if (srcu_max_nodelay != SRCU_DEFAULT_MAX_NODELAY) 2098 pr_info("\tNon-default max no-delay of %lu.\n", srcu_max_nodelay); 2099 pr_info("\tMax phase no-delay instances is %lu.\n", srcu_max_nodelay_phase); 2100 return 0; 2101} 2102early_initcall(srcu_bootup_announce); 2103 2104void __init srcu_init(void) 2105{ 2106 struct srcu_usage *sup; 2107 2108 /* Decide on srcu_struct-size strategy. */ 2109 if (SRCU_SIZING_IS(SRCU_SIZING_AUTO)) { 2110 if (nr_cpu_ids >= big_cpu_lim) { 2111 convert_to_big = SRCU_SIZING_INIT; // Don't bother waiting for contention. 2112 pr_info("%s: Setting srcu_struct sizes to big.\n", __func__); 2113 } else { 2114 convert_to_big = SRCU_SIZING_NONE | SRCU_SIZING_CONTEND; 2115 pr_info("%s: Setting srcu_struct sizes based on contention.\n", __func__); 2116 } 2117 } 2118 2119 /* 2120 * Once that is set, call_srcu() can follow the normal path and 2121 * queue delayed work. This must follow RCU workqueues creation 2122 * and timers initialization. 2123 */ 2124 srcu_init_done = true; 2125 while (!list_empty(&srcu_boot_list)) { 2126 sup = list_first_entry(&srcu_boot_list, struct srcu_usage, 2127 work.work.entry); 2128 list_del_init(&sup->work.work.entry); 2129 if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && 2130 sup->srcu_size_state == SRCU_SIZE_SMALL) 2131 sup->srcu_size_state = SRCU_SIZE_ALLOC; 2132 queue_work(rcu_gp_wq, &sup->work.work); 2133 } 2134} 2135 2136#ifdef CONFIG_MODULES 2137 2138/* Initialize any global-scope srcu_struct structures used by this module. */ 2139static int srcu_module_coming(struct module *mod) 2140{ 2141 int i; 2142 struct srcu_struct *ssp; 2143 struct srcu_struct **sspp = mod->srcu_struct_ptrs; 2144 2145 for (i = 0; i < mod->num_srcu_structs; i++) { 2146 ssp = *(sspp++); 2147 ssp->sda = alloc_percpu(struct srcu_data); 2148 if (WARN_ON_ONCE(!ssp->sda)) 2149 return -ENOMEM; 2150 ssp->srcu_ctrp = &ssp->sda->srcu_ctrs[0]; 2151 } 2152 return 0; 2153} 2154 2155/* Clean up any global-scope srcu_struct structures used by this module. */ 2156static void srcu_module_going(struct module *mod) 2157{ 2158 int i; 2159 struct srcu_struct *ssp; 2160 struct srcu_struct **sspp = mod->srcu_struct_ptrs; 2161 2162 for (i = 0; i < mod->num_srcu_structs; i++) { 2163 ssp = *(sspp++); 2164 if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) && 2165 !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static)) 2166 cleanup_srcu_struct(ssp); 2167 if (!WARN_ON(srcu_readers_active(ssp))) 2168 free_percpu(ssp->sda); 2169 } 2170} 2171 2172/* Handle one module, either coming or going. */ 2173static int srcu_module_notify(struct notifier_block *self, 2174 unsigned long val, void *data) 2175{ 2176 struct module *mod = data; 2177 int ret = 0; 2178 2179 switch (val) { 2180 case MODULE_STATE_COMING: 2181 ret = srcu_module_coming(mod); 2182 break; 2183 case MODULE_STATE_GOING: 2184 srcu_module_going(mod); 2185 break; 2186 default: 2187 break; 2188 } 2189 return ret; 2190} 2191 2192static struct notifier_block srcu_module_nb = { 2193 .notifier_call = srcu_module_notify, 2194 .priority = 0, 2195}; 2196 2197static __init int init_srcu_module_notifier(void) 2198{ 2199 int ret; 2200 2201 ret = register_module_notifier(&srcu_module_nb); 2202 if (ret) 2203 pr_warn("Failed to register srcu module notifier\n"); 2204 return ret; 2205} 2206late_initcall(init_srcu_module_notifier); 2207 2208#endif /* #ifdef CONFIG_MODULES */
~/
Read-only file preview.