FILE
core.c
File preview with syntax highlighting.
linux-6.19.14/kernel/sched/core.c
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * kernel/sched/core.c 4 * 5 * Core kernel CPU scheduler code 6 * 7 * Copyright (C) 1991-2002 Linus Torvalds 8 * Copyright (C) 1998-2024 Ingo Molnar, Red Hat 9 */ 10#define INSTANTIATE_EXPORTED_MIGRATE_DISABLE 11#include <linux/sched.h> 12#include <linux/highmem.h> 13#include <linux/hrtimer_api.h> 14#include <linux/ktime_api.h> 15#include <linux/sched/signal.h> 16#include <linux/syscalls_api.h> 17#include <linux/debug_locks.h> 18#include <linux/prefetch.h> 19#include <linux/capability.h> 20#include <linux/pgtable_api.h> 21#include <linux/wait_bit.h> 22#include <linux/jiffies.h> 23#include <linux/spinlock_api.h> 24#include <linux/cpumask_api.h> 25#include <linux/lockdep_api.h> 26#include <linux/hardirq.h> 27#include <linux/softirq.h> 28#include <linux/refcount_api.h> 29#include <linux/topology.h> 30#include <linux/sched/clock.h> 31#include <linux/sched/cond_resched.h> 32#include <linux/sched/cputime.h> 33#include <linux/sched/debug.h> 34#include <linux/sched/hotplug.h> 35#include <linux/sched/init.h> 36#include <linux/sched/isolation.h> 37#include <linux/sched/loadavg.h> 38#include <linux/sched/mm.h> 39#include <linux/sched/nohz.h> 40#include <linux/sched/rseq_api.h> 41#include <linux/sched/rt.h> 42 43#include <linux/blkdev.h> 44#include <linux/context_tracking.h> 45#include <linux/cpuset.h> 46#include <linux/delayacct.h> 47#include <linux/init_task.h> 48#include <linux/interrupt.h> 49#include <linux/ioprio.h> 50#include <linux/kallsyms.h> 51#include <linux/kcov.h> 52#include <linux/kprobes.h> 53#include <linux/llist_api.h> 54#include <linux/mmu_context.h> 55#include <linux/mmzone.h> 56#include <linux/mutex_api.h> 57#include <linux/nmi.h> 58#include <linux/nospec.h> 59#include <linux/perf_event_api.h> 60#include <linux/profile.h> 61#include <linux/psi.h> 62#include <linux/rcuwait_api.h> 63#include <linux/rseq.h> 64#include <linux/sched/wake_q.h> 65#include <linux/scs.h> 66#include <linux/slab.h> 67#include <linux/syscalls.h> 68#include <linux/vtime.h> 69#include <linux/wait_api.h> 70#include <linux/workqueue_api.h> 71#include <linux/livepatch_sched.h> 72 73#ifdef CONFIG_PREEMPT_DYNAMIC 74# ifdef CONFIG_GENERIC_IRQ_ENTRY 75# include <linux/irq-entry-common.h> 76# endif 77#endif 78 79#include <uapi/linux/sched/types.h> 80 81#include <asm/irq_regs.h> 82#include <asm/switch_to.h> 83#include <asm/tlb.h> 84 85#define CREATE_TRACE_POINTS 86#include <linux/sched/rseq_api.h> 87#include <trace/events/sched.h> 88#include <trace/events/ipi.h> 89#undef CREATE_TRACE_POINTS 90 91#include "sched.h" 92#include "stats.h" 93 94#include "autogroup.h" 95#include "pelt.h" 96#include "smp.h" 97 98#include "../workqueue_internal.h" 99#include "../../io_uring/io-wq.h" 100#include "../smpboot.h" 101#include "../locking/mutex.h" 102 103EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); 104EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); 105 106/* 107 * Export tracepoints that act as a bare tracehook (ie: have no trace event 108 * associated with them) to allow external modules to probe them. 109 */ 110EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp); 111EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp); 112EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp); 113EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp); 114EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp); 115EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_hw_tp); 116EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp); 117EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp); 118EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp); 119EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp); 120EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp); 121EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp); 122EXPORT_TRACEPOINT_SYMBOL_GPL(sched_entry_tp); 123EXPORT_TRACEPOINT_SYMBOL_GPL(sched_exit_tp); 124EXPORT_TRACEPOINT_SYMBOL_GPL(sched_set_need_resched_tp); 125 126DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 127DEFINE_PER_CPU(struct rnd_state, sched_rnd_state); 128 129#ifdef CONFIG_SCHED_PROXY_EXEC 130DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec); 131static int __init setup_proxy_exec(char *str) 132{ 133 bool proxy_enable = true; 134 135 if (*str && kstrtobool(str + 1, &proxy_enable)) { 136 pr_warn("Unable to parse sched_proxy_exec=\n"); 137 return 0; 138 } 139 140 if (proxy_enable) { 141 pr_info("sched_proxy_exec enabled via boot arg\n"); 142 static_branch_enable(&__sched_proxy_exec); 143 } else { 144 pr_info("sched_proxy_exec disabled via boot arg\n"); 145 static_branch_disable(&__sched_proxy_exec); 146 } 147 return 1; 148} 149#else 150static int __init setup_proxy_exec(char *str) 151{ 152 pr_warn("CONFIG_SCHED_PROXY_EXEC=n, so it cannot be enabled or disabled at boot time\n"); 153 return 0; 154} 155#endif 156__setup("sched_proxy_exec", setup_proxy_exec); 157 158/* 159 * Debugging: various feature bits 160 * 161 * If SCHED_DEBUG is disabled, each compilation unit has its own copy of 162 * sysctl_sched_features, defined in sched.h, to allow constants propagation 163 * at compile time and compiler optimization based on features default. 164 */ 165#define SCHED_FEAT(name, enabled) \ 166 (1UL << __SCHED_FEAT_##name) * enabled | 167__read_mostly unsigned int sysctl_sched_features = 168#include "features.h" 169 0; 170#undef SCHED_FEAT 171 172/* 173 * Print a warning if need_resched is set for the given duration (if 174 * LATENCY_WARN is enabled). 175 * 176 * If sysctl_resched_latency_warn_once is set, only one warning will be shown 177 * per boot. 178 */ 179__read_mostly int sysctl_resched_latency_warn_ms = 100; 180__read_mostly int sysctl_resched_latency_warn_once = 1; 181 182/* 183 * Number of tasks to iterate in a single balance run. 184 * Limited because this is done with IRQs disabled. 185 */ 186__read_mostly unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK; 187 188__read_mostly int scheduler_running; 189 190#ifdef CONFIG_SCHED_CORE 191 192DEFINE_STATIC_KEY_FALSE(__sched_core_enabled); 193 194/* kernel prio, less is more */ 195static inline int __task_prio(const struct task_struct *p) 196{ 197 if (p->sched_class == &stop_sched_class) /* trumps deadline */ 198 return -2; 199 200 if (p->dl_server) 201 return -1; /* deadline */ 202 203 if (rt_or_dl_prio(p->prio)) 204 return p->prio; /* [-1, 99] */ 205 206 if (p->sched_class == &idle_sched_class) 207 return MAX_RT_PRIO + NICE_WIDTH; /* 140 */ 208 209 if (task_on_scx(p)) 210 return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */ 211 212 return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */ 213} 214 215/* 216 * l(a,b) 217 * le(a,b) := !l(b,a) 218 * g(a,b) := l(b,a) 219 * ge(a,b) := !l(a,b) 220 */ 221 222/* real prio, less is less */ 223static inline bool prio_less(const struct task_struct *a, 224 const struct task_struct *b, bool in_fi) 225{ 226 227 int pa = __task_prio(a), pb = __task_prio(b); 228 229 if (-pa < -pb) 230 return true; 231 232 if (-pb < -pa) 233 return false; 234 235 if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */ 236 const struct sched_dl_entity *a_dl, *b_dl; 237 238 a_dl = &a->dl; 239 /* 240 * Since,'a' and 'b' can be CFS tasks served by DL server, 241 * __task_prio() can return -1 (for DL) even for those. In that 242 * case, get to the dl_server's DL entity. 243 */ 244 if (a->dl_server) 245 a_dl = a->dl_server; 246 247 b_dl = &b->dl; 248 if (b->dl_server) 249 b_dl = b->dl_server; 250 251 return !dl_time_before(a_dl->deadline, b_dl->deadline); 252 } 253 254 if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */ 255 return cfs_prio_less(a, b, in_fi); 256 257#ifdef CONFIG_SCHED_CLASS_EXT 258 if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */ 259 return scx_prio_less(a, b, in_fi); 260#endif 261 262 return false; 263} 264 265static inline bool __sched_core_less(const struct task_struct *a, 266 const struct task_struct *b) 267{ 268 if (a->core_cookie < b->core_cookie) 269 return true; 270 271 if (a->core_cookie > b->core_cookie) 272 return false; 273 274 /* flip prio, so high prio is leftmost */ 275 if (prio_less(b, a, !!task_rq(a)->core->core_forceidle_count)) 276 return true; 277 278 return false; 279} 280 281#define __node_2_sc(node) rb_entry((node), struct task_struct, core_node) 282 283static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b) 284{ 285 return __sched_core_less(__node_2_sc(a), __node_2_sc(b)); 286} 287 288static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node) 289{ 290 const struct task_struct *p = __node_2_sc(node); 291 unsigned long cookie = (unsigned long)key; 292 293 if (cookie < p->core_cookie) 294 return -1; 295 296 if (cookie > p->core_cookie) 297 return 1; 298 299 return 0; 300} 301 302void sched_core_enqueue(struct rq *rq, struct task_struct *p) 303{ 304 if (p->se.sched_delayed) 305 return; 306 307 rq->core->core_task_seq++; 308 309 if (!p->core_cookie) 310 return; 311 312 rb_add(&p->core_node, &rq->core_tree, rb_sched_core_less); 313} 314 315void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) 316{ 317 if (p->se.sched_delayed) 318 return; 319 320 rq->core->core_task_seq++; 321 322 if (sched_core_enqueued(p)) { 323 rb_erase(&p->core_node, &rq->core_tree); 324 RB_CLEAR_NODE(&p->core_node); 325 } 326 327 /* 328 * Migrating the last task off the cpu, with the cpu in forced idle 329 * state. Reschedule to create an accounting edge for forced idle, 330 * and re-examine whether the core is still in forced idle state. 331 */ 332 if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 && 333 rq->core->core_forceidle_count && rq->curr == rq->idle) 334 resched_curr(rq); 335} 336 337static int sched_task_is_throttled(struct task_struct *p, int cpu) 338{ 339 if (p->sched_class->task_is_throttled) 340 return p->sched_class->task_is_throttled(p, cpu); 341 342 return 0; 343} 344 345static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie) 346{ 347 struct rb_node *node = &p->core_node; 348 int cpu = task_cpu(p); 349 350 do { 351 node = rb_next(node); 352 if (!node) 353 return NULL; 354 355 p = __node_2_sc(node); 356 if (p->core_cookie != cookie) 357 return NULL; 358 359 } while (sched_task_is_throttled(p, cpu)); 360 361 return p; 362} 363 364/* 365 * Find left-most (aka, highest priority) and unthrottled task matching @cookie. 366 * If no suitable task is found, NULL will be returned. 367 */ 368static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie) 369{ 370 struct task_struct *p; 371 struct rb_node *node; 372 373 node = rb_find_first((void *)cookie, &rq->core_tree, rb_sched_core_cmp); 374 if (!node) 375 return NULL; 376 377 p = __node_2_sc(node); 378 if (!sched_task_is_throttled(p, rq->cpu)) 379 return p; 380 381 return sched_core_next(p, cookie); 382} 383 384/* 385 * Magic required such that: 386 * 387 * raw_spin_rq_lock(rq); 388 * ... 389 * raw_spin_rq_unlock(rq); 390 * 391 * ends up locking and unlocking the _same_ lock, and all CPUs 392 * always agree on what rq has what lock. 393 * 394 * XXX entirely possible to selectively enable cores, don't bother for now. 395 */ 396 397static DEFINE_MUTEX(sched_core_mutex); 398static atomic_t sched_core_count; 399static struct cpumask sched_core_mask; 400 401static void sched_core_lock(int cpu, unsigned long *flags) 402{ 403 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 404 int t, i = 0; 405 406 local_irq_save(*flags); 407 for_each_cpu(t, smt_mask) 408 raw_spin_lock_nested(&cpu_rq(t)->__lock, i++); 409} 410 411static void sched_core_unlock(int cpu, unsigned long *flags) 412{ 413 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 414 int t; 415 416 for_each_cpu(t, smt_mask) 417 raw_spin_unlock(&cpu_rq(t)->__lock); 418 local_irq_restore(*flags); 419} 420 421static void __sched_core_flip(bool enabled) 422{ 423 unsigned long flags; 424 int cpu, t; 425 426 cpus_read_lock(); 427 428 /* 429 * Toggle the online cores, one by one. 430 */ 431 cpumask_copy(&sched_core_mask, cpu_online_mask); 432 for_each_cpu(cpu, &sched_core_mask) { 433 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 434 435 sched_core_lock(cpu, &flags); 436 437 for_each_cpu(t, smt_mask) 438 cpu_rq(t)->core_enabled = enabled; 439 440 cpu_rq(cpu)->core->core_forceidle_start = 0; 441 442 sched_core_unlock(cpu, &flags); 443 444 cpumask_andnot(&sched_core_mask, &sched_core_mask, smt_mask); 445 } 446 447 /* 448 * Toggle the offline CPUs. 449 */ 450 for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask) 451 cpu_rq(cpu)->core_enabled = enabled; 452 453 cpus_read_unlock(); 454} 455 456static void sched_core_assert_empty(void) 457{ 458 int cpu; 459 460 for_each_possible_cpu(cpu) 461 WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree)); 462} 463 464static void __sched_core_enable(void) 465{ 466 static_branch_enable(&__sched_core_enabled); 467 /* 468 * Ensure all previous instances of raw_spin_rq_*lock() have finished 469 * and future ones will observe !sched_core_disabled(). 470 */ 471 synchronize_rcu(); 472 __sched_core_flip(true); 473 sched_core_assert_empty(); 474} 475 476static void __sched_core_disable(void) 477{ 478 sched_core_assert_empty(); 479 __sched_core_flip(false); 480 static_branch_disable(&__sched_core_enabled); 481} 482 483void sched_core_get(void) 484{ 485 if (atomic_inc_not_zero(&sched_core_count)) 486 return; 487 488 mutex_lock(&sched_core_mutex); 489 if (!atomic_read(&sched_core_count)) 490 __sched_core_enable(); 491 492 smp_mb__before_atomic(); 493 atomic_inc(&sched_core_count); 494 mutex_unlock(&sched_core_mutex); 495} 496 497static void __sched_core_put(struct work_struct *work) 498{ 499 if (atomic_dec_and_mutex_lock(&sched_core_count, &sched_core_mutex)) { 500 __sched_core_disable(); 501 mutex_unlock(&sched_core_mutex); 502 } 503} 504 505void sched_core_put(void) 506{ 507 static DECLARE_WORK(_work, __sched_core_put); 508 509 /* 510 * "There can be only one" 511 * 512 * Either this is the last one, or we don't actually need to do any 513 * 'work'. If it is the last *again*, we rely on 514 * WORK_STRUCT_PENDING_BIT. 515 */ 516 if (!atomic_add_unless(&sched_core_count, -1, 1)) 517 schedule_work(&_work); 518} 519 520#else /* !CONFIG_SCHED_CORE: */ 521 522static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { } 523static inline void 524sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } 525 526#endif /* !CONFIG_SCHED_CORE */ 527 528/* need a wrapper since we may need to trace from modules */ 529EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp); 530 531/* Call via the helper macro trace_set_current_state. */ 532void __trace_set_current_state(int state_value) 533{ 534 trace_sched_set_state_tp(current, state_value); 535} 536EXPORT_SYMBOL(__trace_set_current_state); 537 538/* 539 * Serialization rules: 540 * 541 * Lock order: 542 * 543 * p->pi_lock 544 * rq->lock 545 * hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls) 546 * 547 * rq1->lock 548 * rq2->lock where: rq1 < rq2 549 * 550 * Regular state: 551 * 552 * Normal scheduling state is serialized by rq->lock. __schedule() takes the 553 * local CPU's rq->lock, it optionally removes the task from the runqueue and 554 * always looks at the local rq data structures to find the most eligible task 555 * to run next. 556 * 557 * Task enqueue is also under rq->lock, possibly taken from another CPU. 558 * Wakeups from another LLC domain might use an IPI to transfer the enqueue to 559 * the local CPU to avoid bouncing the runqueue state around [ see 560 * ttwu_queue_wakelist() ] 561 * 562 * Task wakeup, specifically wakeups that involve migration, are horribly 563 * complicated to avoid having to take two rq->locks. 564 * 565 * Special state: 566 * 567 * System-calls and anything external will use task_rq_lock() which acquires 568 * both p->pi_lock and rq->lock. As a consequence the state they change is 569 * stable while holding either lock: 570 * 571 * - sched_setaffinity()/ 572 * set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed 573 * - set_user_nice(): p->se.load, p->*prio 574 * - __sched_setscheduler(): p->sched_class, p->policy, p->*prio, 575 * p->se.load, p->rt_priority, 576 * p->dl.dl_{runtime, deadline, period, flags, bw, density} 577 * - sched_setnuma(): p->numa_preferred_nid 578 * - sched_move_task(): p->sched_task_group 579 * - uclamp_update_active() p->uclamp* 580 * 581 * p->state <- TASK_*: 582 * 583 * is changed locklessly using set_current_state(), __set_current_state() or 584 * set_special_state(), see their respective comments, or by 585 * try_to_wake_up(). This latter uses p->pi_lock to serialize against 586 * concurrent self. 587 * 588 * p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }: 589 * 590 * is set by activate_task() and cleared by deactivate_task()/block_task(), 591 * under rq->lock. Non-zero indicates the task is runnable, the special 592 * ON_RQ_MIGRATING state is used for migration without holding both 593 * rq->locks. It indicates task_cpu() is not stable, see task_rq_lock(). 594 * 595 * Additionally it is possible to be ->on_rq but still be considered not 596 * runnable when p->se.sched_delayed is true. These tasks are on the runqueue 597 * but will be dequeued as soon as they get picked again. See the 598 * task_is_runnable() helper. 599 * 600 * p->on_cpu <- { 0, 1 }: 601 * 602 * is set by prepare_task() and cleared by finish_task() such that it will be 603 * set before p is scheduled-in and cleared after p is scheduled-out, both 604 * under rq->lock. Non-zero indicates the task is running on its CPU. 605 * 606 * [ The astute reader will observe that it is possible for two tasks on one 607 * CPU to have ->on_cpu = 1 at the same time. ] 608 * 609 * task_cpu(p): is changed by set_task_cpu(), the rules are: 610 * 611 * - Don't call set_task_cpu() on a blocked task: 612 * 613 * We don't care what CPU we're not running on, this simplifies hotplug, 614 * the CPU assignment of blocked tasks isn't required to be valid. 615 * 616 * - for try_to_wake_up(), called under p->pi_lock: 617 * 618 * This allows try_to_wake_up() to only take one rq->lock, see its comment. 619 * 620 * - for migration called under rq->lock: 621 * [ see task_on_rq_migrating() in task_rq_lock() ] 622 * 623 * o move_queued_task() 624 * o detach_task() 625 * 626 * - for migration called under double_rq_lock(): 627 * 628 * o __migrate_swap_task() 629 * o push_rt_task() / pull_rt_task() 630 * o push_dl_task() / pull_dl_task() 631 * o dl_task_offline_migration() 632 * 633 */ 634 635void raw_spin_rq_lock_nested(struct rq *rq, int subclass) 636{ 637 raw_spinlock_t *lock; 638 639 /* Matches synchronize_rcu() in __sched_core_enable() */ 640 preempt_disable(); 641 if (sched_core_disabled()) { 642 raw_spin_lock_nested(&rq->__lock, subclass); 643 /* preempt_count *MUST* be > 1 */ 644 preempt_enable_no_resched(); 645 return; 646 } 647 648 for (;;) { 649 lock = __rq_lockp(rq); 650 raw_spin_lock_nested(lock, subclass); 651 if (likely(lock == __rq_lockp(rq))) { 652 /* preempt_count *MUST* be > 1 */ 653 preempt_enable_no_resched(); 654 return; 655 } 656 raw_spin_unlock(lock); 657 } 658} 659 660bool raw_spin_rq_trylock(struct rq *rq) 661{ 662 raw_spinlock_t *lock; 663 bool ret; 664 665 /* Matches synchronize_rcu() in __sched_core_enable() */ 666 preempt_disable(); 667 if (sched_core_disabled()) { 668 ret = raw_spin_trylock(&rq->__lock); 669 preempt_enable(); 670 return ret; 671 } 672 673 for (;;) { 674 lock = __rq_lockp(rq); 675 ret = raw_spin_trylock(lock); 676 if (!ret || (likely(lock == __rq_lockp(rq)))) { 677 preempt_enable(); 678 return ret; 679 } 680 raw_spin_unlock(lock); 681 } 682} 683 684void raw_spin_rq_unlock(struct rq *rq) 685{ 686 raw_spin_unlock(rq_lockp(rq)); 687} 688 689/* 690 * double_rq_lock - safely lock two runqueues 691 */ 692void double_rq_lock(struct rq *rq1, struct rq *rq2) 693{ 694 lockdep_assert_irqs_disabled(); 695 696 if (rq_order_less(rq2, rq1)) 697 swap(rq1, rq2); 698 699 raw_spin_rq_lock(rq1); 700 if (__rq_lockp(rq1) != __rq_lockp(rq2)) 701 raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); 702 703 double_rq_clock_clear_update(rq1, rq2); 704} 705 706/* 707 * __task_rq_lock - lock the rq @p resides on. 708 */ 709struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) 710 __acquires(rq->lock) 711{ 712 struct rq *rq; 713 714 lockdep_assert_held(&p->pi_lock); 715 716 for (;;) { 717 rq = task_rq(p); 718 raw_spin_rq_lock(rq); 719 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 720 rq_pin_lock(rq, rf); 721 return rq; 722 } 723 raw_spin_rq_unlock(rq); 724 725 while (unlikely(task_on_rq_migrating(p))) 726 cpu_relax(); 727 } 728} 729 730/* 731 * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. 732 */ 733struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) 734 __acquires(p->pi_lock) 735 __acquires(rq->lock) 736{ 737 struct rq *rq; 738 739 for (;;) { 740 raw_spin_lock_irqsave(&p->pi_lock, rf->flags); 741 rq = task_rq(p); 742 raw_spin_rq_lock(rq); 743 /* 744 * move_queued_task() task_rq_lock() 745 * 746 * ACQUIRE (rq->lock) 747 * [S] ->on_rq = MIGRATING [L] rq = task_rq() 748 * WMB (__set_task_cpu()) ACQUIRE (rq->lock); 749 * [S] ->cpu = new_cpu [L] task_rq() 750 * [L] ->on_rq 751 * RELEASE (rq->lock) 752 * 753 * If we observe the old CPU in task_rq_lock(), the acquire of 754 * the old rq->lock will fully serialize against the stores. 755 * 756 * If we observe the new CPU in task_rq_lock(), the address 757 * dependency headed by '[L] rq = task_rq()' and the acquire 758 * will pair with the WMB to ensure we then also see migrating. 759 */ 760 if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) { 761 rq_pin_lock(rq, rf); 762 return rq; 763 } 764 raw_spin_rq_unlock(rq); 765 raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); 766 767 while (unlikely(task_on_rq_migrating(p))) 768 cpu_relax(); 769 } 770} 771 772/* 773 * RQ-clock updating methods: 774 */ 775 776static void update_rq_clock_task(struct rq *rq, s64 delta) 777{ 778/* 779 * In theory, the compile should just see 0 here, and optimize out the call 780 * to sched_rt_avg_update. But I don't trust it... 781 */ 782 s64 __maybe_unused steal = 0, irq_delta = 0; 783 784#ifdef CONFIG_IRQ_TIME_ACCOUNTING 785 if (irqtime_enabled()) { 786 irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; 787 788 /* 789 * Since irq_time is only updated on {soft,}irq_exit, we might run into 790 * this case when a previous update_rq_clock() happened inside a 791 * {soft,}IRQ region. 792 * 793 * When this happens, we stop ->clock_task and only update the 794 * prev_irq_time stamp to account for the part that fit, so that a next 795 * update will consume the rest. This ensures ->clock_task is 796 * monotonic. 797 * 798 * It does however cause some slight miss-attribution of {soft,}IRQ 799 * time, a more accurate solution would be to update the irq_time using 800 * the current rq->clock timestamp, except that would require using 801 * atomic ops. 802 */ 803 if (irq_delta > delta) 804 irq_delta = delta; 805 806 rq->prev_irq_time += irq_delta; 807 delta -= irq_delta; 808 delayacct_irq(rq->curr, irq_delta); 809 } 810#endif 811#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING 812 if (static_key_false((¶virt_steal_rq_enabled))) { 813 u64 prev_steal; 814 815 steal = prev_steal = paravirt_steal_clock(cpu_of(rq)); 816 steal -= rq->prev_steal_time_rq; 817 818 if (unlikely(steal > delta)) 819 steal = delta; 820 821 rq->prev_steal_time_rq = prev_steal; 822 delta -= steal; 823 } 824#endif 825 826 rq->clock_task += delta; 827 828#ifdef CONFIG_HAVE_SCHED_AVG_IRQ 829 if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) 830 update_irq_load_avg(rq, irq_delta + steal); 831#endif 832 update_rq_clock_pelt(rq, delta); 833} 834 835void update_rq_clock(struct rq *rq) 836{ 837 s64 delta; 838 u64 clock; 839 840 lockdep_assert_rq_held(rq); 841 842 if (rq->clock_update_flags & RQCF_ACT_SKIP) 843 return; 844 845 if (sched_feat(WARN_DOUBLE_CLOCK)) 846 WARN_ON_ONCE(rq->clock_update_flags & RQCF_UPDATED); 847 rq->clock_update_flags |= RQCF_UPDATED; 848 849 clock = sched_clock_cpu(cpu_of(rq)); 850 scx_rq_clock_update(rq, clock); 851 852 delta = clock - rq->clock; 853 if (delta < 0) 854 return; 855 rq->clock += delta; 856 857 update_rq_clock_task(rq, delta); 858} 859 860#ifdef CONFIG_SCHED_HRTICK 861/* 862 * Use HR-timers to deliver accurate preemption points. 863 */ 864 865static void hrtick_clear(struct rq *rq) 866{ 867 if (hrtimer_active(&rq->hrtick_timer)) 868 hrtimer_cancel(&rq->hrtick_timer); 869} 870 871/* 872 * High-resolution timer tick. 873 * Runs from hardirq context with interrupts disabled. 874 */ 875static enum hrtimer_restart hrtick(struct hrtimer *timer) 876{ 877 struct rq *rq = container_of(timer, struct rq, hrtick_timer); 878 struct rq_flags rf; 879 880 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); 881 882 rq_lock(rq, &rf); 883 update_rq_clock(rq); 884 rq->donor->sched_class->task_tick(rq, rq->donor, 1); 885 rq_unlock(rq, &rf); 886 887 return HRTIMER_NORESTART; 888} 889 890static void __hrtick_restart(struct rq *rq) 891{ 892 struct hrtimer *timer = &rq->hrtick_timer; 893 ktime_t time = rq->hrtick_time; 894 895 hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD); 896} 897 898/* 899 * called from hardirq (IPI) context 900 */ 901static void __hrtick_start(void *arg) 902{ 903 struct rq *rq = arg; 904 struct rq_flags rf; 905 906 rq_lock(rq, &rf); 907 __hrtick_restart(rq); 908 rq_unlock(rq, &rf); 909} 910 911/* 912 * Called to set the hrtick timer state. 913 * 914 * called with rq->lock held and IRQs disabled 915 */ 916void hrtick_start(struct rq *rq, u64 delay) 917{ 918 struct hrtimer *timer = &rq->hrtick_timer; 919 s64 delta; 920 921 /* 922 * Don't schedule slices shorter than 10000ns, that just 923 * doesn't make sense and can cause timer DoS. 924 */ 925 delta = max_t(s64, delay, 10000LL); 926 rq->hrtick_time = ktime_add_ns(hrtimer_cb_get_time(timer), delta); 927 928 if (rq == this_rq()) 929 __hrtick_restart(rq); 930 else 931 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); 932} 933 934static void hrtick_rq_init(struct rq *rq) 935{ 936 INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq); 937 hrtimer_setup(&rq->hrtick_timer, hrtick, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 938} 939#else /* !CONFIG_SCHED_HRTICK: */ 940static inline void hrtick_clear(struct rq *rq) 941{ 942} 943 944static inline void hrtick_rq_init(struct rq *rq) 945{ 946} 947#endif /* !CONFIG_SCHED_HRTICK */ 948 949/* 950 * try_cmpxchg based fetch_or() macro so it works for different integer types: 951 */ 952#define fetch_or(ptr, mask) \ 953 ({ \ 954 typeof(ptr) _ptr = (ptr); \ 955 typeof(mask) _mask = (mask); \ 956 typeof(*_ptr) _val = *_ptr; \ 957 \ 958 do { \ 959 } while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \ 960 _val; \ 961}) 962 963#ifdef TIF_POLLING_NRFLAG 964/* 965 * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, 966 * this avoids any races wrt polling state changes and thereby avoids 967 * spurious IPIs. 968 */ 969static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif) 970{ 971 return !(fetch_or(&ti->flags, 1 << tif) & _TIF_POLLING_NRFLAG); 972} 973 974/* 975 * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. 976 * 977 * If this returns true, then the idle task promises to call 978 * sched_ttwu_pending() and reschedule soon. 979 */ 980static bool set_nr_if_polling(struct task_struct *p) 981{ 982 struct thread_info *ti = task_thread_info(p); 983 typeof(ti->flags) val = READ_ONCE(ti->flags); 984 985 do { 986 if (!(val & _TIF_POLLING_NRFLAG)) 987 return false; 988 if (val & _TIF_NEED_RESCHED) 989 return true; 990 } while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED)); 991 992 return true; 993} 994 995#else 996static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif) 997{ 998 set_ti_thread_flag(ti, tif); 999 return true; 1000} 1001 1002static inline bool set_nr_if_polling(struct task_struct *p) 1003{ 1004 return false; 1005} 1006#endif 1007 1008static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task) 1009{ 1010 struct wake_q_node *node = &task->wake_q; 1011 1012 /* 1013 * Atomically grab the task, if ->wake_q is !nil already it means 1014 * it's already queued (either by us or someone else) and will get the 1015 * wakeup due to that. 1016 * 1017 * In order to ensure that a pending wakeup will observe our pending 1018 * state, even in the failed case, an explicit smp_mb() must be used. 1019 */ 1020 smp_mb__before_atomic(); 1021 if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL))) 1022 return false; 1023 1024 /* 1025 * The head is context local, there can be no concurrency. 1026 */ 1027 *head->lastp = node; 1028 head->lastp = &node->next; 1029 return true; 1030} 1031 1032/** 1033 * wake_q_add() - queue a wakeup for 'later' waking. 1034 * @head: the wake_q_head to add @task to 1035 * @task: the task to queue for 'later' wakeup 1036 * 1037 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 1038 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 1039 * instantly. 1040 * 1041 * This function must be used as-if it were wake_up_process(); IOW the task 1042 * must be ready to be woken at this location. 1043 */ 1044void wake_q_add(struct wake_q_head *head, struct task_struct *task) 1045{ 1046 if (__wake_q_add(head, task)) 1047 get_task_struct(task); 1048} 1049 1050/** 1051 * wake_q_add_safe() - safely queue a wakeup for 'later' waking. 1052 * @head: the wake_q_head to add @task to 1053 * @task: the task to queue for 'later' wakeup 1054 * 1055 * Queue a task for later wakeup, most likely by the wake_up_q() call in the 1056 * same context, _HOWEVER_ this is not guaranteed, the wakeup can come 1057 * instantly. 1058 * 1059 * This function must be used as-if it were wake_up_process(); IOW the task 1060 * must be ready to be woken at this location. 1061 * 1062 * This function is essentially a task-safe equivalent to wake_q_add(). Callers 1063 * that already hold reference to @task can call the 'safe' version and trust 1064 * wake_q to do the right thing depending whether or not the @task is already 1065 * queued for wakeup. 1066 */ 1067void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task) 1068{ 1069 if (!__wake_q_add(head, task)) 1070 put_task_struct(task); 1071} 1072 1073void wake_up_q(struct wake_q_head *head) 1074{ 1075 struct wake_q_node *node = head->first; 1076 1077 while (node != WAKE_Q_TAIL) { 1078 struct task_struct *task; 1079 1080 task = container_of(node, struct task_struct, wake_q); 1081 node = node->next; 1082 /* pairs with cmpxchg_relaxed() in __wake_q_add() */ 1083 WRITE_ONCE(task->wake_q.next, NULL); 1084 /* Task can safely be re-inserted now. */ 1085 1086 /* 1087 * wake_up_process() executes a full barrier, which pairs with 1088 * the queueing in wake_q_add() so as not to miss wakeups. 1089 */ 1090 wake_up_process(task); 1091 put_task_struct(task); 1092 } 1093} 1094 1095/* 1096 * resched_curr - mark rq's current task 'to be rescheduled now'. 1097 * 1098 * On UP this means the setting of the need_resched flag, on SMP it 1099 * might also involve a cross-CPU call to trigger the scheduler on 1100 * the target CPU. 1101 */ 1102static void __resched_curr(struct rq *rq, int tif) 1103{ 1104 struct task_struct *curr = rq->curr; 1105 struct thread_info *cti = task_thread_info(curr); 1106 int cpu; 1107 1108 lockdep_assert_rq_held(rq); 1109 1110 /* 1111 * Always immediately preempt the idle task; no point in delaying doing 1112 * actual work. 1113 */ 1114 if (is_idle_task(curr) && tif == TIF_NEED_RESCHED_LAZY) 1115 tif = TIF_NEED_RESCHED; 1116 1117 if (cti->flags & ((1 << tif) | _TIF_NEED_RESCHED)) 1118 return; 1119 1120 cpu = cpu_of(rq); 1121 1122 trace_sched_set_need_resched_tp(curr, cpu, tif); 1123 if (cpu == smp_processor_id()) { 1124 set_ti_thread_flag(cti, tif); 1125 if (tif == TIF_NEED_RESCHED) 1126 set_preempt_need_resched(); 1127 return; 1128 } 1129 1130 if (set_nr_and_not_polling(cti, tif)) { 1131 if (tif == TIF_NEED_RESCHED) 1132 smp_send_reschedule(cpu); 1133 } else { 1134 trace_sched_wake_idle_without_ipi(cpu); 1135 } 1136} 1137 1138void __trace_set_need_resched(struct task_struct *curr, int tif) 1139{ 1140 trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif); 1141} 1142EXPORT_SYMBOL_GPL(__trace_set_need_resched); 1143 1144void resched_curr(struct rq *rq) 1145{ 1146 __resched_curr(rq, TIF_NEED_RESCHED); 1147} 1148 1149#ifdef CONFIG_PREEMPT_DYNAMIC 1150static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy); 1151static __always_inline bool dynamic_preempt_lazy(void) 1152{ 1153 return static_branch_unlikely(&sk_dynamic_preempt_lazy); 1154} 1155#else 1156static __always_inline bool dynamic_preempt_lazy(void) 1157{ 1158 return IS_ENABLED(CONFIG_PREEMPT_LAZY); 1159} 1160#endif 1161 1162static __always_inline int get_lazy_tif_bit(void) 1163{ 1164 if (dynamic_preempt_lazy()) 1165 return TIF_NEED_RESCHED_LAZY; 1166 1167 return TIF_NEED_RESCHED; 1168} 1169 1170void resched_curr_lazy(struct rq *rq) 1171{ 1172 __resched_curr(rq, get_lazy_tif_bit()); 1173} 1174 1175void resched_cpu(int cpu) 1176{ 1177 struct rq *rq = cpu_rq(cpu); 1178 unsigned long flags; 1179 1180 raw_spin_rq_lock_irqsave(rq, flags); 1181 if (cpu_online(cpu) || cpu == smp_processor_id()) 1182 resched_curr(rq); 1183 raw_spin_rq_unlock_irqrestore(rq, flags); 1184} 1185 1186#ifdef CONFIG_NO_HZ_COMMON 1187/* 1188 * In the semi idle case, use the nearest busy CPU for migrating timers 1189 * from an idle CPU. This is good for power-savings. 1190 * 1191 * We don't do similar optimization for completely idle system, as 1192 * selecting an idle CPU will add more delays to the timers than intended 1193 * (as that CPU's timer base may not be up to date wrt jiffies etc). 1194 */ 1195int get_nohz_timer_target(void) 1196{ 1197 int i, cpu = smp_processor_id(), default_cpu = -1; 1198 struct sched_domain *sd; 1199 const struct cpumask *hk_mask; 1200 1201 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) { 1202 if (!idle_cpu(cpu)) 1203 return cpu; 1204 default_cpu = cpu; 1205 } 1206 1207 hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); 1208 1209 guard(rcu)(); 1210 1211 for_each_domain(cpu, sd) { 1212 for_each_cpu_and(i, sched_domain_span(sd), hk_mask) { 1213 if (cpu == i) 1214 continue; 1215 1216 if (!idle_cpu(i)) 1217 return i; 1218 } 1219 } 1220 1221 if (default_cpu == -1) 1222 default_cpu = housekeeping_any_cpu(HK_TYPE_KERNEL_NOISE); 1223 1224 return default_cpu; 1225} 1226 1227/* 1228 * When add_timer_on() enqueues a timer into the timer wheel of an 1229 * idle CPU then this timer might expire before the next timer event 1230 * which is scheduled to wake up that CPU. In case of a completely 1231 * idle system the next event might even be infinite time into the 1232 * future. wake_up_idle_cpu() ensures that the CPU is woken up and 1233 * leaves the inner idle loop so the newly added timer is taken into 1234 * account when the CPU goes back to idle and evaluates the timer 1235 * wheel for the next timer event. 1236 */ 1237static void wake_up_idle_cpu(int cpu) 1238{ 1239 struct rq *rq = cpu_rq(cpu); 1240 1241 if (cpu == smp_processor_id()) 1242 return; 1243 1244 /* 1245 * Set TIF_NEED_RESCHED and send an IPI if in the non-polling 1246 * part of the idle loop. This forces an exit from the idle loop 1247 * and a round trip to schedule(). Now this could be optimized 1248 * because a simple new idle loop iteration is enough to 1249 * re-evaluate the next tick. Provided some re-ordering of tick 1250 * nohz functions that would need to follow TIF_NR_POLLING 1251 * clearing: 1252 * 1253 * - On most architectures, a simple fetch_or on ti::flags with a 1254 * "0" value would be enough to know if an IPI needs to be sent. 1255 * 1256 * - x86 needs to perform a last need_resched() check between 1257 * monitor and mwait which doesn't take timers into account. 1258 * There a dedicated TIF_TIMER flag would be required to 1259 * fetch_or here and be checked along with TIF_NEED_RESCHED 1260 * before mwait(). 1261 * 1262 * However, remote timer enqueue is not such a frequent event 1263 * and testing of the above solutions didn't appear to report 1264 * much benefits. 1265 */ 1266 if (set_nr_and_not_polling(task_thread_info(rq->idle), TIF_NEED_RESCHED)) 1267 smp_send_reschedule(cpu); 1268 else 1269 trace_sched_wake_idle_without_ipi(cpu); 1270} 1271 1272static bool wake_up_full_nohz_cpu(int cpu) 1273{ 1274 /* 1275 * We just need the target to call irq_exit() and re-evaluate 1276 * the next tick. The nohz full kick at least implies that. 1277 * If needed we can still optimize that later with an 1278 * empty IRQ. 1279 */ 1280 if (cpu_is_offline(cpu)) 1281 return true; /* Don't try to wake offline CPUs. */ 1282 if (tick_nohz_full_cpu(cpu)) { 1283 if (cpu != smp_processor_id() || 1284 tick_nohz_tick_stopped()) 1285 tick_nohz_full_kick_cpu(cpu); 1286 return true; 1287 } 1288 1289 return false; 1290} 1291 1292/* 1293 * Wake up the specified CPU. If the CPU is going offline, it is the 1294 * caller's responsibility to deal with the lost wakeup, for example, 1295 * by hooking into the CPU_DEAD notifier like timers and hrtimers do. 1296 */ 1297void wake_up_nohz_cpu(int cpu) 1298{ 1299 if (!wake_up_full_nohz_cpu(cpu)) 1300 wake_up_idle_cpu(cpu); 1301} 1302 1303static void nohz_csd_func(void *info) 1304{ 1305 struct rq *rq = info; 1306 int cpu = cpu_of(rq); 1307 unsigned int flags; 1308 1309 /* 1310 * Release the rq::nohz_csd. 1311 */ 1312 flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); 1313 WARN_ON(!(flags & NOHZ_KICK_MASK)); 1314 1315 rq->idle_balance = idle_cpu(cpu); 1316 if (rq->idle_balance) { 1317 rq->nohz_idle_balance = flags; 1318 __raise_softirq_irqoff(SCHED_SOFTIRQ); 1319 } 1320} 1321 1322#endif /* CONFIG_NO_HZ_COMMON */ 1323 1324#ifdef CONFIG_NO_HZ_FULL 1325static inline bool __need_bw_check(struct rq *rq, struct task_struct *p) 1326{ 1327 if (rq->nr_running != 1) 1328 return false; 1329 1330 if (p->sched_class != &fair_sched_class) 1331 return false; 1332 1333 if (!task_on_rq_queued(p)) 1334 return false; 1335 1336 return true; 1337} 1338 1339bool sched_can_stop_tick(struct rq *rq) 1340{ 1341 int fifo_nr_running; 1342 1343 /* Deadline tasks, even if single, need the tick */ 1344 if (rq->dl.dl_nr_running) 1345 return false; 1346 1347 /* 1348 * If there are more than one RR tasks, we need the tick to affect the 1349 * actual RR behaviour. 1350 */ 1351 if (rq->rt.rr_nr_running) { 1352 if (rq->rt.rr_nr_running == 1) 1353 return true; 1354 else 1355 return false; 1356 } 1357 1358 /* 1359 * If there's no RR tasks, but FIFO tasks, we can skip the tick, no 1360 * forced preemption between FIFO tasks. 1361 */ 1362 fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running; 1363 if (fifo_nr_running) 1364 return true; 1365 1366 /* 1367 * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks 1368 * left. For CFS, if there's more than one we need the tick for 1369 * involuntary preemption. For SCX, ask. 1370 */ 1371 if (scx_enabled() && !scx_can_stop_tick(rq)) 1372 return false; 1373 1374 if (rq->cfs.h_nr_queued > 1) 1375 return false; 1376 1377 /* 1378 * If there is one task and it has CFS runtime bandwidth constraints 1379 * and it's on the cpu now we don't want to stop the tick. 1380 * This check prevents clearing the bit if a newly enqueued task here is 1381 * dequeued by migrating while the constrained task continues to run. 1382 * E.g. going from 2->1 without going through pick_next_task(). 1383 */ 1384 if (__need_bw_check(rq, rq->curr)) { 1385 if (cfs_task_bw_constrained(rq->curr)) 1386 return false; 1387 } 1388 1389 return true; 1390} 1391#endif /* CONFIG_NO_HZ_FULL */ 1392 1393#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_FAIR_GROUP_SCHED) 1394/* 1395 * Iterate task_group tree rooted at *from, calling @down when first entering a 1396 * node and @up when leaving it for the final time. 1397 * 1398 * Caller must hold rcu_lock or sufficient equivalent. 1399 */ 1400int walk_tg_tree_from(struct task_group *from, 1401 tg_visitor down, tg_visitor up, void *data) 1402{ 1403 struct task_group *parent, *child; 1404 int ret; 1405 1406 parent = from; 1407 1408down: 1409 ret = (*down)(parent, data); 1410 if (ret) 1411 goto out; 1412 list_for_each_entry_rcu(child, &parent->children, siblings) { 1413 parent = child; 1414 goto down; 1415 1416up: 1417 continue; 1418 } 1419 ret = (*up)(parent, data); 1420 if (ret || parent == from) 1421 goto out; 1422 1423 child = parent; 1424 parent = parent->parent; 1425 if (parent) 1426 goto up; 1427out: 1428 return ret; 1429} 1430 1431int tg_nop(struct task_group *tg, void *data) 1432{ 1433 return 0; 1434} 1435#endif 1436 1437void set_load_weight(struct task_struct *p, bool update_load) 1438{ 1439 int prio = p->static_prio - MAX_RT_PRIO; 1440 struct load_weight lw; 1441 1442 if (task_has_idle_policy(p)) { 1443 lw.weight = scale_load(WEIGHT_IDLEPRIO); 1444 lw.inv_weight = WMULT_IDLEPRIO; 1445 } else { 1446 lw.weight = scale_load(sched_prio_to_weight[prio]); 1447 lw.inv_weight = sched_prio_to_wmult[prio]; 1448 } 1449 1450 /* 1451 * SCHED_OTHER tasks have to update their load when changing their 1452 * weight 1453 */ 1454 if (update_load && p->sched_class->reweight_task) 1455 p->sched_class->reweight_task(task_rq(p), p, &lw); 1456 else 1457 p->se.load = lw; 1458} 1459 1460#ifdef CONFIG_UCLAMP_TASK 1461/* 1462 * Serializes updates of utilization clamp values 1463 * 1464 * The (slow-path) user-space triggers utilization clamp value updates which 1465 * can require updates on (fast-path) scheduler's data structures used to 1466 * support enqueue/dequeue operations. 1467 * While the per-CPU rq lock protects fast-path update operations, user-space 1468 * requests are serialized using a mutex to reduce the risk of conflicting 1469 * updates or API abuses. 1470 */ 1471static __maybe_unused DEFINE_MUTEX(uclamp_mutex); 1472 1473/* Max allowed minimum utilization */ 1474static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; 1475 1476/* Max allowed maximum utilization */ 1477static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; 1478 1479/* 1480 * By default RT tasks run at the maximum performance point/capacity of the 1481 * system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to 1482 * SCHED_CAPACITY_SCALE. 1483 * 1484 * This knob allows admins to change the default behavior when uclamp is being 1485 * used. In battery powered devices, particularly, running at the maximum 1486 * capacity and frequency will increase energy consumption and shorten the 1487 * battery life. 1488 * 1489 * This knob only affects RT tasks that their uclamp_se->user_defined == false. 1490 * 1491 * This knob will not override the system default sched_util_clamp_min defined 1492 * above. 1493 */ 1494unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; 1495 1496/* All clamps are required to be less or equal than these values */ 1497static struct uclamp_se uclamp_default[UCLAMP_CNT]; 1498 1499/* 1500 * This static key is used to reduce the uclamp overhead in the fast path. It 1501 * primarily disables the call to uclamp_rq_{inc, dec}() in 1502 * enqueue/dequeue_task(). 1503 * 1504 * This allows users to continue to enable uclamp in their kernel config with 1505 * minimum uclamp overhead in the fast path. 1506 * 1507 * As soon as userspace modifies any of the uclamp knobs, the static key is 1508 * enabled, since we have an actual users that make use of uclamp 1509 * functionality. 1510 * 1511 * The knobs that would enable this static key are: 1512 * 1513 * * A task modifying its uclamp value with sched_setattr(). 1514 * * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs. 1515 * * An admin modifying the cgroup cpu.uclamp.{min, max} 1516 */ 1517DEFINE_STATIC_KEY_FALSE(sched_uclamp_used); 1518 1519static inline unsigned int 1520uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id, 1521 unsigned int clamp_value) 1522{ 1523 /* 1524 * Avoid blocked utilization pushing up the frequency when we go 1525 * idle (which drops the max-clamp) by retaining the last known 1526 * max-clamp. 1527 */ 1528 if (clamp_id == UCLAMP_MAX) { 1529 rq->uclamp_flags |= UCLAMP_FLAG_IDLE; 1530 return clamp_value; 1531 } 1532 1533 return uclamp_none(UCLAMP_MIN); 1534} 1535 1536static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id, 1537 unsigned int clamp_value) 1538{ 1539 /* Reset max-clamp retention only on idle exit */ 1540 if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1541 return; 1542 1543 uclamp_rq_set(rq, clamp_id, clamp_value); 1544} 1545 1546static inline 1547unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id, 1548 unsigned int clamp_value) 1549{ 1550 struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket; 1551 int bucket_id = UCLAMP_BUCKETS - 1; 1552 1553 /* 1554 * Since both min and max clamps are max aggregated, find the 1555 * top most bucket with tasks in. 1556 */ 1557 for ( ; bucket_id >= 0; bucket_id--) { 1558 if (!bucket[bucket_id].tasks) 1559 continue; 1560 return bucket[bucket_id].value; 1561 } 1562 1563 /* No tasks -- default clamp values */ 1564 return uclamp_idle_value(rq, clamp_id, clamp_value); 1565} 1566 1567static void __uclamp_update_util_min_rt_default(struct task_struct *p) 1568{ 1569 unsigned int default_util_min; 1570 struct uclamp_se *uc_se; 1571 1572 lockdep_assert_held(&p->pi_lock); 1573 1574 uc_se = &p->uclamp_req[UCLAMP_MIN]; 1575 1576 /* Only sync if user didn't override the default */ 1577 if (uc_se->user_defined) 1578 return; 1579 1580 default_util_min = sysctl_sched_uclamp_util_min_rt_default; 1581 uclamp_se_set(uc_se, default_util_min, false); 1582} 1583 1584static void uclamp_update_util_min_rt_default(struct task_struct *p) 1585{ 1586 if (!rt_task(p)) 1587 return; 1588 1589 /* Protect updates to p->uclamp_* */ 1590 guard(task_rq_lock)(p); 1591 __uclamp_update_util_min_rt_default(p); 1592} 1593 1594static inline struct uclamp_se 1595uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) 1596{ 1597 /* Copy by value as we could modify it */ 1598 struct uclamp_se uc_req = p->uclamp_req[clamp_id]; 1599#ifdef CONFIG_UCLAMP_TASK_GROUP 1600 unsigned int tg_min, tg_max, value; 1601 1602 /* 1603 * Tasks in autogroups or root task group will be 1604 * restricted by system defaults. 1605 */ 1606 if (task_group_is_autogroup(task_group(p))) 1607 return uc_req; 1608 if (task_group(p) == &root_task_group) 1609 return uc_req; 1610 1611 tg_min = task_group(p)->uclamp[UCLAMP_MIN].value; 1612 tg_max = task_group(p)->uclamp[UCLAMP_MAX].value; 1613 value = uc_req.value; 1614 value = clamp(value, tg_min, tg_max); 1615 uclamp_se_set(&uc_req, value, false); 1616#endif 1617 1618 return uc_req; 1619} 1620 1621/* 1622 * The effective clamp bucket index of a task depends on, by increasing 1623 * priority: 1624 * - the task specific clamp value, when explicitly requested from userspace 1625 * - the task group effective clamp value, for tasks not either in the root 1626 * group or in an autogroup 1627 * - the system default clamp value, defined by the sysadmin 1628 */ 1629static inline struct uclamp_se 1630uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id) 1631{ 1632 struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id); 1633 struct uclamp_se uc_max = uclamp_default[clamp_id]; 1634 1635 /* System default restrictions always apply */ 1636 if (unlikely(uc_req.value > uc_max.value)) 1637 return uc_max; 1638 1639 return uc_req; 1640} 1641 1642unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id) 1643{ 1644 struct uclamp_se uc_eff; 1645 1646 /* Task currently refcounted: use back-annotated (effective) value */ 1647 if (p->uclamp[clamp_id].active) 1648 return (unsigned long)p->uclamp[clamp_id].value; 1649 1650 uc_eff = uclamp_eff_get(p, clamp_id); 1651 1652 return (unsigned long)uc_eff.value; 1653} 1654 1655/* 1656 * When a task is enqueued on a rq, the clamp bucket currently defined by the 1657 * task's uclamp::bucket_id is refcounted on that rq. This also immediately 1658 * updates the rq's clamp value if required. 1659 * 1660 * Tasks can have a task-specific value requested from user-space, track 1661 * within each bucket the maximum value for tasks refcounted in it. 1662 * This "local max aggregation" allows to track the exact "requested" value 1663 * for each bucket when all its RUNNABLE tasks require the same clamp. 1664 */ 1665static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p, 1666 enum uclamp_id clamp_id) 1667{ 1668 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1669 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1670 struct uclamp_bucket *bucket; 1671 1672 lockdep_assert_rq_held(rq); 1673 1674 /* Update task effective clamp */ 1675 p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id); 1676 1677 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1678 bucket->tasks++; 1679 uc_se->active = true; 1680 1681 uclamp_idle_reset(rq, clamp_id, uc_se->value); 1682 1683 /* 1684 * Local max aggregation: rq buckets always track the max 1685 * "requested" clamp value of its RUNNABLE tasks. 1686 */ 1687 if (bucket->tasks == 1 || uc_se->value > bucket->value) 1688 bucket->value = uc_se->value; 1689 1690 if (uc_se->value > uclamp_rq_get(rq, clamp_id)) 1691 uclamp_rq_set(rq, clamp_id, uc_se->value); 1692} 1693 1694/* 1695 * When a task is dequeued from a rq, the clamp bucket refcounted by the task 1696 * is released. If this is the last task reference counting the rq's max 1697 * active clamp value, then the rq's clamp value is updated. 1698 * 1699 * Both refcounted tasks and rq's cached clamp values are expected to be 1700 * always valid. If it's detected they are not, as defensive programming, 1701 * enforce the expected state and warn. 1702 */ 1703static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p, 1704 enum uclamp_id clamp_id) 1705{ 1706 struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id]; 1707 struct uclamp_se *uc_se = &p->uclamp[clamp_id]; 1708 struct uclamp_bucket *bucket; 1709 unsigned int bkt_clamp; 1710 unsigned int rq_clamp; 1711 1712 lockdep_assert_rq_held(rq); 1713 1714 /* 1715 * If sched_uclamp_used was enabled after task @p was enqueued, 1716 * we could end up with unbalanced call to uclamp_rq_dec_id(). 1717 * 1718 * In this case the uc_se->active flag should be false since no uclamp 1719 * accounting was performed at enqueue time and we can just return 1720 * here. 1721 * 1722 * Need to be careful of the following enqueue/dequeue ordering 1723 * problem too 1724 * 1725 * enqueue(taskA) 1726 * // sched_uclamp_used gets enabled 1727 * enqueue(taskB) 1728 * dequeue(taskA) 1729 * // Must not decrement bucket->tasks here 1730 * dequeue(taskB) 1731 * 1732 * where we could end up with stale data in uc_se and 1733 * bucket[uc_se->bucket_id]. 1734 * 1735 * The following check here eliminates the possibility of such race. 1736 */ 1737 if (unlikely(!uc_se->active)) 1738 return; 1739 1740 bucket = &uc_rq->bucket[uc_se->bucket_id]; 1741 1742 WARN_ON_ONCE(!bucket->tasks); 1743 if (likely(bucket->tasks)) 1744 bucket->tasks--; 1745 1746 uc_se->active = false; 1747 1748 /* 1749 * Keep "local max aggregation" simple and accept to (possibly) 1750 * overboost some RUNNABLE tasks in the same bucket. 1751 * The rq clamp bucket value is reset to its base value whenever 1752 * there are no more RUNNABLE tasks refcounting it. 1753 */ 1754 if (likely(bucket->tasks)) 1755 return; 1756 1757 rq_clamp = uclamp_rq_get(rq, clamp_id); 1758 /* 1759 * Defensive programming: this should never happen. If it happens, 1760 * e.g. due to future modification, warn and fix up the expected value. 1761 */ 1762 WARN_ON_ONCE(bucket->value > rq_clamp); 1763 if (bucket->value >= rq_clamp) { 1764 bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value); 1765 uclamp_rq_set(rq, clamp_id, bkt_clamp); 1766 } 1767} 1768 1769static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags) 1770{ 1771 enum uclamp_id clamp_id; 1772 1773 /* 1774 * Avoid any overhead until uclamp is actually used by the userspace. 1775 * 1776 * The condition is constructed such that a NOP is generated when 1777 * sched_uclamp_used is disabled. 1778 */ 1779 if (!uclamp_is_used()) 1780 return; 1781 1782 if (unlikely(!p->sched_class->uclamp_enabled)) 1783 return; 1784 1785 /* Only inc the delayed task which being woken up. */ 1786 if (p->se.sched_delayed && !(flags & ENQUEUE_DELAYED)) 1787 return; 1788 1789 for_each_clamp_id(clamp_id) 1790 uclamp_rq_inc_id(rq, p, clamp_id); 1791 1792 /* Reset clamp idle holding when there is one RUNNABLE task */ 1793 if (rq->uclamp_flags & UCLAMP_FLAG_IDLE) 1794 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1795} 1796 1797static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) 1798{ 1799 enum uclamp_id clamp_id; 1800 1801 /* 1802 * Avoid any overhead until uclamp is actually used by the userspace. 1803 * 1804 * The condition is constructed such that a NOP is generated when 1805 * sched_uclamp_used is disabled. 1806 */ 1807 if (!uclamp_is_used()) 1808 return; 1809 1810 if (unlikely(!p->sched_class->uclamp_enabled)) 1811 return; 1812 1813 if (p->se.sched_delayed) 1814 return; 1815 1816 for_each_clamp_id(clamp_id) 1817 uclamp_rq_dec_id(rq, p, clamp_id); 1818} 1819 1820static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p, 1821 enum uclamp_id clamp_id) 1822{ 1823 if (!p->uclamp[clamp_id].active) 1824 return; 1825 1826 uclamp_rq_dec_id(rq, p, clamp_id); 1827 uclamp_rq_inc_id(rq, p, clamp_id); 1828 1829 /* 1830 * Make sure to clear the idle flag if we've transiently reached 0 1831 * active tasks on rq. 1832 */ 1833 if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE)) 1834 rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE; 1835} 1836 1837static inline void 1838uclamp_update_active(struct task_struct *p) 1839{ 1840 enum uclamp_id clamp_id; 1841 struct rq_flags rf; 1842 struct rq *rq; 1843 1844 /* 1845 * Lock the task and the rq where the task is (or was) queued. 1846 * 1847 * We might lock the (previous) rq of a !RUNNABLE task, but that's the 1848 * price to pay to safely serialize util_{min,max} updates with 1849 * enqueues, dequeues and migration operations. 1850 * This is the same locking schema used by __set_cpus_allowed_ptr(). 1851 */ 1852 rq = task_rq_lock(p, &rf); 1853 1854 /* 1855 * Setting the clamp bucket is serialized by task_rq_lock(). 1856 * If the task is not yet RUNNABLE and its task_struct is not 1857 * affecting a valid clamp bucket, the next time it's enqueued, 1858 * it will already see the updated clamp bucket value. 1859 */ 1860 for_each_clamp_id(clamp_id) 1861 uclamp_rq_reinc_id(rq, p, clamp_id); 1862 1863 task_rq_unlock(rq, p, &rf); 1864} 1865 1866#ifdef CONFIG_UCLAMP_TASK_GROUP 1867static inline void 1868uclamp_update_active_tasks(struct cgroup_subsys_state *css) 1869{ 1870 struct css_task_iter it; 1871 struct task_struct *p; 1872 1873 css_task_iter_start(css, 0, &it); 1874 while ((p = css_task_iter_next(&it))) 1875 uclamp_update_active(p); 1876 css_task_iter_end(&it); 1877} 1878 1879static void cpu_util_update_eff(struct cgroup_subsys_state *css); 1880#endif 1881 1882#ifdef CONFIG_SYSCTL 1883#ifdef CONFIG_UCLAMP_TASK_GROUP 1884static void uclamp_update_root_tg(void) 1885{ 1886 struct task_group *tg = &root_task_group; 1887 1888 uclamp_se_set(&tg->uclamp_req[UCLAMP_MIN], 1889 sysctl_sched_uclamp_util_min, false); 1890 uclamp_se_set(&tg->uclamp_req[UCLAMP_MAX], 1891 sysctl_sched_uclamp_util_max, false); 1892 1893 guard(rcu)(); 1894 cpu_util_update_eff(&root_task_group.css); 1895} 1896#else 1897static void uclamp_update_root_tg(void) { } 1898#endif 1899 1900static void uclamp_sync_util_min_rt_default(void) 1901{ 1902 struct task_struct *g, *p; 1903 1904 /* 1905 * copy_process() sysctl_uclamp 1906 * uclamp_min_rt = X; 1907 * write_lock(&tasklist_lock) read_lock(&tasklist_lock) 1908 * // link thread smp_mb__after_spinlock() 1909 * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); 1910 * sched_post_fork() for_each_process_thread() 1911 * __uclamp_sync_rt() __uclamp_sync_rt() 1912 * 1913 * Ensures that either sched_post_fork() will observe the new 1914 * uclamp_min_rt or for_each_process_thread() will observe the new 1915 * task. 1916 */ 1917 read_lock(&tasklist_lock); 1918 smp_mb__after_spinlock(); 1919 read_unlock(&tasklist_lock); 1920 1921 guard(rcu)(); 1922 for_each_process_thread(g, p) 1923 uclamp_update_util_min_rt_default(p); 1924} 1925 1926static int sysctl_sched_uclamp_handler(const struct ctl_table *table, int write, 1927 void *buffer, size_t *lenp, loff_t *ppos) 1928{ 1929 bool update_root_tg = false; 1930 int old_min, old_max, old_min_rt; 1931 int result; 1932 1933 guard(mutex)(&uclamp_mutex); 1934 1935 old_min = sysctl_sched_uclamp_util_min; 1936 old_max = sysctl_sched_uclamp_util_max; 1937 old_min_rt = sysctl_sched_uclamp_util_min_rt_default; 1938 1939 result = proc_dointvec(table, write, buffer, lenp, ppos); 1940 if (result) 1941 goto undo; 1942 if (!write) 1943 return 0; 1944 1945 if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max || 1946 sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE || 1947 sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) { 1948 1949 result = -EINVAL; 1950 goto undo; 1951 } 1952 1953 if (old_min != sysctl_sched_uclamp_util_min) { 1954 uclamp_se_set(&uclamp_default[UCLAMP_MIN], 1955 sysctl_sched_uclamp_util_min, false); 1956 update_root_tg = true; 1957 } 1958 if (old_max != sysctl_sched_uclamp_util_max) { 1959 uclamp_se_set(&uclamp_default[UCLAMP_MAX], 1960 sysctl_sched_uclamp_util_max, false); 1961 update_root_tg = true; 1962 } 1963 1964 if (update_root_tg) { 1965 sched_uclamp_enable(); 1966 uclamp_update_root_tg(); 1967 } 1968 1969 if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) { 1970 sched_uclamp_enable(); 1971 uclamp_sync_util_min_rt_default(); 1972 } 1973 1974 /* 1975 * We update all RUNNABLE tasks only when task groups are in use. 1976 * Otherwise, keep it simple and do just a lazy update at each next 1977 * task enqueue time. 1978 */ 1979 return 0; 1980 1981undo: 1982 sysctl_sched_uclamp_util_min = old_min; 1983 sysctl_sched_uclamp_util_max = old_max; 1984 sysctl_sched_uclamp_util_min_rt_default = old_min_rt; 1985 return result; 1986} 1987#endif /* CONFIG_SYSCTL */ 1988 1989static void uclamp_fork(struct task_struct *p) 1990{ 1991 enum uclamp_id clamp_id; 1992 1993 /* 1994 * We don't need to hold task_rq_lock() when updating p->uclamp_* here 1995 * as the task is still at its early fork stages. 1996 */ 1997 for_each_clamp_id(clamp_id) 1998 p->uclamp[clamp_id].active = false; 1999 2000 if (likely(!p->sched_reset_on_fork)) 2001 return; 2002 2003 for_each_clamp_id(clamp_id) { 2004 uclamp_se_set(&p->uclamp_req[clamp_id], 2005 uclamp_none(clamp_id), false); 2006 } 2007} 2008 2009static void uclamp_post_fork(struct task_struct *p) 2010{ 2011 uclamp_update_util_min_rt_default(p); 2012} 2013 2014static void __init init_uclamp_rq(struct rq *rq) 2015{ 2016 enum uclamp_id clamp_id; 2017 struct uclamp_rq *uc_rq = rq->uclamp; 2018 2019 for_each_clamp_id(clamp_id) { 2020 uc_rq[clamp_id] = (struct uclamp_rq) { 2021 .value = uclamp_none(clamp_id) 2022 }; 2023 } 2024 2025 rq->uclamp_flags = UCLAMP_FLAG_IDLE; 2026} 2027 2028static void __init init_uclamp(void) 2029{ 2030 struct uclamp_se uc_max = {}; 2031 enum uclamp_id clamp_id; 2032 int cpu; 2033 2034 for_each_possible_cpu(cpu) 2035 init_uclamp_rq(cpu_rq(cpu)); 2036 2037 for_each_clamp_id(clamp_id) { 2038 uclamp_se_set(&init_task.uclamp_req[clamp_id], 2039 uclamp_none(clamp_id), false); 2040 } 2041 2042 /* System defaults allow max clamp values for both indexes */ 2043 uclamp_se_set(&uc_max, uclamp_none(UCLAMP_MAX), false); 2044 for_each_clamp_id(clamp_id) { 2045 uclamp_default[clamp_id] = uc_max; 2046#ifdef CONFIG_UCLAMP_TASK_GROUP 2047 root_task_group.uclamp_req[clamp_id] = uc_max; 2048 root_task_group.uclamp[clamp_id] = uc_max; 2049#endif 2050 } 2051} 2052 2053#else /* !CONFIG_UCLAMP_TASK: */ 2054static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags) { } 2055static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { } 2056static inline void uclamp_fork(struct task_struct *p) { } 2057static inline void uclamp_post_fork(struct task_struct *p) { } 2058static inline void init_uclamp(void) { } 2059#endif /* !CONFIG_UCLAMP_TASK */ 2060 2061bool sched_task_on_rq(struct task_struct *p) 2062{ 2063 return task_on_rq_queued(p); 2064} 2065 2066unsigned long get_wchan(struct task_struct *p) 2067{ 2068 unsigned long ip = 0; 2069 unsigned int state; 2070 2071 if (!p || p == current) 2072 return 0; 2073 2074 /* Only get wchan if task is blocked and we can keep it that way. */ 2075 raw_spin_lock_irq(&p->pi_lock); 2076 state = READ_ONCE(p->__state); 2077 smp_rmb(); /* see try_to_wake_up() */ 2078 if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq) 2079 ip = __get_wchan(p); 2080 raw_spin_unlock_irq(&p->pi_lock); 2081 2082 return ip; 2083} 2084 2085void enqueue_task(struct rq *rq, struct task_struct *p, int flags) 2086{ 2087 if (!(flags & ENQUEUE_NOCLOCK)) 2088 update_rq_clock(rq); 2089 2090 /* 2091 * Can be before ->enqueue_task() because uclamp considers the 2092 * ENQUEUE_DELAYED task before its ->sched_delayed gets cleared 2093 * in ->enqueue_task(). 2094 */ 2095 uclamp_rq_inc(rq, p, flags); 2096 2097 rq->queue_mask |= p->sched_class->queue_mask; 2098 p->sched_class->enqueue_task(rq, p, flags); 2099 2100 psi_enqueue(p, flags); 2101 2102 if (!(flags & ENQUEUE_RESTORE)) 2103 sched_info_enqueue(rq, p); 2104 2105 if (sched_core_enabled(rq)) 2106 sched_core_enqueue(rq, p); 2107} 2108 2109/* 2110 * Must only return false when DEQUEUE_SLEEP. 2111 */ 2112inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags) 2113{ 2114 if (sched_core_enabled(rq)) 2115 sched_core_dequeue(rq, p, flags); 2116 2117 if (!(flags & DEQUEUE_NOCLOCK)) 2118 update_rq_clock(rq); 2119 2120 if (!(flags & DEQUEUE_SAVE)) 2121 sched_info_dequeue(rq, p); 2122 2123 psi_dequeue(p, flags); 2124 2125 /* 2126 * Must be before ->dequeue_task() because ->dequeue_task() can 'fail' 2127 * and mark the task ->sched_delayed. 2128 */ 2129 uclamp_rq_dec(rq, p); 2130 rq->queue_mask |= p->sched_class->queue_mask; 2131 return p->sched_class->dequeue_task(rq, p, flags); 2132} 2133 2134void activate_task(struct rq *rq, struct task_struct *p, int flags) 2135{ 2136 if (task_on_rq_migrating(p)) 2137 flags |= ENQUEUE_MIGRATED; 2138 2139 enqueue_task(rq, p, flags); 2140 2141 WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED); 2142 ASSERT_EXCLUSIVE_WRITER(p->on_rq); 2143} 2144 2145void deactivate_task(struct rq *rq, struct task_struct *p, int flags) 2146{ 2147 WARN_ON_ONCE(flags & DEQUEUE_SLEEP); 2148 2149 WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING); 2150 ASSERT_EXCLUSIVE_WRITER(p->on_rq); 2151 2152 /* 2153 * Code explicitly relies on TASK_ON_RQ_MIGRATING begin set *before* 2154 * dequeue_task() and cleared *after* enqueue_task(). 2155 */ 2156 2157 dequeue_task(rq, p, flags); 2158} 2159 2160static void block_task(struct rq *rq, struct task_struct *p, int flags) 2161{ 2162 if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags)) 2163 __block_task(rq, p); 2164} 2165 2166/** 2167 * task_curr - is this task currently executing on a CPU? 2168 * @p: the task in question. 2169 * 2170 * Return: 1 if the task is currently executing. 0 otherwise. 2171 */ 2172inline int task_curr(const struct task_struct *p) 2173{ 2174 return cpu_curr(task_cpu(p)) == p; 2175} 2176 2177void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags) 2178{ 2179 struct task_struct *donor = rq->donor; 2180 2181 if (p->sched_class == donor->sched_class) 2182 donor->sched_class->wakeup_preempt(rq, p, flags); 2183 else if (sched_class_above(p->sched_class, donor->sched_class)) 2184 resched_curr(rq); 2185 2186 /* 2187 * A queue event has occurred, and we're going to schedule. In 2188 * this case, we can save a useless back to back clock update. 2189 */ 2190 if (task_on_rq_queued(donor) && test_tsk_need_resched(rq->curr)) 2191 rq_clock_skip_update(rq); 2192} 2193 2194static __always_inline 2195int __task_state_match(struct task_struct *p, unsigned int state) 2196{ 2197 if (READ_ONCE(p->__state) & state) 2198 return 1; 2199 2200 if (READ_ONCE(p->saved_state) & state) 2201 return -1; 2202 2203 return 0; 2204} 2205 2206static __always_inline 2207int task_state_match(struct task_struct *p, unsigned int state) 2208{ 2209 /* 2210 * Serialize against current_save_and_set_rtlock_wait_state(), 2211 * current_restore_rtlock_saved_state(), and __refrigerator(). 2212 */ 2213 guard(raw_spinlock_irq)(&p->pi_lock); 2214 return __task_state_match(p, state); 2215} 2216 2217/* 2218 * wait_task_inactive - wait for a thread to unschedule. 2219 * 2220 * Wait for the thread to block in any of the states set in @match_state. 2221 * If it changes, i.e. @p might have woken up, then return zero. When we 2222 * succeed in waiting for @p to be off its CPU, we return a positive number 2223 * (its total switch count). If a second call a short while later returns the 2224 * same number, the caller can be sure that @p has remained unscheduled the 2225 * whole time. 2226 * 2227 * The caller must ensure that the task *will* unschedule sometime soon, 2228 * else this function might spin for a *long* time. This function can't 2229 * be called with interrupts off, or it may introduce deadlock with 2230 * smp_call_function() if an IPI is sent by the same process we are 2231 * waiting to become inactive. 2232 */ 2233unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state) 2234{ 2235 int running, queued, match; 2236 struct rq_flags rf; 2237 unsigned long ncsw; 2238 struct rq *rq; 2239 2240 for (;;) { 2241 /* 2242 * We do the initial early heuristics without holding 2243 * any task-queue locks at all. We'll only try to get 2244 * the runqueue lock when things look like they will 2245 * work out! 2246 */ 2247 rq = task_rq(p); 2248 2249 /* 2250 * If the task is actively running on another CPU 2251 * still, just relax and busy-wait without holding 2252 * any locks. 2253 * 2254 * NOTE! Since we don't hold any locks, it's not 2255 * even sure that "rq" stays as the right runqueue! 2256 * But we don't care, since "task_on_cpu()" will 2257 * return false if the runqueue has changed and p 2258 * is actually now running somewhere else! 2259 */ 2260 while (task_on_cpu(rq, p)) { 2261 if (!task_state_match(p, match_state)) 2262 return 0; 2263 cpu_relax(); 2264 } 2265 2266 /* 2267 * Ok, time to look more closely! We need the rq 2268 * lock now, to be *sure*. If we're wrong, we'll 2269 * just go back and repeat. 2270 */ 2271 rq = task_rq_lock(p, &rf); 2272 /* 2273 * If task is sched_delayed, force dequeue it, to avoid always 2274 * hitting the tick timeout in the queued case 2275 */ 2276 if (p->se.sched_delayed) 2277 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 2278 trace_sched_wait_task(p); 2279 running = task_on_cpu(rq, p); 2280 queued = task_on_rq_queued(p); 2281 ncsw = 0; 2282 if ((match = __task_state_match(p, match_state))) { 2283 /* 2284 * When matching on p->saved_state, consider this task 2285 * still queued so it will wait. 2286 */ 2287 if (match < 0) 2288 queued = 1; 2289 ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ 2290 } 2291 task_rq_unlock(rq, p, &rf); 2292 2293 /* 2294 * If it changed from the expected state, bail out now. 2295 */ 2296 if (unlikely(!ncsw)) 2297 break; 2298 2299 /* 2300 * Was it really running after all now that we 2301 * checked with the proper locks actually held? 2302 * 2303 * Oops. Go back and try again.. 2304 */ 2305 if (unlikely(running)) { 2306 cpu_relax(); 2307 continue; 2308 } 2309 2310 /* 2311 * It's not enough that it's not actively running, 2312 * it must be off the runqueue _entirely_, and not 2313 * preempted! 2314 * 2315 * So if it was still runnable (but just not actively 2316 * running right now), it's preempted, and we should 2317 * yield - it could be a while. 2318 */ 2319 if (unlikely(queued)) { 2320 ktime_t to = NSEC_PER_SEC / HZ; 2321 2322 set_current_state(TASK_UNINTERRUPTIBLE); 2323 schedule_hrtimeout(&to, HRTIMER_MODE_REL_HARD); 2324 continue; 2325 } 2326 2327 /* 2328 * Ahh, all good. It wasn't running, and it wasn't 2329 * runnable, which means that it will never become 2330 * running in the future either. We're all done! 2331 */ 2332 break; 2333 } 2334 2335 return ncsw; 2336} 2337 2338static void 2339do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx); 2340 2341static void migrate_disable_switch(struct rq *rq, struct task_struct *p) 2342{ 2343 struct affinity_context ac = { 2344 .new_mask = cpumask_of(rq->cpu), 2345 .flags = SCA_MIGRATE_DISABLE, 2346 }; 2347 2348 if (likely(!p->migration_disabled)) 2349 return; 2350 2351 if (p->cpus_ptr != &p->cpus_mask) 2352 return; 2353 2354 scoped_guard (task_rq_lock, p) 2355 do_set_cpus_allowed(p, &ac); 2356} 2357 2358void ___migrate_enable(void) 2359{ 2360 struct task_struct *p = current; 2361 struct affinity_context ac = { 2362 .new_mask = &p->cpus_mask, 2363 .flags = SCA_MIGRATE_ENABLE, 2364 }; 2365 2366 __set_cpus_allowed_ptr(p, &ac); 2367} 2368EXPORT_SYMBOL_GPL(___migrate_enable); 2369 2370void migrate_disable(void) 2371{ 2372 __migrate_disable(); 2373} 2374EXPORT_SYMBOL_GPL(migrate_disable); 2375 2376void migrate_enable(void) 2377{ 2378 __migrate_enable(); 2379} 2380EXPORT_SYMBOL_GPL(migrate_enable); 2381 2382static inline bool rq_has_pinned_tasks(struct rq *rq) 2383{ 2384 return rq->nr_pinned; 2385} 2386 2387/* 2388 * Per-CPU kthreads are allowed to run on !active && online CPUs, see 2389 * __set_cpus_allowed_ptr() and select_fallback_rq(). 2390 */ 2391static inline bool is_cpu_allowed(struct task_struct *p, int cpu) 2392{ 2393 /* When not in the task's cpumask, no point in looking further. */ 2394 if (!task_allowed_on_cpu(p, cpu)) 2395 return false; 2396 2397 /* migrate_disabled() must be allowed to finish. */ 2398 if (is_migration_disabled(p)) 2399 return cpu_online(cpu); 2400 2401 /* Non kernel threads are not allowed during either online or offline. */ 2402 if (!(p->flags & PF_KTHREAD)) 2403 return cpu_active(cpu); 2404 2405 /* KTHREAD_IS_PER_CPU is always allowed. */ 2406 if (kthread_is_per_cpu(p)) 2407 return cpu_online(cpu); 2408 2409 /* Regular kernel threads don't get to stay during offline. */ 2410 if (cpu_dying(cpu)) 2411 return false; 2412 2413 /* But are allowed during online. */ 2414 return cpu_online(cpu); 2415} 2416 2417/* 2418 * This is how migration works: 2419 * 2420 * 1) we invoke migration_cpu_stop() on the target CPU using 2421 * stop_one_cpu(). 2422 * 2) stopper starts to run (implicitly forcing the migrated thread 2423 * off the CPU) 2424 * 3) it checks whether the migrated task is still in the wrong runqueue. 2425 * 4) if it's in the wrong runqueue then the migration thread removes 2426 * it and puts it into the right queue. 2427 * 5) stopper completes and stop_one_cpu() returns and the migration 2428 * is done. 2429 */ 2430 2431/* 2432 * move_queued_task - move a queued task to new rq. 2433 * 2434 * Returns (locked) new rq. Old rq's lock is released. 2435 */ 2436static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, 2437 struct task_struct *p, int new_cpu) 2438{ 2439 lockdep_assert_rq_held(rq); 2440 2441 deactivate_task(rq, p, DEQUEUE_NOCLOCK); 2442 set_task_cpu(p, new_cpu); 2443 rq_unlock(rq, rf); 2444 2445 rq = cpu_rq(new_cpu); 2446 2447 rq_lock(rq, rf); 2448 WARN_ON_ONCE(task_cpu(p) != new_cpu); 2449 activate_task(rq, p, 0); 2450 wakeup_preempt(rq, p, 0); 2451 2452 return rq; 2453} 2454 2455struct migration_arg { 2456 struct task_struct *task; 2457 int dest_cpu; 2458 struct set_affinity_pending *pending; 2459}; 2460 2461/* 2462 * @refs: number of wait_for_completion() 2463 * @stop_pending: is @stop_work in use 2464 */ 2465struct set_affinity_pending { 2466 refcount_t refs; 2467 unsigned int stop_pending; 2468 struct completion done; 2469 struct cpu_stop_work stop_work; 2470 struct migration_arg arg; 2471}; 2472 2473/* 2474 * Move (not current) task off this CPU, onto the destination CPU. We're doing 2475 * this because either it can't run here any more (set_cpus_allowed() 2476 * away from this CPU, or CPU going down), or because we're 2477 * attempting to rebalance this task on exec (sched_exec). 2478 * 2479 * So we race with normal scheduler movements, but that's OK, as long 2480 * as the task is no longer on this CPU. 2481 */ 2482static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, 2483 struct task_struct *p, int dest_cpu) 2484{ 2485 /* Affinity changed (again). */ 2486 if (!is_cpu_allowed(p, dest_cpu)) 2487 return rq; 2488 2489 rq = move_queued_task(rq, rf, p, dest_cpu); 2490 2491 return rq; 2492} 2493 2494/* 2495 * migration_cpu_stop - this will be executed by a high-prio stopper thread 2496 * and performs thread migration by bumping thread off CPU then 2497 * 'pushing' onto another runqueue. 2498 */ 2499static int migration_cpu_stop(void *data) 2500{ 2501 struct migration_arg *arg = data; 2502 struct set_affinity_pending *pending = arg->pending; 2503 struct task_struct *p = arg->task; 2504 struct rq *rq = this_rq(); 2505 bool complete = false; 2506 struct rq_flags rf; 2507 2508 /* 2509 * The original target CPU might have gone down and we might 2510 * be on another CPU but it doesn't matter. 2511 */ 2512 local_irq_save(rf.flags); 2513 /* 2514 * We need to explicitly wake pending tasks before running 2515 * __migrate_task() such that we will not miss enforcing cpus_ptr 2516 * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. 2517 */ 2518 flush_smp_call_function_queue(); 2519 2520 raw_spin_lock(&p->pi_lock); 2521 rq_lock(rq, &rf); 2522 2523 /* 2524 * If we were passed a pending, then ->stop_pending was set, thus 2525 * p->migration_pending must have remained stable. 2526 */ 2527 WARN_ON_ONCE(pending && pending != p->migration_pending); 2528 2529 /* 2530 * If task_rq(p) != rq, it cannot be migrated here, because we're 2531 * holding rq->lock, if p->on_rq == 0 it cannot get enqueued because 2532 * we're holding p->pi_lock. 2533 */ 2534 if (task_rq(p) == rq) { 2535 if (is_migration_disabled(p)) 2536 goto out; 2537 2538 if (pending) { 2539 p->migration_pending = NULL; 2540 complete = true; 2541 2542 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) 2543 goto out; 2544 } 2545 2546 if (task_on_rq_queued(p)) { 2547 update_rq_clock(rq); 2548 rq = __migrate_task(rq, &rf, p, arg->dest_cpu); 2549 } else { 2550 p->wake_cpu = arg->dest_cpu; 2551 } 2552 2553 /* 2554 * XXX __migrate_task() can fail, at which point we might end 2555 * up running on a dodgy CPU, AFAICT this can only happen 2556 * during CPU hotplug, at which point we'll get pushed out 2557 * anyway, so it's probably not a big deal. 2558 */ 2559 2560 } else if (pending) { 2561 /* 2562 * This happens when we get migrated between migrate_enable()'s 2563 * preempt_enable() and scheduling the stopper task. At that 2564 * point we're a regular task again and not current anymore. 2565 * 2566 * A !PREEMPT kernel has a giant hole here, which makes it far 2567 * more likely. 2568 */ 2569 2570 /* 2571 * The task moved before the stopper got to run. We're holding 2572 * ->pi_lock, so the allowed mask is stable - if it got 2573 * somewhere allowed, we're done. 2574 */ 2575 if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) { 2576 p->migration_pending = NULL; 2577 complete = true; 2578 goto out; 2579 } 2580 2581 /* 2582 * When migrate_enable() hits a rq mis-match we can't reliably 2583 * determine is_migration_disabled() and so have to chase after 2584 * it. 2585 */ 2586 WARN_ON_ONCE(!pending->stop_pending); 2587 preempt_disable(); 2588 rq_unlock(rq, &rf); 2589 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 2590 stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop, 2591 &pending->arg, &pending->stop_work); 2592 preempt_enable(); 2593 return 0; 2594 } 2595out: 2596 if (pending) 2597 pending->stop_pending = false; 2598 rq_unlock(rq, &rf); 2599 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 2600 2601 if (complete) 2602 complete_all(&pending->done); 2603 2604 return 0; 2605} 2606 2607int push_cpu_stop(void *arg) 2608{ 2609 struct rq *lowest_rq = NULL, *rq = this_rq(); 2610 struct task_struct *p = arg; 2611 2612 raw_spin_lock_irq(&p->pi_lock); 2613 raw_spin_rq_lock(rq); 2614 2615 if (task_rq(p) != rq) 2616 goto out_unlock; 2617 2618 if (is_migration_disabled(p)) { 2619 p->migration_flags |= MDF_PUSH; 2620 goto out_unlock; 2621 } 2622 2623 p->migration_flags &= ~MDF_PUSH; 2624 2625 if (p->sched_class->find_lock_rq) 2626 lowest_rq = p->sched_class->find_lock_rq(p, rq); 2627 2628 if (!lowest_rq) 2629 goto out_unlock; 2630 2631 // XXX validate p is still the highest prio task 2632 if (task_rq(p) == rq) { 2633 move_queued_task_locked(rq, lowest_rq, p); 2634 resched_curr(lowest_rq); 2635 } 2636 2637 double_unlock_balance(rq, lowest_rq); 2638 2639out_unlock: 2640 rq->push_busy = false; 2641 raw_spin_rq_unlock(rq); 2642 raw_spin_unlock_irq(&p->pi_lock); 2643 2644 put_task_struct(p); 2645 return 0; 2646} 2647 2648static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask); 2649 2650/* 2651 * sched_class::set_cpus_allowed must do the below, but is not required to 2652 * actually call this function. 2653 */ 2654void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx) 2655{ 2656 if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) { 2657 p->cpus_ptr = ctx->new_mask; 2658 return; 2659 } 2660 2661 cpumask_copy(&p->cpus_mask, ctx->new_mask); 2662 p->nr_cpus_allowed = cpumask_weight(ctx->new_mask); 2663 mm_update_cpus_allowed(p->mm, ctx->new_mask); 2664 2665 /* 2666 * Swap in a new user_cpus_ptr if SCA_USER flag set 2667 */ 2668 if (ctx->flags & SCA_USER) 2669 swap(p->user_cpus_ptr, ctx->user_mask); 2670} 2671 2672static void 2673do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx) 2674{ 2675 scoped_guard (sched_change, p, DEQUEUE_SAVE) 2676 p->sched_class->set_cpus_allowed(p, ctx); 2677} 2678 2679/* 2680 * Used for kthread_bind() and select_fallback_rq(), in both cases the user 2681 * affinity (if any) should be destroyed too. 2682 */ 2683void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask) 2684{ 2685 struct affinity_context ac = { 2686 .new_mask = new_mask, 2687 .user_mask = NULL, 2688 .flags = SCA_USER, /* clear the user requested mask */ 2689 }; 2690 union cpumask_rcuhead { 2691 cpumask_t cpumask; 2692 struct rcu_head rcu; 2693 }; 2694 2695 scoped_guard (__task_rq_lock, p) 2696 do_set_cpus_allowed(p, &ac); 2697 2698 /* 2699 * Because this is called with p->pi_lock held, it is not possible 2700 * to use kfree() here (when PREEMPT_RT=y), therefore punt to using 2701 * kfree_rcu(). 2702 */ 2703 kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); 2704} 2705 2706int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, 2707 int node) 2708{ 2709 cpumask_t *user_mask; 2710 unsigned long flags; 2711 2712 /* 2713 * Always clear dst->user_cpus_ptr first as their user_cpus_ptr's 2714 * may differ by now due to racing. 2715 */ 2716 dst->user_cpus_ptr = NULL; 2717 2718 /* 2719 * This check is racy and losing the race is a valid situation. 2720 * It is not worth the extra overhead of taking the pi_lock on 2721 * every fork/clone. 2722 */ 2723 if (data_race(!src->user_cpus_ptr)) 2724 return 0; 2725 2726 user_mask = alloc_user_cpus_ptr(node); 2727 if (!user_mask) 2728 return -ENOMEM; 2729 2730 /* 2731 * Use pi_lock to protect content of user_cpus_ptr 2732 * 2733 * Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent 2734 * set_cpus_allowed_force(). 2735 */ 2736 raw_spin_lock_irqsave(&src->pi_lock, flags); 2737 if (src->user_cpus_ptr) { 2738 swap(dst->user_cpus_ptr, user_mask); 2739 cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr); 2740 } 2741 raw_spin_unlock_irqrestore(&src->pi_lock, flags); 2742 2743 if (unlikely(user_mask)) 2744 kfree(user_mask); 2745 2746 return 0; 2747} 2748 2749static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p) 2750{ 2751 struct cpumask *user_mask = NULL; 2752 2753 swap(p->user_cpus_ptr, user_mask); 2754 2755 return user_mask; 2756} 2757 2758void release_user_cpus_ptr(struct task_struct *p) 2759{ 2760 kfree(clear_user_cpus_ptr(p)); 2761} 2762 2763/* 2764 * This function is wildly self concurrent; here be dragons. 2765 * 2766 * 2767 * When given a valid mask, __set_cpus_allowed_ptr() must block until the 2768 * designated task is enqueued on an allowed CPU. If that task is currently 2769 * running, we have to kick it out using the CPU stopper. 2770 * 2771 * Migrate-Disable comes along and tramples all over our nice sandcastle. 2772 * Consider: 2773 * 2774 * Initial conditions: P0->cpus_mask = [0, 1] 2775 * 2776 * P0@CPU0 P1 2777 * 2778 * migrate_disable(); 2779 * <preempted> 2780 * set_cpus_allowed_ptr(P0, [1]); 2781 * 2782 * P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes 2783 * its outermost migrate_enable() (i.e. it exits its Migrate-Disable region). 2784 * This means we need the following scheme: 2785 * 2786 * P0@CPU0 P1 2787 * 2788 * migrate_disable(); 2789 * <preempted> 2790 * set_cpus_allowed_ptr(P0, [1]); 2791 * <blocks> 2792 * <resumes> 2793 * migrate_enable(); 2794 * __set_cpus_allowed_ptr(); 2795 * <wakes local stopper> 2796 * `--> <woken on migration completion> 2797 * 2798 * Now the fun stuff: there may be several P1-like tasks, i.e. multiple 2799 * concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any 2800 * task p are serialized by p->pi_lock, which we can leverage: the one that 2801 * should come into effect at the end of the Migrate-Disable region is the last 2802 * one. This means we only need to track a single cpumask (i.e. p->cpus_mask), 2803 * but we still need to properly signal those waiting tasks at the appropriate 2804 * moment. 2805 * 2806 * This is implemented using struct set_affinity_pending. The first 2807 * __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will 2808 * setup an instance of that struct and install it on the targeted task_struct. 2809 * Any and all further callers will reuse that instance. Those then wait for 2810 * a completion signaled at the tail of the CPU stopper callback (1), triggered 2811 * on the end of the Migrate-Disable region (i.e. outermost migrate_enable()). 2812 * 2813 * 2814 * (1) In the cases covered above. There is one more where the completion is 2815 * signaled within affine_move_task() itself: when a subsequent affinity request 2816 * occurs after the stopper bailed out due to the targeted task still being 2817 * Migrate-Disable. Consider: 2818 * 2819 * Initial conditions: P0->cpus_mask = [0, 1] 2820 * 2821 * CPU0 P1 P2 2822 * <P0> 2823 * migrate_disable(); 2824 * <preempted> 2825 * set_cpus_allowed_ptr(P0, [1]); 2826 * <blocks> 2827 * <migration/0> 2828 * migration_cpu_stop() 2829 * is_migration_disabled() 2830 * <bails> 2831 * set_cpus_allowed_ptr(P0, [0, 1]); 2832 * <signal completion> 2833 * <awakes> 2834 * 2835 * Note that the above is safe vs a concurrent migrate_enable(), as any 2836 * pending affinity completion is preceded by an uninstallation of 2837 * p->migration_pending done with p->pi_lock held. 2838 */ 2839static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, 2840 int dest_cpu, unsigned int flags) 2841 __releases(rq->lock) 2842 __releases(p->pi_lock) 2843{ 2844 struct set_affinity_pending my_pending = { }, *pending = NULL; 2845 bool stop_pending, complete = false; 2846 2847 /* 2848 * Can the task run on the task's current CPU? If so, we're done 2849 * 2850 * We are also done if the task is the current donor, boosting a lock- 2851 * holding proxy, (and potentially has been migrated outside its 2852 * current or previous affinity mask) 2853 */ 2854 if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask) || 2855 (task_current_donor(rq, p) && !task_current(rq, p))) { 2856 struct task_struct *push_task = NULL; 2857 2858 if ((flags & SCA_MIGRATE_ENABLE) && 2859 (p->migration_flags & MDF_PUSH) && !rq->push_busy) { 2860 rq->push_busy = true; 2861 push_task = get_task_struct(p); 2862 } 2863 2864 /* 2865 * If there are pending waiters, but no pending stop_work, 2866 * then complete now. 2867 */ 2868 pending = p->migration_pending; 2869 if (pending && !pending->stop_pending) { 2870 p->migration_pending = NULL; 2871 complete = true; 2872 } 2873 2874 preempt_disable(); 2875 task_rq_unlock(rq, p, rf); 2876 if (push_task) { 2877 stop_one_cpu_nowait(rq->cpu, push_cpu_stop, 2878 p, &rq->push_work); 2879 } 2880 preempt_enable(); 2881 2882 if (complete) 2883 complete_all(&pending->done); 2884 2885 return 0; 2886 } 2887 2888 if (!(flags & SCA_MIGRATE_ENABLE)) { 2889 /* serialized by p->pi_lock */ 2890 if (!p->migration_pending) { 2891 /* Install the request */ 2892 refcount_set(&my_pending.refs, 1); 2893 init_completion(&my_pending.done); 2894 my_pending.arg = (struct migration_arg) { 2895 .task = p, 2896 .dest_cpu = dest_cpu, 2897 .pending = &my_pending, 2898 }; 2899 2900 p->migration_pending = &my_pending; 2901 } else { 2902 pending = p->migration_pending; 2903 refcount_inc(&pending->refs); 2904 /* 2905 * Affinity has changed, but we've already installed a 2906 * pending. migration_cpu_stop() *must* see this, else 2907 * we risk a completion of the pending despite having a 2908 * task on a disallowed CPU. 2909 * 2910 * Serialized by p->pi_lock, so this is safe. 2911 */ 2912 pending->arg.dest_cpu = dest_cpu; 2913 } 2914 } 2915 pending = p->migration_pending; 2916 /* 2917 * - !MIGRATE_ENABLE: 2918 * we'll have installed a pending if there wasn't one already. 2919 * 2920 * - MIGRATE_ENABLE: 2921 * we're here because the current CPU isn't matching anymore, 2922 * the only way that can happen is because of a concurrent 2923 * set_cpus_allowed_ptr() call, which should then still be 2924 * pending completion. 2925 * 2926 * Either way, we really should have a @pending here. 2927 */ 2928 if (WARN_ON_ONCE(!pending)) { 2929 task_rq_unlock(rq, p, rf); 2930 return -EINVAL; 2931 } 2932 2933 if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) { 2934 /* 2935 * MIGRATE_ENABLE gets here because 'p == current', but for 2936 * anything else we cannot do is_migration_disabled(), punt 2937 * and have the stopper function handle it all race-free. 2938 */ 2939 stop_pending = pending->stop_pending; 2940 if (!stop_pending) 2941 pending->stop_pending = true; 2942 2943 if (flags & SCA_MIGRATE_ENABLE) 2944 p->migration_flags &= ~MDF_PUSH; 2945 2946 preempt_disable(); 2947 task_rq_unlock(rq, p, rf); 2948 if (!stop_pending) { 2949 stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop, 2950 &pending->arg, &pending->stop_work); 2951 } 2952 preempt_enable(); 2953 2954 if (flags & SCA_MIGRATE_ENABLE) 2955 return 0; 2956 } else { 2957 2958 if (!is_migration_disabled(p)) { 2959 if (task_on_rq_queued(p)) 2960 rq = move_queued_task(rq, rf, p, dest_cpu); 2961 2962 if (!pending->stop_pending) { 2963 p->migration_pending = NULL; 2964 complete = true; 2965 } 2966 } 2967 task_rq_unlock(rq, p, rf); 2968 2969 if (complete) 2970 complete_all(&pending->done); 2971 } 2972 2973 wait_for_completion(&pending->done); 2974 2975 if (refcount_dec_and_test(&pending->refs)) 2976 wake_up_var(&pending->refs); /* No UaF, just an address */ 2977 2978 /* 2979 * Block the original owner of &pending until all subsequent callers 2980 * have seen the completion and decremented the refcount 2981 */ 2982 wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs)); 2983 2984 /* ARGH */ 2985 WARN_ON_ONCE(my_pending.stop_pending); 2986 2987 return 0; 2988} 2989 2990/* 2991 * Called with both p->pi_lock and rq->lock held; drops both before returning. 2992 */ 2993static int __set_cpus_allowed_ptr_locked(struct task_struct *p, 2994 struct affinity_context *ctx, 2995 struct rq *rq, 2996 struct rq_flags *rf) 2997 __releases(rq->lock) 2998 __releases(p->pi_lock) 2999{ 3000 const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); 3001 const struct cpumask *cpu_valid_mask = cpu_active_mask; 3002 bool kthread = p->flags & PF_KTHREAD; 3003 unsigned int dest_cpu; 3004 int ret = 0; 3005 3006 if (kthread || is_migration_disabled(p)) { 3007 /* 3008 * Kernel threads are allowed on online && !active CPUs, 3009 * however, during cpu-hot-unplug, even these might get pushed 3010 * away if not KTHREAD_IS_PER_CPU. 3011 * 3012 * Specifically, migration_disabled() tasks must not fail the 3013 * cpumask_any_and_distribute() pick below, esp. so on 3014 * SCA_MIGRATE_ENABLE, otherwise we'll not call 3015 * set_cpus_allowed_common() and actually reset p->cpus_ptr. 3016 */ 3017 cpu_valid_mask = cpu_online_mask; 3018 } 3019 3020 if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) { 3021 ret = -EINVAL; 3022 goto out; 3023 } 3024 3025 /* 3026 * Must re-check here, to close a race against __kthread_bind(), 3027 * sched_setaffinity() is not guaranteed to observe the flag. 3028 */ 3029 if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) { 3030 ret = -EINVAL; 3031 goto out; 3032 } 3033 3034 if (!(ctx->flags & SCA_MIGRATE_ENABLE)) { 3035 if (cpumask_equal(&p->cpus_mask, ctx->new_mask)) { 3036 if (ctx->flags & SCA_USER) 3037 swap(p->user_cpus_ptr, ctx->user_mask); 3038 goto out; 3039 } 3040 3041 if (WARN_ON_ONCE(p == current && 3042 is_migration_disabled(p) && 3043 !cpumask_test_cpu(task_cpu(p), ctx->new_mask))) { 3044 ret = -EBUSY; 3045 goto out; 3046 } 3047 } 3048 3049 /* 3050 * Picking a ~random cpu helps in cases where we are changing affinity 3051 * for groups of tasks (ie. cpuset), so that load balancing is not 3052 * immediately required to distribute the tasks within their new mask. 3053 */ 3054 dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask); 3055 if (dest_cpu >= nr_cpu_ids) { 3056 ret = -EINVAL; 3057 goto out; 3058 } 3059 3060 do_set_cpus_allowed(p, ctx); 3061 3062 return affine_move_task(rq, p, rf, dest_cpu, ctx->flags); 3063 3064out: 3065 task_rq_unlock(rq, p, rf); 3066 3067 return ret; 3068} 3069 3070/* 3071 * Change a given task's CPU affinity. Migrate the thread to a 3072 * proper CPU and schedule it away if the CPU it's executing on 3073 * is removed from the allowed bitmask. 3074 * 3075 * NOTE: the caller must have a valid reference to the task, the 3076 * task must not exit() & deallocate itself prematurely. The 3077 * call is not atomic; no spinlocks may be held. 3078 */ 3079int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx) 3080{ 3081 struct rq_flags rf; 3082 struct rq *rq; 3083 3084 rq = task_rq_lock(p, &rf); 3085 /* 3086 * Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_* 3087 * flags are set. 3088 */ 3089 if (p->user_cpus_ptr && 3090 !(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) && 3091 cpumask_and(rq->scratch_mask, ctx->new_mask, p->user_cpus_ptr)) 3092 ctx->new_mask = rq->scratch_mask; 3093 3094 return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf); 3095} 3096 3097int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) 3098{ 3099 struct affinity_context ac = { 3100 .new_mask = new_mask, 3101 .flags = 0, 3102 }; 3103 3104 return __set_cpus_allowed_ptr(p, &ac); 3105} 3106EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); 3107 3108/* 3109 * Change a given task's CPU affinity to the intersection of its current 3110 * affinity mask and @subset_mask, writing the resulting mask to @new_mask. 3111 * If user_cpus_ptr is defined, use it as the basis for restricting CPU 3112 * affinity or use cpu_online_mask instead. 3113 * 3114 * If the resulting mask is empty, leave the affinity unchanged and return 3115 * -EINVAL. 3116 */ 3117static int restrict_cpus_allowed_ptr(struct task_struct *p, 3118 struct cpumask *new_mask, 3119 const struct cpumask *subset_mask) 3120{ 3121 struct affinity_context ac = { 3122 .new_mask = new_mask, 3123 .flags = 0, 3124 }; 3125 struct rq_flags rf; 3126 struct rq *rq; 3127 int err; 3128 3129 rq = task_rq_lock(p, &rf); 3130 3131 /* 3132 * Forcefully restricting the affinity of a deadline task is 3133 * likely to cause problems, so fail and noisily override the 3134 * mask entirely. 3135 */ 3136 if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { 3137 err = -EPERM; 3138 goto err_unlock; 3139 } 3140 3141 if (!cpumask_and(new_mask, task_user_cpus(p), subset_mask)) { 3142 err = -EINVAL; 3143 goto err_unlock; 3144 } 3145 3146 return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf); 3147 3148err_unlock: 3149 task_rq_unlock(rq, p, &rf); 3150 return err; 3151} 3152 3153/* 3154 * Restrict the CPU affinity of task @p so that it is a subset of 3155 * task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the 3156 * old affinity mask. If the resulting mask is empty, we warn and walk 3157 * up the cpuset hierarchy until we find a suitable mask. 3158 */ 3159void force_compatible_cpus_allowed_ptr(struct task_struct *p) 3160{ 3161 cpumask_var_t new_mask; 3162 const struct cpumask *override_mask = task_cpu_possible_mask(p); 3163 3164 alloc_cpumask_var(&new_mask, GFP_KERNEL); 3165 3166 /* 3167 * __migrate_task() can fail silently in the face of concurrent 3168 * offlining of the chosen destination CPU, so take the hotplug 3169 * lock to ensure that the migration succeeds. 3170 */ 3171 cpus_read_lock(); 3172 if (!cpumask_available(new_mask)) 3173 goto out_set_mask; 3174 3175 if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask)) 3176 goto out_free_mask; 3177 3178 /* 3179 * We failed to find a valid subset of the affinity mask for the 3180 * task, so override it based on its cpuset hierarchy. 3181 */ 3182 cpuset_cpus_allowed(p, new_mask); 3183 override_mask = new_mask; 3184 3185out_set_mask: 3186 if (printk_ratelimit()) { 3187 printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n", 3188 task_pid_nr(p), p->comm, 3189 cpumask_pr_args(override_mask)); 3190 } 3191 3192 WARN_ON(set_cpus_allowed_ptr(p, override_mask)); 3193out_free_mask: 3194 cpus_read_unlock(); 3195 free_cpumask_var(new_mask); 3196} 3197 3198/* 3199 * Restore the affinity of a task @p which was previously restricted by a 3200 * call to force_compatible_cpus_allowed_ptr(). 3201 * 3202 * It is the caller's responsibility to serialise this with any calls to 3203 * force_compatible_cpus_allowed_ptr(@p). 3204 */ 3205void relax_compatible_cpus_allowed_ptr(struct task_struct *p) 3206{ 3207 struct affinity_context ac = { 3208 .new_mask = task_user_cpus(p), 3209 .flags = 0, 3210 }; 3211 int ret; 3212 3213 /* 3214 * Try to restore the old affinity mask with __sched_setaffinity(). 3215 * Cpuset masking will be done there too. 3216 */ 3217 ret = __sched_setaffinity(p, &ac); 3218 WARN_ON_ONCE(ret); 3219} 3220 3221#ifdef CONFIG_SMP 3222 3223void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 3224{ 3225 unsigned int state = READ_ONCE(p->__state); 3226 3227 /* 3228 * We should never call set_task_cpu() on a blocked task, 3229 * ttwu() will sort out the placement. 3230 */ 3231 WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq); 3232 3233 /* 3234 * Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING, 3235 * because schedstat_wait_{start,end} rebase migrating task's wait_start 3236 * time relying on p->on_rq. 3237 */ 3238 WARN_ON_ONCE(state == TASK_RUNNING && 3239 p->sched_class == &fair_sched_class && 3240 (p->on_rq && !task_on_rq_migrating(p))); 3241 3242#ifdef CONFIG_LOCKDEP 3243 /* 3244 * The caller should hold either p->pi_lock or rq->lock, when changing 3245 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. 3246 * 3247 * sched_move_task() holds both and thus holding either pins the cgroup, 3248 * see task_group(). 3249 * 3250 * Furthermore, all task_rq users should acquire both locks, see 3251 * task_rq_lock(). 3252 */ 3253 WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || 3254 lockdep_is_held(__rq_lockp(task_rq(p))))); 3255#endif 3256 /* 3257 * Clearly, migrating tasks to offline CPUs is a fairly daft thing. 3258 */ 3259 WARN_ON_ONCE(!cpu_online(new_cpu)); 3260 3261 WARN_ON_ONCE(is_migration_disabled(p)); 3262 3263 trace_sched_migrate_task(p, new_cpu); 3264 3265 if (task_cpu(p) != new_cpu) { 3266 if (p->sched_class->migrate_task_rq) 3267 p->sched_class->migrate_task_rq(p, new_cpu); 3268 p->se.nr_migrations++; 3269 perf_event_task_migrate(p); 3270 } 3271 3272 __set_task_cpu(p, new_cpu); 3273} 3274#endif /* CONFIG_SMP */ 3275 3276#ifdef CONFIG_NUMA_BALANCING 3277static void __migrate_swap_task(struct task_struct *p, int cpu) 3278{ 3279 if (task_on_rq_queued(p)) { 3280 struct rq *src_rq, *dst_rq; 3281 struct rq_flags srf, drf; 3282 3283 src_rq = task_rq(p); 3284 dst_rq = cpu_rq(cpu); 3285 3286 rq_pin_lock(src_rq, &srf); 3287 rq_pin_lock(dst_rq, &drf); 3288 3289 move_queued_task_locked(src_rq, dst_rq, p); 3290 wakeup_preempt(dst_rq, p, 0); 3291 3292 rq_unpin_lock(dst_rq, &drf); 3293 rq_unpin_lock(src_rq, &srf); 3294 3295 } else { 3296 /* 3297 * Task isn't running anymore; make it appear like we migrated 3298 * it before it went to sleep. This means on wakeup we make the 3299 * previous CPU our target instead of where it really is. 3300 */ 3301 p->wake_cpu = cpu; 3302 } 3303} 3304 3305struct migration_swap_arg { 3306 struct task_struct *src_task, *dst_task; 3307 int src_cpu, dst_cpu; 3308}; 3309 3310static int migrate_swap_stop(void *data) 3311{ 3312 struct migration_swap_arg *arg = data; 3313 struct rq *src_rq, *dst_rq; 3314 3315 if (!cpu_active(arg->src_cpu) || !cpu_active(arg->dst_cpu)) 3316 return -EAGAIN; 3317 3318 src_rq = cpu_rq(arg->src_cpu); 3319 dst_rq = cpu_rq(arg->dst_cpu); 3320 3321 guard(double_raw_spinlock)(&arg->src_task->pi_lock, &arg->dst_task->pi_lock); 3322 guard(double_rq_lock)(src_rq, dst_rq); 3323 3324 if (task_cpu(arg->dst_task) != arg->dst_cpu) 3325 return -EAGAIN; 3326 3327 if (task_cpu(arg->src_task) != arg->src_cpu) 3328 return -EAGAIN; 3329 3330 if (!cpumask_test_cpu(arg->dst_cpu, arg->src_task->cpus_ptr)) 3331 return -EAGAIN; 3332 3333 if (!cpumask_test_cpu(arg->src_cpu, arg->dst_task->cpus_ptr)) 3334 return -EAGAIN; 3335 3336 __migrate_swap_task(arg->src_task, arg->dst_cpu); 3337 __migrate_swap_task(arg->dst_task, arg->src_cpu); 3338 3339 return 0; 3340} 3341 3342/* 3343 * Cross migrate two tasks 3344 */ 3345int migrate_swap(struct task_struct *cur, struct task_struct *p, 3346 int target_cpu, int curr_cpu) 3347{ 3348 struct migration_swap_arg arg; 3349 int ret = -EINVAL; 3350 3351 arg = (struct migration_swap_arg){ 3352 .src_task = cur, 3353 .src_cpu = curr_cpu, 3354 .dst_task = p, 3355 .dst_cpu = target_cpu, 3356 }; 3357 3358 if (arg.src_cpu == arg.dst_cpu) 3359 goto out; 3360 3361 /* 3362 * These three tests are all lockless; this is OK since all of them 3363 * will be re-checked with proper locks held further down the line. 3364 */ 3365 if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) 3366 goto out; 3367 3368 if (!cpumask_test_cpu(arg.dst_cpu, arg.src_task->cpus_ptr)) 3369 goto out; 3370 3371 if (!cpumask_test_cpu(arg.src_cpu, arg.dst_task->cpus_ptr)) 3372 goto out; 3373 3374 trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); 3375 ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); 3376 3377out: 3378 return ret; 3379} 3380#endif /* CONFIG_NUMA_BALANCING */ 3381 3382/*** 3383 * kick_process - kick a running thread to enter/exit the kernel 3384 * @p: the to-be-kicked thread 3385 * 3386 * Cause a process which is running on another CPU to enter 3387 * kernel-mode, without any delay. (to get signals handled.) 3388 * 3389 * NOTE: this function doesn't have to take the runqueue lock, 3390 * because all it wants to ensure is that the remote task enters 3391 * the kernel. If the IPI races and the task has been migrated 3392 * to another CPU then no harm is done and the purpose has been 3393 * achieved as well. 3394 */ 3395void kick_process(struct task_struct *p) 3396{ 3397 guard(preempt)(); 3398 int cpu = task_cpu(p); 3399 3400 if ((cpu != smp_processor_id()) && task_curr(p)) 3401 smp_send_reschedule(cpu); 3402} 3403EXPORT_SYMBOL_GPL(kick_process); 3404 3405/* 3406 * ->cpus_ptr is protected by both rq->lock and p->pi_lock 3407 * 3408 * A few notes on cpu_active vs cpu_online: 3409 * 3410 * - cpu_active must be a subset of cpu_online 3411 * 3412 * - on CPU-up we allow per-CPU kthreads on the online && !active CPU, 3413 * see __set_cpus_allowed_ptr(). At this point the newly online 3414 * CPU isn't yet part of the sched domains, and balancing will not 3415 * see it. 3416 * 3417 * - on CPU-down we clear cpu_active() to mask the sched domains and 3418 * avoid the load balancer to place new tasks on the to be removed 3419 * CPU. Existing tasks will remain running there and will be taken 3420 * off. 3421 * 3422 * This means that fallback selection must not select !active CPUs. 3423 * And can assume that any active CPU must be online. Conversely 3424 * select_task_rq() below may allow selection of !active CPUs in order 3425 * to satisfy the above rules. 3426 */ 3427static int select_fallback_rq(int cpu, struct task_struct *p) 3428{ 3429 int nid = cpu_to_node(cpu); 3430 const struct cpumask *nodemask = NULL; 3431 enum { cpuset, possible, fail } state = cpuset; 3432 int dest_cpu; 3433 3434 /* 3435 * If the node that the CPU is on has been offlined, cpu_to_node() 3436 * will return -1. There is no CPU on the node, and we should 3437 * select the CPU on the other node. 3438 */ 3439 if (nid != -1) { 3440 nodemask = cpumask_of_node(nid); 3441 3442 /* Look for allowed, online CPU in same node. */ 3443 for_each_cpu(dest_cpu, nodemask) { 3444 if (is_cpu_allowed(p, dest_cpu)) 3445 return dest_cpu; 3446 } 3447 } 3448 3449 for (;;) { 3450 /* Any allowed, online CPU? */ 3451 for_each_cpu(dest_cpu, p->cpus_ptr) { 3452 if (!is_cpu_allowed(p, dest_cpu)) 3453 continue; 3454 3455 goto out; 3456 } 3457 3458 /* No more Mr. Nice Guy. */ 3459 switch (state) { 3460 case cpuset: 3461 if (cpuset_cpus_allowed_fallback(p)) { 3462 state = possible; 3463 break; 3464 } 3465 fallthrough; 3466 case possible: 3467 set_cpus_allowed_force(p, task_cpu_fallback_mask(p)); 3468 state = fail; 3469 break; 3470 case fail: 3471 BUG(); 3472 break; 3473 } 3474 } 3475 3476out: 3477 if (state != cpuset) { 3478 /* 3479 * Don't tell them about moving exiting tasks or 3480 * kernel threads (both mm NULL), since they never 3481 * leave kernel. 3482 */ 3483 if (p->mm && printk_ratelimit()) { 3484 printk_deferred("process %d (%s) no longer affine to cpu%d\n", 3485 task_pid_nr(p), p->comm, cpu); 3486 } 3487 } 3488 3489 return dest_cpu; 3490} 3491 3492/* 3493 * The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable. 3494 */ 3495static inline 3496int select_task_rq(struct task_struct *p, int cpu, int *wake_flags) 3497{ 3498 lockdep_assert_held(&p->pi_lock); 3499 3500 if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) { 3501 cpu = p->sched_class->select_task_rq(p, cpu, *wake_flags); 3502 *wake_flags |= WF_RQ_SELECTED; 3503 } else { 3504 cpu = cpumask_any(p->cpus_ptr); 3505 } 3506 3507 /* 3508 * In order not to call set_task_cpu() on a blocking task we need 3509 * to rely on ttwu() to place the task on a valid ->cpus_ptr 3510 * CPU. 3511 * 3512 * Since this is common to all placement strategies, this lives here. 3513 * 3514 * [ this allows ->select_task() to simply return task_cpu(p) and 3515 * not worry about this generic constraint ] 3516 */ 3517 if (unlikely(!is_cpu_allowed(p, cpu))) 3518 cpu = select_fallback_rq(task_cpu(p), p); 3519 3520 return cpu; 3521} 3522 3523void sched_set_stop_task(int cpu, struct task_struct *stop) 3524{ 3525 static struct lock_class_key stop_pi_lock; 3526 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 3527 struct task_struct *old_stop = cpu_rq(cpu)->stop; 3528 3529 if (stop) { 3530 /* 3531 * Make it appear like a SCHED_FIFO task, its something 3532 * userspace knows about and won't get confused about. 3533 * 3534 * Also, it will make PI more or less work without too 3535 * much confusion -- but then, stop work should not 3536 * rely on PI working anyway. 3537 */ 3538 sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); 3539 3540 stop->sched_class = &stop_sched_class; 3541 3542 /* 3543 * The PI code calls rt_mutex_setprio() with ->pi_lock held to 3544 * adjust the effective priority of a task. As a result, 3545 * rt_mutex_setprio() can trigger (RT) balancing operations, 3546 * which can then trigger wakeups of the stop thread to push 3547 * around the current task. 3548 * 3549 * The stop task itself will never be part of the PI-chain, it 3550 * never blocks, therefore that ->pi_lock recursion is safe. 3551 * Tell lockdep about this by placing the stop->pi_lock in its 3552 * own class. 3553 */ 3554 lockdep_set_class(&stop->pi_lock, &stop_pi_lock); 3555 } 3556 3557 cpu_rq(cpu)->stop = stop; 3558 3559 if (old_stop) { 3560 /* 3561 * Reset it back to a normal scheduling class so that 3562 * it can die in pieces. 3563 */ 3564 old_stop->sched_class = &rt_sched_class; 3565 } 3566} 3567 3568static void 3569ttwu_stat(struct task_struct *p, int cpu, int wake_flags) 3570{ 3571 struct rq *rq; 3572 3573 if (!schedstat_enabled()) 3574 return; 3575 3576 rq = this_rq(); 3577 3578 if (cpu == rq->cpu) { 3579 __schedstat_inc(rq->ttwu_local); 3580 __schedstat_inc(p->stats.nr_wakeups_local); 3581 } else { 3582 struct sched_domain *sd; 3583 3584 __schedstat_inc(p->stats.nr_wakeups_remote); 3585 3586 guard(rcu)(); 3587 for_each_domain(rq->cpu, sd) { 3588 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { 3589 __schedstat_inc(sd->ttwu_wake_remote); 3590 break; 3591 } 3592 } 3593 } 3594 3595 if (wake_flags & WF_MIGRATED) 3596 __schedstat_inc(p->stats.nr_wakeups_migrate); 3597 3598 __schedstat_inc(rq->ttwu_count); 3599 __schedstat_inc(p->stats.nr_wakeups); 3600 3601 if (wake_flags & WF_SYNC) 3602 __schedstat_inc(p->stats.nr_wakeups_sync); 3603} 3604 3605/* 3606 * Mark the task runnable. 3607 */ 3608static inline void ttwu_do_wakeup(struct task_struct *p) 3609{ 3610 WRITE_ONCE(p->__state, TASK_RUNNING); 3611 trace_sched_wakeup(p); 3612} 3613 3614static void 3615ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags, 3616 struct rq_flags *rf) 3617{ 3618 int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK; 3619 3620 lockdep_assert_rq_held(rq); 3621 3622 if (p->sched_contributes_to_load) 3623 rq->nr_uninterruptible--; 3624 3625 if (wake_flags & WF_RQ_SELECTED) 3626 en_flags |= ENQUEUE_RQ_SELECTED; 3627 if (wake_flags & WF_MIGRATED) 3628 en_flags |= ENQUEUE_MIGRATED; 3629 else 3630 if (p->in_iowait) { 3631 delayacct_blkio_end(p); 3632 atomic_dec(&task_rq(p)->nr_iowait); 3633 } 3634 3635 activate_task(rq, p, en_flags); 3636 wakeup_preempt(rq, p, wake_flags); 3637 3638 ttwu_do_wakeup(p); 3639 3640 if (p->sched_class->task_woken) { 3641 /* 3642 * Our task @p is fully woken up and running; so it's safe to 3643 * drop the rq->lock, hereafter rq is only used for statistics. 3644 */ 3645 rq_unpin_lock(rq, rf); 3646 p->sched_class->task_woken(rq, p); 3647 rq_repin_lock(rq, rf); 3648 } 3649 3650 if (rq->idle_stamp) { 3651 u64 delta = rq_clock(rq) - rq->idle_stamp; 3652 u64 max = 2*rq->max_idle_balance_cost; 3653 3654 update_avg(&rq->avg_idle, delta); 3655 3656 if (rq->avg_idle > max) 3657 rq->avg_idle = max; 3658 3659 rq->idle_stamp = 0; 3660 } 3661} 3662 3663/* 3664 * Consider @p being inside a wait loop: 3665 * 3666 * for (;;) { 3667 * set_current_state(TASK_UNINTERRUPTIBLE); 3668 * 3669 * if (CONDITION) 3670 * break; 3671 * 3672 * schedule(); 3673 * } 3674 * __set_current_state(TASK_RUNNING); 3675 * 3676 * between set_current_state() and schedule(). In this case @p is still 3677 * runnable, so all that needs doing is change p->state back to TASK_RUNNING in 3678 * an atomic manner. 3679 * 3680 * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq 3681 * then schedule() must still happen and p->state can be changed to 3682 * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we 3683 * need to do a full wakeup with enqueue. 3684 * 3685 * Returns: %true when the wakeup is done, 3686 * %false otherwise. 3687 */ 3688static int ttwu_runnable(struct task_struct *p, int wake_flags) 3689{ 3690 struct rq_flags rf; 3691 struct rq *rq; 3692 int ret = 0; 3693 3694 rq = __task_rq_lock(p, &rf); 3695 if (task_on_rq_queued(p)) { 3696 update_rq_clock(rq); 3697 if (p->se.sched_delayed) 3698 enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED); 3699 if (!task_on_cpu(rq, p)) { 3700 /* 3701 * When on_rq && !on_cpu the task is preempted, see if 3702 * it should preempt the task that is current now. 3703 */ 3704 wakeup_preempt(rq, p, wake_flags); 3705 } 3706 ttwu_do_wakeup(p); 3707 ret = 1; 3708 } 3709 __task_rq_unlock(rq, p, &rf); 3710 3711 return ret; 3712} 3713 3714void sched_ttwu_pending(void *arg) 3715{ 3716 struct llist_node *llist = arg; 3717 struct rq *rq = this_rq(); 3718 struct task_struct *p, *t; 3719 struct rq_flags rf; 3720 3721 if (!llist) 3722 return; 3723 3724 rq_lock_irqsave(rq, &rf); 3725 update_rq_clock(rq); 3726 3727 llist_for_each_entry_safe(p, t, llist, wake_entry.llist) { 3728 if (WARN_ON_ONCE(p->on_cpu)) 3729 smp_cond_load_acquire(&p->on_cpu, !VAL); 3730 3731 if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq))) 3732 set_task_cpu(p, cpu_of(rq)); 3733 3734 ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0, &rf); 3735 } 3736 3737 /* 3738 * Must be after enqueueing at least once task such that 3739 * idle_cpu() does not observe a false-negative -- if it does, 3740 * it is possible for select_idle_siblings() to stack a number 3741 * of tasks on this CPU during that window. 3742 * 3743 * It is OK to clear ttwu_pending when another task pending. 3744 * We will receive IPI after local IRQ enabled and then enqueue it. 3745 * Since now nr_running > 0, idle_cpu() will always get correct result. 3746 */ 3747 WRITE_ONCE(rq->ttwu_pending, 0); 3748 rq_unlock_irqrestore(rq, &rf); 3749} 3750 3751/* 3752 * Prepare the scene for sending an IPI for a remote smp_call 3753 * 3754 * Returns true if the caller can proceed with sending the IPI. 3755 * Returns false otherwise. 3756 */ 3757bool call_function_single_prep_ipi(int cpu) 3758{ 3759 if (set_nr_if_polling(cpu_rq(cpu)->idle)) { 3760 trace_sched_wake_idle_without_ipi(cpu); 3761 return false; 3762 } 3763 3764 return true; 3765} 3766 3767/* 3768 * Queue a task on the target CPUs wake_list and wake the CPU via IPI if 3769 * necessary. The wakee CPU on receipt of the IPI will queue the task 3770 * via sched_ttwu_wakeup() for activation so the wakee incurs the cost 3771 * of the wakeup instead of the waker. 3772 */ 3773static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3774{ 3775 struct rq *rq = cpu_rq(cpu); 3776 3777 p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); 3778 3779 WRITE_ONCE(rq->ttwu_pending, 1); 3780#ifdef CONFIG_SMP 3781 __smp_call_single_queue(cpu, &p->wake_entry.llist); 3782#endif 3783} 3784 3785void wake_up_if_idle(int cpu) 3786{ 3787 struct rq *rq = cpu_rq(cpu); 3788 3789 guard(rcu)(); 3790 if (is_idle_task(rcu_dereference(rq->curr))) { 3791 guard(rq_lock_irqsave)(rq); 3792 if (is_idle_task(rq->curr)) 3793 resched_curr(rq); 3794 } 3795} 3796 3797bool cpus_equal_capacity(int this_cpu, int that_cpu) 3798{ 3799 if (!sched_asym_cpucap_active()) 3800 return true; 3801 3802 if (this_cpu == that_cpu) 3803 return true; 3804 3805 return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu); 3806} 3807 3808bool cpus_share_cache(int this_cpu, int that_cpu) 3809{ 3810 if (this_cpu == that_cpu) 3811 return true; 3812 3813 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); 3814} 3815 3816/* 3817 * Whether CPUs are share cache resources, which means LLC on non-cluster 3818 * machines and LLC tag or L2 on machines with clusters. 3819 */ 3820bool cpus_share_resources(int this_cpu, int that_cpu) 3821{ 3822 if (this_cpu == that_cpu) 3823 return true; 3824 3825 return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu); 3826} 3827 3828static inline bool ttwu_queue_cond(struct task_struct *p, int cpu) 3829{ 3830 /* See SCX_OPS_ALLOW_QUEUED_WAKEUP. */ 3831 if (!scx_allow_ttwu_queue(p)) 3832 return false; 3833 3834#ifdef CONFIG_SMP 3835 if (p->sched_class == &stop_sched_class) 3836 return false; 3837#endif 3838 3839 /* 3840 * Do not complicate things with the async wake_list while the CPU is 3841 * in hotplug state. 3842 */ 3843 if (!cpu_active(cpu)) 3844 return false; 3845 3846 /* Ensure the task will still be allowed to run on the CPU. */ 3847 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 3848 return false; 3849 3850 /* 3851 * If the CPU does not share cache, then queue the task on the 3852 * remote rqs wakelist to avoid accessing remote data. 3853 */ 3854 if (!cpus_share_cache(smp_processor_id(), cpu)) 3855 return true; 3856 3857 if (cpu == smp_processor_id()) 3858 return false; 3859 3860 /* 3861 * If the wakee cpu is idle, or the task is descheduling and the 3862 * only running task on the CPU, then use the wakelist to offload 3863 * the task activation to the idle (or soon-to-be-idle) CPU as 3864 * the current CPU is likely busy. nr_running is checked to 3865 * avoid unnecessary task stacking. 3866 * 3867 * Note that we can only get here with (wakee) p->on_rq=0, 3868 * p->on_cpu can be whatever, we've done the dequeue, so 3869 * the wakee has been accounted out of ->nr_running. 3870 */ 3871 if (!cpu_rq(cpu)->nr_running) 3872 return true; 3873 3874 return false; 3875} 3876 3877static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) 3878{ 3879 if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) { 3880 sched_clock_cpu(cpu); /* Sync clocks across CPUs */ 3881 __ttwu_queue_wakelist(p, cpu, wake_flags); 3882 return true; 3883 } 3884 3885 return false; 3886} 3887 3888static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) 3889{ 3890 struct rq *rq = cpu_rq(cpu); 3891 struct rq_flags rf; 3892 3893 if (ttwu_queue_wakelist(p, cpu, wake_flags)) 3894 return; 3895 3896 rq_lock(rq, &rf); 3897 update_rq_clock(rq); 3898 ttwu_do_activate(rq, p, wake_flags, &rf); 3899 rq_unlock(rq, &rf); 3900} 3901 3902/* 3903 * Invoked from try_to_wake_up() to check whether the task can be woken up. 3904 * 3905 * The caller holds p::pi_lock if p != current or has preemption 3906 * disabled when p == current. 3907 * 3908 * The rules of saved_state: 3909 * 3910 * The related locking code always holds p::pi_lock when updating 3911 * p::saved_state, which means the code is fully serialized in both cases. 3912 * 3913 * For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT. 3914 * No other bits set. This allows to distinguish all wakeup scenarios. 3915 * 3916 * For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This 3917 * allows us to prevent early wakeup of tasks before they can be run on 3918 * asymmetric ISA architectures (eg ARMv9). 3919 */ 3920static __always_inline 3921bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success) 3922{ 3923 int match; 3924 3925 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 3926 WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) && 3927 state != TASK_RTLOCK_WAIT); 3928 } 3929 3930 *success = !!(match = __task_state_match(p, state)); 3931 3932 /* 3933 * Saved state preserves the task state across blocking on 3934 * an RT lock or TASK_FREEZABLE tasks. If the state matches, 3935 * set p::saved_state to TASK_RUNNING, but do not wake the task 3936 * because it waits for a lock wakeup or __thaw_task(). Also 3937 * indicate success because from the regular waker's point of 3938 * view this has succeeded. 3939 * 3940 * After acquiring the lock the task will restore p::__state 3941 * from p::saved_state which ensures that the regular 3942 * wakeup is not lost. The restore will also set 3943 * p::saved_state to TASK_RUNNING so any further tests will 3944 * not result in false positives vs. @success 3945 */ 3946 if (match < 0) 3947 p->saved_state = TASK_RUNNING; 3948 3949 return match > 0; 3950} 3951 3952/* 3953 * Notes on Program-Order guarantees on SMP systems. 3954 * 3955 * MIGRATION 3956 * 3957 * The basic program-order guarantee on SMP systems is that when a task [t] 3958 * migrates, all its activity on its old CPU [c0] happens-before any subsequent 3959 * execution on its new CPU [c1]. 3960 * 3961 * For migration (of runnable tasks) this is provided by the following means: 3962 * 3963 * A) UNLOCK of the rq(c0)->lock scheduling out task t 3964 * B) migration for t is required to synchronize *both* rq(c0)->lock and 3965 * rq(c1)->lock (if not at the same time, then in that order). 3966 * C) LOCK of the rq(c1)->lock scheduling in task 3967 * 3968 * Release/acquire chaining guarantees that B happens after A and C after B. 3969 * Note: the CPU doing B need not be c0 or c1 3970 * 3971 * Example: 3972 * 3973 * CPU0 CPU1 CPU2 3974 * 3975 * LOCK rq(0)->lock 3976 * sched-out X 3977 * sched-in Y 3978 * UNLOCK rq(0)->lock 3979 * 3980 * LOCK rq(0)->lock // orders against CPU0 3981 * dequeue X 3982 * UNLOCK rq(0)->lock 3983 * 3984 * LOCK rq(1)->lock 3985 * enqueue X 3986 * UNLOCK rq(1)->lock 3987 * 3988 * LOCK rq(1)->lock // orders against CPU2 3989 * sched-out Z 3990 * sched-in X 3991 * UNLOCK rq(1)->lock 3992 * 3993 * 3994 * BLOCKING -- aka. SLEEP + WAKEUP 3995 * 3996 * For blocking we (obviously) need to provide the same guarantee as for 3997 * migration. However the means are completely different as there is no lock 3998 * chain to provide order. Instead we do: 3999 * 4000 * 1) smp_store_release(X->on_cpu, 0) -- finish_task() 4001 * 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up() 4002 * 4003 * Example: 4004 * 4005 * CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule) 4006 * 4007 * LOCK rq(0)->lock LOCK X->pi_lock 4008 * dequeue X 4009 * sched-out X 4010 * smp_store_release(X->on_cpu, 0); 4011 * 4012 * smp_cond_load_acquire(&X->on_cpu, !VAL); 4013 * X->state = WAKING 4014 * set_task_cpu(X,2) 4015 * 4016 * LOCK rq(2)->lock 4017 * enqueue X 4018 * X->state = RUNNING 4019 * UNLOCK rq(2)->lock 4020 * 4021 * LOCK rq(2)->lock // orders against CPU1 4022 * sched-out Z 4023 * sched-in X 4024 * UNLOCK rq(2)->lock 4025 * 4026 * UNLOCK X->pi_lock 4027 * UNLOCK rq(0)->lock 4028 * 4029 * 4030 * However, for wakeups there is a second guarantee we must provide, namely we 4031 * must ensure that CONDITION=1 done by the caller can not be reordered with 4032 * accesses to the task state; see try_to_wake_up() and set_current_state(). 4033 */ 4034 4035/** 4036 * try_to_wake_up - wake up a thread 4037 * @p: the thread to be awakened 4038 * @state: the mask of task states that can be woken 4039 * @wake_flags: wake modifier flags (WF_*) 4040 * 4041 * Conceptually does: 4042 * 4043 * If (@state & @p->state) @p->state = TASK_RUNNING. 4044 * 4045 * If the task was not queued/runnable, also place it back on a runqueue. 4046 * 4047 * This function is atomic against schedule() which would dequeue the task. 4048 * 4049 * It issues a full memory barrier before accessing @p->state, see the comment 4050 * with set_current_state(). 4051 * 4052 * Uses p->pi_lock to serialize against concurrent wake-ups. 4053 * 4054 * Relies on p->pi_lock stabilizing: 4055 * - p->sched_class 4056 * - p->cpus_ptr 4057 * - p->sched_task_group 4058 * in order to do migration, see its use of select_task_rq()/set_task_cpu(). 4059 * 4060 * Tries really hard to only take one task_rq(p)->lock for performance. 4061 * Takes rq->lock in: 4062 * - ttwu_runnable() -- old rq, unavoidable, see comment there; 4063 * - ttwu_queue() -- new rq, for enqueue of the task; 4064 * - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us. 4065 * 4066 * As a consequence we race really badly with just about everything. See the 4067 * many memory barriers and their comments for details. 4068 * 4069 * Return: %true if @p->state changes (an actual wakeup was done), 4070 * %false otherwise. 4071 */ 4072int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) 4073{ 4074 guard(preempt)(); 4075 int cpu, success = 0; 4076 4077 wake_flags |= WF_TTWU; 4078 4079 if (p == current) { 4080 /* 4081 * We're waking current, this means 'p->on_rq' and 'task_cpu(p) 4082 * == smp_processor_id()'. Together this means we can special 4083 * case the whole 'p->on_rq && ttwu_runnable()' case below 4084 * without taking any locks. 4085 * 4086 * Specifically, given current runs ttwu() we must be before 4087 * schedule()'s block_task(), as such this must not observe 4088 * sched_delayed. 4089 * 4090 * In particular: 4091 * - we rely on Program-Order guarantees for all the ordering, 4092 * - we're serialized against set_special_state() by virtue of 4093 * it disabling IRQs (this allows not taking ->pi_lock). 4094 */ 4095 WARN_ON_ONCE(p->se.sched_delayed); 4096 if (!ttwu_state_match(p, state, &success)) 4097 goto out; 4098 4099 trace_sched_waking(p); 4100 ttwu_do_wakeup(p); 4101 goto out; 4102 } 4103 4104 /* 4105 * If we are going to wake up a thread waiting for CONDITION we 4106 * need to ensure that CONDITION=1 done by the caller can not be 4107 * reordered with p->state check below. This pairs with smp_store_mb() 4108 * in set_current_state() that the waiting thread does. 4109 */ 4110 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { 4111 smp_mb__after_spinlock(); 4112 if (!ttwu_state_match(p, state, &success)) 4113 break; 4114 4115 trace_sched_waking(p); 4116 4117 /* 4118 * Ensure we load p->on_rq _after_ p->state, otherwise it would 4119 * be possible to, falsely, observe p->on_rq == 0 and get stuck 4120 * in smp_cond_load_acquire() below. 4121 * 4122 * sched_ttwu_pending() try_to_wake_up() 4123 * STORE p->on_rq = 1 LOAD p->state 4124 * UNLOCK rq->lock 4125 * 4126 * __schedule() (switch to task 'p') 4127 * LOCK rq->lock smp_rmb(); 4128 * smp_mb__after_spinlock(); 4129 * UNLOCK rq->lock 4130 * 4131 * [task p] 4132 * STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq 4133 * 4134 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4135 * __schedule(). See the comment for smp_mb__after_spinlock(). 4136 * 4137 * A similar smp_rmb() lives in __task_needs_rq_lock(). 4138 */ 4139 smp_rmb(); 4140 if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags)) 4141 break; 4142 4143 /* 4144 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be 4145 * possible to, falsely, observe p->on_cpu == 0. 4146 * 4147 * One must be running (->on_cpu == 1) in order to remove oneself 4148 * from the runqueue. 4149 * 4150 * __schedule() (switch to task 'p') try_to_wake_up() 4151 * STORE p->on_cpu = 1 LOAD p->on_rq 4152 * UNLOCK rq->lock 4153 * 4154 * __schedule() (put 'p' to sleep) 4155 * LOCK rq->lock smp_rmb(); 4156 * smp_mb__after_spinlock(); 4157 * STORE p->on_rq = 0 LOAD p->on_cpu 4158 * 4159 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in 4160 * __schedule(). See the comment for smp_mb__after_spinlock(). 4161 * 4162 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure 4163 * schedule()'s block_task() has 'happened' and p will no longer 4164 * care about it's own p->state. See the comment in __schedule(). 4165 */ 4166 smp_acquire__after_ctrl_dep(); 4167 4168 /* 4169 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq 4170 * == 0), which means we need to do an enqueue, change p->state to 4171 * TASK_WAKING such that we can unlock p->pi_lock before doing the 4172 * enqueue, such as ttwu_queue_wakelist(). 4173 */ 4174 WRITE_ONCE(p->__state, TASK_WAKING); 4175 4176 /* 4177 * If the owning (remote) CPU is still in the middle of schedule() with 4178 * this task as prev, considering queueing p on the remote CPUs wake_list 4179 * which potentially sends an IPI instead of spinning on p->on_cpu to 4180 * let the waker make forward progress. This is safe because IRQs are 4181 * disabled and the IPI will deliver after on_cpu is cleared. 4182 * 4183 * Ensure we load task_cpu(p) after p->on_cpu: 4184 * 4185 * set_task_cpu(p, cpu); 4186 * STORE p->cpu = @cpu 4187 * __schedule() (switch to task 'p') 4188 * LOCK rq->lock 4189 * smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu) 4190 * STORE p->on_cpu = 1 LOAD p->cpu 4191 * 4192 * to ensure we observe the correct CPU on which the task is currently 4193 * scheduling. 4194 */ 4195 if (smp_load_acquire(&p->on_cpu) && 4196 ttwu_queue_wakelist(p, task_cpu(p), wake_flags)) 4197 break; 4198 4199 /* 4200 * If the owning (remote) CPU is still in the middle of schedule() with 4201 * this task as prev, wait until it's done referencing the task. 4202 * 4203 * Pairs with the smp_store_release() in finish_task(). 4204 * 4205 * This ensures that tasks getting woken will be fully ordered against 4206 * their previous state and preserve Program Order. 4207 */ 4208 smp_cond_load_acquire(&p->on_cpu, !VAL); 4209 4210 cpu = select_task_rq(p, p->wake_cpu, &wake_flags); 4211 if (task_cpu(p) != cpu) { 4212 if (p->in_iowait) { 4213 delayacct_blkio_end(p); 4214 atomic_dec(&task_rq(p)->nr_iowait); 4215 } 4216 4217 wake_flags |= WF_MIGRATED; 4218 psi_ttwu_dequeue(p); 4219 set_task_cpu(p, cpu); 4220 } 4221 4222 ttwu_queue(p, cpu, wake_flags); 4223 } 4224out: 4225 if (success) 4226 ttwu_stat(p, task_cpu(p), wake_flags); 4227 4228 return success; 4229} 4230 4231static bool __task_needs_rq_lock(struct task_struct *p) 4232{ 4233 unsigned int state = READ_ONCE(p->__state); 4234 4235 /* 4236 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when 4237 * the task is blocked. Make sure to check @state since ttwu() can drop 4238 * locks at the end, see ttwu_queue_wakelist(). 4239 */ 4240 if (state == TASK_RUNNING || state == TASK_WAKING) 4241 return true; 4242 4243 /* 4244 * Ensure we load p->on_rq after p->__state, otherwise it would be 4245 * possible to, falsely, observe p->on_rq == 0. 4246 * 4247 * See try_to_wake_up() for a longer comment. 4248 */ 4249 smp_rmb(); 4250 if (p->on_rq) 4251 return true; 4252 4253 /* 4254 * Ensure the task has finished __schedule() and will not be referenced 4255 * anymore. Again, see try_to_wake_up() for a longer comment. 4256 */ 4257 smp_rmb(); 4258 smp_cond_load_acquire(&p->on_cpu, !VAL); 4259 4260 return false; 4261} 4262 4263/** 4264 * task_call_func - Invoke a function on task in fixed state 4265 * @p: Process for which the function is to be invoked, can be @current. 4266 * @func: Function to invoke. 4267 * @arg: Argument to function. 4268 * 4269 * Fix the task in it's current state by avoiding wakeups and or rq operations 4270 * and call @func(@arg) on it. This function can use task_is_runnable() and 4271 * task_curr() to work out what the state is, if required. Given that @func 4272 * can be invoked with a runqueue lock held, it had better be quite 4273 * lightweight. 4274 * 4275 * Returns: 4276 * Whatever @func returns 4277 */ 4278int task_call_func(struct task_struct *p, task_call_f func, void *arg) 4279{ 4280 struct rq *rq = NULL; 4281 struct rq_flags rf; 4282 int ret; 4283 4284 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4285 4286 if (__task_needs_rq_lock(p)) 4287 rq = __task_rq_lock(p, &rf); 4288 4289 /* 4290 * At this point the task is pinned; either: 4291 * - blocked and we're holding off wakeups (pi->lock) 4292 * - woken, and we're holding off enqueue (rq->lock) 4293 * - queued, and we're holding off schedule (rq->lock) 4294 * - running, and we're holding off de-schedule (rq->lock) 4295 * 4296 * The called function (@func) can use: task_curr(), p->on_rq and 4297 * p->__state to differentiate between these states. 4298 */ 4299 ret = func(p, arg); 4300 4301 if (rq) 4302 __task_rq_unlock(rq, p, &rf); 4303 4304 raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); 4305 return ret; 4306} 4307 4308/** 4309 * cpu_curr_snapshot - Return a snapshot of the currently running task 4310 * @cpu: The CPU on which to snapshot the task. 4311 * 4312 * Returns the task_struct pointer of the task "currently" running on 4313 * the specified CPU. 4314 * 4315 * If the specified CPU was offline, the return value is whatever it 4316 * is, perhaps a pointer to the task_struct structure of that CPU's idle 4317 * task, but there is no guarantee. Callers wishing a useful return 4318 * value must take some action to ensure that the specified CPU remains 4319 * online throughout. 4320 * 4321 * This function executes full memory barriers before and after fetching 4322 * the pointer, which permits the caller to confine this function's fetch 4323 * with respect to the caller's accesses to other shared variables. 4324 */ 4325struct task_struct *cpu_curr_snapshot(int cpu) 4326{ 4327 struct rq *rq = cpu_rq(cpu); 4328 struct task_struct *t; 4329 struct rq_flags rf; 4330 4331 rq_lock_irqsave(rq, &rf); 4332 smp_mb__after_spinlock(); /* Pairing determined by caller's synchronization design. */ 4333 t = rcu_dereference(cpu_curr(cpu)); 4334 rq_unlock_irqrestore(rq, &rf); 4335 smp_mb(); /* Pairing determined by caller's synchronization design. */ 4336 4337 return t; 4338} 4339 4340/** 4341 * wake_up_process - Wake up a specific process 4342 * @p: The process to be woken up. 4343 * 4344 * Attempt to wake up the nominated process and move it to the set of runnable 4345 * processes. 4346 * 4347 * Return: 1 if the process was woken up, 0 if it was already running. 4348 * 4349 * This function executes a full memory barrier before accessing the task state. 4350 */ 4351int wake_up_process(struct task_struct *p) 4352{ 4353 return try_to_wake_up(p, TASK_NORMAL, 0); 4354} 4355EXPORT_SYMBOL(wake_up_process); 4356 4357int wake_up_state(struct task_struct *p, unsigned int state) 4358{ 4359 return try_to_wake_up(p, state, 0); 4360} 4361 4362/* 4363 * Perform scheduler related setup for a newly forked process p. 4364 * p is forked by current. 4365 * 4366 * __sched_fork() is basic setup which is also used by sched_init() to 4367 * initialize the boot CPU's idle task. 4368 */ 4369static void __sched_fork(u64 clone_flags, struct task_struct *p) 4370{ 4371 p->on_rq = 0; 4372 4373 p->se.on_rq = 0; 4374 p->se.exec_start = 0; 4375 p->se.sum_exec_runtime = 0; 4376 p->se.prev_sum_exec_runtime = 0; 4377 p->se.nr_migrations = 0; 4378 p->se.vruntime = 0; 4379 p->se.vlag = 0; 4380 INIT_LIST_HEAD(&p->se.group_node); 4381 4382 /* A delayed task cannot be in clone(). */ 4383 WARN_ON_ONCE(p->se.sched_delayed); 4384 4385#ifdef CONFIG_FAIR_GROUP_SCHED 4386 p->se.cfs_rq = NULL; 4387#ifdef CONFIG_CFS_BANDWIDTH 4388 init_cfs_throttle_work(p); 4389#endif 4390#endif 4391 4392#ifdef CONFIG_SCHEDSTATS 4393 /* Even if schedstat is disabled, there should not be garbage */ 4394 memset(&p->stats, 0, sizeof(p->stats)); 4395#endif 4396 4397 init_dl_entity(&p->dl); 4398 4399 INIT_LIST_HEAD(&p->rt.run_list); 4400 p->rt.timeout = 0; 4401 p->rt.time_slice = sched_rr_timeslice; 4402 p->rt.on_rq = 0; 4403 p->rt.on_list = 0; 4404 4405#ifdef CONFIG_SCHED_CLASS_EXT 4406 init_scx_entity(&p->scx); 4407#endif 4408 4409#ifdef CONFIG_PREEMPT_NOTIFIERS 4410 INIT_HLIST_HEAD(&p->preempt_notifiers); 4411#endif 4412 4413#ifdef CONFIG_COMPACTION 4414 p->capture_control = NULL; 4415#endif 4416 init_numa_balancing(clone_flags, p); 4417 p->wake_entry.u_flags = CSD_TYPE_TTWU; 4418 p->migration_pending = NULL; 4419} 4420 4421DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); 4422 4423#ifdef CONFIG_NUMA_BALANCING 4424 4425int sysctl_numa_balancing_mode; 4426 4427static void __set_numabalancing_state(bool enabled) 4428{ 4429 if (enabled) 4430 static_branch_enable(&sched_numa_balancing); 4431 else 4432 static_branch_disable(&sched_numa_balancing); 4433} 4434 4435void set_numabalancing_state(bool enabled) 4436{ 4437 if (enabled) 4438 sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL; 4439 else 4440 sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED; 4441 __set_numabalancing_state(enabled); 4442} 4443 4444#ifdef CONFIG_PROC_SYSCTL 4445static void reset_memory_tiering(void) 4446{ 4447 struct pglist_data *pgdat; 4448 4449 for_each_online_pgdat(pgdat) { 4450 pgdat->nbp_threshold = 0; 4451 pgdat->nbp_th_nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 4452 pgdat->nbp_th_start = jiffies_to_msecs(jiffies); 4453 } 4454} 4455 4456static int sysctl_numa_balancing(const struct ctl_table *table, int write, 4457 void *buffer, size_t *lenp, loff_t *ppos) 4458{ 4459 struct ctl_table t; 4460 int err; 4461 int state = sysctl_numa_balancing_mode; 4462 4463 if (write && !capable(CAP_SYS_ADMIN)) 4464 return -EPERM; 4465 4466 t = *table; 4467 t.data = &state; 4468 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4469 if (err < 0) 4470 return err; 4471 if (write) { 4472 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 4473 (state & NUMA_BALANCING_MEMORY_TIERING)) 4474 reset_memory_tiering(); 4475 sysctl_numa_balancing_mode = state; 4476 __set_numabalancing_state(state); 4477 } 4478 return err; 4479} 4480#endif /* CONFIG_PROC_SYSCTL */ 4481#endif /* CONFIG_NUMA_BALANCING */ 4482 4483#ifdef CONFIG_SCHEDSTATS 4484 4485DEFINE_STATIC_KEY_FALSE(sched_schedstats); 4486 4487static void set_schedstats(bool enabled) 4488{ 4489 if (enabled) 4490 static_branch_enable(&sched_schedstats); 4491 else 4492 static_branch_disable(&sched_schedstats); 4493} 4494 4495void force_schedstat_enabled(void) 4496{ 4497 if (!schedstat_enabled()) { 4498 pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n"); 4499 static_branch_enable(&sched_schedstats); 4500 } 4501} 4502 4503static int __init setup_schedstats(char *str) 4504{ 4505 int ret = 0; 4506 if (!str) 4507 goto out; 4508 4509 if (!strcmp(str, "enable")) { 4510 set_schedstats(true); 4511 ret = 1; 4512 } else if (!strcmp(str, "disable")) { 4513 set_schedstats(false); 4514 ret = 1; 4515 } 4516out: 4517 if (!ret) 4518 pr_warn("Unable to parse schedstats=\n"); 4519 4520 return ret; 4521} 4522__setup("schedstats=", setup_schedstats); 4523 4524#ifdef CONFIG_PROC_SYSCTL 4525static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer, 4526 size_t *lenp, loff_t *ppos) 4527{ 4528 struct ctl_table t; 4529 int err; 4530 int state = static_branch_likely(&sched_schedstats); 4531 4532 if (write && !capable(CAP_SYS_ADMIN)) 4533 return -EPERM; 4534 4535 t = *table; 4536 t.data = &state; 4537 err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); 4538 if (err < 0) 4539 return err; 4540 if (write) 4541 set_schedstats(state); 4542 return err; 4543} 4544#endif /* CONFIG_PROC_SYSCTL */ 4545#endif /* CONFIG_SCHEDSTATS */ 4546 4547#ifdef CONFIG_SYSCTL 4548static const struct ctl_table sched_core_sysctls[] = { 4549#ifdef CONFIG_SCHEDSTATS 4550 { 4551 .procname = "sched_schedstats", 4552 .data = NULL, 4553 .maxlen = sizeof(unsigned int), 4554 .mode = 0644, 4555 .proc_handler = sysctl_schedstats, 4556 .extra1 = SYSCTL_ZERO, 4557 .extra2 = SYSCTL_ONE, 4558 }, 4559#endif /* CONFIG_SCHEDSTATS */ 4560#ifdef CONFIG_UCLAMP_TASK 4561 { 4562 .procname = "sched_util_clamp_min", 4563 .data = &sysctl_sched_uclamp_util_min, 4564 .maxlen = sizeof(unsigned int), 4565 .mode = 0644, 4566 .proc_handler = sysctl_sched_uclamp_handler, 4567 }, 4568 { 4569 .procname = "sched_util_clamp_max", 4570 .data = &sysctl_sched_uclamp_util_max, 4571 .maxlen = sizeof(unsigned int), 4572 .mode = 0644, 4573 .proc_handler = sysctl_sched_uclamp_handler, 4574 }, 4575 { 4576 .procname = "sched_util_clamp_min_rt_default", 4577 .data = &sysctl_sched_uclamp_util_min_rt_default, 4578 .maxlen = sizeof(unsigned int), 4579 .mode = 0644, 4580 .proc_handler = sysctl_sched_uclamp_handler, 4581 }, 4582#endif /* CONFIG_UCLAMP_TASK */ 4583#ifdef CONFIG_NUMA_BALANCING 4584 { 4585 .procname = "numa_balancing", 4586 .data = NULL, /* filled in by handler */ 4587 .maxlen = sizeof(unsigned int), 4588 .mode = 0644, 4589 .proc_handler = sysctl_numa_balancing, 4590 .extra1 = SYSCTL_ZERO, 4591 .extra2 = SYSCTL_FOUR, 4592 }, 4593#endif /* CONFIG_NUMA_BALANCING */ 4594}; 4595static int __init sched_core_sysctl_init(void) 4596{ 4597 register_sysctl_init("kernel", sched_core_sysctls); 4598 return 0; 4599} 4600late_initcall(sched_core_sysctl_init); 4601#endif /* CONFIG_SYSCTL */ 4602 4603/* 4604 * fork()/clone()-time setup: 4605 */ 4606int sched_fork(u64 clone_flags, struct task_struct *p) 4607{ 4608 __sched_fork(clone_flags, p); 4609 /* 4610 * We mark the process as NEW here. This guarantees that 4611 * nobody will actually run it, and a signal or other external 4612 * event cannot wake it up and insert it on the runqueue either. 4613 */ 4614 p->__state = TASK_NEW; 4615 4616 /* 4617 * Make sure we do not leak PI boosting priority to the child. 4618 */ 4619 p->prio = current->normal_prio; 4620 4621 uclamp_fork(p); 4622 4623 /* 4624 * Revert to default priority/policy on fork if requested. 4625 */ 4626 if (unlikely(p->sched_reset_on_fork)) { 4627 if (task_has_dl_policy(p) || task_has_rt_policy(p)) { 4628 p->policy = SCHED_NORMAL; 4629 p->static_prio = NICE_TO_PRIO(0); 4630 p->rt_priority = 0; 4631 } else if (PRIO_TO_NICE(p->static_prio) < 0) 4632 p->static_prio = NICE_TO_PRIO(0); 4633 4634 p->prio = p->normal_prio = p->static_prio; 4635 set_load_weight(p, false); 4636 p->se.custom_slice = 0; 4637 p->se.slice = sysctl_sched_base_slice; 4638 4639 /* 4640 * We don't need the reset flag anymore after the fork. It has 4641 * fulfilled its duty: 4642 */ 4643 p->sched_reset_on_fork = 0; 4644 } 4645 4646 if (dl_prio(p->prio)) 4647 return -EAGAIN; 4648 4649 scx_pre_fork(p); 4650 4651 if (rt_prio(p->prio)) { 4652 p->sched_class = &rt_sched_class; 4653#ifdef CONFIG_SCHED_CLASS_EXT 4654 } else if (task_should_scx(p->policy)) { 4655 p->sched_class = &ext_sched_class; 4656#endif 4657 } else { 4658 p->sched_class = &fair_sched_class; 4659 } 4660 4661 init_entity_runnable_average(&p->se); 4662 4663 4664#ifdef CONFIG_SCHED_INFO 4665 if (likely(sched_info_on())) 4666 memset(&p->sched_info, 0, sizeof(p->sched_info)); 4667#endif 4668 p->on_cpu = 0; 4669 init_task_preempt_count(p); 4670 plist_node_init(&p->pushable_tasks, MAX_PRIO); 4671 RB_CLEAR_NODE(&p->pushable_dl_tasks); 4672 4673 return 0; 4674} 4675 4676int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) 4677{ 4678 unsigned long flags; 4679 4680 /* 4681 * Because we're not yet on the pid-hash, p->pi_lock isn't strictly 4682 * required yet, but lockdep gets upset if rules are violated. 4683 */ 4684 raw_spin_lock_irqsave(&p->pi_lock, flags); 4685#ifdef CONFIG_CGROUP_SCHED 4686 if (1) { 4687 struct task_group *tg; 4688 tg = container_of(kargs->cset->subsys[cpu_cgrp_id], 4689 struct task_group, css); 4690 tg = autogroup_task_group(p, tg); 4691 p->sched_task_group = tg; 4692 } 4693#endif 4694 /* 4695 * We're setting the CPU for the first time, we don't migrate, 4696 * so use __set_task_cpu(). 4697 */ 4698 __set_task_cpu(p, smp_processor_id()); 4699 if (p->sched_class->task_fork) 4700 p->sched_class->task_fork(p); 4701 raw_spin_unlock_irqrestore(&p->pi_lock, flags); 4702 4703 return scx_fork(p); 4704} 4705 4706void sched_cancel_fork(struct task_struct *p) 4707{ 4708 scx_cancel_fork(p); 4709} 4710 4711static void sched_mm_cid_fork(struct task_struct *t); 4712 4713void sched_post_fork(struct task_struct *p) 4714{ 4715 sched_mm_cid_fork(p); 4716 uclamp_post_fork(p); 4717 scx_post_fork(p); 4718} 4719 4720unsigned long to_ratio(u64 period, u64 runtime) 4721{ 4722 if (runtime == RUNTIME_INF) 4723 return BW_UNIT; 4724 4725 /* 4726 * Doing this here saves a lot of checks in all 4727 * the calling paths, and returning zero seems 4728 * safe for them anyway. 4729 */ 4730 if (period == 0) 4731 return 0; 4732 4733 return div64_u64(runtime << BW_SHIFT, period); 4734} 4735 4736/* 4737 * wake_up_new_task - wake up a newly created task for the first time. 4738 * 4739 * This function will do some initial scheduler statistics housekeeping 4740 * that must be done for every newly created context, then puts the task 4741 * on the runqueue and wakes it. 4742 */ 4743void wake_up_new_task(struct task_struct *p) 4744{ 4745 struct rq_flags rf; 4746 struct rq *rq; 4747 int wake_flags = WF_FORK; 4748 4749 raw_spin_lock_irqsave(&p->pi_lock, rf.flags); 4750 WRITE_ONCE(p->__state, TASK_RUNNING); 4751 /* 4752 * Fork balancing, do it here and not earlier because: 4753 * - cpus_ptr can change in the fork path 4754 * - any previously selected CPU might disappear through hotplug 4755 * 4756 * Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq, 4757 * as we're not fully set-up yet. 4758 */ 4759 p->recent_used_cpu = task_cpu(p); 4760 __set_task_cpu(p, select_task_rq(p, task_cpu(p), &wake_flags)); 4761 rq = __task_rq_lock(p, &rf); 4762 update_rq_clock(rq); 4763 post_init_entity_util_avg(p); 4764 4765 activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL); 4766 trace_sched_wakeup_new(p); 4767 wakeup_preempt(rq, p, wake_flags); 4768 if (p->sched_class->task_woken) { 4769 /* 4770 * Nothing relies on rq->lock after this, so it's fine to 4771 * drop it. 4772 */ 4773 rq_unpin_lock(rq, &rf); 4774 p->sched_class->task_woken(rq, p); 4775 rq_repin_lock(rq, &rf); 4776 } 4777 task_rq_unlock(rq, p, &rf); 4778} 4779 4780#ifdef CONFIG_PREEMPT_NOTIFIERS 4781 4782static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key); 4783 4784void preempt_notifier_inc(void) 4785{ 4786 static_branch_inc(&preempt_notifier_key); 4787} 4788EXPORT_SYMBOL_GPL(preempt_notifier_inc); 4789 4790void preempt_notifier_dec(void) 4791{ 4792 static_branch_dec(&preempt_notifier_key); 4793} 4794EXPORT_SYMBOL_GPL(preempt_notifier_dec); 4795 4796/** 4797 * preempt_notifier_register - tell me when current is being preempted & rescheduled 4798 * @notifier: notifier struct to register 4799 */ 4800void preempt_notifier_register(struct preempt_notifier *notifier) 4801{ 4802 if (!static_branch_unlikely(&preempt_notifier_key)) 4803 WARN(1, "registering preempt_notifier while notifiers disabled\n"); 4804 4805 hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); 4806} 4807EXPORT_SYMBOL_GPL(preempt_notifier_register); 4808 4809/** 4810 * preempt_notifier_unregister - no longer interested in preemption notifications 4811 * @notifier: notifier struct to unregister 4812 * 4813 * This is *not* safe to call from within a preemption notifier. 4814 */ 4815void preempt_notifier_unregister(struct preempt_notifier *notifier) 4816{ 4817 hlist_del(¬ifier->link); 4818} 4819EXPORT_SYMBOL_GPL(preempt_notifier_unregister); 4820 4821static void __fire_sched_in_preempt_notifiers(struct task_struct *curr) 4822{ 4823 struct preempt_notifier *notifier; 4824 4825 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4826 notifier->ops->sched_in(notifier, raw_smp_processor_id()); 4827} 4828 4829static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4830{ 4831 if (static_branch_unlikely(&preempt_notifier_key)) 4832 __fire_sched_in_preempt_notifiers(curr); 4833} 4834 4835static void 4836__fire_sched_out_preempt_notifiers(struct task_struct *curr, 4837 struct task_struct *next) 4838{ 4839 struct preempt_notifier *notifier; 4840 4841 hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) 4842 notifier->ops->sched_out(notifier, next); 4843} 4844 4845static __always_inline void 4846fire_sched_out_preempt_notifiers(struct task_struct *curr, 4847 struct task_struct *next) 4848{ 4849 if (static_branch_unlikely(&preempt_notifier_key)) 4850 __fire_sched_out_preempt_notifiers(curr, next); 4851} 4852 4853#else /* !CONFIG_PREEMPT_NOTIFIERS: */ 4854 4855static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr) 4856{ 4857} 4858 4859static inline void 4860fire_sched_out_preempt_notifiers(struct task_struct *curr, 4861 struct task_struct *next) 4862{ 4863} 4864 4865#endif /* !CONFIG_PREEMPT_NOTIFIERS */ 4866 4867static inline void prepare_task(struct task_struct *next) 4868{ 4869 /* 4870 * Claim the task as running, we do this before switching to it 4871 * such that any running task will have this set. 4872 * 4873 * See the smp_load_acquire(&p->on_cpu) case in ttwu() and 4874 * its ordering comment. 4875 */ 4876 WRITE_ONCE(next->on_cpu, 1); 4877} 4878 4879static inline void finish_task(struct task_struct *prev) 4880{ 4881 /* 4882 * This must be the very last reference to @prev from this CPU. After 4883 * p->on_cpu is cleared, the task can be moved to a different CPU. We 4884 * must ensure this doesn't happen until the switch is completely 4885 * finished. 4886 * 4887 * In particular, the load of prev->state in finish_task_switch() must 4888 * happen before this. 4889 * 4890 * Pairs with the smp_cond_load_acquire() in try_to_wake_up(). 4891 */ 4892 smp_store_release(&prev->on_cpu, 0); 4893} 4894 4895static void do_balance_callbacks(struct rq *rq, struct balance_callback *head) 4896{ 4897 void (*func)(struct rq *rq); 4898 struct balance_callback *next; 4899 4900 lockdep_assert_rq_held(rq); 4901 4902 while (head) { 4903 func = (void (*)(struct rq *))head->func; 4904 next = head->next; 4905 head->next = NULL; 4906 head = next; 4907 4908 func(rq); 4909 } 4910} 4911 4912static void balance_push(struct rq *rq); 4913 4914/* 4915 * balance_push_callback is a right abuse of the callback interface and plays 4916 * by significantly different rules. 4917 * 4918 * Where the normal balance_callback's purpose is to be ran in the same context 4919 * that queued it (only later, when it's safe to drop rq->lock again), 4920 * balance_push_callback is specifically targeted at __schedule(). 4921 * 4922 * This abuse is tolerated because it places all the unlikely/odd cases behind 4923 * a single test, namely: rq->balance_callback == NULL. 4924 */ 4925struct balance_callback balance_push_callback = { 4926 .next = NULL, 4927 .func = balance_push, 4928}; 4929 4930static inline struct balance_callback * 4931__splice_balance_callbacks(struct rq *rq, bool split) 4932{ 4933 struct balance_callback *head = rq->balance_callback; 4934 4935 if (likely(!head)) 4936 return NULL; 4937 4938 lockdep_assert_rq_held(rq); 4939 /* 4940 * Must not take balance_push_callback off the list when 4941 * splice_balance_callbacks() and balance_callbacks() are not 4942 * in the same rq->lock section. 4943 * 4944 * In that case it would be possible for __schedule() to interleave 4945 * and observe the list empty. 4946 */ 4947 if (split && head == &balance_push_callback) 4948 head = NULL; 4949 else 4950 rq->balance_callback = NULL; 4951 4952 return head; 4953} 4954 4955struct balance_callback *splice_balance_callbacks(struct rq *rq) 4956{ 4957 return __splice_balance_callbacks(rq, true); 4958} 4959 4960void __balance_callbacks(struct rq *rq, struct rq_flags *rf) 4961{ 4962 if (rf) 4963 rq_unpin_lock(rq, rf); 4964 do_balance_callbacks(rq, __splice_balance_callbacks(rq, false)); 4965 if (rf) 4966 rq_repin_lock(rq, rf); 4967} 4968 4969void balance_callbacks(struct rq *rq, struct balance_callback *head) 4970{ 4971 unsigned long flags; 4972 4973 if (unlikely(head)) { 4974 raw_spin_rq_lock_irqsave(rq, flags); 4975 do_balance_callbacks(rq, head); 4976 raw_spin_rq_unlock_irqrestore(rq, flags); 4977 } 4978} 4979 4980static inline void 4981prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) 4982{ 4983 /* 4984 * Since the runqueue lock will be released by the next 4985 * task (which is an invalid locking op but in the case 4986 * of the scheduler it's an obvious special-case), so we 4987 * do an early lockdep release here: 4988 */ 4989 rq_unpin_lock(rq, rf); 4990 spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_); 4991#ifdef CONFIG_DEBUG_SPINLOCK 4992 /* this is a valid case when another task releases the spinlock */ 4993 rq_lockp(rq)->owner = next; 4994#endif 4995} 4996 4997static inline void finish_lock_switch(struct rq *rq) 4998{ 4999 /* 5000 * If we are tracking spinlock dependencies then we have to 5001 * fix up the runqueue lock - which gets 'carried over' from 5002 * prev into current: 5003 */ 5004 spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_); 5005 __balance_callbacks(rq, NULL); 5006 raw_spin_rq_unlock_irq(rq); 5007} 5008 5009/* 5010 * NOP if the arch has not defined these: 5011 */ 5012 5013#ifndef prepare_arch_switch 5014# define prepare_arch_switch(next) do { } while (0) 5015#endif 5016 5017#ifndef finish_arch_post_lock_switch 5018# define finish_arch_post_lock_switch() do { } while (0) 5019#endif 5020 5021static inline void kmap_local_sched_out(void) 5022{ 5023#ifdef CONFIG_KMAP_LOCAL 5024 if (unlikely(current->kmap_ctrl.idx)) 5025 __kmap_local_sched_out(); 5026#endif 5027} 5028 5029static inline void kmap_local_sched_in(void) 5030{ 5031#ifdef CONFIG_KMAP_LOCAL 5032 if (unlikely(current->kmap_ctrl.idx)) 5033 __kmap_local_sched_in(); 5034#endif 5035} 5036 5037/** 5038 * prepare_task_switch - prepare to switch tasks 5039 * @rq: the runqueue preparing to switch 5040 * @prev: the current task that is being switched out 5041 * @next: the task we are going to switch to. 5042 * 5043 * This is called with the rq lock held and interrupts off. It must 5044 * be paired with a subsequent finish_task_switch after the context 5045 * switch. 5046 * 5047 * prepare_task_switch sets up locking and calls architecture specific 5048 * hooks. 5049 */ 5050static inline void 5051prepare_task_switch(struct rq *rq, struct task_struct *prev, 5052 struct task_struct *next) 5053{ 5054 kcov_prepare_switch(prev); 5055 sched_info_switch(rq, prev, next); 5056 perf_event_task_sched_out(prev, next); 5057 fire_sched_out_preempt_notifiers(prev, next); 5058 kmap_local_sched_out(); 5059 prepare_task(next); 5060 prepare_arch_switch(next); 5061} 5062 5063/** 5064 * finish_task_switch - clean up after a task-switch 5065 * @prev: the thread we just switched away from. 5066 * 5067 * finish_task_switch must be called after the context switch, paired 5068 * with a prepare_task_switch call before the context switch. 5069 * finish_task_switch will reconcile locking set up by prepare_task_switch, 5070 * and do any other architecture-specific cleanup actions. 5071 * 5072 * Note that we may have delayed dropping an mm in context_switch(). If 5073 * so, we finish that here outside of the runqueue lock. (Doing it 5074 * with the lock held can cause deadlocks; see schedule() for 5075 * details.) 5076 * 5077 * The context switch have flipped the stack from under us and restored the 5078 * local variables which were saved when this task called schedule() in the 5079 * past. 'prev == current' is still correct but we need to recalculate this_rq 5080 * because prev may have moved to another CPU. 5081 */ 5082static struct rq *finish_task_switch(struct task_struct *prev) 5083 __releases(rq->lock) 5084{ 5085 struct rq *rq = this_rq(); 5086 struct mm_struct *mm = rq->prev_mm; 5087 unsigned int prev_state; 5088 5089 /* 5090 * The previous task will have left us with a preempt_count of 2 5091 * because it left us after: 5092 * 5093 * schedule() 5094 * preempt_disable(); // 1 5095 * __schedule() 5096 * raw_spin_lock_irq(&rq->lock) // 2 5097 * 5098 * Also, see FORK_PREEMPT_COUNT. 5099 */ 5100 if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET, 5101 "corrupted preempt_count: %s/%d/0x%x\n", 5102 current->comm, current->pid, preempt_count())) 5103 preempt_count_set(FORK_PREEMPT_COUNT); 5104 5105 rq->prev_mm = NULL; 5106 5107 /* 5108 * A task struct has one reference for the use as "current". 5109 * If a task dies, then it sets TASK_DEAD in tsk->state and calls 5110 * schedule one last time. The schedule call will never return, and 5111 * the scheduled task must drop that reference. 5112 * 5113 * We must observe prev->state before clearing prev->on_cpu (in 5114 * finish_task), otherwise a concurrent wakeup can get prev 5115 * running on another CPU and we could rave with its RUNNING -> DEAD 5116 * transition, resulting in a double drop. 5117 */ 5118 prev_state = READ_ONCE(prev->__state); 5119 vtime_task_switch(prev); 5120 perf_event_task_sched_in(prev, current); 5121 finish_task(prev); 5122 tick_nohz_task_switch(); 5123 finish_lock_switch(rq); 5124 finish_arch_post_lock_switch(); 5125 kcov_finish_switch(current); 5126 /* 5127 * kmap_local_sched_out() is invoked with rq::lock held and 5128 * interrupts disabled. There is no requirement for that, but the 5129 * sched out code does not have an interrupt enabled section. 5130 * Restoring the maps on sched in does not require interrupts being 5131 * disabled either. 5132 */ 5133 kmap_local_sched_in(); 5134 5135 fire_sched_in_preempt_notifiers(current); 5136 /* 5137 * When switching through a kernel thread, the loop in 5138 * membarrier_{private,global}_expedited() may have observed that 5139 * kernel thread and not issued an IPI. It is therefore possible to 5140 * schedule between user->kernel->user threads without passing though 5141 * switch_mm(). Membarrier requires a barrier after storing to 5142 * rq->curr, before returning to userspace, so provide them here: 5143 * 5144 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly 5145 * provided by mmdrop_lazy_tlb(), 5146 * - a sync_core for SYNC_CORE. 5147 */ 5148 if (mm) { 5149 membarrier_mm_sync_core_before_usermode(mm); 5150 mmdrop_lazy_tlb_sched(mm); 5151 } 5152 5153 if (unlikely(prev_state == TASK_DEAD)) { 5154 if (prev->sched_class->task_dead) 5155 prev->sched_class->task_dead(prev); 5156 5157 /* 5158 * sched_ext_dead() must come before cgroup_task_dead() to 5159 * prevent cgroups from being removed while its member tasks are 5160 * visible to SCX schedulers. 5161 */ 5162 sched_ext_dead(prev); 5163 cgroup_task_dead(prev); 5164 5165 /* Task is done with its stack. */ 5166 put_task_stack(prev); 5167 5168 put_task_struct_rcu_user(prev); 5169 } 5170 5171 return rq; 5172} 5173 5174/** 5175 * schedule_tail - first thing a freshly forked thread must call. 5176 * @prev: the thread we just switched away from. 5177 */ 5178asmlinkage __visible void schedule_tail(struct task_struct *prev) 5179 __releases(rq->lock) 5180{ 5181 /* 5182 * New tasks start with FORK_PREEMPT_COUNT, see there and 5183 * finish_task_switch() for details. 5184 * 5185 * finish_task_switch() will drop rq->lock() and lower preempt_count 5186 * and the preempt_enable() will end up enabling preemption (on 5187 * PREEMPT_COUNT kernels). 5188 */ 5189 5190 finish_task_switch(prev); 5191 /* 5192 * This is a special case: the newly created task has just 5193 * switched the context for the first time. It is returning from 5194 * schedule for the first time in this path. 5195 */ 5196 trace_sched_exit_tp(true); 5197 preempt_enable(); 5198 5199 if (current->set_child_tid) 5200 put_user(task_pid_vnr(current), current->set_child_tid); 5201 5202 calculate_sigpending(); 5203} 5204 5205/* 5206 * context_switch - switch to the new MM and the new thread's register state. 5207 */ 5208static __always_inline struct rq * 5209context_switch(struct rq *rq, struct task_struct *prev, 5210 struct task_struct *next, struct rq_flags *rf) 5211{ 5212 prepare_task_switch(rq, prev, next); 5213 5214 /* 5215 * For paravirt, this is coupled with an exit in switch_to to 5216 * combine the page table reload and the switch backend into 5217 * one hypercall. 5218 */ 5219 arch_start_context_switch(prev); 5220 5221 /* 5222 * kernel -> kernel lazy + transfer active 5223 * user -> kernel lazy + mmgrab_lazy_tlb() active 5224 * 5225 * kernel -> user switch + mmdrop_lazy_tlb() active 5226 * user -> user switch 5227 */ 5228 if (!next->mm) { // to kernel 5229 enter_lazy_tlb(prev->active_mm, next); 5230 5231 next->active_mm = prev->active_mm; 5232 if (prev->mm) // from user 5233 mmgrab_lazy_tlb(prev->active_mm); 5234 else 5235 prev->active_mm = NULL; 5236 } else { // to user 5237 membarrier_switch_mm(rq, prev->active_mm, next->mm); 5238 /* 5239 * sys_membarrier() requires an smp_mb() between setting 5240 * rq->curr / membarrier_switch_mm() and returning to userspace. 5241 * 5242 * The below provides this either through switch_mm(), or in 5243 * case 'prev->active_mm == next->mm' through 5244 * finish_task_switch()'s mmdrop(). 5245 */ 5246 switch_mm_irqs_off(prev->active_mm, next->mm, next); 5247 lru_gen_use_mm(next->mm); 5248 5249 if (!prev->mm) { // from kernel 5250 /* will mmdrop_lazy_tlb() in finish_task_switch(). */ 5251 rq->prev_mm = prev->active_mm; 5252 prev->active_mm = NULL; 5253 } 5254 } 5255 5256 mm_cid_switch_to(prev, next); 5257 5258 /* 5259 * Tell rseq that the task was scheduled in. Must be after 5260 * switch_mm_cid() to get the TIF flag set. 5261 */ 5262 rseq_sched_switch_event(next); 5263 5264 prepare_lock_switch(rq, next, rf); 5265 5266 /* Here we just switch the register state and the stack. */ 5267 switch_to(prev, next, prev); 5268 barrier(); 5269 5270 return finish_task_switch(prev); 5271} 5272 5273/* 5274 * nr_running and nr_context_switches: 5275 * 5276 * externally visible scheduler statistics: current number of runnable 5277 * threads, total number of context switches performed since bootup. 5278 */ 5279unsigned int nr_running(void) 5280{ 5281 unsigned int i, sum = 0; 5282 5283 for_each_online_cpu(i) 5284 sum += cpu_rq(i)->nr_running; 5285 5286 return sum; 5287} 5288 5289/* 5290 * Check if only the current task is running on the CPU. 5291 * 5292 * Caution: this function does not check that the caller has disabled 5293 * preemption, thus the result might have a time-of-check-to-time-of-use 5294 * race. The caller is responsible to use it correctly, for example: 5295 * 5296 * - from a non-preemptible section (of course) 5297 * 5298 * - from a thread that is bound to a single CPU 5299 * 5300 * - in a loop with very short iterations (e.g. a polling loop) 5301 */ 5302bool single_task_running(void) 5303{ 5304 return raw_rq()->nr_running == 1; 5305} 5306EXPORT_SYMBOL(single_task_running); 5307 5308unsigned long long nr_context_switches_cpu(int cpu) 5309{ 5310 return cpu_rq(cpu)->nr_switches; 5311} 5312 5313unsigned long long nr_context_switches(void) 5314{ 5315 int i; 5316 unsigned long long sum = 0; 5317 5318 for_each_possible_cpu(i) 5319 sum += cpu_rq(i)->nr_switches; 5320 5321 return sum; 5322} 5323 5324/* 5325 * Consumers of these two interfaces, like for example the cpuidle menu 5326 * governor, are using nonsensical data. Preferring shallow idle state selection 5327 * for a CPU that has IO-wait which might not even end up running the task when 5328 * it does become runnable. 5329 */ 5330 5331unsigned int nr_iowait_cpu(int cpu) 5332{ 5333 return atomic_read(&cpu_rq(cpu)->nr_iowait); 5334} 5335 5336/* 5337 * IO-wait accounting, and how it's mostly bollocks (on SMP). 5338 * 5339 * The idea behind IO-wait account is to account the idle time that we could 5340 * have spend running if it were not for IO. That is, if we were to improve the 5341 * storage performance, we'd have a proportional reduction in IO-wait time. 5342 * 5343 * This all works nicely on UP, where, when a task blocks on IO, we account 5344 * idle time as IO-wait, because if the storage were faster, it could've been 5345 * running and we'd not be idle. 5346 * 5347 * This has been extended to SMP, by doing the same for each CPU. This however 5348 * is broken. 5349 * 5350 * Imagine for instance the case where two tasks block on one CPU, only the one 5351 * CPU will have IO-wait accounted, while the other has regular idle. Even 5352 * though, if the storage were faster, both could've ran at the same time, 5353 * utilising both CPUs. 5354 * 5355 * This means, that when looking globally, the current IO-wait accounting on 5356 * SMP is a lower bound, by reason of under accounting. 5357 * 5358 * Worse, since the numbers are provided per CPU, they are sometimes 5359 * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly 5360 * associated with any one particular CPU, it can wake to another CPU than it 5361 * blocked on. This means the per CPU IO-wait number is meaningless. 5362 * 5363 * Task CPU affinities can make all that even more 'interesting'. 5364 */ 5365 5366unsigned int nr_iowait(void) 5367{ 5368 unsigned int i, sum = 0; 5369 5370 for_each_possible_cpu(i) 5371 sum += nr_iowait_cpu(i); 5372 5373 return sum; 5374} 5375 5376/* 5377 * sched_exec - execve() is a valuable balancing opportunity, because at 5378 * this point the task has the smallest effective memory and cache footprint. 5379 */ 5380void sched_exec(void) 5381{ 5382 struct task_struct *p = current; 5383 struct migration_arg arg; 5384 int dest_cpu; 5385 5386 scoped_guard (raw_spinlock_irqsave, &p->pi_lock) { 5387 dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC); 5388 if (dest_cpu == smp_processor_id()) 5389 return; 5390 5391 if (unlikely(!cpu_active(dest_cpu))) 5392 return; 5393 5394 arg = (struct migration_arg){ p, dest_cpu }; 5395 } 5396 stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); 5397} 5398 5399DEFINE_PER_CPU(struct kernel_stat, kstat); 5400DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); 5401 5402EXPORT_PER_CPU_SYMBOL(kstat); 5403EXPORT_PER_CPU_SYMBOL(kernel_cpustat); 5404 5405/* 5406 * The function fair_sched_class.update_curr accesses the struct curr 5407 * and its field curr->exec_start; when called from task_sched_runtime(), 5408 * we observe a high rate of cache misses in practice. 5409 * Prefetching this data results in improved performance. 5410 */ 5411static inline void prefetch_curr_exec_start(struct task_struct *p) 5412{ 5413#ifdef CONFIG_FAIR_GROUP_SCHED 5414 struct sched_entity *curr = p->se.cfs_rq->curr; 5415#else 5416 struct sched_entity *curr = task_rq(p)->cfs.curr; 5417#endif 5418 prefetch(curr); 5419 prefetch(&curr->exec_start); 5420} 5421 5422/* 5423 * Return accounted runtime for the task. 5424 * In case the task is currently running, return the runtime plus current's 5425 * pending runtime that have not been accounted yet. 5426 */ 5427unsigned long long task_sched_runtime(struct task_struct *p) 5428{ 5429 struct rq_flags rf; 5430 struct rq *rq; 5431 u64 ns; 5432 5433#ifdef CONFIG_64BIT 5434 /* 5435 * 64-bit doesn't need locks to atomically read a 64-bit value. 5436 * So we have a optimization chance when the task's delta_exec is 0. 5437 * Reading ->on_cpu is racy, but this is OK. 5438 * 5439 * If we race with it leaving CPU, we'll take a lock. So we're correct. 5440 * If we race with it entering CPU, unaccounted time is 0. This is 5441 * indistinguishable from the read occurring a few cycles earlier. 5442 * If we see ->on_cpu without ->on_rq, the task is leaving, and has 5443 * been accounted, so we're correct here as well. 5444 */ 5445 if (!p->on_cpu || !task_on_rq_queued(p)) 5446 return p->se.sum_exec_runtime; 5447#endif 5448 5449 rq = task_rq_lock(p, &rf); 5450 /* 5451 * Must be ->curr _and_ ->on_rq. If dequeued, we would 5452 * project cycles that may never be accounted to this 5453 * thread, breaking clock_gettime(). 5454 */ 5455 if (task_current_donor(rq, p) && task_on_rq_queued(p)) { 5456 prefetch_curr_exec_start(p); 5457 update_rq_clock(rq); 5458 p->sched_class->update_curr(rq); 5459 } 5460 ns = p->se.sum_exec_runtime; 5461 task_rq_unlock(rq, p, &rf); 5462 5463 return ns; 5464} 5465 5466static u64 cpu_resched_latency(struct rq *rq) 5467{ 5468 int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms); 5469 u64 resched_latency, now = rq_clock(rq); 5470 static bool warned_once; 5471 5472 if (sysctl_resched_latency_warn_once && warned_once) 5473 return 0; 5474 5475 if (!need_resched() || !latency_warn_ms) 5476 return 0; 5477 5478 if (system_state == SYSTEM_BOOTING) 5479 return 0; 5480 5481 if (!rq->last_seen_need_resched_ns) { 5482 rq->last_seen_need_resched_ns = now; 5483 rq->ticks_without_resched = 0; 5484 return 0; 5485 } 5486 5487 rq->ticks_without_resched++; 5488 resched_latency = now - rq->last_seen_need_resched_ns; 5489 if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC) 5490 return 0; 5491 5492 warned_once = true; 5493 5494 return resched_latency; 5495} 5496 5497static int __init setup_resched_latency_warn_ms(char *str) 5498{ 5499 long val; 5500 5501 if ((kstrtol(str, 0, &val))) { 5502 pr_warn("Unable to set resched_latency_warn_ms\n"); 5503 return 1; 5504 } 5505 5506 sysctl_resched_latency_warn_ms = val; 5507 return 1; 5508} 5509__setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms); 5510 5511/* 5512 * This function gets called by the timer code, with HZ frequency. 5513 * We call it with interrupts disabled. 5514 */ 5515void sched_tick(void) 5516{ 5517 int cpu = smp_processor_id(); 5518 struct rq *rq = cpu_rq(cpu); 5519 /* accounting goes to the donor task */ 5520 struct task_struct *donor; 5521 struct rq_flags rf; 5522 unsigned long hw_pressure; 5523 u64 resched_latency; 5524 5525 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) 5526 arch_scale_freq_tick(); 5527 5528 sched_clock_tick(); 5529 5530 rq_lock(rq, &rf); 5531 donor = rq->donor; 5532 5533 psi_account_irqtime(rq, donor, NULL); 5534 5535 update_rq_clock(rq); 5536 hw_pressure = arch_scale_hw_pressure(cpu_of(rq)); 5537 update_hw_load_avg(rq_clock_task(rq), rq, hw_pressure); 5538 5539 if (dynamic_preempt_lazy() && tif_test_bit(TIF_NEED_RESCHED_LAZY)) 5540 resched_curr(rq); 5541 5542 donor->sched_class->task_tick(rq, donor, 0); 5543 if (sched_feat(LATENCY_WARN)) 5544 resched_latency = cpu_resched_latency(rq); 5545 calc_global_load_tick(rq); 5546 sched_core_tick(rq); 5547 scx_tick(rq); 5548 5549 rq_unlock(rq, &rf); 5550 5551 if (sched_feat(LATENCY_WARN) && resched_latency) 5552 resched_latency_warn(cpu, resched_latency); 5553 5554 perf_event_task_tick(); 5555 5556 if (donor->flags & PF_WQ_WORKER) 5557 wq_worker_tick(donor); 5558 5559 if (!scx_switched_all()) { 5560 rq->idle_balance = idle_cpu(cpu); 5561 sched_balance_trigger(rq); 5562 } 5563} 5564 5565#ifdef CONFIG_NO_HZ_FULL 5566 5567struct tick_work { 5568 int cpu; 5569 atomic_t state; 5570 struct delayed_work work; 5571}; 5572/* Values for ->state, see diagram below. */ 5573#define TICK_SCHED_REMOTE_OFFLINE 0 5574#define TICK_SCHED_REMOTE_OFFLINING 1 5575#define TICK_SCHED_REMOTE_RUNNING 2 5576 5577/* 5578 * State diagram for ->state: 5579 * 5580 * 5581 * TICK_SCHED_REMOTE_OFFLINE 5582 * | ^ 5583 * | | 5584 * | | sched_tick_remote() 5585 * | | 5586 * | | 5587 * +--TICK_SCHED_REMOTE_OFFLINING 5588 * | ^ 5589 * | | 5590 * sched_tick_start() | | sched_tick_stop() 5591 * | | 5592 * V | 5593 * TICK_SCHED_REMOTE_RUNNING 5594 * 5595 * 5596 * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote() 5597 * and sched_tick_start() are happy to leave the state in RUNNING. 5598 */ 5599 5600static struct tick_work __percpu *tick_work_cpu; 5601 5602static void sched_tick_remote(struct work_struct *work) 5603{ 5604 struct delayed_work *dwork = to_delayed_work(work); 5605 struct tick_work *twork = container_of(dwork, struct tick_work, work); 5606 int cpu = twork->cpu; 5607 struct rq *rq = cpu_rq(cpu); 5608 int os; 5609 5610 /* 5611 * Handle the tick only if it appears the remote CPU is running in full 5612 * dynticks mode. The check is racy by nature, but missing a tick or 5613 * having one too much is no big deal because the scheduler tick updates 5614 * statistics and checks timeslices in a time-independent way, regardless 5615 * of when exactly it is running. 5616 */ 5617 if (tick_nohz_tick_stopped_cpu(cpu)) { 5618 guard(rq_lock_irq)(rq); 5619 struct task_struct *curr = rq->curr; 5620 5621 if (cpu_online(cpu)) { 5622 /* 5623 * Since this is a remote tick for full dynticks mode, 5624 * we are always sure that there is no proxy (only a 5625 * single task is running). 5626 */ 5627 WARN_ON_ONCE(rq->curr != rq->donor); 5628 update_rq_clock(rq); 5629 5630 if (!is_idle_task(curr)) { 5631 /* 5632 * Make sure the next tick runs within a 5633 * reasonable amount of time. 5634 */ 5635 u64 delta = rq_clock_task(rq) - curr->se.exec_start; 5636 WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 30); 5637 } 5638 curr->sched_class->task_tick(rq, curr, 0); 5639 5640 calc_load_nohz_remote(rq); 5641 } 5642 } 5643 5644 /* 5645 * Run the remote tick once per second (1Hz). This arbitrary 5646 * frequency is large enough to avoid overload but short enough 5647 * to keep scheduler internal stats reasonably up to date. But 5648 * first update state to reflect hotplug activity if required. 5649 */ 5650 os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING); 5651 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE); 5652 if (os == TICK_SCHED_REMOTE_RUNNING) 5653 queue_delayed_work(system_unbound_wq, dwork, HZ); 5654} 5655 5656static void sched_tick_start(int cpu) 5657{ 5658 int os; 5659 struct tick_work *twork; 5660 5661 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) 5662 return; 5663 5664 WARN_ON_ONCE(!tick_work_cpu); 5665 5666 twork = per_cpu_ptr(tick_work_cpu, cpu); 5667 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING); 5668 WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING); 5669 if (os == TICK_SCHED_REMOTE_OFFLINE) { 5670 twork->cpu = cpu; 5671 INIT_DELAYED_WORK(&twork->work, sched_tick_remote); 5672 queue_delayed_work(system_unbound_wq, &twork->work, HZ); 5673 } 5674} 5675 5676#ifdef CONFIG_HOTPLUG_CPU 5677static void sched_tick_stop(int cpu) 5678{ 5679 struct tick_work *twork; 5680 int os; 5681 5682 if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE)) 5683 return; 5684 5685 WARN_ON_ONCE(!tick_work_cpu); 5686 5687 twork = per_cpu_ptr(tick_work_cpu, cpu); 5688 /* There cannot be competing actions, but don't rely on stop-machine. */ 5689 os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING); 5690 WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING); 5691 /* Don't cancel, as this would mess up the state machine. */ 5692} 5693#endif /* CONFIG_HOTPLUG_CPU */ 5694 5695int __init sched_tick_offload_init(void) 5696{ 5697 tick_work_cpu = alloc_percpu(struct tick_work); 5698 BUG_ON(!tick_work_cpu); 5699 return 0; 5700} 5701 5702#else /* !CONFIG_NO_HZ_FULL: */ 5703static inline void sched_tick_start(int cpu) { } 5704static inline void sched_tick_stop(int cpu) { } 5705#endif /* !CONFIG_NO_HZ_FULL */ 5706 5707#if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \ 5708 defined(CONFIG_TRACE_PREEMPT_TOGGLE)) 5709/* 5710 * If the value passed in is equal to the current preempt count 5711 * then we just disabled preemption. Start timing the latency. 5712 */ 5713static inline void preempt_latency_start(int val) 5714{ 5715 if (preempt_count() == val) { 5716 unsigned long ip = get_lock_parent_ip(); 5717#ifdef CONFIG_DEBUG_PREEMPT 5718 current->preempt_disable_ip = ip; 5719#endif 5720 trace_preempt_off(CALLER_ADDR0, ip); 5721 } 5722} 5723 5724void preempt_count_add(int val) 5725{ 5726#ifdef CONFIG_DEBUG_PREEMPT 5727 /* 5728 * Underflow? 5729 */ 5730 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 5731 return; 5732#endif 5733 __preempt_count_add(val); 5734#ifdef CONFIG_DEBUG_PREEMPT 5735 /* 5736 * Spinlock count overflowing soon? 5737 */ 5738 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 5739 PREEMPT_MASK - 10); 5740#endif 5741 preempt_latency_start(val); 5742} 5743EXPORT_SYMBOL(preempt_count_add); 5744NOKPROBE_SYMBOL(preempt_count_add); 5745 5746/* 5747 * If the value passed in equals to the current preempt count 5748 * then we just enabled preemption. Stop timing the latency. 5749 */ 5750static inline void preempt_latency_stop(int val) 5751{ 5752 if (preempt_count() == val) 5753 trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip()); 5754} 5755 5756void preempt_count_sub(int val) 5757{ 5758#ifdef CONFIG_DEBUG_PREEMPT 5759 /* 5760 * Underflow? 5761 */ 5762 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 5763 return; 5764 /* 5765 * Is the spinlock portion underflowing? 5766 */ 5767 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 5768 !(preempt_count() & PREEMPT_MASK))) 5769 return; 5770#endif 5771 5772 preempt_latency_stop(val); 5773 __preempt_count_sub(val); 5774} 5775EXPORT_SYMBOL(preempt_count_sub); 5776NOKPROBE_SYMBOL(preempt_count_sub); 5777 5778#else 5779static inline void preempt_latency_start(int val) { } 5780static inline void preempt_latency_stop(int val) { } 5781#endif 5782 5783static inline unsigned long get_preempt_disable_ip(struct task_struct *p) 5784{ 5785#ifdef CONFIG_DEBUG_PREEMPT 5786 return p->preempt_disable_ip; 5787#else 5788 return 0; 5789#endif 5790} 5791 5792/* 5793 * Print scheduling while atomic bug: 5794 */ 5795static noinline void __schedule_bug(struct task_struct *prev) 5796{ 5797 /* Save this before calling printk(), since that will clobber it */ 5798 unsigned long preempt_disable_ip = get_preempt_disable_ip(current); 5799 5800 if (oops_in_progress) 5801 return; 5802 5803 printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", 5804 prev->comm, prev->pid, preempt_count()); 5805 5806 debug_show_held_locks(prev); 5807 print_modules(); 5808 if (irqs_disabled()) 5809 print_irqtrace_events(prev); 5810 if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) { 5811 pr_err("Preemption disabled at:"); 5812 print_ip_sym(KERN_ERR, preempt_disable_ip); 5813 } 5814 check_panic_on_warn("scheduling while atomic"); 5815 5816 dump_stack(); 5817 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5818} 5819 5820/* 5821 * Various schedule()-time debugging checks and statistics: 5822 */ 5823static inline void schedule_debug(struct task_struct *prev, bool preempt) 5824{ 5825#ifdef CONFIG_SCHED_STACK_END_CHECK 5826 if (task_stack_end_corrupted(prev)) 5827 panic("corrupted stack end detected inside scheduler\n"); 5828 5829 if (task_scs_end_corrupted(prev)) 5830 panic("corrupted shadow stack detected inside scheduler\n"); 5831#endif 5832 5833#ifdef CONFIG_DEBUG_ATOMIC_SLEEP 5834 if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) { 5835 printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n", 5836 prev->comm, prev->pid, prev->non_block_count); 5837 dump_stack(); 5838 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 5839 } 5840#endif 5841 5842 if (unlikely(in_atomic_preempt_off())) { 5843 __schedule_bug(prev); 5844 preempt_count_set(PREEMPT_DISABLED); 5845 } 5846 rcu_sleep_check(); 5847 WARN_ON_ONCE(ct_state() == CT_STATE_USER); 5848 5849 profile_hit(SCHED_PROFILING, __builtin_return_address(0)); 5850 5851 schedstat_inc(this_rq()->sched_count); 5852} 5853 5854static void prev_balance(struct rq *rq, struct task_struct *prev, 5855 struct rq_flags *rf) 5856{ 5857 const struct sched_class *start_class = prev->sched_class; 5858 const struct sched_class *class; 5859 5860 /* 5861 * We must do the balancing pass before put_prev_task(), such 5862 * that when we release the rq->lock the task is in the same 5863 * state as before we took rq->lock. 5864 * 5865 * We can terminate the balance pass as soon as we know there is 5866 * a runnable task of @class priority or higher. 5867 */ 5868 for_active_class_range(class, start_class, &idle_sched_class) { 5869 if (class->balance && class->balance(rq, prev, rf)) 5870 break; 5871 } 5872} 5873 5874/* 5875 * Pick up the highest-prio task: 5876 */ 5877static inline struct task_struct * 5878__pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5879{ 5880 const struct sched_class *class; 5881 struct task_struct *p; 5882 5883 rq->dl_server = NULL; 5884 5885 if (scx_enabled()) 5886 goto restart; 5887 5888 /* 5889 * Optimization: we know that if all tasks are in the fair class we can 5890 * call that function directly, but only if the @prev task wasn't of a 5891 * higher scheduling class, because otherwise those lose the 5892 * opportunity to pull in more work from other CPUs. 5893 */ 5894 if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && 5895 rq->nr_running == rq->cfs.h_nr_queued)) { 5896 5897 p = pick_next_task_fair(rq, prev, rf); 5898 if (unlikely(p == RETRY_TASK)) 5899 goto restart; 5900 5901 /* Assume the next prioritized class is idle_sched_class */ 5902 if (!p) { 5903 p = pick_task_idle(rq, rf); 5904 put_prev_set_next_task(rq, prev, p); 5905 } 5906 5907 return p; 5908 } 5909 5910restart: 5911 prev_balance(rq, prev, rf); 5912 5913 for_each_active_class(class) { 5914 if (class->pick_next_task) { 5915 p = class->pick_next_task(rq, prev, rf); 5916 if (unlikely(p == RETRY_TASK)) 5917 goto restart; 5918 if (p) 5919 return p; 5920 } else { 5921 p = class->pick_task(rq, rf); 5922 if (unlikely(p == RETRY_TASK)) 5923 goto restart; 5924 if (p) { 5925 put_prev_set_next_task(rq, prev, p); 5926 return p; 5927 } 5928 } 5929 } 5930 5931 BUG(); /* The idle class should always have a runnable task. */ 5932} 5933 5934#ifdef CONFIG_SCHED_CORE 5935static inline bool is_task_rq_idle(struct task_struct *t) 5936{ 5937 return (task_rq(t)->idle == t); 5938} 5939 5940static inline bool cookie_equals(struct task_struct *a, unsigned long cookie) 5941{ 5942 return is_task_rq_idle(a) || (a->core_cookie == cookie); 5943} 5944 5945static inline bool cookie_match(struct task_struct *a, struct task_struct *b) 5946{ 5947 if (is_task_rq_idle(a) || is_task_rq_idle(b)) 5948 return true; 5949 5950 return a->core_cookie == b->core_cookie; 5951} 5952 5953/* 5954 * Careful; this can return RETRY_TASK, it does not include the retry-loop 5955 * itself due to the whole SMT pick retry thing below. 5956 */ 5957static inline struct task_struct *pick_task(struct rq *rq, struct rq_flags *rf) 5958{ 5959 const struct sched_class *class; 5960 struct task_struct *p; 5961 5962 rq->dl_server = NULL; 5963 5964 for_each_active_class(class) { 5965 p = class->pick_task(rq, rf); 5966 if (p) 5967 return p; 5968 } 5969 5970 BUG(); /* The idle class should always have a runnable task. */ 5971} 5972 5973extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi); 5974 5975static void queue_core_balance(struct rq *rq); 5976 5977static struct task_struct * 5978pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 5979{ 5980 struct task_struct *next, *p, *max; 5981 const struct cpumask *smt_mask; 5982 bool fi_before = false; 5983 bool core_clock_updated = (rq == rq->core); 5984 unsigned long cookie; 5985 int i, cpu, occ = 0; 5986 struct rq *rq_i; 5987 bool need_sync; 5988 5989 if (!sched_core_enabled(rq)) 5990 return __pick_next_task(rq, prev, rf); 5991 5992 cpu = cpu_of(rq); 5993 5994 /* Stopper task is switching into idle, no need core-wide selection. */ 5995 if (cpu_is_offline(cpu)) { 5996 /* 5997 * Reset core_pick so that we don't enter the fastpath when 5998 * coming online. core_pick would already be migrated to 5999 * another cpu during offline. 6000 */ 6001 rq->core_pick = NULL; 6002 rq->core_dl_server = NULL; 6003 return __pick_next_task(rq, prev, rf); 6004 } 6005 6006 /* 6007 * If there were no {en,de}queues since we picked (IOW, the task 6008 * pointers are all still valid), and we haven't scheduled the last 6009 * pick yet, do so now. 6010 * 6011 * rq->core_pick can be NULL if no selection was made for a CPU because 6012 * it was either offline or went offline during a sibling's core-wide 6013 * selection. In this case, do a core-wide selection. 6014 */ 6015 if (rq->core->core_pick_seq == rq->core->core_task_seq && 6016 rq->core->core_pick_seq != rq->core_sched_seq && 6017 rq->core_pick) { 6018 WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq); 6019 6020 next = rq->core_pick; 6021 rq->dl_server = rq->core_dl_server; 6022 rq->core_pick = NULL; 6023 rq->core_dl_server = NULL; 6024 goto out_set_next; 6025 } 6026 6027 prev_balance(rq, prev, rf); 6028 6029 smt_mask = cpu_smt_mask(cpu); 6030 need_sync = !!rq->core->core_cookie; 6031 6032 /* reset state */ 6033 rq->core->core_cookie = 0UL; 6034 if (rq->core->core_forceidle_count) { 6035 if (!core_clock_updated) { 6036 update_rq_clock(rq->core); 6037 core_clock_updated = true; 6038 } 6039 sched_core_account_forceidle(rq); 6040 /* reset after accounting force idle */ 6041 rq->core->core_forceidle_start = 0; 6042 rq->core->core_forceidle_count = 0; 6043 rq->core->core_forceidle_occupation = 0; 6044 need_sync = true; 6045 fi_before = true; 6046 } 6047 6048 /* 6049 * core->core_task_seq, core->core_pick_seq, rq->core_sched_seq 6050 * 6051 * @task_seq guards the task state ({en,de}queues) 6052 * @pick_seq is the @task_seq we did a selection on 6053 * @sched_seq is the @pick_seq we scheduled 6054 * 6055 * However, preemptions can cause multiple picks on the same task set. 6056 * 'Fix' this by also increasing @task_seq for every pick. 6057 */ 6058 rq->core->core_task_seq++; 6059 6060 /* 6061 * Optimize for common case where this CPU has no cookies 6062 * and there are no cookied tasks running on siblings. 6063 */ 6064 if (!need_sync) { 6065restart_single: 6066 next = pick_task(rq, rf); 6067 if (unlikely(next == RETRY_TASK)) 6068 goto restart_single; 6069 if (!next->core_cookie) { 6070 rq->core_pick = NULL; 6071 rq->core_dl_server = NULL; 6072 /* 6073 * For robustness, update the min_vruntime_fi for 6074 * unconstrained picks as well. 6075 */ 6076 WARN_ON_ONCE(fi_before); 6077 task_vruntime_update(rq, next, false); 6078 goto out_set_next; 6079 } 6080 } 6081 6082 /* 6083 * For each thread: do the regular task pick and find the max prio task 6084 * amongst them. 6085 * 6086 * Tie-break prio towards the current CPU 6087 */ 6088restart_multi: 6089 max = NULL; 6090 for_each_cpu_wrap(i, smt_mask, cpu) { 6091 rq_i = cpu_rq(i); 6092 6093 /* 6094 * Current cpu always has its clock updated on entrance to 6095 * pick_next_task(). If the current cpu is not the core, 6096 * the core may also have been updated above. 6097 */ 6098 if (i != cpu && (rq_i != rq->core || !core_clock_updated)) 6099 update_rq_clock(rq_i); 6100 6101 p = pick_task(rq_i, rf); 6102 if (unlikely(p == RETRY_TASK)) 6103 goto restart_multi; 6104 6105 rq_i->core_pick = p; 6106 rq_i->core_dl_server = rq_i->dl_server; 6107 6108 if (!max || prio_less(max, p, fi_before)) 6109 max = p; 6110 } 6111 6112 cookie = rq->core->core_cookie = max->core_cookie; 6113 6114 /* 6115 * For each thread: try and find a runnable task that matches @max or 6116 * force idle. 6117 */ 6118 for_each_cpu(i, smt_mask) { 6119 rq_i = cpu_rq(i); 6120 p = rq_i->core_pick; 6121 6122 if (!cookie_equals(p, cookie)) { 6123 p = NULL; 6124 if (cookie) 6125 p = sched_core_find(rq_i, cookie); 6126 if (!p) 6127 p = idle_sched_class.pick_task(rq_i, rf); 6128 } 6129 6130 rq_i->core_pick = p; 6131 rq_i->core_dl_server = NULL; 6132 6133 if (p == rq_i->idle) { 6134 if (rq_i->nr_running) { 6135 rq->core->core_forceidle_count++; 6136 if (!fi_before) 6137 rq->core->core_forceidle_seq++; 6138 } 6139 } else { 6140 occ++; 6141 } 6142 } 6143 6144 if (schedstat_enabled() && rq->core->core_forceidle_count) { 6145 rq->core->core_forceidle_start = rq_clock(rq->core); 6146 rq->core->core_forceidle_occupation = occ; 6147 } 6148 6149 rq->core->core_pick_seq = rq->core->core_task_seq; 6150 next = rq->core_pick; 6151 rq->core_sched_seq = rq->core->core_pick_seq; 6152 6153 /* Something should have been selected for current CPU */ 6154 WARN_ON_ONCE(!next); 6155 6156 /* 6157 * Reschedule siblings 6158 * 6159 * NOTE: L1TF -- at this point we're no longer running the old task and 6160 * sending an IPI (below) ensures the sibling will no longer be running 6161 * their task. This ensures there is no inter-sibling overlap between 6162 * non-matching user state. 6163 */ 6164 for_each_cpu(i, smt_mask) { 6165 rq_i = cpu_rq(i); 6166 6167 /* 6168 * An online sibling might have gone offline before a task 6169 * could be picked for it, or it might be offline but later 6170 * happen to come online, but its too late and nothing was 6171 * picked for it. That's Ok - it will pick tasks for itself, 6172 * so ignore it. 6173 */ 6174 if (!rq_i->core_pick) 6175 continue; 6176 6177 /* 6178 * Update for new !FI->FI transitions, or if continuing to be in !FI: 6179 * fi_before fi update? 6180 * 0 0 1 6181 * 0 1 1 6182 * 1 0 1 6183 * 1 1 0 6184 */ 6185 if (!(fi_before && rq->core->core_forceidle_count)) 6186 task_vruntime_update(rq_i, rq_i->core_pick, !!rq->core->core_forceidle_count); 6187 6188 rq_i->core_pick->core_occupation = occ; 6189 6190 if (i == cpu) { 6191 rq_i->core_pick = NULL; 6192 rq_i->core_dl_server = NULL; 6193 continue; 6194 } 6195 6196 /* Did we break L1TF mitigation requirements? */ 6197 WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick)); 6198 6199 if (rq_i->curr == rq_i->core_pick) { 6200 rq_i->core_pick = NULL; 6201 rq_i->core_dl_server = NULL; 6202 continue; 6203 } 6204 6205 resched_curr(rq_i); 6206 } 6207 6208out_set_next: 6209 put_prev_set_next_task(rq, prev, next); 6210 if (rq->core->core_forceidle_count && next == rq->idle) 6211 queue_core_balance(rq); 6212 6213 return next; 6214} 6215 6216static bool try_steal_cookie(int this, int that) 6217{ 6218 struct rq *dst = cpu_rq(this), *src = cpu_rq(that); 6219 struct task_struct *p; 6220 unsigned long cookie; 6221 bool success = false; 6222 6223 guard(irq)(); 6224 guard(double_rq_lock)(dst, src); 6225 6226 cookie = dst->core->core_cookie; 6227 if (!cookie) 6228 return false; 6229 6230 if (dst->curr != dst->idle) 6231 return false; 6232 6233 p = sched_core_find(src, cookie); 6234 if (!p) 6235 return false; 6236 6237 do { 6238 if (p == src->core_pick || p == src->curr) 6239 goto next; 6240 6241 if (!is_cpu_allowed(p, this)) 6242 goto next; 6243 6244 if (p->core_occupation > dst->idle->core_occupation) 6245 goto next; 6246 /* 6247 * sched_core_find() and sched_core_next() will ensure 6248 * that task @p is not throttled now, we also need to 6249 * check whether the runqueue of the destination CPU is 6250 * being throttled. 6251 */ 6252 if (sched_task_is_throttled(p, this)) 6253 goto next; 6254 6255 move_queued_task_locked(src, dst, p); 6256 resched_curr(dst); 6257 6258 success = true; 6259 break; 6260 6261next: 6262 p = sched_core_next(p, cookie); 6263 } while (p); 6264 6265 return success; 6266} 6267 6268static bool steal_cookie_task(int cpu, struct sched_domain *sd) 6269{ 6270 int i; 6271 6272 for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) { 6273 if (i == cpu) 6274 continue; 6275 6276 if (need_resched()) 6277 break; 6278 6279 if (try_steal_cookie(cpu, i)) 6280 return true; 6281 } 6282 6283 return false; 6284} 6285 6286static void sched_core_balance(struct rq *rq) 6287{ 6288 struct sched_domain *sd; 6289 int cpu = cpu_of(rq); 6290 6291 guard(preempt)(); 6292 guard(rcu)(); 6293 6294 raw_spin_rq_unlock_irq(rq); 6295 for_each_domain(cpu, sd) { 6296 if (need_resched()) 6297 break; 6298 6299 if (steal_cookie_task(cpu, sd)) 6300 break; 6301 } 6302 raw_spin_rq_lock_irq(rq); 6303} 6304 6305static DEFINE_PER_CPU(struct balance_callback, core_balance_head); 6306 6307static void queue_core_balance(struct rq *rq) 6308{ 6309 if (!sched_core_enabled(rq)) 6310 return; 6311 6312 if (!rq->core->core_cookie) 6313 return; 6314 6315 if (!rq->nr_running) /* not forced idle */ 6316 return; 6317 6318 queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance); 6319} 6320 6321DEFINE_LOCK_GUARD_1(core_lock, int, 6322 sched_core_lock(*_T->lock, &_T->flags), 6323 sched_core_unlock(*_T->lock, &_T->flags), 6324 unsigned long flags) 6325 6326static void sched_core_cpu_starting(unsigned int cpu) 6327{ 6328 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6329 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6330 int t; 6331 6332 guard(core_lock)(&cpu); 6333 6334 WARN_ON_ONCE(rq->core != rq); 6335 6336 /* if we're the first, we'll be our own leader */ 6337 if (cpumask_weight(smt_mask) == 1) 6338 return; 6339 6340 /* find the leader */ 6341 for_each_cpu(t, smt_mask) { 6342 if (t == cpu) 6343 continue; 6344 rq = cpu_rq(t); 6345 if (rq->core == rq) { 6346 core_rq = rq; 6347 break; 6348 } 6349 } 6350 6351 if (WARN_ON_ONCE(!core_rq)) /* whoopsie */ 6352 return; 6353 6354 /* install and validate core_rq */ 6355 for_each_cpu(t, smt_mask) { 6356 rq = cpu_rq(t); 6357 6358 if (t == cpu) 6359 rq->core = core_rq; 6360 6361 WARN_ON_ONCE(rq->core != core_rq); 6362 } 6363} 6364 6365static void sched_core_cpu_deactivate(unsigned int cpu) 6366{ 6367 const struct cpumask *smt_mask = cpu_smt_mask(cpu); 6368 struct rq *rq = cpu_rq(cpu), *core_rq = NULL; 6369 int t; 6370 6371 guard(core_lock)(&cpu); 6372 6373 /* if we're the last man standing, nothing to do */ 6374 if (cpumask_weight(smt_mask) == 1) { 6375 WARN_ON_ONCE(rq->core != rq); 6376 return; 6377 } 6378 6379 /* if we're not the leader, nothing to do */ 6380 if (rq->core != rq) 6381 return; 6382 6383 /* find a new leader */ 6384 for_each_cpu(t, smt_mask) { 6385 if (t == cpu) 6386 continue; 6387 core_rq = cpu_rq(t); 6388 break; 6389 } 6390 6391 if (WARN_ON_ONCE(!core_rq)) /* impossible */ 6392 return; 6393 6394 /* copy the shared state to the new leader */ 6395 core_rq->core_task_seq = rq->core_task_seq; 6396 core_rq->core_pick_seq = rq->core_pick_seq; 6397 core_rq->core_cookie = rq->core_cookie; 6398 core_rq->core_forceidle_count = rq->core_forceidle_count; 6399 core_rq->core_forceidle_seq = rq->core_forceidle_seq; 6400 core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; 6401 6402 /* 6403 * Accounting edge for forced idle is handled in pick_next_task(). 6404 * Don't need another one here, since the hotplug thread shouldn't 6405 * have a cookie. 6406 */ 6407 core_rq->core_forceidle_start = 0; 6408 6409 /* install new leader */ 6410 for_each_cpu(t, smt_mask) { 6411 rq = cpu_rq(t); 6412 rq->core = core_rq; 6413 } 6414} 6415 6416static inline void sched_core_cpu_dying(unsigned int cpu) 6417{ 6418 struct rq *rq = cpu_rq(cpu); 6419 6420 if (rq->core != rq) 6421 rq->core = rq; 6422} 6423 6424#else /* !CONFIG_SCHED_CORE: */ 6425 6426static inline void sched_core_cpu_starting(unsigned int cpu) {} 6427static inline void sched_core_cpu_deactivate(unsigned int cpu) {} 6428static inline void sched_core_cpu_dying(unsigned int cpu) {} 6429 6430static struct task_struct * 6431pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 6432{ 6433 return __pick_next_task(rq, prev, rf); 6434} 6435 6436#endif /* !CONFIG_SCHED_CORE */ 6437 6438/* 6439 * Constants for the sched_mode argument of __schedule(). 6440 * 6441 * The mode argument allows RT enabled kernels to differentiate a 6442 * preemption from blocking on an 'sleeping' spin/rwlock. 6443 */ 6444#define SM_IDLE (-1) 6445#define SM_NONE 0 6446#define SM_PREEMPT 1 6447#define SM_RTLOCK_WAIT 2 6448 6449/* 6450 * Helper function for __schedule() 6451 * 6452 * Tries to deactivate the task, unless the should_block arg 6453 * is false or if a signal is pending. In the case a signal 6454 * is pending, marks the task's __state as RUNNING (and clear 6455 * blocked_on). 6456 */ 6457static bool try_to_block_task(struct rq *rq, struct task_struct *p, 6458 unsigned long *task_state_p, bool should_block) 6459{ 6460 unsigned long task_state = *task_state_p; 6461 int flags = DEQUEUE_NOCLOCK; 6462 6463 if (signal_pending_state(task_state, p)) { 6464 WRITE_ONCE(p->__state, TASK_RUNNING); 6465 *task_state_p = TASK_RUNNING; 6466 return false; 6467 } 6468 6469 /* 6470 * We check should_block after signal_pending because we 6471 * will want to wake the task in that case. But if 6472 * should_block is false, its likely due to the task being 6473 * blocked on a mutex, and we want to keep it on the runqueue 6474 * to be selectable for proxy-execution. 6475 */ 6476 if (!should_block) 6477 return false; 6478 6479 p->sched_contributes_to_load = 6480 (task_state & TASK_UNINTERRUPTIBLE) && 6481 !(task_state & TASK_NOLOAD) && 6482 !(task_state & TASK_FROZEN); 6483 6484 if (unlikely(is_special_task_state(task_state))) 6485 flags |= DEQUEUE_SPECIAL; 6486 6487 /* 6488 * __schedule() ttwu() 6489 * prev_state = prev->state; if (p->on_rq && ...) 6490 * if (prev_state) goto out; 6491 * p->on_rq = 0; smp_acquire__after_ctrl_dep(); 6492 * p->state = TASK_WAKING 6493 * 6494 * Where __schedule() and ttwu() have matching control dependencies. 6495 * 6496 * After this, schedule() must not care about p->state any more. 6497 */ 6498 block_task(rq, p, flags); 6499 return true; 6500} 6501 6502#ifdef CONFIG_SCHED_PROXY_EXEC 6503static inline struct task_struct *proxy_resched_idle(struct rq *rq) 6504{ 6505 put_prev_set_next_task(rq, rq->donor, rq->idle); 6506 rq_set_donor(rq, rq->idle); 6507 set_tsk_need_resched(rq->idle); 6508 return rq->idle; 6509} 6510 6511static bool __proxy_deactivate(struct rq *rq, struct task_struct *donor) 6512{ 6513 unsigned long state = READ_ONCE(donor->__state); 6514 6515 /* Don't deactivate if the state has been changed to TASK_RUNNING */ 6516 if (state == TASK_RUNNING) 6517 return false; 6518 /* 6519 * Because we got donor from pick_next_task(), it is *crucial* 6520 * that we call proxy_resched_idle() before we deactivate it. 6521 * As once we deactivate donor, donor->on_rq is set to zero, 6522 * which allows ttwu() to immediately try to wake the task on 6523 * another rq. So we cannot use *any* references to donor 6524 * after that point. So things like cfs_rq->curr or rq->donor 6525 * need to be changed from next *before* we deactivate. 6526 */ 6527 proxy_resched_idle(rq); 6528 return try_to_block_task(rq, donor, &state, true); 6529} 6530 6531static struct task_struct *proxy_deactivate(struct rq *rq, struct task_struct *donor) 6532{ 6533 if (!__proxy_deactivate(rq, donor)) { 6534 /* 6535 * XXX: For now, if deactivation failed, set donor 6536 * as unblocked, as we aren't doing proxy-migrations 6537 * yet (more logic will be needed then). 6538 */ 6539 donor->blocked_on = NULL; 6540 } 6541 return NULL; 6542} 6543 6544/* 6545 * Find runnable lock owner to proxy for mutex blocked donor 6546 * 6547 * Follow the blocked-on relation: 6548 * task->blocked_on -> mutex->owner -> task... 6549 * 6550 * Lock order: 6551 * 6552 * p->pi_lock 6553 * rq->lock 6554 * mutex->wait_lock 6555 * 6556 * Returns the task that is going to be used as execution context (the one 6557 * that is actually going to be run on cpu_of(rq)). 6558 */ 6559static struct task_struct * 6560find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) 6561{ 6562 struct task_struct *owner = NULL; 6563 int this_cpu = cpu_of(rq); 6564 struct task_struct *p; 6565 struct mutex *mutex; 6566 6567 /* Follow blocked_on chain. */ 6568 for (p = donor; task_is_blocked(p); p = owner) { 6569 mutex = p->blocked_on; 6570 /* Something changed in the chain, so pick again */ 6571 if (!mutex) 6572 return NULL; 6573 /* 6574 * By taking mutex->wait_lock we hold off concurrent mutex_unlock() 6575 * and ensure @owner sticks around. 6576 */ 6577 guard(raw_spinlock)(&mutex->wait_lock); 6578 6579 /* Check again that p is blocked with wait_lock held */ 6580 if (mutex != __get_task_blocked_on(p)) { 6581 /* 6582 * Something changed in the blocked_on chain and 6583 * we don't know if only at this level. So, let's 6584 * just bail out completely and let __schedule() 6585 * figure things out (pick_again loop). 6586 */ 6587 return NULL; 6588 } 6589 6590 owner = __mutex_owner(mutex); 6591 if (!owner) { 6592 __clear_task_blocked_on(p, mutex); 6593 return p; 6594 } 6595 6596 if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) { 6597 /* XXX Don't handle blocked owners/delayed dequeue yet */ 6598 return proxy_deactivate(rq, donor); 6599 } 6600 6601 if (task_cpu(owner) != this_cpu) { 6602 /* XXX Don't handle migrations yet */ 6603 return proxy_deactivate(rq, donor); 6604 } 6605 6606 if (task_on_rq_migrating(owner)) { 6607 /* 6608 * One of the chain of mutex owners is currently migrating to this 6609 * CPU, but has not yet been enqueued because we are holding the 6610 * rq lock. As a simple solution, just schedule rq->idle to give 6611 * the migration a chance to complete. Much like the migrate_task 6612 * case we should end up back in find_proxy_task(), this time 6613 * hopefully with all relevant tasks already enqueued. 6614 */ 6615 return proxy_resched_idle(rq); 6616 } 6617 6618 /* 6619 * Its possible to race where after we check owner->on_rq 6620 * but before we check (owner_cpu != this_cpu) that the 6621 * task on another cpu was migrated back to this cpu. In 6622 * that case it could slip by our checks. So double check 6623 * we are still on this cpu and not migrating. If we get 6624 * inconsistent results, try again. 6625 */ 6626 if (!task_on_rq_queued(owner) || task_cpu(owner) != this_cpu) 6627 return NULL; 6628 6629 if (owner == p) { 6630 /* 6631 * It's possible we interleave with mutex_unlock like: 6632 * 6633 * lock(&rq->lock); 6634 * find_proxy_task() 6635 * mutex_unlock() 6636 * lock(&wait_lock); 6637 * donor(owner) = current->blocked_donor; 6638 * unlock(&wait_lock); 6639 * 6640 * wake_up_q(); 6641 * ... 6642 * ttwu_runnable() 6643 * __task_rq_lock() 6644 * lock(&wait_lock); 6645 * owner == p 6646 * 6647 * Which leaves us to finish the ttwu_runnable() and make it go. 6648 * 6649 * So schedule rq->idle so that ttwu_runnable() can get the rq 6650 * lock and mark owner as running. 6651 */ 6652 return proxy_resched_idle(rq); 6653 } 6654 /* 6655 * OK, now we're absolutely sure @owner is on this 6656 * rq, therefore holding @rq->lock is sufficient to 6657 * guarantee its existence, as per ttwu_remote(). 6658 */ 6659 } 6660 6661 WARN_ON_ONCE(owner && !owner->on_rq); 6662 return owner; 6663} 6664#else /* SCHED_PROXY_EXEC */ 6665static struct task_struct * 6666find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) 6667{ 6668 WARN_ONCE(1, "This should never be called in the !SCHED_PROXY_EXEC case\n"); 6669 return donor; 6670} 6671#endif /* SCHED_PROXY_EXEC */ 6672 6673static inline void proxy_tag_curr(struct rq *rq, struct task_struct *owner) 6674{ 6675 if (!sched_proxy_exec()) 6676 return; 6677 /* 6678 * pick_next_task() calls set_next_task() on the chosen task 6679 * at some point, which ensures it is not push/pullable. 6680 * However, the chosen/donor task *and* the mutex owner form an 6681 * atomic pair wrt push/pull. 6682 * 6683 * Make sure owner we run is not pushable. Unfortunately we can 6684 * only deal with that by means of a dequeue/enqueue cycle. :-/ 6685 */ 6686 dequeue_task(rq, owner, DEQUEUE_NOCLOCK | DEQUEUE_SAVE); 6687 enqueue_task(rq, owner, ENQUEUE_NOCLOCK | ENQUEUE_RESTORE); 6688} 6689 6690/* 6691 * __schedule() is the main scheduler function. 6692 * 6693 * The main means of driving the scheduler and thus entering this function are: 6694 * 6695 * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. 6696 * 6697 * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return 6698 * paths. For example, see arch/x86/entry_64.S. 6699 * 6700 * To drive preemption between tasks, the scheduler sets the flag in timer 6701 * interrupt handler sched_tick(). 6702 * 6703 * 3. Wakeups don't really cause entry into schedule(). They add a 6704 * task to the run-queue and that's it. 6705 * 6706 * Now, if the new task added to the run-queue preempts the current 6707 * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets 6708 * called on the nearest possible occasion: 6709 * 6710 * - If the kernel is preemptible (CONFIG_PREEMPTION=y): 6711 * 6712 * - in syscall or exception context, at the next outmost 6713 * preempt_enable(). (this might be as soon as the wake_up()'s 6714 * spin_unlock()!) 6715 * 6716 * - in IRQ context, return from interrupt-handler to 6717 * preemptible context 6718 * 6719 * - If the kernel is not preemptible (CONFIG_PREEMPTION is not set) 6720 * then at the next: 6721 * 6722 * - cond_resched() call 6723 * - explicit schedule() call 6724 * - return from syscall or exception to user-space 6725 * - return from interrupt-handler to user-space 6726 * 6727 * WARNING: must be called with preemption disabled! 6728 */ 6729static void __sched notrace __schedule(int sched_mode) 6730{ 6731 struct task_struct *prev, *next; 6732 /* 6733 * On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted 6734 * as a preemption by schedule_debug() and RCU. 6735 */ 6736 bool preempt = sched_mode > SM_NONE; 6737 bool is_switch = false; 6738 unsigned long *switch_count; 6739 unsigned long prev_state; 6740 struct rq_flags rf; 6741 struct rq *rq; 6742 int cpu; 6743 6744 /* Trace preemptions consistently with task switches */ 6745 trace_sched_entry_tp(sched_mode == SM_PREEMPT); 6746 6747 cpu = smp_processor_id(); 6748 rq = cpu_rq(cpu); 6749 prev = rq->curr; 6750 6751 schedule_debug(prev, preempt); 6752 6753 if (sched_feat(HRTICK) || sched_feat(HRTICK_DL)) 6754 hrtick_clear(rq); 6755 6756 klp_sched_try_switch(prev); 6757 6758 local_irq_disable(); 6759 rcu_note_context_switch(preempt); 6760 migrate_disable_switch(rq, prev); 6761 6762 /* 6763 * Make sure that signal_pending_state()->signal_pending() below 6764 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) 6765 * done by the caller to avoid the race with signal_wake_up(): 6766 * 6767 * __set_current_state(@state) signal_wake_up() 6768 * schedule() set_tsk_thread_flag(p, TIF_SIGPENDING) 6769 * wake_up_state(p, state) 6770 * LOCK rq->lock LOCK p->pi_state 6771 * smp_mb__after_spinlock() smp_mb__after_spinlock() 6772 * if (signal_pending_state()) if (p->state & @state) 6773 * 6774 * Also, the membarrier system call requires a full memory barrier 6775 * after coming from user-space, before storing to rq->curr; this 6776 * barrier matches a full barrier in the proximity of the membarrier 6777 * system call exit. 6778 */ 6779 rq_lock(rq, &rf); 6780 smp_mb__after_spinlock(); 6781 6782 /* Promote REQ to ACT */ 6783 rq->clock_update_flags <<= 1; 6784 update_rq_clock(rq); 6785 rq->clock_update_flags = RQCF_UPDATED; 6786 6787 switch_count = &prev->nivcsw; 6788 6789 /* Task state changes only considers SM_PREEMPT as preemption */ 6790 preempt = sched_mode == SM_PREEMPT; 6791 6792 /* 6793 * We must load prev->state once (task_struct::state is volatile), such 6794 * that we form a control dependency vs deactivate_task() below. 6795 */ 6796 prev_state = READ_ONCE(prev->__state); 6797 if (sched_mode == SM_IDLE) { 6798 /* SCX must consult the BPF scheduler to tell if rq is empty */ 6799 if (!rq->nr_running && !scx_enabled()) { 6800 next = prev; 6801 goto picked; 6802 } 6803 } else if (!preempt && prev_state) { 6804 /* 6805 * We pass task_is_blocked() as the should_block arg 6806 * in order to keep mutex-blocked tasks on the runqueue 6807 * for slection with proxy-exec (without proxy-exec 6808 * task_is_blocked() will always be false). 6809 */ 6810 try_to_block_task(rq, prev, &prev_state, 6811 !task_is_blocked(prev)); 6812 switch_count = &prev->nvcsw; 6813 } 6814 6815pick_again: 6816 next = pick_next_task(rq, rq->donor, &rf); 6817 rq_set_donor(rq, next); 6818 if (unlikely(task_is_blocked(next))) { 6819 next = find_proxy_task(rq, next, &rf); 6820 if (!next) 6821 goto pick_again; 6822 if (next == rq->idle) 6823 goto keep_resched; 6824 } 6825picked: 6826 clear_tsk_need_resched(prev); 6827 clear_preempt_need_resched(); 6828keep_resched: 6829 rq->last_seen_need_resched_ns = 0; 6830 6831 is_switch = prev != next; 6832 if (likely(is_switch)) { 6833 rq->nr_switches++; 6834 /* 6835 * RCU users of rcu_dereference(rq->curr) may not see 6836 * changes to task_struct made by pick_next_task(). 6837 */ 6838 RCU_INIT_POINTER(rq->curr, next); 6839 6840 if (!task_current_donor(rq, next)) 6841 proxy_tag_curr(rq, next); 6842 6843 /* 6844 * The membarrier system call requires each architecture 6845 * to have a full memory barrier after updating 6846 * rq->curr, before returning to user-space. 6847 * 6848 * Here are the schemes providing that barrier on the 6849 * various architectures: 6850 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC, 6851 * RISC-V. switch_mm() relies on membarrier_arch_switch_mm() 6852 * on PowerPC and on RISC-V. 6853 * - finish_lock_switch() for weakly-ordered 6854 * architectures where spin_unlock is a full barrier, 6855 * - switch_to() for arm64 (weakly-ordered, spin_unlock 6856 * is a RELEASE barrier), 6857 * 6858 * The barrier matches a full barrier in the proximity of 6859 * the membarrier system call entry. 6860 * 6861 * On RISC-V, this barrier pairing is also needed for the 6862 * SYNC_CORE command when switching between processes, cf. 6863 * the inline comments in membarrier_arch_switch_mm(). 6864 */ 6865 ++*switch_count; 6866 6867 psi_account_irqtime(rq, prev, next); 6868 psi_sched_switch(prev, next, !task_on_rq_queued(prev) || 6869 prev->se.sched_delayed); 6870 6871 trace_sched_switch(preempt, prev, next, prev_state); 6872 6873 /* Also unlocks the rq: */ 6874 rq = context_switch(rq, prev, next, &rf); 6875 } else { 6876 /* In case next was already curr but just got blocked_donor */ 6877 if (!task_current_donor(rq, next)) 6878 proxy_tag_curr(rq, next); 6879 6880 rq_unpin_lock(rq, &rf); 6881 __balance_callbacks(rq, NULL); 6882 raw_spin_rq_unlock_irq(rq); 6883 } 6884 trace_sched_exit_tp(is_switch); 6885} 6886 6887void __noreturn do_task_dead(void) 6888{ 6889 /* Causes final put_task_struct in finish_task_switch(): */ 6890 set_special_state(TASK_DEAD); 6891 6892 /* Tell freezer to ignore us: */ 6893 current->flags |= PF_NOFREEZE; 6894 6895 __schedule(SM_NONE); 6896 BUG(); 6897 6898 /* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */ 6899 for (;;) 6900 cpu_relax(); 6901} 6902 6903static inline void sched_submit_work(struct task_struct *tsk) 6904{ 6905 static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG); 6906 unsigned int task_flags; 6907 6908 /* 6909 * Establish LD_WAIT_CONFIG context to ensure none of the code called 6910 * will use a blocking primitive -- which would lead to recursion. 6911 */ 6912 lock_map_acquire_try(&sched_map); 6913 6914 task_flags = tsk->flags; 6915 /* 6916 * If a worker goes to sleep, notify and ask workqueue whether it 6917 * wants to wake up a task to maintain concurrency. 6918 */ 6919 if (task_flags & PF_WQ_WORKER) 6920 wq_worker_sleeping(tsk); 6921 else if (task_flags & PF_IO_WORKER) 6922 io_wq_worker_sleeping(tsk); 6923 6924 /* 6925 * spinlock and rwlock must not flush block requests. This will 6926 * deadlock if the callback attempts to acquire a lock which is 6927 * already acquired. 6928 */ 6929 WARN_ON_ONCE(current->__state & TASK_RTLOCK_WAIT); 6930 6931 /* 6932 * If we are going to sleep and we have plugged IO queued, 6933 * make sure to submit it to avoid deadlocks. 6934 */ 6935 blk_flush_plug(tsk->plug, true); 6936 6937 lock_map_release(&sched_map); 6938} 6939 6940static void sched_update_worker(struct task_struct *tsk) 6941{ 6942 if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { 6943 if (tsk->flags & PF_BLOCK_TS) 6944 blk_plug_invalidate_ts(tsk); 6945 if (tsk->flags & PF_WQ_WORKER) 6946 wq_worker_running(tsk); 6947 else if (tsk->flags & PF_IO_WORKER) 6948 io_wq_worker_running(tsk); 6949 } 6950} 6951 6952static __always_inline void __schedule_loop(int sched_mode) 6953{ 6954 do { 6955 preempt_disable(); 6956 __schedule(sched_mode); 6957 sched_preempt_enable_no_resched(); 6958 } while (need_resched()); 6959} 6960 6961asmlinkage __visible void __sched schedule(void) 6962{ 6963 struct task_struct *tsk = current; 6964 6965#ifdef CONFIG_RT_MUTEXES 6966 lockdep_assert(!tsk->sched_rt_mutex); 6967#endif 6968 6969 if (!task_is_running(tsk)) 6970 sched_submit_work(tsk); 6971 __schedule_loop(SM_NONE); 6972 sched_update_worker(tsk); 6973} 6974EXPORT_SYMBOL(schedule); 6975 6976/* 6977 * synchronize_rcu_tasks() makes sure that no task is stuck in preempted 6978 * state (have scheduled out non-voluntarily) by making sure that all 6979 * tasks have either left the run queue or have gone into user space. 6980 * As idle tasks do not do either, they must not ever be preempted 6981 * (schedule out non-voluntarily). 6982 * 6983 * schedule_idle() is similar to schedule_preempt_disable() except that it 6984 * never enables preemption because it does not call sched_submit_work(). 6985 */ 6986void __sched schedule_idle(void) 6987{ 6988 /* 6989 * As this skips calling sched_submit_work(), which the idle task does 6990 * regardless because that function is a NOP when the task is in a 6991 * TASK_RUNNING state, make sure this isn't used someplace that the 6992 * current task can be in any other state. Note, idle is always in the 6993 * TASK_RUNNING state. 6994 */ 6995 WARN_ON_ONCE(current->__state); 6996 do { 6997 __schedule(SM_IDLE); 6998 } while (need_resched()); 6999} 7000 7001#if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK) 7002asmlinkage __visible void __sched schedule_user(void) 7003{ 7004 /* 7005 * If we come here after a random call to set_need_resched(), 7006 * or we have been woken up remotely but the IPI has not yet arrived, 7007 * we haven't yet exited the RCU idle mode. Do it here manually until 7008 * we find a better solution. 7009 * 7010 * NB: There are buggy callers of this function. Ideally we 7011 * should warn if prev_state != CT_STATE_USER, but that will trigger 7012 * too frequently to make sense yet. 7013 */ 7014 enum ctx_state prev_state = exception_enter(); 7015 schedule(); 7016 exception_exit(prev_state); 7017} 7018#endif 7019 7020/** 7021 * schedule_preempt_disabled - called with preemption disabled 7022 * 7023 * Returns with preemption disabled. Note: preempt_count must be 1 7024 */ 7025void __sched schedule_preempt_disabled(void) 7026{ 7027 sched_preempt_enable_no_resched(); 7028 schedule(); 7029 preempt_disable(); 7030} 7031 7032#ifdef CONFIG_PREEMPT_RT 7033void __sched notrace schedule_rtlock(void) 7034{ 7035 __schedule_loop(SM_RTLOCK_WAIT); 7036} 7037NOKPROBE_SYMBOL(schedule_rtlock); 7038#endif 7039 7040static void __sched notrace preempt_schedule_common(void) 7041{ 7042 do { 7043 /* 7044 * Because the function tracer can trace preempt_count_sub() 7045 * and it also uses preempt_enable/disable_notrace(), if 7046 * NEED_RESCHED is set, the preempt_enable_notrace() called 7047 * by the function tracer will call this function again and 7048 * cause infinite recursion. 7049 * 7050 * Preemption must be disabled here before the function 7051 * tracer can trace. Break up preempt_disable() into two 7052 * calls. One to disable preemption without fear of being 7053 * traced. The other to still record the preemption latency, 7054 * which can also be traced by the function tracer. 7055 */ 7056 preempt_disable_notrace(); 7057 preempt_latency_start(1); 7058 __schedule(SM_PREEMPT); 7059 preempt_latency_stop(1); 7060 preempt_enable_no_resched_notrace(); 7061 7062 /* 7063 * Check again in case we missed a preemption opportunity 7064 * between schedule and now. 7065 */ 7066 } while (need_resched()); 7067} 7068 7069#ifdef CONFIG_PREEMPTION 7070/* 7071 * This is the entry point to schedule() from in-kernel preemption 7072 * off of preempt_enable. 7073 */ 7074asmlinkage __visible void __sched notrace preempt_schedule(void) 7075{ 7076 /* 7077 * If there is a non-zero preempt_count or interrupts are disabled, 7078 * we do not want to preempt the current task. Just return.. 7079 */ 7080 if (likely(!preemptible())) 7081 return; 7082 preempt_schedule_common(); 7083} 7084NOKPROBE_SYMBOL(preempt_schedule); 7085EXPORT_SYMBOL(preempt_schedule); 7086 7087#ifdef CONFIG_PREEMPT_DYNAMIC 7088# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 7089# ifndef preempt_schedule_dynamic_enabled 7090# define preempt_schedule_dynamic_enabled preempt_schedule 7091# define preempt_schedule_dynamic_disabled NULL 7092# endif 7093DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled); 7094EXPORT_STATIC_CALL_TRAMP(preempt_schedule); 7095# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7096static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule); 7097void __sched notrace dynamic_preempt_schedule(void) 7098{ 7099 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule)) 7100 return; 7101 preempt_schedule(); 7102} 7103NOKPROBE_SYMBOL(dynamic_preempt_schedule); 7104EXPORT_SYMBOL(dynamic_preempt_schedule); 7105# endif 7106#endif /* CONFIG_PREEMPT_DYNAMIC */ 7107 7108/** 7109 * preempt_schedule_notrace - preempt_schedule called by tracing 7110 * 7111 * The tracing infrastructure uses preempt_enable_notrace to prevent 7112 * recursion and tracing preempt enabling caused by the tracing 7113 * infrastructure itself. But as tracing can happen in areas coming 7114 * from userspace or just about to enter userspace, a preempt enable 7115 * can occur before user_exit() is called. This will cause the scheduler 7116 * to be called when the system is still in usermode. 7117 * 7118 * To prevent this, the preempt_enable_notrace will use this function 7119 * instead of preempt_schedule() to exit user context if needed before 7120 * calling the scheduler. 7121 */ 7122asmlinkage __visible void __sched notrace preempt_schedule_notrace(void) 7123{ 7124 enum ctx_state prev_ctx; 7125 7126 if (likely(!preemptible())) 7127 return; 7128 7129 do { 7130 /* 7131 * Because the function tracer can trace preempt_count_sub() 7132 * and it also uses preempt_enable/disable_notrace(), if 7133 * NEED_RESCHED is set, the preempt_enable_notrace() called 7134 * by the function tracer will call this function again and 7135 * cause infinite recursion. 7136 * 7137 * Preemption must be disabled here before the function 7138 * tracer can trace. Break up preempt_disable() into two 7139 * calls. One to disable preemption without fear of being 7140 * traced. The other to still record the preemption latency, 7141 * which can also be traced by the function tracer. 7142 */ 7143 preempt_disable_notrace(); 7144 preempt_latency_start(1); 7145 /* 7146 * Needs preempt disabled in case user_exit() is traced 7147 * and the tracer calls preempt_enable_notrace() causing 7148 * an infinite recursion. 7149 */ 7150 prev_ctx = exception_enter(); 7151 __schedule(SM_PREEMPT); 7152 exception_exit(prev_ctx); 7153 7154 preempt_latency_stop(1); 7155 preempt_enable_no_resched_notrace(); 7156 } while (need_resched()); 7157} 7158EXPORT_SYMBOL_GPL(preempt_schedule_notrace); 7159 7160#ifdef CONFIG_PREEMPT_DYNAMIC 7161# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 7162# ifndef preempt_schedule_notrace_dynamic_enabled 7163# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace 7164# define preempt_schedule_notrace_dynamic_disabled NULL 7165# endif 7166DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled); 7167EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace); 7168# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7169static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace); 7170void __sched notrace dynamic_preempt_schedule_notrace(void) 7171{ 7172 if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace)) 7173 return; 7174 preempt_schedule_notrace(); 7175} 7176NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace); 7177EXPORT_SYMBOL(dynamic_preempt_schedule_notrace); 7178# endif 7179#endif 7180 7181#endif /* CONFIG_PREEMPTION */ 7182 7183/* 7184 * This is the entry point to schedule() from kernel preemption 7185 * off of IRQ context. 7186 * Note, that this is called and return with IRQs disabled. This will 7187 * protect us against recursive calling from IRQ contexts. 7188 */ 7189asmlinkage __visible void __sched preempt_schedule_irq(void) 7190{ 7191 enum ctx_state prev_state; 7192 7193 /* Catch callers which need to be fixed */ 7194 BUG_ON(preempt_count() || !irqs_disabled()); 7195 7196 prev_state = exception_enter(); 7197 7198 do { 7199 preempt_disable(); 7200 local_irq_enable(); 7201 __schedule(SM_PREEMPT); 7202 local_irq_disable(); 7203 sched_preempt_enable_no_resched(); 7204 } while (need_resched()); 7205 7206 exception_exit(prev_state); 7207} 7208 7209int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags, 7210 void *key) 7211{ 7212 WARN_ON_ONCE(wake_flags & ~(WF_SYNC|WF_CURRENT_CPU)); 7213 return try_to_wake_up(curr->private, mode, wake_flags); 7214} 7215EXPORT_SYMBOL(default_wake_function); 7216 7217const struct sched_class *__setscheduler_class(int policy, int prio) 7218{ 7219 if (dl_prio(prio)) 7220 return &dl_sched_class; 7221 7222 if (rt_prio(prio)) 7223 return &rt_sched_class; 7224 7225#ifdef CONFIG_SCHED_CLASS_EXT 7226 if (task_should_scx(policy)) 7227 return &ext_sched_class; 7228#endif 7229 7230 return &fair_sched_class; 7231} 7232 7233#ifdef CONFIG_RT_MUTEXES 7234 7235/* 7236 * Would be more useful with typeof()/auto_type but they don't mix with 7237 * bit-fields. Since it's a local thing, use int. Keep the generic sounding 7238 * name such that if someone were to implement this function we get to compare 7239 * notes. 7240 */ 7241#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; }) 7242 7243void rt_mutex_pre_schedule(void) 7244{ 7245 lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); 7246 sched_submit_work(current); 7247} 7248 7249void rt_mutex_schedule(void) 7250{ 7251 lockdep_assert(current->sched_rt_mutex); 7252 __schedule_loop(SM_NONE); 7253} 7254 7255void rt_mutex_post_schedule(void) 7256{ 7257 sched_update_worker(current); 7258 lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); 7259} 7260 7261/* 7262 * rt_mutex_setprio - set the current priority of a task 7263 * @p: task to boost 7264 * @pi_task: donor task 7265 * 7266 * This function changes the 'effective' priority of a task. It does 7267 * not touch ->normal_prio like __setscheduler(). 7268 * 7269 * Used by the rt_mutex code to implement priority inheritance 7270 * logic. Call site only calls if the priority of the task changed. 7271 */ 7272void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) 7273{ 7274 int prio, oldprio, queue_flag = 7275 DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; 7276 const struct sched_class *prev_class, *next_class; 7277 struct rq_flags rf; 7278 struct rq *rq; 7279 7280 /* XXX used to be waiter->prio, not waiter->task->prio */ 7281 prio = __rt_effective_prio(pi_task, p->normal_prio); 7282 7283 /* 7284 * If nothing changed; bail early. 7285 */ 7286 if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio)) 7287 return; 7288 7289 rq = __task_rq_lock(p, &rf); 7290 update_rq_clock(rq); 7291 /* 7292 * Set under pi_lock && rq->lock, such that the value can be used under 7293 * either lock. 7294 * 7295 * Note that there is loads of tricky to make this pointer cache work 7296 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to 7297 * ensure a task is de-boosted (pi_task is set to NULL) before the 7298 * task is allowed to run again (and can exit). This ensures the pointer 7299 * points to a blocked task -- which guarantees the task is present. 7300 */ 7301 p->pi_top_task = pi_task; 7302 7303 /* 7304 * For FIFO/RR we only need to set prio, if that matches we're done. 7305 */ 7306 if (prio == p->prio && !dl_prio(prio)) 7307 goto out_unlock; 7308 7309 /* 7310 * Idle task boosting is a no-no in general. There is one 7311 * exception, when PREEMPT_RT and NOHZ is active: 7312 * 7313 * The idle task calls get_next_timer_interrupt() and holds 7314 * the timer wheel base->lock on the CPU and another CPU wants 7315 * to access the timer (probably to cancel it). We can safely 7316 * ignore the boosting request, as the idle CPU runs this code 7317 * with interrupts disabled and will complete the lock 7318 * protected section without being interrupted. So there is no 7319 * real need to boost. 7320 */ 7321 if (unlikely(p == rq->idle)) { 7322 WARN_ON(p != rq->curr); 7323 WARN_ON(p->pi_blocked_on); 7324 goto out_unlock; 7325 } 7326 7327 trace_sched_pi_setprio(p, pi_task); 7328 oldprio = p->prio; 7329 7330 if (oldprio == prio && !dl_prio(prio)) 7331 queue_flag &= ~DEQUEUE_MOVE; 7332 7333 prev_class = p->sched_class; 7334 next_class = __setscheduler_class(p->policy, prio); 7335 7336 if (prev_class != next_class) 7337 queue_flag |= DEQUEUE_CLASS; 7338 7339 scoped_guard (sched_change, p, queue_flag) { 7340 /* 7341 * Boosting condition are: 7342 * 1. -rt task is running and holds mutex A 7343 * --> -dl task blocks on mutex A 7344 * 7345 * 2. -dl task is running and holds mutex A 7346 * --> -dl task blocks on mutex A and could preempt the 7347 * running task 7348 */ 7349 if (dl_prio(prio)) { 7350 if (!dl_prio(p->normal_prio) || 7351 (pi_task && dl_prio(pi_task->prio) && 7352 dl_entity_preempt(&pi_task->dl, &p->dl))) { 7353 p->dl.pi_se = pi_task->dl.pi_se; 7354 scope->flags |= ENQUEUE_REPLENISH; 7355 } else { 7356 p->dl.pi_se = &p->dl; 7357 } 7358 } else if (rt_prio(prio)) { 7359 if (dl_prio(oldprio)) 7360 p->dl.pi_se = &p->dl; 7361 if (oldprio < prio) 7362 scope->flags |= ENQUEUE_HEAD; 7363 } else { 7364 if (dl_prio(oldprio)) 7365 p->dl.pi_se = &p->dl; 7366 if (rt_prio(oldprio)) 7367 p->rt.timeout = 0; 7368 } 7369 7370 p->sched_class = next_class; 7371 p->prio = prio; 7372 } 7373out_unlock: 7374 /* Caller holds task_struct::pi_lock, IRQs are still disabled */ 7375 7376 __balance_callbacks(rq, &rf); 7377 __task_rq_unlock(rq, p, &rf); 7378} 7379#endif /* CONFIG_RT_MUTEXES */ 7380 7381#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 7382int __sched __cond_resched(void) 7383{ 7384 if (should_resched(0) && !irqs_disabled()) { 7385 preempt_schedule_common(); 7386 return 1; 7387 } 7388 /* 7389 * In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick 7390 * whether the current CPU is in an RCU read-side critical section, 7391 * so the tick can report quiescent states even for CPUs looping 7392 * in kernel context. In contrast, in non-preemptible kernels, 7393 * RCU readers leave no in-memory hints, which means that CPU-bound 7394 * processes executing in kernel context might never report an 7395 * RCU quiescent state. Therefore, the following code causes 7396 * cond_resched() to report a quiescent state, but only when RCU 7397 * is in urgent need of one. 7398 * A third case, preemptible, but non-PREEMPT_RCU provides for 7399 * urgently needed quiescent states via rcu_flavor_sched_clock_irq(). 7400 */ 7401#ifndef CONFIG_PREEMPT_RCU 7402 rcu_all_qs(); 7403#endif 7404 return 0; 7405} 7406EXPORT_SYMBOL(__cond_resched); 7407#endif 7408 7409#ifdef CONFIG_PREEMPT_DYNAMIC 7410# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL 7411# define cond_resched_dynamic_enabled __cond_resched 7412# define cond_resched_dynamic_disabled ((void *)&__static_call_return0) 7413DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched); 7414EXPORT_STATIC_CALL_TRAMP(cond_resched); 7415 7416# define might_resched_dynamic_enabled __cond_resched 7417# define might_resched_dynamic_disabled ((void *)&__static_call_return0) 7418DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched); 7419EXPORT_STATIC_CALL_TRAMP(might_resched); 7420# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7421static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched); 7422int __sched dynamic_cond_resched(void) 7423{ 7424 if (!static_branch_unlikely(&sk_dynamic_cond_resched)) 7425 return 0; 7426 return __cond_resched(); 7427} 7428EXPORT_SYMBOL(dynamic_cond_resched); 7429 7430static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched); 7431int __sched dynamic_might_resched(void) 7432{ 7433 if (!static_branch_unlikely(&sk_dynamic_might_resched)) 7434 return 0; 7435 return __cond_resched(); 7436} 7437EXPORT_SYMBOL(dynamic_might_resched); 7438# endif 7439#endif /* CONFIG_PREEMPT_DYNAMIC */ 7440 7441/* 7442 * __cond_resched_lock() - if a reschedule is pending, drop the given lock, 7443 * call schedule, and on return reacquire the lock. 7444 * 7445 * This works OK both with and without CONFIG_PREEMPTION. We do strange low-level 7446 * operations here to prevent schedule() from being called twice (once via 7447 * spin_unlock(), once by hand). 7448 */ 7449int __cond_resched_lock(spinlock_t *lock) 7450{ 7451 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7452 int ret = 0; 7453 7454 lockdep_assert_held(lock); 7455 7456 if (spin_needbreak(lock) || resched) { 7457 spin_unlock(lock); 7458 if (!_cond_resched()) 7459 cpu_relax(); 7460 ret = 1; 7461 spin_lock(lock); 7462 } 7463 return ret; 7464} 7465EXPORT_SYMBOL(__cond_resched_lock); 7466 7467int __cond_resched_rwlock_read(rwlock_t *lock) 7468{ 7469 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7470 int ret = 0; 7471 7472 lockdep_assert_held_read(lock); 7473 7474 if (rwlock_needbreak(lock) || resched) { 7475 read_unlock(lock); 7476 if (!_cond_resched()) 7477 cpu_relax(); 7478 ret = 1; 7479 read_lock(lock); 7480 } 7481 return ret; 7482} 7483EXPORT_SYMBOL(__cond_resched_rwlock_read); 7484 7485int __cond_resched_rwlock_write(rwlock_t *lock) 7486{ 7487 int resched = should_resched(PREEMPT_LOCK_OFFSET); 7488 int ret = 0; 7489 7490 lockdep_assert_held_write(lock); 7491 7492 if (rwlock_needbreak(lock) || resched) { 7493 write_unlock(lock); 7494 if (!_cond_resched()) 7495 cpu_relax(); 7496 ret = 1; 7497 write_lock(lock); 7498 } 7499 return ret; 7500} 7501EXPORT_SYMBOL(__cond_resched_rwlock_write); 7502 7503#ifdef CONFIG_PREEMPT_DYNAMIC 7504 7505# ifdef CONFIG_GENERIC_IRQ_ENTRY 7506# include <linux/irq-entry-common.h> 7507# endif 7508 7509/* 7510 * SC:cond_resched 7511 * SC:might_resched 7512 * SC:preempt_schedule 7513 * SC:preempt_schedule_notrace 7514 * SC:irqentry_exit_cond_resched 7515 * 7516 * 7517 * NONE: 7518 * cond_resched <- __cond_resched 7519 * might_resched <- RET0 7520 * preempt_schedule <- NOP 7521 * preempt_schedule_notrace <- NOP 7522 * irqentry_exit_cond_resched <- NOP 7523 * dynamic_preempt_lazy <- false 7524 * 7525 * VOLUNTARY: 7526 * cond_resched <- __cond_resched 7527 * might_resched <- __cond_resched 7528 * preempt_schedule <- NOP 7529 * preempt_schedule_notrace <- NOP 7530 * irqentry_exit_cond_resched <- NOP 7531 * dynamic_preempt_lazy <- false 7532 * 7533 * FULL: 7534 * cond_resched <- RET0 7535 * might_resched <- RET0 7536 * preempt_schedule <- preempt_schedule 7537 * preempt_schedule_notrace <- preempt_schedule_notrace 7538 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 7539 * dynamic_preempt_lazy <- false 7540 * 7541 * LAZY: 7542 * cond_resched <- RET0 7543 * might_resched <- RET0 7544 * preempt_schedule <- preempt_schedule 7545 * preempt_schedule_notrace <- preempt_schedule_notrace 7546 * irqentry_exit_cond_resched <- irqentry_exit_cond_resched 7547 * dynamic_preempt_lazy <- true 7548 */ 7549 7550enum { 7551 preempt_dynamic_undefined = -1, 7552 preempt_dynamic_none, 7553 preempt_dynamic_voluntary, 7554 preempt_dynamic_full, 7555 preempt_dynamic_lazy, 7556}; 7557 7558int preempt_dynamic_mode = preempt_dynamic_undefined; 7559 7560int sched_dynamic_mode(const char *str) 7561{ 7562# ifndef CONFIG_PREEMPT_RT 7563 if (!strcmp(str, "none")) 7564 return preempt_dynamic_none; 7565 7566 if (!strcmp(str, "voluntary")) 7567 return preempt_dynamic_voluntary; 7568# endif 7569 7570 if (!strcmp(str, "full")) 7571 return preempt_dynamic_full; 7572 7573# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY 7574 if (!strcmp(str, "lazy")) 7575 return preempt_dynamic_lazy; 7576# endif 7577 7578 return -EINVAL; 7579} 7580 7581# define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key) 7582# define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key) 7583 7584# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 7585# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled) 7586# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled) 7587# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 7588# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f) 7589# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f) 7590# else 7591# error "Unsupported PREEMPT_DYNAMIC mechanism" 7592# endif 7593 7594static DEFINE_MUTEX(sched_dynamic_mutex); 7595 7596static void __sched_dynamic_update(int mode) 7597{ 7598 /* 7599 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in 7600 * the ZERO state, which is invalid. 7601 */ 7602 preempt_dynamic_enable(cond_resched); 7603 preempt_dynamic_enable(might_resched); 7604 preempt_dynamic_enable(preempt_schedule); 7605 preempt_dynamic_enable(preempt_schedule_notrace); 7606 preempt_dynamic_enable(irqentry_exit_cond_resched); 7607 preempt_dynamic_key_disable(preempt_lazy); 7608 7609 switch (mode) { 7610 case preempt_dynamic_none: 7611 preempt_dynamic_enable(cond_resched); 7612 preempt_dynamic_disable(might_resched); 7613 preempt_dynamic_disable(preempt_schedule); 7614 preempt_dynamic_disable(preempt_schedule_notrace); 7615 preempt_dynamic_disable(irqentry_exit_cond_resched); 7616 preempt_dynamic_key_disable(preempt_lazy); 7617 if (mode != preempt_dynamic_mode) 7618 pr_info("Dynamic Preempt: none\n"); 7619 break; 7620 7621 case preempt_dynamic_voluntary: 7622 preempt_dynamic_enable(cond_resched); 7623 preempt_dynamic_enable(might_resched); 7624 preempt_dynamic_disable(preempt_schedule); 7625 preempt_dynamic_disable(preempt_schedule_notrace); 7626 preempt_dynamic_disable(irqentry_exit_cond_resched); 7627 preempt_dynamic_key_disable(preempt_lazy); 7628 if (mode != preempt_dynamic_mode) 7629 pr_info("Dynamic Preempt: voluntary\n"); 7630 break; 7631 7632 case preempt_dynamic_full: 7633 preempt_dynamic_disable(cond_resched); 7634 preempt_dynamic_disable(might_resched); 7635 preempt_dynamic_enable(preempt_schedule); 7636 preempt_dynamic_enable(preempt_schedule_notrace); 7637 preempt_dynamic_enable(irqentry_exit_cond_resched); 7638 preempt_dynamic_key_disable(preempt_lazy); 7639 if (mode != preempt_dynamic_mode) 7640 pr_info("Dynamic Preempt: full\n"); 7641 break; 7642 7643 case preempt_dynamic_lazy: 7644 preempt_dynamic_disable(cond_resched); 7645 preempt_dynamic_disable(might_resched); 7646 preempt_dynamic_enable(preempt_schedule); 7647 preempt_dynamic_enable(preempt_schedule_notrace); 7648 preempt_dynamic_enable(irqentry_exit_cond_resched); 7649 preempt_dynamic_key_enable(preempt_lazy); 7650 if (mode != preempt_dynamic_mode) 7651 pr_info("Dynamic Preempt: lazy\n"); 7652 break; 7653 } 7654 7655 preempt_dynamic_mode = mode; 7656} 7657 7658void sched_dynamic_update(int mode) 7659{ 7660 mutex_lock(&sched_dynamic_mutex); 7661 __sched_dynamic_update(mode); 7662 mutex_unlock(&sched_dynamic_mutex); 7663} 7664 7665static int __init setup_preempt_mode(char *str) 7666{ 7667 int mode = sched_dynamic_mode(str); 7668 if (mode < 0) { 7669 pr_warn("Dynamic Preempt: unsupported mode: %s\n", str); 7670 return 0; 7671 } 7672 7673 sched_dynamic_update(mode); 7674 return 1; 7675} 7676__setup("preempt=", setup_preempt_mode); 7677 7678static void __init preempt_dynamic_init(void) 7679{ 7680 if (preempt_dynamic_mode == preempt_dynamic_undefined) { 7681 if (IS_ENABLED(CONFIG_PREEMPT_NONE)) { 7682 sched_dynamic_update(preempt_dynamic_none); 7683 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) { 7684 sched_dynamic_update(preempt_dynamic_voluntary); 7685 } else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { 7686 sched_dynamic_update(preempt_dynamic_lazy); 7687 } else { 7688 /* Default static call setting, nothing to do */ 7689 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT)); 7690 preempt_dynamic_mode = preempt_dynamic_full; 7691 pr_info("Dynamic Preempt: full\n"); 7692 } 7693 } 7694} 7695 7696# define PREEMPT_MODEL_ACCESSOR(mode) \ 7697 bool preempt_model_##mode(void) \ 7698 { \ 7699 WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \ 7700 return preempt_dynamic_mode == preempt_dynamic_##mode; \ 7701 } \ 7702 EXPORT_SYMBOL_GPL(preempt_model_##mode) 7703 7704PREEMPT_MODEL_ACCESSOR(none); 7705PREEMPT_MODEL_ACCESSOR(voluntary); 7706PREEMPT_MODEL_ACCESSOR(full); 7707PREEMPT_MODEL_ACCESSOR(lazy); 7708 7709#else /* !CONFIG_PREEMPT_DYNAMIC: */ 7710 7711#define preempt_dynamic_mode -1 7712 7713static inline void preempt_dynamic_init(void) { } 7714 7715#endif /* CONFIG_PREEMPT_DYNAMIC */ 7716 7717const char *preempt_modes[] = { 7718 "none", "voluntary", "full", "lazy", NULL, 7719}; 7720 7721const char *preempt_model_str(void) 7722{ 7723 bool brace = IS_ENABLED(CONFIG_PREEMPT_RT) && 7724 (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC) || 7725 IS_ENABLED(CONFIG_PREEMPT_LAZY)); 7726 static char buf[128]; 7727 7728 if (IS_ENABLED(CONFIG_PREEMPT_BUILD)) { 7729 struct seq_buf s; 7730 7731 seq_buf_init(&s, buf, sizeof(buf)); 7732 seq_buf_puts(&s, "PREEMPT"); 7733 7734 if (IS_ENABLED(CONFIG_PREEMPT_RT)) 7735 seq_buf_printf(&s, "%sRT%s", 7736 brace ? "_{" : "_", 7737 brace ? "," : ""); 7738 7739 if (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC)) { 7740 seq_buf_printf(&s, "(%s)%s", 7741 preempt_dynamic_mode >= 0 ? 7742 preempt_modes[preempt_dynamic_mode] : "undef", 7743 brace ? "}" : ""); 7744 return seq_buf_str(&s); 7745 } 7746 7747 if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) { 7748 seq_buf_printf(&s, "LAZY%s", 7749 brace ? "}" : ""); 7750 return seq_buf_str(&s); 7751 } 7752 7753 return seq_buf_str(&s); 7754 } 7755 7756 if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BUILD)) 7757 return "VOLUNTARY"; 7758 7759 return "NONE"; 7760} 7761 7762int io_schedule_prepare(void) 7763{ 7764 int old_iowait = current->in_iowait; 7765 7766 current->in_iowait = 1; 7767 blk_flush_plug(current->plug, true); 7768 return old_iowait; 7769} 7770 7771void io_schedule_finish(int token) 7772{ 7773 current->in_iowait = token; 7774} 7775 7776/* 7777 * This task is about to go to sleep on IO. Increment rq->nr_iowait so 7778 * that process accounting knows that this is a task in IO wait state. 7779 */ 7780long __sched io_schedule_timeout(long timeout) 7781{ 7782 int token; 7783 long ret; 7784 7785 token = io_schedule_prepare(); 7786 ret = schedule_timeout(timeout); 7787 io_schedule_finish(token); 7788 7789 return ret; 7790} 7791EXPORT_SYMBOL(io_schedule_timeout); 7792 7793void __sched io_schedule(void) 7794{ 7795 int token; 7796 7797 token = io_schedule_prepare(); 7798 schedule(); 7799 io_schedule_finish(token); 7800} 7801EXPORT_SYMBOL(io_schedule); 7802 7803void sched_show_task(struct task_struct *p) 7804{ 7805 unsigned long free; 7806 int ppid; 7807 7808 if (!try_get_task_stack(p)) 7809 return; 7810 7811 pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p)); 7812 7813 if (task_is_running(p)) 7814 pr_cont(" running task "); 7815 free = stack_not_used(p); 7816 ppid = 0; 7817 rcu_read_lock(); 7818 if (pid_alive(p)) 7819 ppid = task_pid_nr(rcu_dereference(p->real_parent)); 7820 rcu_read_unlock(); 7821 pr_cont(" stack:%-5lu pid:%-5d tgid:%-5d ppid:%-6d task_flags:0x%04x flags:0x%08lx\n", 7822 free, task_pid_nr(p), task_tgid_nr(p), 7823 ppid, p->flags, read_task_thread_flags(p)); 7824 7825 print_worker_info(KERN_INFO, p); 7826 print_stop_info(KERN_INFO, p); 7827 print_scx_info(KERN_INFO, p); 7828 show_stack(p, NULL, KERN_INFO); 7829 put_task_stack(p); 7830} 7831EXPORT_SYMBOL_GPL(sched_show_task); 7832 7833static inline bool 7834state_filter_match(unsigned long state_filter, struct task_struct *p) 7835{ 7836 unsigned int state = READ_ONCE(p->__state); 7837 7838 /* no filter, everything matches */ 7839 if (!state_filter) 7840 return true; 7841 7842 /* filter, but doesn't match */ 7843 if (!(state & state_filter)) 7844 return false; 7845 7846 /* 7847 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows 7848 * TASK_KILLABLE). 7849 */ 7850 if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD)) 7851 return false; 7852 7853 return true; 7854} 7855 7856 7857void show_state_filter(unsigned int state_filter) 7858{ 7859 struct task_struct *g, *p; 7860 7861 rcu_read_lock(); 7862 for_each_process_thread(g, p) { 7863 /* 7864 * reset the NMI-timeout, listing all files on a slow 7865 * console might take a lot of time: 7866 * Also, reset softlockup watchdogs on all CPUs, because 7867 * another CPU might be blocked waiting for us to process 7868 * an IPI. 7869 */ 7870 touch_nmi_watchdog(); 7871 touch_all_softlockup_watchdogs(); 7872 if (state_filter_match(state_filter, p)) 7873 sched_show_task(p); 7874 } 7875 7876 if (!state_filter) 7877 sysrq_sched_debug_show(); 7878 7879 rcu_read_unlock(); 7880 /* 7881 * Only show locks if all tasks are dumped: 7882 */ 7883 if (!state_filter) 7884 debug_show_all_locks(); 7885} 7886 7887/** 7888 * init_idle - set up an idle thread for a given CPU 7889 * @idle: task in question 7890 * @cpu: CPU the idle task belongs to 7891 * 7892 * NOTE: this function does not set the idle thread's NEED_RESCHED 7893 * flag, to make booting more robust. 7894 */ 7895void __init init_idle(struct task_struct *idle, int cpu) 7896{ 7897 struct affinity_context ac = (struct affinity_context) { 7898 .new_mask = cpumask_of(cpu), 7899 .flags = 0, 7900 }; 7901 struct rq *rq = cpu_rq(cpu); 7902 unsigned long flags; 7903 7904 raw_spin_lock_irqsave(&idle->pi_lock, flags); 7905 raw_spin_rq_lock(rq); 7906 7907 idle->__state = TASK_RUNNING; 7908 idle->se.exec_start = sched_clock(); 7909 /* 7910 * PF_KTHREAD should already be set at this point; regardless, make it 7911 * look like a proper per-CPU kthread. 7912 */ 7913 idle->flags |= PF_KTHREAD | PF_NO_SETAFFINITY; 7914 kthread_set_per_cpu(idle, cpu); 7915 7916 /* 7917 * No validation and serialization required at boot time and for 7918 * setting up the idle tasks of not yet online CPUs. 7919 */ 7920 set_cpus_allowed_common(idle, &ac); 7921 /* 7922 * We're having a chicken and egg problem, even though we are 7923 * holding rq->lock, the CPU isn't yet set to this CPU so the 7924 * lockdep check in task_group() will fail. 7925 * 7926 * Similar case to sched_fork(). / Alternatively we could 7927 * use task_rq_lock() here and obtain the other rq->lock. 7928 * 7929 * Silence PROVE_RCU 7930 */ 7931 rcu_read_lock(); 7932 __set_task_cpu(idle, cpu); 7933 rcu_read_unlock(); 7934 7935 rq->idle = idle; 7936 rq_set_donor(rq, idle); 7937 rcu_assign_pointer(rq->curr, idle); 7938 idle->on_rq = TASK_ON_RQ_QUEUED; 7939 idle->on_cpu = 1; 7940 raw_spin_rq_unlock(rq); 7941 raw_spin_unlock_irqrestore(&idle->pi_lock, flags); 7942 7943 /* Set the preempt count _outside_ the spinlocks! */ 7944 init_idle_preempt_count(idle, cpu); 7945 7946 /* 7947 * The idle tasks have their own, simple scheduling class: 7948 */ 7949 idle->sched_class = &idle_sched_class; 7950 ftrace_graph_init_idle_task(idle, cpu); 7951 vtime_init_idle(idle, cpu); 7952 sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); 7953} 7954 7955int cpuset_cpumask_can_shrink(const struct cpumask *cur, 7956 const struct cpumask *trial) 7957{ 7958 int ret = 1; 7959 7960 if (cpumask_empty(cur)) 7961 return ret; 7962 7963 ret = dl_cpuset_cpumask_can_shrink(cur, trial); 7964 7965 return ret; 7966} 7967 7968int task_can_attach(struct task_struct *p) 7969{ 7970 int ret = 0; 7971 7972 /* 7973 * Kthreads which disallow setaffinity shouldn't be moved 7974 * to a new cpuset; we don't want to change their CPU 7975 * affinity and isolating such threads by their set of 7976 * allowed nodes is unnecessary. Thus, cpusets are not 7977 * applicable for such threads. This prevents checking for 7978 * success of set_cpus_allowed_ptr() on all attached tasks 7979 * before cpus_mask may be changed. 7980 */ 7981 if (p->flags & PF_NO_SETAFFINITY) 7982 ret = -EINVAL; 7983 7984 return ret; 7985} 7986 7987bool sched_smp_initialized __read_mostly; 7988 7989#ifdef CONFIG_NUMA_BALANCING 7990/* Migrate current task p to target_cpu */ 7991int migrate_task_to(struct task_struct *p, int target_cpu) 7992{ 7993 struct migration_arg arg = { p, target_cpu }; 7994 int curr_cpu = task_cpu(p); 7995 7996 if (curr_cpu == target_cpu) 7997 return 0; 7998 7999 if (!cpumask_test_cpu(target_cpu, p->cpus_ptr)) 8000 return -EINVAL; 8001 8002 /* TODO: This is not properly updating schedstats */ 8003 8004 trace_sched_move_numa(p, curr_cpu, target_cpu); 8005 return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); 8006} 8007 8008/* 8009 * Requeue a task on a given node and accurately track the number of NUMA 8010 * tasks on the runqueues 8011 */ 8012void sched_setnuma(struct task_struct *p, int nid) 8013{ 8014 guard(task_rq_lock)(p); 8015 scoped_guard (sched_change, p, DEQUEUE_SAVE) 8016 p->numa_preferred_nid = nid; 8017} 8018#endif /* CONFIG_NUMA_BALANCING */ 8019 8020#ifdef CONFIG_HOTPLUG_CPU 8021/* 8022 * Invoked on the outgoing CPU in context of the CPU hotplug thread 8023 * after ensuring that there are no user space tasks left on the CPU. 8024 * 8025 * If there is a lazy mm in use on the hotplug thread, drop it and 8026 * switch to init_mm. 8027 * 8028 * The reference count on init_mm is dropped in finish_cpu(). 8029 */ 8030static void sched_force_init_mm(void) 8031{ 8032 struct mm_struct *mm = current->active_mm; 8033 8034 if (mm != &init_mm) { 8035 mmgrab_lazy_tlb(&init_mm); 8036 local_irq_disable(); 8037 current->active_mm = &init_mm; 8038 switch_mm_irqs_off(mm, &init_mm, current); 8039 local_irq_enable(); 8040 finish_arch_post_lock_switch(); 8041 mmdrop_lazy_tlb(mm); 8042 } 8043 8044 /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ 8045} 8046 8047static int __balance_push_cpu_stop(void *arg) 8048{ 8049 struct task_struct *p = arg; 8050 struct rq *rq = this_rq(); 8051 struct rq_flags rf; 8052 int cpu; 8053 8054 scoped_guard (raw_spinlock_irq, &p->pi_lock) { 8055 cpu = select_fallback_rq(rq->cpu, p); 8056 8057 rq_lock(rq, &rf); 8058 update_rq_clock(rq); 8059 if (task_rq(p) == rq && task_on_rq_queued(p)) 8060 rq = __migrate_task(rq, &rf, p, cpu); 8061 rq_unlock(rq, &rf); 8062 } 8063 8064 put_task_struct(p); 8065 8066 return 0; 8067} 8068 8069static DEFINE_PER_CPU(struct cpu_stop_work, push_work); 8070 8071/* 8072 * Ensure we only run per-cpu kthreads once the CPU goes !active. 8073 * 8074 * This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only 8075 * effective when the hotplug motion is down. 8076 */ 8077static void balance_push(struct rq *rq) 8078{ 8079 struct task_struct *push_task = rq->curr; 8080 8081 lockdep_assert_rq_held(rq); 8082 8083 /* 8084 * Ensure the thing is persistent until balance_push_set(.on = false); 8085 */ 8086 rq->balance_callback = &balance_push_callback; 8087 8088 /* 8089 * Only active while going offline and when invoked on the outgoing 8090 * CPU. 8091 */ 8092 if (!cpu_dying(rq->cpu) || rq != this_rq()) 8093 return; 8094 8095 /* 8096 * Both the cpu-hotplug and stop task are in this case and are 8097 * required to complete the hotplug process. 8098 */ 8099 if (kthread_is_per_cpu(push_task) || 8100 is_migration_disabled(push_task)) { 8101 8102 /* 8103 * If this is the idle task on the outgoing CPU try to wake 8104 * up the hotplug control thread which might wait for the 8105 * last task to vanish. The rcuwait_active() check is 8106 * accurate here because the waiter is pinned on this CPU 8107 * and can't obviously be running in parallel. 8108 * 8109 * On RT kernels this also has to check whether there are 8110 * pinned and scheduled out tasks on the runqueue. They 8111 * need to leave the migrate disabled section first. 8112 */ 8113 if (!rq->nr_running && !rq_has_pinned_tasks(rq) && 8114 rcuwait_active(&rq->hotplug_wait)) { 8115 raw_spin_rq_unlock(rq); 8116 rcuwait_wake_up(&rq->hotplug_wait); 8117 raw_spin_rq_lock(rq); 8118 } 8119 return; 8120 } 8121 8122 get_task_struct(push_task); 8123 /* 8124 * Temporarily drop rq->lock such that we can wake-up the stop task. 8125 * Both preemption and IRQs are still disabled. 8126 */ 8127 preempt_disable(); 8128 raw_spin_rq_unlock(rq); 8129 stop_one_cpu_nowait(rq->cpu, __balance_push_cpu_stop, push_task, 8130 this_cpu_ptr(&push_work)); 8131 preempt_enable(); 8132 /* 8133 * At this point need_resched() is true and we'll take the loop in 8134 * schedule(). The next pick is obviously going to be the stop task 8135 * which kthread_is_per_cpu() and will push this task away. 8136 */ 8137 raw_spin_rq_lock(rq); 8138} 8139 8140static void balance_push_set(int cpu, bool on) 8141{ 8142 struct rq *rq = cpu_rq(cpu); 8143 struct rq_flags rf; 8144 8145 rq_lock_irqsave(rq, &rf); 8146 if (on) { 8147 WARN_ON_ONCE(rq->balance_callback); 8148 rq->balance_callback = &balance_push_callback; 8149 } else if (rq->balance_callback == &balance_push_callback) { 8150 rq->balance_callback = NULL; 8151 } 8152 rq_unlock_irqrestore(rq, &rf); 8153} 8154 8155/* 8156 * Invoked from a CPUs hotplug control thread after the CPU has been marked 8157 * inactive. All tasks which are not per CPU kernel threads are either 8158 * pushed off this CPU now via balance_push() or placed on a different CPU 8159 * during wakeup. Wait until the CPU is quiescent. 8160 */ 8161static void balance_hotplug_wait(void) 8162{ 8163 struct rq *rq = this_rq(); 8164 8165 rcuwait_wait_event(&rq->hotplug_wait, 8166 rq->nr_running == 1 && !rq_has_pinned_tasks(rq), 8167 TASK_UNINTERRUPTIBLE); 8168} 8169 8170#else /* !CONFIG_HOTPLUG_CPU: */ 8171 8172static inline void balance_push(struct rq *rq) 8173{ 8174} 8175 8176static inline void balance_push_set(int cpu, bool on) 8177{ 8178} 8179 8180static inline void balance_hotplug_wait(void) 8181{ 8182} 8183 8184#endif /* !CONFIG_HOTPLUG_CPU */ 8185 8186void set_rq_online(struct rq *rq) 8187{ 8188 if (!rq->online) { 8189 const struct sched_class *class; 8190 8191 cpumask_set_cpu(rq->cpu, rq->rd->online); 8192 rq->online = 1; 8193 8194 for_each_class(class) { 8195 if (class->rq_online) 8196 class->rq_online(rq); 8197 } 8198 } 8199} 8200 8201void set_rq_offline(struct rq *rq) 8202{ 8203 if (rq->online) { 8204 const struct sched_class *class; 8205 8206 update_rq_clock(rq); 8207 for_each_class(class) { 8208 if (class->rq_offline) 8209 class->rq_offline(rq); 8210 } 8211 8212 cpumask_clear_cpu(rq->cpu, rq->rd->online); 8213 rq->online = 0; 8214 } 8215} 8216 8217static inline void sched_set_rq_online(struct rq *rq, int cpu) 8218{ 8219 struct rq_flags rf; 8220 8221 rq_lock_irqsave(rq, &rf); 8222 if (rq->rd) { 8223 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 8224 set_rq_online(rq); 8225 } 8226 rq_unlock_irqrestore(rq, &rf); 8227} 8228 8229static inline void sched_set_rq_offline(struct rq *rq, int cpu) 8230{ 8231 struct rq_flags rf; 8232 8233 rq_lock_irqsave(rq, &rf); 8234 if (rq->rd) { 8235 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); 8236 set_rq_offline(rq); 8237 } 8238 rq_unlock_irqrestore(rq, &rf); 8239} 8240 8241/* 8242 * used to mark begin/end of suspend/resume: 8243 */ 8244static int num_cpus_frozen; 8245 8246/* 8247 * Update cpusets according to cpu_active mask. If cpusets are 8248 * disabled, cpuset_update_active_cpus() becomes a simple wrapper 8249 * around partition_sched_domains(). 8250 * 8251 * If we come here as part of a suspend/resume, don't touch cpusets because we 8252 * want to restore it back to its original state upon resume anyway. 8253 */ 8254static void cpuset_cpu_active(void) 8255{ 8256 if (cpuhp_tasks_frozen) { 8257 /* 8258 * num_cpus_frozen tracks how many CPUs are involved in suspend 8259 * resume sequence. As long as this is not the last online 8260 * operation in the resume sequence, just build a single sched 8261 * domain, ignoring cpusets. 8262 */ 8263 cpuset_reset_sched_domains(); 8264 if (--num_cpus_frozen) 8265 return; 8266 /* 8267 * This is the last CPU online operation. So fall through and 8268 * restore the original sched domains by considering the 8269 * cpuset configurations. 8270 */ 8271 cpuset_force_rebuild(); 8272 } 8273 cpuset_update_active_cpus(); 8274} 8275 8276static void cpuset_cpu_inactive(unsigned int cpu) 8277{ 8278 if (!cpuhp_tasks_frozen) { 8279 cpuset_update_active_cpus(); 8280 } else { 8281 num_cpus_frozen++; 8282 cpuset_reset_sched_domains(); 8283 } 8284} 8285 8286static inline void sched_smt_present_inc(int cpu) 8287{ 8288#ifdef CONFIG_SCHED_SMT 8289 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8290 static_branch_inc_cpuslocked(&sched_smt_present); 8291#endif 8292} 8293 8294static inline void sched_smt_present_dec(int cpu) 8295{ 8296#ifdef CONFIG_SCHED_SMT 8297 if (cpumask_weight(cpu_smt_mask(cpu)) == 2) 8298 static_branch_dec_cpuslocked(&sched_smt_present); 8299#endif 8300} 8301 8302int sched_cpu_activate(unsigned int cpu) 8303{ 8304 struct rq *rq = cpu_rq(cpu); 8305 8306 /* 8307 * Clear the balance_push callback and prepare to schedule 8308 * regular tasks. 8309 */ 8310 balance_push_set(cpu, false); 8311 8312 /* 8313 * When going up, increment the number of cores with SMT present. 8314 */ 8315 sched_smt_present_inc(cpu); 8316 set_cpu_active(cpu, true); 8317 8318 if (sched_smp_initialized) { 8319 sched_update_numa(cpu, true); 8320 sched_domains_numa_masks_set(cpu); 8321 cpuset_cpu_active(); 8322 } 8323 8324 scx_rq_activate(rq); 8325 8326 /* 8327 * Put the rq online, if not already. This happens: 8328 * 8329 * 1) In the early boot process, because we build the real domains 8330 * after all CPUs have been brought up. 8331 * 8332 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the 8333 * domains. 8334 */ 8335 sched_set_rq_online(rq, cpu); 8336 8337 return 0; 8338} 8339 8340int sched_cpu_deactivate(unsigned int cpu) 8341{ 8342 struct rq *rq = cpu_rq(cpu); 8343 int ret; 8344 8345 ret = dl_bw_deactivate(cpu); 8346 8347 if (ret) 8348 return ret; 8349 8350 /* 8351 * Remove CPU from nohz.idle_cpus_mask to prevent participating in 8352 * load balancing when not active 8353 */ 8354 nohz_balance_exit_idle(rq); 8355 8356 set_cpu_active(cpu, false); 8357 8358 /* 8359 * From this point forward, this CPU will refuse to run any task that 8360 * is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively 8361 * push those tasks away until this gets cleared, see 8362 * sched_cpu_dying(). 8363 */ 8364 balance_push_set(cpu, true); 8365 8366 /* 8367 * We've cleared cpu_active_mask / set balance_push, wait for all 8368 * preempt-disabled and RCU users of this state to go away such that 8369 * all new such users will observe it. 8370 * 8371 * Specifically, we rely on ttwu to no longer target this CPU, see 8372 * ttwu_queue_cond() and is_cpu_allowed(). 8373 * 8374 * Do sync before park smpboot threads to take care the RCU boost case. 8375 */ 8376 synchronize_rcu(); 8377 8378 sched_set_rq_offline(rq, cpu); 8379 8380 scx_rq_deactivate(rq); 8381 8382 /* 8383 * When going down, decrement the number of cores with SMT present. 8384 */ 8385 sched_smt_present_dec(cpu); 8386 8387#ifdef CONFIG_SCHED_SMT 8388 sched_core_cpu_deactivate(cpu); 8389#endif 8390 8391 if (!sched_smp_initialized) 8392 return 0; 8393 8394 sched_update_numa(cpu, false); 8395 cpuset_cpu_inactive(cpu); 8396 sched_domains_numa_masks_clear(cpu); 8397 return 0; 8398} 8399 8400static void sched_rq_cpu_starting(unsigned int cpu) 8401{ 8402 struct rq *rq = cpu_rq(cpu); 8403 8404 rq->calc_load_update = calc_load_update; 8405 update_max_interval(); 8406} 8407 8408int sched_cpu_starting(unsigned int cpu) 8409{ 8410 sched_core_cpu_starting(cpu); 8411 sched_rq_cpu_starting(cpu); 8412 sched_tick_start(cpu); 8413 return 0; 8414} 8415 8416#ifdef CONFIG_HOTPLUG_CPU 8417 8418/* 8419 * Invoked immediately before the stopper thread is invoked to bring the 8420 * CPU down completely. At this point all per CPU kthreads except the 8421 * hotplug thread (current) and the stopper thread (inactive) have been 8422 * either parked or have been unbound from the outgoing CPU. Ensure that 8423 * any of those which might be on the way out are gone. 8424 * 8425 * If after this point a bound task is being woken on this CPU then the 8426 * responsible hotplug callback has failed to do it's job. 8427 * sched_cpu_dying() will catch it with the appropriate fireworks. 8428 */ 8429int sched_cpu_wait_empty(unsigned int cpu) 8430{ 8431 balance_hotplug_wait(); 8432 sched_force_init_mm(); 8433 return 0; 8434} 8435 8436/* 8437 * Since this CPU is going 'away' for a while, fold any nr_active delta we 8438 * might have. Called from the CPU stopper task after ensuring that the 8439 * stopper is the last running task on the CPU, so nr_active count is 8440 * stable. We need to take the tear-down thread which is calling this into 8441 * account, so we hand in adjust = 1 to the load calculation. 8442 * 8443 * Also see the comment "Global load-average calculations". 8444 */ 8445static void calc_load_migrate(struct rq *rq) 8446{ 8447 long delta = calc_load_fold_active(rq, 1); 8448 8449 if (delta) 8450 atomic_long_add(delta, &calc_load_tasks); 8451} 8452 8453static void dump_rq_tasks(struct rq *rq, const char *loglvl) 8454{ 8455 struct task_struct *g, *p; 8456 int cpu = cpu_of(rq); 8457 8458 lockdep_assert_rq_held(rq); 8459 8460 printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running); 8461 for_each_process_thread(g, p) { 8462 if (task_cpu(p) != cpu) 8463 continue; 8464 8465 if (!task_on_rq_queued(p)) 8466 continue; 8467 8468 printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm); 8469 } 8470} 8471 8472int sched_cpu_dying(unsigned int cpu) 8473{ 8474 struct rq *rq = cpu_rq(cpu); 8475 struct rq_flags rf; 8476 8477 /* Handle pending wakeups and then migrate everything off */ 8478 sched_tick_stop(cpu); 8479 8480 rq_lock_irqsave(rq, &rf); 8481 update_rq_clock(rq); 8482 if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) { 8483 WARN(true, "Dying CPU not properly vacated!"); 8484 dump_rq_tasks(rq, KERN_WARNING); 8485 } 8486 dl_server_stop(&rq->fair_server); 8487 rq_unlock_irqrestore(rq, &rf); 8488 8489 calc_load_migrate(rq); 8490 update_max_interval(); 8491 hrtick_clear(rq); 8492 sched_core_cpu_dying(cpu); 8493 return 0; 8494} 8495#endif /* CONFIG_HOTPLUG_CPU */ 8496 8497void __init sched_init_smp(void) 8498{ 8499 sched_init_numa(NUMA_NO_NODE); 8500 8501 prandom_init_once(&sched_rnd_state); 8502 8503 /* 8504 * There's no userspace yet to cause hotplug operations; hence all the 8505 * CPU masks are stable and all blatant races in the below code cannot 8506 * happen. 8507 */ 8508 sched_domains_mutex_lock(); 8509 sched_init_domains(cpu_active_mask); 8510 sched_domains_mutex_unlock(); 8511 8512 /* Move init over to a non-isolated CPU */ 8513 if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0) 8514 BUG(); 8515 current->flags &= ~PF_NO_SETAFFINITY; 8516 sched_init_granularity(); 8517 8518 init_sched_rt_class(); 8519 init_sched_dl_class(); 8520 8521 sched_init_dl_servers(); 8522 8523 sched_smp_initialized = true; 8524} 8525 8526static int __init migration_init(void) 8527{ 8528 sched_cpu_starting(smp_processor_id()); 8529 return 0; 8530} 8531early_initcall(migration_init); 8532 8533int in_sched_functions(unsigned long addr) 8534{ 8535 return in_lock_functions(addr) || 8536 (addr >= (unsigned long)__sched_text_start 8537 && addr < (unsigned long)__sched_text_end); 8538} 8539 8540#ifdef CONFIG_CGROUP_SCHED 8541/* 8542 * Default task group. 8543 * Every task in system belongs to this group at bootup. 8544 */ 8545struct task_group root_task_group; 8546LIST_HEAD(task_groups); 8547 8548/* Cacheline aligned slab cache for task_group */ 8549static struct kmem_cache *task_group_cache __ro_after_init; 8550#endif 8551 8552void __init sched_init(void) 8553{ 8554 unsigned long ptr = 0; 8555 int i; 8556 8557 /* Make sure the linker didn't screw up */ 8558 BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class)); 8559 BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class)); 8560 BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class)); 8561 BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class)); 8562#ifdef CONFIG_SCHED_CLASS_EXT 8563 BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class)); 8564 BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class)); 8565#endif 8566 8567 wait_bit_init(); 8568 8569#ifdef CONFIG_FAIR_GROUP_SCHED 8570 ptr += 2 * nr_cpu_ids * sizeof(void **); 8571#endif 8572#ifdef CONFIG_RT_GROUP_SCHED 8573 ptr += 2 * nr_cpu_ids * sizeof(void **); 8574#endif 8575 if (ptr) { 8576 ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT); 8577 8578#ifdef CONFIG_FAIR_GROUP_SCHED 8579 root_task_group.se = (struct sched_entity **)ptr; 8580 ptr += nr_cpu_ids * sizeof(void **); 8581 8582 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 8583 ptr += nr_cpu_ids * sizeof(void **); 8584 8585 root_task_group.shares = ROOT_TASK_GROUP_LOAD; 8586 init_cfs_bandwidth(&root_task_group.cfs_bandwidth, NULL); 8587#endif /* CONFIG_FAIR_GROUP_SCHED */ 8588#ifdef CONFIG_EXT_GROUP_SCHED 8589 scx_tg_init(&root_task_group); 8590#endif /* CONFIG_EXT_GROUP_SCHED */ 8591#ifdef CONFIG_RT_GROUP_SCHED 8592 root_task_group.rt_se = (struct sched_rt_entity **)ptr; 8593 ptr += nr_cpu_ids * sizeof(void **); 8594 8595 root_task_group.rt_rq = (struct rt_rq **)ptr; 8596 ptr += nr_cpu_ids * sizeof(void **); 8597 8598#endif /* CONFIG_RT_GROUP_SCHED */ 8599 } 8600 8601 init_defrootdomain(); 8602 8603#ifdef CONFIG_RT_GROUP_SCHED 8604 init_rt_bandwidth(&root_task_group.rt_bandwidth, 8605 global_rt_period(), global_rt_runtime()); 8606#endif /* CONFIG_RT_GROUP_SCHED */ 8607 8608#ifdef CONFIG_CGROUP_SCHED 8609 task_group_cache = KMEM_CACHE(task_group, 0); 8610 8611 list_add(&root_task_group.list, &task_groups); 8612 INIT_LIST_HEAD(&root_task_group.children); 8613 INIT_LIST_HEAD(&root_task_group.siblings); 8614 autogroup_init(&init_task); 8615#endif /* CONFIG_CGROUP_SCHED */ 8616 8617 for_each_possible_cpu(i) { 8618 struct rq *rq; 8619 8620 rq = cpu_rq(i); 8621 raw_spin_lock_init(&rq->__lock); 8622 rq->nr_running = 0; 8623 rq->calc_load_active = 0; 8624 rq->calc_load_update = jiffies + LOAD_FREQ; 8625 init_cfs_rq(&rq->cfs); 8626 init_rt_rq(&rq->rt); 8627 init_dl_rq(&rq->dl); 8628#ifdef CONFIG_FAIR_GROUP_SCHED 8629 INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); 8630 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 8631 /* 8632 * How much CPU bandwidth does root_task_group get? 8633 * 8634 * In case of task-groups formed through the cgroup filesystem, it 8635 * gets 100% of the CPU resources in the system. This overall 8636 * system CPU resource is divided among the tasks of 8637 * root_task_group and its child task-groups in a fair manner, 8638 * based on each entity's (task or task-group's) weight 8639 * (se->load.weight). 8640 * 8641 * In other words, if root_task_group has 10 tasks of weight 8642 * 1024) and two child groups A0 and A1 (of weight 1024 each), 8643 * then A0's share of the CPU resource is: 8644 * 8645 * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% 8646 * 8647 * We achieve this by letting root_task_group's tasks sit 8648 * directly in rq->cfs (i.e root_task_group->se[] = NULL). 8649 */ 8650 init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); 8651#endif /* CONFIG_FAIR_GROUP_SCHED */ 8652 8653#ifdef CONFIG_RT_GROUP_SCHED 8654 /* 8655 * This is required for init cpu because rt.c:__enable_runtime() 8656 * starts working after scheduler_running, which is not the case 8657 * yet. 8658 */ 8659 rq->rt.rt_runtime = global_rt_runtime(); 8660 init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); 8661#endif 8662 rq->sd = NULL; 8663 rq->rd = NULL; 8664 rq->cpu_capacity = SCHED_CAPACITY_SCALE; 8665 rq->balance_callback = &balance_push_callback; 8666 rq->active_balance = 0; 8667 rq->next_balance = jiffies; 8668 rq->push_cpu = 0; 8669 rq->cpu = i; 8670 rq->online = 0; 8671 rq->idle_stamp = 0; 8672 rq->avg_idle = 2*sysctl_sched_migration_cost; 8673 rq->max_idle_balance_cost = sysctl_sched_migration_cost; 8674 8675 INIT_LIST_HEAD(&rq->cfs_tasks); 8676 8677 rq_attach_root(rq, &def_root_domain); 8678#ifdef CONFIG_NO_HZ_COMMON 8679 rq->last_blocked_load_update_tick = jiffies; 8680 atomic_set(&rq->nohz_flags, 0); 8681 8682 INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq); 8683#endif 8684#ifdef CONFIG_HOTPLUG_CPU 8685 rcuwait_init(&rq->hotplug_wait); 8686#endif 8687 hrtick_rq_init(rq); 8688 atomic_set(&rq->nr_iowait, 0); 8689 fair_server_init(rq); 8690 8691#ifdef CONFIG_SCHED_CORE 8692 rq->core = rq; 8693 rq->core_pick = NULL; 8694 rq->core_dl_server = NULL; 8695 rq->core_enabled = 0; 8696 rq->core_tree = RB_ROOT; 8697 rq->core_forceidle_count = 0; 8698 rq->core_forceidle_occupation = 0; 8699 rq->core_forceidle_start = 0; 8700 8701 rq->core_cookie = 0UL; 8702#endif 8703 zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i)); 8704 } 8705 8706 set_load_weight(&init_task, false); 8707 init_task.se.slice = sysctl_sched_base_slice, 8708 8709 /* 8710 * The boot idle thread does lazy MMU switching as well: 8711 */ 8712 mmgrab_lazy_tlb(&init_mm); 8713 enter_lazy_tlb(&init_mm, current); 8714 8715 /* 8716 * The idle task doesn't need the kthread struct to function, but it 8717 * is dressed up as a per-CPU kthread and thus needs to play the part 8718 * if we want to avoid special-casing it in code that deals with per-CPU 8719 * kthreads. 8720 */ 8721 WARN_ON(!set_kthread_struct(current)); 8722 8723 /* 8724 * Make us the idle thread. Technically, schedule() should not be 8725 * called from this thread, however somewhere below it might be, 8726 * but because we are the idle thread, we just pick up running again 8727 * when this runqueue becomes "idle". 8728 */ 8729 __sched_fork(0, current); 8730 init_idle(current, smp_processor_id()); 8731 8732 calc_load_update = jiffies + LOAD_FREQ; 8733 8734 idle_thread_set_boot_cpu(); 8735 8736 balance_push_set(smp_processor_id(), false); 8737 init_sched_fair_class(); 8738 init_sched_ext_class(); 8739 8740 psi_init(); 8741 8742 init_uclamp(); 8743 8744 preempt_dynamic_init(); 8745 8746 scheduler_running = 1; 8747} 8748 8749#ifdef CONFIG_DEBUG_ATOMIC_SLEEP 8750 8751void __might_sleep(const char *file, int line) 8752{ 8753 unsigned int state = get_current_state(); 8754 /* 8755 * Blocking primitives will set (and therefore destroy) current->state, 8756 * since we will exit with TASK_RUNNING make sure we enter with it, 8757 * otherwise we will destroy state. 8758 */ 8759 WARN_ONCE(state != TASK_RUNNING && current->task_state_change, 8760 "do not call blocking ops when !TASK_RUNNING; " 8761 "state=%x set at [<%p>] %pS\n", state, 8762 (void *)current->task_state_change, 8763 (void *)current->task_state_change); 8764 8765 __might_resched(file, line, 0); 8766} 8767EXPORT_SYMBOL(__might_sleep); 8768 8769static void print_preempt_disable_ip(int preempt_offset, unsigned long ip) 8770{ 8771 if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT)) 8772 return; 8773 8774 if (preempt_count() == preempt_offset) 8775 return; 8776 8777 pr_err("Preemption disabled at:"); 8778 print_ip_sym(KERN_ERR, ip); 8779} 8780 8781static inline bool resched_offsets_ok(unsigned int offsets) 8782{ 8783 unsigned int nested = preempt_count(); 8784 8785 nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT; 8786 8787 return nested == offsets; 8788} 8789 8790void __might_resched(const char *file, int line, unsigned int offsets) 8791{ 8792 /* Ratelimiting timestamp: */ 8793 static unsigned long prev_jiffy; 8794 8795 unsigned long preempt_disable_ip; 8796 8797 /* WARN_ON_ONCE() by default, no rate limit required: */ 8798 rcu_sleep_check(); 8799 8800 if ((resched_offsets_ok(offsets) && !irqs_disabled() && 8801 !is_idle_task(current) && !current->non_block_count) || 8802 system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING || 8803 oops_in_progress) 8804 return; 8805 8806 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8807 return; 8808 prev_jiffy = jiffies; 8809 8810 /* Save this before calling printk(), since that will clobber it: */ 8811 preempt_disable_ip = get_preempt_disable_ip(current); 8812 8813 pr_err("BUG: sleeping function called from invalid context at %s:%d\n", 8814 file, line); 8815 pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n", 8816 in_atomic(), irqs_disabled(), current->non_block_count, 8817 current->pid, current->comm); 8818 pr_err("preempt_count: %x, expected: %x\n", preempt_count(), 8819 offsets & MIGHT_RESCHED_PREEMPT_MASK); 8820 8821 if (IS_ENABLED(CONFIG_PREEMPT_RCU)) { 8822 pr_err("RCU nest depth: %d, expected: %u\n", 8823 rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT); 8824 } 8825 8826 if (task_stack_end_corrupted(current)) 8827 pr_emerg("Thread overran stack, or stack corrupted\n"); 8828 8829 debug_show_held_locks(current); 8830 if (irqs_disabled()) 8831 print_irqtrace_events(current); 8832 8833 print_preempt_disable_ip(offsets & MIGHT_RESCHED_PREEMPT_MASK, 8834 preempt_disable_ip); 8835 8836 dump_stack(); 8837 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8838} 8839EXPORT_SYMBOL(__might_resched); 8840 8841void __cant_sleep(const char *file, int line, int preempt_offset) 8842{ 8843 static unsigned long prev_jiffy; 8844 8845 if (irqs_disabled()) 8846 return; 8847 8848 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 8849 return; 8850 8851 if (preempt_count() > preempt_offset) 8852 return; 8853 8854 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8855 return; 8856 prev_jiffy = jiffies; 8857 8858 printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line); 8859 printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", 8860 in_atomic(), irqs_disabled(), 8861 current->pid, current->comm); 8862 8863 debug_show_held_locks(current); 8864 dump_stack(); 8865 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8866} 8867EXPORT_SYMBOL_GPL(__cant_sleep); 8868 8869# ifdef CONFIG_SMP 8870void __cant_migrate(const char *file, int line) 8871{ 8872 static unsigned long prev_jiffy; 8873 8874 if (irqs_disabled()) 8875 return; 8876 8877 if (is_migration_disabled(current)) 8878 return; 8879 8880 if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) 8881 return; 8882 8883 if (preempt_count() > 0) 8884 return; 8885 8886 if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) 8887 return; 8888 prev_jiffy = jiffies; 8889 8890 pr_err("BUG: assuming non migratable context at %s:%d\n", file, line); 8891 pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n", 8892 in_atomic(), irqs_disabled(), is_migration_disabled(current), 8893 current->pid, current->comm); 8894 8895 debug_show_held_locks(current); 8896 dump_stack(); 8897 add_taint(TAINT_WARN, LOCKDEP_STILL_OK); 8898} 8899EXPORT_SYMBOL_GPL(__cant_migrate); 8900# endif /* CONFIG_SMP */ 8901#endif /* CONFIG_DEBUG_ATOMIC_SLEEP */ 8902 8903#ifdef CONFIG_MAGIC_SYSRQ 8904void normalize_rt_tasks(void) 8905{ 8906 struct task_struct *g, *p; 8907 struct sched_attr attr = { 8908 .sched_policy = SCHED_NORMAL, 8909 }; 8910 8911 read_lock(&tasklist_lock); 8912 for_each_process_thread(g, p) { 8913 /* 8914 * Only normalize user tasks: 8915 */ 8916 if (p->flags & PF_KTHREAD) 8917 continue; 8918 8919 p->se.exec_start = 0; 8920 schedstat_set(p->stats.wait_start, 0); 8921 schedstat_set(p->stats.sleep_start, 0); 8922 schedstat_set(p->stats.block_start, 0); 8923 8924 if (!rt_or_dl_task(p)) { 8925 /* 8926 * Renice negative nice level userspace 8927 * tasks back to 0: 8928 */ 8929 if (task_nice(p) < 0) 8930 set_user_nice(p, 0); 8931 continue; 8932 } 8933 8934 __sched_setscheduler(p, &attr, false, false); 8935 } 8936 read_unlock(&tasklist_lock); 8937} 8938 8939#endif /* CONFIG_MAGIC_SYSRQ */ 8940 8941#ifdef CONFIG_KGDB_KDB 8942/* 8943 * These functions are only useful for KDB. 8944 * 8945 * They can only be called when the whole system has been 8946 * stopped - every CPU needs to be quiescent, and no scheduling 8947 * activity can take place. Using them for anything else would 8948 * be a serious bug, and as a result, they aren't even visible 8949 * under any other configuration. 8950 */ 8951 8952/** 8953 * curr_task - return the current task for a given CPU. 8954 * @cpu: the processor in question. 8955 * 8956 * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! 8957 * 8958 * Return: The current task for @cpu. 8959 */ 8960struct task_struct *curr_task(int cpu) 8961{ 8962 return cpu_curr(cpu); 8963} 8964 8965#endif /* CONFIG_KGDB_KDB */ 8966 8967#ifdef CONFIG_CGROUP_SCHED 8968/* task_group_lock serializes the addition/removal of task groups */ 8969static DEFINE_SPINLOCK(task_group_lock); 8970 8971static inline void alloc_uclamp_sched_group(struct task_group *tg, 8972 struct task_group *parent) 8973{ 8974#ifdef CONFIG_UCLAMP_TASK_GROUP 8975 enum uclamp_id clamp_id; 8976 8977 for_each_clamp_id(clamp_id) { 8978 uclamp_se_set(&tg->uclamp_req[clamp_id], 8979 uclamp_none(clamp_id), false); 8980 tg->uclamp[clamp_id] = parent->uclamp[clamp_id]; 8981 } 8982#endif 8983} 8984 8985static void sched_free_group(struct task_group *tg) 8986{ 8987 free_fair_sched_group(tg); 8988 free_rt_sched_group(tg); 8989 autogroup_free(tg); 8990 kmem_cache_free(task_group_cache, tg); 8991} 8992 8993static void sched_free_group_rcu(struct rcu_head *rcu) 8994{ 8995 sched_free_group(container_of(rcu, struct task_group, rcu)); 8996} 8997 8998static void sched_unregister_group(struct task_group *tg) 8999{ 9000 unregister_fair_sched_group(tg); 9001 unregister_rt_sched_group(tg); 9002 /* 9003 * We have to wait for yet another RCU grace period to expire, as 9004 * print_cfs_stats() might run concurrently. 9005 */ 9006 call_rcu(&tg->rcu, sched_free_group_rcu); 9007} 9008 9009/* allocate runqueue etc for a new task group */ 9010struct task_group *sched_create_group(struct task_group *parent) 9011{ 9012 struct task_group *tg; 9013 9014 tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO); 9015 if (!tg) 9016 return ERR_PTR(-ENOMEM); 9017 9018 if (!alloc_fair_sched_group(tg, parent)) 9019 goto err; 9020 9021 if (!alloc_rt_sched_group(tg, parent)) 9022 goto err; 9023 9024 scx_tg_init(tg); 9025 alloc_uclamp_sched_group(tg, parent); 9026 9027 return tg; 9028 9029err: 9030 sched_free_group(tg); 9031 return ERR_PTR(-ENOMEM); 9032} 9033 9034void sched_online_group(struct task_group *tg, struct task_group *parent) 9035{ 9036 unsigned long flags; 9037 9038 spin_lock_irqsave(&task_group_lock, flags); 9039 list_add_tail_rcu(&tg->list, &task_groups); 9040 9041 /* Root should already exist: */ 9042 WARN_ON(!parent); 9043 9044 tg->parent = parent; 9045 INIT_LIST_HEAD(&tg->children); 9046 list_add_rcu(&tg->siblings, &parent->children); 9047 spin_unlock_irqrestore(&task_group_lock, flags); 9048 9049 online_fair_sched_group(tg); 9050} 9051 9052/* RCU callback to free various structures associated with a task group */ 9053static void sched_unregister_group_rcu(struct rcu_head *rhp) 9054{ 9055 /* Now it should be safe to free those cfs_rqs: */ 9056 sched_unregister_group(container_of(rhp, struct task_group, rcu)); 9057} 9058 9059void sched_destroy_group(struct task_group *tg) 9060{ 9061 /* Wait for possible concurrent references to cfs_rqs complete: */ 9062 call_rcu(&tg->rcu, sched_unregister_group_rcu); 9063} 9064 9065void sched_release_group(struct task_group *tg) 9066{ 9067 unsigned long flags; 9068 9069 /* 9070 * Unlink first, to avoid walk_tg_tree_from() from finding us (via 9071 * sched_cfs_period_timer()). 9072 * 9073 * For this to be effective, we have to wait for all pending users of 9074 * this task group to leave their RCU critical section to ensure no new 9075 * user will see our dying task group any more. Specifically ensure 9076 * that tg_unthrottle_up() won't add decayed cfs_rq's to it. 9077 * 9078 * We therefore defer calling unregister_fair_sched_group() to 9079 * sched_unregister_group() which is guarantied to get called only after the 9080 * current RCU grace period has expired. 9081 */ 9082 spin_lock_irqsave(&task_group_lock, flags); 9083 list_del_rcu(&tg->list); 9084 list_del_rcu(&tg->siblings); 9085 spin_unlock_irqrestore(&task_group_lock, flags); 9086} 9087 9088static void sched_change_group(struct task_struct *tsk) 9089{ 9090 struct task_group *tg; 9091 9092 /* 9093 * All callers are synchronized by task_rq_lock(); we do not use RCU 9094 * which is pointless here. Thus, we pass "true" to task_css_check() 9095 * to prevent lockdep warnings. 9096 */ 9097 tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), 9098 struct task_group, css); 9099 tg = autogroup_task_group(tsk, tg); 9100 tsk->sched_task_group = tg; 9101 9102#ifdef CONFIG_FAIR_GROUP_SCHED 9103 if (tsk->sched_class->task_change_group) 9104 tsk->sched_class->task_change_group(tsk); 9105 else 9106#endif 9107 set_task_rq(tsk, task_cpu(tsk)); 9108} 9109 9110/* 9111 * Change task's runqueue when it moves between groups. 9112 * 9113 * The caller of this function should have put the task in its new group by 9114 * now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect 9115 * its new group. 9116 */ 9117void sched_move_task(struct task_struct *tsk, bool for_autogroup) 9118{ 9119 unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE; 9120 bool resched = false; 9121 bool queued = false; 9122 struct rq *rq; 9123 9124 CLASS(task_rq_lock, rq_guard)(tsk); 9125 rq = rq_guard.rq; 9126 9127 scoped_guard (sched_change, tsk, queue_flags) { 9128 sched_change_group(tsk); 9129 if (!for_autogroup) 9130 scx_cgroup_move_task(tsk); 9131 if (scope->running) 9132 resched = true; 9133 queued = scope->queued; 9134 } 9135 9136 if (resched) 9137 resched_curr(rq); 9138 else if (queued) 9139 wakeup_preempt(rq, tsk, 0); 9140 9141 __balance_callbacks(rq, &rq_guard.rf); 9142} 9143 9144static struct cgroup_subsys_state * 9145cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 9146{ 9147 struct task_group *parent = css_tg(parent_css); 9148 struct task_group *tg; 9149 9150 if (!parent) { 9151 /* This is early initialization for the top cgroup */ 9152 return &root_task_group.css; 9153 } 9154 9155 tg = sched_create_group(parent); 9156 if (IS_ERR(tg)) 9157 return ERR_PTR(-ENOMEM); 9158 9159 return &tg->css; 9160} 9161 9162/* Expose task group only after completing cgroup initialization */ 9163static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) 9164{ 9165 struct task_group *tg = css_tg(css); 9166 struct task_group *parent = css_tg(css->parent); 9167 int ret; 9168 9169 ret = scx_tg_online(tg); 9170 if (ret) 9171 return ret; 9172 9173 if (parent) 9174 sched_online_group(tg, parent); 9175 9176#ifdef CONFIG_UCLAMP_TASK_GROUP 9177 /* Propagate the effective uclamp value for the new group */ 9178 guard(mutex)(&uclamp_mutex); 9179 guard(rcu)(); 9180 cpu_util_update_eff(css); 9181#endif 9182 9183 return 0; 9184} 9185 9186static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css) 9187{ 9188 struct task_group *tg = css_tg(css); 9189 9190 scx_tg_offline(tg); 9191} 9192 9193static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) 9194{ 9195 struct task_group *tg = css_tg(css); 9196 9197 sched_release_group(tg); 9198} 9199 9200static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) 9201{ 9202 struct task_group *tg = css_tg(css); 9203 9204 /* 9205 * Relies on the RCU grace period between css_released() and this. 9206 */ 9207 sched_unregister_group(tg); 9208} 9209 9210static int cpu_cgroup_can_attach(struct cgroup_taskset *tset) 9211{ 9212#ifdef CONFIG_RT_GROUP_SCHED 9213 struct task_struct *task; 9214 struct cgroup_subsys_state *css; 9215 9216 if (!rt_group_sched_enabled()) 9217 goto scx_check; 9218 9219 cgroup_taskset_for_each(task, css, tset) { 9220 if (!sched_rt_can_attach(css_tg(css), task)) 9221 return -EINVAL; 9222 } 9223scx_check: 9224#endif /* CONFIG_RT_GROUP_SCHED */ 9225 return scx_cgroup_can_attach(tset); 9226} 9227 9228static void cpu_cgroup_attach(struct cgroup_taskset *tset) 9229{ 9230 struct task_struct *task; 9231 struct cgroup_subsys_state *css; 9232 9233 cgroup_taskset_for_each(task, css, tset) 9234 sched_move_task(task, false); 9235} 9236 9237static void cpu_cgroup_cancel_attach(struct cgroup_taskset *tset) 9238{ 9239 scx_cgroup_cancel_attach(tset); 9240} 9241 9242#ifdef CONFIG_UCLAMP_TASK_GROUP 9243static void cpu_util_update_eff(struct cgroup_subsys_state *css) 9244{ 9245 struct cgroup_subsys_state *top_css = css; 9246 struct uclamp_se *uc_parent = NULL; 9247 struct uclamp_se *uc_se = NULL; 9248 unsigned int eff[UCLAMP_CNT]; 9249 enum uclamp_id clamp_id; 9250 unsigned int clamps; 9251 9252 lockdep_assert_held(&uclamp_mutex); 9253 WARN_ON_ONCE(!rcu_read_lock_held()); 9254 9255 css_for_each_descendant_pre(css, top_css) { 9256 uc_parent = css_tg(css)->parent 9257 ? css_tg(css)->parent->uclamp : NULL; 9258 9259 for_each_clamp_id(clamp_id) { 9260 /* Assume effective clamps matches requested clamps */ 9261 eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value; 9262 /* Cap effective clamps with parent's effective clamps */ 9263 if (uc_parent && 9264 eff[clamp_id] > uc_parent[clamp_id].value) { 9265 eff[clamp_id] = uc_parent[clamp_id].value; 9266 } 9267 } 9268 /* Ensure protection is always capped by limit */ 9269 eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]); 9270 9271 /* Propagate most restrictive effective clamps */ 9272 clamps = 0x0; 9273 uc_se = css_tg(css)->uclamp; 9274 for_each_clamp_id(clamp_id) { 9275 if (eff[clamp_id] == uc_se[clamp_id].value) 9276 continue; 9277 uc_se[clamp_id].value = eff[clamp_id]; 9278 uc_se[clamp_id].bucket_id = uclamp_bucket_id(eff[clamp_id]); 9279 clamps |= (0x1 << clamp_id); 9280 } 9281 if (!clamps) { 9282 css = css_rightmost_descendant(css); 9283 continue; 9284 } 9285 9286 /* Immediately update descendants RUNNABLE tasks */ 9287 uclamp_update_active_tasks(css); 9288 } 9289} 9290 9291/* 9292 * Integer 10^N with a given N exponent by casting to integer the literal "1eN" 9293 * C expression. Since there is no way to convert a macro argument (N) into a 9294 * character constant, use two levels of macros. 9295 */ 9296#define _POW10(exp) ((unsigned int)1e##exp) 9297#define POW10(exp) _POW10(exp) 9298 9299struct uclamp_request { 9300#define UCLAMP_PERCENT_SHIFT 2 9301#define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT)) 9302 s64 percent; 9303 u64 util; 9304 int ret; 9305}; 9306 9307static inline struct uclamp_request 9308capacity_from_percent(char *buf) 9309{ 9310 struct uclamp_request req = { 9311 .percent = UCLAMP_PERCENT_SCALE, 9312 .util = SCHED_CAPACITY_SCALE, 9313 .ret = 0, 9314 }; 9315 9316 buf = strim(buf); 9317 if (strcmp(buf, "max")) { 9318 req.ret = cgroup_parse_float(buf, UCLAMP_PERCENT_SHIFT, 9319 &req.percent); 9320 if (req.ret) 9321 return req; 9322 if ((u64)req.percent > UCLAMP_PERCENT_SCALE) { 9323 req.ret = -ERANGE; 9324 return req; 9325 } 9326 9327 req.util = req.percent << SCHED_CAPACITY_SHIFT; 9328 req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE); 9329 } 9330 9331 return req; 9332} 9333 9334static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf, 9335 size_t nbytes, loff_t off, 9336 enum uclamp_id clamp_id) 9337{ 9338 struct uclamp_request req; 9339 struct task_group *tg; 9340 9341 req = capacity_from_percent(buf); 9342 if (req.ret) 9343 return req.ret; 9344 9345 sched_uclamp_enable(); 9346 9347 guard(mutex)(&uclamp_mutex); 9348 guard(rcu)(); 9349 9350 tg = css_tg(of_css(of)); 9351 if (tg->uclamp_req[clamp_id].value != req.util) 9352 uclamp_se_set(&tg->uclamp_req[clamp_id], req.util, false); 9353 9354 /* 9355 * Because of not recoverable conversion rounding we keep track of the 9356 * exact requested value 9357 */ 9358 tg->uclamp_pct[clamp_id] = req.percent; 9359 9360 /* Update effective clamps to track the most restrictive value */ 9361 cpu_util_update_eff(of_css(of)); 9362 9363 return nbytes; 9364} 9365 9366static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of, 9367 char *buf, size_t nbytes, 9368 loff_t off) 9369{ 9370 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MIN); 9371} 9372 9373static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of, 9374 char *buf, size_t nbytes, 9375 loff_t off) 9376{ 9377 return cpu_uclamp_write(of, buf, nbytes, off, UCLAMP_MAX); 9378} 9379 9380static inline void cpu_uclamp_print(struct seq_file *sf, 9381 enum uclamp_id clamp_id) 9382{ 9383 struct task_group *tg; 9384 u64 util_clamp; 9385 u64 percent; 9386 u32 rem; 9387 9388 scoped_guard (rcu) { 9389 tg = css_tg(seq_css(sf)); 9390 util_clamp = tg->uclamp_req[clamp_id].value; 9391 } 9392 9393 if (util_clamp == SCHED_CAPACITY_SCALE) { 9394 seq_puts(sf, "max\n"); 9395 return; 9396 } 9397 9398 percent = tg->uclamp_pct[clamp_id]; 9399 percent = div_u64_rem(percent, POW10(UCLAMP_PERCENT_SHIFT), &rem); 9400 seq_printf(sf, "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem); 9401} 9402 9403static int cpu_uclamp_min_show(struct seq_file *sf, void *v) 9404{ 9405 cpu_uclamp_print(sf, UCLAMP_MIN); 9406 return 0; 9407} 9408 9409static int cpu_uclamp_max_show(struct seq_file *sf, void *v) 9410{ 9411 cpu_uclamp_print(sf, UCLAMP_MAX); 9412 return 0; 9413} 9414#endif /* CONFIG_UCLAMP_TASK_GROUP */ 9415 9416#ifdef CONFIG_GROUP_SCHED_WEIGHT 9417static unsigned long tg_weight(struct task_group *tg) 9418{ 9419#ifdef CONFIG_FAIR_GROUP_SCHED 9420 return scale_load_down(tg->shares); 9421#else 9422 return sched_weight_from_cgroup(tg->scx.weight); 9423#endif 9424} 9425 9426static int cpu_shares_write_u64(struct cgroup_subsys_state *css, 9427 struct cftype *cftype, u64 shareval) 9428{ 9429 int ret; 9430 9431 if (shareval > scale_load_down(ULONG_MAX)) 9432 shareval = MAX_SHARES; 9433 ret = sched_group_set_shares(css_tg(css), scale_load(shareval)); 9434 if (!ret) 9435 scx_group_set_weight(css_tg(css), 9436 sched_weight_to_cgroup(shareval)); 9437 return ret; 9438} 9439 9440static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, 9441 struct cftype *cft) 9442{ 9443 return tg_weight(css_tg(css)); 9444} 9445#endif /* CONFIG_GROUP_SCHED_WEIGHT */ 9446 9447#ifdef CONFIG_CFS_BANDWIDTH 9448static DEFINE_MUTEX(cfs_constraints_mutex); 9449 9450static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); 9451 9452static int tg_set_cfs_bandwidth(struct task_group *tg, 9453 u64 period_us, u64 quota_us, u64 burst_us) 9454{ 9455 int i, ret = 0, runtime_enabled, runtime_was_enabled; 9456 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9457 u64 period, quota, burst; 9458 9459 period = (u64)period_us * NSEC_PER_USEC; 9460 9461 if (quota_us == RUNTIME_INF) 9462 quota = RUNTIME_INF; 9463 else 9464 quota = (u64)quota_us * NSEC_PER_USEC; 9465 9466 burst = (u64)burst_us * NSEC_PER_USEC; 9467 9468 /* 9469 * Prevent race between setting of cfs_rq->runtime_enabled and 9470 * unthrottle_offline_cfs_rqs(). 9471 */ 9472 guard(cpus_read_lock)(); 9473 guard(mutex)(&cfs_constraints_mutex); 9474 9475 ret = __cfs_schedulable(tg, period, quota); 9476 if (ret) 9477 return ret; 9478 9479 runtime_enabled = quota != RUNTIME_INF; 9480 runtime_was_enabled = cfs_b->quota != RUNTIME_INF; 9481 /* 9482 * If we need to toggle cfs_bandwidth_used, off->on must occur 9483 * before making related changes, and on->off must occur afterwards 9484 */ 9485 if (runtime_enabled && !runtime_was_enabled) 9486 cfs_bandwidth_usage_inc(); 9487 9488 scoped_guard (raw_spinlock_irq, &cfs_b->lock) { 9489 cfs_b->period = ns_to_ktime(period); 9490 cfs_b->quota = quota; 9491 cfs_b->burst = burst; 9492 9493 __refill_cfs_bandwidth_runtime(cfs_b); 9494 9495 /* 9496 * Restart the period timer (if active) to handle new 9497 * period expiry: 9498 */ 9499 if (runtime_enabled) 9500 start_cfs_bandwidth(cfs_b); 9501 } 9502 9503 for_each_online_cpu(i) { 9504 struct cfs_rq *cfs_rq = tg->cfs_rq[i]; 9505 struct rq *rq = cfs_rq->rq; 9506 9507 guard(rq_lock_irq)(rq); 9508 cfs_rq->runtime_enabled = runtime_enabled; 9509 cfs_rq->runtime_remaining = 1; 9510 9511 if (cfs_rq->throttled) 9512 unthrottle_cfs_rq(cfs_rq); 9513 } 9514 9515 if (runtime_was_enabled && !runtime_enabled) 9516 cfs_bandwidth_usage_dec(); 9517 9518 return 0; 9519} 9520 9521static u64 tg_get_cfs_period(struct task_group *tg) 9522{ 9523 u64 cfs_period_us; 9524 9525 cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); 9526 do_div(cfs_period_us, NSEC_PER_USEC); 9527 9528 return cfs_period_us; 9529} 9530 9531static u64 tg_get_cfs_quota(struct task_group *tg) 9532{ 9533 u64 quota_us; 9534 9535 if (tg->cfs_bandwidth.quota == RUNTIME_INF) 9536 return RUNTIME_INF; 9537 9538 quota_us = tg->cfs_bandwidth.quota; 9539 do_div(quota_us, NSEC_PER_USEC); 9540 9541 return quota_us; 9542} 9543 9544static u64 tg_get_cfs_burst(struct task_group *tg) 9545{ 9546 u64 burst_us; 9547 9548 burst_us = tg->cfs_bandwidth.burst; 9549 do_div(burst_us, NSEC_PER_USEC); 9550 9551 return burst_us; 9552} 9553 9554struct cfs_schedulable_data { 9555 struct task_group *tg; 9556 u64 period, quota; 9557}; 9558 9559/* 9560 * normalize group quota/period to be quota/max_period 9561 * note: units are usecs 9562 */ 9563static u64 normalize_cfs_quota(struct task_group *tg, 9564 struct cfs_schedulable_data *d) 9565{ 9566 u64 quota, period; 9567 9568 if (tg == d->tg) { 9569 period = d->period; 9570 quota = d->quota; 9571 } else { 9572 period = tg_get_cfs_period(tg); 9573 quota = tg_get_cfs_quota(tg); 9574 } 9575 9576 /* note: these should typically be equivalent */ 9577 if (quota == RUNTIME_INF || quota == -1) 9578 return RUNTIME_INF; 9579 9580 return to_ratio(period, quota); 9581} 9582 9583static int tg_cfs_schedulable_down(struct task_group *tg, void *data) 9584{ 9585 struct cfs_schedulable_data *d = data; 9586 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9587 s64 quota = 0, parent_quota = -1; 9588 9589 if (!tg->parent) { 9590 quota = RUNTIME_INF; 9591 } else { 9592 struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; 9593 9594 quota = normalize_cfs_quota(tg, d); 9595 parent_quota = parent_b->hierarchical_quota; 9596 9597 /* 9598 * Ensure max(child_quota) <= parent_quota. On cgroup2, 9599 * always take the non-RUNTIME_INF min. On cgroup1, only 9600 * inherit when no limit is set. In both cases this is used 9601 * by the scheduler to determine if a given CFS task has a 9602 * bandwidth constraint at some higher level. 9603 */ 9604 if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) { 9605 if (quota == RUNTIME_INF) 9606 quota = parent_quota; 9607 else if (parent_quota != RUNTIME_INF) 9608 quota = min(quota, parent_quota); 9609 } else { 9610 if (quota == RUNTIME_INF) 9611 quota = parent_quota; 9612 else if (parent_quota != RUNTIME_INF && quota > parent_quota) 9613 return -EINVAL; 9614 } 9615 } 9616 cfs_b->hierarchical_quota = quota; 9617 9618 return 0; 9619} 9620 9621static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) 9622{ 9623 struct cfs_schedulable_data data = { 9624 .tg = tg, 9625 .period = period, 9626 .quota = quota, 9627 }; 9628 9629 if (quota != RUNTIME_INF) { 9630 do_div(data.period, NSEC_PER_USEC); 9631 do_div(data.quota, NSEC_PER_USEC); 9632 } 9633 9634 guard(rcu)(); 9635 return walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); 9636} 9637 9638static int cpu_cfs_stat_show(struct seq_file *sf, void *v) 9639{ 9640 struct task_group *tg = css_tg(seq_css(sf)); 9641 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9642 9643 seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); 9644 seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); 9645 seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); 9646 9647 if (schedstat_enabled() && tg != &root_task_group) { 9648 struct sched_statistics *stats; 9649 u64 ws = 0; 9650 int i; 9651 9652 for_each_possible_cpu(i) { 9653 stats = __schedstats_from_se(tg->se[i]); 9654 ws += schedstat_val(stats->wait_sum); 9655 } 9656 9657 seq_printf(sf, "wait_sum %llu\n", ws); 9658 } 9659 9660 seq_printf(sf, "nr_bursts %d\n", cfs_b->nr_burst); 9661 seq_printf(sf, "burst_time %llu\n", cfs_b->burst_time); 9662 9663 return 0; 9664} 9665 9666static u64 throttled_time_self(struct task_group *tg) 9667{ 9668 int i; 9669 u64 total = 0; 9670 9671 for_each_possible_cpu(i) { 9672 total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time); 9673 } 9674 9675 return total; 9676} 9677 9678static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v) 9679{ 9680 struct task_group *tg = css_tg(seq_css(sf)); 9681 9682 seq_printf(sf, "throttled_time %llu\n", throttled_time_self(tg)); 9683 9684 return 0; 9685} 9686#endif /* CONFIG_CFS_BANDWIDTH */ 9687 9688#ifdef CONFIG_GROUP_SCHED_BANDWIDTH 9689const u64 max_bw_quota_period_us = 1 * USEC_PER_SEC; /* 1s */ 9690static const u64 min_bw_quota_period_us = 1 * USEC_PER_MSEC; /* 1ms */ 9691/* More than 203 days if BW_SHIFT equals 20. */ 9692static const u64 max_bw_runtime_us = MAX_BW; 9693 9694static void tg_bandwidth(struct task_group *tg, 9695 u64 *period_us_p, u64 *quota_us_p, u64 *burst_us_p) 9696{ 9697#ifdef CONFIG_CFS_BANDWIDTH 9698 if (period_us_p) 9699 *period_us_p = tg_get_cfs_period(tg); 9700 if (quota_us_p) 9701 *quota_us_p = tg_get_cfs_quota(tg); 9702 if (burst_us_p) 9703 *burst_us_p = tg_get_cfs_burst(tg); 9704#else /* !CONFIG_CFS_BANDWIDTH */ 9705 if (period_us_p) 9706 *period_us_p = tg->scx.bw_period_us; 9707 if (quota_us_p) 9708 *quota_us_p = tg->scx.bw_quota_us; 9709 if (burst_us_p) 9710 *burst_us_p = tg->scx.bw_burst_us; 9711#endif /* CONFIG_CFS_BANDWIDTH */ 9712} 9713 9714static u64 cpu_period_read_u64(struct cgroup_subsys_state *css, 9715 struct cftype *cft) 9716{ 9717 u64 period_us; 9718 9719 tg_bandwidth(css_tg(css), &period_us, NULL, NULL); 9720 return period_us; 9721} 9722 9723static int tg_set_bandwidth(struct task_group *tg, 9724 u64 period_us, u64 quota_us, u64 burst_us) 9725{ 9726 const u64 max_usec = U64_MAX / NSEC_PER_USEC; 9727 int ret = 0; 9728 9729 if (tg == &root_task_group) 9730 return -EINVAL; 9731 9732 /* Values should survive translation to nsec */ 9733 if (period_us > max_usec || 9734 (quota_us != RUNTIME_INF && quota_us > max_usec) || 9735 burst_us > max_usec) 9736 return -EINVAL; 9737 9738 /* 9739 * Ensure we have some amount of bandwidth every period. This is to 9740 * prevent reaching a state of large arrears when throttled via 9741 * entity_tick() resulting in prolonged exit starvation. 9742 */ 9743 if (quota_us < min_bw_quota_period_us || 9744 period_us < min_bw_quota_period_us) 9745 return -EINVAL; 9746 9747 /* 9748 * Likewise, bound things on the other side by preventing insane quota 9749 * periods. This also allows us to normalize in computing quota 9750 * feasibility. 9751 */ 9752 if (period_us > max_bw_quota_period_us) 9753 return -EINVAL; 9754 9755 /* 9756 * Bound quota to defend quota against overflow during bandwidth shift. 9757 */ 9758 if (quota_us != RUNTIME_INF && quota_us > max_bw_runtime_us) 9759 return -EINVAL; 9760 9761 if (quota_us != RUNTIME_INF && (burst_us > quota_us || 9762 burst_us + quota_us > max_bw_runtime_us)) 9763 return -EINVAL; 9764 9765#ifdef CONFIG_CFS_BANDWIDTH 9766 ret = tg_set_cfs_bandwidth(tg, period_us, quota_us, burst_us); 9767#endif /* CONFIG_CFS_BANDWIDTH */ 9768 if (!ret) 9769 scx_group_set_bandwidth(tg, period_us, quota_us, burst_us); 9770 return ret; 9771} 9772 9773static s64 cpu_quota_read_s64(struct cgroup_subsys_state *css, 9774 struct cftype *cft) 9775{ 9776 u64 quota_us; 9777 9778 tg_bandwidth(css_tg(css), NULL, "a_us, NULL); 9779 return quota_us; /* (s64)RUNTIME_INF becomes -1 */ 9780} 9781 9782static u64 cpu_burst_read_u64(struct cgroup_subsys_state *css, 9783 struct cftype *cft) 9784{ 9785 u64 burst_us; 9786 9787 tg_bandwidth(css_tg(css), NULL, NULL, &burst_us); 9788 return burst_us; 9789} 9790 9791static int cpu_period_write_u64(struct cgroup_subsys_state *css, 9792 struct cftype *cftype, u64 period_us) 9793{ 9794 struct task_group *tg = css_tg(css); 9795 u64 quota_us, burst_us; 9796 9797 tg_bandwidth(tg, NULL, "a_us, &burst_us); 9798 return tg_set_bandwidth(tg, period_us, quota_us, burst_us); 9799} 9800 9801static int cpu_quota_write_s64(struct cgroup_subsys_state *css, 9802 struct cftype *cftype, s64 quota_us) 9803{ 9804 struct task_group *tg = css_tg(css); 9805 u64 period_us, burst_us; 9806 9807 if (quota_us < 0) 9808 quota_us = RUNTIME_INF; 9809 9810 tg_bandwidth(tg, &period_us, NULL, &burst_us); 9811 return tg_set_bandwidth(tg, period_us, quota_us, burst_us); 9812} 9813 9814static int cpu_burst_write_u64(struct cgroup_subsys_state *css, 9815 struct cftype *cftype, u64 burst_us) 9816{ 9817 struct task_group *tg = css_tg(css); 9818 u64 period_us, quota_us; 9819 9820 tg_bandwidth(tg, &period_us, "a_us, NULL); 9821 return tg_set_bandwidth(tg, period_us, quota_us, burst_us); 9822} 9823#endif /* CONFIG_GROUP_SCHED_BANDWIDTH */ 9824 9825#ifdef CONFIG_RT_GROUP_SCHED 9826static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, 9827 struct cftype *cft, s64 val) 9828{ 9829 return sched_group_set_rt_runtime(css_tg(css), val); 9830} 9831 9832static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, 9833 struct cftype *cft) 9834{ 9835 return sched_group_rt_runtime(css_tg(css)); 9836} 9837 9838static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, 9839 struct cftype *cftype, u64 rt_period_us) 9840{ 9841 return sched_group_set_rt_period(css_tg(css), rt_period_us); 9842} 9843 9844static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, 9845 struct cftype *cft) 9846{ 9847 return sched_group_rt_period(css_tg(css)); 9848} 9849#endif /* CONFIG_RT_GROUP_SCHED */ 9850 9851#ifdef CONFIG_GROUP_SCHED_WEIGHT 9852static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css, 9853 struct cftype *cft) 9854{ 9855 return css_tg(css)->idle; 9856} 9857 9858static int cpu_idle_write_s64(struct cgroup_subsys_state *css, 9859 struct cftype *cft, s64 idle) 9860{ 9861 int ret; 9862 9863 ret = sched_group_set_idle(css_tg(css), idle); 9864 if (!ret) 9865 scx_group_set_idle(css_tg(css), idle); 9866 return ret; 9867} 9868#endif /* CONFIG_GROUP_SCHED_WEIGHT */ 9869 9870static struct cftype cpu_legacy_files[] = { 9871#ifdef CONFIG_GROUP_SCHED_WEIGHT 9872 { 9873 .name = "shares", 9874 .read_u64 = cpu_shares_read_u64, 9875 .write_u64 = cpu_shares_write_u64, 9876 }, 9877 { 9878 .name = "idle", 9879 .read_s64 = cpu_idle_read_s64, 9880 .write_s64 = cpu_idle_write_s64, 9881 }, 9882#endif 9883#ifdef CONFIG_GROUP_SCHED_BANDWIDTH 9884 { 9885 .name = "cfs_period_us", 9886 .read_u64 = cpu_period_read_u64, 9887 .write_u64 = cpu_period_write_u64, 9888 }, 9889 { 9890 .name = "cfs_quota_us", 9891 .read_s64 = cpu_quota_read_s64, 9892 .write_s64 = cpu_quota_write_s64, 9893 }, 9894 { 9895 .name = "cfs_burst_us", 9896 .read_u64 = cpu_burst_read_u64, 9897 .write_u64 = cpu_burst_write_u64, 9898 }, 9899#endif 9900#ifdef CONFIG_CFS_BANDWIDTH 9901 { 9902 .name = "stat", 9903 .seq_show = cpu_cfs_stat_show, 9904 }, 9905 { 9906 .name = "stat.local", 9907 .seq_show = cpu_cfs_local_stat_show, 9908 }, 9909#endif 9910#ifdef CONFIG_UCLAMP_TASK_GROUP 9911 { 9912 .name = "uclamp.min", 9913 .flags = CFTYPE_NOT_ON_ROOT, 9914 .seq_show = cpu_uclamp_min_show, 9915 .write = cpu_uclamp_min_write, 9916 }, 9917 { 9918 .name = "uclamp.max", 9919 .flags = CFTYPE_NOT_ON_ROOT, 9920 .seq_show = cpu_uclamp_max_show, 9921 .write = cpu_uclamp_max_write, 9922 }, 9923#endif 9924 { } /* Terminate */ 9925}; 9926 9927#ifdef CONFIG_RT_GROUP_SCHED 9928static struct cftype rt_group_files[] = { 9929 { 9930 .name = "rt_runtime_us", 9931 .read_s64 = cpu_rt_runtime_read, 9932 .write_s64 = cpu_rt_runtime_write, 9933 }, 9934 { 9935 .name = "rt_period_us", 9936 .read_u64 = cpu_rt_period_read_uint, 9937 .write_u64 = cpu_rt_period_write_uint, 9938 }, 9939 { } /* Terminate */ 9940}; 9941 9942# ifdef CONFIG_RT_GROUP_SCHED_DEFAULT_DISABLED 9943DEFINE_STATIC_KEY_FALSE(rt_group_sched); 9944# else 9945DEFINE_STATIC_KEY_TRUE(rt_group_sched); 9946# endif 9947 9948static int __init setup_rt_group_sched(char *str) 9949{ 9950 long val; 9951 9952 if (kstrtol(str, 0, &val) || val < 0 || val > 1) { 9953 pr_warn("Unable to set rt_group_sched\n"); 9954 return 1; 9955 } 9956 if (val) 9957 static_branch_enable(&rt_group_sched); 9958 else 9959 static_branch_disable(&rt_group_sched); 9960 9961 return 1; 9962} 9963__setup("rt_group_sched=", setup_rt_group_sched); 9964 9965static int __init cpu_rt_group_init(void) 9966{ 9967 if (!rt_group_sched_enabled()) 9968 return 0; 9969 9970 WARN_ON(cgroup_add_legacy_cftypes(&cpu_cgrp_subsys, rt_group_files)); 9971 return 0; 9972} 9973subsys_initcall(cpu_rt_group_init); 9974#endif /* CONFIG_RT_GROUP_SCHED */ 9975 9976static int cpu_extra_stat_show(struct seq_file *sf, 9977 struct cgroup_subsys_state *css) 9978{ 9979#ifdef CONFIG_CFS_BANDWIDTH 9980 { 9981 struct task_group *tg = css_tg(css); 9982 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 9983 u64 throttled_usec, burst_usec; 9984 9985 throttled_usec = cfs_b->throttled_time; 9986 do_div(throttled_usec, NSEC_PER_USEC); 9987 burst_usec = cfs_b->burst_time; 9988 do_div(burst_usec, NSEC_PER_USEC); 9989 9990 seq_printf(sf, "nr_periods %d\n" 9991 "nr_throttled %d\n" 9992 "throttled_usec %llu\n" 9993 "nr_bursts %d\n" 9994 "burst_usec %llu\n", 9995 cfs_b->nr_periods, cfs_b->nr_throttled, 9996 throttled_usec, cfs_b->nr_burst, burst_usec); 9997 } 9998#endif /* CONFIG_CFS_BANDWIDTH */ 9999 return 0; 10000} 10001 10002static int cpu_local_stat_show(struct seq_file *sf, 10003 struct cgroup_subsys_state *css) 10004{ 10005#ifdef CONFIG_CFS_BANDWIDTH 10006 { 10007 struct task_group *tg = css_tg(css); 10008 u64 throttled_self_usec; 10009 10010 throttled_self_usec = throttled_time_self(tg); 10011 do_div(throttled_self_usec, NSEC_PER_USEC); 10012 10013 seq_printf(sf, "throttled_usec %llu\n", 10014 throttled_self_usec); 10015 } 10016#endif 10017 return 0; 10018} 10019 10020#ifdef CONFIG_GROUP_SCHED_WEIGHT 10021 10022static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css, 10023 struct cftype *cft) 10024{ 10025 return sched_weight_to_cgroup(tg_weight(css_tg(css))); 10026} 10027 10028static int cpu_weight_write_u64(struct cgroup_subsys_state *css, 10029 struct cftype *cft, u64 cgrp_weight) 10030{ 10031 unsigned long weight; 10032 int ret; 10033 10034 if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX) 10035 return -ERANGE; 10036 10037 weight = sched_weight_from_cgroup(cgrp_weight); 10038 10039 ret = sched_group_set_shares(css_tg(css), scale_load(weight)); 10040 if (!ret) 10041 scx_group_set_weight(css_tg(css), cgrp_weight); 10042 return ret; 10043} 10044 10045static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css, 10046 struct cftype *cft) 10047{ 10048 unsigned long weight = tg_weight(css_tg(css)); 10049 int last_delta = INT_MAX; 10050 int prio, delta; 10051 10052 /* find the closest nice value to the current weight */ 10053 for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) { 10054 delta = abs(sched_prio_to_weight[prio] - weight); 10055 if (delta >= last_delta) 10056 break; 10057 last_delta = delta; 10058 } 10059 10060 return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO); 10061} 10062 10063static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css, 10064 struct cftype *cft, s64 nice) 10065{ 10066 unsigned long weight; 10067 int idx, ret; 10068 10069 if (nice < MIN_NICE || nice > MAX_NICE) 10070 return -ERANGE; 10071 10072 idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO; 10073 idx = array_index_nospec(idx, 40); 10074 weight = sched_prio_to_weight[idx]; 10075 10076 ret = sched_group_set_shares(css_tg(css), scale_load(weight)); 10077 if (!ret) 10078 scx_group_set_weight(css_tg(css), 10079 sched_weight_to_cgroup(weight)); 10080 return ret; 10081} 10082#endif /* CONFIG_GROUP_SCHED_WEIGHT */ 10083 10084static void __maybe_unused cpu_period_quota_print(struct seq_file *sf, 10085 long period, long quota) 10086{ 10087 if (quota < 0) 10088 seq_puts(sf, "max"); 10089 else 10090 seq_printf(sf, "%ld", quota); 10091 10092 seq_printf(sf, " %ld\n", period); 10093} 10094 10095/* caller should put the current value in *@periodp before calling */ 10096static int __maybe_unused cpu_period_quota_parse(char *buf, u64 *period_us_p, 10097 u64 *quota_us_p) 10098{ 10099 char tok[21]; /* U64_MAX */ 10100 10101 if (sscanf(buf, "%20s %llu", tok, period_us_p) < 1) 10102 return -EINVAL; 10103 10104 if (sscanf(tok, "%llu", quota_us_p) < 1) { 10105 if (!strcmp(tok, "max")) 10106 *quota_us_p = RUNTIME_INF; 10107 else 10108 return -EINVAL; 10109 } 10110 10111 return 0; 10112} 10113 10114#ifdef CONFIG_GROUP_SCHED_BANDWIDTH 10115static int cpu_max_show(struct seq_file *sf, void *v) 10116{ 10117 struct task_group *tg = css_tg(seq_css(sf)); 10118 u64 period_us, quota_us; 10119 10120 tg_bandwidth(tg, &period_us, "a_us, NULL); 10121 cpu_period_quota_print(sf, period_us, quota_us); 10122 return 0; 10123} 10124 10125static ssize_t cpu_max_write(struct kernfs_open_file *of, 10126 char *buf, size_t nbytes, loff_t off) 10127{ 10128 struct task_group *tg = css_tg(of_css(of)); 10129 u64 period_us, quota_us, burst_us; 10130 int ret; 10131 10132 tg_bandwidth(tg, &period_us, NULL, &burst_us); 10133 ret = cpu_period_quota_parse(buf, &period_us, "a_us); 10134 if (!ret) 10135 ret = tg_set_bandwidth(tg, period_us, quota_us, burst_us); 10136 return ret ?: nbytes; 10137} 10138#endif /* CONFIG_CFS_BANDWIDTH */ 10139 10140static struct cftype cpu_files[] = { 10141#ifdef CONFIG_GROUP_SCHED_WEIGHT 10142 { 10143 .name = "weight", 10144 .flags = CFTYPE_NOT_ON_ROOT, 10145 .read_u64 = cpu_weight_read_u64, 10146 .write_u64 = cpu_weight_write_u64, 10147 }, 10148 { 10149 .name = "weight.nice", 10150 .flags = CFTYPE_NOT_ON_ROOT, 10151 .read_s64 = cpu_weight_nice_read_s64, 10152 .write_s64 = cpu_weight_nice_write_s64, 10153 }, 10154 { 10155 .name = "idle", 10156 .flags = CFTYPE_NOT_ON_ROOT, 10157 .read_s64 = cpu_idle_read_s64, 10158 .write_s64 = cpu_idle_write_s64, 10159 }, 10160#endif 10161#ifdef CONFIG_GROUP_SCHED_BANDWIDTH 10162 { 10163 .name = "max", 10164 .flags = CFTYPE_NOT_ON_ROOT, 10165 .seq_show = cpu_max_show, 10166 .write = cpu_max_write, 10167 }, 10168 { 10169 .name = "max.burst", 10170 .flags = CFTYPE_NOT_ON_ROOT, 10171 .read_u64 = cpu_burst_read_u64, 10172 .write_u64 = cpu_burst_write_u64, 10173 }, 10174#endif /* CONFIG_CFS_BANDWIDTH */ 10175#ifdef CONFIG_UCLAMP_TASK_GROUP 10176 { 10177 .name = "uclamp.min", 10178 .flags = CFTYPE_NOT_ON_ROOT, 10179 .seq_show = cpu_uclamp_min_show, 10180 .write = cpu_uclamp_min_write, 10181 }, 10182 { 10183 .name = "uclamp.max", 10184 .flags = CFTYPE_NOT_ON_ROOT, 10185 .seq_show = cpu_uclamp_max_show, 10186 .write = cpu_uclamp_max_write, 10187 }, 10188#endif /* CONFIG_UCLAMP_TASK_GROUP */ 10189 { } /* terminate */ 10190}; 10191 10192struct cgroup_subsys cpu_cgrp_subsys = { 10193 .css_alloc = cpu_cgroup_css_alloc, 10194 .css_online = cpu_cgroup_css_online, 10195 .css_offline = cpu_cgroup_css_offline, 10196 .css_released = cpu_cgroup_css_released, 10197 .css_free = cpu_cgroup_css_free, 10198 .css_extra_stat_show = cpu_extra_stat_show, 10199 .css_local_stat_show = cpu_local_stat_show, 10200 .can_attach = cpu_cgroup_can_attach, 10201 .attach = cpu_cgroup_attach, 10202 .cancel_attach = cpu_cgroup_cancel_attach, 10203 .legacy_cftypes = cpu_legacy_files, 10204 .dfl_cftypes = cpu_files, 10205 .early_init = true, 10206 .threaded = true, 10207}; 10208 10209#endif /* CONFIG_CGROUP_SCHED */ 10210 10211void dump_cpu_task(int cpu) 10212{ 10213 if (in_hardirq() && cpu == smp_processor_id()) { 10214 struct pt_regs *regs; 10215 10216 regs = get_irq_regs(); 10217 if (regs) { 10218 show_regs(regs); 10219 return; 10220 } 10221 } 10222 10223 if (trigger_single_cpu_backtrace(cpu)) 10224 return; 10225 10226 pr_info("Task dump for CPU %d:\n", cpu); 10227 sched_show_task(cpu_curr(cpu)); 10228} 10229 10230/* 10231 * Nice levels are multiplicative, with a gentle 10% change for every 10232 * nice level changed. I.e. when a CPU-bound task goes from nice 0 to 10233 * nice 1, it will get ~10% less CPU time than another CPU-bound task 10234 * that remained on nice 0. 10235 * 10236 * The "10% effect" is relative and cumulative: from _any_ nice level, 10237 * if you go up 1 level, it's -10% CPU usage, if you go down 1 level 10238 * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. 10239 * If a task goes up by ~10% and another task goes down by ~10% then 10240 * the relative distance between them is ~25%.) 10241 */ 10242const int sched_prio_to_weight[40] = { 10243 /* -20 */ 88761, 71755, 56483, 46273, 36291, 10244 /* -15 */ 29154, 23254, 18705, 14949, 11916, 10245 /* -10 */ 9548, 7620, 6100, 4904, 3906, 10246 /* -5 */ 3121, 2501, 1991, 1586, 1277, 10247 /* 0 */ 1024, 820, 655, 526, 423, 10248 /* 5 */ 335, 272, 215, 172, 137, 10249 /* 10 */ 110, 87, 70, 56, 45, 10250 /* 15 */ 36, 29, 23, 18, 15, 10251}; 10252 10253/* 10254 * Inverse (2^32/x) values of the sched_prio_to_weight[] array, pre-calculated. 10255 * 10256 * In cases where the weight does not change often, we can use the 10257 * pre-calculated inverse to speed up arithmetics by turning divisions 10258 * into multiplications: 10259 */ 10260const u32 sched_prio_to_wmult[40] = { 10261 /* -20 */ 48388, 59856, 76040, 92818, 118348, 10262 /* -15 */ 147320, 184698, 229616, 287308, 360437, 10263 /* -10 */ 449829, 563644, 704093, 875809, 1099582, 10264 /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, 10265 /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, 10266 /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, 10267 /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, 10268 /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, 10269}; 10270 10271void call_trace_sched_update_nr_running(struct rq *rq, int count) 10272{ 10273 trace_sched_update_nr_running_tp(rq, count); 10274} 10275 10276#ifdef CONFIG_SCHED_MM_CID 10277/* 10278 * Concurrency IDentifier management 10279 * 10280 * Serialization rules: 10281 * 10282 * mm::mm_cid::mutex: Serializes fork() and exit() and therefore 10283 * protects mm::mm_cid::users and mode switch 10284 * transitions 10285 * 10286 * mm::mm_cid::lock: Serializes mm_update_max_cids() and 10287 * mm_update_cpus_allowed(). Nests in mm_cid::mutex 10288 * and runqueue lock. 10289 * 10290 * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks 10291 * and can only be modified with atomic operations. 10292 * 10293 * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue 10294 * lock. 10295 * 10296 * CID ownership: 10297 * 10298 * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or 10299 * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the 10300 * MM_CID_ONCPU bit set. 10301 * 10302 * During the transition of ownership mode, the MM_CID_TRANSIT bit is set 10303 * on the CIDs. When this bit is set the tasks drop the CID back into the 10304 * pool when scheduling out. 10305 * 10306 * Both bits (ONCPU and TRANSIT) are filtered out by task_cid() when the 10307 * CID is actually handed over to user space in the RSEQ memory. 10308 * 10309 * Mode switching: 10310 * 10311 * The ownership mode is per process and stored in mm:mm_cid::mode with the 10312 * following possible states: 10313 * 10314 * 0: Per task ownership 10315 * 0 | MM_CID_TRANSIT: Transition from per CPU to per task 10316 * MM_CID_ONCPU: Per CPU ownership 10317 * MM_CID_ONCPU | MM_CID_TRANSIT: Transition from per task to per CPU 10318 * 10319 * All transitions of ownership mode happen in two phases: 10320 * 10321 * 1) mm:mm_cid::mode has the MM_CID_TRANSIT bit set. This is OR'ed on the 10322 * CIDs and denotes that the CID is only temporarily owned by a 10323 * task. When the task schedules out it drops the CID back into the 10324 * pool if this bit is set. 10325 * 10326 * 2) The initiating context walks the per CPU space or the tasks to fixup 10327 * or drop the CIDs and after completion it clears MM_CID_TRANSIT in 10328 * mm:mm_cid::mode. After that point the CIDs are strictly task or CPU 10329 * owned again. 10330 * 10331 * This two phase transition is required to prevent CID space exhaustion 10332 * during the transition as a direct transfer of ownership would fail: 10333 * 10334 * - On task to CPU mode switch if a task is scheduled in on one CPU and 10335 * then migrated to another CPU before the fixup freed enough per task 10336 * CIDs. 10337 * 10338 * - On CPU to task mode switch if two tasks are scheduled in on the same 10339 * CPU before the fixup freed per CPU CIDs. 10340 * 10341 * Both scenarios can result in a live lock because sched_in() is invoked 10342 * with runqueue lock held and loops in search of a CID and the fixup 10343 * thread can't make progress freeing them up because it is stuck on the 10344 * same runqueue lock. 10345 * 10346 * While MM_CID_TRANSIT is active during the transition phase the MM_CID 10347 * bitmap can be contended, but that's a temporary contention bound to the 10348 * transition period. After that everything goes back into steady state and 10349 * nothing except fork() and exit() will touch the bitmap. This is an 10350 * acceptable tradeoff as it completely avoids complex serialization, 10351 * memory barriers and atomic operations for the common case. 10352 * 10353 * Aside of that this mechanism also ensures RT compability: 10354 * 10355 * - The task which runs the fixup is fully preemptible except for the 10356 * short runqueue lock held sections. 10357 * 10358 * - The transient impact of the bitmap contention is only problematic 10359 * when there is a thundering herd scenario of tasks scheduling in and 10360 * out concurrently. There is not much which can be done about that 10361 * except for avoiding mode switching by a proper overall system 10362 * configuration. 10363 * 10364 * Switching to per CPU mode happens when the user count becomes greater 10365 * than the maximum number of CIDs, which is calculated by: 10366 * 10367 * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users); 10368 * max_cids = min(1.25 * opt_cids, num_possible_cpus()); 10369 * 10370 * The +25% allowance is useful for tight CPU masks in scenarios where only 10371 * a few threads are created and destroyed to avoid frequent mode 10372 * switches. Though this allowance shrinks, the closer opt_cids becomes to 10373 * num_possible_cpus(), which is the (unfortunate) hard ABI limit. 10374 * 10375 * At the point of switching to per CPU mode the new user is not yet 10376 * visible in the system, so the task which initiated the fork() runs the 10377 * fixup function. mm_cid_fixup_tasks_to_cpu() walks the thread list and 10378 * either marks each task owned CID with MM_CID_TRANSIT if the task is 10379 * running on a CPU or drops it into the CID pool if a task is not on a 10380 * CPU. Tasks which schedule in before the task walk reaches them do the 10381 * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() 10382 * completes it is guaranteed that no task related to that MM owns a CID 10383 * anymore. 10384 * 10385 * Switching back to task mode happens when the user count goes below the 10386 * threshold which was recorded on the per CPU mode switch: 10387 * 10388 * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2); 10389 * 10390 * This threshold is updated when a affinity change increases the number of 10391 * allowed CPUs for the MM, which might cause a switch back to per task 10392 * mode. 10393 * 10394 * If the switch back was initiated by a exiting task, then that task runs 10395 * the fixup function. If it was initiated by a affinity change, then it's 10396 * run either in the deferred update function in context of a workqueue or 10397 * by a task which forks a new one or by a task which exits. Whatever 10398 * happens first. mm_cid_fixup_cpus_to_task() walks through the possible 10399 * CPUs and either marks the CPU owned CIDs with MM_CID_TRANSIT if a 10400 * related task is running on the CPU or drops it into the pool. Tasks 10401 * which are scheduled in before the fixup covered them do the handover 10402 * themself. When mm_cid_fixup_cpus_to_tasks() completes it is guaranteed 10403 * that no CID related to that MM is owned by a CPU anymore. 10404 */ 10405 10406/* 10407 * Update the CID range properties when the constraints change. Invoked via 10408 * fork(), exit() and affinity changes 10409 */ 10410static void __mm_update_max_cids(struct mm_mm_cid *mc) 10411{ 10412 unsigned int opt_cids, max_cids; 10413 10414 /* Calculate the new optimal constraint */ 10415 opt_cids = min(mc->nr_cpus_allowed, mc->users); 10416 10417 /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */ 10418 max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus()); 10419 WRITE_ONCE(mc->max_cids, max_cids); 10420} 10421 10422static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc) 10423{ 10424 unsigned int opt_cids; 10425 10426 opt_cids = min(mc->nr_cpus_allowed, mc->users); 10427 /* Has to be at least 1 because 0 indicates PCPU mode off */ 10428 return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1); 10429} 10430 10431static bool mm_update_max_cids(struct mm_struct *mm) 10432{ 10433 struct mm_mm_cid *mc = &mm->mm_cid; 10434 bool percpu = cid_on_cpu(mc->mode); 10435 10436 lockdep_assert_held(&mm->mm_cid.lock); 10437 10438 /* Clear deferred mode switch flag. A change is handled by the caller */ 10439 mc->update_deferred = false; 10440 __mm_update_max_cids(mc); 10441 10442 /* Check whether owner mode must be changed */ 10443 if (!percpu) { 10444 /* Enable per CPU mode when the number of users is above max_cids */ 10445 if (mc->users > mc->max_cids) 10446 mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc); 10447 } else { 10448 /* Switch back to per task if user count under threshold */ 10449 if (mc->users < mc->pcpu_thrs) 10450 mc->pcpu_thrs = 0; 10451 } 10452 10453 /* Mode change required? */ 10454 if (percpu == !!mc->pcpu_thrs) 10455 return false; 10456 10457 /* Flip the mode and set the transition flag to bridge the transfer */ 10458 WRITE_ONCE(mc->mode, mc->mode ^ (MM_CID_TRANSIT | MM_CID_ONCPU)); 10459 /* 10460 * Order the store against the subsequent fixups so that 10461 * acquire(rq::lock) cannot be reordered by the CPU before the 10462 * store. 10463 */ 10464 smp_mb(); 10465 return true; 10466} 10467 10468static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) 10469{ 10470 struct cpumask *mm_allowed; 10471 struct mm_mm_cid *mc; 10472 unsigned int weight; 10473 10474 if (!mm || !READ_ONCE(mm->mm_cid.users)) 10475 return; 10476 /* 10477 * mm::mm_cid::mm_cpus_allowed is the superset of each threads 10478 * allowed CPUs mask which means it can only grow. 10479 */ 10480 mc = &mm->mm_cid; 10481 guard(raw_spinlock)(&mc->lock); 10482 mm_allowed = mm_cpus_allowed(mm); 10483 weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk); 10484 if (weight == mc->nr_cpus_allowed) 10485 return; 10486 10487 WRITE_ONCE(mc->nr_cpus_allowed, weight); 10488 __mm_update_max_cids(mc); 10489 if (!cid_on_cpu(mc->mode)) 10490 return; 10491 10492 /* Adjust the threshold to the wider set */ 10493 mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc); 10494 /* Switch back to per task mode? */ 10495 if (mc->users >= mc->pcpu_thrs) 10496 return; 10497 10498 /* Don't queue twice */ 10499 if (mc->update_deferred) 10500 return; 10501 10502 /* Queue the irq work, which schedules the real work */ 10503 mc->update_deferred = true; 10504 irq_work_queue(&mc->irq_work); 10505} 10506 10507static inline void mm_cid_complete_transit(struct mm_struct *mm, unsigned int mode) 10508{ 10509 /* 10510 * Ensure that the store removing the TRANSIT bit cannot be 10511 * reordered by the CPU before the fixups have been completed. 10512 */ 10513 smp_mb(); 10514 WRITE_ONCE(mm->mm_cid.mode, mode); 10515} 10516 10517static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp) 10518{ 10519 if (cid_on_cpu(t->mm_cid.cid)) { 10520 unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid); 10521 10522 t->mm_cid.cid = cid_to_transit_cid(cid); 10523 pcp->cid = t->mm_cid.cid; 10524 } 10525} 10526 10527static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm) 10528{ 10529 unsigned int cpu; 10530 10531 /* Walk the CPUs and fixup all stale CIDs */ 10532 for_each_possible_cpu(cpu) { 10533 struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu); 10534 struct rq *rq = cpu_rq(cpu); 10535 10536 /* Remote access to mm::mm_cid::pcpu requires rq_lock */ 10537 guard(rq_lock_irq)(rq); 10538 /* Is the CID still owned by the CPU? */ 10539 if (cid_on_cpu(pcp->cid)) { 10540 /* 10541 * If rq->curr has @mm, transfer it with the 10542 * transition bit set. Otherwise drop it. 10543 */ 10544 if (rq->curr->mm == mm && rq->curr->mm_cid.active) 10545 mm_cid_transit_to_task(rq->curr, pcp); 10546 else 10547 mm_drop_cid_on_cpu(mm, pcp); 10548 10549 } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) { 10550 unsigned int cid = rq->curr->mm_cid.cid; 10551 10552 /* Ensure it has the transition bit set */ 10553 if (!cid_in_transit(cid)) { 10554 cid = cid_to_transit_cid(cid); 10555 rq->curr->mm_cid.cid = cid; 10556 pcp->cid = cid; 10557 } 10558 } 10559 } 10560 mm_cid_complete_transit(mm, 0); 10561} 10562 10563static inline void mm_cid_transit_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp) 10564{ 10565 if (cid_on_task(t->mm_cid.cid)) { 10566 t->mm_cid.cid = cid_to_transit_cid(t->mm_cid.cid); 10567 pcp->cid = t->mm_cid.cid; 10568 } 10569} 10570 10571static void mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm) 10572{ 10573 /* Remote access to mm::mm_cid::pcpu requires rq_lock */ 10574 guard(task_rq_lock)(t); 10575 if (cid_on_task(t->mm_cid.cid)) { 10576 /* If running on the CPU, put the CID in transit mode, otherwise drop it */ 10577 if (task_rq(t)->curr == t) 10578 mm_cid_transit_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t))); 10579 else 10580 mm_unset_cid_on_task(t); 10581 } 10582} 10583 10584static void mm_cid_fixup_tasks_to_cpus(void) 10585{ 10586 struct mm_struct *mm = current->mm; 10587 struct task_struct *t; 10588 10589 lockdep_assert_held(&mm->mm_cid.mutex); 10590 10591 hlist_for_each_entry(t, &mm->mm_cid.user_list, mm_cid.node) { 10592 /* Current has already transferred before invoking the fixup. */ 10593 if (t != current) 10594 mm_cid_fixup_task_to_cpu(t, mm); 10595 } 10596 10597 mm_cid_complete_transit(mm, MM_CID_ONCPU); 10598} 10599 10600static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm) 10601{ 10602 lockdep_assert_held(&mm->mm_cid.lock); 10603 10604 t->mm_cid.active = 1; 10605 hlist_add_head(&t->mm_cid.node, &mm->mm_cid.user_list); 10606 mm->mm_cid.users++; 10607 return mm_update_max_cids(mm); 10608} 10609 10610static void sched_mm_cid_fork(struct task_struct *t) 10611{ 10612 struct mm_struct *mm = t->mm; 10613 bool percpu; 10614 10615 if (!mm) 10616 return; 10617 10618 WARN_ON_ONCE(t->mm_cid.cid != MM_CID_UNSET); 10619 10620 guard(mutex)(&mm->mm_cid.mutex); 10621 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) { 10622 struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu); 10623 10624 /* First user ? */ 10625 if (!mm->mm_cid.users) { 10626 sched_mm_cid_add_user(t, mm); 10627 t->mm_cid.cid = mm_get_cid(mm); 10628 /* Required for execve() */ 10629 pcp->cid = t->mm_cid.cid; 10630 return; 10631 } 10632 10633 if (!sched_mm_cid_add_user(t, mm)) { 10634 if (!cid_on_cpu(mm->mm_cid.mode)) 10635 t->mm_cid.cid = mm_get_cid(mm); 10636 return; 10637 } 10638 10639 /* Handle the mode change and transfer current's CID */ 10640 percpu = cid_on_cpu(mm->mm_cid.mode); 10641 if (!percpu) 10642 mm_cid_transit_to_task(current, pcp); 10643 else 10644 mm_cid_transit_to_cpu(current, pcp); 10645 } 10646 10647 if (percpu) { 10648 mm_cid_fixup_tasks_to_cpus(); 10649 } else { 10650 mm_cid_fixup_cpus_to_tasks(mm); 10651 t->mm_cid.cid = mm_get_cid(mm); 10652 } 10653} 10654 10655static bool sched_mm_cid_remove_user(struct task_struct *t) 10656{ 10657 lockdep_assert_held(&t->mm->mm_cid.lock); 10658 10659 t->mm_cid.active = 0; 10660 /* Clear the transition bit */ 10661 t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid); 10662 mm_unset_cid_on_task(t); 10663 hlist_del_init(&t->mm_cid.node); 10664 t->mm->mm_cid.users--; 10665 return mm_update_max_cids(t->mm); 10666} 10667 10668static bool __sched_mm_cid_exit(struct task_struct *t) 10669{ 10670 struct mm_struct *mm = t->mm; 10671 10672 if (!sched_mm_cid_remove_user(t)) 10673 return false; 10674 /* 10675 * Contrary to fork() this only deals with a switch back to per 10676 * task mode either because the above decreased users or an 10677 * affinity change increased the number of allowed CPUs and the 10678 * deferred fixup did not run yet. 10679 */ 10680 if (WARN_ON_ONCE(cid_on_cpu(mm->mm_cid.mode))) 10681 return false; 10682 /* 10683 * A failed fork(2) cleanup never gets here, so @current must have 10684 * the same MM as @t. That's true for exit() and the failed 10685 * pthread_create() cleanup case. 10686 */ 10687 if (WARN_ON_ONCE(current->mm != mm)) 10688 return false; 10689 return true; 10690} 10691 10692/* 10693 * When a task exits, the MM CID held by the task is not longer required as 10694 * the task cannot return to user space. 10695 */ 10696void sched_mm_cid_exit(struct task_struct *t) 10697{ 10698 struct mm_struct *mm = t->mm; 10699 10700 if (!mm || !t->mm_cid.active) 10701 return; 10702 /* 10703 * Ensure that only one instance is doing MM CID operations within 10704 * a MM. The common case is uncontended. The rare fixup case adds 10705 * some overhead. 10706 */ 10707 scoped_guard(mutex, &mm->mm_cid.mutex) { 10708 /* mm_cid::mutex is sufficient to protect mm_cid::users */ 10709 if (likely(mm->mm_cid.users > 1)) { 10710 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) { 10711 if (!__sched_mm_cid_exit(t)) 10712 return; 10713 /* 10714 * Mode change. The task has the CID unset 10715 * already and dealt with an eventually set 10716 * TRANSIT bit. If the CID is owned by the CPU 10717 * then drop it. 10718 */ 10719 mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu)); 10720 } 10721 mm_cid_fixup_cpus_to_tasks(mm); 10722 return; 10723 } 10724 /* Last user */ 10725 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) { 10726 /* Required across execve() */ 10727 if (t == current) 10728 mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu)); 10729 /* Ignore mode change. There is nothing to do. */ 10730 sched_mm_cid_remove_user(t); 10731 } 10732 } 10733 10734 /* 10735 * As this is the last user (execve(), process exit or failed 10736 * fork(2)) there is no concurrency anymore. 10737 * 10738 * Synchronize eventually pending work to ensure that there are no 10739 * dangling references left. @t->mm_cid.users is zero so nothing 10740 * can queue this work anymore. 10741 */ 10742 irq_work_sync(&mm->mm_cid.irq_work); 10743 cancel_work_sync(&mm->mm_cid.work); 10744} 10745 10746/* Deactivate MM CID allocation across execve() */ 10747void sched_mm_cid_before_execve(struct task_struct *t) 10748{ 10749 sched_mm_cid_exit(t); 10750} 10751 10752/* Reactivate MM CID after execve() */ 10753void sched_mm_cid_after_execve(struct task_struct *t) 10754{ 10755 if (t->mm) 10756 sched_mm_cid_fork(t); 10757} 10758 10759static void mm_cid_work_fn(struct work_struct *work) 10760{ 10761 struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work); 10762 10763 guard(mutex)(&mm->mm_cid.mutex); 10764 /* Did the last user task exit already? */ 10765 if (!mm->mm_cid.users) 10766 return; 10767 10768 scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) { 10769 /* Have fork() or exit() handled it already? */ 10770 if (!mm->mm_cid.update_deferred) 10771 return; 10772 /* This clears mm_cid::update_deferred */ 10773 if (!mm_update_max_cids(mm)) 10774 return; 10775 /* Affinity changes can only switch back to task mode */ 10776 if (WARN_ON_ONCE(cid_on_cpu(mm->mm_cid.mode))) 10777 return; 10778 } 10779 mm_cid_fixup_cpus_to_tasks(mm); 10780} 10781 10782static void mm_cid_irq_work(struct irq_work *work) 10783{ 10784 struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work); 10785 10786 /* 10787 * Needs to be unconditional because mm_cid::lock cannot be held 10788 * when scheduling work as mm_update_cpus_allowed() nests inside 10789 * rq::lock and schedule_work() might end up in wakeup... 10790 */ 10791 schedule_work(&mm->mm_cid.work); 10792} 10793 10794void mm_init_cid(struct mm_struct *mm, struct task_struct *p) 10795{ 10796 mm->mm_cid.max_cids = 0; 10797 mm->mm_cid.mode = 0; 10798 mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed; 10799 mm->mm_cid.users = 0; 10800 mm->mm_cid.pcpu_thrs = 0; 10801 mm->mm_cid.update_deferred = 0; 10802 raw_spin_lock_init(&mm->mm_cid.lock); 10803 mutex_init(&mm->mm_cid.mutex); 10804 mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work); 10805 INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn); 10806 INIT_HLIST_HEAD(&mm->mm_cid.user_list); 10807 cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask); 10808 bitmap_zero(mm_cidmask(mm), num_possible_cpus()); 10809} 10810#else /* CONFIG_SCHED_MM_CID */ 10811static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { } 10812static inline void sched_mm_cid_fork(struct task_struct *t) { } 10813#endif /* !CONFIG_SCHED_MM_CID */ 10814 10815static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx); 10816 10817struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int flags) 10818{ 10819 struct sched_change_ctx *ctx = this_cpu_ptr(&sched_change_ctx); 10820 struct rq *rq = task_rq(p); 10821 10822 /* 10823 * Must exclusively use matched flags since this is both dequeue and 10824 * enqueue. 10825 */ 10826 WARN_ON_ONCE(flags & 0xFFFF0000); 10827 10828 lockdep_assert_rq_held(rq); 10829 10830 if (!(flags & DEQUEUE_NOCLOCK)) { 10831 update_rq_clock(rq); 10832 flags |= DEQUEUE_NOCLOCK; 10833 } 10834 10835 if (flags & DEQUEUE_CLASS) { 10836 if (p->sched_class->switching_from) 10837 p->sched_class->switching_from(rq, p); 10838 } 10839 10840 *ctx = (struct sched_change_ctx){ 10841 .p = p, 10842 .flags = flags, 10843 .queued = task_on_rq_queued(p), 10844 .running = task_current_donor(rq, p), 10845 }; 10846 10847 if (!(flags & DEQUEUE_CLASS)) { 10848 if (p->sched_class->get_prio) 10849 ctx->prio = p->sched_class->get_prio(rq, p); 10850 else 10851 ctx->prio = p->prio; 10852 } 10853 10854 if (ctx->queued) 10855 dequeue_task(rq, p, flags); 10856 if (ctx->running) 10857 put_prev_task(rq, p); 10858 10859 if ((flags & DEQUEUE_CLASS) && p->sched_class->switched_from) 10860 p->sched_class->switched_from(rq, p); 10861 10862 return ctx; 10863} 10864 10865void sched_change_end(struct sched_change_ctx *ctx) 10866{ 10867 struct task_struct *p = ctx->p; 10868 struct rq *rq = task_rq(p); 10869 10870 lockdep_assert_rq_held(rq); 10871 10872 if ((ctx->flags & ENQUEUE_CLASS) && p->sched_class->switching_to) 10873 p->sched_class->switching_to(rq, p); 10874 10875 if (ctx->queued) 10876 enqueue_task(rq, p, ctx->flags); 10877 if (ctx->running) 10878 set_next_task(rq, p); 10879 10880 if (ctx->flags & ENQUEUE_CLASS) { 10881 if (p->sched_class->switched_to) 10882 p->sched_class->switched_to(rq, p); 10883 } else { 10884 p->sched_class->prio_changed(rq, p, ctx->prio); 10885 } 10886}
~/
Read-only file preview.