FILE
sched.h
File preview with syntax highlighting.
linux-6.19.14/include/linux/sched.h
1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_SCHED_H 3#define _LINUX_SCHED_H 4 5/* 6 * Define 'struct task_struct' and provide the main scheduler 7 * APIs (schedule(), wakeup variants, etc.) 8 */ 9 10#include <uapi/linux/sched.h> 11 12#include <asm/current.h> 13#include <asm/processor.h> 14#include <linux/thread_info.h> 15#include <linux/preempt.h> 16#include <linux/cpumask_types.h> 17 18#include <linux/cache.h> 19#include <linux/irqflags_types.h> 20#include <linux/smp_types.h> 21#include <linux/pid_types.h> 22#include <linux/sem_types.h> 23#include <linux/shm.h> 24#include <linux/kmsan_types.h> 25#include <linux/mutex_types.h> 26#include <linux/plist_types.h> 27#include <linux/hrtimer_types.h> 28#include <linux/timer_types.h> 29#include <linux/seccomp_types.h> 30#include <linux/nodemask_types.h> 31#include <linux/refcount_types.h> 32#include <linux/resource.h> 33#include <linux/latencytop.h> 34#include <linux/sched/prio.h> 35#include <linux/sched/types.h> 36#include <linux/signal_types.h> 37#include <linux/spinlock.h> 38#include <linux/syscall_user_dispatch_types.h> 39#include <linux/mm_types_task.h> 40#include <linux/netdevice_xmit.h> 41#include <linux/task_io_accounting.h> 42#include <linux/posix-timers_types.h> 43#include <linux/restart_block.h> 44#include <linux/rseq_types.h> 45#include <linux/seqlock_types.h> 46#include <linux/kcsan.h> 47#include <linux/rv.h> 48#include <linux/uidgid_types.h> 49#include <linux/tracepoint-defs.h> 50#include <linux/unwind_deferred_types.h> 51#include <asm/kmap_size.h> 52#ifndef COMPILE_OFFSETS 53#include <generated/rq-offsets.h> 54#endif 55 56/* task_struct member predeclarations (sorted alphabetically): */ 57struct audit_context; 58struct bio_list; 59struct blk_plug; 60struct bpf_local_storage; 61struct bpf_run_ctx; 62struct bpf_net_context; 63struct capture_control; 64struct cfs_rq; 65struct fs_struct; 66struct futex_pi_state; 67struct io_context; 68struct io_uring_task; 69struct mempolicy; 70struct nameidata; 71struct nsproxy; 72struct perf_event_context; 73struct perf_ctx_data; 74struct pid_namespace; 75struct pipe_inode_info; 76struct rcu_node; 77struct reclaim_state; 78struct robust_list_head; 79struct root_domain; 80struct rq; 81struct sched_attr; 82struct sched_dl_entity; 83struct seq_file; 84struct sighand_struct; 85struct signal_struct; 86struct task_delay_info; 87struct task_group; 88struct task_struct; 89struct user_event_mm; 90 91#include <linux/sched/ext.h> 92 93/* 94 * Task state bitmask. NOTE! These bits are also 95 * encoded in fs/proc/array.c: get_task_state(). 96 * 97 * We have two separate sets of flags: task->__state 98 * is about runnability, while task->exit_state are 99 * about the task exiting. Confusing, but this way 100 * modifying one set can't modify the other one by 101 * mistake. 102 */ 103 104/* Used in tsk->__state: */ 105#define TASK_RUNNING 0x00000000 106#define TASK_INTERRUPTIBLE 0x00000001 107#define TASK_UNINTERRUPTIBLE 0x00000002 108#define __TASK_STOPPED 0x00000004 109#define __TASK_TRACED 0x00000008 110/* Used in tsk->exit_state: */ 111#define EXIT_DEAD 0x00000010 112#define EXIT_ZOMBIE 0x00000020 113#define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) 114/* Used in tsk->__state again: */ 115#define TASK_PARKED 0x00000040 116#define TASK_DEAD 0x00000080 117#define TASK_WAKEKILL 0x00000100 118#define TASK_WAKING 0x00000200 119#define TASK_NOLOAD 0x00000400 120#define TASK_NEW 0x00000800 121#define TASK_RTLOCK_WAIT 0x00001000 122#define TASK_FREEZABLE 0x00002000 123#define __TASK_FREEZABLE_UNSAFE (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP)) 124#define TASK_FROZEN 0x00008000 125#define TASK_STATE_MAX 0x00010000 126 127#define TASK_ANY (TASK_STATE_MAX-1) 128 129/* 130 * DO NOT ADD ANY NEW USERS ! 131 */ 132#define TASK_FREEZABLE_UNSAFE (TASK_FREEZABLE | __TASK_FREEZABLE_UNSAFE) 133 134/* Convenience macros for the sake of set_current_state: */ 135#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) 136#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED) 137#define TASK_TRACED __TASK_TRACED 138 139#define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD) 140 141/* Convenience macros for the sake of wake_up(): */ 142#define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) 143 144/* get_task_state(): */ 145#define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ 146 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ 147 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ 148 TASK_PARKED) 149 150#define task_is_running(task) (READ_ONCE((task)->__state) == TASK_RUNNING) 151 152#define task_is_traced(task) ((READ_ONCE(task->jobctl) & JOBCTL_TRACED) != 0) 153#define task_is_stopped(task) ((READ_ONCE(task->jobctl) & JOBCTL_STOPPED) != 0) 154#define task_is_stopped_or_traced(task) ((READ_ONCE(task->jobctl) & (JOBCTL_STOPPED | JOBCTL_TRACED)) != 0) 155 156/* 157 * Special states are those that do not use the normal wait-loop pattern. See 158 * the comment with set_special_state(). 159 */ 160#define is_special_task_state(state) \ 161 ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | \ 162 TASK_DEAD | TASK_FROZEN)) 163 164#ifdef CONFIG_DEBUG_ATOMIC_SLEEP 165# define debug_normal_state_change(state_value) \ 166 do { \ 167 WARN_ON_ONCE(is_special_task_state(state_value)); \ 168 current->task_state_change = _THIS_IP_; \ 169 } while (0) 170 171# define debug_special_state_change(state_value) \ 172 do { \ 173 WARN_ON_ONCE(!is_special_task_state(state_value)); \ 174 current->task_state_change = _THIS_IP_; \ 175 } while (0) 176 177# define debug_rtlock_wait_set_state() \ 178 do { \ 179 current->saved_state_change = current->task_state_change;\ 180 current->task_state_change = _THIS_IP_; \ 181 } while (0) 182 183# define debug_rtlock_wait_restore_state() \ 184 do { \ 185 current->task_state_change = current->saved_state_change;\ 186 } while (0) 187 188#else 189# define debug_normal_state_change(cond) do { } while (0) 190# define debug_special_state_change(cond) do { } while (0) 191# define debug_rtlock_wait_set_state() do { } while (0) 192# define debug_rtlock_wait_restore_state() do { } while (0) 193#endif 194 195#define trace_set_current_state(state_value) \ 196 do { \ 197 if (tracepoint_enabled(sched_set_state_tp)) \ 198 __trace_set_current_state(state_value); \ 199 } while (0) 200 201/* 202 * set_current_state() includes a barrier so that the write of current->__state 203 * is correctly serialised wrt the caller's subsequent test of whether to 204 * actually sleep: 205 * 206 * for (;;) { 207 * set_current_state(TASK_UNINTERRUPTIBLE); 208 * if (CONDITION) 209 * break; 210 * 211 * schedule(); 212 * } 213 * __set_current_state(TASK_RUNNING); 214 * 215 * If the caller does not need such serialisation (because, for instance, the 216 * CONDITION test and condition change and wakeup are under the same lock) then 217 * use __set_current_state(). 218 * 219 * The above is typically ordered against the wakeup, which does: 220 * 221 * CONDITION = 1; 222 * wake_up_state(p, TASK_UNINTERRUPTIBLE); 223 * 224 * where wake_up_state()/try_to_wake_up() executes a full memory barrier before 225 * accessing p->__state. 226 * 227 * Wakeup will do: if (@state & p->__state) p->__state = TASK_RUNNING, that is, 228 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a 229 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING). 230 * 231 * However, with slightly different timing the wakeup TASK_RUNNING store can 232 * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not 233 * a problem either because that will result in one extra go around the loop 234 * and our @cond test will save the day. 235 * 236 * Also see the comments of try_to_wake_up(). 237 */ 238#define __set_current_state(state_value) \ 239 do { \ 240 debug_normal_state_change((state_value)); \ 241 trace_set_current_state(state_value); \ 242 WRITE_ONCE(current->__state, (state_value)); \ 243 } while (0) 244 245#define set_current_state(state_value) \ 246 do { \ 247 debug_normal_state_change((state_value)); \ 248 trace_set_current_state(state_value); \ 249 smp_store_mb(current->__state, (state_value)); \ 250 } while (0) 251 252/* 253 * set_special_state() should be used for those states when the blocking task 254 * can not use the regular condition based wait-loop. In that case we must 255 * serialize against wakeups such that any possible in-flight TASK_RUNNING 256 * stores will not collide with our state change. 257 */ 258#define set_special_state(state_value) \ 259 do { \ 260 unsigned long flags; /* may shadow */ \ 261 \ 262 raw_spin_lock_irqsave(¤t->pi_lock, flags); \ 263 debug_special_state_change((state_value)); \ 264 trace_set_current_state(state_value); \ 265 WRITE_ONCE(current->__state, (state_value)); \ 266 raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \ 267 } while (0) 268 269/* 270 * PREEMPT_RT specific variants for "sleeping" spin/rwlocks 271 * 272 * RT's spin/rwlock substitutions are state preserving. The state of the 273 * task when blocking on the lock is saved in task_struct::saved_state and 274 * restored after the lock has been acquired. These operations are 275 * serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT 276 * lock related wakeups while the task is blocked on the lock are 277 * redirected to operate on task_struct::saved_state to ensure that these 278 * are not dropped. On restore task_struct::saved_state is set to 279 * TASK_RUNNING so any wakeup attempt redirected to saved_state will fail. 280 * 281 * The lock operation looks like this: 282 * 283 * current_save_and_set_rtlock_wait_state(); 284 * for (;;) { 285 * if (try_lock()) 286 * break; 287 * raw_spin_unlock_irq(&lock->wait_lock); 288 * schedule_rtlock(); 289 * raw_spin_lock_irq(&lock->wait_lock); 290 * set_current_state(TASK_RTLOCK_WAIT); 291 * } 292 * current_restore_rtlock_saved_state(); 293 */ 294#define current_save_and_set_rtlock_wait_state() \ 295 do { \ 296 lockdep_assert_irqs_disabled(); \ 297 raw_spin_lock(¤t->pi_lock); \ 298 current->saved_state = current->__state; \ 299 debug_rtlock_wait_set_state(); \ 300 trace_set_current_state(TASK_RTLOCK_WAIT); \ 301 WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT); \ 302 raw_spin_unlock(¤t->pi_lock); \ 303 } while (0); 304 305#define current_restore_rtlock_saved_state() \ 306 do { \ 307 lockdep_assert_irqs_disabled(); \ 308 raw_spin_lock(¤t->pi_lock); \ 309 debug_rtlock_wait_restore_state(); \ 310 trace_set_current_state(current->saved_state); \ 311 WRITE_ONCE(current->__state, current->saved_state); \ 312 current->saved_state = TASK_RUNNING; \ 313 raw_spin_unlock(¤t->pi_lock); \ 314 } while (0); 315 316#define get_current_state() READ_ONCE(current->__state) 317 318/* 319 * Define the task command name length as enum, then it can be visible to 320 * BPF programs. 321 */ 322enum { 323 TASK_COMM_LEN = 16, 324}; 325 326extern void sched_tick(void); 327 328#define MAX_SCHEDULE_TIMEOUT LONG_MAX 329 330extern long schedule_timeout(long timeout); 331extern long schedule_timeout_interruptible(long timeout); 332extern long schedule_timeout_killable(long timeout); 333extern long schedule_timeout_uninterruptible(long timeout); 334extern long schedule_timeout_idle(long timeout); 335asmlinkage void schedule(void); 336extern void schedule_preempt_disabled(void); 337asmlinkage void preempt_schedule_irq(void); 338#ifdef CONFIG_PREEMPT_RT 339 extern void schedule_rtlock(void); 340#endif 341 342extern int __must_check io_schedule_prepare(void); 343extern void io_schedule_finish(int token); 344extern long io_schedule_timeout(long timeout); 345extern void io_schedule(void); 346 347/* wrapper functions to trace from this header file */ 348DECLARE_TRACEPOINT(sched_set_state_tp); 349extern void __trace_set_current_state(int state_value); 350DECLARE_TRACEPOINT(sched_set_need_resched_tp); 351extern void __trace_set_need_resched(struct task_struct *curr, int tif); 352 353/** 354 * struct prev_cputime - snapshot of system and user cputime 355 * @utime: time spent in user mode 356 * @stime: time spent in system mode 357 * @lock: protects the above two fields 358 * 359 * Stores previous user/system time values such that we can guarantee 360 * monotonicity. 361 */ 362struct prev_cputime { 363#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 364 u64 utime; 365 u64 stime; 366 raw_spinlock_t lock; 367#endif 368}; 369 370enum vtime_state { 371 /* Task is sleeping or running in a CPU with VTIME inactive: */ 372 VTIME_INACTIVE = 0, 373 /* Task is idle */ 374 VTIME_IDLE, 375 /* Task runs in kernelspace in a CPU with VTIME active: */ 376 VTIME_SYS, 377 /* Task runs in userspace in a CPU with VTIME active: */ 378 VTIME_USER, 379 /* Task runs as guests in a CPU with VTIME active: */ 380 VTIME_GUEST, 381}; 382 383struct vtime { 384 seqcount_t seqcount; 385 unsigned long long starttime; 386 enum vtime_state state; 387 unsigned int cpu; 388 u64 utime; 389 u64 stime; 390 u64 gtime; 391}; 392 393/* 394 * Utilization clamp constraints. 395 * @UCLAMP_MIN: Minimum utilization 396 * @UCLAMP_MAX: Maximum utilization 397 * @UCLAMP_CNT: Utilization clamp constraints count 398 */ 399enum uclamp_id { 400 UCLAMP_MIN = 0, 401 UCLAMP_MAX, 402 UCLAMP_CNT 403}; 404 405extern struct root_domain def_root_domain; 406extern struct mutex sched_domains_mutex; 407extern void sched_domains_mutex_lock(void); 408extern void sched_domains_mutex_unlock(void); 409 410struct sched_param { 411 int sched_priority; 412}; 413 414struct sched_info { 415#ifdef CONFIG_SCHED_INFO 416 /* Cumulative counters: */ 417 418 /* # of times we have run on this CPU: */ 419 unsigned long pcount; 420 421 /* Time spent waiting on a runqueue: */ 422 unsigned long long run_delay; 423 424 /* Max time spent waiting on a runqueue: */ 425 unsigned long long max_run_delay; 426 427 /* Min time spent waiting on a runqueue: */ 428 unsigned long long min_run_delay; 429 430 /* Timestamps: */ 431 432 /* When did we last run on a CPU? */ 433 unsigned long long last_arrival; 434 435 /* When were we last queued to run? */ 436 unsigned long long last_queued; 437 438#endif /* CONFIG_SCHED_INFO */ 439}; 440 441/* 442 * Integer metrics need fixed point arithmetic, e.g., sched/fair 443 * has a few: load, load_avg, util_avg, freq, and capacity. 444 * 445 * We define a basic fixed point arithmetic range, and then formalize 446 * all these metrics based on that basic range. 447 */ 448# define SCHED_FIXEDPOINT_SHIFT 10 449# define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT) 450 451/* Increase resolution of cpu_capacity calculations */ 452# define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT 453# define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) 454 455struct load_weight { 456 unsigned long weight; 457 u32 inv_weight; 458}; 459 460/* 461 * The load/runnable/util_avg accumulates an infinite geometric series 462 * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c). 463 * 464 * [load_avg definition] 465 * 466 * load_avg = runnable% * scale_load_down(load) 467 * 468 * [runnable_avg definition] 469 * 470 * runnable_avg = runnable% * SCHED_CAPACITY_SCALE 471 * 472 * [util_avg definition] 473 * 474 * util_avg = running% * SCHED_CAPACITY_SCALE 475 * 476 * where runnable% is the time ratio that a sched_entity is runnable and 477 * running% the time ratio that a sched_entity is running. 478 * 479 * For cfs_rq, they are the aggregated values of all runnable and blocked 480 * sched_entities. 481 * 482 * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU 483 * capacity scaling. The scaling is done through the rq_clock_pelt that is used 484 * for computing those signals (see update_rq_clock_pelt()) 485 * 486 * N.B., the above ratios (runnable% and running%) themselves are in the 487 * range of [0, 1]. To do fixed point arithmetics, we therefore scale them 488 * to as large a range as necessary. This is for example reflected by 489 * util_avg's SCHED_CAPACITY_SCALE. 490 * 491 * [Overflow issue] 492 * 493 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities 494 * with the highest load (=88761), always runnable on a single cfs_rq, 495 * and should not overflow as the number already hits PID_MAX_LIMIT. 496 * 497 * For all other cases (including 32-bit kernels), struct load_weight's 498 * weight will overflow first before we do, because: 499 * 500 * Max(load_avg) <= Max(load.weight) 501 * 502 * Then it is the load_weight's responsibility to consider overflow 503 * issues. 504 */ 505struct sched_avg { 506 u64 last_update_time; 507 u64 load_sum; 508 u64 runnable_sum; 509 u32 util_sum; 510 u32 period_contrib; 511 unsigned long load_avg; 512 unsigned long runnable_avg; 513 unsigned long util_avg; 514 unsigned int util_est; 515} ____cacheline_aligned; 516 517/* 518 * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg 519 * updates. When a task is dequeued, its util_est should not be updated if its 520 * util_avg has not been updated in the meantime. 521 * This information is mapped into the MSB bit of util_est at dequeue time. 522 * Since max value of util_est for a task is 1024 (PELT util_avg for a task) 523 * it is safe to use MSB. 524 */ 525#define UTIL_EST_WEIGHT_SHIFT 2 526#define UTIL_AVG_UNCHANGED 0x80000000 527 528struct sched_statistics { 529#ifdef CONFIG_SCHEDSTATS 530 u64 wait_start; 531 u64 wait_max; 532 u64 wait_count; 533 u64 wait_sum; 534 u64 iowait_count; 535 u64 iowait_sum; 536 537 u64 sleep_start; 538 u64 sleep_max; 539 s64 sum_sleep_runtime; 540 541 u64 block_start; 542 u64 block_max; 543 s64 sum_block_runtime; 544 545 s64 exec_max; 546 u64 slice_max; 547 548 u64 nr_migrations_cold; 549 u64 nr_failed_migrations_affine; 550 u64 nr_failed_migrations_running; 551 u64 nr_failed_migrations_hot; 552 u64 nr_forced_migrations; 553 554 u64 nr_wakeups; 555 u64 nr_wakeups_sync; 556 u64 nr_wakeups_migrate; 557 u64 nr_wakeups_local; 558 u64 nr_wakeups_remote; 559 u64 nr_wakeups_affine; 560 u64 nr_wakeups_affine_attempts; 561 u64 nr_wakeups_passive; 562 u64 nr_wakeups_idle; 563 564#ifdef CONFIG_SCHED_CORE 565 u64 core_forceidle_sum; 566#endif 567#endif /* CONFIG_SCHEDSTATS */ 568} ____cacheline_aligned; 569 570struct sched_entity { 571 /* For load-balancing: */ 572 struct load_weight load; 573 struct rb_node run_node; 574 u64 deadline; 575 u64 min_vruntime; 576 u64 min_slice; 577 u64 max_slice; 578 579 struct list_head group_node; 580 unsigned char on_rq; 581 unsigned char sched_delayed; 582 unsigned char rel_deadline; 583 unsigned char custom_slice; 584 /* hole */ 585 586 u64 exec_start; 587 u64 sum_exec_runtime; 588 u64 prev_sum_exec_runtime; 589 u64 vruntime; 590 union { 591 /* 592 * When !@on_rq this field is vlag. 593 * When cfs_rq->curr == se (which implies @on_rq) 594 * this field is vprot. See protect_slice(). 595 */ 596 s64 vlag; 597 u64 vprot; 598 }; 599 u64 slice; 600 601 u64 nr_migrations; 602 603#ifdef CONFIG_FAIR_GROUP_SCHED 604 int depth; 605 struct sched_entity *parent; 606 /* rq on which this entity is (to be) queued: */ 607 struct cfs_rq *cfs_rq; 608 /* rq "owned" by this entity/group: */ 609 struct cfs_rq *my_q; 610 /* cached value of my_q->h_nr_running */ 611 unsigned long runnable_weight; 612#endif 613 614 /* 615 * Per entity load average tracking. 616 * 617 * Put into separate cache line so it does not 618 * collide with read-mostly values above. 619 */ 620 struct sched_avg avg; 621}; 622 623struct sched_rt_entity { 624 struct list_head run_list; 625 unsigned long timeout; 626 unsigned long watchdog_stamp; 627 unsigned int time_slice; 628 unsigned short on_rq; 629 unsigned short on_list; 630 631 struct sched_rt_entity *back; 632#ifdef CONFIG_RT_GROUP_SCHED 633 struct sched_rt_entity *parent; 634 /* rq on which this entity is (to be) queued: */ 635 struct rt_rq *rt_rq; 636 /* rq "owned" by this entity/group: */ 637 struct rt_rq *my_q; 638#endif 639} __randomize_layout; 640 641struct rq_flags; 642typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *, struct rq_flags *rf); 643 644struct sched_dl_entity { 645 struct rb_node rb_node; 646 647 /* 648 * Original scheduling parameters. Copied here from sched_attr 649 * during sched_setattr(), they will remain the same until 650 * the next sched_setattr(). 651 */ 652 u64 dl_runtime; /* Maximum runtime for each instance */ 653 u64 dl_deadline; /* Relative deadline of each instance */ 654 u64 dl_period; /* Separation of two instances (period) */ 655 u64 dl_bw; /* dl_runtime / dl_period */ 656 u64 dl_density; /* dl_runtime / dl_deadline */ 657 658 /* 659 * Actual scheduling parameters. Initialized with the values above, 660 * they are continuously updated during task execution. Note that 661 * the remaining runtime could be < 0 in case we are in overrun. 662 */ 663 s64 runtime; /* Remaining runtime for this instance */ 664 u64 deadline; /* Absolute deadline for this instance */ 665 unsigned int flags; /* Specifying the scheduler behaviour */ 666 667 /* 668 * Some bool flags: 669 * 670 * @dl_throttled tells if we exhausted the runtime. If so, the 671 * task has to wait for a replenishment to be performed at the 672 * next firing of dl_timer. 673 * 674 * @dl_yielded tells if task gave up the CPU before consuming 675 * all its available runtime during the last job. 676 * 677 * @dl_non_contending tells if the task is inactive while still 678 * contributing to the active utilization. In other words, it 679 * indicates if the inactive timer has been armed and its handler 680 * has not been executed yet. This flag is useful to avoid race 681 * conditions between the inactive timer handler and the wakeup 682 * code. 683 * 684 * @dl_overrun tells if the task asked to be informed about runtime 685 * overruns. 686 * 687 * @dl_server tells if this is a server entity. 688 * 689 * @dl_server_active tells if the dlserver is active(started). 690 * dlserver is started on first cfs enqueue on an idle runqueue 691 * and is stopped when a dequeue results in 0 cfs tasks on the 692 * runqueue. In other words, dlserver is active only when cpu's 693 * runqueue has atleast one cfs task. 694 * 695 * @dl_defer tells if this is a deferred or regular server. For 696 * now only defer server exists. 697 * 698 * @dl_defer_armed tells if the deferrable server is waiting 699 * for the replenishment timer to activate it. 700 * 701 * @dl_defer_running tells if the deferrable server is actually 702 * running, skipping the defer phase. 703 * 704 * @dl_defer_idle tracks idle state 705 */ 706 unsigned int dl_throttled : 1; 707 unsigned int dl_yielded : 1; 708 unsigned int dl_non_contending : 1; 709 unsigned int dl_overrun : 1; 710 unsigned int dl_server : 1; 711 unsigned int dl_server_active : 1; 712 unsigned int dl_defer : 1; 713 unsigned int dl_defer_armed : 1; 714 unsigned int dl_defer_running : 1; 715 unsigned int dl_defer_idle : 1; 716 717 /* 718 * Bandwidth enforcement timer. Each -deadline task has its 719 * own bandwidth to be enforced, thus we need one timer per task. 720 */ 721 struct hrtimer dl_timer; 722 723 /* 724 * Inactive timer, responsible for decreasing the active utilization 725 * at the "0-lag time". When a -deadline task blocks, it contributes 726 * to GRUB's active utilization until the "0-lag time", hence a 727 * timer is needed to decrease the active utilization at the correct 728 * time. 729 */ 730 struct hrtimer inactive_timer; 731 732 /* 733 * Bits for DL-server functionality. Also see the comment near 734 * dl_server_update(). 735 * 736 * @rq the runqueue this server is for 737 */ 738 struct rq *rq; 739 dl_server_pick_f server_pick_task; 740 741#ifdef CONFIG_RT_MUTEXES 742 /* 743 * Priority Inheritance. When a DEADLINE scheduling entity is boosted 744 * pi_se points to the donor, otherwise points to the dl_se it belongs 745 * to (the original one/itself). 746 */ 747 struct sched_dl_entity *pi_se; 748#endif 749}; 750 751#ifdef CONFIG_UCLAMP_TASK 752/* Number of utilization clamp buckets (shorter alias) */ 753#define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT 754 755/* 756 * Utilization clamp for a scheduling entity 757 * @value: clamp value "assigned" to a se 758 * @bucket_id: bucket index corresponding to the "assigned" value 759 * @active: the se is currently refcounted in a rq's bucket 760 * @user_defined: the requested clamp value comes from user-space 761 * 762 * The bucket_id is the index of the clamp bucket matching the clamp value 763 * which is pre-computed and stored to avoid expensive integer divisions from 764 * the fast path. 765 * 766 * The active bit is set whenever a task has got an "effective" value assigned, 767 * which can be different from the clamp value "requested" from user-space. 768 * This allows to know a task is refcounted in the rq's bucket corresponding 769 * to the "effective" bucket_id. 770 * 771 * The user_defined bit is set whenever a task has got a task-specific clamp 772 * value requested from userspace, i.e. the system defaults apply to this task 773 * just as a restriction. This allows to relax default clamps when a less 774 * restrictive task-specific value has been requested, thus allowing to 775 * implement a "nice" semantic. For example, a task running with a 20% 776 * default boost can still drop its own boosting to 0%. 777 */ 778struct uclamp_se { 779 unsigned int value : bits_per(SCHED_CAPACITY_SCALE); 780 unsigned int bucket_id : bits_per(UCLAMP_BUCKETS); 781 unsigned int active : 1; 782 unsigned int user_defined : 1; 783}; 784#endif /* CONFIG_UCLAMP_TASK */ 785 786union rcu_special { 787 struct { 788 u8 blocked; 789 u8 need_qs; 790 u8 exp_hint; /* Hint for performance. */ 791 u8 need_mb; /* Readers need smp_mb(). */ 792 } b; /* Bits. */ 793 u32 s; /* Set of bits. */ 794}; 795 796enum perf_event_task_context { 797 perf_invalid_context = -1, 798 perf_hw_context = 0, 799 perf_sw_context, 800 perf_nr_task_contexts, 801}; 802 803/* 804 * Number of contexts where an event can trigger: 805 * task, softirq, hardirq, nmi. 806 */ 807#define PERF_NR_CONTEXTS 4 808 809struct wake_q_node { 810 struct wake_q_node *next; 811}; 812 813struct kmap_ctrl { 814#ifdef CONFIG_KMAP_LOCAL 815 int idx; 816 pte_t pteval[KM_MAX_IDX]; 817#endif 818}; 819 820struct task_struct { 821#ifdef CONFIG_THREAD_INFO_IN_TASK 822 /* 823 * For reasons of header soup (see current_thread_info()), this 824 * must be the first element of task_struct. 825 */ 826 struct thread_info thread_info; 827#endif 828 unsigned int __state; 829 830 /* saved state for "spinlock sleepers" */ 831 unsigned int saved_state; 832 833 /* 834 * This begins the randomizable portion of task_struct. Only 835 * scheduling-critical items should be added above here. 836 */ 837 randomized_struct_fields_start 838 839 void *stack; 840 refcount_t usage; 841 /* Per task flags (PF_*), defined further below: */ 842 unsigned int flags; 843 unsigned int ptrace; 844 845#ifdef CONFIG_MEM_ALLOC_PROFILING 846 struct alloc_tag *alloc_tag; 847#endif 848 849 int on_cpu; 850 struct __call_single_node wake_entry; 851 unsigned int wakee_flips; 852 unsigned long wakee_flip_decay_ts; 853 struct task_struct *last_wakee; 854 855 /* 856 * recent_used_cpu is initially set as the last CPU used by a task 857 * that wakes affine another task. Waker/wakee relationships can 858 * push tasks around a CPU where each wakeup moves to the next one. 859 * Tracking a recently used CPU allows a quick search for a recently 860 * used CPU that may be idle. 861 */ 862 int recent_used_cpu; 863 int wake_cpu; 864 int on_rq; 865 866 int prio; 867 int static_prio; 868 int normal_prio; 869 unsigned int rt_priority; 870 871 struct sched_entity se; 872 struct sched_rt_entity rt; 873 struct sched_dl_entity dl; 874 struct sched_dl_entity *dl_server; 875#ifdef CONFIG_SCHED_CLASS_EXT 876 struct sched_ext_entity scx; 877#endif 878 const struct sched_class *sched_class; 879 880#ifdef CONFIG_SCHED_CORE 881 struct rb_node core_node; 882 unsigned long core_cookie; 883 unsigned int core_occupation; 884#endif 885 886#ifdef CONFIG_CGROUP_SCHED 887 struct task_group *sched_task_group; 888#ifdef CONFIG_CFS_BANDWIDTH 889 struct callback_head sched_throttle_work; 890 struct list_head throttle_node; 891 bool throttled; 892#endif 893#endif 894 895 896#ifdef CONFIG_UCLAMP_TASK 897 /* 898 * Clamp values requested for a scheduling entity. 899 * Must be updated with task_rq_lock() held. 900 */ 901 struct uclamp_se uclamp_req[UCLAMP_CNT]; 902 /* 903 * Effective clamp values used for a scheduling entity. 904 * Must be updated with task_rq_lock() held. 905 */ 906 struct uclamp_se uclamp[UCLAMP_CNT]; 907#endif 908 909 struct sched_statistics stats; 910 911#ifdef CONFIG_PREEMPT_NOTIFIERS 912 /* List of struct preempt_notifier: */ 913 struct hlist_head preempt_notifiers; 914#endif 915 916#ifdef CONFIG_BLK_DEV_IO_TRACE 917 unsigned int btrace_seq; 918#endif 919 920 unsigned int policy; 921 unsigned long max_allowed_capacity; 922 int nr_cpus_allowed; 923 const cpumask_t *cpus_ptr; 924 cpumask_t *user_cpus_ptr; 925 cpumask_t cpus_mask; 926 void *migration_pending; 927 unsigned short migration_disabled; 928 unsigned short migration_flags; 929 930#ifdef CONFIG_PREEMPT_RCU 931 int rcu_read_lock_nesting; 932 union rcu_special rcu_read_unlock_special; 933 struct list_head rcu_node_entry; 934 struct rcu_node *rcu_blocked_node; 935#endif /* #ifdef CONFIG_PREEMPT_RCU */ 936 937#ifdef CONFIG_TASKS_RCU 938 unsigned long rcu_tasks_nvcsw; 939 u8 rcu_tasks_holdout; 940 u8 rcu_tasks_idx; 941 int rcu_tasks_idle_cpu; 942 struct list_head rcu_tasks_holdout_list; 943 int rcu_tasks_exit_cpu; 944 struct list_head rcu_tasks_exit_list; 945#endif /* #ifdef CONFIG_TASKS_RCU */ 946 947#ifdef CONFIG_TASKS_TRACE_RCU 948 int trc_reader_nesting; 949 int trc_ipi_to_cpu; 950 union rcu_special trc_reader_special; 951 struct list_head trc_holdout_list; 952 struct list_head trc_blkd_node; 953 int trc_blkd_cpu; 954#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ 955 956 struct sched_info sched_info; 957 958 struct list_head tasks; 959 struct plist_node pushable_tasks; 960 struct rb_node pushable_dl_tasks; 961 962 struct mm_struct *mm; 963 struct mm_struct *active_mm; 964 struct address_space *faults_disabled_mapping; 965 966 int exit_state; 967 int exit_code; 968 int exit_signal; 969 /* The signal sent when the parent dies: */ 970 int pdeath_signal; 971 /* JOBCTL_*, siglock protected: */ 972 unsigned long jobctl; 973 974 /* Used for emulating ABI behavior of previous Linux versions: */ 975 unsigned int personality; 976 977 /* Scheduler bits, serialized by scheduler locks: */ 978 unsigned sched_reset_on_fork:1; 979 unsigned sched_contributes_to_load:1; 980 unsigned sched_migrated:1; 981 unsigned sched_task_hot:1; 982 983 /* Force alignment to the next boundary: */ 984 unsigned :0; 985 986 /* Unserialized, strictly 'current' */ 987 988 /* 989 * This field must not be in the scheduler word above due to wakelist 990 * queueing no longer being serialized by p->on_cpu. However: 991 * 992 * p->XXX = X; ttwu() 993 * schedule() if (p->on_rq && ..) // false 994 * smp_mb__after_spinlock(); if (smp_load_acquire(&p->on_cpu) && //true 995 * deactivate_task() ttwu_queue_wakelist()) 996 * p->on_rq = 0; p->sched_remote_wakeup = Y; 997 * 998 * guarantees all stores of 'current' are visible before 999 * ->sched_remote_wakeup gets used, so it can be in this word. 1000 */ 1001 unsigned sched_remote_wakeup:1; 1002#ifdef CONFIG_RT_MUTEXES 1003 unsigned sched_rt_mutex:1; 1004#endif 1005 1006 /* Bit to tell TOMOYO we're in execve(): */ 1007 unsigned in_execve:1; 1008 unsigned in_iowait:1; 1009#ifndef TIF_RESTORE_SIGMASK 1010 unsigned restore_sigmask:1; 1011#endif 1012#ifdef CONFIG_MEMCG_V1 1013 unsigned in_user_fault:1; 1014#endif 1015#ifdef CONFIG_LRU_GEN 1016 /* whether the LRU algorithm may apply to this access */ 1017 unsigned in_lru_fault:1; 1018#endif 1019#ifdef CONFIG_COMPAT_BRK 1020 unsigned brk_randomized:1; 1021#endif 1022#ifdef CONFIG_CGROUPS 1023 /* disallow userland-initiated cgroup migration */ 1024 unsigned no_cgroup_migration:1; 1025 /* task is frozen/stopped (used by the cgroup freezer) */ 1026 unsigned frozen:1; 1027#endif 1028#ifdef CONFIG_BLK_CGROUP 1029 unsigned use_memdelay:1; 1030#endif 1031#ifdef CONFIG_PSI 1032 /* Stalled due to lack of memory */ 1033 unsigned in_memstall:1; 1034#endif 1035#ifdef CONFIG_PAGE_OWNER 1036 /* Used by page_owner=on to detect recursion in page tracking. */ 1037 unsigned in_page_owner:1; 1038#endif 1039#ifdef CONFIG_EVENTFD 1040 /* Recursion prevention for eventfd_signal() */ 1041 unsigned in_eventfd:1; 1042#endif 1043#ifdef CONFIG_ARCH_HAS_CPU_PASID 1044 unsigned pasid_activated:1; 1045#endif 1046#ifdef CONFIG_X86_BUS_LOCK_DETECT 1047 unsigned reported_split_lock:1; 1048#endif 1049#ifdef CONFIG_TASK_DELAY_ACCT 1050 /* delay due to memory thrashing */ 1051 unsigned in_thrashing:1; 1052#endif 1053 unsigned in_nf_duplicate:1; 1054#ifdef CONFIG_PREEMPT_RT 1055 struct netdev_xmit net_xmit; 1056#endif 1057 unsigned long atomic_flags; /* Flags requiring atomic access. */ 1058 1059 struct restart_block restart_block; 1060 1061 pid_t pid; 1062 pid_t tgid; 1063 1064#ifdef CONFIG_STACKPROTECTOR 1065 /* Canary value for the -fstack-protector GCC feature: */ 1066 unsigned long stack_canary; 1067#endif 1068 /* 1069 * Pointers to the (original) parent process, youngest child, younger sibling, 1070 * older sibling, respectively. (p->father can be replaced with 1071 * p->real_parent->pid) 1072 */ 1073 1074 /* Real parent process: */ 1075 struct task_struct __rcu *real_parent; 1076 1077 /* Recipient of SIGCHLD, wait4() reports: */ 1078 struct task_struct __rcu *parent; 1079 1080 /* 1081 * Children/sibling form the list of natural children: 1082 */ 1083 struct list_head children; 1084 struct list_head sibling; 1085 struct task_struct *group_leader; 1086 1087 /* 1088 * 'ptraced' is the list of tasks this task is using ptrace() on. 1089 * 1090 * This includes both natural children and PTRACE_ATTACH targets. 1091 * 'ptrace_entry' is this task's link on the p->parent->ptraced list. 1092 */ 1093 struct list_head ptraced; 1094 struct list_head ptrace_entry; 1095 1096 /* PID/PID hash table linkage. */ 1097 struct pid *thread_pid; 1098 struct hlist_node pid_links[PIDTYPE_MAX]; 1099 struct list_head thread_node; 1100 1101 struct completion *vfork_done; 1102 1103 /* CLONE_CHILD_SETTID: */ 1104 int __user *set_child_tid; 1105 1106 /* CLONE_CHILD_CLEARTID: */ 1107 int __user *clear_child_tid; 1108 1109 /* PF_KTHREAD | PF_IO_WORKER */ 1110 void *worker_private; 1111 1112 u64 utime; 1113 u64 stime; 1114#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME 1115 u64 utimescaled; 1116 u64 stimescaled; 1117#endif 1118 u64 gtime; 1119 struct prev_cputime prev_cputime; 1120#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 1121 struct vtime vtime; 1122#endif 1123 1124#ifdef CONFIG_NO_HZ_FULL 1125 atomic_t tick_dep_mask; 1126#endif 1127 /* Context switch counts: */ 1128 unsigned long nvcsw; 1129 unsigned long nivcsw; 1130 1131 /* Monotonic time in nsecs: */ 1132 u64 start_time; 1133 1134 /* Boot based time in nsecs: */ 1135 u64 start_boottime; 1136 1137 /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */ 1138 unsigned long min_flt; 1139 unsigned long maj_flt; 1140 1141 /* Empty if CONFIG_POSIX_CPUTIMERS=n */ 1142 struct posix_cputimers posix_cputimers; 1143 1144#ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK 1145 struct posix_cputimers_work posix_cputimers_work; 1146#endif 1147 1148 /* Process credentials: */ 1149 1150 /* Tracer's credentials at attach: */ 1151 const struct cred __rcu *ptracer_cred; 1152 1153 /* Objective and real subjective task credentials (COW): */ 1154 const struct cred __rcu *real_cred; 1155 1156 /* Effective (overridable) subjective task credentials (COW): */ 1157 const struct cred __rcu *cred; 1158 1159#ifdef CONFIG_KEYS 1160 /* Cached requested key. */ 1161 struct key *cached_requested_key; 1162#endif 1163 1164 /* 1165 * executable name, excluding path. 1166 * 1167 * - normally initialized begin_new_exec() 1168 * - set it with set_task_comm() 1169 * - strscpy_pad() to ensure it is always NUL-terminated and 1170 * zero-padded 1171 * - task_lock() to ensure the operation is atomic and the name is 1172 * fully updated. 1173 */ 1174 char comm[TASK_COMM_LEN]; 1175 1176 struct nameidata *nameidata; 1177 1178#ifdef CONFIG_SYSVIPC 1179 struct sysv_sem sysvsem; 1180 struct sysv_shm sysvshm; 1181#endif 1182#ifdef CONFIG_DETECT_HUNG_TASK 1183 unsigned long last_switch_count; 1184 unsigned long last_switch_time; 1185#endif 1186 /* Filesystem information: */ 1187 struct fs_struct *fs; 1188 1189 /* Open file information: */ 1190 struct files_struct *files; 1191 1192#ifdef CONFIG_IO_URING 1193 struct io_uring_task *io_uring; 1194#endif 1195 1196 /* Namespaces: */ 1197 struct nsproxy *nsproxy; 1198 1199 /* Signal handlers: */ 1200 struct signal_struct *signal; 1201 struct sighand_struct __rcu *sighand; 1202 sigset_t blocked; 1203 sigset_t real_blocked; 1204 /* Restored if set_restore_sigmask() was used: */ 1205 sigset_t saved_sigmask; 1206 struct sigpending pending; 1207 unsigned long sas_ss_sp; 1208 size_t sas_ss_size; 1209 unsigned int sas_ss_flags; 1210 1211 struct callback_head *task_works; 1212 1213#ifdef CONFIG_AUDIT 1214#ifdef CONFIG_AUDITSYSCALL 1215 struct audit_context *audit_context; 1216#endif 1217 kuid_t loginuid; 1218 unsigned int sessionid; 1219#endif 1220 struct seccomp seccomp; 1221 struct syscall_user_dispatch syscall_dispatch; 1222 1223 /* Thread group tracking: */ 1224 u64 parent_exec_id; 1225 u64 self_exec_id; 1226 1227 /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */ 1228 spinlock_t alloc_lock; 1229 1230 /* Protection of the PI data structures: */ 1231 raw_spinlock_t pi_lock; 1232 1233 struct wake_q_node wake_q; 1234 1235#ifdef CONFIG_RT_MUTEXES 1236 /* PI waiters blocked on a rt_mutex held by this task: */ 1237 struct rb_root_cached pi_waiters; 1238 /* Updated under owner's pi_lock and rq lock */ 1239 struct task_struct *pi_top_task; 1240 /* Deadlock detection and priority inheritance handling: */ 1241 struct rt_mutex_waiter *pi_blocked_on; 1242#endif 1243 1244 struct mutex *blocked_on; /* lock we're blocked on */ 1245 1246#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER 1247 /* 1248 * Encoded lock address causing task block (lower 2 bits = type from 1249 * <linux/hung_task.h>). Accessed via hung_task_*() helpers. 1250 */ 1251 unsigned long blocker; 1252#endif 1253 1254#ifdef CONFIG_DEBUG_ATOMIC_SLEEP 1255 int non_block_count; 1256#endif 1257 1258#ifdef CONFIG_TRACE_IRQFLAGS 1259 struct irqtrace_events irqtrace; 1260 unsigned int hardirq_threaded; 1261 u64 hardirq_chain_key; 1262 int softirqs_enabled; 1263 int softirq_context; 1264 int irq_config; 1265#endif 1266#ifdef CONFIG_PREEMPT_RT 1267 int softirq_disable_cnt; 1268#endif 1269 1270#ifdef CONFIG_LOCKDEP 1271# define MAX_LOCK_DEPTH 48UL 1272 u64 curr_chain_key; 1273 int lockdep_depth; 1274 unsigned int lockdep_recursion; 1275 struct held_lock held_locks[MAX_LOCK_DEPTH]; 1276#endif 1277 1278#if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP) 1279 unsigned int in_ubsan; 1280#endif 1281 1282 /* Journalling filesystem info: */ 1283 void *journal_info; 1284 1285 /* Stacked block device info: */ 1286 struct bio_list *bio_list; 1287 1288 /* Stack plugging: */ 1289 struct blk_plug *plug; 1290 1291 /* VM state: */ 1292 struct reclaim_state *reclaim_state; 1293 1294 struct io_context *io_context; 1295 1296#ifdef CONFIG_COMPACTION 1297 struct capture_control *capture_control; 1298#endif 1299 /* Ptrace state: */ 1300 unsigned long ptrace_message; 1301 kernel_siginfo_t *last_siginfo; 1302 1303 struct task_io_accounting ioac; 1304#ifdef CONFIG_PSI 1305 /* Pressure stall state */ 1306 unsigned int psi_flags; 1307#endif 1308#ifdef CONFIG_TASK_XACCT 1309 /* Accumulated RSS usage: */ 1310 u64 acct_rss_mem1; 1311 /* Accumulated virtual memory usage: */ 1312 u64 acct_vm_mem1; 1313 /* stime + utime since last update: */ 1314 u64 acct_timexpd; 1315#endif 1316#ifdef CONFIG_CPUSETS 1317 /* Protected by ->alloc_lock: */ 1318 nodemask_t mems_allowed; 1319 /* Sequence number to catch updates: */ 1320 seqcount_spinlock_t mems_allowed_seq; 1321 int cpuset_mem_spread_rotor; 1322#endif 1323#ifdef CONFIG_CGROUPS 1324 /* Control Group info protected by css_set_lock: */ 1325 struct css_set __rcu *cgroups; 1326 /* cg_list protected by css_set_lock and tsk->alloc_lock: */ 1327 struct list_head cg_list; 1328#ifdef CONFIG_PREEMPT_RT 1329 struct llist_node cg_dead_lnode; 1330#endif /* CONFIG_PREEMPT_RT */ 1331#endif /* CONFIG_CGROUPS */ 1332#ifdef CONFIG_X86_CPU_RESCTRL 1333 u32 closid; 1334 u32 rmid; 1335#endif 1336#ifdef CONFIG_FUTEX 1337 struct robust_list_head __user *robust_list; 1338#ifdef CONFIG_COMPAT 1339 struct compat_robust_list_head __user *compat_robust_list; 1340#endif 1341 struct list_head pi_state_list; 1342 struct futex_pi_state *pi_state_cache; 1343 struct mutex futex_exit_mutex; 1344 unsigned int futex_state; 1345#endif 1346#ifdef CONFIG_PERF_EVENTS 1347 u8 perf_recursion[PERF_NR_CONTEXTS]; 1348 struct perf_event_context *perf_event_ctxp; 1349 struct mutex perf_event_mutex; 1350 struct list_head perf_event_list; 1351 struct perf_ctx_data __rcu *perf_ctx_data; 1352#endif 1353#ifdef CONFIG_DEBUG_PREEMPT 1354 unsigned long preempt_disable_ip; 1355#endif 1356#ifdef CONFIG_NUMA 1357 /* Protected by alloc_lock: */ 1358 struct mempolicy *mempolicy; 1359 short il_prev; 1360 u8 il_weight; 1361 short pref_node_fork; 1362#endif 1363#ifdef CONFIG_NUMA_BALANCING 1364 int numa_scan_seq; 1365 unsigned int numa_scan_period; 1366 unsigned int numa_scan_period_max; 1367 int numa_preferred_nid; 1368 unsigned long numa_migrate_retry; 1369 /* Migration stamp: */ 1370 u64 node_stamp; 1371 u64 last_task_numa_placement; 1372 u64 last_sum_exec_runtime; 1373 struct callback_head numa_work; 1374 1375 /* 1376 * This pointer is only modified for current in syscall and 1377 * pagefault context (and for tasks being destroyed), so it can be read 1378 * from any of the following contexts: 1379 * - RCU read-side critical section 1380 * - current->numa_group from everywhere 1381 * - task's runqueue locked, task not running 1382 */ 1383 struct numa_group __rcu *numa_group; 1384 1385 /* 1386 * numa_faults is an array split into four regions: 1387 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer 1388 * in this precise order. 1389 * 1390 * faults_memory: Exponential decaying average of faults on a per-node 1391 * basis. Scheduling placement decisions are made based on these 1392 * counts. The values remain static for the duration of a PTE scan. 1393 * faults_cpu: Track the nodes the process was running on when a NUMA 1394 * hinting fault was incurred. 1395 * faults_memory_buffer and faults_cpu_buffer: Record faults per node 1396 * during the current scan window. When the scan completes, the counts 1397 * in faults_memory and faults_cpu decay and these values are copied. 1398 */ 1399 unsigned long *numa_faults; 1400 unsigned long total_numa_faults; 1401 1402 /* 1403 * numa_faults_locality tracks if faults recorded during the last 1404 * scan window were remote/local or failed to migrate. The task scan 1405 * period is adapted based on the locality of the faults with different 1406 * weights depending on whether they were shared or private faults 1407 */ 1408 unsigned long numa_faults_locality[3]; 1409 1410 unsigned long numa_pages_migrated; 1411#endif /* CONFIG_NUMA_BALANCING */ 1412 1413 struct rseq_data rseq; 1414 struct sched_mm_cid mm_cid; 1415 1416 struct tlbflush_unmap_batch tlb_ubc; 1417 1418 /* Cache last used pipe for splice(): */ 1419 struct pipe_inode_info *splice_pipe; 1420 1421 struct page_frag task_frag; 1422 1423#ifdef CONFIG_TASK_DELAY_ACCT 1424 struct task_delay_info *delays; 1425#endif 1426 1427#ifdef CONFIG_FAULT_INJECTION 1428 int make_it_fail; 1429 unsigned int fail_nth; 1430#endif 1431 /* 1432 * When (nr_dirtied >= nr_dirtied_pause), it's time to call 1433 * balance_dirty_pages() for a dirty throttling pause: 1434 */ 1435 int nr_dirtied; 1436 int nr_dirtied_pause; 1437 /* Start of a write-and-pause period: */ 1438 unsigned long dirty_paused_when; 1439 1440#ifdef CONFIG_LATENCYTOP 1441 int latency_record_count; 1442 struct latency_record latency_record[LT_SAVECOUNT]; 1443#endif 1444 /* 1445 * Time slack values; these are used to round up poll() and 1446 * select() etc timeout values. These are in nanoseconds. 1447 */ 1448 u64 timer_slack_ns; 1449 u64 default_timer_slack_ns; 1450 1451#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 1452 unsigned int kasan_depth; 1453#endif 1454 1455#ifdef CONFIG_KCSAN 1456 struct kcsan_ctx kcsan_ctx; 1457#ifdef CONFIG_TRACE_IRQFLAGS 1458 struct irqtrace_events kcsan_save_irqtrace; 1459#endif 1460#ifdef CONFIG_KCSAN_WEAK_MEMORY 1461 int kcsan_stack_depth; 1462#endif 1463#endif 1464 1465#ifdef CONFIG_KMSAN 1466 struct kmsan_ctx kmsan_ctx; 1467#endif 1468 1469#if IS_ENABLED(CONFIG_KUNIT) 1470 struct kunit *kunit_test; 1471#endif 1472 1473#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1474 /* Index of current stored address in ret_stack: */ 1475 int curr_ret_stack; 1476 int curr_ret_depth; 1477 1478 /* Stack of return addresses for return function tracing: */ 1479 unsigned long *ret_stack; 1480 1481 /* Timestamp for last schedule: */ 1482 unsigned long long ftrace_timestamp; 1483 unsigned long long ftrace_sleeptime; 1484 1485 /* 1486 * Number of functions that haven't been traced 1487 * because of depth overrun: 1488 */ 1489 atomic_t trace_overrun; 1490 1491 /* Pause tracing: */ 1492 atomic_t tracing_graph_pause; 1493#endif 1494 1495#ifdef CONFIG_TRACING 1496 /* Bitmask and counter of trace recursion: */ 1497 unsigned long trace_recursion; 1498#endif /* CONFIG_TRACING */ 1499 1500#ifdef CONFIG_KCOV 1501 /* See kernel/kcov.c for more details. */ 1502 1503 /* Coverage collection mode enabled for this task (0 if disabled): */ 1504 unsigned int kcov_mode; 1505 1506 /* Size of the kcov_area: */ 1507 unsigned int kcov_size; 1508 1509 /* Buffer for coverage collection: */ 1510 void *kcov_area; 1511 1512 /* KCOV descriptor wired with this task or NULL: */ 1513 struct kcov *kcov; 1514 1515 /* KCOV common handle for remote coverage collection: */ 1516 u64 kcov_handle; 1517 1518 /* KCOV sequence number: */ 1519 int kcov_sequence; 1520 1521 /* Collect coverage from softirq context: */ 1522 unsigned int kcov_softirq; 1523#endif 1524 1525#ifdef CONFIG_MEMCG_V1 1526 struct mem_cgroup *memcg_in_oom; 1527#endif 1528 1529#ifdef CONFIG_MEMCG 1530 /* Number of pages to reclaim on returning to userland: */ 1531 unsigned int memcg_nr_pages_over_high; 1532 1533 /* Used by memcontrol for targeted memcg charge: */ 1534 struct mem_cgroup *active_memcg; 1535 1536 /* Cache for current->cgroups->memcg->objcg lookups: */ 1537 struct obj_cgroup *objcg; 1538#endif 1539 1540#ifdef CONFIG_BLK_CGROUP 1541 struct gendisk *throttle_disk; 1542#endif 1543 1544#ifdef CONFIG_UPROBES 1545 struct uprobe_task *utask; 1546#endif 1547#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE) 1548 unsigned int sequential_io; 1549 unsigned int sequential_io_avg; 1550#endif 1551 struct kmap_ctrl kmap_ctrl; 1552#ifdef CONFIG_DEBUG_ATOMIC_SLEEP 1553 unsigned long task_state_change; 1554# ifdef CONFIG_PREEMPT_RT 1555 unsigned long saved_state_change; 1556# endif 1557#endif 1558 struct rcu_head rcu; 1559 refcount_t rcu_users; 1560 int pagefault_disabled; 1561#ifdef CONFIG_MMU 1562 struct task_struct *oom_reaper_list; 1563 struct timer_list oom_reaper_timer; 1564#endif 1565#ifdef CONFIG_VMAP_STACK 1566 struct vm_struct *stack_vm_area; 1567#endif 1568#ifdef CONFIG_THREAD_INFO_IN_TASK 1569 /* A live task holds one reference: */ 1570 refcount_t stack_refcount; 1571#endif 1572#ifdef CONFIG_LIVEPATCH 1573 int patch_state; 1574#endif 1575#ifdef CONFIG_SECURITY 1576 /* Used by LSM modules for access restriction: */ 1577 void *security; 1578#endif 1579#ifdef CONFIG_BPF_SYSCALL 1580 /* Used by BPF task local storage */ 1581 struct bpf_local_storage __rcu *bpf_storage; 1582 /* Used for BPF run context */ 1583 struct bpf_run_ctx *bpf_ctx; 1584#endif 1585 /* Used by BPF for per-TASK xdp storage */ 1586 struct bpf_net_context *bpf_net_context; 1587 1588#ifdef CONFIG_KSTACK_ERASE 1589 unsigned long lowest_stack; 1590#endif 1591#ifdef CONFIG_KSTACK_ERASE_METRICS 1592 unsigned long prev_lowest_stack; 1593#endif 1594 1595#ifdef CONFIG_X86_MCE 1596 void __user *mce_vaddr; 1597 __u64 mce_kflags; 1598 u64 mce_addr; 1599 __u64 mce_ripv : 1, 1600 mce_whole_page : 1, 1601 __mce_reserved : 62; 1602 struct callback_head mce_kill_me; 1603 int mce_count; 1604#endif 1605 1606#ifdef CONFIG_KRETPROBES 1607 struct llist_head kretprobe_instances; 1608#endif 1609#ifdef CONFIG_RETHOOK 1610 struct llist_head rethooks; 1611#endif 1612 1613#ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH 1614 /* 1615 * If L1D flush is supported on mm context switch 1616 * then we use this callback head to queue kill work 1617 * to kill tasks that are not running on SMT disabled 1618 * cores 1619 */ 1620 struct callback_head l1d_flush_kill; 1621#endif 1622 1623#ifdef CONFIG_RV 1624 /* 1625 * Per-task RV monitor, fixed in CONFIG_RV_PER_TASK_MONITORS. 1626 * If memory becomes a concern, we can think about a dynamic method. 1627 */ 1628 union rv_task_monitor rv[CONFIG_RV_PER_TASK_MONITORS]; 1629#endif 1630 1631#ifdef CONFIG_USER_EVENTS 1632 struct user_event_mm *user_event_mm; 1633#endif 1634 1635#ifdef CONFIG_UNWIND_USER 1636 struct unwind_task_info unwind_info; 1637#endif 1638 1639 /* CPU-specific state of this task: */ 1640 struct thread_struct thread; 1641 1642 /* 1643 * New fields for task_struct should be added above here, so that 1644 * they are included in the randomized portion of task_struct. 1645 */ 1646 randomized_struct_fields_end 1647} __attribute__ ((aligned (64))); 1648 1649#ifdef CONFIG_SCHED_PROXY_EXEC 1650DECLARE_STATIC_KEY_TRUE(__sched_proxy_exec); 1651static inline bool sched_proxy_exec(void) 1652{ 1653 return static_branch_likely(&__sched_proxy_exec); 1654} 1655#else 1656static inline bool sched_proxy_exec(void) 1657{ 1658 return false; 1659} 1660#endif 1661 1662#define TASK_REPORT_IDLE (TASK_REPORT + 1) 1663#define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1) 1664 1665static inline unsigned int __task_state_index(unsigned int tsk_state, 1666 unsigned int tsk_exit_state) 1667{ 1668 unsigned int state = (tsk_state | tsk_exit_state) & TASK_REPORT; 1669 1670 BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX); 1671 1672 if ((tsk_state & TASK_IDLE) == TASK_IDLE) 1673 state = TASK_REPORT_IDLE; 1674 1675 /* 1676 * We're lying here, but rather than expose a completely new task state 1677 * to userspace, we can make this appear as if the task has gone through 1678 * a regular rt_mutex_lock() call. 1679 * Report frozen tasks as uninterruptible. 1680 */ 1681 if ((tsk_state & TASK_RTLOCK_WAIT) || (tsk_state & TASK_FROZEN)) 1682 state = TASK_UNINTERRUPTIBLE; 1683 1684 return fls(state); 1685} 1686 1687static inline unsigned int task_state_index(struct task_struct *tsk) 1688{ 1689 return __task_state_index(READ_ONCE(tsk->__state), tsk->exit_state); 1690} 1691 1692static inline char task_index_to_char(unsigned int state) 1693{ 1694 static const char state_char[] = "RSDTtXZPI"; 1695 1696 BUILD_BUG_ON(TASK_REPORT_MAX * 2 != 1 << (sizeof(state_char) - 1)); 1697 1698 return state_char[state]; 1699} 1700 1701static inline char task_state_to_char(struct task_struct *tsk) 1702{ 1703 return task_index_to_char(task_state_index(tsk)); 1704} 1705 1706extern struct pid *cad_pid; 1707 1708/* 1709 * Per process flags 1710 */ 1711#define PF_VCPU 0x00000001 /* I'm a virtual CPU */ 1712#define PF_IDLE 0x00000002 /* I am an IDLE thread */ 1713#define PF_EXITING 0x00000004 /* Getting shut down */ 1714#define PF_POSTCOREDUMP 0x00000008 /* Coredumps should ignore this task */ 1715#define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ 1716#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ 1717#define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */ 1718#define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */ 1719#define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */ 1720#define PF_DUMPCORE 0x00000200 /* Dumped core */ 1721#define PF_SIGNALED 0x00000400 /* Killed by a signal */ 1722#define PF_MEMALLOC 0x00000800 /* Allocating memory to free memory. See memalloc_noreclaim_save() */ 1723#define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */ 1724#define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ 1725#define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ 1726#define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ 1727#define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ 1728#define PF_KSWAPD 0x00020000 /* I am kswapd */ 1729#define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ 1730#define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ 1731#define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to, 1732 * I am cleaning dirty pages from some other bdi. */ 1733#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 1734#define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ 1735#define PF__HOLE__00800000 0x00800000 1736#define PF__HOLE__01000000 0x01000000 1737#define PF__HOLE__02000000 0x02000000 1738#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ 1739#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ 1740#define PF_MEMALLOC_PIN 0x10000000 /* Allocations constrained to zones which allow long term pinning. 1741 * See memalloc_pin_save() */ 1742#define PF_BLOCK_TS 0x20000000 /* plug has ts that needs updating */ 1743#define PF__HOLE__40000000 0x40000000 1744#define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */ 1745 1746/* 1747 * Only the _current_ task can read/write to tsk->flags, but other 1748 * tasks can access tsk->flags in readonly mode for example 1749 * with tsk_used_math (like during threaded core dumping). 1750 * There is however an exception to this rule during ptrace 1751 * or during fork: the ptracer task is allowed to write to the 1752 * child->flags of its traced child (same goes for fork, the parent 1753 * can write to the child->flags), because we're guaranteed the 1754 * child is not running and in turn not changing child->flags 1755 * at the same time the parent does it. 1756 */ 1757#define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0) 1758#define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0) 1759#define clear_used_math() clear_stopped_child_used_math(current) 1760#define set_used_math() set_stopped_child_used_math(current) 1761 1762#define conditional_stopped_child_used_math(condition, child) \ 1763 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0) 1764 1765#define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current) 1766 1767#define copy_to_stopped_child_used_math(child) \ 1768 do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0) 1769 1770/* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */ 1771#define tsk_used_math(p) ((p)->flags & PF_USED_MATH) 1772#define used_math() tsk_used_math(current) 1773 1774static __always_inline bool is_percpu_thread(void) 1775{ 1776 return (current->flags & PF_NO_SETAFFINITY) && 1777 (current->nr_cpus_allowed == 1); 1778} 1779 1780static __always_inline bool is_user_task(struct task_struct *task) 1781{ 1782 return task->mm && !(task->flags & (PF_KTHREAD | PF_USER_WORKER)); 1783} 1784 1785/* Per-process atomic flags. */ 1786#define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ 1787#define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ 1788#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ 1789#define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */ 1790#define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/ 1791#define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */ 1792#define PFA_SPEC_IB_FORCE_DISABLE 6 /* Indirect branch speculation permanently restricted */ 1793#define PFA_SPEC_SSB_NOEXEC 7 /* Speculative Store Bypass clear on execve() */ 1794 1795#define TASK_PFA_TEST(name, func) \ 1796 static inline bool task_##func(struct task_struct *p) \ 1797 { return test_bit(PFA_##name, &p->atomic_flags); } 1798 1799#define TASK_PFA_SET(name, func) \ 1800 static inline void task_set_##func(struct task_struct *p) \ 1801 { set_bit(PFA_##name, &p->atomic_flags); } 1802 1803#define TASK_PFA_CLEAR(name, func) \ 1804 static inline void task_clear_##func(struct task_struct *p) \ 1805 { clear_bit(PFA_##name, &p->atomic_flags); } 1806 1807TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) 1808TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) 1809 1810TASK_PFA_TEST(SPREAD_PAGE, spread_page) 1811TASK_PFA_SET(SPREAD_PAGE, spread_page) 1812TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) 1813 1814TASK_PFA_TEST(SPREAD_SLAB, spread_slab) 1815TASK_PFA_SET(SPREAD_SLAB, spread_slab) 1816TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) 1817 1818TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable) 1819TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable) 1820TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable) 1821 1822TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec) 1823TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec) 1824TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec) 1825 1826TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable) 1827TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable) 1828 1829TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable) 1830TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable) 1831TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable) 1832 1833TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable) 1834TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable) 1835 1836static inline void 1837current_restore_flags(unsigned long orig_flags, unsigned long flags) 1838{ 1839 current->flags &= ~flags; 1840 current->flags |= orig_flags & flags; 1841} 1842 1843extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); 1844extern int task_can_attach(struct task_struct *p); 1845extern int dl_bw_alloc(int cpu, u64 dl_bw); 1846extern void dl_bw_free(int cpu, u64 dl_bw); 1847 1848/* set_cpus_allowed_force() - consider using set_cpus_allowed_ptr() instead */ 1849extern void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask); 1850 1851/** 1852 * set_cpus_allowed_ptr - set CPU affinity mask of a task 1853 * @p: the task 1854 * @new_mask: CPU affinity mask 1855 * 1856 * Return: zero if successful, or a negative error code 1857 */ 1858extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); 1859extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node); 1860extern void release_user_cpus_ptr(struct task_struct *p); 1861extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask); 1862extern void force_compatible_cpus_allowed_ptr(struct task_struct *p); 1863extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p); 1864 1865extern int yield_to(struct task_struct *p, bool preempt); 1866extern void set_user_nice(struct task_struct *p, long nice); 1867extern int task_prio(const struct task_struct *p); 1868 1869/** 1870 * task_nice - return the nice value of a given task. 1871 * @p: the task in question. 1872 * 1873 * Return: The nice value [ -20 ... 0 ... 19 ]. 1874 */ 1875static inline int task_nice(const struct task_struct *p) 1876{ 1877 return PRIO_TO_NICE((p)->static_prio); 1878} 1879 1880extern int can_nice(const struct task_struct *p, const int nice); 1881extern int task_curr(const struct task_struct *p); 1882extern int idle_cpu(int cpu); 1883extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *); 1884extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *); 1885extern void sched_set_fifo(struct task_struct *p); 1886extern void sched_set_fifo_low(struct task_struct *p); 1887extern void sched_set_fifo_secondary(struct task_struct *p); 1888extern void sched_set_normal(struct task_struct *p, int nice); 1889extern int sched_setattr(struct task_struct *, const struct sched_attr *); 1890extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *); 1891extern struct task_struct *idle_task(int cpu); 1892 1893/** 1894 * is_idle_task - is the specified task an idle task? 1895 * @p: the task in question. 1896 * 1897 * Return: 1 if @p is an idle task. 0 otherwise. 1898 */ 1899static __always_inline bool is_idle_task(const struct task_struct *p) 1900{ 1901 return !!(p->flags & PF_IDLE); 1902} 1903 1904extern struct task_struct *curr_task(int cpu); 1905extern void ia64_set_curr_task(int cpu, struct task_struct *p); 1906 1907void yield(void); 1908 1909union thread_union { 1910 struct task_struct task; 1911#ifndef CONFIG_THREAD_INFO_IN_TASK 1912 struct thread_info thread_info; 1913#endif 1914 unsigned long stack[THREAD_SIZE/sizeof(long)]; 1915}; 1916 1917#ifndef CONFIG_THREAD_INFO_IN_TASK 1918extern struct thread_info init_thread_info; 1919#endif 1920 1921extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)]; 1922 1923#ifdef CONFIG_THREAD_INFO_IN_TASK 1924# define task_thread_info(task) (&(task)->thread_info) 1925#else 1926# define task_thread_info(task) ((struct thread_info *)(task)->stack) 1927#endif 1928 1929/* 1930 * find a task by one of its numerical ids 1931 * 1932 * find_task_by_pid_ns(): 1933 * finds a task by its pid in the specified namespace 1934 * find_task_by_vpid(): 1935 * finds a task by its virtual pid 1936 * 1937 * see also find_vpid() etc in include/linux/pid.h 1938 */ 1939 1940extern struct task_struct *find_task_by_vpid(pid_t nr); 1941extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); 1942 1943/* 1944 * find a task by its virtual pid and get the task struct 1945 */ 1946extern struct task_struct *find_get_task_by_vpid(pid_t nr); 1947 1948extern int wake_up_state(struct task_struct *tsk, unsigned int state); 1949extern int wake_up_process(struct task_struct *tsk); 1950extern void wake_up_new_task(struct task_struct *tsk); 1951 1952extern void kick_process(struct task_struct *tsk); 1953 1954extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); 1955#define set_task_comm(tsk, from) ({ \ 1956 BUILD_BUG_ON(sizeof(from) != TASK_COMM_LEN); \ 1957 __set_task_comm(tsk, from, false); \ 1958}) 1959 1960/* 1961 * - Why not use task_lock()? 1962 * User space can randomly change their names anyway, so locking for readers 1963 * doesn't make sense. For writers, locking is probably necessary, as a race 1964 * condition could lead to long-term mixed results. 1965 * The strscpy_pad() in __set_task_comm() can ensure that the task comm is 1966 * always NUL-terminated and zero-padded. Therefore the race condition between 1967 * reader and writer is not an issue. 1968 * 1969 * - BUILD_BUG_ON() can help prevent the buf from being truncated. 1970 * Since the callers don't perform any return value checks, this safeguard is 1971 * necessary. 1972 */ 1973#define get_task_comm(buf, tsk) ({ \ 1974 BUILD_BUG_ON(sizeof(buf) < TASK_COMM_LEN); \ 1975 strscpy_pad(buf, (tsk)->comm); \ 1976 buf; \ 1977}) 1978 1979static __always_inline void scheduler_ipi(void) 1980{ 1981 /* 1982 * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting 1983 * TIF_NEED_RESCHED remotely (for the first time) will also send 1984 * this IPI. 1985 */ 1986 preempt_fold_need_resched(); 1987} 1988 1989extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state); 1990 1991/* 1992 * Set thread flags in other task's structures. 1993 * See asm/thread_info.h for TIF_xxxx flags available: 1994 */ 1995static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag) 1996{ 1997 set_ti_thread_flag(task_thread_info(tsk), flag); 1998} 1999 2000static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag) 2001{ 2002 clear_ti_thread_flag(task_thread_info(tsk), flag); 2003} 2004 2005static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag, 2006 bool value) 2007{ 2008 update_ti_thread_flag(task_thread_info(tsk), flag, value); 2009} 2010 2011static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) 2012{ 2013 return test_and_set_ti_thread_flag(task_thread_info(tsk), flag); 2014} 2015 2016static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) 2017{ 2018 return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag); 2019} 2020 2021static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) 2022{ 2023 return test_ti_thread_flag(task_thread_info(tsk), flag); 2024} 2025 2026static inline void set_tsk_need_resched(struct task_struct *tsk) 2027{ 2028 if (tracepoint_enabled(sched_set_need_resched_tp) && 2029 !test_tsk_thread_flag(tsk, TIF_NEED_RESCHED)) 2030 __trace_set_need_resched(tsk, TIF_NEED_RESCHED); 2031 set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); 2032} 2033 2034static inline void clear_tsk_need_resched(struct task_struct *tsk) 2035{ 2036 atomic_long_andnot(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY, 2037 (atomic_long_t *)&task_thread_info(tsk)->flags); 2038} 2039 2040static inline int test_tsk_need_resched(struct task_struct *tsk) 2041{ 2042 return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); 2043} 2044 2045static inline void set_need_resched_current(void) 2046{ 2047 lockdep_assert_irqs_disabled(); 2048 set_tsk_need_resched(current); 2049 set_preempt_need_resched(); 2050} 2051 2052/* 2053 * cond_resched() and cond_resched_lock(): latency reduction via 2054 * explicit rescheduling in places that are safe. The return 2055 * value indicates whether a reschedule was done in fact. 2056 * cond_resched_lock() will drop the spinlock before scheduling, 2057 */ 2058#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) 2059extern int __cond_resched(void); 2060 2061#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) 2062 2063DECLARE_STATIC_CALL(cond_resched, __cond_resched); 2064 2065static __always_inline int _cond_resched(void) 2066{ 2067 return static_call_mod(cond_resched)(); 2068} 2069 2070#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) 2071 2072extern int dynamic_cond_resched(void); 2073 2074static __always_inline int _cond_resched(void) 2075{ 2076 return dynamic_cond_resched(); 2077} 2078 2079#else /* !CONFIG_PREEMPTION */ 2080 2081static inline int _cond_resched(void) 2082{ 2083 return __cond_resched(); 2084} 2085 2086#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ 2087 2088#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ 2089 2090static inline int _cond_resched(void) 2091{ 2092 return 0; 2093} 2094 2095#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ 2096 2097#define cond_resched() ({ \ 2098 __might_resched(__FILE__, __LINE__, 0); \ 2099 _cond_resched(); \ 2100}) 2101 2102extern int __cond_resched_lock(spinlock_t *lock); 2103extern int __cond_resched_rwlock_read(rwlock_t *lock); 2104extern int __cond_resched_rwlock_write(rwlock_t *lock); 2105 2106#define MIGHT_RESCHED_RCU_SHIFT 8 2107#define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1) 2108 2109#ifndef CONFIG_PREEMPT_RT 2110/* 2111 * Non RT kernels have an elevated preempt count due to the held lock, 2112 * but are not allowed to be inside a RCU read side critical section 2113 */ 2114# define PREEMPT_LOCK_RESCHED_OFFSETS PREEMPT_LOCK_OFFSET 2115#else 2116/* 2117 * spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in 2118 * cond_resched*lock() has to take that into account because it checks for 2119 * preempt_count() and rcu_preempt_depth(). 2120 */ 2121# define PREEMPT_LOCK_RESCHED_OFFSETS \ 2122 (PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT)) 2123#endif 2124 2125#define cond_resched_lock(lock) ({ \ 2126 __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \ 2127 __cond_resched_lock(lock); \ 2128}) 2129 2130#define cond_resched_rwlock_read(lock) ({ \ 2131 __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \ 2132 __cond_resched_rwlock_read(lock); \ 2133}) 2134 2135#define cond_resched_rwlock_write(lock) ({ \ 2136 __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \ 2137 __cond_resched_rwlock_write(lock); \ 2138}) 2139 2140#ifndef CONFIG_PREEMPT_RT 2141static inline struct mutex *__get_task_blocked_on(struct task_struct *p) 2142{ 2143 struct mutex *m = p->blocked_on; 2144 2145 if (m) 2146 lockdep_assert_held_once(&m->wait_lock); 2147 return m; 2148} 2149 2150static inline void __set_task_blocked_on(struct task_struct *p, struct mutex *m) 2151{ 2152 struct mutex *blocked_on = READ_ONCE(p->blocked_on); 2153 2154 WARN_ON_ONCE(!m); 2155 /* The task should only be setting itself as blocked */ 2156 WARN_ON_ONCE(p != current); 2157 /* Currently we serialize blocked_on under the mutex::wait_lock */ 2158 lockdep_assert_held_once(&m->wait_lock); 2159 /* 2160 * Check ensure we don't overwrite existing mutex value 2161 * with a different mutex. Note, setting it to the same 2162 * lock repeatedly is ok. 2163 */ 2164 WARN_ON_ONCE(blocked_on && blocked_on != m); 2165 WRITE_ONCE(p->blocked_on, m); 2166} 2167 2168static inline void set_task_blocked_on(struct task_struct *p, struct mutex *m) 2169{ 2170 guard(raw_spinlock_irqsave)(&m->wait_lock); 2171 __set_task_blocked_on(p, m); 2172} 2173 2174static inline void __clear_task_blocked_on(struct task_struct *p, struct mutex *m) 2175{ 2176 if (m) { 2177 struct mutex *blocked_on = READ_ONCE(p->blocked_on); 2178 2179 /* Currently we serialize blocked_on under the mutex::wait_lock */ 2180 lockdep_assert_held_once(&m->wait_lock); 2181 /* 2182 * There may be cases where we re-clear already cleared 2183 * blocked_on relationships, but make sure we are not 2184 * clearing the relationship with a different lock. 2185 */ 2186 WARN_ON_ONCE(blocked_on && blocked_on != m); 2187 } 2188 WRITE_ONCE(p->blocked_on, NULL); 2189} 2190 2191static inline void clear_task_blocked_on(struct task_struct *p, struct mutex *m) 2192{ 2193 guard(raw_spinlock_irqsave)(&m->wait_lock); 2194 __clear_task_blocked_on(p, m); 2195} 2196#else 2197static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m) 2198{ 2199} 2200 2201static inline void clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m) 2202{ 2203} 2204#endif /* !CONFIG_PREEMPT_RT */ 2205 2206static __always_inline bool need_resched(void) 2207{ 2208 return unlikely(tif_need_resched()); 2209} 2210 2211/* 2212 * Wrappers for p->thread_info->cpu access. No-op on UP. 2213 */ 2214#ifdef CONFIG_SMP 2215 2216static inline unsigned int task_cpu(const struct task_struct *p) 2217{ 2218 return READ_ONCE(task_thread_info(p)->cpu); 2219} 2220 2221extern void set_task_cpu(struct task_struct *p, unsigned int cpu); 2222 2223#else 2224 2225static inline unsigned int task_cpu(const struct task_struct *p) 2226{ 2227 return 0; 2228} 2229 2230static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) 2231{ 2232} 2233 2234#endif /* CONFIG_SMP */ 2235 2236static inline bool task_is_runnable(struct task_struct *p) 2237{ 2238 return p->on_rq && !p->se.sched_delayed; 2239} 2240 2241extern bool sched_task_on_rq(struct task_struct *p); 2242extern unsigned long get_wchan(struct task_struct *p); 2243extern struct task_struct *cpu_curr_snapshot(int cpu); 2244 2245/* 2246 * In order to reduce various lock holder preemption latencies provide an 2247 * interface to see if a vCPU is currently running or not. 2248 * 2249 * This allows us to terminate optimistic spin loops and block, analogous to 2250 * the native optimistic spin heuristic of testing if the lock owner task is 2251 * running or not. 2252 */ 2253#ifndef vcpu_is_preempted 2254static inline bool vcpu_is_preempted(int cpu) 2255{ 2256 return false; 2257} 2258#endif 2259 2260extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); 2261extern long sched_getaffinity(pid_t pid, struct cpumask *mask); 2262 2263#ifndef TASK_SIZE_OF 2264#define TASK_SIZE_OF(tsk) TASK_SIZE 2265#endif 2266 2267static inline bool owner_on_cpu(struct task_struct *owner) 2268{ 2269 /* 2270 * As lock holder preemption issue, we both skip spinning if 2271 * task is not on cpu or its cpu is preempted 2272 */ 2273 return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(task_cpu(owner)); 2274} 2275 2276/* Returns effective CPU energy utilization, as seen by the scheduler */ 2277unsigned long sched_cpu_util(int cpu); 2278 2279#ifdef CONFIG_SCHED_CORE 2280extern void sched_core_free(struct task_struct *tsk); 2281extern void sched_core_fork(struct task_struct *p); 2282extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, 2283 unsigned long uaddr); 2284extern int sched_core_idle_cpu(int cpu); 2285#else 2286static inline void sched_core_free(struct task_struct *tsk) { } 2287static inline void sched_core_fork(struct task_struct *p) { } 2288static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); } 2289#endif 2290 2291extern void sched_set_stop_task(int cpu, struct task_struct *stop); 2292 2293#ifdef CONFIG_MEM_ALLOC_PROFILING 2294static __always_inline struct alloc_tag *alloc_tag_save(struct alloc_tag *tag) 2295{ 2296 swap(current->alloc_tag, tag); 2297 return tag; 2298} 2299 2300static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct alloc_tag *old) 2301{ 2302#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG 2303 WARN(current->alloc_tag != tag, "current->alloc_tag was changed:\n"); 2304#endif 2305 current->alloc_tag = old; 2306} 2307#else 2308#define alloc_tag_save(_tag) NULL 2309#define alloc_tag_restore(_tag, _old) do {} while (0) 2310#endif 2311 2312/* Avoids recursive inclusion hell */ 2313#ifdef CONFIG_SCHED_MM_CID 2314void sched_mm_cid_before_execve(struct task_struct *t); 2315void sched_mm_cid_after_execve(struct task_struct *t); 2316void sched_mm_cid_exit(struct task_struct *t); 2317static __always_inline int task_mm_cid(struct task_struct *t) 2318{ 2319 return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT); 2320} 2321#else 2322static inline void sched_mm_cid_before_execve(struct task_struct *t) { } 2323static inline void sched_mm_cid_after_execve(struct task_struct *t) { } 2324static inline void sched_mm_cid_exit(struct task_struct *t) { } 2325static __always_inline int task_mm_cid(struct task_struct *t) 2326{ 2327 /* 2328 * Use the processor id as a fall-back when the mm cid feature is 2329 * disabled. This provides functional per-cpu data structure accesses 2330 * in user-space, althrough it won't provide the memory usage benefits. 2331 */ 2332 return task_cpu(t); 2333} 2334#endif 2335 2336#ifndef MODULE 2337#ifndef COMPILE_OFFSETS 2338 2339extern void ___migrate_enable(void); 2340 2341struct rq; 2342DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); 2343 2344/* 2345 * The "struct rq" is not available here, so we can't access the 2346 * "runqueues" with this_cpu_ptr(), as the compilation will fail in 2347 * this_cpu_ptr() -> raw_cpu_ptr() -> __verify_pcpu_ptr(): 2348 * typeof((ptr) + 0) 2349 * 2350 * So use arch_raw_cpu_ptr()/PERCPU_PTR() directly here. 2351 */ 2352#ifdef CONFIG_SMP 2353#define this_rq_raw() arch_raw_cpu_ptr(&runqueues) 2354#else 2355#define this_rq_raw() PERCPU_PTR(&runqueues) 2356#endif 2357#define this_rq_pinned() (*(unsigned int *)((void *)this_rq_raw() + RQ_nr_pinned)) 2358 2359static inline void __migrate_enable(void) 2360{ 2361 struct task_struct *p = current; 2362 2363#ifdef CONFIG_DEBUG_PREEMPT 2364 /* 2365 * Check both overflow from migrate_disable() and superfluous 2366 * migrate_enable(). 2367 */ 2368 if (WARN_ON_ONCE((s16)p->migration_disabled <= 0)) 2369 return; 2370#endif 2371 2372 if (p->migration_disabled > 1) { 2373 p->migration_disabled--; 2374 return; 2375 } 2376 2377 /* 2378 * Ensure stop_task runs either before or after this, and that 2379 * __set_cpus_allowed_ptr(SCA_MIGRATE_ENABLE) doesn't schedule(). 2380 */ 2381 guard(preempt)(); 2382 if (unlikely(p->cpus_ptr != &p->cpus_mask)) 2383 ___migrate_enable(); 2384 /* 2385 * Mustn't clear migration_disabled() until cpus_ptr points back at the 2386 * regular cpus_mask, otherwise things that race (eg. 2387 * select_fallback_rq) get confused. 2388 */ 2389 barrier(); 2390 p->migration_disabled = 0; 2391 this_rq_pinned()--; 2392} 2393 2394static inline void __migrate_disable(void) 2395{ 2396 struct task_struct *p = current; 2397 2398 if (p->migration_disabled) { 2399#ifdef CONFIG_DEBUG_PREEMPT 2400 /* 2401 *Warn about overflow half-way through the range. 2402 */ 2403 WARN_ON_ONCE((s16)p->migration_disabled < 0); 2404#endif 2405 p->migration_disabled++; 2406 return; 2407 } 2408 2409 guard(preempt)(); 2410 this_rq_pinned()++; 2411 p->migration_disabled = 1; 2412} 2413#else /* !COMPILE_OFFSETS */ 2414static inline void __migrate_disable(void) { } 2415static inline void __migrate_enable(void) { } 2416#endif /* !COMPILE_OFFSETS */ 2417 2418/* 2419 * So that it is possible to not export the runqueues variable, define and 2420 * export migrate_enable/migrate_disable in kernel/sched/core.c too, and use 2421 * them for the modules. The macro "INSTANTIATE_EXPORTED_MIGRATE_DISABLE" will 2422 * be defined in kernel/sched/core.c. 2423 */ 2424#ifndef INSTANTIATE_EXPORTED_MIGRATE_DISABLE 2425static __always_inline void migrate_disable(void) 2426{ 2427 __migrate_disable(); 2428} 2429 2430static __always_inline void migrate_enable(void) 2431{ 2432 __migrate_enable(); 2433} 2434#else /* INSTANTIATE_EXPORTED_MIGRATE_DISABLE */ 2435extern void migrate_disable(void); 2436extern void migrate_enable(void); 2437#endif /* INSTANTIATE_EXPORTED_MIGRATE_DISABLE */ 2438 2439#else /* MODULE */ 2440extern void migrate_disable(void); 2441extern void migrate_enable(void); 2442#endif /* MODULE */ 2443 2444DEFINE_LOCK_GUARD_0(migrate, migrate_disable(), migrate_enable()) 2445 2446#endif
~/
Read-only file preview.