FILE
fair.c
File preview with syntax highlighting.
linux-6.19.14/kernel/sched/fair.c
1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <efault@gmx.de> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23#include <linux/energy_model.h> 24#include <linux/mmap_lock.h> 25#include <linux/hugetlb_inline.h> 26#include <linux/jiffies.h> 27#include <linux/mm_api.h> 28#include <linux/highmem.h> 29#include <linux/spinlock_api.h> 30#include <linux/cpumask_api.h> 31#include <linux/lockdep_api.h> 32#include <linux/softirq.h> 33#include <linux/refcount_api.h> 34#include <linux/topology.h> 35#include <linux/sched/clock.h> 36#include <linux/sched/cond_resched.h> 37#include <linux/sched/cputime.h> 38#include <linux/sched/isolation.h> 39#include <linux/sched/nohz.h> 40#include <linux/sched/prio.h> 41 42#include <linux/cpuidle.h> 43#include <linux/interrupt.h> 44#include <linux/memory-tiers.h> 45#include <linux/mempolicy.h> 46#include <linux/mutex_api.h> 47#include <linux/profile.h> 48#include <linux/psi.h> 49#include <linux/ratelimit.h> 50#include <linux/task_work.h> 51#include <linux/rbtree_augmented.h> 52 53#include <asm/switch_to.h> 54 55#include <uapi/linux/sched/types.h> 56 57#include "sched.h" 58#include "stats.h" 59#include "autogroup.h" 60 61/* 62 * The initial- and re-scaling of tunables is configurable 63 * 64 * Options are: 65 * 66 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 67 * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) 68 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 69 * 70 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 71 */ 72unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 73 74/* 75 * Minimal preemption granularity for CPU-bound tasks: 76 * 77 * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) 78 */ 79unsigned int sysctl_sched_base_slice = 700000ULL; 80static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; 81 82__read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; 83 84static int __init setup_sched_thermal_decay_shift(char *str) 85{ 86 pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n"); 87 return 1; 88} 89__setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 90 91/* 92 * For asym packing, by default the lower numbered CPU has higher priority. 93 */ 94int __weak arch_asym_cpu_priority(int cpu) 95{ 96 return -cpu; 97} 98 99/* 100 * The margin used when comparing utilization with CPU capacity. 101 * 102 * (default: ~20%) 103 */ 104#define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 105 106/* 107 * The margin used when comparing CPU capacities. 108 * is 'cap1' noticeably greater than 'cap2' 109 * 110 * (default: ~5%) 111 */ 112#define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 113 114#ifdef CONFIG_CFS_BANDWIDTH 115/* 116 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 117 * each time a cfs_rq requests quota. 118 * 119 * Note: in the case that the slice exceeds the runtime remaining (either due 120 * to consumption or the quota being specified to be smaller than the slice) 121 * we will always only issue the remaining available time. 122 * 123 * (default: 5 msec, units: microseconds) 124 */ 125static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 126#endif 127 128#ifdef CONFIG_NUMA_BALANCING 129/* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 130static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 131#endif 132 133#ifdef CONFIG_SYSCTL 134static const struct ctl_table sched_fair_sysctls[] = { 135#ifdef CONFIG_CFS_BANDWIDTH 136 { 137 .procname = "sched_cfs_bandwidth_slice_us", 138 .data = &sysctl_sched_cfs_bandwidth_slice, 139 .maxlen = sizeof(unsigned int), 140 .mode = 0644, 141 .proc_handler = proc_dointvec_minmax, 142 .extra1 = SYSCTL_ONE, 143 }, 144#endif 145#ifdef CONFIG_NUMA_BALANCING 146 { 147 .procname = "numa_balancing_promote_rate_limit_MBps", 148 .data = &sysctl_numa_balancing_promote_rate_limit, 149 .maxlen = sizeof(unsigned int), 150 .mode = 0644, 151 .proc_handler = proc_dointvec_minmax, 152 .extra1 = SYSCTL_ZERO, 153 }, 154#endif /* CONFIG_NUMA_BALANCING */ 155}; 156 157static int __init sched_fair_sysctl_init(void) 158{ 159 register_sysctl_init("kernel", sched_fair_sysctls); 160 return 0; 161} 162late_initcall(sched_fair_sysctl_init); 163#endif /* CONFIG_SYSCTL */ 164 165static inline void update_load_add(struct load_weight *lw, unsigned long inc) 166{ 167 lw->weight += inc; 168 lw->inv_weight = 0; 169} 170 171static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 172{ 173 lw->weight -= dec; 174 lw->inv_weight = 0; 175} 176 177static inline void update_load_set(struct load_weight *lw, unsigned long w) 178{ 179 lw->weight = w; 180 lw->inv_weight = 0; 181} 182 183/* 184 * Increase the granularity value when there are more CPUs, 185 * because with more CPUs the 'effective latency' as visible 186 * to users decreases. But the relationship is not linear, 187 * so pick a second-best guess by going with the log2 of the 188 * number of CPUs. 189 * 190 * This idea comes from the SD scheduler of Con Kolivas: 191 */ 192static unsigned int get_update_sysctl_factor(void) 193{ 194 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 195 unsigned int factor; 196 197 switch (sysctl_sched_tunable_scaling) { 198 case SCHED_TUNABLESCALING_NONE: 199 factor = 1; 200 break; 201 case SCHED_TUNABLESCALING_LINEAR: 202 factor = cpus; 203 break; 204 case SCHED_TUNABLESCALING_LOG: 205 default: 206 factor = 1 + ilog2(cpus); 207 break; 208 } 209 210 return factor; 211} 212 213static void update_sysctl(void) 214{ 215 unsigned int factor = get_update_sysctl_factor(); 216 217#define SET_SYSCTL(name) \ 218 (sysctl_##name = (factor) * normalized_sysctl_##name) 219 SET_SYSCTL(sched_base_slice); 220#undef SET_SYSCTL 221} 222 223void __init sched_init_granularity(void) 224{ 225 update_sysctl(); 226} 227 228#define WMULT_CONST (~0U) 229#define WMULT_SHIFT 32 230 231static void __update_inv_weight(struct load_weight *lw) 232{ 233 unsigned long w; 234 235 if (likely(lw->inv_weight)) 236 return; 237 238 w = scale_load_down(lw->weight); 239 240 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 241 lw->inv_weight = 1; 242 else if (unlikely(!w)) 243 lw->inv_weight = WMULT_CONST; 244 else 245 lw->inv_weight = WMULT_CONST / w; 246} 247 248/* 249 * delta_exec * weight / lw.weight 250 * OR 251 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 252 * 253 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 254 * we're guaranteed shift stays positive because inv_weight is guaranteed to 255 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 256 * 257 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 258 * weight/lw.weight <= 1, and therefore our shift will also be positive. 259 */ 260static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 261{ 262 u64 fact = scale_load_down(weight); 263 u32 fact_hi = (u32)(fact >> 32); 264 int shift = WMULT_SHIFT; 265 int fs; 266 267 __update_inv_weight(lw); 268 269 if (unlikely(fact_hi)) { 270 fs = fls(fact_hi); 271 shift -= fs; 272 fact >>= fs; 273 } 274 275 fact = mul_u32_u32(fact, lw->inv_weight); 276 277 fact_hi = (u32)(fact >> 32); 278 if (fact_hi) { 279 fs = fls(fact_hi); 280 shift -= fs; 281 fact >>= fs; 282 } 283 284 return mul_u64_u32_shr(delta_exec, fact, shift); 285} 286 287/* 288 * delta /= w 289 */ 290static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 291{ 292 if (unlikely(se->load.weight != NICE_0_LOAD)) 293 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 294 295 return delta; 296} 297 298const struct sched_class fair_sched_class; 299 300/************************************************************** 301 * CFS operations on generic schedulable entities: 302 */ 303 304#ifdef CONFIG_FAIR_GROUP_SCHED 305 306/* Walk up scheduling entities hierarchy */ 307#define for_each_sched_entity(se) \ 308 for (; se; se = se->parent) 309 310static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 311{ 312 struct rq *rq = rq_of(cfs_rq); 313 int cpu = cpu_of(rq); 314 315 if (cfs_rq->on_list) 316 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 317 318 cfs_rq->on_list = 1; 319 320 /* 321 * Ensure we either appear before our parent (if already 322 * enqueued) or force our parent to appear after us when it is 323 * enqueued. The fact that we always enqueue bottom-up 324 * reduces this to two cases and a special case for the root 325 * cfs_rq. Furthermore, it also means that we will always reset 326 * tmp_alone_branch either when the branch is connected 327 * to a tree or when we reach the top of the tree 328 */ 329 if (cfs_rq->tg->parent && 330 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 331 /* 332 * If parent is already on the list, we add the child 333 * just before. Thanks to circular linked property of 334 * the list, this means to put the child at the tail 335 * of the list that starts by parent. 336 */ 337 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 338 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 339 /* 340 * The branch is now connected to its tree so we can 341 * reset tmp_alone_branch to the beginning of the 342 * list. 343 */ 344 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 345 return true; 346 } 347 348 if (!cfs_rq->tg->parent) { 349 /* 350 * cfs rq without parent should be put 351 * at the tail of the list. 352 */ 353 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 354 &rq->leaf_cfs_rq_list); 355 /* 356 * We have reach the top of a tree so we can reset 357 * tmp_alone_branch to the beginning of the list. 358 */ 359 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 360 return true; 361 } 362 363 /* 364 * The parent has not already been added so we want to 365 * make sure that it will be put after us. 366 * tmp_alone_branch points to the begin of the branch 367 * where we will add parent. 368 */ 369 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 370 /* 371 * update tmp_alone_branch to points to the new begin 372 * of the branch 373 */ 374 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 375 return false; 376} 377 378static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 379{ 380 if (cfs_rq->on_list) { 381 struct rq *rq = rq_of(cfs_rq); 382 383 /* 384 * With cfs_rq being unthrottled/throttled during an enqueue, 385 * it can happen the tmp_alone_branch points to the leaf that 386 * we finally want to delete. In this case, tmp_alone_branch moves 387 * to the prev element but it will point to rq->leaf_cfs_rq_list 388 * at the end of the enqueue. 389 */ 390 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 391 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 392 393 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 394 cfs_rq->on_list = 0; 395 } 396} 397 398static inline void assert_list_leaf_cfs_rq(struct rq *rq) 399{ 400 WARN_ON_ONCE(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 401} 402 403/* Iterate through all leaf cfs_rq's on a runqueue */ 404#define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 405 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 406 leaf_cfs_rq_list) 407 408/* Do the two (enqueued) entities belong to the same group ? */ 409static inline struct cfs_rq * 410is_same_group(struct sched_entity *se, struct sched_entity *pse) 411{ 412 if (se->cfs_rq == pse->cfs_rq) 413 return se->cfs_rq; 414 415 return NULL; 416} 417 418static inline struct sched_entity *parent_entity(const struct sched_entity *se) 419{ 420 return se->parent; 421} 422 423static void 424find_matching_se(struct sched_entity **se, struct sched_entity **pse) 425{ 426 int se_depth, pse_depth; 427 428 /* 429 * preemption test can be made between sibling entities who are in the 430 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 431 * both tasks until we find their ancestors who are siblings of common 432 * parent. 433 */ 434 435 /* First walk up until both entities are at same depth */ 436 se_depth = (*se)->depth; 437 pse_depth = (*pse)->depth; 438 439 while (se_depth > pse_depth) { 440 se_depth--; 441 *se = parent_entity(*se); 442 } 443 444 while (pse_depth > se_depth) { 445 pse_depth--; 446 *pse = parent_entity(*pse); 447 } 448 449 while (!is_same_group(*se, *pse)) { 450 *se = parent_entity(*se); 451 *pse = parent_entity(*pse); 452 } 453} 454 455static int tg_is_idle(struct task_group *tg) 456{ 457 return tg->idle > 0; 458} 459 460static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 461{ 462 return cfs_rq->idle > 0; 463} 464 465static int se_is_idle(struct sched_entity *se) 466{ 467 if (entity_is_task(se)) 468 return task_has_idle_policy(task_of(se)); 469 return cfs_rq_is_idle(group_cfs_rq(se)); 470} 471 472#else /* !CONFIG_FAIR_GROUP_SCHED: */ 473 474#define for_each_sched_entity(se) \ 475 for (; se; se = NULL) 476 477static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 478{ 479 return true; 480} 481 482static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 483{ 484} 485 486static inline void assert_list_leaf_cfs_rq(struct rq *rq) 487{ 488} 489 490#define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 491 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 492 493static inline struct sched_entity *parent_entity(struct sched_entity *se) 494{ 495 return NULL; 496} 497 498static inline void 499find_matching_se(struct sched_entity **se, struct sched_entity **pse) 500{ 501} 502 503static inline int tg_is_idle(struct task_group *tg) 504{ 505 return 0; 506} 507 508static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 509{ 510 return 0; 511} 512 513static int se_is_idle(struct sched_entity *se) 514{ 515 return task_has_idle_policy(task_of(se)); 516} 517 518#endif /* !CONFIG_FAIR_GROUP_SCHED */ 519 520static __always_inline 521void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 522 523/************************************************************** 524 * Scheduling class tree data structure manipulation methods: 525 */ 526 527extern void __BUILD_BUG_vruntime_cmp(void); 528 529/* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ 530 531#define vruntime_cmp(A, CMP_STR, B) ({ \ 532 int __res = 0; \ 533 \ 534 if (!__builtin_strcmp(CMP_STR, "<")) { \ 535 __res = ((s64)((A)-(B)) < 0); \ 536 } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ 537 __res = ((s64)((A)-(B)) <= 0); \ 538 } else if (!__builtin_strcmp(CMP_STR, ">")) { \ 539 __res = ((s64)((A)-(B)) > 0); \ 540 } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ 541 __res = ((s64)((A)-(B)) >= 0); \ 542 } else { \ 543 /* Unknown operator throws linker error: */ \ 544 __BUILD_BUG_vruntime_cmp(); \ 545 } \ 546 \ 547 __res; \ 548}) 549 550extern void __BUILD_BUG_vruntime_op(void); 551 552#define vruntime_op(A, OP_STR, B) ({ \ 553 s64 __res = 0; \ 554 \ 555 if (!__builtin_strcmp(OP_STR, "-")) { \ 556 __res = (s64)((A)-(B)); \ 557 } else { \ 558 /* Unknown operator throws linker error: */ \ 559 __BUILD_BUG_vruntime_op(); \ 560 } \ 561 \ 562 __res; \ 563}) 564 565 566static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) 567{ 568 if (vruntime_cmp(vruntime, ">", max_vruntime)) 569 max_vruntime = vruntime; 570 571 return max_vruntime; 572} 573 574static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) 575{ 576 if (vruntime_cmp(vruntime, "<", min_vruntime)) 577 min_vruntime = vruntime; 578 579 return min_vruntime; 580} 581 582static inline bool entity_before(const struct sched_entity *a, 583 const struct sched_entity *b) 584{ 585 /* 586 * Tiebreak on vruntime seems unnecessary since it can 587 * hardly happen. 588 */ 589 return vruntime_cmp(a->deadline, "<", b->deadline); 590} 591 592/* 593 * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale 594 * and this value should be no more than two lag bounds. Which puts it in the 595 * general order of: 596 * 597 * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT 598 * 599 * which is around 44 bits in size (on 64bit); that is 20 for 600 * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for 601 * however many msec the actual slice+tick ends up begin. 602 * 603 * (disregarding the actual divide-by-weight part makes for the worst case 604 * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually 605 * being the zero-lag point). 606 */ 607static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 608{ 609 return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); 610} 611 612#define __node_2_se(node) \ 613 rb_entry((node), struct sched_entity, run_node) 614 615/* 616 * Compute virtual time from the per-task service numbers: 617 * 618 * Fair schedulers conserve lag: 619 * 620 * \Sum lag_i = 0 621 * 622 * Where lag_i is given by: 623 * 624 * lag_i = S - s_i = w_i * (V - v_i) 625 * 626 * Where S is the ideal service time and V is it's virtual time counterpart. 627 * Therefore: 628 * 629 * \Sum lag_i = 0 630 * \Sum w_i * (V - v_i) = 0 631 * \Sum w_i * V - w_i * v_i = 0 632 * 633 * From which we can solve an expression for V in v_i (which we have in 634 * se->vruntime): 635 * 636 * \Sum v_i * w_i \Sum v_i * w_i 637 * V = -------------- = -------------- 638 * \Sum w_i W 639 * 640 * Specifically, this is the weighted average of all entity virtual runtimes. 641 * 642 * [[ NOTE: this is only equal to the ideal scheduler under the condition 643 * that join/leave operations happen at lag_i = 0, otherwise the 644 * virtual time has non-contiguous motion equivalent to: 645 * 646 * V +-= lag_i / W 647 * 648 * Also see the comment in place_entity() that deals with this. ]] 649 * 650 * However, since v_i is u64, and the multiplication could easily overflow 651 * transform it into a relative form that uses smaller quantities: 652 * 653 * Substitute: v_i == (v_i - v0) + v0 654 * 655 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 656 * V = ---------------------------- = --------------------- + v0 657 * W W 658 * 659 * Which we track using: 660 * 661 * v0 := cfs_rq->zero_vruntime 662 * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime 663 * \Sum w_i := cfs_rq->sum_weight 664 * 665 * Since zero_vruntime closely tracks the per-task service, these 666 * deltas: (v_i - v), will be in the order of the maximal (virtual) lag 667 * induced in the system due to quantisation. 668 * 669 * Also, we use scale_load_down() to reduce the size. 670 * 671 * As measured, the max (key * weight) value was ~44 bits for a kernel build. 672 */ 673static void 674sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 675{ 676 unsigned long weight = scale_load_down(se->load.weight); 677 s64 key = entity_key(cfs_rq, se); 678 679 cfs_rq->sum_w_vruntime += key * weight; 680 cfs_rq->sum_weight += weight; 681} 682 683static void 684sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 685{ 686 unsigned long weight = scale_load_down(se->load.weight); 687 s64 key = entity_key(cfs_rq, se); 688 689 cfs_rq->sum_w_vruntime -= key * weight; 690 cfs_rq->sum_weight -= weight; 691} 692 693static inline 694void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) 695{ 696 /* 697 * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight 698 */ 699 cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; 700 cfs_rq->zero_vruntime += delta; 701} 702 703/* 704 * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true 705 * For this to be so, the result of this function must have a left bias. 706 * 707 * Called in: 708 * - place_entity() -- before enqueue 709 * - update_entity_lag() -- before dequeue 710 * - update_deadline() -- slice expiration 711 * 712 * This means it is one entry 'behind' but that puts it close enough to where 713 * the bound on entity_key() is at most two lag bounds. 714 */ 715u64 avg_vruntime(struct cfs_rq *cfs_rq) 716{ 717 struct sched_entity *curr = cfs_rq->curr; 718 long weight = cfs_rq->sum_weight; 719 s64 delta = 0; 720 721 if (curr && !curr->on_rq) 722 curr = NULL; 723 724 if (weight) { 725 s64 runtime = cfs_rq->sum_w_vruntime; 726 727 if (curr) { 728 unsigned long w = scale_load_down(curr->load.weight); 729 730 runtime += entity_key(cfs_rq, curr) * w; 731 weight += w; 732 } 733 734 /* sign flips effective floor / ceiling */ 735 if (runtime < 0) 736 runtime -= (weight - 1); 737 738 delta = div_s64(runtime, weight); 739 } else if (curr) { 740 /* 741 * When there is but one element, it is the average. 742 */ 743 delta = curr->vruntime - cfs_rq->zero_vruntime; 744 } 745 746 update_zero_vruntime(cfs_rq, delta); 747 748 return cfs_rq->zero_vruntime; 749} 750 751static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); 752 753/* 754 * lag_i = S - s_i = w_i * (V - v_i) 755 * 756 * However, since V is approximated by the weighted average of all entities it 757 * is possible -- by addition/removal/reweight to the tree -- to move V around 758 * and end up with a larger lag than we started with. 759 * 760 * Limit this to either double the slice length with a minimum of TICK_NSEC 761 * since that is the timing granularity. 762 * 763 * EEVDF gives the following limit for a steady state system: 764 * 765 * -r_max < lag < max(r_max, q) 766 */ 767static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 768{ 769 u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; 770 s64 vlag, limit; 771 772 WARN_ON_ONCE(!se->on_rq); 773 774 vlag = avg_vruntime(cfs_rq) - se->vruntime; 775 limit = calc_delta_fair(max_slice, se); 776 777 se->vlag = clamp(vlag, -limit, limit); 778} 779 780/* 781 * Entity is eligible once it received less service than it ought to have, 782 * eg. lag >= 0. 783 * 784 * lag_i = S - s_i = w_i*(V - v_i) 785 * 786 * lag_i >= 0 -> V >= v_i 787 * 788 * \Sum (v_i - v)*w_i 789 * V = ------------------ + v 790 * \Sum w_i 791 * 792 * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i) 793 * 794 * Note: using 'avg_vruntime() > se->vruntime' is inaccurate due 795 * to the loss in precision caused by the division. 796 */ 797static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) 798{ 799 struct sched_entity *curr = cfs_rq->curr; 800 s64 avg = cfs_rq->sum_w_vruntime; 801 long load = cfs_rq->sum_weight; 802 803 if (curr && curr->on_rq) { 804 unsigned long weight = scale_load_down(curr->load.weight); 805 806 avg += entity_key(cfs_rq, curr) * weight; 807 load += weight; 808 } 809 810 return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; 811} 812 813int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 814{ 815 return vruntime_eligible(cfs_rq, se->vruntime); 816} 817 818static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) 819{ 820 struct sched_entity *root = __pick_root_entity(cfs_rq); 821 struct sched_entity *curr = cfs_rq->curr; 822 u64 min_slice = ~0ULL; 823 824 if (curr && curr->on_rq) 825 min_slice = curr->slice; 826 827 if (root) 828 min_slice = min(min_slice, root->min_slice); 829 830 return min_slice; 831} 832 833static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) 834{ 835 struct sched_entity *root = __pick_root_entity(cfs_rq); 836 struct sched_entity *curr = cfs_rq->curr; 837 u64 max_slice = 0ULL; 838 839 if (curr && curr->on_rq) 840 max_slice = curr->slice; 841 842 if (root) 843 max_slice = max(max_slice, root->max_slice); 844 845 return max_slice; 846} 847 848static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 849{ 850 return entity_before(__node_2_se(a), __node_2_se(b)); 851} 852 853static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) 854{ 855 if (node) { 856 struct sched_entity *rse = __node_2_se(node); 857 858 if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) 859 se->min_vruntime = rse->min_vruntime; 860 } 861} 862 863static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) 864{ 865 if (node) { 866 struct sched_entity *rse = __node_2_se(node); 867 if (rse->min_slice < se->min_slice) 868 se->min_slice = rse->min_slice; 869 } 870} 871 872static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) 873{ 874 if (node) { 875 struct sched_entity *rse = __node_2_se(node); 876 if (rse->max_slice > se->max_slice) 877 se->max_slice = rse->max_slice; 878 } 879} 880 881/* 882 * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) 883 */ 884static inline bool min_vruntime_update(struct sched_entity *se, bool exit) 885{ 886 u64 old_min_vruntime = se->min_vruntime; 887 u64 old_min_slice = se->min_slice; 888 u64 old_max_slice = se->max_slice; 889 struct rb_node *node = &se->run_node; 890 891 se->min_vruntime = se->vruntime; 892 __min_vruntime_update(se, node->rb_right); 893 __min_vruntime_update(se, node->rb_left); 894 895 se->min_slice = se->slice; 896 __min_slice_update(se, node->rb_right); 897 __min_slice_update(se, node->rb_left); 898 899 se->max_slice = se->slice; 900 __max_slice_update(se, node->rb_right); 901 __max_slice_update(se, node->rb_left); 902 903 return se->min_vruntime == old_min_vruntime && 904 se->min_slice == old_min_slice && 905 se->max_slice == old_max_slice; 906} 907 908RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, 909 run_node, min_vruntime, min_vruntime_update); 910 911/* 912 * Enqueue an entity into the rb-tree: 913 */ 914static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 915{ 916 sum_w_vruntime_add(cfs_rq, se); 917 se->min_vruntime = se->vruntime; 918 se->min_slice = se->slice; 919 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 920 __entity_less, &min_vruntime_cb); 921} 922 923static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 924{ 925 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 926 &min_vruntime_cb); 927 sum_w_vruntime_sub(cfs_rq, se); 928} 929 930struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) 931{ 932 struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node; 933 934 if (!root) 935 return NULL; 936 937 return __node_2_se(root); 938} 939 940struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 941{ 942 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 943 944 if (!left) 945 return NULL; 946 947 return __node_2_se(left); 948} 949 950/* 951 * Set the vruntime up to which an entity can run before looking 952 * for another entity to pick. 953 * In case of run to parity, we use the shortest slice of the enqueued 954 * entities to set the protected period. 955 * When run to parity is disabled, we give a minimum quantum to the running 956 * entity to ensure progress. 957 */ 958static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 959{ 960 u64 slice = normalized_sysctl_sched_base_slice; 961 u64 vprot = se->deadline; 962 963 if (sched_feat(RUN_TO_PARITY)) 964 slice = cfs_rq_min_slice(cfs_rq); 965 966 slice = min(slice, se->slice); 967 if (slice != se->slice) 968 vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); 969 970 se->vprot = vprot; 971} 972 973static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 974{ 975 u64 slice = cfs_rq_min_slice(cfs_rq); 976 977 se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); 978} 979 980static inline bool protect_slice(struct sched_entity *se) 981{ 982 return vruntime_cmp(se->vruntime, "<", se->vprot); 983} 984 985static inline void cancel_protect_slice(struct sched_entity *se) 986{ 987 if (protect_slice(se)) 988 se->vprot = se->vruntime; 989} 990 991/* 992 * Earliest Eligible Virtual Deadline First 993 * 994 * In order to provide latency guarantees for different request sizes 995 * EEVDF selects the best runnable task from two criteria: 996 * 997 * 1) the task must be eligible (must be owed service) 998 * 999 * 2) from those tasks that meet 1), we select the one 1000 * with the earliest virtual deadline. 1001 * 1002 * We can do this in O(log n) time due to an augmented RB-tree. The 1003 * tree keeps the entries sorted on deadline, but also functions as a 1004 * heap based on the vruntime by keeping: 1005 * 1006 * se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime) 1007 * 1008 * Which allows tree pruning through eligibility. 1009 */ 1010static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect) 1011{ 1012 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 1013 struct sched_entity *se = __pick_first_entity(cfs_rq); 1014 struct sched_entity *curr = cfs_rq->curr; 1015 struct sched_entity *best = NULL; 1016 1017 /* 1018 * We can safely skip eligibility check if there is only one entity 1019 * in this cfs_rq, saving some cycles. 1020 */ 1021 if (cfs_rq->nr_queued == 1) 1022 return curr && curr->on_rq ? curr : se; 1023 1024 /* 1025 * Picking the ->next buddy will affect latency but not fairness. 1026 */ 1027 if (sched_feat(PICK_BUDDY) && 1028 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) { 1029 /* ->next will never be delayed */ 1030 WARN_ON_ONCE(cfs_rq->next->sched_delayed); 1031 return cfs_rq->next; 1032 } 1033 1034 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 1035 curr = NULL; 1036 1037 if (curr && protect && protect_slice(curr)) 1038 return curr; 1039 1040 /* Pick the leftmost entity if it's eligible */ 1041 if (se && entity_eligible(cfs_rq, se)) { 1042 best = se; 1043 goto found; 1044 } 1045 1046 /* Heap search for the EEVD entity */ 1047 while (node) { 1048 struct rb_node *left = node->rb_left; 1049 1050 /* 1051 * Eligible entities in left subtree are always better 1052 * choices, since they have earlier deadlines. 1053 */ 1054 if (left && vruntime_eligible(cfs_rq, 1055 __node_2_se(left)->min_vruntime)) { 1056 node = left; 1057 continue; 1058 } 1059 1060 se = __node_2_se(node); 1061 1062 /* 1063 * The left subtree either is empty or has no eligible 1064 * entity, so check the current node since it is the one 1065 * with earliest deadline that might be eligible. 1066 */ 1067 if (entity_eligible(cfs_rq, se)) { 1068 best = se; 1069 break; 1070 } 1071 1072 node = node->rb_right; 1073 } 1074found: 1075 if (!best || (curr && entity_before(curr, best))) 1076 best = curr; 1077 1078 return best; 1079} 1080 1081static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) 1082{ 1083 return __pick_eevdf(cfs_rq, true); 1084} 1085 1086struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 1087{ 1088 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 1089 1090 if (!last) 1091 return NULL; 1092 1093 return __node_2_se(last); 1094} 1095 1096/************************************************************** 1097 * Scheduling class statistics methods: 1098 */ 1099int sched_update_scaling(void) 1100{ 1101 unsigned int factor = get_update_sysctl_factor(); 1102 1103#define WRT_SYSCTL(name) \ 1104 (normalized_sysctl_##name = sysctl_##name / (factor)) 1105 WRT_SYSCTL(sched_base_slice); 1106#undef WRT_SYSCTL 1107 1108 return 0; 1109} 1110 1111static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1112 1113/* 1114 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1115 * this is probably good enough. 1116 */ 1117static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1118{ 1119 if (vruntime_cmp(se->vruntime, "<", se->deadline)) 1120 return false; 1121 1122 /* 1123 * For EEVDF the virtual time slope is determined by w_i (iow. 1124 * nice) while the request time r_i is determined by 1125 * sysctl_sched_base_slice. 1126 */ 1127 if (!se->custom_slice) 1128 se->slice = sysctl_sched_base_slice; 1129 1130 /* 1131 * EEVDF: vd_i = ve_i + r_i / w_i 1132 */ 1133 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1134 avg_vruntime(cfs_rq); 1135 1136 /* 1137 * The task has consumed its request, reschedule. 1138 */ 1139 return true; 1140} 1141 1142#include "pelt.h" 1143 1144static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1145static unsigned long task_h_load(struct task_struct *p); 1146static unsigned long capacity_of(int cpu); 1147 1148/* Give new sched_entity start runnable values to heavy its load in infant time */ 1149void init_entity_runnable_average(struct sched_entity *se) 1150{ 1151 struct sched_avg *sa = &se->avg; 1152 1153 memset(sa, 0, sizeof(*sa)); 1154 1155 /* 1156 * Tasks are initialized with full load to be seen as heavy tasks until 1157 * they get a chance to stabilize to their real load level. 1158 * Group entities are initialized with zero load to reflect the fact that 1159 * nothing has been attached to the task group yet. 1160 */ 1161 if (entity_is_task(se)) 1162 sa->load_avg = scale_load_down(se->load.weight); 1163 1164 /* when this task is enqueued, it will contribute to its cfs_rq's load_avg */ 1165} 1166 1167/* 1168 * With new tasks being created, their initial util_avgs are extrapolated 1169 * based on the cfs_rq's current util_avg: 1170 * 1171 * util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1) 1172 * * se_weight(se) 1173 * 1174 * However, in many cases, the above util_avg does not give a desired 1175 * value. Moreover, the sum of the util_avgs may be divergent, such 1176 * as when the series is a harmonic series. 1177 * 1178 * To solve this problem, we also cap the util_avg of successive tasks to 1179 * only 1/2 of the left utilization budget: 1180 * 1181 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1182 * 1183 * where n denotes the nth task and cpu_scale the CPU capacity. 1184 * 1185 * For example, for a CPU with 1024 of capacity, a simplest series from 1186 * the beginning would be like: 1187 * 1188 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1189 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1190 * 1191 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1192 * if util_avg > util_avg_cap. 1193 */ 1194void post_init_entity_util_avg(struct task_struct *p) 1195{ 1196 struct sched_entity *se = &p->se; 1197 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1198 struct sched_avg *sa = &se->avg; 1199 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1200 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1201 1202 if (p->sched_class != &fair_sched_class) { 1203 /* 1204 * For !fair tasks do: 1205 * 1206 update_cfs_rq_load_avg(now, cfs_rq); 1207 attach_entity_load_avg(cfs_rq, se); 1208 switched_from_fair(rq, p); 1209 * 1210 * such that the next switched_to_fair() has the 1211 * expected state. 1212 */ 1213 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1214 return; 1215 } 1216 1217 if (cap > 0) { 1218 if (cfs_rq->avg.util_avg != 0) { 1219 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); 1220 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1221 1222 if (sa->util_avg > cap) 1223 sa->util_avg = cap; 1224 } else { 1225 sa->util_avg = cap; 1226 } 1227 } 1228 1229 sa->runnable_avg = sa->util_avg; 1230} 1231 1232static s64 update_se(struct rq *rq, struct sched_entity *se) 1233{ 1234 u64 now = rq_clock_task(rq); 1235 s64 delta_exec; 1236 1237 delta_exec = now - se->exec_start; 1238 if (unlikely(delta_exec <= 0)) 1239 return delta_exec; 1240 1241 se->exec_start = now; 1242 if (entity_is_task(se)) { 1243 struct task_struct *donor = task_of(se); 1244 struct task_struct *running = rq->curr; 1245 /* 1246 * If se is a task, we account the time against the running 1247 * task, as w/ proxy-exec they may not be the same. 1248 */ 1249 running->se.exec_start = now; 1250 running->se.sum_exec_runtime += delta_exec; 1251 1252 trace_sched_stat_runtime(running, delta_exec); 1253 account_group_exec_runtime(running, delta_exec); 1254 1255 /* cgroup time is always accounted against the donor */ 1256 cgroup_account_cputime(donor, delta_exec); 1257 } else { 1258 /* If not task, account the time against donor se */ 1259 se->sum_exec_runtime += delta_exec; 1260 } 1261 1262 if (schedstat_enabled()) { 1263 struct sched_statistics *stats; 1264 1265 stats = __schedstats_from_se(se); 1266 __schedstat_set(stats->exec_max, 1267 max(delta_exec, stats->exec_max)); 1268 } 1269 1270 return delta_exec; 1271} 1272 1273static void set_next_buddy(struct sched_entity *se); 1274 1275/* 1276 * Used by other classes to account runtime. 1277 */ 1278s64 update_curr_common(struct rq *rq) 1279{ 1280 return update_se(rq, &rq->donor->se); 1281} 1282 1283/* 1284 * Update the current task's runtime statistics. 1285 */ 1286static void update_curr(struct cfs_rq *cfs_rq) 1287{ 1288 /* 1289 * Note: cfs_rq->curr corresponds to the task picked to 1290 * run (ie: rq->donor.se) which due to proxy-exec may 1291 * not necessarily be the actual task running 1292 * (rq->curr.se). This is easy to confuse! 1293 */ 1294 struct sched_entity *curr = cfs_rq->curr; 1295 struct rq *rq = rq_of(cfs_rq); 1296 s64 delta_exec; 1297 bool resched; 1298 1299 if (unlikely(!curr)) 1300 return; 1301 1302 delta_exec = update_se(rq, curr); 1303 if (unlikely(delta_exec <= 0)) 1304 return; 1305 1306 curr->vruntime += calc_delta_fair(delta_exec, curr); 1307 resched = update_deadline(cfs_rq, curr); 1308 1309 if (entity_is_task(curr)) { 1310 /* 1311 * If the fair_server is active, we need to account for the 1312 * fair_server time whether or not the task is running on 1313 * behalf of fair_server or not: 1314 * - If the task is running on behalf of fair_server, we need 1315 * to limit its time based on the assigned runtime. 1316 * - Fair task that runs outside of fair_server should account 1317 * against fair_server such that it can account for this time 1318 * and possibly avoid running this period. 1319 */ 1320 dl_server_update(&rq->fair_server, delta_exec); 1321 } 1322 1323 account_cfs_rq_runtime(cfs_rq, delta_exec); 1324 1325 if (cfs_rq->nr_queued == 1) 1326 return; 1327 1328 if (resched || !protect_slice(curr)) { 1329 resched_curr_lazy(rq); 1330 clear_buddies(cfs_rq, curr); 1331 } 1332} 1333 1334static void update_curr_fair(struct rq *rq) 1335{ 1336 update_curr(cfs_rq_of(&rq->donor->se)); 1337} 1338 1339static inline void 1340update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1341{ 1342 struct sched_statistics *stats; 1343 struct task_struct *p = NULL; 1344 1345 if (!schedstat_enabled()) 1346 return; 1347 1348 stats = __schedstats_from_se(se); 1349 1350 if (entity_is_task(se)) 1351 p = task_of(se); 1352 1353 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 1354} 1355 1356static inline void 1357update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1358{ 1359 struct sched_statistics *stats; 1360 struct task_struct *p = NULL; 1361 1362 if (!schedstat_enabled()) 1363 return; 1364 1365 stats = __schedstats_from_se(se); 1366 1367 /* 1368 * When the sched_schedstat changes from 0 to 1, some sched se 1369 * maybe already in the runqueue, the se->statistics.wait_start 1370 * will be 0.So it will let the delta wrong. We need to avoid this 1371 * scenario. 1372 */ 1373 if (unlikely(!schedstat_val(stats->wait_start))) 1374 return; 1375 1376 if (entity_is_task(se)) 1377 p = task_of(se); 1378 1379 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 1380} 1381 1382static inline void 1383update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1384{ 1385 struct sched_statistics *stats; 1386 struct task_struct *tsk = NULL; 1387 1388 if (!schedstat_enabled()) 1389 return; 1390 1391 stats = __schedstats_from_se(se); 1392 1393 if (entity_is_task(se)) 1394 tsk = task_of(se); 1395 1396 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1397} 1398 1399/* 1400 * Task is being enqueued - update stats: 1401 */ 1402static inline void 1403update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1404{ 1405 if (!schedstat_enabled()) 1406 return; 1407 1408 /* 1409 * Are we enqueueing a waiting task? (for current tasks 1410 * a dequeue/enqueue event is a NOP) 1411 */ 1412 if (se != cfs_rq->curr) 1413 update_stats_wait_start_fair(cfs_rq, se); 1414 1415 if (flags & ENQUEUE_WAKEUP) 1416 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1417} 1418 1419static inline void 1420update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1421{ 1422 1423 if (!schedstat_enabled()) 1424 return; 1425 1426 /* 1427 * Mark the end of the wait period if dequeueing a 1428 * waiting task: 1429 */ 1430 if (se != cfs_rq->curr) 1431 update_stats_wait_end_fair(cfs_rq, se); 1432 1433 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1434 struct task_struct *tsk = task_of(se); 1435 unsigned int state; 1436 1437 /* XXX racy against TTWU */ 1438 state = READ_ONCE(tsk->__state); 1439 if (state & TASK_INTERRUPTIBLE) 1440 __schedstat_set(tsk->stats.sleep_start, 1441 rq_clock(rq_of(cfs_rq))); 1442 if (state & TASK_UNINTERRUPTIBLE) 1443 __schedstat_set(tsk->stats.block_start, 1444 rq_clock(rq_of(cfs_rq))); 1445 } 1446} 1447 1448/* 1449 * We are picking a new current task - update its stats: 1450 */ 1451static inline void 1452update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1453{ 1454 /* 1455 * We are starting a new run period: 1456 */ 1457 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1458} 1459 1460/************************************************** 1461 * Scheduling class queueing methods: 1462 */ 1463 1464static inline bool is_core_idle(int cpu) 1465{ 1466#ifdef CONFIG_SCHED_SMT 1467 int sibling; 1468 1469 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1470 if (cpu == sibling) 1471 continue; 1472 1473 if (!idle_cpu(sibling)) 1474 return false; 1475 } 1476#endif 1477 1478 return true; 1479} 1480 1481#ifdef CONFIG_NUMA 1482#define NUMA_IMBALANCE_MIN 2 1483 1484static inline long 1485adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1486{ 1487 /* 1488 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1489 * threshold. Above this threshold, individual tasks may be contending 1490 * for both memory bandwidth and any shared HT resources. This is an 1491 * approximation as the number of running tasks may not be related to 1492 * the number of busy CPUs due to sched_setaffinity. 1493 */ 1494 if (dst_running > imb_numa_nr) 1495 return imbalance; 1496 1497 /* 1498 * Allow a small imbalance based on a simple pair of communicating 1499 * tasks that remain local when the destination is lightly loaded. 1500 */ 1501 if (imbalance <= NUMA_IMBALANCE_MIN) 1502 return 0; 1503 1504 return imbalance; 1505} 1506#endif /* CONFIG_NUMA */ 1507 1508#ifdef CONFIG_NUMA_BALANCING 1509/* 1510 * Approximate time to scan a full NUMA task in ms. The task scan period is 1511 * calculated based on the tasks virtual memory size and 1512 * numa_balancing_scan_size. 1513 */ 1514unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1515unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1516 1517/* Portion of address space to scan in MB */ 1518unsigned int sysctl_numa_balancing_scan_size = 256; 1519 1520/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1521unsigned int sysctl_numa_balancing_scan_delay = 1000; 1522 1523/* The page with hint page fault latency < threshold in ms is considered hot */ 1524unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1525 1526struct numa_group { 1527 refcount_t refcount; 1528 1529 spinlock_t lock; /* nr_tasks, tasks */ 1530 int nr_tasks; 1531 pid_t gid; 1532 int active_nodes; 1533 1534 struct rcu_head rcu; 1535 unsigned long total_faults; 1536 unsigned long max_faults_cpu; 1537 /* 1538 * faults[] array is split into two regions: faults_mem and faults_cpu. 1539 * 1540 * Faults_cpu is used to decide whether memory should move 1541 * towards the CPU. As a consequence, these stats are weighted 1542 * more by CPU use than by memory faults. 1543 */ 1544 unsigned long faults[]; 1545}; 1546 1547/* 1548 * For functions that can be called in multiple contexts that permit reading 1549 * ->numa_group (see struct task_struct for locking rules). 1550 */ 1551static struct numa_group *deref_task_numa_group(struct task_struct *p) 1552{ 1553 return rcu_dereference_check(p->numa_group, p == current || 1554 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1555} 1556 1557static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1558{ 1559 return rcu_dereference_protected(p->numa_group, p == current); 1560} 1561 1562static inline unsigned long group_faults_priv(struct numa_group *ng); 1563static inline unsigned long group_faults_shared(struct numa_group *ng); 1564 1565static unsigned int task_nr_scan_windows(struct task_struct *p) 1566{ 1567 unsigned long rss = 0; 1568 unsigned long nr_scan_pages; 1569 1570 /* 1571 * Calculations based on RSS as non-present and empty pages are skipped 1572 * by the PTE scanner and NUMA hinting faults should be trapped based 1573 * on resident pages 1574 */ 1575 nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); 1576 rss = get_mm_rss(p->mm); 1577 if (!rss) 1578 rss = nr_scan_pages; 1579 1580 rss = round_up(rss, nr_scan_pages); 1581 return rss / nr_scan_pages; 1582} 1583 1584/* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1585#define MAX_SCAN_WINDOW 2560 1586 1587static unsigned int task_scan_min(struct task_struct *p) 1588{ 1589 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1590 unsigned int scan, floor; 1591 unsigned int windows = 1; 1592 1593 if (scan_size < MAX_SCAN_WINDOW) 1594 windows = MAX_SCAN_WINDOW / scan_size; 1595 floor = 1000 / windows; 1596 1597 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1598 return max_t(unsigned int, floor, scan); 1599} 1600 1601static unsigned int task_scan_start(struct task_struct *p) 1602{ 1603 unsigned long smin = task_scan_min(p); 1604 unsigned long period = smin; 1605 struct numa_group *ng; 1606 1607 /* Scale the maximum scan period with the amount of shared memory. */ 1608 rcu_read_lock(); 1609 ng = rcu_dereference(p->numa_group); 1610 if (ng) { 1611 unsigned long shared = group_faults_shared(ng); 1612 unsigned long private = group_faults_priv(ng); 1613 1614 period *= refcount_read(&ng->refcount); 1615 period *= shared + 1; 1616 period /= private + shared + 1; 1617 } 1618 rcu_read_unlock(); 1619 1620 return max(smin, period); 1621} 1622 1623static unsigned int task_scan_max(struct task_struct *p) 1624{ 1625 unsigned long smin = task_scan_min(p); 1626 unsigned long smax; 1627 struct numa_group *ng; 1628 1629 /* Watch for min being lower than max due to floor calculations */ 1630 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1631 1632 /* Scale the maximum scan period with the amount of shared memory. */ 1633 ng = deref_curr_numa_group(p); 1634 if (ng) { 1635 unsigned long shared = group_faults_shared(ng); 1636 unsigned long private = group_faults_priv(ng); 1637 unsigned long period = smax; 1638 1639 period *= refcount_read(&ng->refcount); 1640 period *= shared + 1; 1641 period /= private + shared + 1; 1642 1643 smax = max(smax, period); 1644 } 1645 1646 return max(smin, smax); 1647} 1648 1649static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1650{ 1651 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1652 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1653} 1654 1655static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1656{ 1657 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1658 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1659} 1660 1661/* Shared or private faults. */ 1662#define NR_NUMA_HINT_FAULT_TYPES 2 1663 1664/* Memory and CPU locality */ 1665#define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1666 1667/* Averaged statistics, and temporary buffers. */ 1668#define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1669 1670pid_t task_numa_group_id(struct task_struct *p) 1671{ 1672 struct numa_group *ng; 1673 pid_t gid = 0; 1674 1675 rcu_read_lock(); 1676 ng = rcu_dereference(p->numa_group); 1677 if (ng) 1678 gid = ng->gid; 1679 rcu_read_unlock(); 1680 1681 return gid; 1682} 1683 1684/* 1685 * The averaged statistics, shared & private, memory & CPU, 1686 * occupy the first half of the array. The second half of the 1687 * array is for current counters, which are averaged into the 1688 * first set by task_numa_placement. 1689 */ 1690static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1691{ 1692 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1693} 1694 1695static inline unsigned long task_faults(struct task_struct *p, int nid) 1696{ 1697 if (!p->numa_faults) 1698 return 0; 1699 1700 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1701 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1702} 1703 1704static inline unsigned long group_faults(struct task_struct *p, int nid) 1705{ 1706 struct numa_group *ng = deref_task_numa_group(p); 1707 1708 if (!ng) 1709 return 0; 1710 1711 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1712 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1713} 1714 1715static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1716{ 1717 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1718 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1719} 1720 1721static inline unsigned long group_faults_priv(struct numa_group *ng) 1722{ 1723 unsigned long faults = 0; 1724 int node; 1725 1726 for_each_online_node(node) { 1727 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1728 } 1729 1730 return faults; 1731} 1732 1733static inline unsigned long group_faults_shared(struct numa_group *ng) 1734{ 1735 unsigned long faults = 0; 1736 int node; 1737 1738 for_each_online_node(node) { 1739 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1740 } 1741 1742 return faults; 1743} 1744 1745/* 1746 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1747 * considered part of a numa group's pseudo-interleaving set. Migrations 1748 * between these nodes are slowed down, to allow things to settle down. 1749 */ 1750#define ACTIVE_NODE_FRACTION 3 1751 1752static bool numa_is_active_node(int nid, struct numa_group *ng) 1753{ 1754 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1755} 1756 1757/* Handle placement on systems where not all nodes are directly connected. */ 1758static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1759 int lim_dist, bool task) 1760{ 1761 unsigned long score = 0; 1762 int node, max_dist; 1763 1764 /* 1765 * All nodes are directly connected, and the same distance 1766 * from each other. No need for fancy placement algorithms. 1767 */ 1768 if (sched_numa_topology_type == NUMA_DIRECT) 1769 return 0; 1770 1771 /* sched_max_numa_distance may be changed in parallel. */ 1772 max_dist = READ_ONCE(sched_max_numa_distance); 1773 /* 1774 * This code is called for each node, introducing N^2 complexity, 1775 * which should be OK given the number of nodes rarely exceeds 8. 1776 */ 1777 for_each_online_node(node) { 1778 unsigned long faults; 1779 int dist = node_distance(nid, node); 1780 1781 /* 1782 * The furthest away nodes in the system are not interesting 1783 * for placement; nid was already counted. 1784 */ 1785 if (dist >= max_dist || node == nid) 1786 continue; 1787 1788 /* 1789 * On systems with a backplane NUMA topology, compare groups 1790 * of nodes, and move tasks towards the group with the most 1791 * memory accesses. When comparing two nodes at distance 1792 * "hoplimit", only nodes closer by than "hoplimit" are part 1793 * of each group. Skip other nodes. 1794 */ 1795 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1796 continue; 1797 1798 /* Add up the faults from nearby nodes. */ 1799 if (task) 1800 faults = task_faults(p, node); 1801 else 1802 faults = group_faults(p, node); 1803 1804 /* 1805 * On systems with a glueless mesh NUMA topology, there are 1806 * no fixed "groups of nodes". Instead, nodes that are not 1807 * directly connected bounce traffic through intermediate 1808 * nodes; a numa_group can occupy any set of nodes. 1809 * The further away a node is, the less the faults count. 1810 * This seems to result in good task placement. 1811 */ 1812 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1813 faults *= (max_dist - dist); 1814 faults /= (max_dist - LOCAL_DISTANCE); 1815 } 1816 1817 score += faults; 1818 } 1819 1820 return score; 1821} 1822 1823/* 1824 * These return the fraction of accesses done by a particular task, or 1825 * task group, on a particular numa node. The group weight is given a 1826 * larger multiplier, in order to group tasks together that are almost 1827 * evenly spread out between numa nodes. 1828 */ 1829static inline unsigned long task_weight(struct task_struct *p, int nid, 1830 int dist) 1831{ 1832 unsigned long faults, total_faults; 1833 1834 if (!p->numa_faults) 1835 return 0; 1836 1837 total_faults = p->total_numa_faults; 1838 1839 if (!total_faults) 1840 return 0; 1841 1842 faults = task_faults(p, nid); 1843 faults += score_nearby_nodes(p, nid, dist, true); 1844 1845 return 1000 * faults / total_faults; 1846} 1847 1848static inline unsigned long group_weight(struct task_struct *p, int nid, 1849 int dist) 1850{ 1851 struct numa_group *ng = deref_task_numa_group(p); 1852 unsigned long faults, total_faults; 1853 1854 if (!ng) 1855 return 0; 1856 1857 total_faults = ng->total_faults; 1858 1859 if (!total_faults) 1860 return 0; 1861 1862 faults = group_faults(p, nid); 1863 faults += score_nearby_nodes(p, nid, dist, false); 1864 1865 return 1000 * faults / total_faults; 1866} 1867 1868/* 1869 * If memory tiering mode is enabled, cpupid of slow memory page is 1870 * used to record scan time instead of CPU and PID. When tiering mode 1871 * is disabled at run time, the scan time (in cpupid) will be 1872 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1873 * access out of array bound. 1874 */ 1875static inline bool cpupid_valid(int cpupid) 1876{ 1877 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1878} 1879 1880/* 1881 * For memory tiering mode, if there are enough free pages (more than 1882 * enough watermark defined here) in fast memory node, to take full 1883 * advantage of fast memory capacity, all recently accessed slow 1884 * memory pages will be migrated to fast memory node without 1885 * considering hot threshold. 1886 */ 1887static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1888{ 1889 int z; 1890 unsigned long enough_wmark; 1891 1892 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1893 pgdat->node_present_pages >> 4); 1894 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1895 struct zone *zone = pgdat->node_zones + z; 1896 1897 if (!populated_zone(zone)) 1898 continue; 1899 1900 if (zone_watermark_ok(zone, 0, 1901 promo_wmark_pages(zone) + enough_wmark, 1902 ZONE_MOVABLE, 0)) 1903 return true; 1904 } 1905 return false; 1906} 1907 1908/* 1909 * For memory tiering mode, when page tables are scanned, the scan 1910 * time will be recorded in struct page in addition to make page 1911 * PROT_NONE for slow memory page. So when the page is accessed, in 1912 * hint page fault handler, the hint page fault latency is calculated 1913 * via, 1914 * 1915 * hint page fault latency = hint page fault time - scan time 1916 * 1917 * The smaller the hint page fault latency, the higher the possibility 1918 * for the page to be hot. 1919 */ 1920static int numa_hint_fault_latency(struct folio *folio) 1921{ 1922 int last_time, time; 1923 1924 time = jiffies_to_msecs(jiffies); 1925 last_time = folio_xchg_access_time(folio, time); 1926 1927 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1928} 1929 1930/* 1931 * For memory tiering mode, too high promotion/demotion throughput may 1932 * hurt application latency. So we provide a mechanism to rate limit 1933 * the number of pages that are tried to be promoted. 1934 */ 1935static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1936 unsigned long rate_limit, int nr) 1937{ 1938 unsigned long nr_cand; 1939 unsigned int now, start; 1940 1941 now = jiffies_to_msecs(jiffies); 1942 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1943 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1944 start = pgdat->nbp_rl_start; 1945 if (now - start > MSEC_PER_SEC && 1946 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1947 pgdat->nbp_rl_nr_cand = nr_cand; 1948 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1949 return true; 1950 return false; 1951} 1952 1953#define NUMA_MIGRATION_ADJUST_STEPS 16 1954 1955static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1956 unsigned long rate_limit, 1957 unsigned int ref_th) 1958{ 1959 unsigned int now, start, th_period, unit_th, th; 1960 unsigned long nr_cand, ref_cand, diff_cand; 1961 1962 now = jiffies_to_msecs(jiffies); 1963 th_period = sysctl_numa_balancing_scan_period_max; 1964 start = pgdat->nbp_th_start; 1965 if (now - start > th_period && 1966 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1967 ref_cand = rate_limit * 1968 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1969 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1970 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1971 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1972 th = pgdat->nbp_threshold ? : ref_th; 1973 if (diff_cand > ref_cand * 11 / 10) 1974 th = max(th - unit_th, unit_th); 1975 else if (diff_cand < ref_cand * 9 / 10) 1976 th = min(th + unit_th, ref_th * 2); 1977 pgdat->nbp_th_nr_cand = nr_cand; 1978 pgdat->nbp_threshold = th; 1979 } 1980} 1981 1982bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, 1983 int src_nid, int dst_cpu) 1984{ 1985 struct numa_group *ng = deref_curr_numa_group(p); 1986 int dst_nid = cpu_to_node(dst_cpu); 1987 int last_cpupid, this_cpupid; 1988 1989 /* 1990 * Cannot migrate to memoryless nodes. 1991 */ 1992 if (!node_state(dst_nid, N_MEMORY)) 1993 return false; 1994 1995 /* 1996 * The pages in slow memory node should be migrated according 1997 * to hot/cold instead of private/shared. 1998 */ 1999 if (folio_use_access_time(folio)) { 2000 struct pglist_data *pgdat; 2001 unsigned long rate_limit; 2002 unsigned int latency, th, def_th; 2003 long nr = folio_nr_pages(folio); 2004 2005 pgdat = NODE_DATA(dst_nid); 2006 if (pgdat_free_space_enough(pgdat)) { 2007 /* workload changed, reset hot threshold */ 2008 pgdat->nbp_threshold = 0; 2009 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr); 2010 return true; 2011 } 2012 2013 def_th = sysctl_numa_balancing_hot_threshold; 2014 rate_limit = MB_TO_PAGES(sysctl_numa_balancing_promote_rate_limit); 2015 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 2016 2017 th = pgdat->nbp_threshold ? : def_th; 2018 latency = numa_hint_fault_latency(folio); 2019 if (latency >= th) 2020 return false; 2021 2022 return !numa_promotion_rate_limit(pgdat, rate_limit, nr); 2023 } 2024 2025 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 2026 last_cpupid = folio_xchg_last_cpupid(folio, this_cpupid); 2027 2028 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 2029 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 2030 return false; 2031 2032 /* 2033 * Allow first faults or private faults to migrate immediately early in 2034 * the lifetime of a task. The magic number 4 is based on waiting for 2035 * two full passes of the "multi-stage node selection" test that is 2036 * executed below. 2037 */ 2038 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 2039 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 2040 return true; 2041 2042 /* 2043 * Multi-stage node selection is used in conjunction with a periodic 2044 * migration fault to build a temporal task<->page relation. By using 2045 * a two-stage filter we remove short/unlikely relations. 2046 * 2047 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 2048 * a task's usage of a particular page (n_p) per total usage of this 2049 * page (n_t) (in a given time-span) to a probability. 2050 * 2051 * Our periodic faults will sample this probability and getting the 2052 * same result twice in a row, given these samples are fully 2053 * independent, is then given by P(n)^2, provided our sample period 2054 * is sufficiently short compared to the usage pattern. 2055 * 2056 * This quadric squishes small probabilities, making it less likely we 2057 * act on an unlikely task<->page relation. 2058 */ 2059 if (!cpupid_pid_unset(last_cpupid) && 2060 cpupid_to_nid(last_cpupid) != dst_nid) 2061 return false; 2062 2063 /* Always allow migrate on private faults */ 2064 if (cpupid_match_pid(p, last_cpupid)) 2065 return true; 2066 2067 /* A shared fault, but p->numa_group has not been set up yet. */ 2068 if (!ng) 2069 return true; 2070 2071 /* 2072 * Destination node is much more heavily used than the source 2073 * node? Allow migration. 2074 */ 2075 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 2076 ACTIVE_NODE_FRACTION) 2077 return true; 2078 2079 /* 2080 * Distribute memory according to CPU & memory use on each node, 2081 * with 3/4 hysteresis to avoid unnecessary memory migrations: 2082 * 2083 * faults_cpu(dst) 3 faults_cpu(src) 2084 * --------------- * - > --------------- 2085 * faults_mem(dst) 4 faults_mem(src) 2086 */ 2087 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 2088 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 2089} 2090 2091/* 2092 * 'numa_type' describes the node at the moment of load balancing. 2093 */ 2094enum numa_type { 2095 /* The node has spare capacity that can be used to run more tasks. */ 2096 node_has_spare = 0, 2097 /* 2098 * The node is fully used and the tasks don't compete for more CPU 2099 * cycles. Nevertheless, some tasks might wait before running. 2100 */ 2101 node_fully_busy, 2102 /* 2103 * The node is overloaded and can't provide expected CPU cycles to all 2104 * tasks. 2105 */ 2106 node_overloaded 2107}; 2108 2109/* Cached statistics for all CPUs within a node */ 2110struct numa_stats { 2111 unsigned long load; 2112 unsigned long runnable; 2113 unsigned long util; 2114 /* Total compute capacity of CPUs on a node */ 2115 unsigned long compute_capacity; 2116 unsigned int nr_running; 2117 unsigned int weight; 2118 enum numa_type node_type; 2119 int idle_cpu; 2120}; 2121 2122struct task_numa_env { 2123 struct task_struct *p; 2124 2125 int src_cpu, src_nid; 2126 int dst_cpu, dst_nid; 2127 int imb_numa_nr; 2128 2129 struct numa_stats src_stats, dst_stats; 2130 2131 int imbalance_pct; 2132 int dist; 2133 2134 struct task_struct *best_task; 2135 long best_imp; 2136 int best_cpu; 2137}; 2138 2139static unsigned long cpu_load(struct rq *rq); 2140static unsigned long cpu_runnable(struct rq *rq); 2141 2142static inline enum 2143numa_type numa_classify(unsigned int imbalance_pct, 2144 struct numa_stats *ns) 2145{ 2146 if ((ns->nr_running > ns->weight) && 2147 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 2148 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 2149 return node_overloaded; 2150 2151 if ((ns->nr_running < ns->weight) || 2152 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 2153 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 2154 return node_has_spare; 2155 2156 return node_fully_busy; 2157} 2158 2159#ifdef CONFIG_SCHED_SMT 2160/* Forward declarations of select_idle_sibling helpers */ 2161static inline bool test_idle_cores(int cpu); 2162static inline int numa_idle_core(int idle_core, int cpu) 2163{ 2164 if (!static_branch_likely(&sched_smt_present) || 2165 idle_core >= 0 || !test_idle_cores(cpu)) 2166 return idle_core; 2167 2168 /* 2169 * Prefer cores instead of packing HT siblings 2170 * and triggering future load balancing. 2171 */ 2172 if (is_core_idle(cpu)) 2173 idle_core = cpu; 2174 2175 return idle_core; 2176} 2177#else /* !CONFIG_SCHED_SMT: */ 2178static inline int numa_idle_core(int idle_core, int cpu) 2179{ 2180 return idle_core; 2181} 2182#endif /* !CONFIG_SCHED_SMT */ 2183 2184/* 2185 * Gather all necessary information to make NUMA balancing placement 2186 * decisions that are compatible with standard load balancer. This 2187 * borrows code and logic from update_sg_lb_stats but sharing a 2188 * common implementation is impractical. 2189 */ 2190static void update_numa_stats(struct task_numa_env *env, 2191 struct numa_stats *ns, int nid, 2192 bool find_idle) 2193{ 2194 int cpu, idle_core = -1; 2195 2196 memset(ns, 0, sizeof(*ns)); 2197 ns->idle_cpu = -1; 2198 2199 rcu_read_lock(); 2200 for_each_cpu(cpu, cpumask_of_node(nid)) { 2201 struct rq *rq = cpu_rq(cpu); 2202 2203 ns->load += cpu_load(rq); 2204 ns->runnable += cpu_runnable(rq); 2205 ns->util += cpu_util_cfs(cpu); 2206 ns->nr_running += rq->cfs.h_nr_runnable; 2207 ns->compute_capacity += capacity_of(cpu); 2208 2209 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 2210 if (READ_ONCE(rq->numa_migrate_on) || 2211 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2212 continue; 2213 2214 if (ns->idle_cpu == -1) 2215 ns->idle_cpu = cpu; 2216 2217 idle_core = numa_idle_core(idle_core, cpu); 2218 } 2219 } 2220 rcu_read_unlock(); 2221 2222 ns->weight = cpumask_weight(cpumask_of_node(nid)); 2223 2224 ns->node_type = numa_classify(env->imbalance_pct, ns); 2225 2226 if (idle_core >= 0) 2227 ns->idle_cpu = idle_core; 2228} 2229 2230static void task_numa_assign(struct task_numa_env *env, 2231 struct task_struct *p, long imp) 2232{ 2233 struct rq *rq = cpu_rq(env->dst_cpu); 2234 2235 /* Check if run-queue part of active NUMA balance. */ 2236 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 2237 int cpu; 2238 int start = env->dst_cpu; 2239 2240 /* Find alternative idle CPU. */ 2241 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 2242 if (cpu == env->best_cpu || !idle_cpu(cpu) || 2243 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 2244 continue; 2245 } 2246 2247 env->dst_cpu = cpu; 2248 rq = cpu_rq(env->dst_cpu); 2249 if (!xchg(&rq->numa_migrate_on, 1)) 2250 goto assign; 2251 } 2252 2253 /* Failed to find an alternative idle CPU */ 2254 return; 2255 } 2256 2257assign: 2258 /* 2259 * Clear previous best_cpu/rq numa-migrate flag, since task now 2260 * found a better CPU to move/swap. 2261 */ 2262 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 2263 rq = cpu_rq(env->best_cpu); 2264 WRITE_ONCE(rq->numa_migrate_on, 0); 2265 } 2266 2267 if (env->best_task) 2268 put_task_struct(env->best_task); 2269 if (p) 2270 get_task_struct(p); 2271 2272 env->best_task = p; 2273 env->best_imp = imp; 2274 env->best_cpu = env->dst_cpu; 2275} 2276 2277static bool load_too_imbalanced(long src_load, long dst_load, 2278 struct task_numa_env *env) 2279{ 2280 long imb, old_imb; 2281 long orig_src_load, orig_dst_load; 2282 long src_capacity, dst_capacity; 2283 2284 /* 2285 * The load is corrected for the CPU capacity available on each node. 2286 * 2287 * src_load dst_load 2288 * ------------ vs --------- 2289 * src_capacity dst_capacity 2290 */ 2291 src_capacity = env->src_stats.compute_capacity; 2292 dst_capacity = env->dst_stats.compute_capacity; 2293 2294 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 2295 2296 orig_src_load = env->src_stats.load; 2297 orig_dst_load = env->dst_stats.load; 2298 2299 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 2300 2301 /* Would this change make things worse? */ 2302 return (imb > old_imb); 2303} 2304 2305/* 2306 * Maximum NUMA importance can be 1998 (2*999); 2307 * SMALLIMP @ 30 would be close to 1998/64. 2308 * Used to deter task migration. 2309 */ 2310#define SMALLIMP 30 2311 2312/* 2313 * This checks if the overall compute and NUMA accesses of the system would 2314 * be improved if the source tasks was migrated to the target dst_cpu taking 2315 * into account that it might be best if task running on the dst_cpu should 2316 * be exchanged with the source task 2317 */ 2318static bool task_numa_compare(struct task_numa_env *env, 2319 long taskimp, long groupimp, bool maymove) 2320{ 2321 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 2322 struct rq *dst_rq = cpu_rq(env->dst_cpu); 2323 long imp = p_ng ? groupimp : taskimp; 2324 struct task_struct *cur; 2325 long src_load, dst_load; 2326 int dist = env->dist; 2327 long moveimp = imp; 2328 long load; 2329 bool stopsearch = false; 2330 2331 if (READ_ONCE(dst_rq->numa_migrate_on)) 2332 return false; 2333 2334 rcu_read_lock(); 2335 cur = rcu_dereference(dst_rq->curr); 2336 if (cur && ((cur->flags & (PF_EXITING | PF_KTHREAD)) || 2337 !cur->mm)) 2338 cur = NULL; 2339 2340 /* 2341 * Because we have preemption enabled we can get migrated around and 2342 * end try selecting ourselves (current == env->p) as a swap candidate. 2343 */ 2344 if (cur == env->p) { 2345 stopsearch = true; 2346 goto unlock; 2347 } 2348 2349 if (!cur) { 2350 if (maymove && moveimp >= env->best_imp) 2351 goto assign; 2352 else 2353 goto unlock; 2354 } 2355 2356 /* Skip this swap candidate if cannot move to the source cpu. */ 2357 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 2358 goto unlock; 2359 2360 /* 2361 * Skip this swap candidate if it is not moving to its preferred 2362 * node and the best task is. 2363 */ 2364 if (env->best_task && 2365 env->best_task->numa_preferred_nid == env->src_nid && 2366 cur->numa_preferred_nid != env->src_nid) { 2367 goto unlock; 2368 } 2369 2370 /* 2371 * "imp" is the fault differential for the source task between the 2372 * source and destination node. Calculate the total differential for 2373 * the source task and potential destination task. The more negative 2374 * the value is, the more remote accesses that would be expected to 2375 * be incurred if the tasks were swapped. 2376 * 2377 * If dst and source tasks are in the same NUMA group, or not 2378 * in any group then look only at task weights. 2379 */ 2380 cur_ng = rcu_dereference(cur->numa_group); 2381 if (cur_ng == p_ng) { 2382 /* 2383 * Do not swap within a group or between tasks that have 2384 * no group if there is spare capacity. Swapping does 2385 * not address the load imbalance and helps one task at 2386 * the cost of punishing another. 2387 */ 2388 if (env->dst_stats.node_type == node_has_spare) 2389 goto unlock; 2390 2391 imp = taskimp + task_weight(cur, env->src_nid, dist) - 2392 task_weight(cur, env->dst_nid, dist); 2393 /* 2394 * Add some hysteresis to prevent swapping the 2395 * tasks within a group over tiny differences. 2396 */ 2397 if (cur_ng) 2398 imp -= imp / 16; 2399 } else { 2400 /* 2401 * Compare the group weights. If a task is all by itself 2402 * (not part of a group), use the task weight instead. 2403 */ 2404 if (cur_ng && p_ng) 2405 imp += group_weight(cur, env->src_nid, dist) - 2406 group_weight(cur, env->dst_nid, dist); 2407 else 2408 imp += task_weight(cur, env->src_nid, dist) - 2409 task_weight(cur, env->dst_nid, dist); 2410 } 2411 2412 /* Discourage picking a task already on its preferred node */ 2413 if (cur->numa_preferred_nid == env->dst_nid) 2414 imp -= imp / 16; 2415 2416 /* 2417 * Encourage picking a task that moves to its preferred node. 2418 * This potentially makes imp larger than it's maximum of 2419 * 1998 (see SMALLIMP and task_weight for why) but in this 2420 * case, it does not matter. 2421 */ 2422 if (cur->numa_preferred_nid == env->src_nid) 2423 imp += imp / 8; 2424 2425 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2426 imp = moveimp; 2427 cur = NULL; 2428 goto assign; 2429 } 2430 2431 /* 2432 * Prefer swapping with a task moving to its preferred node over a 2433 * task that is not. 2434 */ 2435 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2436 env->best_task->numa_preferred_nid != env->src_nid) { 2437 goto assign; 2438 } 2439 2440 /* 2441 * If the NUMA importance is less than SMALLIMP, 2442 * task migration might only result in ping pong 2443 * of tasks and also hurt performance due to cache 2444 * misses. 2445 */ 2446 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2447 goto unlock; 2448 2449 /* 2450 * In the overloaded case, try and keep the load balanced. 2451 */ 2452 load = task_h_load(env->p) - task_h_load(cur); 2453 if (!load) 2454 goto assign; 2455 2456 dst_load = env->dst_stats.load + load; 2457 src_load = env->src_stats.load - load; 2458 2459 if (load_too_imbalanced(src_load, dst_load, env)) 2460 goto unlock; 2461 2462assign: 2463 /* Evaluate an idle CPU for a task numa move. */ 2464 if (!cur) { 2465 int cpu = env->dst_stats.idle_cpu; 2466 2467 /* Nothing cached so current CPU went idle since the search. */ 2468 if (cpu < 0) 2469 cpu = env->dst_cpu; 2470 2471 /* 2472 * If the CPU is no longer truly idle and the previous best CPU 2473 * is, keep using it. 2474 */ 2475 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2476 idle_cpu(env->best_cpu)) { 2477 cpu = env->best_cpu; 2478 } 2479 2480 env->dst_cpu = cpu; 2481 } 2482 2483 task_numa_assign(env, cur, imp); 2484 2485 /* 2486 * If a move to idle is allowed because there is capacity or load 2487 * balance improves then stop the search. While a better swap 2488 * candidate may exist, a search is not free. 2489 */ 2490 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2491 stopsearch = true; 2492 2493 /* 2494 * If a swap candidate must be identified and the current best task 2495 * moves its preferred node then stop the search. 2496 */ 2497 if (!maymove && env->best_task && 2498 env->best_task->numa_preferred_nid == env->src_nid) { 2499 stopsearch = true; 2500 } 2501unlock: 2502 rcu_read_unlock(); 2503 2504 return stopsearch; 2505} 2506 2507static void task_numa_find_cpu(struct task_numa_env *env, 2508 long taskimp, long groupimp) 2509{ 2510 bool maymove = false; 2511 int cpu; 2512 2513 /* 2514 * If dst node has spare capacity, then check if there is an 2515 * imbalance that would be overruled by the load balancer. 2516 */ 2517 if (env->dst_stats.node_type == node_has_spare) { 2518 unsigned int imbalance; 2519 int src_running, dst_running; 2520 2521 /* 2522 * Would movement cause an imbalance? Note that if src has 2523 * more running tasks that the imbalance is ignored as the 2524 * move improves the imbalance from the perspective of the 2525 * CPU load balancer. 2526 * */ 2527 src_running = env->src_stats.nr_running - 1; 2528 dst_running = env->dst_stats.nr_running + 1; 2529 imbalance = max(0, dst_running - src_running); 2530 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2531 env->imb_numa_nr); 2532 2533 /* Use idle CPU if there is no imbalance */ 2534 if (!imbalance) { 2535 maymove = true; 2536 if (env->dst_stats.idle_cpu >= 0) { 2537 env->dst_cpu = env->dst_stats.idle_cpu; 2538 task_numa_assign(env, NULL, 0); 2539 return; 2540 } 2541 } 2542 } else { 2543 long src_load, dst_load, load; 2544 /* 2545 * If the improvement from just moving env->p direction is better 2546 * than swapping tasks around, check if a move is possible. 2547 */ 2548 load = task_h_load(env->p); 2549 dst_load = env->dst_stats.load + load; 2550 src_load = env->src_stats.load - load; 2551 maymove = !load_too_imbalanced(src_load, dst_load, env); 2552 } 2553 2554 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 2555 /* Skip this CPU if the source task cannot migrate */ 2556 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2557 continue; 2558 2559 env->dst_cpu = cpu; 2560 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2561 break; 2562 } 2563} 2564 2565static int task_numa_migrate(struct task_struct *p) 2566{ 2567 struct task_numa_env env = { 2568 .p = p, 2569 2570 .src_cpu = task_cpu(p), 2571 .src_nid = task_node(p), 2572 2573 .imbalance_pct = 112, 2574 2575 .best_task = NULL, 2576 .best_imp = 0, 2577 .best_cpu = -1, 2578 }; 2579 unsigned long taskweight, groupweight; 2580 struct sched_domain *sd; 2581 long taskimp, groupimp; 2582 struct numa_group *ng; 2583 struct rq *best_rq; 2584 int nid, ret, dist; 2585 2586 /* 2587 * Pick the lowest SD_NUMA domain, as that would have the smallest 2588 * imbalance and would be the first to start moving tasks about. 2589 * 2590 * And we want to avoid any moving of tasks about, as that would create 2591 * random movement of tasks -- counter the numa conditions we're trying 2592 * to satisfy here. 2593 */ 2594 rcu_read_lock(); 2595 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 2596 if (sd) { 2597 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2598 env.imb_numa_nr = sd->imb_numa_nr; 2599 } 2600 rcu_read_unlock(); 2601 2602 /* 2603 * Cpusets can break the scheduler domain tree into smaller 2604 * balance domains, some of which do not cross NUMA boundaries. 2605 * Tasks that are "trapped" in such domains cannot be migrated 2606 * elsewhere, so there is no point in (re)trying. 2607 */ 2608 if (unlikely(!sd)) { 2609 sched_setnuma(p, task_node(p)); 2610 return -EINVAL; 2611 } 2612 2613 env.dst_nid = p->numa_preferred_nid; 2614 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2615 taskweight = task_weight(p, env.src_nid, dist); 2616 groupweight = group_weight(p, env.src_nid, dist); 2617 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2618 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2619 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2620 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2621 2622 /* Try to find a spot on the preferred nid. */ 2623 task_numa_find_cpu(&env, taskimp, groupimp); 2624 2625 /* 2626 * Look at other nodes in these cases: 2627 * - there is no space available on the preferred_nid 2628 * - the task is part of a numa_group that is interleaved across 2629 * multiple NUMA nodes; in order to better consolidate the group, 2630 * we need to check other locations. 2631 */ 2632 ng = deref_curr_numa_group(p); 2633 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2634 for_each_node_state(nid, N_CPU) { 2635 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2636 continue; 2637 2638 dist = node_distance(env.src_nid, env.dst_nid); 2639 if (sched_numa_topology_type == NUMA_BACKPLANE && 2640 dist != env.dist) { 2641 taskweight = task_weight(p, env.src_nid, dist); 2642 groupweight = group_weight(p, env.src_nid, dist); 2643 } 2644 2645 /* Only consider nodes where both task and groups benefit */ 2646 taskimp = task_weight(p, nid, dist) - taskweight; 2647 groupimp = group_weight(p, nid, dist) - groupweight; 2648 if (taskimp < 0 && groupimp < 0) 2649 continue; 2650 2651 env.dist = dist; 2652 env.dst_nid = nid; 2653 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2654 task_numa_find_cpu(&env, taskimp, groupimp); 2655 } 2656 } 2657 2658 /* 2659 * If the task is part of a workload that spans multiple NUMA nodes, 2660 * and is migrating into one of the workload's active nodes, remember 2661 * this node as the task's preferred numa node, so the workload can 2662 * settle down. 2663 * A task that migrated to a second choice node will be better off 2664 * trying for a better one later. Do not set the preferred node here. 2665 */ 2666 if (ng) { 2667 if (env.best_cpu == -1) 2668 nid = env.src_nid; 2669 else 2670 nid = cpu_to_node(env.best_cpu); 2671 2672 if (nid != p->numa_preferred_nid) 2673 sched_setnuma(p, nid); 2674 } 2675 2676 /* No better CPU than the current one was found. */ 2677 if (env.best_cpu == -1) { 2678 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2679 return -EAGAIN; 2680 } 2681 2682 best_rq = cpu_rq(env.best_cpu); 2683 if (env.best_task == NULL) { 2684 ret = migrate_task_to(p, env.best_cpu); 2685 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2686 if (ret != 0) 2687 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2688 return ret; 2689 } 2690 2691 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2692 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2693 2694 if (ret != 0) 2695 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2696 put_task_struct(env.best_task); 2697 return ret; 2698} 2699 2700/* Attempt to migrate a task to a CPU on the preferred node. */ 2701static void numa_migrate_preferred(struct task_struct *p) 2702{ 2703 unsigned long interval = HZ; 2704 2705 /* This task has no NUMA fault statistics yet */ 2706 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2707 return; 2708 2709 /* Periodically retry migrating the task to the preferred node */ 2710 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2711 p->numa_migrate_retry = jiffies + interval; 2712 2713 /* Success if task is already running on preferred CPU */ 2714 if (task_node(p) == p->numa_preferred_nid) 2715 return; 2716 2717 /* Otherwise, try migrate to a CPU on the preferred node */ 2718 task_numa_migrate(p); 2719} 2720 2721/* 2722 * Find out how many nodes the workload is actively running on. Do this by 2723 * tracking the nodes from which NUMA hinting faults are triggered. This can 2724 * be different from the set of nodes where the workload's memory is currently 2725 * located. 2726 */ 2727static void numa_group_count_active_nodes(struct numa_group *numa_group) 2728{ 2729 unsigned long faults, max_faults = 0; 2730 int nid, active_nodes = 0; 2731 2732 for_each_node_state(nid, N_CPU) { 2733 faults = group_faults_cpu(numa_group, nid); 2734 if (faults > max_faults) 2735 max_faults = faults; 2736 } 2737 2738 for_each_node_state(nid, N_CPU) { 2739 faults = group_faults_cpu(numa_group, nid); 2740 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2741 active_nodes++; 2742 } 2743 2744 numa_group->max_faults_cpu = max_faults; 2745 numa_group->active_nodes = active_nodes; 2746} 2747 2748/* 2749 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2750 * increments. The more local the fault statistics are, the higher the scan 2751 * period will be for the next scan window. If local/(local+remote) ratio is 2752 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2753 * the scan period will decrease. Aim for 70% local accesses. 2754 */ 2755#define NUMA_PERIOD_SLOTS 10 2756#define NUMA_PERIOD_THRESHOLD 7 2757 2758/* 2759 * Increase the scan period (slow down scanning) if the majority of 2760 * our memory is already on our local node, or if the majority of 2761 * the page accesses are shared with other processes. 2762 * Otherwise, decrease the scan period. 2763 */ 2764static void update_task_scan_period(struct task_struct *p, 2765 unsigned long shared, unsigned long private) 2766{ 2767 unsigned int period_slot; 2768 int lr_ratio, ps_ratio; 2769 int diff; 2770 2771 unsigned long remote = p->numa_faults_locality[0]; 2772 unsigned long local = p->numa_faults_locality[1]; 2773 2774 /* 2775 * If there were no record hinting faults then either the task is 2776 * completely idle or all activity is in areas that are not of interest 2777 * to automatic numa balancing. Related to that, if there were failed 2778 * migration then it implies we are migrating too quickly or the local 2779 * node is overloaded. In either case, scan slower 2780 */ 2781 if (local + shared == 0 || p->numa_faults_locality[2]) { 2782 p->numa_scan_period = min(p->numa_scan_period_max, 2783 p->numa_scan_period << 1); 2784 2785 p->mm->numa_next_scan = jiffies + 2786 msecs_to_jiffies(p->numa_scan_period); 2787 2788 return; 2789 } 2790 2791 /* 2792 * Prepare to scale scan period relative to the current period. 2793 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2794 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2795 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2796 */ 2797 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2798 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2799 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2800 2801 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2802 /* 2803 * Most memory accesses are local. There is no need to 2804 * do fast NUMA scanning, since memory is already local. 2805 */ 2806 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2807 if (!slot) 2808 slot = 1; 2809 diff = slot * period_slot; 2810 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2811 /* 2812 * Most memory accesses are shared with other tasks. 2813 * There is no point in continuing fast NUMA scanning, 2814 * since other tasks may just move the memory elsewhere. 2815 */ 2816 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2817 if (!slot) 2818 slot = 1; 2819 diff = slot * period_slot; 2820 } else { 2821 /* 2822 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2823 * yet they are not on the local NUMA node. Speed up 2824 * NUMA scanning to get the memory moved over. 2825 */ 2826 int ratio = max(lr_ratio, ps_ratio); 2827 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2828 } 2829 2830 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2831 task_scan_min(p), task_scan_max(p)); 2832 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2833} 2834 2835/* 2836 * Get the fraction of time the task has been running since the last 2837 * NUMA placement cycle. The scheduler keeps similar statistics, but 2838 * decays those on a 32ms period, which is orders of magnitude off 2839 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2840 * stats only if the task is so new there are no NUMA statistics yet. 2841 */ 2842static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2843{ 2844 u64 runtime, delta, now; 2845 /* Use the start of this time slice to avoid calculations. */ 2846 now = p->se.exec_start; 2847 runtime = p->se.sum_exec_runtime; 2848 2849 if (p->last_task_numa_placement) { 2850 delta = runtime - p->last_sum_exec_runtime; 2851 *period = now - p->last_task_numa_placement; 2852 2853 /* Avoid time going backwards, prevent potential divide error: */ 2854 if (unlikely((s64)*period < 0)) 2855 *period = 0; 2856 } else { 2857 delta = p->se.avg.load_sum; 2858 *period = LOAD_AVG_MAX; 2859 } 2860 2861 p->last_sum_exec_runtime = runtime; 2862 p->last_task_numa_placement = now; 2863 2864 return delta; 2865} 2866 2867/* 2868 * Determine the preferred nid for a task in a numa_group. This needs to 2869 * be done in a way that produces consistent results with group_weight, 2870 * otherwise workloads might not converge. 2871 */ 2872static int preferred_group_nid(struct task_struct *p, int nid) 2873{ 2874 nodemask_t nodes; 2875 int dist; 2876 2877 /* Direct connections between all NUMA nodes. */ 2878 if (sched_numa_topology_type == NUMA_DIRECT) 2879 return nid; 2880 2881 /* 2882 * On a system with glueless mesh NUMA topology, group_weight 2883 * scores nodes according to the number of NUMA hinting faults on 2884 * both the node itself, and on nearby nodes. 2885 */ 2886 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2887 unsigned long score, max_score = 0; 2888 int node, max_node = nid; 2889 2890 dist = sched_max_numa_distance; 2891 2892 for_each_node_state(node, N_CPU) { 2893 score = group_weight(p, node, dist); 2894 if (score > max_score) { 2895 max_score = score; 2896 max_node = node; 2897 } 2898 } 2899 return max_node; 2900 } 2901 2902 /* 2903 * Finding the preferred nid in a system with NUMA backplane 2904 * interconnect topology is more involved. The goal is to locate 2905 * tasks from numa_groups near each other in the system, and 2906 * untangle workloads from different sides of the system. This requires 2907 * searching down the hierarchy of node groups, recursively searching 2908 * inside the highest scoring group of nodes. The nodemask tricks 2909 * keep the complexity of the search down. 2910 */ 2911 nodes = node_states[N_CPU]; 2912 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2913 unsigned long max_faults = 0; 2914 nodemask_t max_group = NODE_MASK_NONE; 2915 int a, b; 2916 2917 /* Are there nodes at this distance from each other? */ 2918 if (!find_numa_distance(dist)) 2919 continue; 2920 2921 for_each_node_mask(a, nodes) { 2922 unsigned long faults = 0; 2923 nodemask_t this_group; 2924 nodes_clear(this_group); 2925 2926 /* Sum group's NUMA faults; includes a==b case. */ 2927 for_each_node_mask(b, nodes) { 2928 if (node_distance(a, b) < dist) { 2929 faults += group_faults(p, b); 2930 node_set(b, this_group); 2931 node_clear(b, nodes); 2932 } 2933 } 2934 2935 /* Remember the top group. */ 2936 if (faults > max_faults) { 2937 max_faults = faults; 2938 max_group = this_group; 2939 /* 2940 * subtle: at the smallest distance there is 2941 * just one node left in each "group", the 2942 * winner is the preferred nid. 2943 */ 2944 nid = a; 2945 } 2946 } 2947 /* Next round, evaluate the nodes within max_group. */ 2948 if (!max_faults) 2949 break; 2950 nodes = max_group; 2951 } 2952 return nid; 2953} 2954 2955static void task_numa_placement(struct task_struct *p) 2956{ 2957 int seq, nid, max_nid = NUMA_NO_NODE; 2958 unsigned long max_faults = 0; 2959 unsigned long fault_types[2] = { 0, 0 }; 2960 unsigned long total_faults; 2961 u64 runtime, period; 2962 spinlock_t *group_lock = NULL; 2963 struct numa_group *ng; 2964 2965 /* 2966 * The p->mm->numa_scan_seq field gets updated without 2967 * exclusive access. Use READ_ONCE() here to ensure 2968 * that the field is read in a single access: 2969 */ 2970 seq = READ_ONCE(p->mm->numa_scan_seq); 2971 if (p->numa_scan_seq == seq) 2972 return; 2973 p->numa_scan_seq = seq; 2974 p->numa_scan_period_max = task_scan_max(p); 2975 2976 total_faults = p->numa_faults_locality[0] + 2977 p->numa_faults_locality[1]; 2978 runtime = numa_get_avg_runtime(p, &period); 2979 2980 /* If the task is part of a group prevent parallel updates to group stats */ 2981 ng = deref_curr_numa_group(p); 2982 if (ng) { 2983 group_lock = &ng->lock; 2984 spin_lock_irq(group_lock); 2985 } 2986 2987 /* Find the node with the highest number of faults */ 2988 for_each_online_node(nid) { 2989 /* Keep track of the offsets in numa_faults array */ 2990 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2991 unsigned long faults = 0, group_faults = 0; 2992 int priv; 2993 2994 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2995 long diff, f_diff, f_weight; 2996 2997 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2998 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2999 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 3000 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 3001 3002 /* Decay existing window, copy faults since last scan */ 3003 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 3004 fault_types[priv] += p->numa_faults[membuf_idx]; 3005 p->numa_faults[membuf_idx] = 0; 3006 3007 /* 3008 * Normalize the faults_from, so all tasks in a group 3009 * count according to CPU use, instead of by the raw 3010 * number of faults. Tasks with little runtime have 3011 * little over-all impact on throughput, and thus their 3012 * faults are less important. 3013 */ 3014 f_weight = div64_u64(runtime << 16, period + 1); 3015 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 3016 (total_faults + 1); 3017 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 3018 p->numa_faults[cpubuf_idx] = 0; 3019 3020 p->numa_faults[mem_idx] += diff; 3021 p->numa_faults[cpu_idx] += f_diff; 3022 faults += p->numa_faults[mem_idx]; 3023 p->total_numa_faults += diff; 3024 if (ng) { 3025 /* 3026 * safe because we can only change our own group 3027 * 3028 * mem_idx represents the offset for a given 3029 * nid and priv in a specific region because it 3030 * is at the beginning of the numa_faults array. 3031 */ 3032 ng->faults[mem_idx] += diff; 3033 ng->faults[cpu_idx] += f_diff; 3034 ng->total_faults += diff; 3035 group_faults += ng->faults[mem_idx]; 3036 } 3037 } 3038 3039 if (!ng) { 3040 if (faults > max_faults) { 3041 max_faults = faults; 3042 max_nid = nid; 3043 } 3044 } else if (group_faults > max_faults) { 3045 max_faults = group_faults; 3046 max_nid = nid; 3047 } 3048 } 3049 3050 /* Cannot migrate task to CPU-less node */ 3051 max_nid = numa_nearest_node(max_nid, N_CPU); 3052 3053 if (ng) { 3054 numa_group_count_active_nodes(ng); 3055 spin_unlock_irq(group_lock); 3056 max_nid = preferred_group_nid(p, max_nid); 3057 } 3058 3059 if (max_faults) { 3060 /* Set the new preferred node */ 3061 if (max_nid != p->numa_preferred_nid) 3062 sched_setnuma(p, max_nid); 3063 } 3064 3065 update_task_scan_period(p, fault_types[0], fault_types[1]); 3066} 3067 3068static inline int get_numa_group(struct numa_group *grp) 3069{ 3070 return refcount_inc_not_zero(&grp->refcount); 3071} 3072 3073static inline void put_numa_group(struct numa_group *grp) 3074{ 3075 if (refcount_dec_and_test(&grp->refcount)) 3076 kfree_rcu(grp, rcu); 3077} 3078 3079static void task_numa_group(struct task_struct *p, int cpupid, int flags, 3080 int *priv) 3081{ 3082 struct numa_group *grp, *my_grp; 3083 struct task_struct *tsk; 3084 bool join = false; 3085 int cpu = cpupid_to_cpu(cpupid); 3086 int i; 3087 3088 if (unlikely(!deref_curr_numa_group(p))) { 3089 unsigned int size = sizeof(struct numa_group) + 3090 NR_NUMA_HINT_FAULT_STATS * 3091 nr_node_ids * sizeof(unsigned long); 3092 3093 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 3094 if (!grp) 3095 return; 3096 3097 refcount_set(&grp->refcount, 1); 3098 grp->active_nodes = 1; 3099 grp->max_faults_cpu = 0; 3100 spin_lock_init(&grp->lock); 3101 grp->gid = p->pid; 3102 3103 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3104 grp->faults[i] = p->numa_faults[i]; 3105 3106 grp->total_faults = p->total_numa_faults; 3107 3108 grp->nr_tasks++; 3109 rcu_assign_pointer(p->numa_group, grp); 3110 } 3111 3112 rcu_read_lock(); 3113 tsk = READ_ONCE(cpu_rq(cpu)->curr); 3114 3115 if (!cpupid_match_pid(tsk, cpupid)) 3116 goto no_join; 3117 3118 grp = rcu_dereference(tsk->numa_group); 3119 if (!grp) 3120 goto no_join; 3121 3122 my_grp = deref_curr_numa_group(p); 3123 if (grp == my_grp) 3124 goto no_join; 3125 3126 /* 3127 * Only join the other group if its bigger; if we're the bigger group, 3128 * the other task will join us. 3129 */ 3130 if (my_grp->nr_tasks > grp->nr_tasks) 3131 goto no_join; 3132 3133 /* 3134 * Tie-break on the grp address. 3135 */ 3136 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 3137 goto no_join; 3138 3139 /* Always join threads in the same process. */ 3140 if (tsk->mm == current->mm) 3141 join = true; 3142 3143 /* Simple filter to avoid false positives due to PID collisions */ 3144 if (flags & TNF_SHARED) 3145 join = true; 3146 3147 /* Update priv based on whether false sharing was detected */ 3148 *priv = !join; 3149 3150 if (join && !get_numa_group(grp)) 3151 goto no_join; 3152 3153 rcu_read_unlock(); 3154 3155 if (!join) 3156 return; 3157 3158 WARN_ON_ONCE(irqs_disabled()); 3159 double_lock_irq(&my_grp->lock, &grp->lock); 3160 3161 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 3162 my_grp->faults[i] -= p->numa_faults[i]; 3163 grp->faults[i] += p->numa_faults[i]; 3164 } 3165 my_grp->total_faults -= p->total_numa_faults; 3166 grp->total_faults += p->total_numa_faults; 3167 3168 my_grp->nr_tasks--; 3169 grp->nr_tasks++; 3170 3171 spin_unlock(&my_grp->lock); 3172 spin_unlock_irq(&grp->lock); 3173 3174 rcu_assign_pointer(p->numa_group, grp); 3175 3176 put_numa_group(my_grp); 3177 return; 3178 3179no_join: 3180 rcu_read_unlock(); 3181 return; 3182} 3183 3184/* 3185 * Get rid of NUMA statistics associated with a task (either current or dead). 3186 * If @final is set, the task is dead and has reached refcount zero, so we can 3187 * safely free all relevant data structures. Otherwise, there might be 3188 * concurrent reads from places like load balancing and procfs, and we should 3189 * reset the data back to default state without freeing ->numa_faults. 3190 */ 3191void task_numa_free(struct task_struct *p, bool final) 3192{ 3193 /* safe: p either is current or is being freed by current */ 3194 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 3195 unsigned long *numa_faults = p->numa_faults; 3196 unsigned long flags; 3197 int i; 3198 3199 if (!numa_faults) 3200 return; 3201 3202 if (grp) { 3203 spin_lock_irqsave(&grp->lock, flags); 3204 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3205 grp->faults[i] -= p->numa_faults[i]; 3206 grp->total_faults -= p->total_numa_faults; 3207 3208 grp->nr_tasks--; 3209 spin_unlock_irqrestore(&grp->lock, flags); 3210 RCU_INIT_POINTER(p->numa_group, NULL); 3211 put_numa_group(grp); 3212 } 3213 3214 if (final) { 3215 p->numa_faults = NULL; 3216 kfree(numa_faults); 3217 } else { 3218 p->total_numa_faults = 0; 3219 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3220 numa_faults[i] = 0; 3221 } 3222} 3223 3224/* 3225 * Got a PROT_NONE fault for a page on @node. 3226 */ 3227void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 3228{ 3229 struct task_struct *p = current; 3230 bool migrated = flags & TNF_MIGRATED; 3231 int cpu_node = task_node(current); 3232 int local = !!(flags & TNF_FAULT_LOCAL); 3233 struct numa_group *ng; 3234 int priv; 3235 3236 if (!static_branch_likely(&sched_numa_balancing)) 3237 return; 3238 3239 /* for example, ksmd faulting in a user's mm */ 3240 if (!p->mm) 3241 return; 3242 3243 /* 3244 * NUMA faults statistics are unnecessary for the slow memory 3245 * node for memory tiering mode. 3246 */ 3247 if (!node_is_toptier(mem_node) && 3248 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 3249 !cpupid_valid(last_cpupid))) 3250 return; 3251 3252 /* Allocate buffer to track faults on a per-node basis */ 3253 if (unlikely(!p->numa_faults)) { 3254 int size = sizeof(*p->numa_faults) * 3255 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 3256 3257 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 3258 if (!p->numa_faults) 3259 return; 3260 3261 p->total_numa_faults = 0; 3262 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 3263 } 3264 3265 /* 3266 * First accesses are treated as private, otherwise consider accesses 3267 * to be private if the accessing pid has not changed 3268 */ 3269 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 3270 priv = 1; 3271 } else { 3272 priv = cpupid_match_pid(p, last_cpupid); 3273 if (!priv && !(flags & TNF_NO_GROUP)) 3274 task_numa_group(p, last_cpupid, flags, &priv); 3275 } 3276 3277 /* 3278 * If a workload spans multiple NUMA nodes, a shared fault that 3279 * occurs wholly within the set of nodes that the workload is 3280 * actively using should be counted as local. This allows the 3281 * scan rate to slow down when a workload has settled down. 3282 */ 3283 ng = deref_curr_numa_group(p); 3284 if (!priv && !local && ng && ng->active_nodes > 1 && 3285 numa_is_active_node(cpu_node, ng) && 3286 numa_is_active_node(mem_node, ng)) 3287 local = 1; 3288 3289 /* 3290 * Retry to migrate task to preferred node periodically, in case it 3291 * previously failed, or the scheduler moved us. 3292 */ 3293 if (time_after(jiffies, p->numa_migrate_retry)) { 3294 task_numa_placement(p); 3295 numa_migrate_preferred(p); 3296 } 3297 3298 if (migrated) 3299 p->numa_pages_migrated += pages; 3300 if (flags & TNF_MIGRATE_FAIL) 3301 p->numa_faults_locality[2] += pages; 3302 3303 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 3304 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 3305 p->numa_faults_locality[local] += pages; 3306} 3307 3308static void reset_ptenuma_scan(struct task_struct *p) 3309{ 3310 /* 3311 * We only did a read acquisition of the mmap sem, so 3312 * p->mm->numa_scan_seq is written to without exclusive access 3313 * and the update is not guaranteed to be atomic. That's not 3314 * much of an issue though, since this is just used for 3315 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 3316 * expensive, to avoid any form of compiler optimizations: 3317 */ 3318 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 3319 p->mm->numa_scan_offset = 0; 3320} 3321 3322static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) 3323{ 3324 unsigned long pids; 3325 /* 3326 * Allow unconditional access first two times, so that all the (pages) 3327 * of VMAs get prot_none fault introduced irrespective of accesses. 3328 * This is also done to avoid any side effect of task scanning 3329 * amplifying the unfairness of disjoint set of VMAs' access. 3330 */ 3331 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) 3332 return true; 3333 3334 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; 3335 if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids)) 3336 return true; 3337 3338 /* 3339 * Complete a scan that has already started regardless of PID access, or 3340 * some VMAs may never be scanned in multi-threaded applications: 3341 */ 3342 if (mm->numa_scan_offset > vma->vm_start) { 3343 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); 3344 return true; 3345 } 3346 3347 /* 3348 * This vma has not been accessed for a while, and if the number 3349 * the threads in the same process is low, which means no other 3350 * threads can help scan this vma, force a vma scan. 3351 */ 3352 if (READ_ONCE(mm->numa_scan_seq) > 3353 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) 3354 return true; 3355 3356 return false; 3357} 3358 3359#define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 3360 3361/* 3362 * The expensive part of numa migration is done from task_work context. 3363 * Triggered from task_tick_numa(). 3364 */ 3365static void task_numa_work(struct callback_head *work) 3366{ 3367 unsigned long migrate, next_scan, now = jiffies; 3368 struct task_struct *p = current; 3369 struct mm_struct *mm = p->mm; 3370 u64 runtime = p->se.sum_exec_runtime; 3371 struct vm_area_struct *vma; 3372 unsigned long start, end; 3373 unsigned long nr_pte_updates = 0; 3374 long pages, virtpages; 3375 struct vma_iterator vmi; 3376 bool vma_pids_skipped; 3377 bool vma_pids_forced = false; 3378 3379 WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); 3380 3381 work->next = work; 3382 /* 3383 * Who cares about NUMA placement when they're dying. 3384 * 3385 * NOTE: make sure not to dereference p->mm before this check, 3386 * exit_task_work() happens _after_ exit_mm() so we could be called 3387 * without p->mm even though we still had it when we enqueued this 3388 * work. 3389 */ 3390 if (p->flags & PF_EXITING) 3391 return; 3392 3393 /* 3394 * Memory is pinned to only one NUMA node via cpuset.mems, naturally 3395 * no page can be migrated. 3396 */ 3397 if (cpusets_enabled() && nodes_weight(cpuset_current_mems_allowed) == 1) { 3398 trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed); 3399 return; 3400 } 3401 3402 if (!mm->numa_next_scan) { 3403 mm->numa_next_scan = now + 3404 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3405 } 3406 3407 /* 3408 * Enforce maximal scan/migration frequency.. 3409 */ 3410 migrate = mm->numa_next_scan; 3411 if (time_before(now, migrate)) 3412 return; 3413 3414 if (p->numa_scan_period == 0) { 3415 p->numa_scan_period_max = task_scan_max(p); 3416 p->numa_scan_period = task_scan_start(p); 3417 } 3418 3419 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 3420 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 3421 return; 3422 3423 /* 3424 * Delay this task enough that another task of this mm will likely win 3425 * the next time around. 3426 */ 3427 p->node_stamp += 2 * TICK_NSEC; 3428 3429 pages = sysctl_numa_balancing_scan_size; 3430 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3431 virtpages = pages * 8; /* Scan up to this much virtual space */ 3432 if (!pages) 3433 return; 3434 3435 3436 if (!mmap_read_trylock(mm)) 3437 return; 3438 3439 /* 3440 * VMAs are skipped if the current PID has not trapped a fault within 3441 * the VMA recently. Allow scanning to be forced if there is no 3442 * suitable VMA remaining. 3443 */ 3444 vma_pids_skipped = false; 3445 3446retry_pids: 3447 start = mm->numa_scan_offset; 3448 vma_iter_init(&vmi, mm, start); 3449 vma = vma_next(&vmi); 3450 if (!vma) { 3451 reset_ptenuma_scan(p); 3452 start = 0; 3453 vma_iter_set(&vmi, start); 3454 vma = vma_next(&vmi); 3455 } 3456 3457 for (; vma; vma = vma_next(&vmi)) { 3458 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3459 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3460 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); 3461 continue; 3462 } 3463 3464 /* 3465 * Shared library pages mapped by multiple processes are not 3466 * migrated as it is expected they are cache replicated. Avoid 3467 * hinting faults in read-only file-backed mappings or the vDSO 3468 * as migrating the pages will be of marginal benefit. 3469 */ 3470 if (!vma->vm_mm || 3471 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { 3472 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); 3473 continue; 3474 } 3475 3476 /* 3477 * Skip inaccessible VMAs to avoid any confusion between 3478 * PROT_NONE and NUMA hinting PTEs 3479 */ 3480 if (!vma_is_accessible(vma)) { 3481 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); 3482 continue; 3483 } 3484 3485 /* Initialise new per-VMA NUMAB state. */ 3486 if (!vma->numab_state) { 3487 struct vma_numab_state *ptr; 3488 3489 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 3490 if (!ptr) 3491 continue; 3492 3493 if (cmpxchg(&vma->numab_state, NULL, ptr)) { 3494 kfree(ptr); 3495 continue; 3496 } 3497 3498 vma->numab_state->start_scan_seq = mm->numa_scan_seq; 3499 3500 vma->numab_state->next_scan = now + 3501 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3502 3503 /* Reset happens after 4 times scan delay of scan start */ 3504 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + 3505 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3506 3507 /* 3508 * Ensure prev_scan_seq does not match numa_scan_seq, 3509 * to prevent VMAs being skipped prematurely on the 3510 * first scan: 3511 */ 3512 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; 3513 } 3514 3515 /* 3516 * Scanning the VMAs of short lived tasks add more overhead. So 3517 * delay the scan for new VMAs. 3518 */ 3519 if (mm->numa_scan_seq && time_before(jiffies, 3520 vma->numab_state->next_scan)) { 3521 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); 3522 continue; 3523 } 3524 3525 /* RESET access PIDs regularly for old VMAs. */ 3526 if (mm->numa_scan_seq && 3527 time_after(jiffies, vma->numab_state->pids_active_reset)) { 3528 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + 3529 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3530 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); 3531 vma->numab_state->pids_active[1] = 0; 3532 } 3533 3534 /* Do not rescan VMAs twice within the same sequence. */ 3535 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { 3536 mm->numa_scan_offset = vma->vm_end; 3537 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); 3538 continue; 3539 } 3540 3541 /* 3542 * Do not scan the VMA if task has not accessed it, unless no other 3543 * VMA candidate exists. 3544 */ 3545 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { 3546 vma_pids_skipped = true; 3547 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); 3548 continue; 3549 } 3550 3551 do { 3552 start = max(start, vma->vm_start); 3553 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3554 end = min(end, vma->vm_end); 3555 nr_pte_updates = change_prot_numa(vma, start, end); 3556 3557 /* 3558 * Try to scan sysctl_numa_balancing_size worth of 3559 * hpages that have at least one present PTE that 3560 * is not already PTE-numa. If the VMA contains 3561 * areas that are unused or already full of prot_numa 3562 * PTEs, scan up to virtpages, to skip through those 3563 * areas faster. 3564 */ 3565 if (nr_pte_updates) 3566 pages -= (end - start) >> PAGE_SHIFT; 3567 virtpages -= (end - start) >> PAGE_SHIFT; 3568 3569 start = end; 3570 if (pages <= 0 || virtpages <= 0) 3571 goto out; 3572 3573 cond_resched(); 3574 } while (end != vma->vm_end); 3575 3576 /* VMA scan is complete, do not scan until next sequence. */ 3577 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; 3578 3579 /* 3580 * Only force scan within one VMA at a time, to limit the 3581 * cost of scanning a potentially uninteresting VMA. 3582 */ 3583 if (vma_pids_forced) 3584 break; 3585 } 3586 3587 /* 3588 * If no VMAs are remaining and VMAs were skipped due to the PID 3589 * not accessing the VMA previously, then force a scan to ensure 3590 * forward progress: 3591 */ 3592 if (!vma && !vma_pids_forced && vma_pids_skipped) { 3593 vma_pids_forced = true; 3594 goto retry_pids; 3595 } 3596 3597out: 3598 /* 3599 * It is possible to reach the end of the VMA list but the last few 3600 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3601 * would find the !migratable VMA on the next scan but not reset the 3602 * scanner to the start so check it now. 3603 */ 3604 if (vma) 3605 mm->numa_scan_offset = start; 3606 else 3607 reset_ptenuma_scan(p); 3608 mmap_read_unlock(mm); 3609 3610 /* 3611 * Make sure tasks use at least 32x as much time to run other code 3612 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3613 * Usually update_task_scan_period slows down scanning enough; on an 3614 * overloaded system we need to limit overhead on a per task basis. 3615 */ 3616 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3617 u64 diff = p->se.sum_exec_runtime - runtime; 3618 p->node_stamp += 32 * diff; 3619 } 3620} 3621 3622void init_numa_balancing(u64 clone_flags, struct task_struct *p) 3623{ 3624 int mm_users = 0; 3625 struct mm_struct *mm = p->mm; 3626 3627 if (mm) { 3628 mm_users = atomic_read(&mm->mm_users); 3629 if (mm_users == 1) { 3630 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3631 mm->numa_scan_seq = 0; 3632 } 3633 } 3634 p->node_stamp = 0; 3635 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3636 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3637 p->numa_migrate_retry = 0; 3638 /* Protect against double add, see task_tick_numa and task_numa_work */ 3639 p->numa_work.next = &p->numa_work; 3640 p->numa_faults = NULL; 3641 p->numa_pages_migrated = 0; 3642 p->total_numa_faults = 0; 3643 RCU_INIT_POINTER(p->numa_group, NULL); 3644 p->last_task_numa_placement = 0; 3645 p->last_sum_exec_runtime = 0; 3646 3647 init_task_work(&p->numa_work, task_numa_work); 3648 3649 /* New address space, reset the preferred nid */ 3650 if (!(clone_flags & CLONE_VM)) { 3651 p->numa_preferred_nid = NUMA_NO_NODE; 3652 return; 3653 } 3654 3655 /* 3656 * New thread, keep existing numa_preferred_nid which should be copied 3657 * already by arch_dup_task_struct but stagger when scans start. 3658 */ 3659 if (mm) { 3660 unsigned int delay; 3661 3662 delay = min_t(unsigned int, task_scan_max(current), 3663 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3664 delay += 2 * TICK_NSEC; 3665 p->node_stamp = delay; 3666 } 3667} 3668 3669/* 3670 * Drive the periodic memory faults.. 3671 */ 3672static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3673{ 3674 struct callback_head *work = &curr->numa_work; 3675 u64 period, now; 3676 3677 /* 3678 * We don't care about NUMA placement if we don't have memory. 3679 */ 3680 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3681 return; 3682 3683 /* 3684 * Using runtime rather than walltime has the dual advantage that 3685 * we (mostly) drive the selection from busy threads and that the 3686 * task needs to have done some actual work before we bother with 3687 * NUMA placement. 3688 */ 3689 now = curr->se.sum_exec_runtime; 3690 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3691 3692 if (now > curr->node_stamp + period) { 3693 if (!curr->node_stamp) 3694 curr->numa_scan_period = task_scan_start(curr); 3695 curr->node_stamp += period; 3696 3697 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3698 task_work_add(curr, work, TWA_RESUME); 3699 } 3700} 3701 3702static void update_scan_period(struct task_struct *p, int new_cpu) 3703{ 3704 int src_nid = cpu_to_node(task_cpu(p)); 3705 int dst_nid = cpu_to_node(new_cpu); 3706 3707 if (!static_branch_likely(&sched_numa_balancing)) 3708 return; 3709 3710 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3711 return; 3712 3713 if (src_nid == dst_nid) 3714 return; 3715 3716 /* 3717 * Allow resets if faults have been trapped before one scan 3718 * has completed. This is most likely due to a new task that 3719 * is pulled cross-node due to wakeups or load balancing. 3720 */ 3721 if (p->numa_scan_seq) { 3722 /* 3723 * Avoid scan adjustments if moving to the preferred 3724 * node or if the task was not previously running on 3725 * the preferred node. 3726 */ 3727 if (dst_nid == p->numa_preferred_nid || 3728 (p->numa_preferred_nid != NUMA_NO_NODE && 3729 src_nid != p->numa_preferred_nid)) 3730 return; 3731 } 3732 3733 p->numa_scan_period = task_scan_start(p); 3734} 3735 3736#else /* !CONFIG_NUMA_BALANCING: */ 3737 3738static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3739{ 3740} 3741 3742static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3743{ 3744} 3745 3746static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3747{ 3748} 3749 3750static inline void update_scan_period(struct task_struct *p, int new_cpu) 3751{ 3752} 3753 3754#endif /* !CONFIG_NUMA_BALANCING */ 3755 3756static void 3757account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3758{ 3759 update_load_add(&cfs_rq->load, se->load.weight); 3760 if (entity_is_task(se)) { 3761 struct rq *rq = rq_of(cfs_rq); 3762 3763 account_numa_enqueue(rq, task_of(se)); 3764 list_add(&se->group_node, &rq->cfs_tasks); 3765 } 3766 cfs_rq->nr_queued++; 3767} 3768 3769static void 3770account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3771{ 3772 update_load_sub(&cfs_rq->load, se->load.weight); 3773 if (entity_is_task(se)) { 3774 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3775 list_del_init(&se->group_node); 3776 } 3777 cfs_rq->nr_queued--; 3778} 3779 3780/* 3781 * Signed add and clamp on underflow. 3782 * 3783 * Explicitly do a load-store to ensure the intermediate value never hits 3784 * memory. This allows lockless observations without ever seeing the negative 3785 * values. 3786 */ 3787#define add_positive(_ptr, _val) do { \ 3788 typeof(_ptr) ptr = (_ptr); \ 3789 typeof(_val) val = (_val); \ 3790 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3791 \ 3792 res = var + val; \ 3793 \ 3794 if (val < 0 && res > var) \ 3795 res = 0; \ 3796 \ 3797 WRITE_ONCE(*ptr, res); \ 3798} while (0) 3799 3800/* 3801 * Unsigned subtract and clamp on underflow. 3802 * 3803 * Explicitly do a load-store to ensure the intermediate value never hits 3804 * memory. This allows lockless observations without ever seeing the negative 3805 * values. 3806 */ 3807#define sub_positive(_ptr, _val) do { \ 3808 typeof(_ptr) ptr = (_ptr); \ 3809 typeof(*ptr) val = (_val); \ 3810 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3811 res = var - val; \ 3812 if (res > var) \ 3813 res = 0; \ 3814 WRITE_ONCE(*ptr, res); \ 3815} while (0) 3816 3817/* 3818 * Remove and clamp on negative, from a local variable. 3819 * 3820 * A variant of sub_positive(), which does not use explicit load-store 3821 * and is thus optimized for local variable updates. 3822 */ 3823#define lsub_positive(_ptr, _val) do { \ 3824 typeof(_ptr) ptr = (_ptr); \ 3825 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3826} while (0) 3827 3828static inline void 3829enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3830{ 3831 cfs_rq->avg.load_avg += se->avg.load_avg; 3832 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3833} 3834 3835static inline void 3836dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3837{ 3838 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3839 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3840 /* See update_cfs_rq_load_avg() */ 3841 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3842 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3843} 3844 3845static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags); 3846 3847static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3848 unsigned long weight) 3849{ 3850 bool curr = cfs_rq->curr == se; 3851 bool rel_vprot = false; 3852 u64 vprot; 3853 3854 if (se->on_rq) { 3855 /* commit outstanding execution time */ 3856 update_curr(cfs_rq); 3857 update_entity_lag(cfs_rq, se); 3858 se->deadline -= se->vruntime; 3859 se->rel_deadline = 1; 3860 if (curr && protect_slice(se)) { 3861 vprot = se->vprot - se->vruntime; 3862 rel_vprot = true; 3863 } 3864 3865 cfs_rq->nr_queued--; 3866 if (!curr) 3867 __dequeue_entity(cfs_rq, se); 3868 update_load_sub(&cfs_rq->load, se->load.weight); 3869 } 3870 dequeue_load_avg(cfs_rq, se); 3871 3872 /* 3873 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), 3874 * we need to scale se->vlag when w_i changes. 3875 */ 3876 se->vlag = div_s64(se->vlag * se->load.weight, weight); 3877 if (se->rel_deadline) 3878 se->deadline = div_s64(se->deadline * se->load.weight, weight); 3879 3880 if (rel_vprot) 3881 vprot = div_s64(vprot * se->load.weight, weight); 3882 3883 update_load_set(&se->load, weight); 3884 3885 do { 3886 u32 divider = get_pelt_divider(&se->avg); 3887 3888 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3889 } while (0); 3890 3891 enqueue_load_avg(cfs_rq, se); 3892 if (se->on_rq) { 3893 place_entity(cfs_rq, se, 0); 3894 if (rel_vprot) 3895 se->vprot = se->vruntime + vprot; 3896 update_load_add(&cfs_rq->load, se->load.weight); 3897 if (!curr) 3898 __enqueue_entity(cfs_rq, se); 3899 cfs_rq->nr_queued++; 3900 } 3901} 3902 3903static void reweight_task_fair(struct rq *rq, struct task_struct *p, 3904 const struct load_weight *lw) 3905{ 3906 struct sched_entity *se = &p->se; 3907 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3908 struct load_weight *load = &se->load; 3909 3910 reweight_entity(cfs_rq, se, lw->weight); 3911 load->inv_weight = lw->inv_weight; 3912} 3913 3914static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3915 3916#ifdef CONFIG_FAIR_GROUP_SCHED 3917/* 3918 * All this does is approximate the hierarchical proportion which includes that 3919 * global sum we all love to hate. 3920 * 3921 * That is, the weight of a group entity, is the proportional share of the 3922 * group weight based on the group runqueue weights. That is: 3923 * 3924 * tg->weight * grq->load.weight 3925 * ge->load.weight = ----------------------------- (1) 3926 * \Sum grq->load.weight 3927 * 3928 * Now, because computing that sum is prohibitively expensive to compute (been 3929 * there, done that) we approximate it with this average stuff. The average 3930 * moves slower and therefore the approximation is cheaper and more stable. 3931 * 3932 * So instead of the above, we substitute: 3933 * 3934 * grq->load.weight -> grq->avg.load_avg (2) 3935 * 3936 * which yields the following: 3937 * 3938 * tg->weight * grq->avg.load_avg 3939 * ge->load.weight = ------------------------------ (3) 3940 * tg->load_avg 3941 * 3942 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3943 * 3944 * That is shares_avg, and it is right (given the approximation (2)). 3945 * 3946 * The problem with it is that because the average is slow -- it was designed 3947 * to be exactly that of course -- this leads to transients in boundary 3948 * conditions. In specific, the case where the group was idle and we start the 3949 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3950 * yielding bad latency etc.. 3951 * 3952 * Now, in that special case (1) reduces to: 3953 * 3954 * tg->weight * grq->load.weight 3955 * ge->load.weight = ----------------------------- = tg->weight (4) 3956 * grp->load.weight 3957 * 3958 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3959 * 3960 * So what we do is modify our approximation (3) to approach (4) in the (near) 3961 * UP case, like: 3962 * 3963 * ge->load.weight = 3964 * 3965 * tg->weight * grq->load.weight 3966 * --------------------------------------------------- (5) 3967 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3968 * 3969 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3970 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3971 * 3972 * 3973 * tg->weight * grq->load.weight 3974 * ge->load.weight = ----------------------------- (6) 3975 * tg_load_avg' 3976 * 3977 * Where: 3978 * 3979 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3980 * max(grq->load.weight, grq->avg.load_avg) 3981 * 3982 * And that is shares_weight and is icky. In the (near) UP case it approaches 3983 * (4) while in the normal case it approaches (3). It consistently 3984 * overestimates the ge->load.weight and therefore: 3985 * 3986 * \Sum ge->load.weight >= tg->weight 3987 * 3988 * hence icky! 3989 */ 3990static long calc_group_shares(struct cfs_rq *cfs_rq) 3991{ 3992 long tg_weight, tg_shares, load, shares; 3993 struct task_group *tg = cfs_rq->tg; 3994 3995 tg_shares = READ_ONCE(tg->shares); 3996 3997 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3998 3999 tg_weight = atomic_long_read(&tg->load_avg); 4000 4001 /* Ensure tg_weight >= load */ 4002 tg_weight -= cfs_rq->tg_load_avg_contrib; 4003 tg_weight += load; 4004 4005 shares = (tg_shares * load); 4006 if (tg_weight) 4007 shares /= tg_weight; 4008 4009 /* 4010 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 4011 * of a group with small tg->shares value. It is a floor value which is 4012 * assigned as a minimum load.weight to the sched_entity representing 4013 * the group on a CPU. 4014 * 4015 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 4016 * on an 8-core system with 8 tasks each runnable on one CPU shares has 4017 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 4018 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 4019 * instead of 0. 4020 */ 4021 return clamp_t(long, shares, MIN_SHARES, tg_shares); 4022} 4023 4024/* 4025 * Recomputes the group entity based on the current state of its group 4026 * runqueue. 4027 */ 4028static void update_cfs_group(struct sched_entity *se) 4029{ 4030 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4031 long shares; 4032 4033 /* 4034 * When a group becomes empty, preserve its weight. This matters for 4035 * DELAY_DEQUEUE. 4036 */ 4037 if (!gcfs_rq || !gcfs_rq->load.weight) 4038 return; 4039 4040 shares = calc_group_shares(gcfs_rq); 4041 if (unlikely(se->load.weight != shares)) 4042 reweight_entity(cfs_rq_of(se), se, shares); 4043} 4044 4045#else /* !CONFIG_FAIR_GROUP_SCHED: */ 4046static inline void update_cfs_group(struct sched_entity *se) 4047{ 4048} 4049#endif /* !CONFIG_FAIR_GROUP_SCHED */ 4050 4051static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 4052{ 4053 struct rq *rq = rq_of(cfs_rq); 4054 4055 if (&rq->cfs == cfs_rq) { 4056 /* 4057 * There are a few boundary cases this might miss but it should 4058 * get called often enough that that should (hopefully) not be 4059 * a real problem. 4060 * 4061 * It will not get called when we go idle, because the idle 4062 * thread is a different class (!fair), nor will the utilization 4063 * number include things like RT tasks. 4064 * 4065 * As is, the util number is not freq-invariant (we'd have to 4066 * implement arch_scale_freq_capacity() for that). 4067 * 4068 * See cpu_util_cfs(). 4069 */ 4070 cpufreq_update_util(rq, flags); 4071 } 4072} 4073 4074static inline bool load_avg_is_decayed(struct sched_avg *sa) 4075{ 4076 if (sa->load_sum) 4077 return false; 4078 4079 if (sa->util_sum) 4080 return false; 4081 4082 if (sa->runnable_sum) 4083 return false; 4084 4085 /* 4086 * _avg must be null when _sum are null because _avg = _sum / divider 4087 * Make sure that rounding and/or propagation of PELT values never 4088 * break this. 4089 */ 4090 WARN_ON_ONCE(sa->load_avg || 4091 sa->util_avg || 4092 sa->runnable_avg); 4093 4094 return true; 4095} 4096 4097static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 4098{ 4099 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 4100 cfs_rq->last_update_time_copy); 4101} 4102#ifdef CONFIG_FAIR_GROUP_SCHED 4103/* 4104 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 4105 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 4106 * bottom-up, we only have to test whether the cfs_rq before us on the list 4107 * is our child. 4108 * If cfs_rq is not on the list, test whether a child needs its to be added to 4109 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 4110 */ 4111static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 4112{ 4113 struct cfs_rq *prev_cfs_rq; 4114 struct list_head *prev; 4115 struct rq *rq = rq_of(cfs_rq); 4116 4117 if (cfs_rq->on_list) { 4118 prev = cfs_rq->leaf_cfs_rq_list.prev; 4119 } else { 4120 prev = rq->tmp_alone_branch; 4121 } 4122 4123 if (prev == &rq->leaf_cfs_rq_list) 4124 return false; 4125 4126 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 4127 4128 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 4129} 4130 4131static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4132{ 4133 if (cfs_rq->load.weight) 4134 return false; 4135 4136 if (!load_avg_is_decayed(&cfs_rq->avg)) 4137 return false; 4138 4139 if (child_cfs_rq_on_list(cfs_rq)) 4140 return false; 4141 4142 if (cfs_rq->tg_load_avg_contrib) 4143 return false; 4144 4145 return true; 4146} 4147 4148/** 4149 * update_tg_load_avg - update the tg's load avg 4150 * @cfs_rq: the cfs_rq whose avg changed 4151 * 4152 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 4153 * However, because tg->load_avg is a global value there are performance 4154 * considerations. 4155 * 4156 * In order to avoid having to look at the other cfs_rq's, we use a 4157 * differential update where we store the last value we propagated. This in 4158 * turn allows skipping updates if the differential is 'small'. 4159 * 4160 * Updating tg's load_avg is necessary before update_cfs_share(). 4161 */ 4162static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 4163{ 4164 long delta; 4165 u64 now; 4166 4167 /* 4168 * No need to update load_avg for root_task_group as it is not used. 4169 */ 4170 if (cfs_rq->tg == &root_task_group) 4171 return; 4172 4173 /* rq has been offline and doesn't contribute to the share anymore: */ 4174 if (!cpu_active(cpu_of(rq_of(cfs_rq)))) 4175 return; 4176 4177 /* 4178 * For migration heavy workloads, access to tg->load_avg can be 4179 * unbound. Limit the update rate to at most once per ms. 4180 */ 4181 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4182 if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) 4183 return; 4184 4185 delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 4186 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 4187 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4188 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 4189 cfs_rq->last_update_tg_load_avg = now; 4190 } 4191} 4192 4193static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) 4194{ 4195 long delta; 4196 u64 now; 4197 4198 /* 4199 * No need to update load_avg for root_task_group, as it is not used. 4200 */ 4201 if (cfs_rq->tg == &root_task_group) 4202 return; 4203 4204 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4205 delta = 0 - cfs_rq->tg_load_avg_contrib; 4206 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4207 cfs_rq->tg_load_avg_contrib = 0; 4208 cfs_rq->last_update_tg_load_avg = now; 4209} 4210 4211/* CPU offline callback: */ 4212static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) 4213{ 4214 struct task_group *tg; 4215 4216 lockdep_assert_rq_held(rq); 4217 4218 /* 4219 * The rq clock has already been updated in 4220 * set_rq_offline(), so we should skip updating 4221 * the rq clock again in unthrottle_cfs_rq(). 4222 */ 4223 rq_clock_start_loop_update(rq); 4224 4225 rcu_read_lock(); 4226 list_for_each_entry_rcu(tg, &task_groups, list) { 4227 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4228 4229 clear_tg_load_avg(cfs_rq); 4230 } 4231 rcu_read_unlock(); 4232 4233 rq_clock_stop_loop_update(rq); 4234} 4235 4236/* 4237 * Called within set_task_rq() right before setting a task's CPU. The 4238 * caller only guarantees p->pi_lock is held; no other assumptions, 4239 * including the state of rq->lock, should be made. 4240 */ 4241void set_task_rq_fair(struct sched_entity *se, 4242 struct cfs_rq *prev, struct cfs_rq *next) 4243{ 4244 u64 p_last_update_time; 4245 u64 n_last_update_time; 4246 4247 if (!sched_feat(ATTACH_AGE_LOAD)) 4248 return; 4249 4250 /* 4251 * We are supposed to update the task to "current" time, then its up to 4252 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 4253 * getting what current time is, so simply throw away the out-of-date 4254 * time. This will result in the wakee task is less decayed, but giving 4255 * the wakee more load sounds not bad. 4256 */ 4257 if (!(se->avg.last_update_time && prev)) 4258 return; 4259 4260 p_last_update_time = cfs_rq_last_update_time(prev); 4261 n_last_update_time = cfs_rq_last_update_time(next); 4262 4263 __update_load_avg_blocked_se(p_last_update_time, se); 4264 se->avg.last_update_time = n_last_update_time; 4265} 4266 4267/* 4268 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 4269 * propagate its contribution. The key to this propagation is the invariant 4270 * that for each group: 4271 * 4272 * ge->avg == grq->avg (1) 4273 * 4274 * _IFF_ we look at the pure running and runnable sums. Because they 4275 * represent the very same entity, just at different points in the hierarchy. 4276 * 4277 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 4278 * and simply copies the running/runnable sum over (but still wrong, because 4279 * the group entity and group rq do not have their PELT windows aligned). 4280 * 4281 * However, update_tg_cfs_load() is more complex. So we have: 4282 * 4283 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 4284 * 4285 * And since, like util, the runnable part should be directly transferable, 4286 * the following would _appear_ to be the straight forward approach: 4287 * 4288 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 4289 * 4290 * And per (1) we have: 4291 * 4292 * ge->avg.runnable_avg == grq->avg.runnable_avg 4293 * 4294 * Which gives: 4295 * 4296 * ge->load.weight * grq->avg.load_avg 4297 * ge->avg.load_avg = ----------------------------------- (4) 4298 * grq->load.weight 4299 * 4300 * Except that is wrong! 4301 * 4302 * Because while for entities historical weight is not important and we 4303 * really only care about our future and therefore can consider a pure 4304 * runnable sum, runqueues can NOT do this. 4305 * 4306 * We specifically want runqueues to have a load_avg that includes 4307 * historical weights. Those represent the blocked load, the load we expect 4308 * to (shortly) return to us. This only works by keeping the weights as 4309 * integral part of the sum. We therefore cannot decompose as per (3). 4310 * 4311 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 4312 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 4313 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 4314 * runnable section of these tasks overlap (or not). If they were to perfectly 4315 * align the rq as a whole would be runnable 2/3 of the time. If however we 4316 * always have at least 1 runnable task, the rq as a whole is always runnable. 4317 * 4318 * So we'll have to approximate.. :/ 4319 * 4320 * Given the constraint: 4321 * 4322 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 4323 * 4324 * We can construct a rule that adds runnable to a rq by assuming minimal 4325 * overlap. 4326 * 4327 * On removal, we'll assume each task is equally runnable; which yields: 4328 * 4329 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 4330 * 4331 * XXX: only do this for the part of runnable > running ? 4332 * 4333 */ 4334static inline void 4335update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4336{ 4337 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 4338 u32 new_sum, divider; 4339 4340 /* Nothing to update */ 4341 if (!delta_avg) 4342 return; 4343 4344 /* 4345 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4346 * See ___update_load_avg() for details. 4347 */ 4348 divider = get_pelt_divider(&cfs_rq->avg); 4349 4350 4351 /* Set new sched_entity's utilization */ 4352 se->avg.util_avg = gcfs_rq->avg.util_avg; 4353 new_sum = se->avg.util_avg * divider; 4354 delta_sum = (long)new_sum - (long)se->avg.util_sum; 4355 se->avg.util_sum = new_sum; 4356 4357 /* Update parent cfs_rq utilization */ 4358 add_positive(&cfs_rq->avg.util_avg, delta_avg); 4359 add_positive(&cfs_rq->avg.util_sum, delta_sum); 4360 4361 /* See update_cfs_rq_load_avg() */ 4362 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4363 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4364} 4365 4366static inline void 4367update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4368{ 4369 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 4370 u32 new_sum, divider; 4371 4372 /* Nothing to update */ 4373 if (!delta_avg) 4374 return; 4375 4376 /* 4377 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4378 * See ___update_load_avg() for details. 4379 */ 4380 divider = get_pelt_divider(&cfs_rq->avg); 4381 4382 /* Set new sched_entity's runnable */ 4383 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 4384 new_sum = se->avg.runnable_avg * divider; 4385 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 4386 se->avg.runnable_sum = new_sum; 4387 4388 /* Update parent cfs_rq runnable */ 4389 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); 4390 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); 4391 /* See update_cfs_rq_load_avg() */ 4392 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4393 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4394} 4395 4396static inline void 4397update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4398{ 4399 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 4400 unsigned long load_avg; 4401 u64 load_sum = 0; 4402 s64 delta_sum; 4403 u32 divider; 4404 4405 if (!runnable_sum) 4406 return; 4407 4408 gcfs_rq->prop_runnable_sum = 0; 4409 4410 /* 4411 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4412 * See ___update_load_avg() for details. 4413 */ 4414 divider = get_pelt_divider(&cfs_rq->avg); 4415 4416 if (runnable_sum >= 0) { 4417 /* 4418 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 4419 * the CPU is saturated running == runnable. 4420 */ 4421 runnable_sum += se->avg.load_sum; 4422 runnable_sum = min_t(long, runnable_sum, divider); 4423 } else { 4424 /* 4425 * Estimate the new unweighted runnable_sum of the gcfs_rq by 4426 * assuming all tasks are equally runnable. 4427 */ 4428 if (scale_load_down(gcfs_rq->load.weight)) { 4429 load_sum = div_u64(gcfs_rq->avg.load_sum, 4430 scale_load_down(gcfs_rq->load.weight)); 4431 } 4432 4433 /* But make sure to not inflate se's runnable */ 4434 runnable_sum = min(se->avg.load_sum, load_sum); 4435 } 4436 4437 /* 4438 * runnable_sum can't be lower than running_sum 4439 * Rescale running sum to be in the same range as runnable sum 4440 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 4441 * runnable_sum is in [0 : LOAD_AVG_MAX] 4442 */ 4443 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 4444 runnable_sum = max(runnable_sum, running_sum); 4445 4446 load_sum = se_weight(se) * runnable_sum; 4447 load_avg = div_u64(load_sum, divider); 4448 4449 delta_avg = load_avg - se->avg.load_avg; 4450 if (!delta_avg) 4451 return; 4452 4453 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 4454 4455 se->avg.load_sum = runnable_sum; 4456 se->avg.load_avg = load_avg; 4457 add_positive(&cfs_rq->avg.load_avg, delta_avg); 4458 add_positive(&cfs_rq->avg.load_sum, delta_sum); 4459 /* See update_cfs_rq_load_avg() */ 4460 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 4461 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 4462} 4463 4464static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 4465{ 4466 cfs_rq->propagate = 1; 4467 cfs_rq->prop_runnable_sum += runnable_sum; 4468} 4469 4470/* Update task and its cfs_rq load average */ 4471static inline int propagate_entity_load_avg(struct sched_entity *se) 4472{ 4473 struct cfs_rq *cfs_rq, *gcfs_rq; 4474 4475 if (entity_is_task(se)) 4476 return 0; 4477 4478 gcfs_rq = group_cfs_rq(se); 4479 if (!gcfs_rq->propagate) 4480 return 0; 4481 4482 gcfs_rq->propagate = 0; 4483 4484 cfs_rq = cfs_rq_of(se); 4485 4486 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 4487 4488 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 4489 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 4490 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 4491 4492 trace_pelt_cfs_tp(cfs_rq); 4493 trace_pelt_se_tp(se); 4494 4495 return 1; 4496} 4497 4498/* 4499 * Check if we need to update the load and the utilization of a blocked 4500 * group_entity: 4501 */ 4502static inline bool skip_blocked_update(struct sched_entity *se) 4503{ 4504 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4505 4506 /* 4507 * If sched_entity still have not zero load or utilization, we have to 4508 * decay it: 4509 */ 4510 if (se->avg.load_avg || se->avg.util_avg) 4511 return false; 4512 4513 /* 4514 * If there is a pending propagation, we have to update the load and 4515 * the utilization of the sched_entity: 4516 */ 4517 if (gcfs_rq->propagate) 4518 return false; 4519 4520 /* 4521 * Otherwise, the load and the utilization of the sched_entity is 4522 * already zero and there is no pending propagation, so it will be a 4523 * waste of time to try to decay it: 4524 */ 4525 return true; 4526} 4527 4528#else /* !CONFIG_FAIR_GROUP_SCHED: */ 4529 4530static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 4531 4532static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {} 4533 4534static inline int propagate_entity_load_avg(struct sched_entity *se) 4535{ 4536 return 0; 4537} 4538 4539static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 4540 4541#endif /* !CONFIG_FAIR_GROUP_SCHED */ 4542 4543#ifdef CONFIG_NO_HZ_COMMON 4544static inline void migrate_se_pelt_lag(struct sched_entity *se) 4545{ 4546 u64 throttled = 0, now, lut; 4547 struct cfs_rq *cfs_rq; 4548 struct rq *rq; 4549 bool is_idle; 4550 4551 if (load_avg_is_decayed(&se->avg)) 4552 return; 4553 4554 cfs_rq = cfs_rq_of(se); 4555 rq = rq_of(cfs_rq); 4556 4557 rcu_read_lock(); 4558 is_idle = is_idle_task(rcu_dereference(rq->curr)); 4559 rcu_read_unlock(); 4560 4561 /* 4562 * The lag estimation comes with a cost we don't want to pay all the 4563 * time. Hence, limiting to the case where the source CPU is idle and 4564 * we know we are at the greatest risk to have an outdated clock. 4565 */ 4566 if (!is_idle) 4567 return; 4568 4569 /* 4570 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4571 * 4572 * last_update_time (the cfs_rq's last_update_time) 4573 * = cfs_rq_clock_pelt()@cfs_rq_idle 4574 * = rq_clock_pelt()@cfs_rq_idle 4575 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4576 * 4577 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4578 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4579 * 4580 * rq_idle_lag (delta between now and rq's update) 4581 * = sched_clock_cpu() - rq_clock()@rq_idle 4582 * 4583 * We can then write: 4584 * 4585 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4586 * sched_clock_cpu() - rq_clock()@rq_idle 4587 * Where: 4588 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4589 * rq_clock()@rq_idle is rq->clock_idle 4590 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4591 * is cfs_rq->throttled_pelt_idle 4592 */ 4593 4594#ifdef CONFIG_CFS_BANDWIDTH 4595 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4596 /* The clock has been stopped for throttling */ 4597 if (throttled == U64_MAX) 4598 return; 4599#endif 4600 now = u64_u32_load(rq->clock_pelt_idle); 4601 /* 4602 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4603 * is observed the old clock_pelt_idle value and the new clock_idle, 4604 * which lead to an underestimation. The opposite would lead to an 4605 * overestimation. 4606 */ 4607 smp_rmb(); 4608 lut = cfs_rq_last_update_time(cfs_rq); 4609 4610 now -= throttled; 4611 if (now < lut) 4612 /* 4613 * cfs_rq->avg.last_update_time is more recent than our 4614 * estimation, let's use it. 4615 */ 4616 now = lut; 4617 else 4618 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4619 4620 __update_load_avg_blocked_se(now, se); 4621} 4622#else /* !CONFIG_NO_HZ_COMMON: */ 4623static void migrate_se_pelt_lag(struct sched_entity *se) {} 4624#endif /* !CONFIG_NO_HZ_COMMON */ 4625 4626/** 4627 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4628 * @now: current time, as per cfs_rq_clock_pelt() 4629 * @cfs_rq: cfs_rq to update 4630 * 4631 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4632 * avg. The immediate corollary is that all (fair) tasks must be attached. 4633 * 4634 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4635 * 4636 * Return: true if the load decayed or we removed load. 4637 * 4638 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4639 * call update_tg_load_avg() when this function returns true. 4640 */ 4641static inline int 4642update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4643{ 4644 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4645 struct sched_avg *sa = &cfs_rq->avg; 4646 int decayed = 0; 4647 4648 if (cfs_rq->removed.nr) { 4649 unsigned long r; 4650 u32 divider = get_pelt_divider(&cfs_rq->avg); 4651 4652 raw_spin_lock(&cfs_rq->removed.lock); 4653 swap(cfs_rq->removed.util_avg, removed_util); 4654 swap(cfs_rq->removed.load_avg, removed_load); 4655 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4656 cfs_rq->removed.nr = 0; 4657 raw_spin_unlock(&cfs_rq->removed.lock); 4658 4659 r = removed_load; 4660 sub_positive(&sa->load_avg, r); 4661 sub_positive(&sa->load_sum, r * divider); 4662 /* See sa->util_sum below */ 4663 sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER); 4664 4665 r = removed_util; 4666 sub_positive(&sa->util_avg, r); 4667 sub_positive(&sa->util_sum, r * divider); 4668 /* 4669 * Because of rounding, se->util_sum might ends up being +1 more than 4670 * cfs->util_sum. Although this is not a problem by itself, detaching 4671 * a lot of tasks with the rounding problem between 2 updates of 4672 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4673 * cfs_util_avg is not. 4674 * Check that util_sum is still above its lower bound for the new 4675 * util_avg. Given that period_contrib might have moved since the last 4676 * sync, we are only sure that util_sum must be above or equal to 4677 * util_avg * minimum possible divider 4678 */ 4679 sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER); 4680 4681 r = removed_runnable; 4682 sub_positive(&sa->runnable_avg, r); 4683 sub_positive(&sa->runnable_sum, r * divider); 4684 /* See sa->util_sum above */ 4685 sa->runnable_sum = max_t(u32, sa->runnable_sum, 4686 sa->runnable_avg * PELT_MIN_DIVIDER); 4687 4688 /* 4689 * removed_runnable is the unweighted version of removed_load so we 4690 * can use it to estimate removed_load_sum. 4691 */ 4692 add_tg_cfs_propagate(cfs_rq, 4693 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4694 4695 decayed = 1; 4696 } 4697 4698 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4699 u64_u32_store_copy(sa->last_update_time, 4700 cfs_rq->last_update_time_copy, 4701 sa->last_update_time); 4702 return decayed; 4703} 4704 4705/** 4706 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4707 * @cfs_rq: cfs_rq to attach to 4708 * @se: sched_entity to attach 4709 * 4710 * Must call update_cfs_rq_load_avg() before this, since we rely on 4711 * cfs_rq->avg.last_update_time being current. 4712 */ 4713static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4714{ 4715 /* 4716 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4717 * See ___update_load_avg() for details. 4718 */ 4719 u32 divider = get_pelt_divider(&cfs_rq->avg); 4720 4721 /* 4722 * When we attach the @se to the @cfs_rq, we must align the decay 4723 * window because without that, really weird and wonderful things can 4724 * happen. 4725 * 4726 * XXX illustrate 4727 */ 4728 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4729 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4730 4731 /* 4732 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4733 * period_contrib. This isn't strictly correct, but since we're 4734 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4735 * _sum a little. 4736 */ 4737 se->avg.util_sum = se->avg.util_avg * divider; 4738 4739 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4740 4741 se->avg.load_sum = se->avg.load_avg * divider; 4742 if (se_weight(se) < se->avg.load_sum) 4743 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4744 else 4745 se->avg.load_sum = 1; 4746 4747 enqueue_load_avg(cfs_rq, se); 4748 cfs_rq->avg.util_avg += se->avg.util_avg; 4749 cfs_rq->avg.util_sum += se->avg.util_sum; 4750 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4751 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4752 4753 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4754 4755 cfs_rq_util_change(cfs_rq, 0); 4756 4757 trace_pelt_cfs_tp(cfs_rq); 4758} 4759 4760/** 4761 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4762 * @cfs_rq: cfs_rq to detach from 4763 * @se: sched_entity to detach 4764 * 4765 * Must call update_cfs_rq_load_avg() before this, since we rely on 4766 * cfs_rq->avg.last_update_time being current. 4767 */ 4768static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4769{ 4770 dequeue_load_avg(cfs_rq, se); 4771 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 4772 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 4773 /* See update_cfs_rq_load_avg() */ 4774 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4775 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4776 4777 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 4778 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 4779 /* See update_cfs_rq_load_avg() */ 4780 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4781 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4782 4783 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4784 4785 cfs_rq_util_change(cfs_rq, 0); 4786 4787 trace_pelt_cfs_tp(cfs_rq); 4788} 4789 4790/* 4791 * Optional action to be done while updating the load average 4792 */ 4793#define UPDATE_TG 0x1 4794#define SKIP_AGE_LOAD 0x2 4795#define DO_ATTACH 0x4 4796#define DO_DETACH 0x8 4797 4798/* Update task and its cfs_rq load average */ 4799static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4800{ 4801 u64 now = cfs_rq_clock_pelt(cfs_rq); 4802 int decayed; 4803 4804 /* 4805 * Track task load average for carrying it to new CPU after migrated, and 4806 * track group sched_entity load average for task_h_load calculation in migration 4807 */ 4808 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4809 __update_load_avg_se(now, cfs_rq, se); 4810 4811 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4812 decayed |= propagate_entity_load_avg(se); 4813 4814 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4815 4816 /* 4817 * DO_ATTACH means we're here from enqueue_entity(). 4818 * !last_update_time means we've passed through 4819 * migrate_task_rq_fair() indicating we migrated. 4820 * 4821 * IOW we're enqueueing a task on a new CPU. 4822 */ 4823 attach_entity_load_avg(cfs_rq, se); 4824 update_tg_load_avg(cfs_rq); 4825 4826 } else if (flags & DO_DETACH) { 4827 /* 4828 * DO_DETACH means we're here from dequeue_entity() 4829 * and we are migrating task out of the CPU. 4830 */ 4831 detach_entity_load_avg(cfs_rq, se); 4832 update_tg_load_avg(cfs_rq); 4833 } else if (decayed) { 4834 cfs_rq_util_change(cfs_rq, 0); 4835 4836 if (flags & UPDATE_TG) 4837 update_tg_load_avg(cfs_rq); 4838 } 4839} 4840 4841/* 4842 * Synchronize entity load avg of dequeued entity without locking 4843 * the previous rq. 4844 */ 4845static void sync_entity_load_avg(struct sched_entity *se) 4846{ 4847 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4848 u64 last_update_time; 4849 4850 last_update_time = cfs_rq_last_update_time(cfs_rq); 4851 __update_load_avg_blocked_se(last_update_time, se); 4852} 4853 4854/* 4855 * Task first catches up with cfs_rq, and then subtract 4856 * itself from the cfs_rq (task must be off the queue now). 4857 */ 4858static void remove_entity_load_avg(struct sched_entity *se) 4859{ 4860 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4861 unsigned long flags; 4862 4863 /* 4864 * tasks cannot exit without having gone through wake_up_new_task() -> 4865 * enqueue_task_fair() which will have added things to the cfs_rq, 4866 * so we can remove unconditionally. 4867 */ 4868 4869 sync_entity_load_avg(se); 4870 4871 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4872 ++cfs_rq->removed.nr; 4873 cfs_rq->removed.util_avg += se->avg.util_avg; 4874 cfs_rq->removed.load_avg += se->avg.load_avg; 4875 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4876 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4877} 4878 4879static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4880{ 4881 return cfs_rq->avg.runnable_avg; 4882} 4883 4884static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4885{ 4886 return cfs_rq->avg.load_avg; 4887} 4888 4889static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf); 4890 4891static inline unsigned long task_util(struct task_struct *p) 4892{ 4893 return READ_ONCE(p->se.avg.util_avg); 4894} 4895 4896static inline unsigned long task_runnable(struct task_struct *p) 4897{ 4898 return READ_ONCE(p->se.avg.runnable_avg); 4899} 4900 4901static inline unsigned long _task_util_est(struct task_struct *p) 4902{ 4903 return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; 4904} 4905 4906static inline unsigned long task_util_est(struct task_struct *p) 4907{ 4908 return max(task_util(p), _task_util_est(p)); 4909} 4910 4911static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4912 struct task_struct *p) 4913{ 4914 unsigned int enqueued; 4915 4916 if (!sched_feat(UTIL_EST)) 4917 return; 4918 4919 /* Update root cfs_rq's estimated utilization */ 4920 enqueued = cfs_rq->avg.util_est; 4921 enqueued += _task_util_est(p); 4922 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4923 4924 trace_sched_util_est_cfs_tp(cfs_rq); 4925} 4926 4927static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4928 struct task_struct *p) 4929{ 4930 unsigned int enqueued; 4931 4932 if (!sched_feat(UTIL_EST)) 4933 return; 4934 4935 /* Update root cfs_rq's estimated utilization */ 4936 enqueued = cfs_rq->avg.util_est; 4937 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4938 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4939 4940 trace_sched_util_est_cfs_tp(cfs_rq); 4941} 4942 4943#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4944 4945static inline void util_est_update(struct cfs_rq *cfs_rq, 4946 struct task_struct *p, 4947 bool task_sleep) 4948{ 4949 unsigned int ewma, dequeued, last_ewma_diff; 4950 4951 if (!sched_feat(UTIL_EST)) 4952 return; 4953 4954 /* 4955 * Skip update of task's estimated utilization when the task has not 4956 * yet completed an activation, e.g. being migrated. 4957 */ 4958 if (!task_sleep) 4959 return; 4960 4961 /* Get current estimate of utilization */ 4962 ewma = READ_ONCE(p->se.avg.util_est); 4963 4964 /* 4965 * If the PELT values haven't changed since enqueue time, 4966 * skip the util_est update. 4967 */ 4968 if (ewma & UTIL_AVG_UNCHANGED) 4969 return; 4970 4971 /* Get utilization at dequeue */ 4972 dequeued = task_util(p); 4973 4974 /* 4975 * Reset EWMA on utilization increases, the moving average is used only 4976 * to smooth utilization decreases. 4977 */ 4978 if (ewma <= dequeued) { 4979 ewma = dequeued; 4980 goto done; 4981 } 4982 4983 /* 4984 * Skip update of task's estimated utilization when its members are 4985 * already ~1% close to its last activation value. 4986 */ 4987 last_ewma_diff = ewma - dequeued; 4988 if (last_ewma_diff < UTIL_EST_MARGIN) 4989 goto done; 4990 4991 /* 4992 * To avoid underestimate of task utilization, skip updates of EWMA if 4993 * we cannot grant that thread got all CPU time it wanted. 4994 */ 4995 if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p)) 4996 goto done; 4997 4998 4999 /* 5000 * Update Task's estimated utilization 5001 * 5002 * When *p completes an activation we can consolidate another sample 5003 * of the task size. This is done by using this value to update the 5004 * Exponential Weighted Moving Average (EWMA): 5005 * 5006 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 5007 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 5008 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 5009 * = w * ( -last_ewma_diff ) + ewma(t-1) 5010 * = w * (-last_ewma_diff + ewma(t-1) / w) 5011 * 5012 * Where 'w' is the weight of new samples, which is configured to be 5013 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 5014 */ 5015 ewma <<= UTIL_EST_WEIGHT_SHIFT; 5016 ewma -= last_ewma_diff; 5017 ewma >>= UTIL_EST_WEIGHT_SHIFT; 5018done: 5019 ewma |= UTIL_AVG_UNCHANGED; 5020 WRITE_ONCE(p->se.avg.util_est, ewma); 5021 5022 trace_sched_util_est_se_tp(&p->se); 5023} 5024 5025static inline unsigned long get_actual_cpu_capacity(int cpu) 5026{ 5027 unsigned long capacity = arch_scale_cpu_capacity(cpu); 5028 5029 capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu)); 5030 5031 return capacity; 5032} 5033 5034static inline int util_fits_cpu(unsigned long util, 5035 unsigned long uclamp_min, 5036 unsigned long uclamp_max, 5037 int cpu) 5038{ 5039 unsigned long capacity = capacity_of(cpu); 5040 unsigned long capacity_orig; 5041 bool fits, uclamp_max_fits; 5042 5043 /* 5044 * Check if the real util fits without any uclamp boost/cap applied. 5045 */ 5046 fits = fits_capacity(util, capacity); 5047 5048 if (!uclamp_is_used()) 5049 return fits; 5050 5051 /* 5052 * We must use arch_scale_cpu_capacity() for comparing against uclamp_min and 5053 * uclamp_max. We only care about capacity pressure (by using 5054 * capacity_of()) for comparing against the real util. 5055 * 5056 * If a task is boosted to 1024 for example, we don't want a tiny 5057 * pressure to skew the check whether it fits a CPU or not. 5058 * 5059 * Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it 5060 * should fit a little cpu even if there's some pressure. 5061 * 5062 * Only exception is for HW or cpufreq pressure since it has a direct impact 5063 * on available OPP of the system. 5064 * 5065 * We honour it for uclamp_min only as a drop in performance level 5066 * could result in not getting the requested minimum performance level. 5067 * 5068 * For uclamp_max, we can tolerate a drop in performance level as the 5069 * goal is to cap the task. So it's okay if it's getting less. 5070 */ 5071 capacity_orig = arch_scale_cpu_capacity(cpu); 5072 5073 /* 5074 * We want to force a task to fit a cpu as implied by uclamp_max. 5075 * But we do have some corner cases to cater for.. 5076 * 5077 * 5078 * C=z 5079 * | ___ 5080 * | C=y | | 5081 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5082 * | C=x | | | | 5083 * | ___ | | | | 5084 * | | | | | | | (util somewhere in this region) 5085 * | | | | | | | 5086 * | | | | | | | 5087 * +---------------------------------------- 5088 * CPU0 CPU1 CPU2 5089 * 5090 * In the above example if a task is capped to a specific performance 5091 * point, y, then when: 5092 * 5093 * * util = 80% of x then it does not fit on CPU0 and should migrate 5094 * to CPU1 5095 * * util = 80% of y then it is forced to fit on CPU1 to honour 5096 * uclamp_max request. 5097 * 5098 * which is what we're enforcing here. A task always fits if 5099 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 5100 * the normal upmigration rules should withhold still. 5101 * 5102 * Only exception is when we are on max capacity, then we need to be 5103 * careful not to block overutilized state. This is so because: 5104 * 5105 * 1. There's no concept of capping at max_capacity! We can't go 5106 * beyond this performance level anyway. 5107 * 2. The system is being saturated when we're operating near 5108 * max capacity, it doesn't make sense to block overutilized. 5109 */ 5110 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 5111 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 5112 fits = fits || uclamp_max_fits; 5113 5114 /* 5115 * 5116 * C=z 5117 * | ___ (region a, capped, util >= uclamp_max) 5118 * | C=y | | 5119 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5120 * | C=x | | | | 5121 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 5122 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 5123 * | | | | | | | 5124 * | | | | | | | (region c, boosted, util < uclamp_min) 5125 * +---------------------------------------- 5126 * CPU0 CPU1 CPU2 5127 * 5128 * a) If util > uclamp_max, then we're capped, we don't care about 5129 * actual fitness value here. We only care if uclamp_max fits 5130 * capacity without taking margin/pressure into account. 5131 * See comment above. 5132 * 5133 * b) If uclamp_min <= util <= uclamp_max, then the normal 5134 * fits_capacity() rules apply. Except we need to ensure that we 5135 * enforce we remain within uclamp_max, see comment above. 5136 * 5137 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 5138 * need to take into account the boosted value fits the CPU without 5139 * taking margin/pressure into account. 5140 * 5141 * Cases (a) and (b) are handled in the 'fits' variable already. We 5142 * just need to consider an extra check for case (c) after ensuring we 5143 * handle the case uclamp_min > uclamp_max. 5144 */ 5145 uclamp_min = min(uclamp_min, uclamp_max); 5146 if (fits && (util < uclamp_min) && 5147 (uclamp_min > get_actual_cpu_capacity(cpu))) 5148 return -1; 5149 5150 return fits; 5151} 5152 5153static inline int task_fits_cpu(struct task_struct *p, int cpu) 5154{ 5155 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 5156 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 5157 unsigned long util = task_util_est(p); 5158 /* 5159 * Return true only if the cpu fully fits the task requirements, which 5160 * include the utilization but also the performance hints. 5161 */ 5162 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 5163} 5164 5165static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 5166{ 5167 int cpu = cpu_of(rq); 5168 5169 if (!sched_asym_cpucap_active()) 5170 return; 5171 5172 /* 5173 * Affinity allows us to go somewhere higher? Or are we on biggest 5174 * available CPU already? Or do we fit into this CPU ? 5175 */ 5176 if (!p || (p->nr_cpus_allowed == 1) || 5177 (arch_scale_cpu_capacity(cpu) == p->max_allowed_capacity) || 5178 task_fits_cpu(p, cpu)) { 5179 5180 rq->misfit_task_load = 0; 5181 return; 5182 } 5183 5184 /* 5185 * Make sure that misfit_task_load will not be null even if 5186 * task_h_load() returns 0. 5187 */ 5188 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 5189} 5190 5191void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) 5192{ 5193 struct sched_entity *se = &p->se; 5194 5195 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 5196 if (attr->sched_runtime) { 5197 se->custom_slice = 1; 5198 se->slice = clamp_t(u64, attr->sched_runtime, 5199 NSEC_PER_MSEC/10, /* HZ=1000 * 10 */ 5200 NSEC_PER_MSEC*100); /* HZ=100 / 10 */ 5201 } else { 5202 se->custom_slice = 0; 5203 se->slice = sysctl_sched_base_slice; 5204 } 5205} 5206 5207static void 5208place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5209{ 5210 u64 vslice, vruntime = avg_vruntime(cfs_rq); 5211 s64 lag = 0; 5212 5213 if (!se->custom_slice) 5214 se->slice = sysctl_sched_base_slice; 5215 vslice = calc_delta_fair(se->slice, se); 5216 5217 /* 5218 * Due to how V is constructed as the weighted average of entities, 5219 * adding tasks with positive lag, or removing tasks with negative lag 5220 * will move 'time' backwards, this can screw around with the lag of 5221 * other tasks. 5222 * 5223 * EEVDF: placement strategy #1 / #2 5224 */ 5225 if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { 5226 struct sched_entity *curr = cfs_rq->curr; 5227 unsigned long load; 5228 5229 lag = se->vlag; 5230 5231 /* 5232 * If we want to place a task and preserve lag, we have to 5233 * consider the effect of the new entity on the weighted 5234 * average and compensate for this, otherwise lag can quickly 5235 * evaporate. 5236 * 5237 * Lag is defined as: 5238 * 5239 * lag_i = S - s_i = w_i * (V - v_i) 5240 * 5241 * To avoid the 'w_i' term all over the place, we only track 5242 * the virtual lag: 5243 * 5244 * vl_i = V - v_i <=> v_i = V - vl_i 5245 * 5246 * And we take V to be the weighted average of all v: 5247 * 5248 * V = (\Sum w_j*v_j) / W 5249 * 5250 * Where W is: \Sum w_j 5251 * 5252 * Then, the weighted average after adding an entity with lag 5253 * vl_i is given by: 5254 * 5255 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 5256 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 5257 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 5258 * = (V*(W + w_i) - w_i*vl_i) / (W + w_i) 5259 * = V - w_i*vl_i / (W + w_i) 5260 * 5261 * And the actual lag after adding an entity with vl_i is: 5262 * 5263 * vl'_i = V' - v_i 5264 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 5265 * = vl_i - w_i*vl_i / (W + w_i) 5266 * 5267 * Which is strictly less than vl_i. So in order to preserve lag 5268 * we should inflate the lag before placement such that the 5269 * effective lag after placement comes out right. 5270 * 5271 * As such, invert the above relation for vl'_i to get the vl_i 5272 * we need to use such that the lag after placement is the lag 5273 * we computed before dequeue. 5274 * 5275 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 5276 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 5277 * 5278 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 5279 * = W*vl_i 5280 * 5281 * vl_i = (W + w_i)*vl'_i / W 5282 */ 5283 load = cfs_rq->sum_weight; 5284 if (curr && curr->on_rq) 5285 load += scale_load_down(curr->load.weight); 5286 5287 lag *= load + scale_load_down(se->load.weight); 5288 if (WARN_ON_ONCE(!load)) 5289 load = 1; 5290 lag = div_s64(lag, load); 5291 } 5292 5293 se->vruntime = vruntime - lag; 5294 5295 if (se->rel_deadline) { 5296 se->deadline += se->vruntime; 5297 se->rel_deadline = 0; 5298 return; 5299 } 5300 5301 /* 5302 * When joining the competition; the existing tasks will be, 5303 * on average, halfway through their slice, as such start tasks 5304 * off with half a slice to ease into the competition. 5305 */ 5306 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 5307 vslice /= 2; 5308 5309 /* 5310 * EEVDF: vd_i = ve_i + r_i/w_i 5311 */ 5312 se->deadline = se->vruntime + vslice; 5313} 5314 5315static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 5316static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 5317 5318static void 5319requeue_delayed_entity(struct sched_entity *se); 5320 5321static void 5322enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5323{ 5324 bool curr = cfs_rq->curr == se; 5325 5326 /* 5327 * If we're the current task, we must renormalise before calling 5328 * update_curr(). 5329 */ 5330 if (curr) 5331 place_entity(cfs_rq, se, flags); 5332 5333 update_curr(cfs_rq); 5334 5335 /* 5336 * When enqueuing a sched_entity, we must: 5337 * - Update loads to have both entity and cfs_rq synced with now. 5338 * - For group_entity, update its runnable_weight to reflect the new 5339 * h_nr_runnable of its group cfs_rq. 5340 * - For group_entity, update its weight to reflect the new share of 5341 * its group cfs_rq 5342 * - Add its new weight to cfs_rq->load.weight 5343 */ 5344 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 5345 se_update_runnable(se); 5346 /* 5347 * XXX update_load_avg() above will have attached us to the pelt sum; 5348 * but update_cfs_group() here will re-adjust the weight and have to 5349 * undo/redo all that. Seems wasteful. 5350 */ 5351 update_cfs_group(se); 5352 5353 /* 5354 * XXX now that the entity has been re-weighted, and it's lag adjusted, 5355 * we can place the entity. 5356 */ 5357 if (!curr) 5358 place_entity(cfs_rq, se, flags); 5359 5360 account_entity_enqueue(cfs_rq, se); 5361 5362 /* Entity has migrated, no longer consider this task hot */ 5363 if (flags & ENQUEUE_MIGRATED) 5364 se->exec_start = 0; 5365 5366 check_schedstat_required(); 5367 update_stats_enqueue_fair(cfs_rq, se, flags); 5368 if (!curr) 5369 __enqueue_entity(cfs_rq, se); 5370 se->on_rq = 1; 5371 5372 if (cfs_rq->nr_queued == 1) { 5373 check_enqueue_throttle(cfs_rq); 5374 list_add_leaf_cfs_rq(cfs_rq); 5375#ifdef CONFIG_CFS_BANDWIDTH 5376 if (cfs_rq->pelt_clock_throttled) { 5377 struct rq *rq = rq_of(cfs_rq); 5378 5379 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5380 cfs_rq->throttled_clock_pelt; 5381 cfs_rq->pelt_clock_throttled = 0; 5382 } 5383#endif 5384 } 5385} 5386 5387static void __clear_buddies_next(struct sched_entity *se) 5388{ 5389 for_each_sched_entity(se) { 5390 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5391 if (cfs_rq->next != se) 5392 break; 5393 5394 cfs_rq->next = NULL; 5395 } 5396} 5397 5398static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 5399{ 5400 if (cfs_rq->next == se) 5401 __clear_buddies_next(se); 5402} 5403 5404static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5405 5406static void set_delayed(struct sched_entity *se) 5407{ 5408 se->sched_delayed = 1; 5409 5410 /* 5411 * Delayed se of cfs_rq have no tasks queued on them. 5412 * Do not adjust h_nr_runnable since dequeue_entities() 5413 * will account it for blocked tasks. 5414 */ 5415 if (!entity_is_task(se)) 5416 return; 5417 5418 for_each_sched_entity(se) { 5419 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5420 5421 cfs_rq->h_nr_runnable--; 5422 } 5423} 5424 5425static void clear_delayed(struct sched_entity *se) 5426{ 5427 se->sched_delayed = 0; 5428 5429 /* 5430 * Delayed se of cfs_rq have no tasks queued on them. 5431 * Do not adjust h_nr_runnable since a dequeue has 5432 * already accounted for it or an enqueue of a task 5433 * below it will account for it in enqueue_task_fair(). 5434 */ 5435 if (!entity_is_task(se)) 5436 return; 5437 5438 for_each_sched_entity(se) { 5439 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5440 5441 cfs_rq->h_nr_runnable++; 5442 } 5443} 5444 5445static inline void finish_delayed_dequeue_entity(struct sched_entity *se) 5446{ 5447 clear_delayed(se); 5448 if (sched_feat(DELAY_ZERO) && se->vlag > 0) 5449 se->vlag = 0; 5450} 5451 5452static bool 5453dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5454{ 5455 bool sleep = flags & DEQUEUE_SLEEP; 5456 int action = UPDATE_TG; 5457 5458 update_curr(cfs_rq); 5459 clear_buddies(cfs_rq, se); 5460 5461 if (flags & DEQUEUE_DELAYED) { 5462 WARN_ON_ONCE(!se->sched_delayed); 5463 } else { 5464 bool delay = sleep; 5465 /* 5466 * DELAY_DEQUEUE relies on spurious wakeups, special task 5467 * states must not suffer spurious wakeups, excempt them. 5468 */ 5469 if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) 5470 delay = false; 5471 5472 WARN_ON_ONCE(delay && se->sched_delayed); 5473 5474 if (sched_feat(DELAY_DEQUEUE) && delay && 5475 !entity_eligible(cfs_rq, se)) { 5476 update_load_avg(cfs_rq, se, 0); 5477 set_delayed(se); 5478 return false; 5479 } 5480 } 5481 5482 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 5483 action |= DO_DETACH; 5484 5485 /* 5486 * When dequeuing a sched_entity, we must: 5487 * - Update loads to have both entity and cfs_rq synced with now. 5488 * - For group_entity, update its runnable_weight to reflect the new 5489 * h_nr_runnable of its group cfs_rq. 5490 * - Subtract its previous weight from cfs_rq->load.weight. 5491 * - For group entity, update its weight to reflect the new share 5492 * of its group cfs_rq. 5493 */ 5494 update_load_avg(cfs_rq, se, action); 5495 se_update_runnable(se); 5496 5497 update_stats_dequeue_fair(cfs_rq, se, flags); 5498 5499 update_entity_lag(cfs_rq, se); 5500 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { 5501 se->deadline -= se->vruntime; 5502 se->rel_deadline = 1; 5503 } 5504 5505 if (se != cfs_rq->curr) 5506 __dequeue_entity(cfs_rq, se); 5507 se->on_rq = 0; 5508 account_entity_dequeue(cfs_rq, se); 5509 5510 /* return excess runtime on last dequeue */ 5511 return_cfs_rq_runtime(cfs_rq); 5512 5513 update_cfs_group(se); 5514 5515 if (flags & DEQUEUE_DELAYED) 5516 finish_delayed_dequeue_entity(se); 5517 5518 if (cfs_rq->nr_queued == 0) { 5519 update_idle_cfs_rq_clock_pelt(cfs_rq); 5520#ifdef CONFIG_CFS_BANDWIDTH 5521 if (throttled_hierarchy(cfs_rq)) { 5522 struct rq *rq = rq_of(cfs_rq); 5523 5524 list_del_leaf_cfs_rq(cfs_rq); 5525 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5526 cfs_rq->pelt_clock_throttled = 1; 5527 } 5528#endif 5529 } 5530 5531 return true; 5532} 5533 5534static void 5535set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) 5536{ 5537 clear_buddies(cfs_rq, se); 5538 5539 /* 'current' is not kept within the tree. */ 5540 if (se->on_rq) { 5541 /* 5542 * Any task has to be enqueued before it get to execute on 5543 * a CPU. So account for the time it spent waiting on the 5544 * runqueue. 5545 */ 5546 update_stats_wait_end_fair(cfs_rq, se); 5547 __dequeue_entity(cfs_rq, se); 5548 update_load_avg(cfs_rq, se, UPDATE_TG); 5549 5550 if (first) 5551 set_protect_slice(cfs_rq, se); 5552 } 5553 5554 update_stats_curr_start(cfs_rq, se); 5555 WARN_ON_ONCE(cfs_rq->curr); 5556 cfs_rq->curr = se; 5557 5558 /* 5559 * Track our maximum slice length, if the CPU's load is at 5560 * least twice that of our own weight (i.e. don't track it 5561 * when there are only lesser-weight tasks around): 5562 */ 5563 if (schedstat_enabled() && 5564 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5565 struct sched_statistics *stats; 5566 5567 stats = __schedstats_from_se(se); 5568 __schedstat_set(stats->slice_max, 5569 max((u64)stats->slice_max, 5570 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5571 } 5572 5573 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5574} 5575 5576static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); 5577 5578/* 5579 * Pick the next process, keeping these things in mind, in this order: 5580 * 1) keep things fair between processes/task groups 5581 * 2) pick the "next" process, since someone really wants that to run 5582 * 3) pick the "last" process, for cache locality 5583 * 4) do not run the "skip" process, if something else is available 5584 */ 5585static struct sched_entity * 5586pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq) 5587{ 5588 struct sched_entity *se; 5589 5590 se = pick_eevdf(cfs_rq); 5591 if (se->sched_delayed) { 5592 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 5593 /* 5594 * Must not reference @se again, see __block_task(). 5595 */ 5596 return NULL; 5597 } 5598 return se; 5599} 5600 5601static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5602 5603static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5604{ 5605 /* 5606 * If still on the runqueue then deactivate_task() 5607 * was not called and update_curr() has to be done: 5608 */ 5609 if (prev->on_rq) 5610 update_curr(cfs_rq); 5611 5612 /* throttle cfs_rqs exceeding runtime */ 5613 check_cfs_rq_runtime(cfs_rq); 5614 5615 if (prev->on_rq) { 5616 update_stats_wait_start_fair(cfs_rq, prev); 5617 /* Put 'current' back into the tree. */ 5618 __enqueue_entity(cfs_rq, prev); 5619 /* in !on_rq case, update occurred at dequeue */ 5620 update_load_avg(cfs_rq, prev, 0); 5621 } 5622 WARN_ON_ONCE(cfs_rq->curr != prev); 5623 cfs_rq->curr = NULL; 5624} 5625 5626static void 5627entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5628{ 5629 /* 5630 * Update run-time statistics of the 'current'. 5631 */ 5632 update_curr(cfs_rq); 5633 5634 /* 5635 * Ensure that runnable average is periodically updated. 5636 */ 5637 update_load_avg(cfs_rq, curr, UPDATE_TG); 5638 update_cfs_group(curr); 5639 5640#ifdef CONFIG_SCHED_HRTICK 5641 /* 5642 * queued ticks are scheduled to match the slice, so don't bother 5643 * validating it and just reschedule. 5644 */ 5645 if (queued) { 5646 resched_curr_lazy(rq_of(cfs_rq)); 5647 return; 5648 } 5649#endif 5650} 5651 5652 5653/************************************************** 5654 * CFS bandwidth control machinery 5655 */ 5656 5657#ifdef CONFIG_CFS_BANDWIDTH 5658 5659#ifdef CONFIG_JUMP_LABEL 5660static struct static_key __cfs_bandwidth_used; 5661 5662static inline bool cfs_bandwidth_used(void) 5663{ 5664 return static_key_false(&__cfs_bandwidth_used); 5665} 5666 5667void cfs_bandwidth_usage_inc(void) 5668{ 5669 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5670} 5671 5672void cfs_bandwidth_usage_dec(void) 5673{ 5674 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5675} 5676#else /* !CONFIG_JUMP_LABEL: */ 5677static bool cfs_bandwidth_used(void) 5678{ 5679 return true; 5680} 5681 5682void cfs_bandwidth_usage_inc(void) {} 5683void cfs_bandwidth_usage_dec(void) {} 5684#endif /* !CONFIG_JUMP_LABEL */ 5685 5686static inline u64 sched_cfs_bandwidth_slice(void) 5687{ 5688 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5689} 5690 5691/* 5692 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5693 * directly instead of rq->clock to avoid adding additional synchronization 5694 * around rq->lock. 5695 * 5696 * requires cfs_b->lock 5697 */ 5698void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5699{ 5700 s64 runtime; 5701 5702 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5703 return; 5704 5705 cfs_b->runtime += cfs_b->quota; 5706 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5707 if (runtime > 0) { 5708 cfs_b->burst_time += runtime; 5709 cfs_b->nr_burst++; 5710 } 5711 5712 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5713 cfs_b->runtime_snap = cfs_b->runtime; 5714} 5715 5716static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5717{ 5718 return &tg->cfs_bandwidth; 5719} 5720 5721/* returns 0 on failure to allocate runtime */ 5722static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5723 struct cfs_rq *cfs_rq, u64 target_runtime) 5724{ 5725 u64 min_amount, amount = 0; 5726 5727 lockdep_assert_held(&cfs_b->lock); 5728 5729 /* note: this is a positive sum as runtime_remaining <= 0 */ 5730 min_amount = target_runtime - cfs_rq->runtime_remaining; 5731 5732 if (cfs_b->quota == RUNTIME_INF) 5733 amount = min_amount; 5734 else { 5735 start_cfs_bandwidth(cfs_b); 5736 5737 if (cfs_b->runtime > 0) { 5738 amount = min(cfs_b->runtime, min_amount); 5739 cfs_b->runtime -= amount; 5740 cfs_b->idle = 0; 5741 } 5742 } 5743 5744 cfs_rq->runtime_remaining += amount; 5745 5746 return cfs_rq->runtime_remaining > 0; 5747} 5748 5749/* returns 0 on failure to allocate runtime */ 5750static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5751{ 5752 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5753 int ret; 5754 5755 raw_spin_lock(&cfs_b->lock); 5756 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5757 raw_spin_unlock(&cfs_b->lock); 5758 5759 return ret; 5760} 5761 5762static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5763{ 5764 /* dock delta_exec before expiring quota (as it could span periods) */ 5765 cfs_rq->runtime_remaining -= delta_exec; 5766 5767 if (likely(cfs_rq->runtime_remaining > 0)) 5768 return; 5769 5770 if (cfs_rq->throttled) 5771 return; 5772 /* 5773 * if we're unable to extend our runtime we resched so that the active 5774 * hierarchy can be throttled 5775 */ 5776 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5777 resched_curr(rq_of(cfs_rq)); 5778} 5779 5780static __always_inline 5781void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5782{ 5783 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5784 return; 5785 5786 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5787} 5788 5789static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5790{ 5791 return cfs_bandwidth_used() && cfs_rq->throttled; 5792} 5793 5794static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 5795{ 5796 return cfs_bandwidth_used() && cfs_rq->pelt_clock_throttled; 5797} 5798 5799/* check whether cfs_rq, or any parent, is throttled */ 5800static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5801{ 5802 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5803} 5804 5805static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 5806{ 5807 return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]); 5808} 5809 5810static inline bool task_is_throttled(struct task_struct *p) 5811{ 5812 return cfs_bandwidth_used() && p->throttled; 5813} 5814 5815static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5816static void throttle_cfs_rq_work(struct callback_head *work) 5817{ 5818 struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work); 5819 struct sched_entity *se; 5820 struct cfs_rq *cfs_rq; 5821 struct rq *rq; 5822 5823 WARN_ON_ONCE(p != current); 5824 p->sched_throttle_work.next = &p->sched_throttle_work; 5825 5826 /* 5827 * If task is exiting, then there won't be a return to userspace, so we 5828 * don't have to bother with any of this. 5829 */ 5830 if ((p->flags & PF_EXITING)) 5831 return; 5832 5833 scoped_guard(task_rq_lock, p) { 5834 se = &p->se; 5835 cfs_rq = cfs_rq_of(se); 5836 5837 /* Raced, forget */ 5838 if (p->sched_class != &fair_sched_class) 5839 return; 5840 5841 /* 5842 * If not in limbo, then either replenish has happened or this 5843 * task got migrated out of the throttled cfs_rq, move along. 5844 */ 5845 if (!cfs_rq->throttle_count) 5846 return; 5847 rq = scope.rq; 5848 update_rq_clock(rq); 5849 WARN_ON_ONCE(p->throttled || !list_empty(&p->throttle_node)); 5850 dequeue_task_fair(rq, p, DEQUEUE_SLEEP | DEQUEUE_THROTTLE); 5851 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5852 /* 5853 * Must not set throttled before dequeue or dequeue will 5854 * mistakenly regard this task as an already throttled one. 5855 */ 5856 p->throttled = true; 5857 resched_curr(rq); 5858 } 5859} 5860 5861void init_cfs_throttle_work(struct task_struct *p) 5862{ 5863 init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work); 5864 /* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */ 5865 p->sched_throttle_work.next = &p->sched_throttle_work; 5866 INIT_LIST_HEAD(&p->throttle_node); 5867} 5868 5869/* 5870 * Task is throttled and someone wants to dequeue it again: 5871 * it could be sched/core when core needs to do things like 5872 * task affinity change, task group change, task sched class 5873 * change etc. and in these cases, DEQUEUE_SLEEP is not set; 5874 * or the task is blocked after throttled due to freezer etc. 5875 * and in these cases, DEQUEUE_SLEEP is set. 5876 */ 5877static void detach_task_cfs_rq(struct task_struct *p); 5878static void dequeue_throttled_task(struct task_struct *p, int flags) 5879{ 5880 WARN_ON_ONCE(p->se.on_rq); 5881 list_del_init(&p->throttle_node); 5882 5883 /* task blocked after throttled */ 5884 if (flags & DEQUEUE_SLEEP) { 5885 p->throttled = false; 5886 return; 5887 } 5888 5889 /* 5890 * task is migrating off its old cfs_rq, detach 5891 * the task's load from its old cfs_rq. 5892 */ 5893 if (task_on_rq_migrating(p)) 5894 detach_task_cfs_rq(p); 5895} 5896 5897static bool enqueue_throttled_task(struct task_struct *p) 5898{ 5899 struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); 5900 5901 /* @p should have gone through dequeue_throttled_task() first */ 5902 WARN_ON_ONCE(!list_empty(&p->throttle_node)); 5903 5904 /* 5905 * If the throttled task @p is enqueued to a throttled cfs_rq, 5906 * take the fast path by directly putting the task on the 5907 * target cfs_rq's limbo list. 5908 * 5909 * Do not do that when @p is current because the following race can 5910 * cause @p's group_node to be incorectly re-insterted in its rq's 5911 * cfs_tasks list, despite being throttled: 5912 * 5913 * cpuX cpuY 5914 * p ret2user 5915 * throttle_cfs_rq_work() sched_move_task(p) 5916 * LOCK task_rq_lock 5917 * dequeue_task_fair(p) 5918 * UNLOCK task_rq_lock 5919 * LOCK task_rq_lock 5920 * task_current_donor(p) == true 5921 * task_on_rq_queued(p) == true 5922 * dequeue_task(p) 5923 * put_prev_task(p) 5924 * sched_change_group() 5925 * enqueue_task(p) -> p's new cfs_rq 5926 * is throttled, go 5927 * fast path and skip 5928 * actual enqueue 5929 * set_next_task(p) 5930 * list_move(&se->group_node, &rq->cfs_tasks); // bug 5931 * schedule() 5932 * 5933 * In the above race case, @p current cfs_rq is in the same rq as 5934 * its previous cfs_rq because sched_move_task() only moves a task 5935 * to a different group from the same rq, so we can use its current 5936 * cfs_rq to derive rq and test if the task is current. 5937 */ 5938 if (throttled_hierarchy(cfs_rq) && 5939 !task_current_donor(rq_of(cfs_rq), p)) { 5940 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5941 return true; 5942 } 5943 5944 /* we can't take the fast path, do an actual enqueue*/ 5945 p->throttled = false; 5946 return false; 5947} 5948 5949static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5950static int tg_unthrottle_up(struct task_group *tg, void *data) 5951{ 5952 struct rq *rq = data; 5953 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5954 struct task_struct *p, *tmp; 5955 5956 if (--cfs_rq->throttle_count) 5957 return 0; 5958 5959 if (cfs_rq->pelt_clock_throttled) { 5960 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5961 cfs_rq->throttled_clock_pelt; 5962 cfs_rq->pelt_clock_throttled = 0; 5963 } 5964 5965 if (cfs_rq->throttled_clock_self) { 5966 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 5967 5968 cfs_rq->throttled_clock_self = 0; 5969 5970 if (WARN_ON_ONCE((s64)delta < 0)) 5971 delta = 0; 5972 5973 cfs_rq->throttled_clock_self_time += delta; 5974 } 5975 5976 /* Re-enqueue the tasks that have been throttled at this level. */ 5977 list_for_each_entry_safe(p, tmp, &cfs_rq->throttled_limbo_list, throttle_node) { 5978 list_del_init(&p->throttle_node); 5979 p->throttled = false; 5980 enqueue_task_fair(rq_of(cfs_rq), p, ENQUEUE_WAKEUP); 5981 } 5982 5983 /* Add cfs_rq with load or one or more already running entities to the list */ 5984 if (!cfs_rq_is_decayed(cfs_rq)) 5985 list_add_leaf_cfs_rq(cfs_rq); 5986 5987 return 0; 5988} 5989 5990static inline bool task_has_throttle_work(struct task_struct *p) 5991{ 5992 return p->sched_throttle_work.next != &p->sched_throttle_work; 5993} 5994 5995static inline void task_throttle_setup_work(struct task_struct *p) 5996{ 5997 if (task_has_throttle_work(p)) 5998 return; 5999 6000 /* 6001 * Kthreads and exiting tasks don't return to userspace, so adding the 6002 * work is pointless 6003 */ 6004 if ((p->flags & (PF_EXITING | PF_KTHREAD))) 6005 return; 6006 6007 task_work_add(p, &p->sched_throttle_work, TWA_RESUME); 6008} 6009 6010static void record_throttle_clock(struct cfs_rq *cfs_rq) 6011{ 6012 struct rq *rq = rq_of(cfs_rq); 6013 6014 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 6015 cfs_rq->throttled_clock = rq_clock(rq); 6016 6017 if (!cfs_rq->throttled_clock_self) 6018 cfs_rq->throttled_clock_self = rq_clock(rq); 6019} 6020 6021static int tg_throttle_down(struct task_group *tg, void *data) 6022{ 6023 struct rq *rq = data; 6024 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6025 6026 if (cfs_rq->throttle_count++) 6027 return 0; 6028 6029 /* 6030 * For cfs_rqs that still have entities enqueued, PELT clock 6031 * stop happens at dequeue time when all entities are dequeued. 6032 */ 6033 if (!cfs_rq->nr_queued) { 6034 list_del_leaf_cfs_rq(cfs_rq); 6035 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 6036 cfs_rq->pelt_clock_throttled = 1; 6037 } 6038 6039 WARN_ON_ONCE(cfs_rq->throttled_clock_self); 6040 WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_limbo_list)); 6041 return 0; 6042} 6043 6044static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 6045{ 6046 struct rq *rq = rq_of(cfs_rq); 6047 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6048 int dequeue = 1; 6049 6050 raw_spin_lock(&cfs_b->lock); 6051 /* This will start the period timer if necessary */ 6052 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 6053 /* 6054 * We have raced with bandwidth becoming available, and if we 6055 * actually throttled the timer might not unthrottle us for an 6056 * entire period. We additionally needed to make sure that any 6057 * subsequent check_cfs_rq_runtime calls agree not to throttle 6058 * us, as we may commit to do cfs put_prev+pick_next, so we ask 6059 * for 1ns of runtime rather than just check cfs_b. 6060 */ 6061 dequeue = 0; 6062 } else { 6063 list_add_tail_rcu(&cfs_rq->throttled_list, 6064 &cfs_b->throttled_cfs_rq); 6065 } 6066 raw_spin_unlock(&cfs_b->lock); 6067 6068 if (!dequeue) 6069 return false; /* Throttle no longer required. */ 6070 6071 /* freeze hierarchy runnable averages while throttled */ 6072 rcu_read_lock(); 6073 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 6074 rcu_read_unlock(); 6075 6076 /* 6077 * Note: distribution will already see us throttled via the 6078 * throttled-list. rq->lock protects completion. 6079 */ 6080 cfs_rq->throttled = 1; 6081 WARN_ON_ONCE(cfs_rq->throttled_clock); 6082 return true; 6083} 6084 6085void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 6086{ 6087 struct rq *rq = rq_of(cfs_rq); 6088 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6089 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 6090 6091 /* 6092 * It's possible we are called with runtime_remaining < 0 due to things 6093 * like async unthrottled us with a positive runtime_remaining but other 6094 * still running entities consumed those runtime before we reached here. 6095 * 6096 * We can't unthrottle this cfs_rq without any runtime remaining because 6097 * any enqueue in tg_unthrottle_up() will immediately trigger a throttle, 6098 * which is not supposed to happen on unthrottle path. 6099 */ 6100 if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0) 6101 return; 6102 6103 cfs_rq->throttled = 0; 6104 6105 update_rq_clock(rq); 6106 6107 raw_spin_lock(&cfs_b->lock); 6108 if (cfs_rq->throttled_clock) { 6109 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 6110 cfs_rq->throttled_clock = 0; 6111 } 6112 list_del_rcu(&cfs_rq->throttled_list); 6113 raw_spin_unlock(&cfs_b->lock); 6114 6115 /* update hierarchical throttle state */ 6116 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 6117 6118 if (!cfs_rq->load.weight) { 6119 if (!cfs_rq->on_list) 6120 return; 6121 /* 6122 * Nothing to run but something to decay (on_list)? 6123 * Complete the branch. 6124 */ 6125 for_each_sched_entity(se) { 6126 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 6127 break; 6128 } 6129 } 6130 6131 assert_list_leaf_cfs_rq(rq); 6132 6133 /* Determine whether we need to wake up potentially idle CPU: */ 6134 if (rq->curr == rq->idle && rq->cfs.nr_queued) 6135 resched_curr(rq); 6136} 6137 6138static void __cfsb_csd_unthrottle(void *arg) 6139{ 6140 struct cfs_rq *cursor, *tmp; 6141 struct rq *rq = arg; 6142 struct rq_flags rf; 6143 6144 rq_lock(rq, &rf); 6145 6146 /* 6147 * Iterating over the list can trigger several call to 6148 * update_rq_clock() in unthrottle_cfs_rq(). 6149 * Do it once and skip the potential next ones. 6150 */ 6151 update_rq_clock(rq); 6152 rq_clock_start_loop_update(rq); 6153 6154 /* 6155 * Since we hold rq lock we're safe from concurrent manipulation of 6156 * the CSD list. However, this RCU critical section annotates the 6157 * fact that we pair with sched_free_group_rcu(), so that we cannot 6158 * race with group being freed in the window between removing it 6159 * from the list and advancing to the next entry in the list. 6160 */ 6161 rcu_read_lock(); 6162 6163 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 6164 throttled_csd_list) { 6165 list_del_init(&cursor->throttled_csd_list); 6166 6167 if (cfs_rq_throttled(cursor)) 6168 unthrottle_cfs_rq(cursor); 6169 } 6170 6171 rcu_read_unlock(); 6172 6173 rq_clock_stop_loop_update(rq); 6174 rq_unlock(rq, &rf); 6175} 6176 6177static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6178{ 6179 struct rq *rq = rq_of(cfs_rq); 6180 bool first; 6181 6182 if (rq == this_rq()) { 6183 unthrottle_cfs_rq(cfs_rq); 6184 return; 6185 } 6186 6187 /* Already enqueued */ 6188 if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list))) 6189 return; 6190 6191 first = list_empty(&rq->cfsb_csd_list); 6192 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 6193 if (first) 6194 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 6195} 6196 6197static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6198{ 6199 lockdep_assert_rq_held(rq_of(cfs_rq)); 6200 6201 if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) || 6202 cfs_rq->runtime_remaining <= 0)) 6203 return; 6204 6205 __unthrottle_cfs_rq_async(cfs_rq); 6206} 6207 6208static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 6209{ 6210 int this_cpu = smp_processor_id(); 6211 u64 runtime, remaining = 1; 6212 bool throttled = false; 6213 struct cfs_rq *cfs_rq, *tmp; 6214 struct rq_flags rf; 6215 struct rq *rq; 6216 LIST_HEAD(local_unthrottle); 6217 6218 rcu_read_lock(); 6219 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 6220 throttled_list) { 6221 rq = rq_of(cfs_rq); 6222 6223 if (!remaining) { 6224 throttled = true; 6225 break; 6226 } 6227 6228 rq_lock_irqsave(rq, &rf); 6229 if (!cfs_rq_throttled(cfs_rq)) 6230 goto next; 6231 6232 /* Already queued for async unthrottle */ 6233 if (!list_empty(&cfs_rq->throttled_csd_list)) 6234 goto next; 6235 6236 /* By the above checks, this should never be true */ 6237 WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); 6238 6239 raw_spin_lock(&cfs_b->lock); 6240 runtime = -cfs_rq->runtime_remaining + 1; 6241 if (runtime > cfs_b->runtime) 6242 runtime = cfs_b->runtime; 6243 cfs_b->runtime -= runtime; 6244 remaining = cfs_b->runtime; 6245 raw_spin_unlock(&cfs_b->lock); 6246 6247 cfs_rq->runtime_remaining += runtime; 6248 6249 /* we check whether we're throttled above */ 6250 if (cfs_rq->runtime_remaining > 0) { 6251 if (cpu_of(rq) != this_cpu) { 6252 unthrottle_cfs_rq_async(cfs_rq); 6253 } else { 6254 /* 6255 * We currently only expect to be unthrottling 6256 * a single cfs_rq locally. 6257 */ 6258 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6259 list_add_tail(&cfs_rq->throttled_csd_list, 6260 &local_unthrottle); 6261 } 6262 } else { 6263 throttled = true; 6264 } 6265 6266next: 6267 rq_unlock_irqrestore(rq, &rf); 6268 } 6269 6270 list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, 6271 throttled_csd_list) { 6272 struct rq *rq = rq_of(cfs_rq); 6273 6274 rq_lock_irqsave(rq, &rf); 6275 6276 list_del_init(&cfs_rq->throttled_csd_list); 6277 6278 if (cfs_rq_throttled(cfs_rq)) 6279 unthrottle_cfs_rq(cfs_rq); 6280 6281 rq_unlock_irqrestore(rq, &rf); 6282 } 6283 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6284 6285 rcu_read_unlock(); 6286 6287 return throttled; 6288} 6289 6290/* 6291 * Responsible for refilling a task_group's bandwidth and unthrottling its 6292 * cfs_rqs as appropriate. If there has been no activity within the last 6293 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 6294 * used to track this state. 6295 */ 6296static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 6297{ 6298 int throttled; 6299 6300 /* no need to continue the timer with no bandwidth constraint */ 6301 if (cfs_b->quota == RUNTIME_INF) 6302 goto out_deactivate; 6303 6304 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 6305 cfs_b->nr_periods += overrun; 6306 6307 /* Refill extra burst quota even if cfs_b->idle */ 6308 __refill_cfs_bandwidth_runtime(cfs_b); 6309 6310 /* 6311 * idle depends on !throttled (for the case of a large deficit), and if 6312 * we're going inactive then everything else can be deferred 6313 */ 6314 if (cfs_b->idle && !throttled) 6315 goto out_deactivate; 6316 6317 if (!throttled) { 6318 /* mark as potentially idle for the upcoming period */ 6319 cfs_b->idle = 1; 6320 return 0; 6321 } 6322 6323 /* account preceding periods in which throttling occurred */ 6324 cfs_b->nr_throttled += overrun; 6325 6326 /* 6327 * This check is repeated as we release cfs_b->lock while we unthrottle. 6328 */ 6329 while (throttled && cfs_b->runtime > 0) { 6330 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6331 /* we can't nest cfs_b->lock while distributing bandwidth */ 6332 throttled = distribute_cfs_runtime(cfs_b); 6333 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6334 } 6335 6336 /* 6337 * While we are ensured activity in the period following an 6338 * unthrottle, this also covers the case in which the new bandwidth is 6339 * insufficient to cover the existing bandwidth deficit. (Forcing the 6340 * timer to remain active while there are any throttled entities.) 6341 */ 6342 cfs_b->idle = 0; 6343 6344 return 0; 6345 6346out_deactivate: 6347 return 1; 6348} 6349 6350/* a cfs_rq won't donate quota below this amount */ 6351static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 6352/* minimum remaining period time to redistribute slack quota */ 6353static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 6354/* how long we wait to gather additional slack before distributing */ 6355static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 6356 6357/* 6358 * Are we near the end of the current quota period? 6359 * 6360 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 6361 * hrtimer base being cleared by hrtimer_start. In the case of 6362 * migrate_hrtimers, base is never cleared, so we are fine. 6363 */ 6364static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 6365{ 6366 struct hrtimer *refresh_timer = &cfs_b->period_timer; 6367 s64 remaining; 6368 6369 /* if the call-back is running a quota refresh is already occurring */ 6370 if (hrtimer_callback_running(refresh_timer)) 6371 return 1; 6372 6373 /* is a quota refresh about to occur? */ 6374 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 6375 if (remaining < (s64)min_expire) 6376 return 1; 6377 6378 return 0; 6379} 6380 6381static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 6382{ 6383 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 6384 6385 /* if there's a quota refresh soon don't bother with slack */ 6386 if (runtime_refresh_within(cfs_b, min_left)) 6387 return; 6388 6389 /* don't push forwards an existing deferred unthrottle */ 6390 if (cfs_b->slack_started) 6391 return; 6392 cfs_b->slack_started = true; 6393 6394 hrtimer_start(&cfs_b->slack_timer, 6395 ns_to_ktime(cfs_bandwidth_slack_period), 6396 HRTIMER_MODE_REL); 6397} 6398 6399/* we know any runtime found here is valid as update_curr() precedes return */ 6400static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6401{ 6402 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6403 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 6404 6405 if (slack_runtime <= 0) 6406 return; 6407 6408 raw_spin_lock(&cfs_b->lock); 6409 if (cfs_b->quota != RUNTIME_INF) { 6410 cfs_b->runtime += slack_runtime; 6411 6412 /* we are under rq->lock, defer unthrottling using a timer */ 6413 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 6414 !list_empty(&cfs_b->throttled_cfs_rq)) 6415 start_cfs_slack_bandwidth(cfs_b); 6416 } 6417 raw_spin_unlock(&cfs_b->lock); 6418 6419 /* even if it's not valid for return we don't want to try again */ 6420 cfs_rq->runtime_remaining -= slack_runtime; 6421} 6422 6423static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6424{ 6425 if (!cfs_bandwidth_used()) 6426 return; 6427 6428 if (!cfs_rq->runtime_enabled || cfs_rq->nr_queued) 6429 return; 6430 6431 __return_cfs_rq_runtime(cfs_rq); 6432} 6433 6434/* 6435 * This is done with a timer (instead of inline with bandwidth return) since 6436 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 6437 */ 6438static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 6439{ 6440 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 6441 unsigned long flags; 6442 6443 /* confirm we're still not at a refresh boundary */ 6444 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6445 cfs_b->slack_started = false; 6446 6447 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 6448 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6449 return; 6450 } 6451 6452 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 6453 runtime = cfs_b->runtime; 6454 6455 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6456 6457 if (!runtime) 6458 return; 6459 6460 distribute_cfs_runtime(cfs_b); 6461} 6462 6463/* 6464 * When a group wakes up we want to make sure that its quota is not already 6465 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 6466 * runtime as update_curr() throttling can not trigger until it's on-rq. 6467 */ 6468static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 6469{ 6470 if (!cfs_bandwidth_used()) 6471 return; 6472 6473 /* an active group must be handled by the update_curr()->put() path */ 6474 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 6475 return; 6476 6477 /* ensure the group is not already throttled */ 6478 if (cfs_rq_throttled(cfs_rq)) 6479 return; 6480 6481 /* update runtime allocation */ 6482 account_cfs_rq_runtime(cfs_rq, 0); 6483 if (cfs_rq->runtime_remaining <= 0) 6484 throttle_cfs_rq(cfs_rq); 6485} 6486 6487static void sync_throttle(struct task_group *tg, int cpu) 6488{ 6489 struct cfs_rq *pcfs_rq, *cfs_rq; 6490 6491 if (!cfs_bandwidth_used()) 6492 return; 6493 6494 if (!tg->parent) 6495 return; 6496 6497 cfs_rq = tg->cfs_rq[cpu]; 6498 pcfs_rq = tg->parent->cfs_rq[cpu]; 6499 6500 cfs_rq->throttle_count = pcfs_rq->throttle_count; 6501 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 6502 6503 /* 6504 * It is not enough to sync the "pelt_clock_throttled" indicator 6505 * with the parent cfs_rq when the hierarchy is not queued. 6506 * Always join a throttled hierarchy with PELT clock throttled 6507 * and leaf it to the first enqueue, or distribution to 6508 * unthrottle the PELT clock. 6509 */ 6510 if (cfs_rq->throttle_count) 6511 cfs_rq->pelt_clock_throttled = 1; 6512} 6513 6514/* conditionally throttle active cfs_rq's from put_prev_entity() */ 6515static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6516{ 6517 if (!cfs_bandwidth_used()) 6518 return false; 6519 6520 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 6521 return false; 6522 6523 /* 6524 * it's possible for a throttled entity to be forced into a running 6525 * state (e.g. set_curr_task), in this case we're finished. 6526 */ 6527 if (cfs_rq_throttled(cfs_rq)) 6528 return true; 6529 6530 return throttle_cfs_rq(cfs_rq); 6531} 6532 6533static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 6534{ 6535 struct cfs_bandwidth *cfs_b = 6536 container_of(timer, struct cfs_bandwidth, slack_timer); 6537 6538 do_sched_cfs_slack_timer(cfs_b); 6539 6540 return HRTIMER_NORESTART; 6541} 6542 6543static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 6544{ 6545 struct cfs_bandwidth *cfs_b = 6546 container_of(timer, struct cfs_bandwidth, period_timer); 6547 unsigned long flags; 6548 int overrun; 6549 int idle = 0; 6550 int count = 0; 6551 6552 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6553 for (;;) { 6554 overrun = hrtimer_forward_now(timer, cfs_b->period); 6555 if (!overrun) 6556 break; 6557 6558 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 6559 6560 if (++count > 3) { 6561 u64 new, old = ktime_to_ns(cfs_b->period); 6562 6563 /* 6564 * Grow period by a factor of 2 to avoid losing precision. 6565 * Precision loss in the quota/period ratio can cause __cfs_schedulable 6566 * to fail. 6567 */ 6568 new = old * 2; 6569 if (new < max_bw_quota_period_us * NSEC_PER_USEC) { 6570 cfs_b->period = ns_to_ktime(new); 6571 cfs_b->quota *= 2; 6572 cfs_b->burst *= 2; 6573 6574 pr_warn_ratelimited( 6575 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6576 smp_processor_id(), 6577 div_u64(new, NSEC_PER_USEC), 6578 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6579 } else { 6580 pr_warn_ratelimited( 6581 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6582 smp_processor_id(), 6583 div_u64(old, NSEC_PER_USEC), 6584 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6585 } 6586 6587 /* reset count so we don't come right back in here */ 6588 count = 0; 6589 } 6590 } 6591 if (idle) 6592 cfs_b->period_active = 0; 6593 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6594 6595 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6596} 6597 6598void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 6599{ 6600 raw_spin_lock_init(&cfs_b->lock); 6601 cfs_b->runtime = 0; 6602 cfs_b->quota = RUNTIME_INF; 6603 cfs_b->period = us_to_ktime(default_bw_period_us()); 6604 cfs_b->burst = 0; 6605 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 6606 6607 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6608 hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC, 6609 HRTIMER_MODE_ABS_PINNED); 6610 6611 /* Add a random offset so that timers interleave */ 6612 hrtimer_set_expires(&cfs_b->period_timer, 6613 get_random_u32_below(cfs_b->period)); 6614 hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC, 6615 HRTIMER_MODE_REL); 6616 cfs_b->slack_started = false; 6617} 6618 6619static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6620{ 6621 cfs_rq->runtime_enabled = 0; 6622 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6623 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6624 INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list); 6625} 6626 6627void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6628{ 6629 lockdep_assert_held(&cfs_b->lock); 6630 6631 if (cfs_b->period_active) 6632 return; 6633 6634 cfs_b->period_active = 1; 6635 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6636 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6637} 6638 6639static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6640{ 6641 int __maybe_unused i; 6642 6643 /* init_cfs_bandwidth() was not called */ 6644 if (!cfs_b->throttled_cfs_rq.next) 6645 return; 6646 6647 hrtimer_cancel(&cfs_b->period_timer); 6648 hrtimer_cancel(&cfs_b->slack_timer); 6649 6650 /* 6651 * It is possible that we still have some cfs_rq's pending on a CSD 6652 * list, though this race is very rare. In order for this to occur, we 6653 * must have raced with the last task leaving the group while there 6654 * exist throttled cfs_rq(s), and the period_timer must have queued the 6655 * CSD item but the remote cpu has not yet processed it. To handle this, 6656 * we can simply flush all pending CSD work inline here. We're 6657 * guaranteed at this point that no additional cfs_rq of this group can 6658 * join a CSD list. 6659 */ 6660 for_each_possible_cpu(i) { 6661 struct rq *rq = cpu_rq(i); 6662 unsigned long flags; 6663 6664 if (list_empty(&rq->cfsb_csd_list)) 6665 continue; 6666 6667 local_irq_save(flags); 6668 __cfsb_csd_unthrottle(rq); 6669 local_irq_restore(flags); 6670 } 6671} 6672 6673/* 6674 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6675 * 6676 * The race is harmless, since modifying bandwidth settings of unhooked group 6677 * bits doesn't do much. 6678 */ 6679 6680/* cpu online callback */ 6681static void __maybe_unused update_runtime_enabled(struct rq *rq) 6682{ 6683 struct task_group *tg; 6684 6685 lockdep_assert_rq_held(rq); 6686 6687 rcu_read_lock(); 6688 list_for_each_entry_rcu(tg, &task_groups, list) { 6689 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6690 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6691 6692 raw_spin_lock(&cfs_b->lock); 6693 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6694 raw_spin_unlock(&cfs_b->lock); 6695 } 6696 rcu_read_unlock(); 6697} 6698 6699/* cpu offline callback */ 6700static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6701{ 6702 struct task_group *tg; 6703 6704 lockdep_assert_rq_held(rq); 6705 6706 // Do not unthrottle for an active CPU 6707 if (cpumask_test_cpu(cpu_of(rq), cpu_active_mask)) 6708 return; 6709 6710 /* 6711 * The rq clock has already been updated in the 6712 * set_rq_offline(), so we should skip updating 6713 * the rq clock again in unthrottle_cfs_rq(). 6714 */ 6715 rq_clock_start_loop_update(rq); 6716 6717 rcu_read_lock(); 6718 list_for_each_entry_rcu(tg, &task_groups, list) { 6719 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6720 6721 if (!cfs_rq->runtime_enabled) 6722 continue; 6723 6724 /* 6725 * Offline rq is schedulable till CPU is completely disabled 6726 * in take_cpu_down(), so we prevent new cfs throttling here. 6727 */ 6728 cfs_rq->runtime_enabled = 0; 6729 6730 if (!cfs_rq_throttled(cfs_rq)) 6731 continue; 6732 6733 /* 6734 * clock_task is not advancing so we just need to make sure 6735 * there's some valid quota amount 6736 */ 6737 cfs_rq->runtime_remaining = 1; 6738 unthrottle_cfs_rq(cfs_rq); 6739 } 6740 rcu_read_unlock(); 6741 6742 rq_clock_stop_loop_update(rq); 6743} 6744 6745bool cfs_task_bw_constrained(struct task_struct *p) 6746{ 6747 struct cfs_rq *cfs_rq = task_cfs_rq(p); 6748 6749 if (!cfs_bandwidth_used()) 6750 return false; 6751 6752 if (cfs_rq->runtime_enabled || 6753 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 6754 return true; 6755 6756 return false; 6757} 6758 6759#ifdef CONFIG_NO_HZ_FULL 6760/* called from pick_next_task_fair() */ 6761static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 6762{ 6763 int cpu = cpu_of(rq); 6764 6765 if (!cfs_bandwidth_used()) 6766 return; 6767 6768 if (!tick_nohz_full_cpu(cpu)) 6769 return; 6770 6771 if (rq->nr_running != 1) 6772 return; 6773 6774 /* 6775 * We know there is only one task runnable and we've just picked it. The 6776 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 6777 * be otherwise able to stop the tick. Just need to check if we are using 6778 * bandwidth control. 6779 */ 6780 if (cfs_task_bw_constrained(p)) 6781 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 6782} 6783#endif /* CONFIG_NO_HZ_FULL */ 6784 6785#else /* !CONFIG_CFS_BANDWIDTH: */ 6786 6787static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6788static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6789static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6790static inline void sync_throttle(struct task_group *tg, int cpu) {} 6791static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6792static void task_throttle_setup_work(struct task_struct *p) {} 6793static bool task_is_throttled(struct task_struct *p) { return false; } 6794static void dequeue_throttled_task(struct task_struct *p, int flags) {} 6795static bool enqueue_throttled_task(struct task_struct *p) { return false; } 6796static void record_throttle_clock(struct cfs_rq *cfs_rq) {} 6797 6798static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6799{ 6800 return 0; 6801} 6802 6803static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 6804{ 6805 return false; 6806} 6807 6808static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6809{ 6810 return 0; 6811} 6812 6813static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 6814{ 6815 return 0; 6816} 6817 6818#ifdef CONFIG_FAIR_GROUP_SCHED 6819void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 6820static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6821#endif 6822 6823static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6824{ 6825 return NULL; 6826} 6827static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6828static inline void update_runtime_enabled(struct rq *rq) {} 6829static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6830#ifdef CONFIG_CGROUP_SCHED 6831bool cfs_task_bw_constrained(struct task_struct *p) 6832{ 6833 return false; 6834} 6835#endif 6836#endif /* !CONFIG_CFS_BANDWIDTH */ 6837 6838#if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 6839static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 6840#endif 6841 6842/************************************************** 6843 * CFS operations on tasks: 6844 */ 6845 6846#ifdef CONFIG_SCHED_HRTICK 6847static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6848{ 6849 struct sched_entity *se = &p->se; 6850 6851 WARN_ON_ONCE(task_rq(p) != rq); 6852 6853 if (rq->cfs.h_nr_queued > 1) { 6854 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6855 u64 slice = se->slice; 6856 s64 delta = slice - ran; 6857 6858 if (delta < 0) { 6859 if (task_current_donor(rq, p)) 6860 resched_curr(rq); 6861 return; 6862 } 6863 hrtick_start(rq, delta); 6864 } 6865} 6866 6867/* 6868 * called from enqueue/dequeue and updates the hrtick when the 6869 * current task is from our class and nr_running is low enough 6870 * to matter. 6871 */ 6872static void hrtick_update(struct rq *rq) 6873{ 6874 struct task_struct *donor = rq->donor; 6875 6876 if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class) 6877 return; 6878 6879 hrtick_start_fair(rq, donor); 6880} 6881#else /* !CONFIG_SCHED_HRTICK: */ 6882static inline void 6883hrtick_start_fair(struct rq *rq, struct task_struct *p) 6884{ 6885} 6886 6887static inline void hrtick_update(struct rq *rq) 6888{ 6889} 6890#endif /* !CONFIG_SCHED_HRTICK */ 6891 6892static inline bool cpu_overutilized(int cpu) 6893{ 6894 unsigned long rq_util_min, rq_util_max; 6895 6896 if (!sched_energy_enabled()) 6897 return false; 6898 6899 rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6900 rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6901 6902 /* Return true only if the utilization doesn't fit CPU's capacity */ 6903 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6904} 6905 6906/* 6907 * overutilized value make sense only if EAS is enabled 6908 */ 6909static inline bool is_rd_overutilized(struct root_domain *rd) 6910{ 6911 return !sched_energy_enabled() || READ_ONCE(rd->overutilized); 6912} 6913 6914static inline void set_rd_overutilized(struct root_domain *rd, bool flag) 6915{ 6916 if (!sched_energy_enabled()) 6917 return; 6918 6919 WRITE_ONCE(rd->overutilized, flag); 6920 trace_sched_overutilized_tp(rd, flag); 6921} 6922 6923static inline void check_update_overutilized_status(struct rq *rq) 6924{ 6925 /* 6926 * overutilized field is used for load balancing decisions only 6927 * if energy aware scheduler is being used 6928 */ 6929 6930 if (!is_rd_overutilized(rq->rd) && cpu_overutilized(rq->cpu)) 6931 set_rd_overutilized(rq->rd, 1); 6932} 6933 6934/* Runqueue only has SCHED_IDLE tasks enqueued */ 6935static int sched_idle_rq(struct rq *rq) 6936{ 6937 return unlikely(rq->nr_running == rq->cfs.h_nr_idle && 6938 rq->nr_running); 6939} 6940 6941static int sched_idle_cpu(int cpu) 6942{ 6943 return sched_idle_rq(cpu_rq(cpu)); 6944} 6945 6946static void 6947requeue_delayed_entity(struct sched_entity *se) 6948{ 6949 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6950 6951 /* 6952 * se->sched_delayed should imply: se->on_rq == 1. 6953 * Because a delayed entity is one that is still on 6954 * the runqueue competing until elegibility. 6955 */ 6956 WARN_ON_ONCE(!se->sched_delayed); 6957 WARN_ON_ONCE(!se->on_rq); 6958 6959 if (sched_feat(DELAY_ZERO)) { 6960 update_entity_lag(cfs_rq, se); 6961 if (se->vlag > 0) { 6962 cfs_rq->nr_queued--; 6963 if (se != cfs_rq->curr) 6964 __dequeue_entity(cfs_rq, se); 6965 se->vlag = 0; 6966 place_entity(cfs_rq, se, 0); 6967 if (se != cfs_rq->curr) 6968 __enqueue_entity(cfs_rq, se); 6969 cfs_rq->nr_queued++; 6970 } 6971 } 6972 6973 update_load_avg(cfs_rq, se, 0); 6974 clear_delayed(se); 6975} 6976 6977/* 6978 * The enqueue_task method is called before nr_running is 6979 * increased. Here we update the fair scheduling stats and 6980 * then put the task into the rbtree: 6981 */ 6982static void 6983enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6984{ 6985 struct cfs_rq *cfs_rq; 6986 struct sched_entity *se = &p->se; 6987 int h_nr_idle = task_has_idle_policy(p); 6988 int h_nr_runnable = 1; 6989 int task_new = !(flags & ENQUEUE_WAKEUP); 6990 int rq_h_nr_queued = rq->cfs.h_nr_queued; 6991 u64 slice = 0; 6992 6993 if (task_is_throttled(p) && enqueue_throttled_task(p)) 6994 return; 6995 6996 /* 6997 * The code below (indirectly) updates schedutil which looks at 6998 * the cfs_rq utilization to select a frequency. 6999 * Let's add the task's estimated utilization to the cfs_rq's 7000 * estimated utilization, before we update schedutil. 7001 */ 7002 if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) 7003 util_est_enqueue(&rq->cfs, p); 7004 7005 if (flags & ENQUEUE_DELAYED) { 7006 requeue_delayed_entity(se); 7007 return; 7008 } 7009 7010 /* 7011 * If in_iowait is set, the code below may not trigger any cpufreq 7012 * utilization updates, so do it here explicitly with the IOWAIT flag 7013 * passed. 7014 */ 7015 if (p->in_iowait) 7016 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 7017 7018 if (task_new && se->sched_delayed) 7019 h_nr_runnable = 0; 7020 7021 for_each_sched_entity(se) { 7022 if (se->on_rq) { 7023 if (se->sched_delayed) 7024 requeue_delayed_entity(se); 7025 break; 7026 } 7027 cfs_rq = cfs_rq_of(se); 7028 7029 /* 7030 * Basically set the slice of group entries to the min_slice of 7031 * their respective cfs_rq. This ensures the group can service 7032 * its entities in the desired time-frame. 7033 */ 7034 if (slice) { 7035 se->slice = slice; 7036 se->custom_slice = 1; 7037 } 7038 enqueue_entity(cfs_rq, se, flags); 7039 slice = cfs_rq_min_slice(cfs_rq); 7040 7041 cfs_rq->h_nr_runnable += h_nr_runnable; 7042 cfs_rq->h_nr_queued++; 7043 cfs_rq->h_nr_idle += h_nr_idle; 7044 7045 if (cfs_rq_is_idle(cfs_rq)) 7046 h_nr_idle = 1; 7047 7048 flags = ENQUEUE_WAKEUP; 7049 } 7050 7051 for_each_sched_entity(se) { 7052 cfs_rq = cfs_rq_of(se); 7053 7054 update_load_avg(cfs_rq, se, UPDATE_TG); 7055 se_update_runnable(se); 7056 update_cfs_group(se); 7057 7058 se->slice = slice; 7059 if (se != cfs_rq->curr) 7060 min_vruntime_cb_propagate(&se->run_node, NULL); 7061 slice = cfs_rq_min_slice(cfs_rq); 7062 7063 cfs_rq->h_nr_runnable += h_nr_runnable; 7064 cfs_rq->h_nr_queued++; 7065 cfs_rq->h_nr_idle += h_nr_idle; 7066 7067 if (cfs_rq_is_idle(cfs_rq)) 7068 h_nr_idle = 1; 7069 } 7070 7071 if (!rq_h_nr_queued && rq->cfs.h_nr_queued) 7072 dl_server_start(&rq->fair_server); 7073 7074 /* At this point se is NULL and we are at root level*/ 7075 add_nr_running(rq, 1); 7076 7077 /* 7078 * Since new tasks are assigned an initial util_avg equal to 7079 * half of the spare capacity of their CPU, tiny tasks have the 7080 * ability to cross the overutilized threshold, which will 7081 * result in the load balancer ruining all the task placement 7082 * done by EAS. As a way to mitigate that effect, do not account 7083 * for the first enqueue operation of new tasks during the 7084 * overutilized flag detection. 7085 * 7086 * A better way of solving this problem would be to wait for 7087 * the PELT signals of tasks to converge before taking them 7088 * into account, but that is not straightforward to implement, 7089 * and the following generally works well enough in practice. 7090 */ 7091 if (!task_new) 7092 check_update_overutilized_status(rq); 7093 7094 assert_list_leaf_cfs_rq(rq); 7095 7096 hrtick_update(rq); 7097} 7098 7099/* 7100 * Basically dequeue_task_fair(), except it can deal with dequeue_entity() 7101 * failing half-way through and resume the dequeue later. 7102 * 7103 * Returns: 7104 * -1 - dequeue delayed 7105 * 0 - dequeue throttled 7106 * 1 - dequeue complete 7107 */ 7108static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) 7109{ 7110 bool was_sched_idle = sched_idle_rq(rq); 7111 bool task_sleep = flags & DEQUEUE_SLEEP; 7112 bool task_delayed = flags & DEQUEUE_DELAYED; 7113 bool task_throttled = flags & DEQUEUE_THROTTLE; 7114 struct task_struct *p = NULL; 7115 int h_nr_idle = 0; 7116 int h_nr_queued = 0; 7117 int h_nr_runnable = 0; 7118 struct cfs_rq *cfs_rq; 7119 u64 slice = 0; 7120 7121 if (entity_is_task(se)) { 7122 p = task_of(se); 7123 h_nr_queued = 1; 7124 h_nr_idle = task_has_idle_policy(p); 7125 if (task_sleep || task_delayed || !se->sched_delayed) 7126 h_nr_runnable = 1; 7127 } 7128 7129 for_each_sched_entity(se) { 7130 cfs_rq = cfs_rq_of(se); 7131 7132 if (!dequeue_entity(cfs_rq, se, flags)) { 7133 if (p && &p->se == se) 7134 return -1; 7135 7136 slice = cfs_rq_min_slice(cfs_rq); 7137 break; 7138 } 7139 7140 cfs_rq->h_nr_runnable -= h_nr_runnable; 7141 cfs_rq->h_nr_queued -= h_nr_queued; 7142 cfs_rq->h_nr_idle -= h_nr_idle; 7143 7144 if (cfs_rq_is_idle(cfs_rq)) 7145 h_nr_idle = h_nr_queued; 7146 7147 if (throttled_hierarchy(cfs_rq) && task_throttled) 7148 record_throttle_clock(cfs_rq); 7149 7150 /* Don't dequeue parent if it has other entities besides us */ 7151 if (cfs_rq->load.weight) { 7152 slice = cfs_rq_min_slice(cfs_rq); 7153 7154 /* Avoid re-evaluating load for this entity: */ 7155 se = parent_entity(se); 7156 /* 7157 * Bias pick_next to pick a task from this cfs_rq, as 7158 * p is sleeping when it is within its sched_slice. 7159 */ 7160 if (task_sleep && se) 7161 set_next_buddy(se); 7162 break; 7163 } 7164 flags |= DEQUEUE_SLEEP; 7165 flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); 7166 } 7167 7168 for_each_sched_entity(se) { 7169 cfs_rq = cfs_rq_of(se); 7170 7171 update_load_avg(cfs_rq, se, UPDATE_TG); 7172 se_update_runnable(se); 7173 update_cfs_group(se); 7174 7175 se->slice = slice; 7176 if (se != cfs_rq->curr) 7177 min_vruntime_cb_propagate(&se->run_node, NULL); 7178 slice = cfs_rq_min_slice(cfs_rq); 7179 7180 cfs_rq->h_nr_runnable -= h_nr_runnable; 7181 cfs_rq->h_nr_queued -= h_nr_queued; 7182 cfs_rq->h_nr_idle -= h_nr_idle; 7183 7184 if (cfs_rq_is_idle(cfs_rq)) 7185 h_nr_idle = h_nr_queued; 7186 7187 if (throttled_hierarchy(cfs_rq) && task_throttled) 7188 record_throttle_clock(cfs_rq); 7189 } 7190 7191 sub_nr_running(rq, h_nr_queued); 7192 7193 /* balance early to pull high priority tasks */ 7194 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 7195 rq->next_balance = jiffies; 7196 7197 if (p && task_delayed) { 7198 WARN_ON_ONCE(!task_sleep); 7199 WARN_ON_ONCE(p->on_rq != 1); 7200 7201 /* Fix-up what dequeue_task_fair() skipped */ 7202 hrtick_update(rq); 7203 7204 /* 7205 * Fix-up what block_task() skipped. 7206 * 7207 * Must be last, @p might not be valid after this. 7208 */ 7209 __block_task(rq, p); 7210 } 7211 7212 return 1; 7213} 7214 7215/* 7216 * The dequeue_task method is called before nr_running is 7217 * decreased. We remove the task from the rbtree and 7218 * update the fair scheduling stats: 7219 */ 7220static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 7221{ 7222 if (task_is_throttled(p)) { 7223 dequeue_throttled_task(p, flags); 7224 return true; 7225 } 7226 7227 if (!p->se.sched_delayed) 7228 util_est_dequeue(&rq->cfs, p); 7229 7230 util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP); 7231 if (dequeue_entities(rq, &p->se, flags) < 0) 7232 return false; 7233 7234 /* 7235 * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). 7236 */ 7237 7238 hrtick_update(rq); 7239 return true; 7240} 7241 7242static inline unsigned int cfs_h_nr_delayed(struct rq *rq) 7243{ 7244 return (rq->cfs.h_nr_queued - rq->cfs.h_nr_runnable); 7245} 7246 7247/* Working cpumask for: sched_balance_rq(), sched_balance_newidle(). */ 7248static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 7249static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 7250static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 7251 7252#ifdef CONFIG_NO_HZ_COMMON 7253 7254static struct { 7255 cpumask_var_t idle_cpus_mask; 7256 atomic_t nr_cpus; 7257 int has_blocked; /* Idle CPUS has blocked load */ 7258 int needs_update; /* Newly idle CPUs need their next_balance collated */ 7259 unsigned long next_balance; /* in jiffy units */ 7260 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 7261} nohz ____cacheline_aligned; 7262 7263#endif /* CONFIG_NO_HZ_COMMON */ 7264 7265static unsigned long cpu_load(struct rq *rq) 7266{ 7267 return cfs_rq_load_avg(&rq->cfs); 7268} 7269 7270/* 7271 * cpu_load_without - compute CPU load without any contributions from *p 7272 * @cpu: the CPU which load is requested 7273 * @p: the task which load should be discounted 7274 * 7275 * The load of a CPU is defined by the load of tasks currently enqueued on that 7276 * CPU as well as tasks which are currently sleeping after an execution on that 7277 * CPU. 7278 * 7279 * This method returns the load of the specified CPU by discounting the load of 7280 * the specified task, whenever the task is currently contributing to the CPU 7281 * load. 7282 */ 7283static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 7284{ 7285 struct cfs_rq *cfs_rq; 7286 unsigned int load; 7287 7288 /* Task has no contribution or is new */ 7289 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7290 return cpu_load(rq); 7291 7292 cfs_rq = &rq->cfs; 7293 load = READ_ONCE(cfs_rq->avg.load_avg); 7294 7295 /* Discount task's util from CPU's util */ 7296 lsub_positive(&load, task_h_load(p)); 7297 7298 return load; 7299} 7300 7301static unsigned long cpu_runnable(struct rq *rq) 7302{ 7303 return cfs_rq_runnable_avg(&rq->cfs); 7304} 7305 7306static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 7307{ 7308 struct cfs_rq *cfs_rq; 7309 unsigned int runnable; 7310 7311 /* Task has no contribution or is new */ 7312 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7313 return cpu_runnable(rq); 7314 7315 cfs_rq = &rq->cfs; 7316 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7317 7318 /* Discount task's runnable from CPU's runnable */ 7319 lsub_positive(&runnable, p->se.avg.runnable_avg); 7320 7321 return runnable; 7322} 7323 7324static unsigned long capacity_of(int cpu) 7325{ 7326 return cpu_rq(cpu)->cpu_capacity; 7327} 7328 7329static void record_wakee(struct task_struct *p) 7330{ 7331 /* 7332 * Only decay a single time; tasks that have less then 1 wakeup per 7333 * jiffy will not have built up many flips. 7334 */ 7335 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 7336 current->wakee_flips >>= 1; 7337 current->wakee_flip_decay_ts = jiffies; 7338 } 7339 7340 if (current->last_wakee != p) { 7341 current->last_wakee = p; 7342 current->wakee_flips++; 7343 } 7344} 7345 7346/* 7347 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 7348 * 7349 * A waker of many should wake a different task than the one last awakened 7350 * at a frequency roughly N times higher than one of its wakees. 7351 * 7352 * In order to determine whether we should let the load spread vs consolidating 7353 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 7354 * partner, and a factor of lls_size higher frequency in the other. 7355 * 7356 * With both conditions met, we can be relatively sure that the relationship is 7357 * non-monogamous, with partner count exceeding socket size. 7358 * 7359 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 7360 * whatever is irrelevant, spread criteria is apparent partner count exceeds 7361 * socket size. 7362 */ 7363static int wake_wide(struct task_struct *p) 7364{ 7365 unsigned int master = current->wakee_flips; 7366 unsigned int slave = p->wakee_flips; 7367 int factor = __this_cpu_read(sd_llc_size); 7368 7369 if (master < slave) 7370 swap(master, slave); 7371 if (slave < factor || master < slave * factor) 7372 return 0; 7373 return 1; 7374} 7375 7376/* 7377 * The purpose of wake_affine() is to quickly determine on which CPU we can run 7378 * soonest. For the purpose of speed we only consider the waking and previous 7379 * CPU. 7380 * 7381 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 7382 * cache-affine and is (or will be) idle. 7383 * 7384 * wake_affine_weight() - considers the weight to reflect the average 7385 * scheduling latency of the CPUs. This seems to work 7386 * for the overloaded case. 7387 */ 7388static int 7389wake_affine_idle(int this_cpu, int prev_cpu, int sync) 7390{ 7391 /* 7392 * If this_cpu is idle, it implies the wakeup is from interrupt 7393 * context. Only allow the move if cache is shared. Otherwise an 7394 * interrupt intensive workload could force all tasks onto one 7395 * node depending on the IO topology or IRQ affinity settings. 7396 * 7397 * If the prev_cpu is idle and cache affine then avoid a migration. 7398 * There is no guarantee that the cache hot data from an interrupt 7399 * is more important than cache hot data on the prev_cpu and from 7400 * a cpufreq perspective, it's better to have higher utilisation 7401 * on one CPU. 7402 */ 7403 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 7404 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 7405 7406 if (sync) { 7407 struct rq *rq = cpu_rq(this_cpu); 7408 7409 if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) 7410 return this_cpu; 7411 } 7412 7413 if (available_idle_cpu(prev_cpu)) 7414 return prev_cpu; 7415 7416 return nr_cpumask_bits; 7417} 7418 7419static int 7420wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 7421 int this_cpu, int prev_cpu, int sync) 7422{ 7423 s64 this_eff_load, prev_eff_load; 7424 unsigned long task_load; 7425 7426 this_eff_load = cpu_load(cpu_rq(this_cpu)); 7427 7428 if (sync) { 7429 unsigned long current_load = task_h_load(current); 7430 7431 if (current_load > this_eff_load) 7432 return this_cpu; 7433 7434 this_eff_load -= current_load; 7435 } 7436 7437 task_load = task_h_load(p); 7438 7439 this_eff_load += task_load; 7440 if (sched_feat(WA_BIAS)) 7441 this_eff_load *= 100; 7442 this_eff_load *= capacity_of(prev_cpu); 7443 7444 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 7445 prev_eff_load -= task_load; 7446 if (sched_feat(WA_BIAS)) 7447 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 7448 prev_eff_load *= capacity_of(this_cpu); 7449 7450 /* 7451 * If sync, adjust the weight of prev_eff_load such that if 7452 * prev_eff == this_eff that select_idle_sibling() will consider 7453 * stacking the wakee on top of the waker if no other CPU is 7454 * idle. 7455 */ 7456 if (sync) 7457 prev_eff_load += 1; 7458 7459 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 7460} 7461 7462static int wake_affine(struct sched_domain *sd, struct task_struct *p, 7463 int this_cpu, int prev_cpu, int sync) 7464{ 7465 int target = nr_cpumask_bits; 7466 7467 if (sched_feat(WA_IDLE)) 7468 target = wake_affine_idle(this_cpu, prev_cpu, sync); 7469 7470 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 7471 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 7472 7473 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 7474 if (target != this_cpu) 7475 return prev_cpu; 7476 7477 schedstat_inc(sd->ttwu_move_affine); 7478 schedstat_inc(p->stats.nr_wakeups_affine); 7479 return target; 7480} 7481 7482static struct sched_group * 7483sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 7484 7485/* 7486 * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group. 7487 */ 7488static int 7489sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 7490{ 7491 unsigned long load, min_load = ULONG_MAX; 7492 unsigned int min_exit_latency = UINT_MAX; 7493 u64 latest_idle_timestamp = 0; 7494 int least_loaded_cpu = this_cpu; 7495 int shallowest_idle_cpu = -1; 7496 int i; 7497 7498 /* Check if we have any choice: */ 7499 if (group->group_weight == 1) 7500 return cpumask_first(sched_group_span(group)); 7501 7502 /* Traverse only the allowed CPUs */ 7503 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 7504 struct rq *rq = cpu_rq(i); 7505 7506 if (!sched_core_cookie_match(rq, p)) 7507 continue; 7508 7509 if (sched_idle_cpu(i)) 7510 return i; 7511 7512 if (available_idle_cpu(i)) { 7513 struct cpuidle_state *idle = idle_get_state(rq); 7514 if (idle && idle->exit_latency < min_exit_latency) { 7515 /* 7516 * We give priority to a CPU whose idle state 7517 * has the smallest exit latency irrespective 7518 * of any idle timestamp. 7519 */ 7520 min_exit_latency = idle->exit_latency; 7521 latest_idle_timestamp = rq->idle_stamp; 7522 shallowest_idle_cpu = i; 7523 } else if ((!idle || idle->exit_latency == min_exit_latency) && 7524 rq->idle_stamp > latest_idle_timestamp) { 7525 /* 7526 * If equal or no active idle state, then 7527 * the most recently idled CPU might have 7528 * a warmer cache. 7529 */ 7530 latest_idle_timestamp = rq->idle_stamp; 7531 shallowest_idle_cpu = i; 7532 } 7533 } else if (shallowest_idle_cpu == -1) { 7534 load = cpu_load(cpu_rq(i)); 7535 if (load < min_load) { 7536 min_load = load; 7537 least_loaded_cpu = i; 7538 } 7539 } 7540 } 7541 7542 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 7543} 7544 7545static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p, 7546 int cpu, int prev_cpu, int sd_flag) 7547{ 7548 int new_cpu = cpu; 7549 7550 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 7551 return prev_cpu; 7552 7553 /* 7554 * We need task's util for cpu_util_without, sync it up to 7555 * prev_cpu's last_update_time. 7556 */ 7557 if (!(sd_flag & SD_BALANCE_FORK)) 7558 sync_entity_load_avg(&p->se); 7559 7560 while (sd) { 7561 struct sched_group *group; 7562 struct sched_domain *tmp; 7563 int weight; 7564 7565 if (!(sd->flags & sd_flag)) { 7566 sd = sd->child; 7567 continue; 7568 } 7569 7570 group = sched_balance_find_dst_group(sd, p, cpu); 7571 if (!group) { 7572 sd = sd->child; 7573 continue; 7574 } 7575 7576 new_cpu = sched_balance_find_dst_group_cpu(group, p, cpu); 7577 if (new_cpu == cpu) { 7578 /* Now try balancing at a lower domain level of 'cpu': */ 7579 sd = sd->child; 7580 continue; 7581 } 7582 7583 /* Now try balancing at a lower domain level of 'new_cpu': */ 7584 cpu = new_cpu; 7585 weight = sd->span_weight; 7586 sd = NULL; 7587 for_each_domain(cpu, tmp) { 7588 if (weight <= tmp->span_weight) 7589 break; 7590 if (tmp->flags & sd_flag) 7591 sd = tmp; 7592 } 7593 } 7594 7595 return new_cpu; 7596} 7597 7598static inline int __select_idle_cpu(int cpu, struct task_struct *p) 7599{ 7600 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 7601 sched_cpu_cookie_match(cpu_rq(cpu), p)) 7602 return cpu; 7603 7604 return -1; 7605} 7606 7607#ifdef CONFIG_SCHED_SMT 7608DEFINE_STATIC_KEY_FALSE(sched_smt_present); 7609EXPORT_SYMBOL_GPL(sched_smt_present); 7610 7611static inline void set_idle_cores(int cpu, int val) 7612{ 7613 struct sched_domain_shared *sds; 7614 7615 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7616 if (sds) 7617 WRITE_ONCE(sds->has_idle_cores, val); 7618} 7619 7620static inline bool test_idle_cores(int cpu) 7621{ 7622 struct sched_domain_shared *sds; 7623 7624 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7625 if (sds) 7626 return READ_ONCE(sds->has_idle_cores); 7627 7628 return false; 7629} 7630 7631/* 7632 * Scans the local SMT mask to see if the entire core is idle, and records this 7633 * information in sd_llc_shared->has_idle_cores. 7634 * 7635 * Since SMT siblings share all cache levels, inspecting this limited remote 7636 * state should be fairly cheap. 7637 */ 7638void __update_idle_core(struct rq *rq) 7639{ 7640 int core = cpu_of(rq); 7641 int cpu; 7642 7643 rcu_read_lock(); 7644 if (test_idle_cores(core)) 7645 goto unlock; 7646 7647 for_each_cpu(cpu, cpu_smt_mask(core)) { 7648 if (cpu == core) 7649 continue; 7650 7651 if (!available_idle_cpu(cpu)) 7652 goto unlock; 7653 } 7654 7655 set_idle_cores(core, 1); 7656unlock: 7657 rcu_read_unlock(); 7658} 7659 7660/* 7661 * Scan the entire LLC domain for idle cores; this dynamically switches off if 7662 * there are no idle cores left in the system; tracked through 7663 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 7664 */ 7665static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7666{ 7667 bool idle = true; 7668 int cpu; 7669 7670 for_each_cpu(cpu, cpu_smt_mask(core)) { 7671 if (!available_idle_cpu(cpu)) { 7672 idle = false; 7673 if (*idle_cpu == -1) { 7674 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpus)) { 7675 *idle_cpu = cpu; 7676 break; 7677 } 7678 continue; 7679 } 7680 break; 7681 } 7682 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus)) 7683 *idle_cpu = cpu; 7684 } 7685 7686 if (idle) 7687 return core; 7688 7689 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 7690 return -1; 7691} 7692 7693/* 7694 * Scan the local SMT mask for idle CPUs. 7695 */ 7696static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7697{ 7698 int cpu; 7699 7700 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 7701 if (cpu == target) 7702 continue; 7703 /* 7704 * Check if the CPU is in the LLC scheduling domain of @target. 7705 * Due to isolcpus, there is no guarantee that all the siblings are in the domain. 7706 */ 7707 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7708 continue; 7709 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 7710 return cpu; 7711 } 7712 7713 return -1; 7714} 7715 7716#else /* !CONFIG_SCHED_SMT: */ 7717 7718static inline void set_idle_cores(int cpu, int val) 7719{ 7720} 7721 7722static inline bool test_idle_cores(int cpu) 7723{ 7724 return false; 7725} 7726 7727static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7728{ 7729 return __select_idle_cpu(core, p); 7730} 7731 7732static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7733{ 7734 return -1; 7735} 7736 7737#endif /* !CONFIG_SCHED_SMT */ 7738 7739/* 7740 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 7741 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 7742 * average idle time for this rq (as found in rq->avg_idle). 7743 */ 7744static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 7745{ 7746 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7747 int i, cpu, idle_cpu = -1, nr = INT_MAX; 7748 struct sched_domain_shared *sd_share; 7749 7750 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7751 7752 if (sched_feat(SIS_UTIL)) { 7753 sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); 7754 if (sd_share) { 7755 /* because !--nr is the condition to stop scan */ 7756 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7757 /* overloaded LLC is unlikely to have idle cpu/core */ 7758 if (nr == 1) 7759 return -1; 7760 } 7761 } 7762 7763 if (static_branch_unlikely(&sched_cluster_active)) { 7764 struct sched_group *sg = sd->groups; 7765 7766 if (sg->flags & SD_CLUSTER) { 7767 for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) { 7768 if (!cpumask_test_cpu(cpu, cpus)) 7769 continue; 7770 7771 if (has_idle_core) { 7772 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7773 if ((unsigned int)i < nr_cpumask_bits) 7774 return i; 7775 } else { 7776 if (--nr <= 0) 7777 return -1; 7778 idle_cpu = __select_idle_cpu(cpu, p); 7779 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7780 return idle_cpu; 7781 } 7782 } 7783 cpumask_andnot(cpus, cpus, sched_group_span(sg)); 7784 } 7785 } 7786 7787 for_each_cpu_wrap(cpu, cpus, target + 1) { 7788 if (has_idle_core) { 7789 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7790 if ((unsigned int)i < nr_cpumask_bits) 7791 return i; 7792 7793 } else { 7794 if (--nr <= 0) 7795 return -1; 7796 idle_cpu = __select_idle_cpu(cpu, p); 7797 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7798 break; 7799 } 7800 } 7801 7802 if (has_idle_core) 7803 set_idle_cores(target, false); 7804 7805 return idle_cpu; 7806} 7807 7808/* 7809 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7810 * the task fits. If no CPU is big enough, but there are idle ones, try to 7811 * maximize capacity. 7812 */ 7813static int 7814select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7815{ 7816 unsigned long task_util, util_min, util_max, best_cap = 0; 7817 int fits, best_fits = 0; 7818 int cpu, best_cpu = -1; 7819 struct cpumask *cpus; 7820 7821 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7822 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7823 7824 task_util = task_util_est(p); 7825 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7826 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7827 7828 for_each_cpu_wrap(cpu, cpus, target) { 7829 unsigned long cpu_cap = capacity_of(cpu); 7830 7831 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7832 continue; 7833 7834 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7835 7836 /* This CPU fits with all requirements */ 7837 if (fits > 0) 7838 return cpu; 7839 /* 7840 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7841 * Look for the CPU with best capacity. 7842 */ 7843 else if (fits < 0) 7844 cpu_cap = get_actual_cpu_capacity(cpu); 7845 7846 /* 7847 * First, select CPU which fits better (-1 being better than 0). 7848 * Then, select the one with best capacity at same level. 7849 */ 7850 if ((fits < best_fits) || 7851 ((fits == best_fits) && (cpu_cap > best_cap))) { 7852 best_cap = cpu_cap; 7853 best_cpu = cpu; 7854 best_fits = fits; 7855 } 7856 } 7857 7858 return best_cpu; 7859} 7860 7861static inline bool asym_fits_cpu(unsigned long util, 7862 unsigned long util_min, 7863 unsigned long util_max, 7864 int cpu) 7865{ 7866 if (sched_asym_cpucap_active()) 7867 /* 7868 * Return true only if the cpu fully fits the task requirements 7869 * which include the utilization and the performance hints. 7870 */ 7871 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7872 7873 return true; 7874} 7875 7876/* 7877 * Try and locate an idle core/thread in the LLC cache domain. 7878 */ 7879static int select_idle_sibling(struct task_struct *p, int prev, int target) 7880{ 7881 bool has_idle_core = false; 7882 struct sched_domain *sd; 7883 unsigned long task_util, util_min, util_max; 7884 int i, recent_used_cpu, prev_aff = -1; 7885 7886 /* 7887 * On asymmetric system, update task utilization because we will check 7888 * that the task fits with CPU's capacity. 7889 */ 7890 if (sched_asym_cpucap_active()) { 7891 sync_entity_load_avg(&p->se); 7892 task_util = task_util_est(p); 7893 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7894 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7895 } 7896 7897 /* 7898 * per-cpu select_rq_mask usage 7899 */ 7900 lockdep_assert_irqs_disabled(); 7901 7902 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7903 asym_fits_cpu(task_util, util_min, util_max, target)) 7904 return target; 7905 7906 /* 7907 * If the previous CPU is cache affine and idle, don't be stupid: 7908 */ 7909 if (prev != target && cpus_share_cache(prev, target) && 7910 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7911 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7912 7913 if (!static_branch_unlikely(&sched_cluster_active) || 7914 cpus_share_resources(prev, target)) 7915 return prev; 7916 7917 prev_aff = prev; 7918 } 7919 7920 /* 7921 * Allow a per-cpu kthread to stack with the wakee if the 7922 * kworker thread and the tasks previous CPUs are the same. 7923 * The assumption is that the wakee queued work for the 7924 * per-cpu kthread that is now complete and the wakeup is 7925 * essentially a sync wakeup. An obvious example of this 7926 * pattern is IO completions. 7927 */ 7928 if (is_per_cpu_kthread(current) && 7929 in_task() && 7930 prev == smp_processor_id() && 7931 this_rq()->nr_running <= 1 && 7932 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7933 return prev; 7934 } 7935 7936 /* Check a recently used CPU as a potential idle candidate: */ 7937 recent_used_cpu = p->recent_used_cpu; 7938 p->recent_used_cpu = prev; 7939 if (recent_used_cpu != prev && 7940 recent_used_cpu != target && 7941 cpus_share_cache(recent_used_cpu, target) && 7942 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7943 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7944 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7945 7946 if (!static_branch_unlikely(&sched_cluster_active) || 7947 cpus_share_resources(recent_used_cpu, target)) 7948 return recent_used_cpu; 7949 7950 } else { 7951 recent_used_cpu = -1; 7952 } 7953 7954 /* 7955 * For asymmetric CPU capacity systems, our domain of interest is 7956 * sd_asym_cpucapacity rather than sd_llc. 7957 */ 7958 if (sched_asym_cpucap_active()) { 7959 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 7960 /* 7961 * On an asymmetric CPU capacity system where an exclusive 7962 * cpuset defines a symmetric island (i.e. one unique 7963 * capacity_orig value through the cpuset), the key will be set 7964 * but the CPUs within that cpuset will not have a domain with 7965 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7966 * capacity path. 7967 */ 7968 if (sd) { 7969 i = select_idle_capacity(p, sd, target); 7970 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7971 } 7972 } 7973 7974 sd = rcu_dereference(per_cpu(sd_llc, target)); 7975 if (!sd) 7976 return target; 7977 7978 if (sched_smt_active()) { 7979 has_idle_core = test_idle_cores(target); 7980 7981 if (!has_idle_core && cpus_share_cache(prev, target)) { 7982 i = select_idle_smt(p, sd, prev); 7983 if ((unsigned int)i < nr_cpumask_bits) 7984 return i; 7985 } 7986 } 7987 7988 i = select_idle_cpu(p, sd, has_idle_core, target); 7989 if ((unsigned)i < nr_cpumask_bits) 7990 return i; 7991 7992 /* 7993 * For cluster machines which have lower sharing cache like L2 or 7994 * LLC Tag, we tend to find an idle CPU in the target's cluster 7995 * first. But prev_cpu or recent_used_cpu may also be a good candidate, 7996 * use them if possible when no idle CPU found in select_idle_cpu(). 7997 */ 7998 if ((unsigned int)prev_aff < nr_cpumask_bits) 7999 return prev_aff; 8000 if ((unsigned int)recent_used_cpu < nr_cpumask_bits) 8001 return recent_used_cpu; 8002 8003 return target; 8004} 8005 8006/** 8007 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 8008 * @cpu: the CPU to get the utilization for 8009 * @p: task for which the CPU utilization should be predicted or NULL 8010 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 8011 * @boost: 1 to enable boosting, otherwise 0 8012 * 8013 * The unit of the return value must be the same as the one of CPU capacity 8014 * so that CPU utilization can be compared with CPU capacity. 8015 * 8016 * CPU utilization is the sum of running time of runnable tasks plus the 8017 * recent utilization of currently non-runnable tasks on that CPU. 8018 * It represents the amount of CPU capacity currently used by CFS tasks in 8019 * the range [0..max CPU capacity] with max CPU capacity being the CPU 8020 * capacity at f_max. 8021 * 8022 * The estimated CPU utilization is defined as the maximum between CPU 8023 * utilization and sum of the estimated utilization of the currently 8024 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 8025 * previously-executed tasks, which helps better deduce how busy a CPU will 8026 * be when a long-sleeping task wakes up. The contribution to CPU utilization 8027 * of such a task would be significantly decayed at this point of time. 8028 * 8029 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 8030 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 8031 * utilization. Boosting is implemented in cpu_util() so that internal 8032 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 8033 * latter via cpu_util_cfs_boost(). 8034 * 8035 * CPU utilization can be higher than the current CPU capacity 8036 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 8037 * of rounding errors as well as task migrations or wakeups of new tasks. 8038 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 8039 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 8040 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 8041 * capacity. CPU utilization is allowed to overshoot current CPU capacity 8042 * though since this is useful for predicting the CPU capacity required 8043 * after task migrations (scheduler-driven DVFS). 8044 * 8045 * Return: (Boosted) (estimated) utilization for the specified CPU. 8046 */ 8047static unsigned long 8048cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 8049{ 8050 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 8051 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 8052 unsigned long runnable; 8053 8054 if (boost) { 8055 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 8056 util = max(util, runnable); 8057 } 8058 8059 /* 8060 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 8061 * contribution. If @p migrates from another CPU to @cpu add its 8062 * contribution. In all the other cases @cpu is not impacted by the 8063 * migration so its util_avg is already correct. 8064 */ 8065 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 8066 lsub_positive(&util, task_util(p)); 8067 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 8068 util += task_util(p); 8069 8070 if (sched_feat(UTIL_EST)) { 8071 unsigned long util_est; 8072 8073 util_est = READ_ONCE(cfs_rq->avg.util_est); 8074 8075 /* 8076 * During wake-up @p isn't enqueued yet and doesn't contribute 8077 * to any cpu_rq(cpu)->cfs.avg.util_est. 8078 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 8079 * has been enqueued. 8080 * 8081 * During exec (@dst_cpu = -1) @p is enqueued and does 8082 * contribute to cpu_rq(cpu)->cfs.util_est. 8083 * Remove it to "simulate" cpu_util without @p's contribution. 8084 * 8085 * Despite the task_on_rq_queued(@p) check there is still a 8086 * small window for a possible race when an exec 8087 * select_task_rq_fair() races with LB's detach_task(). 8088 * 8089 * detach_task() 8090 * deactivate_task() 8091 * p->on_rq = TASK_ON_RQ_MIGRATING; 8092 * -------------------------------- A 8093 * dequeue_task() \ 8094 * dequeue_task_fair() + Race Time 8095 * util_est_dequeue() / 8096 * -------------------------------- B 8097 * 8098 * The additional check "current == p" is required to further 8099 * reduce the race window. 8100 */ 8101 if (dst_cpu == cpu) 8102 util_est += _task_util_est(p); 8103 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 8104 lsub_positive(&util_est, _task_util_est(p)); 8105 8106 util = max(util, util_est); 8107 } 8108 8109 return min(util, arch_scale_cpu_capacity(cpu)); 8110} 8111 8112unsigned long cpu_util_cfs(int cpu) 8113{ 8114 return cpu_util(cpu, NULL, -1, 0); 8115} 8116 8117unsigned long cpu_util_cfs_boost(int cpu) 8118{ 8119 return cpu_util(cpu, NULL, -1, 1); 8120} 8121 8122/* 8123 * cpu_util_without: compute cpu utilization without any contributions from *p 8124 * @cpu: the CPU which utilization is requested 8125 * @p: the task which utilization should be discounted 8126 * 8127 * The utilization of a CPU is defined by the utilization of tasks currently 8128 * enqueued on that CPU as well as tasks which are currently sleeping after an 8129 * execution on that CPU. 8130 * 8131 * This method returns the utilization of the specified CPU by discounting the 8132 * utilization of the specified task, whenever the task is currently 8133 * contributing to the CPU utilization. 8134 */ 8135static unsigned long cpu_util_without(int cpu, struct task_struct *p) 8136{ 8137 /* Task has no contribution or is new */ 8138 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8139 p = NULL; 8140 8141 return cpu_util(cpu, p, -1, 0); 8142} 8143 8144/* 8145 * This function computes an effective utilization for the given CPU, to be 8146 * used for frequency selection given the linear relation: f = u * f_max. 8147 * 8148 * The scheduler tracks the following metrics: 8149 * 8150 * cpu_util_{cfs,rt,dl,irq}() 8151 * cpu_bw_dl() 8152 * 8153 * Where the cfs,rt and dl util numbers are tracked with the same metric and 8154 * synchronized windows and are thus directly comparable. 8155 * 8156 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 8157 * which excludes things like IRQ and steal-time. These latter are then accrued 8158 * in the IRQ utilization. 8159 * 8160 * The DL bandwidth number OTOH is not a measured metric but a value computed 8161 * based on the task model parameters and gives the minimal utilization 8162 * required to meet deadlines. 8163 */ 8164unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 8165 unsigned long *min, 8166 unsigned long *max) 8167{ 8168 unsigned long util, irq, scale; 8169 struct rq *rq = cpu_rq(cpu); 8170 8171 scale = arch_scale_cpu_capacity(cpu); 8172 8173 /* 8174 * Early check to see if IRQ/steal time saturates the CPU, can be 8175 * because of inaccuracies in how we track these -- see 8176 * update_irq_load_avg(). 8177 */ 8178 irq = cpu_util_irq(rq); 8179 if (unlikely(irq >= scale)) { 8180 if (min) 8181 *min = scale; 8182 if (max) 8183 *max = scale; 8184 return scale; 8185 } 8186 8187 if (min) { 8188 /* 8189 * The minimum utilization returns the highest level between: 8190 * - the computed DL bandwidth needed with the IRQ pressure which 8191 * steals time to the deadline task. 8192 * - The minimum performance requirement for CFS and/or RT. 8193 */ 8194 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); 8195 8196 /* 8197 * When an RT task is runnable and uclamp is not used, we must 8198 * ensure that the task will run at maximum compute capacity. 8199 */ 8200 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) 8201 *min = max(*min, scale); 8202 } 8203 8204 /* 8205 * Because the time spend on RT/DL tasks is visible as 'lost' time to 8206 * CFS tasks and we use the same metric to track the effective 8207 * utilization (PELT windows are synchronized) we can directly add them 8208 * to obtain the CPU's actual utilization. 8209 */ 8210 util = util_cfs + cpu_util_rt(rq); 8211 util += cpu_util_dl(rq); 8212 8213 /* 8214 * The maximum hint is a soft bandwidth requirement, which can be lower 8215 * than the actual utilization because of uclamp_max requirements. 8216 */ 8217 if (max) 8218 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); 8219 8220 if (util >= scale) 8221 return scale; 8222 8223 /* 8224 * There is still idle time; further improve the number by using the 8225 * IRQ metric. Because IRQ/steal time is hidden from the task clock we 8226 * need to scale the task numbers: 8227 * 8228 * max - irq 8229 * U' = irq + --------- * U 8230 * max 8231 */ 8232 util = scale_irq_capacity(util, irq, scale); 8233 util += irq; 8234 8235 return min(scale, util); 8236} 8237 8238unsigned long sched_cpu_util(int cpu) 8239{ 8240 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); 8241} 8242 8243/* 8244 * energy_env - Utilization landscape for energy estimation. 8245 * @task_busy_time: Utilization contribution by the task for which we test the 8246 * placement. Given by eenv_task_busy_time(). 8247 * @pd_busy_time: Utilization of the whole perf domain without the task 8248 * contribution. Given by eenv_pd_busy_time(). 8249 * @cpu_cap: Maximum CPU capacity for the perf domain. 8250 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 8251 */ 8252struct energy_env { 8253 unsigned long task_busy_time; 8254 unsigned long pd_busy_time; 8255 unsigned long cpu_cap; 8256 unsigned long pd_cap; 8257}; 8258 8259/* 8260 * Compute the task busy time for compute_energy(). This time cannot be 8261 * injected directly into effective_cpu_util() because of the IRQ scaling. 8262 * The latter only makes sense with the most recent CPUs where the task has 8263 * run. 8264 */ 8265static inline void eenv_task_busy_time(struct energy_env *eenv, 8266 struct task_struct *p, int prev_cpu) 8267{ 8268 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 8269 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 8270 8271 if (unlikely(irq >= max_cap)) 8272 busy_time = max_cap; 8273 else 8274 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 8275 8276 eenv->task_busy_time = busy_time; 8277} 8278 8279/* 8280 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 8281 * utilization for each @pd_cpus, it however doesn't take into account 8282 * clamping since the ratio (utilization / cpu_capacity) is already enough to 8283 * scale the EM reported power consumption at the (eventually clamped) 8284 * cpu_capacity. 8285 * 8286 * The contribution of the task @p for which we want to estimate the 8287 * energy cost is removed (by cpu_util()) and must be calculated 8288 * separately (see eenv_task_busy_time). This ensures: 8289 * 8290 * - A stable PD utilization, no matter which CPU of that PD we want to place 8291 * the task on. 8292 * 8293 * - A fair comparison between CPUs as the task contribution (task_util()) 8294 * will always be the same no matter which CPU utilization we rely on 8295 * (util_avg or util_est). 8296 * 8297 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 8298 * exceed @eenv->pd_cap. 8299 */ 8300static inline void eenv_pd_busy_time(struct energy_env *eenv, 8301 struct cpumask *pd_cpus, 8302 struct task_struct *p) 8303{ 8304 unsigned long busy_time = 0; 8305 int cpu; 8306 8307 for_each_cpu(cpu, pd_cpus) { 8308 unsigned long util = cpu_util(cpu, p, -1, 0); 8309 8310 busy_time += effective_cpu_util(cpu, util, NULL, NULL); 8311 } 8312 8313 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 8314} 8315 8316/* 8317 * Compute the maximum utilization for compute_energy() when the task @p 8318 * is placed on the cpu @dst_cpu. 8319 * 8320 * Returns the maximum utilization among @eenv->cpus. This utilization can't 8321 * exceed @eenv->cpu_cap. 8322 */ 8323static inline unsigned long 8324eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 8325 struct task_struct *p, int dst_cpu) 8326{ 8327 unsigned long max_util = 0; 8328 int cpu; 8329 8330 for_each_cpu(cpu, pd_cpus) { 8331 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 8332 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 8333 unsigned long eff_util, min, max; 8334 8335 /* 8336 * Performance domain frequency: utilization clamping 8337 * must be considered since it affects the selection 8338 * of the performance domain frequency. 8339 * NOTE: in case RT tasks are running, by default the min 8340 * utilization can be max OPP. 8341 */ 8342 eff_util = effective_cpu_util(cpu, util, &min, &max); 8343 8344 /* Task's uclamp can modify min and max value */ 8345 if (tsk && uclamp_is_used()) { 8346 min = max(min, uclamp_eff_value(p, UCLAMP_MIN)); 8347 8348 /* 8349 * If there is no active max uclamp constraint, 8350 * directly use task's one, otherwise keep max. 8351 */ 8352 if (uclamp_rq_is_idle(cpu_rq(cpu))) 8353 max = uclamp_eff_value(p, UCLAMP_MAX); 8354 else 8355 max = max(max, uclamp_eff_value(p, UCLAMP_MAX)); 8356 } 8357 8358 eff_util = sugov_effective_cpu_perf(cpu, eff_util, min, max); 8359 max_util = max(max_util, eff_util); 8360 } 8361 8362 return min(max_util, eenv->cpu_cap); 8363} 8364 8365/* 8366 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 8367 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 8368 * contribution is ignored. 8369 */ 8370static inline unsigned long 8371compute_energy(struct energy_env *eenv, struct perf_domain *pd, 8372 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 8373{ 8374 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 8375 unsigned long busy_time = eenv->pd_busy_time; 8376 unsigned long energy; 8377 8378 if (dst_cpu >= 0) 8379 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 8380 8381 energy = em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 8382 8383 trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time); 8384 8385 return energy; 8386} 8387 8388/* 8389 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 8390 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 8391 * spare capacity in each performance domain and uses it as a potential 8392 * candidate to execute the task. Then, it uses the Energy Model to figure 8393 * out which of the CPU candidates is the most energy-efficient. 8394 * 8395 * The rationale for this heuristic is as follows. In a performance domain, 8396 * all the most energy efficient CPU candidates (according to the Energy 8397 * Model) are those for which we'll request a low frequency. When there are 8398 * several CPUs for which the frequency request will be the same, we don't 8399 * have enough data to break the tie between them, because the Energy Model 8400 * only includes active power costs. With this model, if we assume that 8401 * frequency requests follow utilization (e.g. using schedutil), the CPU with 8402 * the maximum spare capacity in a performance domain is guaranteed to be among 8403 * the best candidates of the performance domain. 8404 * 8405 * In practice, it could be preferable from an energy standpoint to pack 8406 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 8407 * but that could also hurt our chances to go cluster idle, and we have no 8408 * ways to tell with the current Energy Model if this is actually a good 8409 * idea or not. So, find_energy_efficient_cpu() basically favors 8410 * cluster-packing, and spreading inside a cluster. That should at least be 8411 * a good thing for latency, and this is consistent with the idea that most 8412 * of the energy savings of EAS come from the asymmetry of the system, and 8413 * not so much from breaking the tie between identical CPUs. That's also the 8414 * reason why EAS is enabled in the topology code only for systems where 8415 * SD_ASYM_CPUCAPACITY is set. 8416 * 8417 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 8418 * they don't have any useful utilization data yet and it's not possible to 8419 * forecast their impact on energy consumption. Consequently, they will be 8420 * placed by sched_balance_find_dst_cpu() on the least loaded CPU, which might turn out 8421 * to be energy-inefficient in some use-cases. The alternative would be to 8422 * bias new tasks towards specific types of CPUs first, or to try to infer 8423 * their util_avg from the parent task, but those heuristics could hurt 8424 * other use-cases too. So, until someone finds a better way to solve this, 8425 * let's keep things simple by re-using the existing slow path. 8426 */ 8427static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 8428{ 8429 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 8430 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 8431 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 8432 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 8433 struct root_domain *rd = this_rq()->rd; 8434 int cpu, best_energy_cpu, target = -1; 8435 int prev_fits = -1, best_fits = -1; 8436 unsigned long best_actual_cap = 0; 8437 unsigned long prev_actual_cap = 0; 8438 struct sched_domain *sd; 8439 struct perf_domain *pd; 8440 struct energy_env eenv; 8441 8442 rcu_read_lock(); 8443 pd = rcu_dereference(rd->pd); 8444 if (!pd) 8445 goto unlock; 8446 8447 /* 8448 * Energy-aware wake-up happens on the lowest sched_domain starting 8449 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 8450 */ 8451 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 8452 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 8453 sd = sd->parent; 8454 if (!sd) 8455 goto unlock; 8456 8457 target = prev_cpu; 8458 8459 sync_entity_load_avg(&p->se); 8460 if (!task_util_est(p) && p_util_min == 0) 8461 goto unlock; 8462 8463 eenv_task_busy_time(&eenv, p, prev_cpu); 8464 8465 for (; pd; pd = pd->next) { 8466 unsigned long util_min = p_util_min, util_max = p_util_max; 8467 unsigned long cpu_cap, cpu_actual_cap, util; 8468 long prev_spare_cap = -1, max_spare_cap = -1; 8469 unsigned long rq_util_min, rq_util_max; 8470 unsigned long cur_delta, base_energy; 8471 int max_spare_cap_cpu = -1; 8472 int fits, max_fits = -1; 8473 8474 cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); 8475 8476 if (cpumask_empty(cpus)) 8477 continue; 8478 8479 /* Account external pressure for the energy estimation */ 8480 cpu = cpumask_first(cpus); 8481 cpu_actual_cap = get_actual_cpu_capacity(cpu); 8482 8483 eenv.cpu_cap = cpu_actual_cap; 8484 eenv.pd_cap = 0; 8485 8486 for_each_cpu(cpu, cpus) { 8487 struct rq *rq = cpu_rq(cpu); 8488 8489 eenv.pd_cap += cpu_actual_cap; 8490 8491 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 8492 continue; 8493 8494 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 8495 continue; 8496 8497 util = cpu_util(cpu, p, cpu, 0); 8498 cpu_cap = capacity_of(cpu); 8499 8500 /* 8501 * Skip CPUs that cannot satisfy the capacity request. 8502 * IOW, placing the task there would make the CPU 8503 * overutilized. Take uclamp into account to see how 8504 * much capacity we can get out of the CPU; this is 8505 * aligned with sched_cpu_util(). 8506 */ 8507 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 8508 /* 8509 * Open code uclamp_rq_util_with() except for 8510 * the clamp() part. I.e.: apply max aggregation 8511 * only. util_fits_cpu() logic requires to 8512 * operate on non clamped util but must use the 8513 * max-aggregated uclamp_{min, max}. 8514 */ 8515 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 8516 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 8517 8518 util_min = max(rq_util_min, p_util_min); 8519 util_max = max(rq_util_max, p_util_max); 8520 } 8521 8522 fits = util_fits_cpu(util, util_min, util_max, cpu); 8523 if (!fits) 8524 continue; 8525 8526 lsub_positive(&cpu_cap, util); 8527 8528 if (cpu == prev_cpu) { 8529 /* Always use prev_cpu as a candidate. */ 8530 prev_spare_cap = cpu_cap; 8531 prev_fits = fits; 8532 } else if ((fits > max_fits) || 8533 ((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) { 8534 /* 8535 * Find the CPU with the maximum spare capacity 8536 * among the remaining CPUs in the performance 8537 * domain. 8538 */ 8539 max_spare_cap = cpu_cap; 8540 max_spare_cap_cpu = cpu; 8541 max_fits = fits; 8542 } 8543 } 8544 8545 if (max_spare_cap_cpu < 0 && prev_spare_cap < 0) 8546 continue; 8547 8548 eenv_pd_busy_time(&eenv, cpus, p); 8549 /* Compute the 'base' energy of the pd, without @p */ 8550 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 8551 8552 /* Evaluate the energy impact of using prev_cpu. */ 8553 if (prev_spare_cap > -1) { 8554 prev_delta = compute_energy(&eenv, pd, cpus, p, 8555 prev_cpu); 8556 /* CPU utilization has changed */ 8557 if (prev_delta < base_energy) 8558 goto unlock; 8559 prev_delta -= base_energy; 8560 prev_actual_cap = cpu_actual_cap; 8561 best_delta = min(best_delta, prev_delta); 8562 } 8563 8564 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 8565 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 8566 /* Current best energy cpu fits better */ 8567 if (max_fits < best_fits) 8568 continue; 8569 8570 /* 8571 * Both don't fit performance hint (i.e. uclamp_min) 8572 * but best energy cpu has better capacity. 8573 */ 8574 if ((max_fits < 0) && 8575 (cpu_actual_cap <= best_actual_cap)) 8576 continue; 8577 8578 cur_delta = compute_energy(&eenv, pd, cpus, p, 8579 max_spare_cap_cpu); 8580 /* CPU utilization has changed */ 8581 if (cur_delta < base_energy) 8582 goto unlock; 8583 cur_delta -= base_energy; 8584 8585 /* 8586 * Both fit for the task but best energy cpu has lower 8587 * energy impact. 8588 */ 8589 if ((max_fits > 0) && (best_fits > 0) && 8590 (cur_delta >= best_delta)) 8591 continue; 8592 8593 best_delta = cur_delta; 8594 best_energy_cpu = max_spare_cap_cpu; 8595 best_fits = max_fits; 8596 best_actual_cap = cpu_actual_cap; 8597 } 8598 } 8599 rcu_read_unlock(); 8600 8601 if ((best_fits > prev_fits) || 8602 ((best_fits > 0) && (best_delta < prev_delta)) || 8603 ((best_fits < 0) && (best_actual_cap > prev_actual_cap))) 8604 target = best_energy_cpu; 8605 8606 return target; 8607 8608unlock: 8609 rcu_read_unlock(); 8610 8611 return target; 8612} 8613 8614/* 8615 * select_task_rq_fair: Select target runqueue for the waking task in domains 8616 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 8617 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 8618 * 8619 * Balances load by selecting the idlest CPU in the idlest group, or under 8620 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 8621 * 8622 * Returns the target CPU number. 8623 */ 8624static int 8625select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 8626{ 8627 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 8628 struct sched_domain *tmp, *sd = NULL; 8629 int cpu = smp_processor_id(); 8630 int new_cpu = prev_cpu; 8631 int want_affine = 0; 8632 /* SD_flags and WF_flags share the first nibble */ 8633 int sd_flag = wake_flags & 0xF; 8634 8635 /* 8636 * required for stable ->cpus_allowed 8637 */ 8638 lockdep_assert_held(&p->pi_lock); 8639 if (wake_flags & WF_TTWU) { 8640 record_wakee(p); 8641 8642 if ((wake_flags & WF_CURRENT_CPU) && 8643 cpumask_test_cpu(cpu, p->cpus_ptr)) 8644 return cpu; 8645 8646 if (!is_rd_overutilized(this_rq()->rd)) { 8647 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 8648 if (new_cpu >= 0) 8649 return new_cpu; 8650 new_cpu = prev_cpu; 8651 } 8652 8653 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 8654 } 8655 8656 rcu_read_lock(); 8657 for_each_domain(cpu, tmp) { 8658 /* 8659 * If both 'cpu' and 'prev_cpu' are part of this domain, 8660 * cpu is a valid SD_WAKE_AFFINE target. 8661 */ 8662 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 8663 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 8664 if (cpu != prev_cpu) 8665 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 8666 8667 sd = NULL; /* Prefer wake_affine over balance flags */ 8668 break; 8669 } 8670 8671 /* 8672 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 8673 * usually do not have SD_BALANCE_WAKE set. That means wakeup 8674 * will usually go to the fast path. 8675 */ 8676 if (tmp->flags & sd_flag) 8677 sd = tmp; 8678 else if (!want_affine) 8679 break; 8680 } 8681 8682 if (unlikely(sd)) { 8683 /* Slow path */ 8684 new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); 8685 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 8686 /* Fast path */ 8687 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 8688 } 8689 rcu_read_unlock(); 8690 8691 return new_cpu; 8692} 8693 8694/* 8695 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 8696 * cfs_rq_of(p) references at time of call are still valid and identify the 8697 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 8698 */ 8699static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 8700{ 8701 struct sched_entity *se = &p->se; 8702 8703 if (!task_on_rq_migrating(p)) { 8704 remove_entity_load_avg(se); 8705 8706 /* 8707 * Here, the task's PELT values have been updated according to 8708 * the current rq's clock. But if that clock hasn't been 8709 * updated in a while, a substantial idle time will be missed, 8710 * leading to an inflation after wake-up on the new rq. 8711 * 8712 * Estimate the missing time from the cfs_rq last_update_time 8713 * and update sched_avg to improve the PELT continuity after 8714 * migration. 8715 */ 8716 migrate_se_pelt_lag(se); 8717 } 8718 8719 /* Tell new CPU we are migrated */ 8720 se->avg.last_update_time = 0; 8721 8722 update_scan_period(p, new_cpu); 8723} 8724 8725static void task_dead_fair(struct task_struct *p) 8726{ 8727 struct sched_entity *se = &p->se; 8728 8729 if (se->sched_delayed) { 8730 struct rq_flags rf; 8731 struct rq *rq; 8732 8733 rq = task_rq_lock(p, &rf); 8734 if (se->sched_delayed) { 8735 update_rq_clock(rq); 8736 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 8737 } 8738 task_rq_unlock(rq, p, &rf); 8739 } 8740 8741 remove_entity_load_avg(se); 8742} 8743 8744/* 8745 * Set the max capacity the task is allowed to run at for misfit detection. 8746 */ 8747static void set_task_max_allowed_capacity(struct task_struct *p) 8748{ 8749 struct asym_cap_data *entry; 8750 8751 if (!sched_asym_cpucap_active()) 8752 return; 8753 8754 rcu_read_lock(); 8755 list_for_each_entry_rcu(entry, &asym_cap_list, link) { 8756 cpumask_t *cpumask; 8757 8758 cpumask = cpu_capacity_span(entry); 8759 if (!cpumask_intersects(p->cpus_ptr, cpumask)) 8760 continue; 8761 8762 p->max_allowed_capacity = entry->capacity; 8763 break; 8764 } 8765 rcu_read_unlock(); 8766} 8767 8768static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context *ctx) 8769{ 8770 set_cpus_allowed_common(p, ctx); 8771 set_task_max_allowed_capacity(p); 8772} 8773 8774static void set_next_buddy(struct sched_entity *se) 8775{ 8776 for_each_sched_entity(se) { 8777 if (WARN_ON_ONCE(!se->on_rq)) 8778 return; 8779 if (se_is_idle(se)) 8780 return; 8781 cfs_rq_of(se)->next = se; 8782 } 8783} 8784 8785enum preempt_wakeup_action { 8786 PREEMPT_WAKEUP_NONE, /* No preemption. */ 8787 PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ 8788 PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ 8789 PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ 8790}; 8791 8792static inline bool 8793set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, 8794 struct sched_entity *pse, struct sched_entity *se) 8795{ 8796 /* 8797 * Keep existing buddy if the deadline is sooner than pse. 8798 * The older buddy may be cache cold and completely unrelated 8799 * to the current wakeup but that is unpredictable where as 8800 * obeying the deadline is more in line with EEVDF objectives. 8801 */ 8802 if (cfs_rq->next && entity_before(cfs_rq->next, pse)) 8803 return false; 8804 8805 set_next_buddy(pse); 8806 return true; 8807} 8808 8809/* 8810 * WF_SYNC|WF_TTWU indicates the waker expects to sleep but it is not 8811 * strictly enforced because the hint is either misunderstood or 8812 * multiple tasks must be woken up. 8813 */ 8814static inline enum preempt_wakeup_action 8815preempt_sync(struct rq *rq, int wake_flags, 8816 struct sched_entity *pse, struct sched_entity *se) 8817{ 8818 u64 threshold, delta; 8819 8820 /* 8821 * WF_SYNC without WF_TTWU is not expected so warn if it happens even 8822 * though it is likely harmless. 8823 */ 8824 WARN_ON_ONCE(!(wake_flags & WF_TTWU)); 8825 8826 threshold = sysctl_sched_migration_cost; 8827 delta = rq_clock_task(rq) - se->exec_start; 8828 if ((s64)delta < 0) 8829 delta = 0; 8830 8831 /* 8832 * WF_RQ_SELECTED implies the tasks are stacking on a CPU when they 8833 * could run on other CPUs. Reduce the threshold before preemption is 8834 * allowed to an arbitrary lower value as it is more likely (but not 8835 * guaranteed) the waker requires the wakee to finish. 8836 */ 8837 if (wake_flags & WF_RQ_SELECTED) 8838 threshold >>= 2; 8839 8840 /* 8841 * As WF_SYNC is not strictly obeyed, allow some runtime for batch 8842 * wakeups to be issued. 8843 */ 8844 if (entity_before(pse, se) && delta >= threshold) 8845 return PREEMPT_WAKEUP_RESCHED; 8846 8847 return PREEMPT_WAKEUP_NONE; 8848} 8849 8850/* 8851 * Preempt the current task with a newly woken task if needed: 8852 */ 8853static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int wake_flags) 8854{ 8855 enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; 8856 struct task_struct *donor = rq->donor; 8857 struct sched_entity *se = &donor->se, *pse = &p->se; 8858 struct cfs_rq *cfs_rq = task_cfs_rq(donor); 8859 int cse_is_idle, pse_is_idle; 8860 8861 if (unlikely(se == pse)) 8862 return; 8863 8864 /* 8865 * This is possible from callers such as attach_tasks(), in which we 8866 * unconditionally wakeup_preempt() after an enqueue (which may have 8867 * lead to a throttle). This both saves work and prevents false 8868 * next-buddy nomination below. 8869 */ 8870 if (task_is_throttled(p)) 8871 return; 8872 8873 /* 8874 * We can come here with TIF_NEED_RESCHED already set from new task 8875 * wake up path. 8876 * 8877 * Note: this also catches the edge-case of curr being in a throttled 8878 * group (e.g. via set_curr_task), since update_curr() (in the 8879 * enqueue of curr) will have resulted in resched being set. This 8880 * prevents us from potentially nominating it as a false LAST_BUDDY 8881 * below. 8882 */ 8883 if (test_tsk_need_resched(rq->curr)) 8884 return; 8885 8886 if (!sched_feat(WAKEUP_PREEMPTION)) 8887 return; 8888 8889 find_matching_se(&se, &pse); 8890 WARN_ON_ONCE(!pse); 8891 8892 cse_is_idle = se_is_idle(se); 8893 pse_is_idle = se_is_idle(pse); 8894 8895 /* 8896 * Preempt an idle entity in favor of a non-idle entity (and don't preempt 8897 * in the inverse case). 8898 */ 8899 if (cse_is_idle && !pse_is_idle) { 8900 /* 8901 * When non-idle entity preempt an idle entity, 8902 * don't give idle entity slice protection. 8903 */ 8904 preempt_action = PREEMPT_WAKEUP_SHORT; 8905 goto preempt; 8906 } 8907 8908 if (cse_is_idle != pse_is_idle) 8909 return; 8910 8911 /* 8912 * BATCH and IDLE tasks do not preempt others. 8913 */ 8914 if (unlikely(!normal_policy(p->policy))) 8915 return; 8916 8917 cfs_rq = cfs_rq_of(se); 8918 update_curr(cfs_rq); 8919 /* 8920 * If @p has a shorter slice than current and @p is eligible, override 8921 * current's slice protection in order to allow preemption. 8922 */ 8923 if (sched_feat(PREEMPT_SHORT) && (pse->slice < se->slice)) { 8924 preempt_action = PREEMPT_WAKEUP_SHORT; 8925 goto pick; 8926 } 8927 8928 /* 8929 * Ignore wakee preemption on WF_FORK as it is less likely that 8930 * there is shared data as exec often follow fork. Do not 8931 * preempt for tasks that are sched_delayed as it would violate 8932 * EEVDF to forcibly queue an ineligible task. 8933 */ 8934 if ((wake_flags & WF_FORK) || pse->sched_delayed) 8935 return; 8936 8937 /* Prefer picking wakee soon if appropriate. */ 8938 if (sched_feat(NEXT_BUDDY) && 8939 set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { 8940 8941 /* 8942 * Decide whether to obey WF_SYNC hint for a new buddy. Old 8943 * buddies are ignored as they may not be relevant to the 8944 * waker and less likely to be cache hot. 8945 */ 8946 if (wake_flags & WF_SYNC) 8947 preempt_action = preempt_sync(rq, wake_flags, pse, se); 8948 } 8949 8950 switch (preempt_action) { 8951 case PREEMPT_WAKEUP_NONE: 8952 return; 8953 case PREEMPT_WAKEUP_RESCHED: 8954 goto preempt; 8955 case PREEMPT_WAKEUP_SHORT: 8956 fallthrough; 8957 case PREEMPT_WAKEUP_PICK: 8958 break; 8959 } 8960 8961pick: 8962 /* 8963 * If @p has become the most eligible task, force preemption. 8964 */ 8965 if (__pick_eevdf(cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT) == pse) 8966 goto preempt; 8967 8968 if (sched_feat(RUN_TO_PARITY)) 8969 update_protect_slice(cfs_rq, se); 8970 8971 return; 8972 8973preempt: 8974 if (preempt_action == PREEMPT_WAKEUP_SHORT) 8975 cancel_protect_slice(se); 8976 8977 resched_curr_lazy(rq); 8978} 8979 8980static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) 8981{ 8982 struct sched_entity *se; 8983 struct cfs_rq *cfs_rq; 8984 struct task_struct *p; 8985 bool throttled; 8986 8987again: 8988 cfs_rq = &rq->cfs; 8989 if (!cfs_rq->nr_queued) 8990 return NULL; 8991 8992 throttled = false; 8993 8994 do { 8995 /* Might not have done put_prev_entity() */ 8996 if (cfs_rq->curr && cfs_rq->curr->on_rq) 8997 update_curr(cfs_rq); 8998 8999 throttled |= check_cfs_rq_runtime(cfs_rq); 9000 9001 se = pick_next_entity(rq, cfs_rq); 9002 if (!se) 9003 goto again; 9004 cfs_rq = group_cfs_rq(se); 9005 } while (cfs_rq); 9006 9007 p = task_of(se); 9008 if (unlikely(throttled)) 9009 task_throttle_setup_work(p); 9010 return p; 9011} 9012 9013static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 9014static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 9015 9016struct task_struct * 9017pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 9018{ 9019 struct sched_entity *se; 9020 struct task_struct *p; 9021 int new_tasks; 9022 9023again: 9024 p = pick_task_fair(rq, rf); 9025 if (!p) 9026 goto idle; 9027 se = &p->se; 9028 9029#ifdef CONFIG_FAIR_GROUP_SCHED 9030 if (prev->sched_class != &fair_sched_class) 9031 goto simple; 9032 9033 __put_prev_set_next_dl_server(rq, prev, p); 9034 9035 /* 9036 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 9037 * likely that a next task is from the same cgroup as the current. 9038 * 9039 * Therefore attempt to avoid putting and setting the entire cgroup 9040 * hierarchy, only change the part that actually changes. 9041 * 9042 * Since we haven't yet done put_prev_entity and if the selected task 9043 * is a different task than we started out with, try and touch the 9044 * least amount of cfs_rqs. 9045 */ 9046 if (prev != p) { 9047 struct sched_entity *pse = &prev->se; 9048 struct cfs_rq *cfs_rq; 9049 9050 while (!(cfs_rq = is_same_group(se, pse))) { 9051 int se_depth = se->depth; 9052 int pse_depth = pse->depth; 9053 9054 if (se_depth <= pse_depth) { 9055 put_prev_entity(cfs_rq_of(pse), pse); 9056 pse = parent_entity(pse); 9057 } 9058 if (se_depth >= pse_depth) { 9059 set_next_entity(cfs_rq_of(se), se, true); 9060 se = parent_entity(se); 9061 } 9062 } 9063 9064 put_prev_entity(cfs_rq, pse); 9065 set_next_entity(cfs_rq, se, true); 9066 9067 __set_next_task_fair(rq, p, true); 9068 } 9069 9070 return p; 9071 9072simple: 9073#endif /* CONFIG_FAIR_GROUP_SCHED */ 9074 put_prev_set_next_task(rq, prev, p); 9075 return p; 9076 9077idle: 9078 if (rf) { 9079 new_tasks = sched_balance_newidle(rq, rf); 9080 9081 /* 9082 * Because sched_balance_newidle() releases (and re-acquires) 9083 * rq->lock, it is possible for any higher priority task to 9084 * appear. In that case we must re-start the pick_next_entity() 9085 * loop. 9086 */ 9087 if (new_tasks < 0) 9088 return RETRY_TASK; 9089 9090 if (new_tasks > 0) 9091 goto again; 9092 } 9093 9094 return NULL; 9095} 9096 9097static struct task_struct * 9098fair_server_pick_task(struct sched_dl_entity *dl_se, struct rq_flags *rf) 9099{ 9100 return pick_task_fair(dl_se->rq, rf); 9101} 9102 9103void fair_server_init(struct rq *rq) 9104{ 9105 struct sched_dl_entity *dl_se = &rq->fair_server; 9106 9107 init_dl_entity(dl_se); 9108 9109 dl_server_init(dl_se, rq, fair_server_pick_task); 9110} 9111 9112/* 9113 * Account for a descheduled task: 9114 */ 9115static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) 9116{ 9117 struct sched_entity *se = &prev->se; 9118 struct cfs_rq *cfs_rq; 9119 9120 for_each_sched_entity(se) { 9121 cfs_rq = cfs_rq_of(se); 9122 put_prev_entity(cfs_rq, se); 9123 } 9124} 9125 9126/* 9127 * sched_yield() is very simple 9128 */ 9129static void yield_task_fair(struct rq *rq) 9130{ 9131 struct task_struct *curr = rq->donor; 9132 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 9133 struct sched_entity *se = &curr->se; 9134 9135 /* 9136 * Are we the only task in the tree? 9137 */ 9138 if (unlikely(rq->nr_running == 1)) 9139 return; 9140 9141 clear_buddies(cfs_rq, se); 9142 9143 update_rq_clock(rq); 9144 /* 9145 * Update run-time statistics of the 'current'. 9146 */ 9147 update_curr(cfs_rq); 9148 /* 9149 * Tell update_rq_clock() that we've just updated, 9150 * so we don't do microscopic update in schedule() 9151 * and double the fastpath cost. 9152 */ 9153 rq_clock_skip_update(rq); 9154 9155 /* 9156 * Forfeit the remaining vruntime, only if the entity is eligible. This 9157 * condition is necessary because in core scheduling we prefer to run 9158 * ineligible tasks rather than force idling. If this happens we may 9159 * end up in a loop where the core scheduler picks the yielding task, 9160 * which yields immediately again; without the condition the vruntime 9161 * ends up quickly running away. 9162 */ 9163 if (entity_eligible(cfs_rq, se)) { 9164 se->vruntime = se->deadline; 9165 update_deadline(cfs_rq, se); 9166 } 9167} 9168 9169static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 9170{ 9171 struct sched_entity *se = &p->se; 9172 9173 /* !se->on_rq also covers throttled task */ 9174 if (!se->on_rq) 9175 return false; 9176 9177 /* Tell the scheduler that we'd really like se to run next. */ 9178 set_next_buddy(se); 9179 9180 yield_task_fair(rq); 9181 9182 return true; 9183} 9184 9185/************************************************** 9186 * Fair scheduling class load-balancing methods. 9187 * 9188 * BASICS 9189 * 9190 * The purpose of load-balancing is to achieve the same basic fairness the 9191 * per-CPU scheduler provides, namely provide a proportional amount of compute 9192 * time to each task. This is expressed in the following equation: 9193 * 9194 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 9195 * 9196 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 9197 * W_i,0 is defined as: 9198 * 9199 * W_i,0 = \Sum_j w_i,j (2) 9200 * 9201 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 9202 * is derived from the nice value as per sched_prio_to_weight[]. 9203 * 9204 * The weight average is an exponential decay average of the instantaneous 9205 * weight: 9206 * 9207 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 9208 * 9209 * C_i is the compute capacity of CPU i, typically it is the 9210 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 9211 * can also include other factors [XXX]. 9212 * 9213 * To achieve this balance we define a measure of imbalance which follows 9214 * directly from (1): 9215 * 9216 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 9217 * 9218 * We them move tasks around to minimize the imbalance. In the continuous 9219 * function space it is obvious this converges, in the discrete case we get 9220 * a few fun cases generally called infeasible weight scenarios. 9221 * 9222 * [XXX expand on: 9223 * - infeasible weights; 9224 * - local vs global optima in the discrete case. ] 9225 * 9226 * 9227 * SCHED DOMAINS 9228 * 9229 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 9230 * for all i,j solution, we create a tree of CPUs that follows the hardware 9231 * topology where each level pairs two lower groups (or better). This results 9232 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 9233 * tree to only the first of the previous level and we decrease the frequency 9234 * of load-balance at each level inversely proportional to the number of CPUs in 9235 * the groups. 9236 * 9237 * This yields: 9238 * 9239 * log_2 n 1 n 9240 * \Sum { --- * --- * 2^i } = O(n) (5) 9241 * i = 0 2^i 2^i 9242 * `- size of each group 9243 * | | `- number of CPUs doing load-balance 9244 * | `- freq 9245 * `- sum over all levels 9246 * 9247 * Coupled with a limit on how many tasks we can migrate every balance pass, 9248 * this makes (5) the runtime complexity of the balancer. 9249 * 9250 * An important property here is that each CPU is still (indirectly) connected 9251 * to every other CPU in at most O(log n) steps: 9252 * 9253 * The adjacency matrix of the resulting graph is given by: 9254 * 9255 * log_2 n 9256 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 9257 * k = 0 9258 * 9259 * And you'll find that: 9260 * 9261 * A^(log_2 n)_i,j != 0 for all i,j (7) 9262 * 9263 * Showing there's indeed a path between every CPU in at most O(log n) steps. 9264 * The task movement gives a factor of O(m), giving a convergence complexity 9265 * of: 9266 * 9267 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 9268 * 9269 * 9270 * WORK CONSERVING 9271 * 9272 * In order to avoid CPUs going idle while there's still work to do, new idle 9273 * balancing is more aggressive and has the newly idle CPU iterate up the domain 9274 * tree itself instead of relying on other CPUs to bring it work. 9275 * 9276 * This adds some complexity to both (5) and (8) but it reduces the total idle 9277 * time. 9278 * 9279 * [XXX more?] 9280 * 9281 * 9282 * CGROUPS 9283 * 9284 * Cgroups make a horror show out of (2), instead of a simple sum we get: 9285 * 9286 * s_k,i 9287 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 9288 * S_k 9289 * 9290 * Where 9291 * 9292 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 9293 * 9294 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 9295 * 9296 * The big problem is S_k, its a global sum needed to compute a local (W_i) 9297 * property. 9298 * 9299 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 9300 * rewrite all of this once again.] 9301 */ 9302 9303static unsigned long __read_mostly max_load_balance_interval = HZ/10; 9304 9305enum fbq_type { regular, remote, all }; 9306 9307/* 9308 * 'group_type' describes the group of CPUs at the moment of load balancing. 9309 * 9310 * The enum is ordered by pulling priority, with the group with lowest priority 9311 * first so the group_type can simply be compared when selecting the busiest 9312 * group. See update_sd_pick_busiest(). 9313 */ 9314enum group_type { 9315 /* The group has spare capacity that can be used to run more tasks. */ 9316 group_has_spare = 0, 9317 /* 9318 * The group is fully used and the tasks don't compete for more CPU 9319 * cycles. Nevertheless, some tasks might wait before running. 9320 */ 9321 group_fully_busy, 9322 /* 9323 * One task doesn't fit with CPU's capacity and must be migrated to a 9324 * more powerful CPU. 9325 */ 9326 group_misfit_task, 9327 /* 9328 * Balance SMT group that's fully busy. Can benefit from migration 9329 * a task on SMT with busy sibling to another CPU on idle core. 9330 */ 9331 group_smt_balance, 9332 /* 9333 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 9334 * and the task should be migrated to it instead of running on the 9335 * current CPU. 9336 */ 9337 group_asym_packing, 9338 /* 9339 * The tasks' affinity constraints previously prevented the scheduler 9340 * from balancing the load across the system. 9341 */ 9342 group_imbalanced, 9343 /* 9344 * The CPU is overloaded and can't provide expected CPU cycles to all 9345 * tasks. 9346 */ 9347 group_overloaded 9348}; 9349 9350enum migration_type { 9351 migrate_load = 0, 9352 migrate_util, 9353 migrate_task, 9354 migrate_misfit 9355}; 9356 9357#define LBF_ALL_PINNED 0x01 9358#define LBF_NEED_BREAK 0x02 9359#define LBF_DST_PINNED 0x04 9360#define LBF_SOME_PINNED 0x08 9361#define LBF_ACTIVE_LB 0x10 9362 9363struct lb_env { 9364 struct sched_domain *sd; 9365 9366 struct rq *src_rq; 9367 int src_cpu; 9368 9369 int dst_cpu; 9370 struct rq *dst_rq; 9371 9372 struct cpumask *dst_grpmask; 9373 int new_dst_cpu; 9374 enum cpu_idle_type idle; 9375 long imbalance; 9376 /* The set of CPUs under consideration for load-balancing */ 9377 struct cpumask *cpus; 9378 9379 unsigned int flags; 9380 9381 unsigned int loop; 9382 unsigned int loop_break; 9383 unsigned int loop_max; 9384 9385 enum fbq_type fbq_type; 9386 enum migration_type migration_type; 9387 struct list_head tasks; 9388}; 9389 9390/* 9391 * Is this task likely cache-hot: 9392 */ 9393static int task_hot(struct task_struct *p, struct lb_env *env) 9394{ 9395 s64 delta; 9396 9397 lockdep_assert_rq_held(env->src_rq); 9398 9399 if (p->sched_class != &fair_sched_class) 9400 return 0; 9401 9402 if (unlikely(task_has_idle_policy(p))) 9403 return 0; 9404 9405 /* SMT siblings share cache */ 9406 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 9407 return 0; 9408 9409 /* 9410 * Buddy candidates are cache hot: 9411 */ 9412 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 9413 (&p->se == cfs_rq_of(&p->se)->next)) 9414 return 1; 9415 9416 if (sysctl_sched_migration_cost == -1) 9417 return 1; 9418 9419 /* 9420 * Don't migrate task if the task's cookie does not match 9421 * with the destination CPU's core cookie. 9422 */ 9423 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 9424 return 1; 9425 9426 if (sysctl_sched_migration_cost == 0) 9427 return 0; 9428 9429 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 9430 9431 return delta < (s64)sysctl_sched_migration_cost; 9432} 9433 9434#ifdef CONFIG_NUMA_BALANCING 9435/* 9436 * Returns a positive value, if task migration degrades locality. 9437 * Returns 0, if task migration is not affected by locality. 9438 * Returns a negative value, if task migration improves locality i.e migration preferred. 9439 */ 9440static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 9441{ 9442 struct numa_group *numa_group = rcu_dereference(p->numa_group); 9443 unsigned long src_weight, dst_weight; 9444 int src_nid, dst_nid, dist; 9445 9446 if (!static_branch_likely(&sched_numa_balancing)) 9447 return 0; 9448 9449 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 9450 return 0; 9451 9452 src_nid = cpu_to_node(env->src_cpu); 9453 dst_nid = cpu_to_node(env->dst_cpu); 9454 9455 if (src_nid == dst_nid) 9456 return 0; 9457 9458 /* Migrating away from the preferred node is always bad. */ 9459 if (src_nid == p->numa_preferred_nid) { 9460 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 9461 return 1; 9462 else 9463 return 0; 9464 } 9465 9466 /* Encourage migration to the preferred node. */ 9467 if (dst_nid == p->numa_preferred_nid) 9468 return -1; 9469 9470 /* Leaving a core idle is often worse than degrading locality. */ 9471 if (env->idle == CPU_IDLE) 9472 return 0; 9473 9474 dist = node_distance(src_nid, dst_nid); 9475 if (numa_group) { 9476 src_weight = group_weight(p, src_nid, dist); 9477 dst_weight = group_weight(p, dst_nid, dist); 9478 } else { 9479 src_weight = task_weight(p, src_nid, dist); 9480 dst_weight = task_weight(p, dst_nid, dist); 9481 } 9482 9483 return src_weight - dst_weight; 9484} 9485 9486#else /* !CONFIG_NUMA_BALANCING: */ 9487static inline long migrate_degrades_locality(struct task_struct *p, 9488 struct lb_env *env) 9489{ 9490 return 0; 9491} 9492#endif /* !CONFIG_NUMA_BALANCING */ 9493 9494/* 9495 * Check whether the task is ineligible on the destination cpu 9496 * 9497 * When the PLACE_LAG scheduling feature is enabled and 9498 * dst_cfs_rq->nr_queued is greater than 1, if the task 9499 * is ineligible, it will also be ineligible when 9500 * it is migrated to the destination cpu. 9501 */ 9502static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) 9503{ 9504 struct cfs_rq *dst_cfs_rq; 9505 9506#ifdef CONFIG_FAIR_GROUP_SCHED 9507 dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu]; 9508#else 9509 dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; 9510#endif 9511 if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && 9512 !entity_eligible(task_cfs_rq(p), &p->se)) 9513 return 1; 9514 9515 return 0; 9516} 9517 9518/* 9519 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 9520 */ 9521static 9522int can_migrate_task(struct task_struct *p, struct lb_env *env) 9523{ 9524 long degrades, hot; 9525 9526 lockdep_assert_rq_held(env->src_rq); 9527 if (p->sched_task_hot) 9528 p->sched_task_hot = 0; 9529 9530 /* 9531 * We do not migrate tasks that are: 9532 * 1) delayed dequeued unless we migrate load, or 9533 * 2) target cfs_rq is in throttled hierarchy, or 9534 * 3) cannot be migrated to this CPU due to cpus_ptr, or 9535 * 4) running (obviously), or 9536 * 5) are cache-hot on their current CPU, or 9537 * 6) are blocked on mutexes (if SCHED_PROXY_EXEC is enabled) 9538 */ 9539 if ((p->se.sched_delayed) && (env->migration_type != migrate_load)) 9540 return 0; 9541 9542 if (lb_throttled_hierarchy(p, env->dst_cpu)) 9543 return 0; 9544 9545 /* 9546 * We want to prioritize the migration of eligible tasks. 9547 * For ineligible tasks we soft-limit them and only allow 9548 * them to migrate when nr_balance_failed is non-zero to 9549 * avoid load-balancing trying very hard to balance the load. 9550 */ 9551 if (!env->sd->nr_balance_failed && 9552 task_is_ineligible_on_dst_cpu(p, env->dst_cpu)) 9553 return 0; 9554 9555 /* Disregard percpu kthreads; they are where they need to be. */ 9556 if (kthread_is_per_cpu(p)) 9557 return 0; 9558 9559 if (task_is_blocked(p)) 9560 return 0; 9561 9562 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 9563 int cpu; 9564 9565 schedstat_inc(p->stats.nr_failed_migrations_affine); 9566 9567 env->flags |= LBF_SOME_PINNED; 9568 9569 /* 9570 * Remember if this task can be migrated to any other CPU in 9571 * our sched_group. We may want to revisit it if we couldn't 9572 * meet load balance goals by pulling other tasks on src_cpu. 9573 * 9574 * Avoid computing new_dst_cpu 9575 * - for NEWLY_IDLE 9576 * - if we have already computed one in current iteration 9577 * - if it's an active balance 9578 */ 9579 if (env->idle == CPU_NEWLY_IDLE || 9580 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 9581 return 0; 9582 9583 /* Prevent to re-select dst_cpu via env's CPUs: */ 9584 cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr); 9585 9586 if (cpu < nr_cpu_ids) { 9587 env->flags |= LBF_DST_PINNED; 9588 env->new_dst_cpu = cpu; 9589 } 9590 9591 return 0; 9592 } 9593 9594 /* Record that we found at least one task that could run on dst_cpu */ 9595 env->flags &= ~LBF_ALL_PINNED; 9596 9597 if (task_on_cpu(env->src_rq, p) || 9598 task_current_donor(env->src_rq, p)) { 9599 schedstat_inc(p->stats.nr_failed_migrations_running); 9600 return 0; 9601 } 9602 9603 /* 9604 * Aggressive migration if: 9605 * 1) active balance 9606 * 2) destination numa is preferred 9607 * 3) task is cache cold, or 9608 * 4) too many balance attempts have failed. 9609 */ 9610 if (env->flags & LBF_ACTIVE_LB) 9611 return 1; 9612 9613 degrades = migrate_degrades_locality(p, env); 9614 if (!degrades) 9615 hot = task_hot(p, env); 9616 else 9617 hot = degrades > 0; 9618 9619 if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 9620 if (hot) 9621 p->sched_task_hot = 1; 9622 return 1; 9623 } 9624 9625 schedstat_inc(p->stats.nr_failed_migrations_hot); 9626 return 0; 9627} 9628 9629/* 9630 * detach_task() -- detach the task for the migration specified in env 9631 */ 9632static void detach_task(struct task_struct *p, struct lb_env *env) 9633{ 9634 lockdep_assert_rq_held(env->src_rq); 9635 9636 if (p->sched_task_hot) { 9637 p->sched_task_hot = 0; 9638 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 9639 schedstat_inc(p->stats.nr_forced_migrations); 9640 } 9641 9642 WARN_ON(task_current(env->src_rq, p)); 9643 WARN_ON(task_current_donor(env->src_rq, p)); 9644 9645 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 9646 set_task_cpu(p, env->dst_cpu); 9647} 9648 9649/* 9650 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 9651 * part of active balancing operations within "domain". 9652 * 9653 * Returns a task if successful and NULL otherwise. 9654 */ 9655static struct task_struct *detach_one_task(struct lb_env *env) 9656{ 9657 struct task_struct *p; 9658 9659 lockdep_assert_rq_held(env->src_rq); 9660 9661 list_for_each_entry_reverse(p, 9662 &env->src_rq->cfs_tasks, se.group_node) { 9663 if (!can_migrate_task(p, env)) 9664 continue; 9665 9666 detach_task(p, env); 9667 9668 /* 9669 * Right now, this is only the second place where 9670 * lb_gained[env->idle] is updated (other is detach_tasks) 9671 * so we can safely collect stats here rather than 9672 * inside detach_tasks(). 9673 */ 9674 schedstat_inc(env->sd->lb_gained[env->idle]); 9675 return p; 9676 } 9677 return NULL; 9678} 9679 9680/* 9681 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 9682 * busiest_rq, as part of a balancing operation within domain "sd". 9683 * 9684 * Returns number of detached tasks if successful and 0 otherwise. 9685 */ 9686static int detach_tasks(struct lb_env *env) 9687{ 9688 struct list_head *tasks = &env->src_rq->cfs_tasks; 9689 unsigned long util, load; 9690 struct task_struct *p; 9691 int detached = 0; 9692 9693 lockdep_assert_rq_held(env->src_rq); 9694 9695 /* 9696 * Source run queue has been emptied by another CPU, clear 9697 * LBF_ALL_PINNED flag as we will not test any task. 9698 */ 9699 if (env->src_rq->nr_running <= 1) { 9700 env->flags &= ~LBF_ALL_PINNED; 9701 return 0; 9702 } 9703 9704 if (env->imbalance <= 0) 9705 return 0; 9706 9707 while (!list_empty(tasks)) { 9708 /* 9709 * We don't want to steal all, otherwise we may be treated likewise, 9710 * which could at worst lead to a livelock crash. 9711 */ 9712 if (env->idle && env->src_rq->nr_running <= 1) 9713 break; 9714 9715 env->loop++; 9716 /* We've more or less seen every task there is, call it quits */ 9717 if (env->loop > env->loop_max) 9718 break; 9719 9720 /* take a breather every nr_migrate tasks */ 9721 if (env->loop > env->loop_break) { 9722 env->loop_break += SCHED_NR_MIGRATE_BREAK; 9723 env->flags |= LBF_NEED_BREAK; 9724 break; 9725 } 9726 9727 p = list_last_entry(tasks, struct task_struct, se.group_node); 9728 9729 if (!can_migrate_task(p, env)) 9730 goto next; 9731 9732 switch (env->migration_type) { 9733 case migrate_load: 9734 /* 9735 * Depending of the number of CPUs and tasks and the 9736 * cgroup hierarchy, task_h_load() can return a null 9737 * value. Make sure that env->imbalance decreases 9738 * otherwise detach_tasks() will stop only after 9739 * detaching up to loop_max tasks. 9740 */ 9741 load = max_t(unsigned long, task_h_load(p), 1); 9742 9743 if (sched_feat(LB_MIN) && 9744 load < 16 && !env->sd->nr_balance_failed) 9745 goto next; 9746 9747 /* 9748 * Make sure that we don't migrate too much load. 9749 * Nevertheless, let relax the constraint if 9750 * scheduler fails to find a good waiting task to 9751 * migrate. 9752 */ 9753 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 9754 goto next; 9755 9756 env->imbalance -= load; 9757 break; 9758 9759 case migrate_util: 9760 util = task_util_est(p); 9761 9762 if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance) 9763 goto next; 9764 9765 env->imbalance -= util; 9766 break; 9767 9768 case migrate_task: 9769 env->imbalance--; 9770 break; 9771 9772 case migrate_misfit: 9773 /* This is not a misfit task */ 9774 if (task_fits_cpu(p, env->src_cpu)) 9775 goto next; 9776 9777 env->imbalance = 0; 9778 break; 9779 } 9780 9781 detach_task(p, env); 9782 list_add(&p->se.group_node, &env->tasks); 9783 9784 detached++; 9785 9786#ifdef CONFIG_PREEMPTION 9787 /* 9788 * NEWIDLE balancing is a source of latency, so preemptible 9789 * kernels will stop after the first task is detached to minimize 9790 * the critical section. 9791 */ 9792 if (env->idle == CPU_NEWLY_IDLE) 9793 break; 9794#endif 9795 9796 /* 9797 * We only want to steal up to the prescribed amount of 9798 * load/util/tasks. 9799 */ 9800 if (env->imbalance <= 0) 9801 break; 9802 9803 continue; 9804next: 9805 if (p->sched_task_hot) 9806 schedstat_inc(p->stats.nr_failed_migrations_hot); 9807 9808 list_move(&p->se.group_node, tasks); 9809 } 9810 9811 /* 9812 * Right now, this is one of only two places we collect this stat 9813 * so we can safely collect detach_one_task() stats here rather 9814 * than inside detach_one_task(). 9815 */ 9816 schedstat_add(env->sd->lb_gained[env->idle], detached); 9817 9818 return detached; 9819} 9820 9821/* 9822 * attach_task() -- attach the task detached by detach_task() to its new rq. 9823 */ 9824static void attach_task(struct rq *rq, struct task_struct *p) 9825{ 9826 lockdep_assert_rq_held(rq); 9827 9828 WARN_ON_ONCE(task_rq(p) != rq); 9829 activate_task(rq, p, ENQUEUE_NOCLOCK); 9830 wakeup_preempt(rq, p, 0); 9831} 9832 9833/* 9834 * attach_one_task() -- attaches the task returned from detach_one_task() to 9835 * its new rq. 9836 */ 9837static void attach_one_task(struct rq *rq, struct task_struct *p) 9838{ 9839 struct rq_flags rf; 9840 9841 rq_lock(rq, &rf); 9842 update_rq_clock(rq); 9843 attach_task(rq, p); 9844 rq_unlock(rq, &rf); 9845} 9846 9847/* 9848 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 9849 * new rq. 9850 */ 9851static void attach_tasks(struct lb_env *env) 9852{ 9853 struct list_head *tasks = &env->tasks; 9854 struct task_struct *p; 9855 struct rq_flags rf; 9856 9857 rq_lock(env->dst_rq, &rf); 9858 update_rq_clock(env->dst_rq); 9859 9860 while (!list_empty(tasks)) { 9861 p = list_first_entry(tasks, struct task_struct, se.group_node); 9862 list_del_init(&p->se.group_node); 9863 9864 attach_task(env->dst_rq, p); 9865 } 9866 9867 rq_unlock(env->dst_rq, &rf); 9868} 9869 9870#ifdef CONFIG_NO_HZ_COMMON 9871static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 9872{ 9873 if (cfs_rq->avg.load_avg) 9874 return true; 9875 9876 if (cfs_rq->avg.util_avg) 9877 return true; 9878 9879 return false; 9880} 9881 9882static inline bool others_have_blocked(struct rq *rq) 9883{ 9884 if (cpu_util_rt(rq)) 9885 return true; 9886 9887 if (cpu_util_dl(rq)) 9888 return true; 9889 9890 if (hw_load_avg(rq)) 9891 return true; 9892 9893 if (cpu_util_irq(rq)) 9894 return true; 9895 9896 return false; 9897} 9898 9899static inline void update_blocked_load_tick(struct rq *rq) 9900{ 9901 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 9902} 9903 9904static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 9905{ 9906 if (!has_blocked) 9907 rq->has_blocked_load = 0; 9908} 9909#else /* !CONFIG_NO_HZ_COMMON: */ 9910static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 9911static inline bool others_have_blocked(struct rq *rq) { return false; } 9912static inline void update_blocked_load_tick(struct rq *rq) {} 9913static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 9914#endif /* !CONFIG_NO_HZ_COMMON */ 9915 9916static bool __update_blocked_others(struct rq *rq, bool *done) 9917{ 9918 bool updated; 9919 9920 /* 9921 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 9922 * DL and IRQ signals have been updated before updating CFS. 9923 */ 9924 updated = update_other_load_avgs(rq); 9925 9926 if (others_have_blocked(rq)) 9927 *done = false; 9928 9929 return updated; 9930} 9931 9932#ifdef CONFIG_FAIR_GROUP_SCHED 9933 9934static bool __update_blocked_fair(struct rq *rq, bool *done) 9935{ 9936 struct cfs_rq *cfs_rq, *pos; 9937 bool decayed = false; 9938 int cpu = cpu_of(rq); 9939 9940 /* 9941 * Iterates the task_group tree in a bottom up fashion, see 9942 * list_add_leaf_cfs_rq() for details. 9943 */ 9944 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 9945 struct sched_entity *se; 9946 9947 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9948 update_tg_load_avg(cfs_rq); 9949 9950 if (cfs_rq->nr_queued == 0) 9951 update_idle_cfs_rq_clock_pelt(cfs_rq); 9952 9953 if (cfs_rq == &rq->cfs) 9954 decayed = true; 9955 } 9956 9957 /* Propagate pending load changes to the parent, if any: */ 9958 se = cfs_rq->tg->se[cpu]; 9959 if (se && !skip_blocked_update(se)) 9960 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9961 9962 /* 9963 * There can be a lot of idle CPU cgroups. Don't let fully 9964 * decayed cfs_rqs linger on the list. 9965 */ 9966 if (cfs_rq_is_decayed(cfs_rq)) 9967 list_del_leaf_cfs_rq(cfs_rq); 9968 9969 /* Don't need periodic decay once load/util_avg are null */ 9970 if (cfs_rq_has_blocked(cfs_rq)) 9971 *done = false; 9972 } 9973 9974 return decayed; 9975} 9976 9977/* 9978 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9979 * This needs to be done in a top-down fashion because the load of a child 9980 * group is a fraction of its parents load. 9981 */ 9982static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 9983{ 9984 struct rq *rq = rq_of(cfs_rq); 9985 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 9986 unsigned long now = jiffies; 9987 unsigned long load; 9988 9989 if (cfs_rq->last_h_load_update == now) 9990 return; 9991 9992 WRITE_ONCE(cfs_rq->h_load_next, NULL); 9993 for_each_sched_entity(se) { 9994 cfs_rq = cfs_rq_of(se); 9995 WRITE_ONCE(cfs_rq->h_load_next, se); 9996 if (cfs_rq->last_h_load_update == now) 9997 break; 9998 } 9999 10000 if (!se) { 10001 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 10002 cfs_rq->last_h_load_update = now; 10003 } 10004 10005 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 10006 load = cfs_rq->h_load; 10007 load = div64_ul(load * se->avg.load_avg, 10008 cfs_rq_load_avg(cfs_rq) + 1); 10009 cfs_rq = group_cfs_rq(se); 10010 cfs_rq->h_load = load; 10011 cfs_rq->last_h_load_update = now; 10012 } 10013} 10014 10015static unsigned long task_h_load(struct task_struct *p) 10016{ 10017 struct cfs_rq *cfs_rq = task_cfs_rq(p); 10018 10019 update_cfs_rq_h_load(cfs_rq); 10020 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 10021 cfs_rq_load_avg(cfs_rq) + 1); 10022} 10023#else /* !CONFIG_FAIR_GROUP_SCHED: */ 10024static bool __update_blocked_fair(struct rq *rq, bool *done) 10025{ 10026 struct cfs_rq *cfs_rq = &rq->cfs; 10027 bool decayed; 10028 10029 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 10030 if (cfs_rq_has_blocked(cfs_rq)) 10031 *done = false; 10032 10033 return decayed; 10034} 10035 10036static unsigned long task_h_load(struct task_struct *p) 10037{ 10038 return p->se.avg.load_avg; 10039} 10040#endif /* !CONFIG_FAIR_GROUP_SCHED */ 10041 10042static void sched_balance_update_blocked_averages(int cpu) 10043{ 10044 bool decayed = false, done = true; 10045 struct rq *rq = cpu_rq(cpu); 10046 struct rq_flags rf; 10047 10048 rq_lock_irqsave(rq, &rf); 10049 update_blocked_load_tick(rq); 10050 update_rq_clock(rq); 10051 10052 decayed |= __update_blocked_others(rq, &done); 10053 decayed |= __update_blocked_fair(rq, &done); 10054 10055 update_blocked_load_status(rq, !done); 10056 if (decayed) 10057 cpufreq_update_util(rq, 0); 10058 rq_unlock_irqrestore(rq, &rf); 10059} 10060 10061/********** Helpers for sched_balance_find_src_group ************************/ 10062 10063/* 10064 * sg_lb_stats - stats of a sched_group required for load-balancing: 10065 */ 10066struct sg_lb_stats { 10067 unsigned long avg_load; /* Avg load over the CPUs of the group */ 10068 unsigned long group_load; /* Total load over the CPUs of the group */ 10069 unsigned long group_capacity; /* Capacity over the CPUs of the group */ 10070 unsigned long group_util; /* Total utilization over the CPUs of the group */ 10071 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 10072 unsigned int sum_nr_running; /* Nr of all tasks running in the group */ 10073 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 10074 unsigned int idle_cpus; /* Nr of idle CPUs in the group */ 10075 unsigned int group_weight; 10076 enum group_type group_type; 10077 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 10078 unsigned int group_smt_balance; /* Task on busy SMT be moved */ 10079 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 10080#ifdef CONFIG_NUMA_BALANCING 10081 unsigned int nr_numa_running; 10082 unsigned int nr_preferred_running; 10083#endif 10084}; 10085 10086/* 10087 * sd_lb_stats - stats of a sched_domain required for load-balancing: 10088 */ 10089struct sd_lb_stats { 10090 struct sched_group *busiest; /* Busiest group in this sd */ 10091 struct sched_group *local; /* Local group in this sd */ 10092 unsigned long total_load; /* Total load of all groups in sd */ 10093 unsigned long total_capacity; /* Total capacity of all groups in sd */ 10094 unsigned long avg_load; /* Average load across all groups in sd */ 10095 unsigned int prefer_sibling; /* Tasks should go to sibling first */ 10096 10097 struct sg_lb_stats busiest_stat; /* Statistics of the busiest group */ 10098 struct sg_lb_stats local_stat; /* Statistics of the local group */ 10099}; 10100 10101static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 10102{ 10103 /* 10104 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 10105 * local_stat because update_sg_lb_stats() does a full clear/assignment. 10106 * We must however set busiest_stat::group_type and 10107 * busiest_stat::idle_cpus to the worst busiest group because 10108 * update_sd_pick_busiest() reads these before assignment. 10109 */ 10110 *sds = (struct sd_lb_stats){ 10111 .busiest = NULL, 10112 .local = NULL, 10113 .total_load = 0UL, 10114 .total_capacity = 0UL, 10115 .busiest_stat = { 10116 .idle_cpus = UINT_MAX, 10117 .group_type = group_has_spare, 10118 }, 10119 }; 10120} 10121 10122static unsigned long scale_rt_capacity(int cpu) 10123{ 10124 unsigned long max = get_actual_cpu_capacity(cpu); 10125 struct rq *rq = cpu_rq(cpu); 10126 unsigned long used, free; 10127 unsigned long irq; 10128 10129 irq = cpu_util_irq(rq); 10130 10131 if (unlikely(irq >= max)) 10132 return 1; 10133 10134 /* 10135 * avg_rt.util_avg and avg_dl.util_avg track binary signals 10136 * (running and not running) with weights 0 and 1024 respectively. 10137 */ 10138 used = cpu_util_rt(rq); 10139 used += cpu_util_dl(rq); 10140 10141 if (unlikely(used >= max)) 10142 return 1; 10143 10144 free = max - used; 10145 10146 return scale_irq_capacity(free, irq, max); 10147} 10148 10149static void update_cpu_capacity(struct sched_domain *sd, int cpu) 10150{ 10151 unsigned long capacity = scale_rt_capacity(cpu); 10152 struct sched_group *sdg = sd->groups; 10153 10154 if (!capacity) 10155 capacity = 1; 10156 10157 cpu_rq(cpu)->cpu_capacity = capacity; 10158 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 10159 10160 sdg->sgc->capacity = capacity; 10161 sdg->sgc->min_capacity = capacity; 10162 sdg->sgc->max_capacity = capacity; 10163} 10164 10165void update_group_capacity(struct sched_domain *sd, int cpu) 10166{ 10167 struct sched_domain *child = sd->child; 10168 struct sched_group *group, *sdg = sd->groups; 10169 unsigned long capacity, min_capacity, max_capacity; 10170 unsigned long interval; 10171 10172 interval = msecs_to_jiffies(sd->balance_interval); 10173 interval = clamp(interval, 1UL, max_load_balance_interval); 10174 sdg->sgc->next_update = jiffies + interval; 10175 10176 if (!child) { 10177 update_cpu_capacity(sd, cpu); 10178 return; 10179 } 10180 10181 capacity = 0; 10182 min_capacity = ULONG_MAX; 10183 max_capacity = 0; 10184 10185 if (child->flags & SD_NUMA) { 10186 /* 10187 * SD_NUMA domains cannot assume that child groups 10188 * span the current group. 10189 */ 10190 10191 for_each_cpu(cpu, sched_group_span(sdg)) { 10192 unsigned long cpu_cap = capacity_of(cpu); 10193 10194 capacity += cpu_cap; 10195 min_capacity = min(cpu_cap, min_capacity); 10196 max_capacity = max(cpu_cap, max_capacity); 10197 } 10198 } else { 10199 /* 10200 * !SD_NUMA domains can assume that child groups 10201 * span the current group. 10202 */ 10203 10204 group = child->groups; 10205 do { 10206 struct sched_group_capacity *sgc = group->sgc; 10207 10208 capacity += sgc->capacity; 10209 min_capacity = min(sgc->min_capacity, min_capacity); 10210 max_capacity = max(sgc->max_capacity, max_capacity); 10211 group = group->next; 10212 } while (group != child->groups); 10213 } 10214 10215 sdg->sgc->capacity = capacity; 10216 sdg->sgc->min_capacity = min_capacity; 10217 sdg->sgc->max_capacity = max_capacity; 10218} 10219 10220/* 10221 * Check whether the capacity of the rq has been noticeably reduced by side 10222 * activity. The imbalance_pct is used for the threshold. 10223 * Return true is the capacity is reduced 10224 */ 10225static inline int 10226check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 10227{ 10228 return ((rq->cpu_capacity * sd->imbalance_pct) < 10229 (arch_scale_cpu_capacity(cpu_of(rq)) * 100)); 10230} 10231 10232/* Check if the rq has a misfit task */ 10233static inline bool check_misfit_status(struct rq *rq) 10234{ 10235 return rq->misfit_task_load; 10236} 10237 10238/* 10239 * Group imbalance indicates (and tries to solve) the problem where balancing 10240 * groups is inadequate due to ->cpus_ptr constraints. 10241 * 10242 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 10243 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 10244 * Something like: 10245 * 10246 * { 0 1 2 3 } { 4 5 6 7 } 10247 * * * * * 10248 * 10249 * If we were to balance group-wise we'd place two tasks in the first group and 10250 * two tasks in the second group. Clearly this is undesired as it will overload 10251 * cpu 3 and leave one of the CPUs in the second group unused. 10252 * 10253 * The current solution to this issue is detecting the skew in the first group 10254 * by noticing the lower domain failed to reach balance and had difficulty 10255 * moving tasks due to affinity constraints. 10256 * 10257 * When this is so detected; this group becomes a candidate for busiest; see 10258 * update_sd_pick_busiest(). And calculate_imbalance() and 10259 * sched_balance_find_src_group() avoid some of the usual balance conditions to allow it 10260 * to create an effective group imbalance. 10261 * 10262 * This is a somewhat tricky proposition since the next run might not find the 10263 * group imbalance and decide the groups need to be balanced again. A most 10264 * subtle and fragile situation. 10265 */ 10266 10267static inline int sg_imbalanced(struct sched_group *group) 10268{ 10269 return group->sgc->imbalance; 10270} 10271 10272/* 10273 * group_has_capacity returns true if the group has spare capacity that could 10274 * be used by some tasks. 10275 * We consider that a group has spare capacity if the number of task is 10276 * smaller than the number of CPUs or if the utilization is lower than the 10277 * available capacity for CFS tasks. 10278 * For the latter, we use a threshold to stabilize the state, to take into 10279 * account the variance of the tasks' load and to return true if the available 10280 * capacity in meaningful for the load balancer. 10281 * As an example, an available capacity of 1% can appear but it doesn't make 10282 * any benefit for the load balance. 10283 */ 10284static inline bool 10285group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 10286{ 10287 if (sgs->sum_nr_running < sgs->group_weight) 10288 return true; 10289 10290 if ((sgs->group_capacity * imbalance_pct) < 10291 (sgs->group_runnable * 100)) 10292 return false; 10293 10294 if ((sgs->group_capacity * 100) > 10295 (sgs->group_util * imbalance_pct)) 10296 return true; 10297 10298 return false; 10299} 10300 10301/* 10302 * group_is_overloaded returns true if the group has more tasks than it can 10303 * handle. 10304 * group_is_overloaded is not equals to !group_has_capacity because a group 10305 * with the exact right number of tasks, has no more spare capacity but is not 10306 * overloaded so both group_has_capacity and group_is_overloaded return 10307 * false. 10308 */ 10309static inline bool 10310group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 10311{ 10312 if (sgs->sum_nr_running <= sgs->group_weight) 10313 return false; 10314 10315 if ((sgs->group_capacity * 100) < 10316 (sgs->group_util * imbalance_pct)) 10317 return true; 10318 10319 if ((sgs->group_capacity * imbalance_pct) < 10320 (sgs->group_runnable * 100)) 10321 return true; 10322 10323 return false; 10324} 10325 10326static inline enum 10327group_type group_classify(unsigned int imbalance_pct, 10328 struct sched_group *group, 10329 struct sg_lb_stats *sgs) 10330{ 10331 if (group_is_overloaded(imbalance_pct, sgs)) 10332 return group_overloaded; 10333 10334 if (sg_imbalanced(group)) 10335 return group_imbalanced; 10336 10337 if (sgs->group_asym_packing) 10338 return group_asym_packing; 10339 10340 if (sgs->group_smt_balance) 10341 return group_smt_balance; 10342 10343 if (sgs->group_misfit_task_load) 10344 return group_misfit_task; 10345 10346 if (!group_has_capacity(imbalance_pct, sgs)) 10347 return group_fully_busy; 10348 10349 return group_has_spare; 10350} 10351 10352/** 10353 * sched_use_asym_prio - Check whether asym_packing priority must be used 10354 * @sd: The scheduling domain of the load balancing 10355 * @cpu: A CPU 10356 * 10357 * Always use CPU priority when balancing load between SMT siblings. When 10358 * balancing load between cores, it is not sufficient that @cpu is idle. Only 10359 * use CPU priority if the whole core is idle. 10360 * 10361 * Returns: True if the priority of @cpu must be followed. False otherwise. 10362 */ 10363static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 10364{ 10365 if (!(sd->flags & SD_ASYM_PACKING)) 10366 return false; 10367 10368 if (!sched_smt_active()) 10369 return true; 10370 10371 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 10372} 10373 10374static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu) 10375{ 10376 /* 10377 * First check if @dst_cpu can do asym_packing load balance. Only do it 10378 * if it has higher priority than @src_cpu. 10379 */ 10380 return sched_use_asym_prio(sd, dst_cpu) && 10381 sched_asym_prefer(dst_cpu, src_cpu); 10382} 10383 10384/** 10385 * sched_group_asym - Check if the destination CPU can do asym_packing balance 10386 * @env: The load balancing environment 10387 * @sgs: Load-balancing statistics of the candidate busiest group 10388 * @group: The candidate busiest group 10389 * 10390 * @env::dst_cpu can do asym_packing if it has higher priority than the 10391 * preferred CPU of @group. 10392 * 10393 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 10394 * otherwise. 10395 */ 10396static inline bool 10397sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) 10398{ 10399 /* 10400 * CPU priorities do not make sense for SMT cores with more than one 10401 * busy sibling. 10402 */ 10403 if ((group->flags & SD_SHARE_CPUCAPACITY) && 10404 (sgs->group_weight - sgs->idle_cpus != 1)) 10405 return false; 10406 10407 return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu)); 10408} 10409 10410/* One group has more than one SMT CPU while the other group does not */ 10411static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1, 10412 struct sched_group *sg2) 10413{ 10414 if (!sg1 || !sg2) 10415 return false; 10416 10417 return (sg1->flags & SD_SHARE_CPUCAPACITY) != 10418 (sg2->flags & SD_SHARE_CPUCAPACITY); 10419} 10420 10421static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs, 10422 struct sched_group *group) 10423{ 10424 if (!env->idle) 10425 return false; 10426 10427 /* 10428 * For SMT source group, it is better to move a task 10429 * to a CPU that doesn't have multiple tasks sharing its CPU capacity. 10430 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY 10431 * will not be on. 10432 */ 10433 if (group->flags & SD_SHARE_CPUCAPACITY && 10434 sgs->sum_h_nr_running > 1) 10435 return true; 10436 10437 return false; 10438} 10439 10440static inline long sibling_imbalance(struct lb_env *env, 10441 struct sd_lb_stats *sds, 10442 struct sg_lb_stats *busiest, 10443 struct sg_lb_stats *local) 10444{ 10445 int ncores_busiest, ncores_local; 10446 long imbalance; 10447 10448 if (!env->idle || !busiest->sum_nr_running) 10449 return 0; 10450 10451 ncores_busiest = sds->busiest->cores; 10452 ncores_local = sds->local->cores; 10453 10454 if (ncores_busiest == ncores_local) { 10455 imbalance = busiest->sum_nr_running; 10456 lsub_positive(&imbalance, local->sum_nr_running); 10457 return imbalance; 10458 } 10459 10460 /* Balance such that nr_running/ncores ratio are same on both groups */ 10461 imbalance = ncores_local * busiest->sum_nr_running; 10462 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running); 10463 /* Normalize imbalance and do rounding on normalization */ 10464 imbalance = 2 * imbalance + ncores_local + ncores_busiest; 10465 imbalance /= ncores_local + ncores_busiest; 10466 10467 /* Take advantage of resource in an empty sched group */ 10468 if (imbalance <= 1 && local->sum_nr_running == 0 && 10469 busiest->sum_nr_running > 1) 10470 imbalance = 2; 10471 10472 return imbalance; 10473} 10474 10475static inline bool 10476sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 10477{ 10478 /* 10479 * When there is more than 1 task, the group_overloaded case already 10480 * takes care of cpu with reduced capacity 10481 */ 10482 if (rq->cfs.h_nr_runnable != 1) 10483 return false; 10484 10485 return check_cpu_capacity(rq, sd); 10486} 10487 10488/** 10489 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 10490 * @env: The load balancing environment. 10491 * @sds: Load-balancing data with statistics of the local group. 10492 * @group: sched_group whose statistics are to be updated. 10493 * @sgs: variable to hold the statistics for this group. 10494 * @sg_overloaded: sched_group is overloaded 10495 * @sg_overutilized: sched_group is overutilized 10496 */ 10497static inline void update_sg_lb_stats(struct lb_env *env, 10498 struct sd_lb_stats *sds, 10499 struct sched_group *group, 10500 struct sg_lb_stats *sgs, 10501 bool *sg_overloaded, 10502 bool *sg_overutilized) 10503{ 10504 int i, nr_running, local_group, sd_flags = env->sd->flags; 10505 bool balancing_at_rd = !env->sd->parent; 10506 10507 memset(sgs, 0, sizeof(*sgs)); 10508 10509 local_group = group == sds->local; 10510 10511 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 10512 struct rq *rq = cpu_rq(i); 10513 unsigned long load = cpu_load(rq); 10514 10515 sgs->group_load += load; 10516 sgs->group_util += cpu_util_cfs(i); 10517 sgs->group_runnable += cpu_runnable(rq); 10518 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable; 10519 10520 nr_running = rq->nr_running; 10521 sgs->sum_nr_running += nr_running; 10522 10523 if (cpu_overutilized(i)) 10524 *sg_overutilized = 1; 10525 10526 /* 10527 * No need to call idle_cpu() if nr_running is not 0 10528 */ 10529 if (!nr_running && idle_cpu(i)) { 10530 sgs->idle_cpus++; 10531 /* Idle cpu can't have misfit task */ 10532 continue; 10533 } 10534 10535 /* Overload indicator is only updated at root domain */ 10536 if (balancing_at_rd && nr_running > 1) 10537 *sg_overloaded = 1; 10538 10539#ifdef CONFIG_NUMA_BALANCING 10540 /* Only fbq_classify_group() uses this to classify NUMA groups */ 10541 if (sd_flags & SD_NUMA) { 10542 sgs->nr_numa_running += rq->nr_numa_running; 10543 sgs->nr_preferred_running += rq->nr_preferred_running; 10544 } 10545#endif 10546 if (local_group) 10547 continue; 10548 10549 if (sd_flags & SD_ASYM_CPUCAPACITY) { 10550 /* Check for a misfit task on the cpu */ 10551 if (sgs->group_misfit_task_load < rq->misfit_task_load) { 10552 sgs->group_misfit_task_load = rq->misfit_task_load; 10553 *sg_overloaded = 1; 10554 } 10555 } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { 10556 /* Check for a task running on a CPU with reduced capacity */ 10557 if (sgs->group_misfit_task_load < load) 10558 sgs->group_misfit_task_load = load; 10559 } 10560 } 10561 10562 sgs->group_capacity = group->sgc->capacity; 10563 10564 sgs->group_weight = group->group_weight; 10565 10566 /* Check if dst CPU is idle and preferred to this group */ 10567 if (!local_group && env->idle && sgs->sum_h_nr_running && 10568 sched_group_asym(env, sgs, group)) 10569 sgs->group_asym_packing = 1; 10570 10571 /* Check for loaded SMT group to be balanced to dst CPU */ 10572 if (!local_group && smt_balance(env, sgs, group)) 10573 sgs->group_smt_balance = 1; 10574 10575 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 10576 10577 /* Computing avg_load makes sense only when group is overloaded */ 10578 if (sgs->group_type == group_overloaded) 10579 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10580 sgs->group_capacity; 10581} 10582 10583/** 10584 * update_sd_pick_busiest - return 1 on busiest group 10585 * @env: The load balancing environment. 10586 * @sds: sched_domain statistics 10587 * @sg: sched_group candidate to be checked for being the busiest 10588 * @sgs: sched_group statistics 10589 * 10590 * Determine if @sg is a busier group than the previously selected 10591 * busiest group. 10592 * 10593 * Return: %true if @sg is a busier group than the previously selected 10594 * busiest group. %false otherwise. 10595 */ 10596static bool update_sd_pick_busiest(struct lb_env *env, 10597 struct sd_lb_stats *sds, 10598 struct sched_group *sg, 10599 struct sg_lb_stats *sgs) 10600{ 10601 struct sg_lb_stats *busiest = &sds->busiest_stat; 10602 10603 /* Make sure that there is at least one task to pull */ 10604 if (!sgs->sum_h_nr_running) 10605 return false; 10606 10607 /* 10608 * Don't try to pull misfit tasks we can't help. 10609 * We can use max_capacity here as reduction in capacity on some 10610 * CPUs in the group should either be possible to resolve 10611 * internally or be covered by avg_load imbalance (eventually). 10612 */ 10613 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10614 (sgs->group_type == group_misfit_task) && 10615 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 10616 sds->local_stat.group_type != group_has_spare)) 10617 return false; 10618 10619 if (sgs->group_type > busiest->group_type) 10620 return true; 10621 10622 if (sgs->group_type < busiest->group_type) 10623 return false; 10624 10625 /* 10626 * The candidate and the current busiest group are the same type of 10627 * group. Let check which one is the busiest according to the type. 10628 */ 10629 10630 switch (sgs->group_type) { 10631 case group_overloaded: 10632 /* Select the overloaded group with highest avg_load. */ 10633 return sgs->avg_load > busiest->avg_load; 10634 10635 case group_imbalanced: 10636 /* 10637 * Select the 1st imbalanced group as we don't have any way to 10638 * choose one more than another. 10639 */ 10640 return false; 10641 10642 case group_asym_packing: 10643 /* Prefer to move from lowest priority CPU's work */ 10644 return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu), 10645 READ_ONCE(sg->asym_prefer_cpu)); 10646 10647 case group_misfit_task: 10648 /* 10649 * If we have more than one misfit sg go with the biggest 10650 * misfit. 10651 */ 10652 return sgs->group_misfit_task_load > busiest->group_misfit_task_load; 10653 10654 case group_smt_balance: 10655 /* 10656 * Check if we have spare CPUs on either SMT group to 10657 * choose has spare or fully busy handling. 10658 */ 10659 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0) 10660 goto has_spare; 10661 10662 fallthrough; 10663 10664 case group_fully_busy: 10665 /* 10666 * Select the fully busy group with highest avg_load. In 10667 * theory, there is no need to pull task from such kind of 10668 * group because tasks have all compute capacity that they need 10669 * but we can still improve the overall throughput by reducing 10670 * contention when accessing shared HW resources. 10671 * 10672 * XXX for now avg_load is not computed and always 0 so we 10673 * select the 1st one, except if @sg is composed of SMT 10674 * siblings. 10675 */ 10676 10677 if (sgs->avg_load < busiest->avg_load) 10678 return false; 10679 10680 if (sgs->avg_load == busiest->avg_load) { 10681 /* 10682 * SMT sched groups need more help than non-SMT groups. 10683 * If @sg happens to also be SMT, either choice is good. 10684 */ 10685 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 10686 return false; 10687 } 10688 10689 break; 10690 10691 case group_has_spare: 10692 /* 10693 * Do not pick sg with SMT CPUs over sg with pure CPUs, 10694 * as we do not want to pull task off SMT core with one task 10695 * and make the core idle. 10696 */ 10697 if (smt_vs_nonsmt_groups(sds->busiest, sg)) { 10698 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1) 10699 return false; 10700 else 10701 return true; 10702 } 10703has_spare: 10704 10705 /* 10706 * Select not overloaded group with lowest number of idle CPUs 10707 * and highest number of running tasks. We could also compare 10708 * the spare capacity which is more stable but it can end up 10709 * that the group has less spare capacity but finally more idle 10710 * CPUs which means less opportunity to pull tasks. 10711 */ 10712 if (sgs->idle_cpus > busiest->idle_cpus) 10713 return false; 10714 else if ((sgs->idle_cpus == busiest->idle_cpus) && 10715 (sgs->sum_nr_running <= busiest->sum_nr_running)) 10716 return false; 10717 10718 break; 10719 } 10720 10721 /* 10722 * Candidate sg has no more than one task per CPU and has higher 10723 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 10724 * throughput. Maximize throughput, power/energy consequences are not 10725 * considered. 10726 */ 10727 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10728 (sgs->group_type <= group_fully_busy) && 10729 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 10730 return false; 10731 10732 return true; 10733} 10734 10735#ifdef CONFIG_NUMA_BALANCING 10736static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10737{ 10738 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 10739 return regular; 10740 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 10741 return remote; 10742 return all; 10743} 10744 10745static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10746{ 10747 if (rq->nr_running > rq->nr_numa_running) 10748 return regular; 10749 if (rq->nr_running > rq->nr_preferred_running) 10750 return remote; 10751 return all; 10752} 10753#else /* !CONFIG_NUMA_BALANCING: */ 10754static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10755{ 10756 return all; 10757} 10758 10759static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10760{ 10761 return regular; 10762} 10763#endif /* !CONFIG_NUMA_BALANCING */ 10764 10765 10766struct sg_lb_stats; 10767 10768/* 10769 * task_running_on_cpu - return 1 if @p is running on @cpu. 10770 */ 10771 10772static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 10773{ 10774 /* Task has no contribution or is new */ 10775 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 10776 return 0; 10777 10778 if (task_on_rq_queued(p)) 10779 return 1; 10780 10781 return 0; 10782} 10783 10784/** 10785 * idle_cpu_without - would a given CPU be idle without p ? 10786 * @cpu: the processor on which idleness is tested. 10787 * @p: task which should be ignored. 10788 * 10789 * Return: 1 if the CPU would be idle. 0 otherwise. 10790 */ 10791static int idle_cpu_without(int cpu, struct task_struct *p) 10792{ 10793 struct rq *rq = cpu_rq(cpu); 10794 10795 if (rq->curr != rq->idle && rq->curr != p) 10796 return 0; 10797 10798 /* 10799 * rq->nr_running can't be used but an updated version without the 10800 * impact of p on cpu must be used instead. The updated nr_running 10801 * be computed and tested before calling idle_cpu_without(). 10802 */ 10803 10804 if (rq->ttwu_pending) 10805 return 0; 10806 10807 return 1; 10808} 10809 10810/* 10811 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 10812 * @sd: The sched_domain level to look for idlest group. 10813 * @group: sched_group whose statistics are to be updated. 10814 * @sgs: variable to hold the statistics for this group. 10815 * @p: The task for which we look for the idlest group/CPU. 10816 */ 10817static inline void update_sg_wakeup_stats(struct sched_domain *sd, 10818 struct sched_group *group, 10819 struct sg_lb_stats *sgs, 10820 struct task_struct *p) 10821{ 10822 int i, nr_running; 10823 10824 memset(sgs, 0, sizeof(*sgs)); 10825 10826 /* Assume that task can't fit any CPU of the group */ 10827 if (sd->flags & SD_ASYM_CPUCAPACITY) 10828 sgs->group_misfit_task_load = 1; 10829 10830 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 10831 struct rq *rq = cpu_rq(i); 10832 unsigned int local; 10833 10834 sgs->group_load += cpu_load_without(rq, p); 10835 sgs->group_util += cpu_util_without(i, p); 10836 sgs->group_runnable += cpu_runnable_without(rq, p); 10837 local = task_running_on_cpu(i, p); 10838 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable - local; 10839 10840 nr_running = rq->nr_running - local; 10841 sgs->sum_nr_running += nr_running; 10842 10843 /* 10844 * No need to call idle_cpu_without() if nr_running is not 0 10845 */ 10846 if (!nr_running && idle_cpu_without(i, p)) 10847 sgs->idle_cpus++; 10848 10849 /* Check if task fits in the CPU */ 10850 if (sd->flags & SD_ASYM_CPUCAPACITY && 10851 sgs->group_misfit_task_load && 10852 task_fits_cpu(p, i)) 10853 sgs->group_misfit_task_load = 0; 10854 10855 } 10856 10857 sgs->group_capacity = group->sgc->capacity; 10858 10859 sgs->group_weight = group->group_weight; 10860 10861 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 10862 10863 /* 10864 * Computing avg_load makes sense only when group is fully busy or 10865 * overloaded 10866 */ 10867 if (sgs->group_type == group_fully_busy || 10868 sgs->group_type == group_overloaded) 10869 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10870 sgs->group_capacity; 10871} 10872 10873static bool update_pick_idlest(struct sched_group *idlest, 10874 struct sg_lb_stats *idlest_sgs, 10875 struct sched_group *group, 10876 struct sg_lb_stats *sgs) 10877{ 10878 if (sgs->group_type < idlest_sgs->group_type) 10879 return true; 10880 10881 if (sgs->group_type > idlest_sgs->group_type) 10882 return false; 10883 10884 /* 10885 * The candidate and the current idlest group are the same type of 10886 * group. Let check which one is the idlest according to the type. 10887 */ 10888 10889 switch (sgs->group_type) { 10890 case group_overloaded: 10891 case group_fully_busy: 10892 /* Select the group with lowest avg_load. */ 10893 if (idlest_sgs->avg_load <= sgs->avg_load) 10894 return false; 10895 break; 10896 10897 case group_imbalanced: 10898 case group_asym_packing: 10899 case group_smt_balance: 10900 /* Those types are not used in the slow wakeup path */ 10901 return false; 10902 10903 case group_misfit_task: 10904 /* Select group with the highest max capacity */ 10905 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 10906 return false; 10907 break; 10908 10909 case group_has_spare: 10910 /* Select group with most idle CPUs */ 10911 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 10912 return false; 10913 10914 /* Select group with lowest group_util */ 10915 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 10916 idlest_sgs->group_util <= sgs->group_util) 10917 return false; 10918 10919 break; 10920 } 10921 10922 return true; 10923} 10924 10925/* 10926 * sched_balance_find_dst_group() finds and returns the least busy CPU group within the 10927 * domain. 10928 * 10929 * Assumes p is allowed on at least one CPU in sd. 10930 */ 10931static struct sched_group * 10932sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 10933{ 10934 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 10935 struct sg_lb_stats local_sgs, tmp_sgs; 10936 struct sg_lb_stats *sgs; 10937 unsigned long imbalance; 10938 struct sg_lb_stats idlest_sgs = { 10939 .avg_load = UINT_MAX, 10940 .group_type = group_overloaded, 10941 }; 10942 10943 do { 10944 int local_group; 10945 10946 /* Skip over this group if it has no CPUs allowed */ 10947 if (!cpumask_intersects(sched_group_span(group), 10948 p->cpus_ptr)) 10949 continue; 10950 10951 /* Skip over this group if no cookie matched */ 10952 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 10953 continue; 10954 10955 local_group = cpumask_test_cpu(this_cpu, 10956 sched_group_span(group)); 10957 10958 if (local_group) { 10959 sgs = &local_sgs; 10960 local = group; 10961 } else { 10962 sgs = &tmp_sgs; 10963 } 10964 10965 update_sg_wakeup_stats(sd, group, sgs, p); 10966 10967 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 10968 idlest = group; 10969 idlest_sgs = *sgs; 10970 } 10971 10972 } while (group = group->next, group != sd->groups); 10973 10974 10975 /* There is no idlest group to push tasks to */ 10976 if (!idlest) 10977 return NULL; 10978 10979 /* The local group has been skipped because of CPU affinity */ 10980 if (!local) 10981 return idlest; 10982 10983 /* 10984 * If the local group is idler than the selected idlest group 10985 * don't try and push the task. 10986 */ 10987 if (local_sgs.group_type < idlest_sgs.group_type) 10988 return NULL; 10989 10990 /* 10991 * If the local group is busier than the selected idlest group 10992 * try and push the task. 10993 */ 10994 if (local_sgs.group_type > idlest_sgs.group_type) 10995 return idlest; 10996 10997 switch (local_sgs.group_type) { 10998 case group_overloaded: 10999 case group_fully_busy: 11000 11001 /* Calculate allowed imbalance based on load */ 11002 imbalance = scale_load_down(NICE_0_LOAD) * 11003 (sd->imbalance_pct-100) / 100; 11004 11005 /* 11006 * When comparing groups across NUMA domains, it's possible for 11007 * the local domain to be very lightly loaded relative to the 11008 * remote domains but "imbalance" skews the comparison making 11009 * remote CPUs look much more favourable. When considering 11010 * cross-domain, add imbalance to the load on the remote node 11011 * and consider staying local. 11012 */ 11013 11014 if ((sd->flags & SD_NUMA) && 11015 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 11016 return NULL; 11017 11018 /* 11019 * If the local group is less loaded than the selected 11020 * idlest group don't try and push any tasks. 11021 */ 11022 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 11023 return NULL; 11024 11025 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 11026 return NULL; 11027 break; 11028 11029 case group_imbalanced: 11030 case group_asym_packing: 11031 case group_smt_balance: 11032 /* Those type are not used in the slow wakeup path */ 11033 return NULL; 11034 11035 case group_misfit_task: 11036 /* Select group with the highest max capacity */ 11037 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 11038 return NULL; 11039 break; 11040 11041 case group_has_spare: 11042#ifdef CONFIG_NUMA 11043 if (sd->flags & SD_NUMA) { 11044 int imb_numa_nr = sd->imb_numa_nr; 11045#ifdef CONFIG_NUMA_BALANCING 11046 int idlest_cpu; 11047 /* 11048 * If there is spare capacity at NUMA, try to select 11049 * the preferred node 11050 */ 11051 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 11052 return NULL; 11053 11054 idlest_cpu = cpumask_first(sched_group_span(idlest)); 11055 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 11056 return idlest; 11057#endif /* CONFIG_NUMA_BALANCING */ 11058 /* 11059 * Otherwise, keep the task close to the wakeup source 11060 * and improve locality if the number of running tasks 11061 * would remain below threshold where an imbalance is 11062 * allowed while accounting for the possibility the 11063 * task is pinned to a subset of CPUs. If there is a 11064 * real need of migration, periodic load balance will 11065 * take care of it. 11066 */ 11067 if (p->nr_cpus_allowed != NR_CPUS) { 11068 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 11069 11070 cpumask_and(cpus, sched_group_span(local), p->cpus_ptr); 11071 imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr); 11072 } 11073 11074 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 11075 if (!adjust_numa_imbalance(imbalance, 11076 local_sgs.sum_nr_running + 1, 11077 imb_numa_nr)) { 11078 return NULL; 11079 } 11080 } 11081#endif /* CONFIG_NUMA */ 11082 11083 /* 11084 * Select group with highest number of idle CPUs. We could also 11085 * compare the utilization which is more stable but it can end 11086 * up that the group has less spare capacity but finally more 11087 * idle CPUs which means more opportunity to run task. 11088 */ 11089 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 11090 return NULL; 11091 break; 11092 } 11093 11094 return idlest; 11095} 11096 11097static void update_idle_cpu_scan(struct lb_env *env, 11098 unsigned long sum_util) 11099{ 11100 struct sched_domain_shared *sd_share; 11101 int llc_weight, pct; 11102 u64 x, y, tmp; 11103 /* 11104 * Update the number of CPUs to scan in LLC domain, which could 11105 * be used as a hint in select_idle_cpu(). The update of sd_share 11106 * could be expensive because it is within a shared cache line. 11107 * So the write of this hint only occurs during periodic load 11108 * balancing, rather than CPU_NEWLY_IDLE, because the latter 11109 * can fire way more frequently than the former. 11110 */ 11111 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 11112 return; 11113 11114 llc_weight = per_cpu(sd_llc_size, env->dst_cpu); 11115 if (env->sd->span_weight != llc_weight) 11116 return; 11117 11118 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); 11119 if (!sd_share) 11120 return; 11121 11122 /* 11123 * The number of CPUs to search drops as sum_util increases, when 11124 * sum_util hits 85% or above, the scan stops. 11125 * The reason to choose 85% as the threshold is because this is the 11126 * imbalance_pct(117) when a LLC sched group is overloaded. 11127 * 11128 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 11129 * and y'= y / SCHED_CAPACITY_SCALE 11130 * 11131 * x is the ratio of sum_util compared to the CPU capacity: 11132 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 11133 * y' is the ratio of CPUs to be scanned in the LLC domain, 11134 * and the number of CPUs to scan is calculated by: 11135 * 11136 * nr_scan = llc_weight * y' [2] 11137 * 11138 * When x hits the threshold of overloaded, AKA, when 11139 * x = 100 / pct, y drops to 0. According to [1], 11140 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 11141 * 11142 * Scale x by SCHED_CAPACITY_SCALE: 11143 * x' = sum_util / llc_weight; [3] 11144 * 11145 * and finally [1] becomes: 11146 * y = SCHED_CAPACITY_SCALE - 11147 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 11148 * 11149 */ 11150 /* equation [3] */ 11151 x = sum_util; 11152 do_div(x, llc_weight); 11153 11154 /* equation [4] */ 11155 pct = env->sd->imbalance_pct; 11156 tmp = x * x * pct * pct; 11157 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 11158 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 11159 y = SCHED_CAPACITY_SCALE - tmp; 11160 11161 /* equation [2] */ 11162 y *= llc_weight; 11163 do_div(y, SCHED_CAPACITY_SCALE); 11164 if ((int)y != sd_share->nr_idle_scan) 11165 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 11166} 11167 11168/** 11169 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 11170 * @env: The load balancing environment. 11171 * @sds: variable to hold the statistics for this sched_domain. 11172 */ 11173 11174static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 11175{ 11176 struct sched_group *sg = env->sd->groups; 11177 struct sg_lb_stats *local = &sds->local_stat; 11178 struct sg_lb_stats tmp_sgs; 11179 unsigned long sum_util = 0; 11180 bool sg_overloaded = 0, sg_overutilized = 0; 11181 11182 do { 11183 struct sg_lb_stats *sgs = &tmp_sgs; 11184 int local_group; 11185 11186 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 11187 if (local_group) { 11188 sds->local = sg; 11189 sgs = local; 11190 11191 if (env->idle != CPU_NEWLY_IDLE || 11192 time_after_eq(jiffies, sg->sgc->next_update)) 11193 update_group_capacity(env->sd, env->dst_cpu); 11194 } 11195 11196 update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized); 11197 11198 if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) { 11199 sds->busiest = sg; 11200 sds->busiest_stat = *sgs; 11201 } 11202 11203 /* Now, start updating sd_lb_stats */ 11204 sds->total_load += sgs->group_load; 11205 sds->total_capacity += sgs->group_capacity; 11206 11207 sum_util += sgs->group_util; 11208 sg = sg->next; 11209 } while (sg != env->sd->groups); 11210 11211 /* 11212 * Indicate that the child domain of the busiest group prefers tasks 11213 * go to a child's sibling domains first. NB the flags of a sched group 11214 * are those of the child domain. 11215 */ 11216 if (sds->busiest) 11217 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 11218 11219 11220 if (env->sd->flags & SD_NUMA) 11221 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 11222 11223 if (!env->sd->parent) { 11224 /* update overload indicator if we are at root domain */ 11225 set_rd_overloaded(env->dst_rq->rd, sg_overloaded); 11226 11227 /* Update over-utilization (tipping point, U >= 0) indicator */ 11228 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 11229 } else if (sg_overutilized) { 11230 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 11231 } 11232 11233 update_idle_cpu_scan(env, sum_util); 11234} 11235 11236/** 11237 * calculate_imbalance - Calculate the amount of imbalance present within the 11238 * groups of a given sched_domain during load balance. 11239 * @env: load balance environment 11240 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 11241 */ 11242static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 11243{ 11244 struct sg_lb_stats *local, *busiest; 11245 11246 local = &sds->local_stat; 11247 busiest = &sds->busiest_stat; 11248 11249 if (busiest->group_type == group_misfit_task) { 11250 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 11251 /* Set imbalance to allow misfit tasks to be balanced. */ 11252 env->migration_type = migrate_misfit; 11253 env->imbalance = 1; 11254 } else { 11255 /* 11256 * Set load imbalance to allow moving task from cpu 11257 * with reduced capacity. 11258 */ 11259 env->migration_type = migrate_load; 11260 env->imbalance = busiest->group_misfit_task_load; 11261 } 11262 return; 11263 } 11264 11265 if (busiest->group_type == group_asym_packing) { 11266 /* 11267 * In case of asym capacity, we will try to migrate all load to 11268 * the preferred CPU. 11269 */ 11270 env->migration_type = migrate_task; 11271 env->imbalance = busiest->sum_h_nr_running; 11272 return; 11273 } 11274 11275 if (busiest->group_type == group_smt_balance) { 11276 /* Reduce number of tasks sharing CPU capacity */ 11277 env->migration_type = migrate_task; 11278 env->imbalance = 1; 11279 return; 11280 } 11281 11282 if (busiest->group_type == group_imbalanced) { 11283 /* 11284 * In the group_imb case we cannot rely on group-wide averages 11285 * to ensure CPU-load equilibrium, try to move any task to fix 11286 * the imbalance. The next load balance will take care of 11287 * balancing back the system. 11288 */ 11289 env->migration_type = migrate_task; 11290 env->imbalance = 1; 11291 return; 11292 } 11293 11294 /* 11295 * Try to use spare capacity of local group without overloading it or 11296 * emptying busiest. 11297 */ 11298 if (local->group_type == group_has_spare) { 11299 if ((busiest->group_type > group_fully_busy) && 11300 !(env->sd->flags & SD_SHARE_LLC)) { 11301 /* 11302 * If busiest is overloaded, try to fill spare 11303 * capacity. This might end up creating spare capacity 11304 * in busiest or busiest still being overloaded but 11305 * there is no simple way to directly compute the 11306 * amount of load to migrate in order to balance the 11307 * system. 11308 */ 11309 env->migration_type = migrate_util; 11310 env->imbalance = max(local->group_capacity, local->group_util) - 11311 local->group_util; 11312 11313 /* 11314 * In some cases, the group's utilization is max or even 11315 * higher than capacity because of migrations but the 11316 * local CPU is (newly) idle. There is at least one 11317 * waiting task in this overloaded busiest group. Let's 11318 * try to pull it. 11319 */ 11320 if (env->idle && env->imbalance == 0) { 11321 env->migration_type = migrate_task; 11322 env->imbalance = 1; 11323 } 11324 11325 return; 11326 } 11327 11328 if (busiest->group_weight == 1 || sds->prefer_sibling) { 11329 /* 11330 * When prefer sibling, evenly spread running tasks on 11331 * groups. 11332 */ 11333 env->migration_type = migrate_task; 11334 env->imbalance = sibling_imbalance(env, sds, busiest, local); 11335 } else { 11336 11337 /* 11338 * If there is no overload, we just want to even the number of 11339 * idle CPUs. 11340 */ 11341 env->migration_type = migrate_task; 11342 env->imbalance = max_t(long, 0, 11343 (local->idle_cpus - busiest->idle_cpus)); 11344 } 11345 11346#ifdef CONFIG_NUMA 11347 /* Consider allowing a small imbalance between NUMA groups */ 11348 if (env->sd->flags & SD_NUMA) { 11349 env->imbalance = adjust_numa_imbalance(env->imbalance, 11350 local->sum_nr_running + 1, 11351 env->sd->imb_numa_nr); 11352 } 11353#endif 11354 11355 /* Number of tasks to move to restore balance */ 11356 env->imbalance >>= 1; 11357 11358 return; 11359 } 11360 11361 /* 11362 * Local is fully busy but has to take more load to relieve the 11363 * busiest group 11364 */ 11365 if (local->group_type < group_overloaded) { 11366 /* 11367 * Local will become overloaded so the avg_load metrics are 11368 * finally needed. 11369 */ 11370 11371 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 11372 local->group_capacity; 11373 11374 /* 11375 * If the local group is more loaded than the selected 11376 * busiest group don't try to pull any tasks. 11377 */ 11378 if (local->avg_load >= busiest->avg_load) { 11379 env->imbalance = 0; 11380 return; 11381 } 11382 11383 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 11384 sds->total_capacity; 11385 11386 /* 11387 * If the local group is more loaded than the average system 11388 * load, don't try to pull any tasks. 11389 */ 11390 if (local->avg_load >= sds->avg_load) { 11391 env->imbalance = 0; 11392 return; 11393 } 11394 11395 } 11396 11397 /* 11398 * Both group are or will become overloaded and we're trying to get all 11399 * the CPUs to the average_load, so we don't want to push ourselves 11400 * above the average load, nor do we wish to reduce the max loaded CPU 11401 * below the average load. At the same time, we also don't want to 11402 * reduce the group load below the group capacity. Thus we look for 11403 * the minimum possible imbalance. 11404 */ 11405 env->migration_type = migrate_load; 11406 env->imbalance = min( 11407 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 11408 (sds->avg_load - local->avg_load) * local->group_capacity 11409 ) / SCHED_CAPACITY_SCALE; 11410} 11411 11412/******* sched_balance_find_src_group() helpers end here *********************/ 11413 11414/* 11415 * Decision matrix according to the local and busiest group type: 11416 * 11417 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 11418 * has_spare nr_idle balanced N/A N/A balanced balanced 11419 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 11420 * misfit_task force N/A N/A N/A N/A N/A 11421 * asym_packing force force N/A N/A force force 11422 * imbalanced force force N/A N/A force force 11423 * overloaded force force N/A N/A force avg_load 11424 * 11425 * N/A : Not Applicable because already filtered while updating 11426 * statistics. 11427 * balanced : The system is balanced for these 2 groups. 11428 * force : Calculate the imbalance as load migration is probably needed. 11429 * avg_load : Only if imbalance is significant enough. 11430 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 11431 * different in groups. 11432 */ 11433 11434/** 11435 * sched_balance_find_src_group - Returns the busiest group within the sched_domain 11436 * if there is an imbalance. 11437 * @env: The load balancing environment. 11438 * 11439 * Also calculates the amount of runnable load which should be moved 11440 * to restore balance. 11441 * 11442 * Return: - The busiest group if imbalance exists. 11443 */ 11444static struct sched_group *sched_balance_find_src_group(struct lb_env *env) 11445{ 11446 struct sg_lb_stats *local, *busiest; 11447 struct sd_lb_stats sds; 11448 11449 init_sd_lb_stats(&sds); 11450 11451 /* 11452 * Compute the various statistics relevant for load balancing at 11453 * this level. 11454 */ 11455 update_sd_lb_stats(env, &sds); 11456 11457 /* There is no busy sibling group to pull tasks from */ 11458 if (!sds.busiest) 11459 goto out_balanced; 11460 11461 busiest = &sds.busiest_stat; 11462 11463 /* Misfit tasks should be dealt with regardless of the avg load */ 11464 if (busiest->group_type == group_misfit_task) 11465 goto force_balance; 11466 11467 if (!is_rd_overutilized(env->dst_rq->rd) && 11468 rcu_dereference(env->dst_rq->rd->pd)) 11469 goto out_balanced; 11470 11471 /* ASYM feature bypasses nice load balance check */ 11472 if (busiest->group_type == group_asym_packing) 11473 goto force_balance; 11474 11475 /* 11476 * If the busiest group is imbalanced the below checks don't 11477 * work because they assume all things are equal, which typically 11478 * isn't true due to cpus_ptr constraints and the like. 11479 */ 11480 if (busiest->group_type == group_imbalanced) 11481 goto force_balance; 11482 11483 local = &sds.local_stat; 11484 /* 11485 * If the local group is busier than the selected busiest group 11486 * don't try and pull any tasks. 11487 */ 11488 if (local->group_type > busiest->group_type) 11489 goto out_balanced; 11490 11491 /* 11492 * When groups are overloaded, use the avg_load to ensure fairness 11493 * between tasks. 11494 */ 11495 if (local->group_type == group_overloaded) { 11496 /* 11497 * If the local group is more loaded than the selected 11498 * busiest group don't try to pull any tasks. 11499 */ 11500 if (local->avg_load >= busiest->avg_load) 11501 goto out_balanced; 11502 11503 /* XXX broken for overlapping NUMA groups */ 11504 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 11505 sds.total_capacity; 11506 11507 /* 11508 * Don't pull any tasks if this group is already above the 11509 * domain average load. 11510 */ 11511 if (local->avg_load >= sds.avg_load) 11512 goto out_balanced; 11513 11514 /* 11515 * If the busiest group is more loaded, use imbalance_pct to be 11516 * conservative. 11517 */ 11518 if (100 * busiest->avg_load <= 11519 env->sd->imbalance_pct * local->avg_load) 11520 goto out_balanced; 11521 } 11522 11523 /* 11524 * Try to move all excess tasks to a sibling domain of the busiest 11525 * group's child domain. 11526 */ 11527 if (sds.prefer_sibling && local->group_type == group_has_spare && 11528 sibling_imbalance(env, &sds, busiest, local) > 1) 11529 goto force_balance; 11530 11531 if (busiest->group_type != group_overloaded) { 11532 if (!env->idle) { 11533 /* 11534 * If the busiest group is not overloaded (and as a 11535 * result the local one too) but this CPU is already 11536 * busy, let another idle CPU try to pull task. 11537 */ 11538 goto out_balanced; 11539 } 11540 11541 if (busiest->group_type == group_smt_balance && 11542 smt_vs_nonsmt_groups(sds.local, sds.busiest)) { 11543 /* Let non SMT CPU pull from SMT CPU sharing with sibling */ 11544 goto force_balance; 11545 } 11546 11547 if (busiest->group_weight > 1 && 11548 local->idle_cpus <= (busiest->idle_cpus + 1)) { 11549 /* 11550 * If the busiest group is not overloaded 11551 * and there is no imbalance between this and busiest 11552 * group wrt idle CPUs, it is balanced. The imbalance 11553 * becomes significant if the diff is greater than 1 11554 * otherwise we might end up to just move the imbalance 11555 * on another group. Of course this applies only if 11556 * there is more than 1 CPU per group. 11557 */ 11558 goto out_balanced; 11559 } 11560 11561 if (busiest->sum_h_nr_running == 1) { 11562 /* 11563 * busiest doesn't have any tasks waiting to run 11564 */ 11565 goto out_balanced; 11566 } 11567 } 11568 11569force_balance: 11570 /* Looks like there is an imbalance. Compute it */ 11571 calculate_imbalance(env, &sds); 11572 return env->imbalance ? sds.busiest : NULL; 11573 11574out_balanced: 11575 env->imbalance = 0; 11576 return NULL; 11577} 11578 11579/* 11580 * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group. 11581 */ 11582static struct rq *sched_balance_find_src_rq(struct lb_env *env, 11583 struct sched_group *group) 11584{ 11585 struct rq *busiest = NULL, *rq; 11586 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 11587 unsigned int busiest_nr = 0; 11588 int i; 11589 11590 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 11591 unsigned long capacity, load, util; 11592 unsigned int nr_running; 11593 enum fbq_type rt; 11594 11595 rq = cpu_rq(i); 11596 rt = fbq_classify_rq(rq); 11597 11598 /* 11599 * We classify groups/runqueues into three groups: 11600 * - regular: there are !numa tasks 11601 * - remote: there are numa tasks that run on the 'wrong' node 11602 * - all: there is no distinction 11603 * 11604 * In order to avoid migrating ideally placed numa tasks, 11605 * ignore those when there's better options. 11606 * 11607 * If we ignore the actual busiest queue to migrate another 11608 * task, the next balance pass can still reduce the busiest 11609 * queue by moving tasks around inside the node. 11610 * 11611 * If we cannot move enough load due to this classification 11612 * the next pass will adjust the group classification and 11613 * allow migration of more tasks. 11614 * 11615 * Both cases only affect the total convergence complexity. 11616 */ 11617 if (rt > env->fbq_type) 11618 continue; 11619 11620 nr_running = rq->cfs.h_nr_runnable; 11621 if (!nr_running) 11622 continue; 11623 11624 capacity = capacity_of(i); 11625 11626 /* 11627 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 11628 * eventually lead to active_balancing high->low capacity. 11629 * Higher per-CPU capacity is considered better than balancing 11630 * average load. 11631 */ 11632 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 11633 !capacity_greater(capacity_of(env->dst_cpu), capacity) && 11634 nr_running == 1) 11635 continue; 11636 11637 /* 11638 * Make sure we only pull tasks from a CPU of lower priority 11639 * when balancing between SMT siblings. 11640 * 11641 * If balancing between cores, let lower priority CPUs help 11642 * SMT cores with more than one busy sibling. 11643 */ 11644 if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1) 11645 continue; 11646 11647 switch (env->migration_type) { 11648 case migrate_load: 11649 /* 11650 * When comparing with load imbalance, use cpu_load() 11651 * which is not scaled with the CPU capacity. 11652 */ 11653 load = cpu_load(rq); 11654 11655 if (nr_running == 1 && load > env->imbalance && 11656 !check_cpu_capacity(rq, env->sd)) 11657 break; 11658 11659 /* 11660 * For the load comparisons with the other CPUs, 11661 * consider the cpu_load() scaled with the CPU 11662 * capacity, so that the load can be moved away 11663 * from the CPU that is potentially running at a 11664 * lower capacity. 11665 * 11666 * Thus we're looking for max(load_i / capacity_i), 11667 * crosswise multiplication to rid ourselves of the 11668 * division works out to: 11669 * load_i * capacity_j > load_j * capacity_i; 11670 * where j is our previous maximum. 11671 */ 11672 if (load * busiest_capacity > busiest_load * capacity) { 11673 busiest_load = load; 11674 busiest_capacity = capacity; 11675 busiest = rq; 11676 } 11677 break; 11678 11679 case migrate_util: 11680 util = cpu_util_cfs_boost(i); 11681 11682 /* 11683 * Don't try to pull utilization from a CPU with one 11684 * running task. Whatever its utilization, we will fail 11685 * detach the task. 11686 */ 11687 if (nr_running <= 1) 11688 continue; 11689 11690 if (busiest_util < util) { 11691 busiest_util = util; 11692 busiest = rq; 11693 } 11694 break; 11695 11696 case migrate_task: 11697 if (busiest_nr < nr_running) { 11698 busiest_nr = nr_running; 11699 busiest = rq; 11700 } 11701 break; 11702 11703 case migrate_misfit: 11704 /* 11705 * For ASYM_CPUCAPACITY domains with misfit tasks we 11706 * simply seek the "biggest" misfit task. 11707 */ 11708 if (rq->misfit_task_load > busiest_load) { 11709 busiest_load = rq->misfit_task_load; 11710 busiest = rq; 11711 } 11712 11713 break; 11714 11715 } 11716 } 11717 11718 return busiest; 11719} 11720 11721/* 11722 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 11723 * so long as it is large enough. 11724 */ 11725#define MAX_PINNED_INTERVAL 512 11726 11727static inline bool 11728asym_active_balance(struct lb_env *env) 11729{ 11730 /* 11731 * ASYM_PACKING needs to force migrate tasks from busy but lower 11732 * priority CPUs in order to pack all tasks in the highest priority 11733 * CPUs. When done between cores, do it only if the whole core if the 11734 * whole core is idle. 11735 * 11736 * If @env::src_cpu is an SMT core with busy siblings, let 11737 * the lower priority @env::dst_cpu help it. Do not follow 11738 * CPU priority. 11739 */ 11740 return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) && 11741 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 11742 !sched_use_asym_prio(env->sd, env->src_cpu)); 11743} 11744 11745static inline bool 11746imbalanced_active_balance(struct lb_env *env) 11747{ 11748 struct sched_domain *sd = env->sd; 11749 11750 /* 11751 * The imbalanced case includes the case of pinned tasks preventing a fair 11752 * distribution of the load on the system but also the even distribution of the 11753 * threads on a system with spare capacity 11754 */ 11755 if ((env->migration_type == migrate_task) && 11756 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 11757 return 1; 11758 11759 return 0; 11760} 11761 11762static int need_active_balance(struct lb_env *env) 11763{ 11764 struct sched_domain *sd = env->sd; 11765 11766 if (asym_active_balance(env)) 11767 return 1; 11768 11769 if (imbalanced_active_balance(env)) 11770 return 1; 11771 11772 /* 11773 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 11774 * It's worth migrating the task if the src_cpu's capacity is reduced 11775 * because of other sched_class or IRQs if more capacity stays 11776 * available on dst_cpu. 11777 */ 11778 if (env->idle && 11779 (env->src_rq->cfs.h_nr_runnable == 1)) { 11780 if ((check_cpu_capacity(env->src_rq, sd)) && 11781 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 11782 return 1; 11783 } 11784 11785 if (env->migration_type == migrate_misfit) 11786 return 1; 11787 11788 return 0; 11789} 11790 11791static int active_load_balance_cpu_stop(void *data); 11792 11793static int should_we_balance(struct lb_env *env) 11794{ 11795 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask); 11796 struct sched_group *sg = env->sd->groups; 11797 int cpu, idle_smt = -1; 11798 11799 /* 11800 * Ensure the balancing environment is consistent; can happen 11801 * when the softirq triggers 'during' hotplug. 11802 */ 11803 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 11804 return 0; 11805 11806 /* 11807 * In the newly idle case, we will allow all the CPUs 11808 * to do the newly idle load balance. 11809 * 11810 * However, we bail out if we already have tasks or a wakeup pending, 11811 * to optimize wakeup latency. 11812 */ 11813 if (env->idle == CPU_NEWLY_IDLE) { 11814 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 11815 return 0; 11816 return 1; 11817 } 11818 11819 cpumask_copy(swb_cpus, group_balance_mask(sg)); 11820 /* Try to find first idle CPU */ 11821 for_each_cpu_and(cpu, swb_cpus, env->cpus) { 11822 if (!idle_cpu(cpu)) 11823 continue; 11824 11825 /* 11826 * Don't balance to idle SMT in busy core right away when 11827 * balancing cores, but remember the first idle SMT CPU for 11828 * later consideration. Find CPU on an idle core first. 11829 */ 11830 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) { 11831 if (idle_smt == -1) 11832 idle_smt = cpu; 11833 /* 11834 * If the core is not idle, and first SMT sibling which is 11835 * idle has been found, then its not needed to check other 11836 * SMT siblings for idleness: 11837 */ 11838#ifdef CONFIG_SCHED_SMT 11839 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); 11840#endif 11841 continue; 11842 } 11843 11844 /* 11845 * Are we the first idle core in a non-SMT domain or higher, 11846 * or the first idle CPU in a SMT domain? 11847 */ 11848 return cpu == env->dst_cpu; 11849 } 11850 11851 /* Are we the first idle CPU with busy siblings? */ 11852 if (idle_smt != -1) 11853 return idle_smt == env->dst_cpu; 11854 11855 /* Are we the first CPU of this group ? */ 11856 return group_balance_cpu(sg) == env->dst_cpu; 11857} 11858 11859static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd, 11860 enum cpu_idle_type idle) 11861{ 11862 if (!schedstat_enabled()) 11863 return; 11864 11865 switch (env->migration_type) { 11866 case migrate_load: 11867 __schedstat_add(sd->lb_imbalance_load[idle], env->imbalance); 11868 break; 11869 case migrate_util: 11870 __schedstat_add(sd->lb_imbalance_util[idle], env->imbalance); 11871 break; 11872 case migrate_task: 11873 __schedstat_add(sd->lb_imbalance_task[idle], env->imbalance); 11874 break; 11875 case migrate_misfit: 11876 __schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance); 11877 break; 11878 } 11879} 11880 11881/* 11882 * This flag serializes load-balancing passes over large domains 11883 * (above the NODE topology level) - only one load-balancing instance 11884 * may run at a time, to reduce overhead on very large systems with 11885 * lots of CPUs and large NUMA distances. 11886 * 11887 * - Note that load-balancing passes triggered while another one 11888 * is executing are skipped and not re-tried. 11889 * 11890 * - Also note that this does not serialize rebalance_domains() 11891 * execution, as non-SD_SERIALIZE domains will still be 11892 * load-balanced in parallel. 11893 */ 11894static atomic_t sched_balance_running = ATOMIC_INIT(0); 11895 11896/* 11897 * Check this_cpu to ensure it is balanced within domain. Attempt to move 11898 * tasks if there is an imbalance. 11899 */ 11900static int sched_balance_rq(int this_cpu, struct rq *this_rq, 11901 struct sched_domain *sd, enum cpu_idle_type idle, 11902 int *continue_balancing) 11903{ 11904 int ld_moved, cur_ld_moved, active_balance = 0; 11905 struct sched_domain *sd_parent = sd->parent; 11906 struct sched_group *group; 11907 struct rq *busiest; 11908 struct rq_flags rf; 11909 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 11910 struct lb_env env = { 11911 .sd = sd, 11912 .dst_cpu = this_cpu, 11913 .dst_rq = this_rq, 11914 .dst_grpmask = group_balance_mask(sd->groups), 11915 .idle = idle, 11916 .loop_break = SCHED_NR_MIGRATE_BREAK, 11917 .cpus = cpus, 11918 .fbq_type = all, 11919 .tasks = LIST_HEAD_INIT(env.tasks), 11920 }; 11921 bool need_unlock = false; 11922 11923 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 11924 11925 schedstat_inc(sd->lb_count[idle]); 11926 11927redo: 11928 if (!should_we_balance(&env)) { 11929 *continue_balancing = 0; 11930 goto out_balanced; 11931 } 11932 11933 if (!need_unlock && (sd->flags & SD_SERIALIZE)) { 11934 int zero = 0; 11935 if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1)) 11936 goto out_balanced; 11937 11938 need_unlock = true; 11939 } 11940 11941 group = sched_balance_find_src_group(&env); 11942 if (!group) { 11943 schedstat_inc(sd->lb_nobusyg[idle]); 11944 goto out_balanced; 11945 } 11946 11947 busiest = sched_balance_find_src_rq(&env, group); 11948 if (!busiest) { 11949 schedstat_inc(sd->lb_nobusyq[idle]); 11950 goto out_balanced; 11951 } 11952 11953 WARN_ON_ONCE(busiest == env.dst_rq); 11954 11955 update_lb_imbalance_stat(&env, sd, idle); 11956 11957 env.src_cpu = busiest->cpu; 11958 env.src_rq = busiest; 11959 11960 ld_moved = 0; 11961 /* Clear this flag as soon as we find a pullable task */ 11962 env.flags |= LBF_ALL_PINNED; 11963 if (busiest->nr_running > 1) { 11964 /* 11965 * Attempt to move tasks. If sched_balance_find_src_group has found 11966 * an imbalance but busiest->nr_running <= 1, the group is 11967 * still unbalanced. ld_moved simply stays zero, so it is 11968 * correctly treated as an imbalance. 11969 */ 11970 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 11971 11972more_balance: 11973 rq_lock_irqsave(busiest, &rf); 11974 update_rq_clock(busiest); 11975 11976 /* 11977 * cur_ld_moved - load moved in current iteration 11978 * ld_moved - cumulative load moved across iterations 11979 */ 11980 cur_ld_moved = detach_tasks(&env); 11981 11982 /* 11983 * We've detached some tasks from busiest_rq. Every 11984 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 11985 * unlock busiest->lock, and we are able to be sure 11986 * that nobody can manipulate the tasks in parallel. 11987 * See task_rq_lock() family for the details. 11988 */ 11989 11990 rq_unlock(busiest, &rf); 11991 11992 if (cur_ld_moved) { 11993 attach_tasks(&env); 11994 ld_moved += cur_ld_moved; 11995 } 11996 11997 local_irq_restore(rf.flags); 11998 11999 if (env.flags & LBF_NEED_BREAK) { 12000 env.flags &= ~LBF_NEED_BREAK; 12001 goto more_balance; 12002 } 12003 12004 /* 12005 * Revisit (affine) tasks on src_cpu that couldn't be moved to 12006 * us and move them to an alternate dst_cpu in our sched_group 12007 * where they can run. The upper limit on how many times we 12008 * iterate on same src_cpu is dependent on number of CPUs in our 12009 * sched_group. 12010 * 12011 * This changes load balance semantics a bit on who can move 12012 * load to a given_cpu. In addition to the given_cpu itself 12013 * (or a ilb_cpu acting on its behalf where given_cpu is 12014 * nohz-idle), we now have balance_cpu in a position to move 12015 * load to given_cpu. In rare situations, this may cause 12016 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 12017 * _independently_ and at _same_ time to move some load to 12018 * given_cpu) causing excess load to be moved to given_cpu. 12019 * This however should not happen so much in practice and 12020 * moreover subsequent load balance cycles should correct the 12021 * excess load moved. 12022 */ 12023 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 12024 12025 /* Prevent to re-select dst_cpu via env's CPUs */ 12026 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 12027 12028 env.dst_rq = cpu_rq(env.new_dst_cpu); 12029 env.dst_cpu = env.new_dst_cpu; 12030 env.flags &= ~LBF_DST_PINNED; 12031 env.loop = 0; 12032 env.loop_break = SCHED_NR_MIGRATE_BREAK; 12033 12034 /* 12035 * Go back to "more_balance" rather than "redo" since we 12036 * need to continue with same src_cpu. 12037 */ 12038 goto more_balance; 12039 } 12040 12041 /* 12042 * We failed to reach balance because of affinity. 12043 */ 12044 if (sd_parent) { 12045 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 12046 12047 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 12048 *group_imbalance = 1; 12049 } 12050 12051 /* All tasks on this runqueue were pinned by CPU affinity */ 12052 if (unlikely(env.flags & LBF_ALL_PINNED)) { 12053 __cpumask_clear_cpu(cpu_of(busiest), cpus); 12054 /* 12055 * Attempting to continue load balancing at the current 12056 * sched_domain level only makes sense if there are 12057 * active CPUs remaining as possible busiest CPUs to 12058 * pull load from which are not contained within the 12059 * destination group that is receiving any migrated 12060 * load. 12061 */ 12062 if (!cpumask_subset(cpus, env.dst_grpmask)) { 12063 env.loop = 0; 12064 env.loop_break = SCHED_NR_MIGRATE_BREAK; 12065 goto redo; 12066 } 12067 goto out_all_pinned; 12068 } 12069 } 12070 12071 if (!ld_moved) { 12072 schedstat_inc(sd->lb_failed[idle]); 12073 /* 12074 * Increment the failure counter only on periodic balance. 12075 * We do not want newidle balance, which can be very 12076 * frequent, pollute the failure counter causing 12077 * excessive cache_hot migrations and active balances. 12078 * 12079 * Similarly for migration_misfit which is not related to 12080 * load/util migration, don't pollute nr_balance_failed. 12081 */ 12082 if (idle != CPU_NEWLY_IDLE && 12083 env.migration_type != migrate_misfit) 12084 sd->nr_balance_failed++; 12085 12086 if (need_active_balance(&env)) { 12087 unsigned long flags; 12088 12089 raw_spin_rq_lock_irqsave(busiest, flags); 12090 12091 /* 12092 * Don't kick the active_load_balance_cpu_stop, 12093 * if the curr task on busiest CPU can't be 12094 * moved to this_cpu: 12095 */ 12096 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 12097 raw_spin_rq_unlock_irqrestore(busiest, flags); 12098 goto out_one_pinned; 12099 } 12100 12101 /* Record that we found at least one task that could run on this_cpu */ 12102 env.flags &= ~LBF_ALL_PINNED; 12103 12104 /* 12105 * ->active_balance synchronizes accesses to 12106 * ->active_balance_work. Once set, it's cleared 12107 * only after active load balance is finished. 12108 */ 12109 if (!busiest->active_balance) { 12110 busiest->active_balance = 1; 12111 busiest->push_cpu = this_cpu; 12112 active_balance = 1; 12113 } 12114 12115 preempt_disable(); 12116 raw_spin_rq_unlock_irqrestore(busiest, flags); 12117 if (active_balance) { 12118 stop_one_cpu_nowait(cpu_of(busiest), 12119 active_load_balance_cpu_stop, busiest, 12120 &busiest->active_balance_work); 12121 } 12122 preempt_enable(); 12123 } 12124 } else { 12125 sd->nr_balance_failed = 0; 12126 } 12127 12128 if (likely(!active_balance) || need_active_balance(&env)) { 12129 /* We were unbalanced, so reset the balancing interval */ 12130 sd->balance_interval = sd->min_interval; 12131 } 12132 12133 goto out; 12134 12135out_balanced: 12136 /* 12137 * We reach balance although we may have faced some affinity 12138 * constraints. Clear the imbalance flag only if other tasks got 12139 * a chance to move and fix the imbalance. 12140 */ 12141 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 12142 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 12143 12144 if (*group_imbalance) 12145 *group_imbalance = 0; 12146 } 12147 12148out_all_pinned: 12149 /* 12150 * We reach balance because all tasks are pinned at this level so 12151 * we can't migrate them. Let the imbalance flag set so parent level 12152 * can try to migrate them. 12153 */ 12154 schedstat_inc(sd->lb_balanced[idle]); 12155 12156 sd->nr_balance_failed = 0; 12157 12158out_one_pinned: 12159 ld_moved = 0; 12160 12161 /* 12162 * sched_balance_newidle() disregards balance intervals, so we could 12163 * repeatedly reach this code, which would lead to balance_interval 12164 * skyrocketing in a short amount of time. Skip the balance_interval 12165 * increase logic to avoid that. 12166 * 12167 * Similarly misfit migration which is not necessarily an indication of 12168 * the system being busy and requires lb to backoff to let it settle 12169 * down. 12170 */ 12171 if (env.idle == CPU_NEWLY_IDLE || 12172 env.migration_type == migrate_misfit) 12173 goto out; 12174 12175 /* tune up the balancing interval */ 12176 if ((env.flags & LBF_ALL_PINNED && 12177 sd->balance_interval < MAX_PINNED_INTERVAL) || 12178 sd->balance_interval < sd->max_interval) 12179 sd->balance_interval *= 2; 12180out: 12181 if (need_unlock) 12182 atomic_set_release(&sched_balance_running, 0); 12183 12184 return ld_moved; 12185} 12186 12187static inline unsigned long 12188get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 12189{ 12190 unsigned long interval = sd->balance_interval; 12191 12192 if (cpu_busy) 12193 interval *= sd->busy_factor; 12194 12195 /* scale ms to jiffies */ 12196 interval = msecs_to_jiffies(interval); 12197 12198 /* 12199 * Reduce likelihood of busy balancing at higher domains racing with 12200 * balancing at lower domains by preventing their balancing periods 12201 * from being multiples of each other. 12202 */ 12203 if (cpu_busy) 12204 interval -= 1; 12205 12206 interval = clamp(interval, 1UL, max_load_balance_interval); 12207 12208 return interval; 12209} 12210 12211static inline void 12212update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 12213{ 12214 unsigned long interval, next; 12215 12216 /* used by idle balance, so cpu_busy = 0 */ 12217 interval = get_sd_balance_interval(sd, 0); 12218 next = sd->last_balance + interval; 12219 12220 if (time_after(*next_balance, next)) 12221 *next_balance = next; 12222} 12223 12224/* 12225 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 12226 * running tasks off the busiest CPU onto idle CPUs. It requires at 12227 * least 1 task to be running on each physical CPU where possible, and 12228 * avoids physical / logical imbalances. 12229 */ 12230static int active_load_balance_cpu_stop(void *data) 12231{ 12232 struct rq *busiest_rq = data; 12233 int busiest_cpu = cpu_of(busiest_rq); 12234 int target_cpu = busiest_rq->push_cpu; 12235 struct rq *target_rq = cpu_rq(target_cpu); 12236 struct sched_domain *sd; 12237 struct task_struct *p = NULL; 12238 struct rq_flags rf; 12239 12240 rq_lock_irq(busiest_rq, &rf); 12241 /* 12242 * Between queueing the stop-work and running it is a hole in which 12243 * CPUs can become inactive. We should not move tasks from or to 12244 * inactive CPUs. 12245 */ 12246 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 12247 goto out_unlock; 12248 12249 /* Make sure the requested CPU hasn't gone down in the meantime: */ 12250 if (unlikely(busiest_cpu != smp_processor_id() || 12251 !busiest_rq->active_balance)) 12252 goto out_unlock; 12253 12254 /* Is there any task to move? */ 12255 if (busiest_rq->nr_running <= 1) 12256 goto out_unlock; 12257 12258 /* 12259 * This condition is "impossible", if it occurs 12260 * we need to fix it. Originally reported by 12261 * Bjorn Helgaas on a 128-CPU setup. 12262 */ 12263 WARN_ON_ONCE(busiest_rq == target_rq); 12264 12265 /* Search for an sd spanning us and the target CPU. */ 12266 rcu_read_lock(); 12267 for_each_domain(target_cpu, sd) { 12268 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 12269 break; 12270 } 12271 12272 if (likely(sd)) { 12273 struct lb_env env = { 12274 .sd = sd, 12275 .dst_cpu = target_cpu, 12276 .dst_rq = target_rq, 12277 .src_cpu = busiest_rq->cpu, 12278 .src_rq = busiest_rq, 12279 .idle = CPU_IDLE, 12280 .flags = LBF_ACTIVE_LB, 12281 }; 12282 12283 schedstat_inc(sd->alb_count); 12284 update_rq_clock(busiest_rq); 12285 12286 p = detach_one_task(&env); 12287 if (p) { 12288 schedstat_inc(sd->alb_pushed); 12289 /* Active balancing done, reset the failure counter. */ 12290 sd->nr_balance_failed = 0; 12291 } else { 12292 schedstat_inc(sd->alb_failed); 12293 } 12294 } 12295 rcu_read_unlock(); 12296out_unlock: 12297 busiest_rq->active_balance = 0; 12298 rq_unlock(busiest_rq, &rf); 12299 12300 if (p) 12301 attach_one_task(target_rq, p); 12302 12303 local_irq_enable(); 12304 12305 return 0; 12306} 12307 12308/* 12309 * Scale the max sched_balance_rq interval with the number of CPUs in the system. 12310 * This trades load-balance latency on larger machines for less cross talk. 12311 */ 12312void update_max_interval(void) 12313{ 12314 max_load_balance_interval = HZ*num_online_cpus()/10; 12315} 12316 12317static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success) 12318{ 12319 sd->newidle_call++; 12320 sd->newidle_success += success; 12321 12322 if (sd->newidle_call >= 1024) { 12323 sd->newidle_ratio = sd->newidle_success; 12324 sd->newidle_call /= 2; 12325 sd->newidle_success /= 2; 12326 } 12327} 12328 12329static inline bool 12330update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success) 12331{ 12332 unsigned long next_decay = sd->last_decay_max_lb_cost + HZ; 12333 unsigned long now = jiffies; 12334 12335 if (cost) 12336 update_newidle_stats(sd, success); 12337 12338 if (cost > sd->max_newidle_lb_cost) { 12339 /* 12340 * Track max cost of a domain to make sure to not delay the 12341 * next wakeup on the CPU. 12342 */ 12343 sd->max_newidle_lb_cost = cost; 12344 sd->last_decay_max_lb_cost = now; 12345 12346 } else if (time_after(now, next_decay)) { 12347 /* 12348 * Decay the newidle max times by ~1% per second to ensure that 12349 * it is not outdated and the current max cost is actually 12350 * shorter. 12351 */ 12352 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 12353 sd->last_decay_max_lb_cost = now; 12354 return true; 12355 } 12356 12357 return false; 12358} 12359 12360/* 12361 * It checks each scheduling domain to see if it is due to be balanced, 12362 * and initiates a balancing operation if so. 12363 * 12364 * Balancing parameters are set up in init_sched_domains. 12365 */ 12366static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) 12367{ 12368 int continue_balancing = 1; 12369 int cpu = rq->cpu; 12370 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 12371 unsigned long interval; 12372 struct sched_domain *sd; 12373 /* Earliest time when we have to do rebalance again */ 12374 unsigned long next_balance = jiffies + 60*HZ; 12375 int update_next_balance = 0; 12376 int need_decay = 0; 12377 u64 max_cost = 0; 12378 12379 rcu_read_lock(); 12380 for_each_domain(cpu, sd) { 12381 /* 12382 * Decay the newidle max times here because this is a regular 12383 * visit to all the domains. 12384 */ 12385 need_decay = update_newidle_cost(sd, 0, 0); 12386 max_cost += sd->max_newidle_lb_cost; 12387 12388 /* 12389 * Stop the load balance at this level. There is another 12390 * CPU in our sched group which is doing load balancing more 12391 * actively. 12392 */ 12393 if (!continue_balancing) { 12394 if (need_decay) 12395 continue; 12396 break; 12397 } 12398 12399 interval = get_sd_balance_interval(sd, busy); 12400 if (time_after_eq(jiffies, sd->last_balance + interval)) { 12401 if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) { 12402 /* 12403 * The LBF_DST_PINNED logic could have changed 12404 * env->dst_cpu, so we can't know our idle 12405 * state even if we migrated tasks. Update it. 12406 */ 12407 idle = idle_cpu(cpu); 12408 busy = !idle && !sched_idle_cpu(cpu); 12409 } 12410 sd->last_balance = jiffies; 12411 interval = get_sd_balance_interval(sd, busy); 12412 } 12413 if (time_after(next_balance, sd->last_balance + interval)) { 12414 next_balance = sd->last_balance + interval; 12415 update_next_balance = 1; 12416 } 12417 } 12418 if (need_decay) { 12419 /* 12420 * Ensure the rq-wide value also decays but keep it at a 12421 * reasonable floor to avoid funnies with rq->avg_idle. 12422 */ 12423 rq->max_idle_balance_cost = 12424 max((u64)sysctl_sched_migration_cost, max_cost); 12425 } 12426 rcu_read_unlock(); 12427 12428 /* 12429 * next_balance will be updated only when there is a need. 12430 * When the cpu is attached to null domain for ex, it will not be 12431 * updated. 12432 */ 12433 if (likely(update_next_balance)) 12434 rq->next_balance = next_balance; 12435 12436} 12437 12438static inline int on_null_domain(struct rq *rq) 12439{ 12440 return unlikely(!rcu_dereference_sched(rq->sd)); 12441} 12442 12443#ifdef CONFIG_NO_HZ_COMMON 12444/* 12445 * NOHZ idle load balancing (ILB) details: 12446 * 12447 * - When one of the busy CPUs notices that there may be an idle rebalancing 12448 * needed, they will kick the idle load balancer, which then does idle 12449 * load balancing for all the idle CPUs. 12450 */ 12451static inline int find_new_ilb(void) 12452{ 12453 const struct cpumask *hk_mask; 12454 int ilb_cpu; 12455 12456 hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); 12457 12458 for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) { 12459 12460 if (ilb_cpu == smp_processor_id()) 12461 continue; 12462 12463 if (idle_cpu(ilb_cpu)) 12464 return ilb_cpu; 12465 } 12466 12467 return -1; 12468} 12469 12470/* 12471 * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU 12472 * SMP function call (IPI). 12473 * 12474 * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set 12475 * (if there is one). 12476 */ 12477static void kick_ilb(unsigned int flags) 12478{ 12479 int ilb_cpu; 12480 12481 /* 12482 * Increase nohz.next_balance only when if full ilb is triggered but 12483 * not if we only update stats. 12484 */ 12485 if (flags & NOHZ_BALANCE_KICK) 12486 nohz.next_balance = jiffies+1; 12487 12488 ilb_cpu = find_new_ilb(); 12489 if (ilb_cpu < 0) 12490 return; 12491 12492 /* 12493 * Don't bother if no new NOHZ balance work items for ilb_cpu, 12494 * i.e. all bits in flags are already set in ilb_cpu. 12495 */ 12496 if ((atomic_read(nohz_flags(ilb_cpu)) & flags) == flags) 12497 return; 12498 12499 /* 12500 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 12501 * the first flag owns it; cleared by nohz_csd_func(). 12502 */ 12503 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 12504 if (flags & NOHZ_KICK_MASK) 12505 return; 12506 12507 /* 12508 * This way we generate an IPI on the target CPU which 12509 * is idle, and the softirq performing NOHZ idle load balancing 12510 * will be run before returning from the IPI. 12511 */ 12512 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 12513} 12514 12515/* 12516 * Current decision point for kicking the idle load balancer in the presence 12517 * of idle CPUs in the system. 12518 */ 12519static void nohz_balancer_kick(struct rq *rq) 12520{ 12521 unsigned long now = jiffies; 12522 struct sched_domain_shared *sds; 12523 struct sched_domain *sd; 12524 int nr_busy, i, cpu = rq->cpu; 12525 unsigned int flags = 0; 12526 12527 if (unlikely(rq->idle_balance)) 12528 return; 12529 12530 /* 12531 * We may be recently in ticked or tickless idle mode. At the first 12532 * busy tick after returning from idle, we will update the busy stats. 12533 */ 12534 nohz_balance_exit_idle(rq); 12535 12536 /* 12537 * None are in tickless mode and hence no need for NOHZ idle load 12538 * balancing: 12539 */ 12540 if (likely(!atomic_read(&nohz.nr_cpus))) 12541 return; 12542 12543 if (READ_ONCE(nohz.has_blocked) && 12544 time_after(now, READ_ONCE(nohz.next_blocked))) 12545 flags = NOHZ_STATS_KICK; 12546 12547 if (time_before(now, nohz.next_balance)) 12548 goto out; 12549 12550 if (rq->nr_running >= 2) { 12551 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12552 goto out; 12553 } 12554 12555 rcu_read_lock(); 12556 12557 sd = rcu_dereference(rq->sd); 12558 if (sd) { 12559 /* 12560 * If there's a runnable CFS task and the current CPU has reduced 12561 * capacity, kick the ILB to see if there's a better CPU to run on: 12562 */ 12563 if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) { 12564 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12565 goto unlock; 12566 } 12567 } 12568 12569 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 12570 if (sd) { 12571 /* 12572 * When ASYM_PACKING; see if there's a more preferred CPU 12573 * currently idle; in which case, kick the ILB to move tasks 12574 * around. 12575 * 12576 * When balancing between cores, all the SMT siblings of the 12577 * preferred CPU must be idle. 12578 */ 12579 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 12580 if (sched_asym(sd, i, cpu)) { 12581 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12582 goto unlock; 12583 } 12584 } 12585 } 12586 12587 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 12588 if (sd) { 12589 /* 12590 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 12591 * to run the misfit task on. 12592 */ 12593 if (check_misfit_status(rq)) { 12594 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12595 goto unlock; 12596 } 12597 12598 /* 12599 * For asymmetric systems, we do not want to nicely balance 12600 * cache use, instead we want to embrace asymmetry and only 12601 * ensure tasks have enough CPU capacity. 12602 * 12603 * Skip the LLC logic because it's not relevant in that case. 12604 */ 12605 goto unlock; 12606 } 12607 12608 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 12609 if (sds) { 12610 /* 12611 * If there is an imbalance between LLC domains (IOW we could 12612 * increase the overall cache utilization), we need a less-loaded LLC 12613 * domain to pull some load from. Likewise, we may need to spread 12614 * load within the current LLC domain (e.g. packed SMT cores but 12615 * other CPUs are idle). We can't really know from here how busy 12616 * the others are - so just get a NOHZ balance going if it looks 12617 * like this LLC domain has tasks we could move. 12618 */ 12619 nr_busy = atomic_read(&sds->nr_busy_cpus); 12620 if (nr_busy > 1) { 12621 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12622 goto unlock; 12623 } 12624 } 12625unlock: 12626 rcu_read_unlock(); 12627out: 12628 if (READ_ONCE(nohz.needs_update)) 12629 flags |= NOHZ_NEXT_KICK; 12630 12631 if (flags) 12632 kick_ilb(flags); 12633} 12634 12635static void set_cpu_sd_state_busy(int cpu) 12636{ 12637 struct sched_domain *sd; 12638 12639 rcu_read_lock(); 12640 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 12641 12642 if (!sd || !sd->nohz_idle) 12643 goto unlock; 12644 sd->nohz_idle = 0; 12645 12646 atomic_inc(&sd->shared->nr_busy_cpus); 12647unlock: 12648 rcu_read_unlock(); 12649} 12650 12651void nohz_balance_exit_idle(struct rq *rq) 12652{ 12653 WARN_ON_ONCE(rq != this_rq()); 12654 12655 if (likely(!rq->nohz_tick_stopped)) 12656 return; 12657 12658 rq->nohz_tick_stopped = 0; 12659 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 12660 atomic_dec(&nohz.nr_cpus); 12661 12662 set_cpu_sd_state_busy(rq->cpu); 12663} 12664 12665static void set_cpu_sd_state_idle(int cpu) 12666{ 12667 struct sched_domain *sd; 12668 12669 rcu_read_lock(); 12670 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 12671 12672 if (!sd || sd->nohz_idle) 12673 goto unlock; 12674 sd->nohz_idle = 1; 12675 12676 atomic_dec(&sd->shared->nr_busy_cpus); 12677unlock: 12678 rcu_read_unlock(); 12679} 12680 12681/* 12682 * This routine will record that the CPU is going idle with tick stopped. 12683 * This info will be used in performing idle load balancing in the future. 12684 */ 12685void nohz_balance_enter_idle(int cpu) 12686{ 12687 struct rq *rq = cpu_rq(cpu); 12688 12689 WARN_ON_ONCE(cpu != smp_processor_id()); 12690 12691 /* If this CPU is going down, then nothing needs to be done: */ 12692 if (!cpu_active(cpu)) 12693 return; 12694 12695 /* 12696 * Can be set safely without rq->lock held 12697 * If a clear happens, it will have evaluated last additions because 12698 * rq->lock is held during the check and the clear 12699 */ 12700 rq->has_blocked_load = 1; 12701 12702 /* 12703 * The tick is still stopped but load could have been added in the 12704 * meantime. We set the nohz.has_blocked flag to trig a check of the 12705 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 12706 * of nohz.has_blocked can only happen after checking the new load 12707 */ 12708 if (rq->nohz_tick_stopped) 12709 goto out; 12710 12711 /* If we're a completely isolated CPU, we don't play: */ 12712 if (on_null_domain(rq)) 12713 return; 12714 12715 rq->nohz_tick_stopped = 1; 12716 12717 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 12718 atomic_inc(&nohz.nr_cpus); 12719 12720 /* 12721 * Ensures that if nohz_idle_balance() fails to observe our 12722 * @idle_cpus_mask store, it must observe the @has_blocked 12723 * and @needs_update stores. 12724 */ 12725 smp_mb__after_atomic(); 12726 12727 set_cpu_sd_state_idle(cpu); 12728 12729 WRITE_ONCE(nohz.needs_update, 1); 12730out: 12731 /* 12732 * Each time a cpu enter idle, we assume that it has blocked load and 12733 * enable the periodic update of the load of idle CPUs 12734 */ 12735 WRITE_ONCE(nohz.has_blocked, 1); 12736} 12737 12738static bool update_nohz_stats(struct rq *rq) 12739{ 12740 unsigned int cpu = rq->cpu; 12741 12742 if (!rq->has_blocked_load) 12743 return false; 12744 12745 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 12746 return false; 12747 12748 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 12749 return true; 12750 12751 sched_balance_update_blocked_averages(cpu); 12752 12753 return rq->has_blocked_load; 12754} 12755 12756/* 12757 * Internal function that runs load balance for all idle CPUs. The load balance 12758 * can be a simple update of blocked load or a complete load balance with 12759 * tasks movement depending of flags. 12760 */ 12761static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 12762{ 12763 /* Earliest time when we have to do rebalance again */ 12764 unsigned long now = jiffies; 12765 unsigned long next_balance = now + 60*HZ; 12766 bool has_blocked_load = false; 12767 int update_next_balance = 0; 12768 int this_cpu = this_rq->cpu; 12769 int balance_cpu; 12770 struct rq *rq; 12771 12772 WARN_ON_ONCE((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 12773 12774 /* 12775 * We assume there will be no idle load after this update and clear 12776 * the has_blocked flag. If a cpu enters idle in the mean time, it will 12777 * set the has_blocked flag and trigger another update of idle load. 12778 * Because a cpu that becomes idle, is added to idle_cpus_mask before 12779 * setting the flag, we are sure to not clear the state and not 12780 * check the load of an idle cpu. 12781 * 12782 * Same applies to idle_cpus_mask vs needs_update. 12783 */ 12784 if (flags & NOHZ_STATS_KICK) 12785 WRITE_ONCE(nohz.has_blocked, 0); 12786 if (flags & NOHZ_NEXT_KICK) 12787 WRITE_ONCE(nohz.needs_update, 0); 12788 12789 /* 12790 * Ensures that if we miss the CPU, we must see the has_blocked 12791 * store from nohz_balance_enter_idle(). 12792 */ 12793 smp_mb(); 12794 12795 /* 12796 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 12797 * chance for other idle cpu to pull load. 12798 */ 12799 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 12800 if (!idle_cpu(balance_cpu)) 12801 continue; 12802 12803 /* 12804 * If this CPU gets work to do, stop the load balancing 12805 * work being done for other CPUs. Next load 12806 * balancing owner will pick it up. 12807 */ 12808 if (!idle_cpu(this_cpu) && need_resched()) { 12809 if (flags & NOHZ_STATS_KICK) 12810 has_blocked_load = true; 12811 if (flags & NOHZ_NEXT_KICK) 12812 WRITE_ONCE(nohz.needs_update, 1); 12813 goto abort; 12814 } 12815 12816 rq = cpu_rq(balance_cpu); 12817 12818 if (flags & NOHZ_STATS_KICK) 12819 has_blocked_load |= update_nohz_stats(rq); 12820 12821 /* 12822 * If time for next balance is due, 12823 * do the balance. 12824 */ 12825 if (time_after_eq(jiffies, rq->next_balance)) { 12826 struct rq_flags rf; 12827 12828 rq_lock_irqsave(rq, &rf); 12829 update_rq_clock(rq); 12830 rq_unlock_irqrestore(rq, &rf); 12831 12832 if (flags & NOHZ_BALANCE_KICK) 12833 sched_balance_domains(rq, CPU_IDLE); 12834 } 12835 12836 if (time_after(next_balance, rq->next_balance)) { 12837 next_balance = rq->next_balance; 12838 update_next_balance = 1; 12839 } 12840 } 12841 12842 /* 12843 * next_balance will be updated only when there is a need. 12844 * When the CPU is attached to null domain for ex, it will not be 12845 * updated. 12846 */ 12847 if (likely(update_next_balance)) 12848 nohz.next_balance = next_balance; 12849 12850 if (flags & NOHZ_STATS_KICK) 12851 WRITE_ONCE(nohz.next_blocked, 12852 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 12853 12854abort: 12855 /* There is still blocked load, enable periodic update */ 12856 if (has_blocked_load) 12857 WRITE_ONCE(nohz.has_blocked, 1); 12858} 12859 12860/* 12861 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 12862 * rebalancing for all the CPUs for whom scheduler ticks are stopped. 12863 */ 12864static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12865{ 12866 unsigned int flags = this_rq->nohz_idle_balance; 12867 12868 if (!flags) 12869 return false; 12870 12871 this_rq->nohz_idle_balance = 0; 12872 12873 if (idle != CPU_IDLE) 12874 return false; 12875 12876 _nohz_idle_balance(this_rq, flags); 12877 12878 return true; 12879} 12880 12881/* 12882 * Check if we need to directly run the ILB for updating blocked load before 12883 * entering idle state. Here we run ILB directly without issuing IPIs. 12884 * 12885 * Note that when this function is called, the tick may not yet be stopped on 12886 * this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and 12887 * cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates 12888 * don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle 12889 * entry/exit rate (usec). So it is possible that _nohz_idle_balance() is 12890 * called from this function on (this) CPU that's not yet in the mask. That's 12891 * OK because the goal of nohz_run_idle_balance() is to run ILB only for 12892 * updating the blocked load of already idle CPUs without waking up one of 12893 * those idle CPUs and outside the preempt disable / IRQ off phase of the local 12894 * cpu about to enter idle, because it can take a long time. 12895 */ 12896void nohz_run_idle_balance(int cpu) 12897{ 12898 unsigned int flags; 12899 12900 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 12901 12902 /* 12903 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 12904 * (i.e. NOHZ_STATS_KICK set) and will do the same. 12905 */ 12906 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 12907 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 12908} 12909 12910static void nohz_newidle_balance(struct rq *this_rq) 12911{ 12912 int this_cpu = this_rq->cpu; 12913 12914 /* Will wake up very soon. No time for doing anything else*/ 12915 if (this_rq->avg_idle < sysctl_sched_migration_cost) 12916 return; 12917 12918 /* Don't need to update blocked load of idle CPUs*/ 12919 if (!READ_ONCE(nohz.has_blocked) || 12920 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 12921 return; 12922 12923 /* 12924 * Set the need to trigger ILB in order to update blocked load 12925 * before entering idle state. 12926 */ 12927 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 12928} 12929 12930#else /* !CONFIG_NO_HZ_COMMON: */ 12931static inline void nohz_balancer_kick(struct rq *rq) { } 12932 12933static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12934{ 12935 return false; 12936} 12937 12938static inline void nohz_newidle_balance(struct rq *this_rq) { } 12939#endif /* !CONFIG_NO_HZ_COMMON */ 12940 12941/* 12942 * sched_balance_newidle is called by schedule() if this_cpu is about to become 12943 * idle. Attempts to pull tasks from other CPUs. 12944 * 12945 * Returns: 12946 * < 0 - we released the lock and there are !fair tasks present 12947 * 0 - failed, no new tasks 12948 * > 0 - success, new (fair) tasks present 12949 */ 12950static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 12951{ 12952 unsigned long next_balance = jiffies + HZ; 12953 int this_cpu = this_rq->cpu; 12954 int continue_balancing = 1; 12955 u64 t0, t1, curr_cost = 0; 12956 struct sched_domain *sd; 12957 int pulled_task = 0; 12958 12959 update_misfit_status(NULL, this_rq); 12960 12961 /* 12962 * There is a task waiting to run. No need to search for one. 12963 * Return 0; the task will be enqueued when switching to idle. 12964 */ 12965 if (this_rq->ttwu_pending) 12966 return 0; 12967 12968 /* 12969 * We must set idle_stamp _before_ calling sched_balance_rq() 12970 * for CPU_NEWLY_IDLE, such that we measure the this duration 12971 * as idle time. 12972 */ 12973 this_rq->idle_stamp = rq_clock(this_rq); 12974 12975 /* 12976 * Do not pull tasks towards !active CPUs... 12977 */ 12978 if (!cpu_active(this_cpu)) 12979 return 0; 12980 12981 /* 12982 * This is OK, because current is on_cpu, which avoids it being picked 12983 * for load-balance and preemption/IRQs are still disabled avoiding 12984 * further scheduler activity on it and we're being very careful to 12985 * re-start the picking loop. 12986 */ 12987 rq_unpin_lock(this_rq, rf); 12988 12989 rcu_read_lock(); 12990 sd = rcu_dereference_check_sched_domain(this_rq->sd); 12991 if (!sd) { 12992 rcu_read_unlock(); 12993 goto out; 12994 } 12995 12996 if (!get_rd_overloaded(this_rq->rd) || 12997 this_rq->avg_idle < sd->max_newidle_lb_cost) { 12998 12999 update_next_balance(sd, &next_balance); 13000 rcu_read_unlock(); 13001 goto out; 13002 } 13003 rcu_read_unlock(); 13004 13005 rq_modified_clear(this_rq); 13006 raw_spin_rq_unlock(this_rq); 13007 13008 t0 = sched_clock_cpu(this_cpu); 13009 sched_balance_update_blocked_averages(this_cpu); 13010 13011 rcu_read_lock(); 13012 for_each_domain(this_cpu, sd) { 13013 u64 domain_cost; 13014 13015 update_next_balance(sd, &next_balance); 13016 13017 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 13018 break; 13019 13020 if (sd->flags & SD_BALANCE_NEWIDLE) { 13021 unsigned int weight = 1; 13022 13023 if (sched_feat(NI_RANDOM)) { 13024 /* 13025 * Throw a 1k sided dice; and only run 13026 * newidle_balance according to the success 13027 * rate. 13028 */ 13029 u32 d1k = sched_rng() % 1024; 13030 weight = 1 + sd->newidle_ratio; 13031 if (d1k > weight) { 13032 update_newidle_stats(sd, 0); 13033 continue; 13034 } 13035 weight = (1024 + weight/2) / weight; 13036 } 13037 13038 pulled_task = sched_balance_rq(this_cpu, this_rq, 13039 sd, CPU_NEWLY_IDLE, 13040 &continue_balancing); 13041 13042 t1 = sched_clock_cpu(this_cpu); 13043 domain_cost = t1 - t0; 13044 curr_cost += domain_cost; 13045 t0 = t1; 13046 13047 /* 13048 * Track max cost of a domain to make sure to not delay the 13049 * next wakeup on the CPU. 13050 */ 13051 update_newidle_cost(sd, domain_cost, weight * !!pulled_task); 13052 } 13053 13054 /* 13055 * Stop searching for tasks to pull if there are 13056 * now runnable tasks on this rq. 13057 */ 13058 if (pulled_task || !continue_balancing) 13059 break; 13060 } 13061 rcu_read_unlock(); 13062 13063 raw_spin_rq_lock(this_rq); 13064 13065 if (curr_cost > this_rq->max_idle_balance_cost) 13066 this_rq->max_idle_balance_cost = curr_cost; 13067 13068 /* 13069 * While browsing the domains, we released the rq lock, a task could 13070 * have been enqueued in the meantime. Since we're not going idle, 13071 * pretend we pulled a task. 13072 */ 13073 if (this_rq->cfs.h_nr_queued && !pulled_task) 13074 pulled_task = 1; 13075 13076 /* If a higher prio class was modified, restart the pick */ 13077 if (rq_modified_above(this_rq, &fair_sched_class)) 13078 pulled_task = -1; 13079 13080out: 13081 /* Move the next balance forward */ 13082 if (time_after(this_rq->next_balance, next_balance)) 13083 this_rq->next_balance = next_balance; 13084 13085 if (pulled_task) 13086 this_rq->idle_stamp = 0; 13087 else 13088 nohz_newidle_balance(this_rq); 13089 13090 rq_repin_lock(this_rq, rf); 13091 13092 return pulled_task; 13093} 13094 13095/* 13096 * This softirq handler is triggered via SCHED_SOFTIRQ from two places: 13097 * 13098 * - directly from the local sched_tick() for periodic load balancing 13099 * 13100 * - indirectly from a remote sched_tick() for NOHZ idle balancing 13101 * through the SMP cross-call nohz_csd_func() 13102 */ 13103static __latent_entropy void sched_balance_softirq(void) 13104{ 13105 struct rq *this_rq = this_rq(); 13106 enum cpu_idle_type idle = this_rq->idle_balance; 13107 /* 13108 * If this CPU has a pending NOHZ_BALANCE_KICK, then do the 13109 * balancing on behalf of the other idle CPUs whose ticks are 13110 * stopped. Do nohz_idle_balance *before* sched_balance_domains to 13111 * give the idle CPUs a chance to load balance. Else we may 13112 * load balance only within the local sched_domain hierarchy 13113 * and abort nohz_idle_balance altogether if we pull some load. 13114 */ 13115 if (nohz_idle_balance(this_rq, idle)) 13116 return; 13117 13118 /* normal load balance */ 13119 sched_balance_update_blocked_averages(this_rq->cpu); 13120 sched_balance_domains(this_rq, idle); 13121} 13122 13123/* 13124 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 13125 */ 13126void sched_balance_trigger(struct rq *rq) 13127{ 13128 /* 13129 * Don't need to rebalance while attached to NULL domain or 13130 * runqueue CPU is not active 13131 */ 13132 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 13133 return; 13134 13135 if (time_after_eq(jiffies, rq->next_balance)) 13136 raise_softirq(SCHED_SOFTIRQ); 13137 13138 nohz_balancer_kick(rq); 13139} 13140 13141static void rq_online_fair(struct rq *rq) 13142{ 13143 update_sysctl(); 13144 13145 update_runtime_enabled(rq); 13146} 13147 13148static void rq_offline_fair(struct rq *rq) 13149{ 13150 update_sysctl(); 13151 13152 /* Ensure any throttled groups are reachable by pick_next_task */ 13153 unthrottle_offline_cfs_rqs(rq); 13154 13155 /* Ensure that we remove rq contribution to group share: */ 13156 clear_tg_offline_cfs_rqs(rq); 13157} 13158 13159#ifdef CONFIG_SCHED_CORE 13160static inline bool 13161__entity_slice_used(struct sched_entity *se, int min_nr_tasks) 13162{ 13163 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 13164 u64 slice = se->slice; 13165 13166 return (rtime * min_nr_tasks > slice); 13167} 13168 13169#define MIN_NR_TASKS_DURING_FORCEIDLE 2 13170static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 13171{ 13172 if (!sched_core_enabled(rq)) 13173 return; 13174 13175 /* 13176 * If runqueue has only one task which used up its slice and 13177 * if the sibling is forced idle, then trigger schedule to 13178 * give forced idle task a chance. 13179 * 13180 * sched_slice() considers only this active rq and it gets the 13181 * whole slice. But during force idle, we have siblings acting 13182 * like a single runqueue and hence we need to consider runnable 13183 * tasks on this CPU and the forced idle CPU. Ideally, we should 13184 * go through the forced idle rq, but that would be a perf hit. 13185 * We can assume that the forced idle CPU has at least 13186 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 13187 * if we need to give up the CPU. 13188 */ 13189 if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && 13190 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 13191 resched_curr(rq); 13192} 13193 13194/* 13195 * Consider any infeasible weight scenario. Take for instance two tasks, 13196 * each bound to their respective sibling, one with weight 1 and one with 13197 * weight 2. Then the lower weight task will run ahead of the higher weight 13198 * task without bound. 13199 * 13200 * This utterly destroys the concept of a shared time base. 13201 * 13202 * Remember; all this is about a proportionally fair scheduling, where each 13203 * tasks receives: 13204 * 13205 * w_i 13206 * dt_i = ---------- dt (1) 13207 * \Sum_j w_j 13208 * 13209 * which we do by tracking a virtual time, s_i: 13210 * 13211 * 1 13212 * s_i = --- d[t]_i (2) 13213 * w_i 13214 * 13215 * Where d[t] is a delta of discrete time, while dt is an infinitesimal. 13216 * The immediate corollary is that the ideal schedule S, where (2) to use 13217 * an infinitesimal delta, is: 13218 * 13219 * 1 13220 * S = ---------- dt (3) 13221 * \Sum_i w_i 13222 * 13223 * From which we can define the lag, or deviation from the ideal, as: 13224 * 13225 * lag(i) = S - s_i (4) 13226 * 13227 * And since the one and only purpose is to approximate S, we get that: 13228 * 13229 * \Sum_i w_i lag(i) := 0 (5) 13230 * 13231 * If this were not so, we no longer converge to S, and we can no longer 13232 * claim our scheduler has any of the properties we derive from S. This is 13233 * exactly what you did above, you broke it! 13234 * 13235 * 13236 * Let's continue for a while though; to see if there is anything useful to 13237 * be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i: 13238 * 13239 * \Sum_i w_i s_i 13240 * S = -------------- (6) 13241 * \Sum_i w_i 13242 * 13243 * Which gives us a way to compute S, given our s_i. Now, if you've read 13244 * our code, you know that we do not in fact do this, the reason for this 13245 * is two-fold. Firstly, computing S in that way requires a 64bit division 13246 * for every time we'd use it (see 12), and secondly, this only describes 13247 * the steady-state, it doesn't handle dynamics. 13248 * 13249 * Anyway, in (6): s_i -> x + (s_i - x), to get: 13250 * 13251 * \Sum_i w_i (s_i - x) 13252 * S - x = -------------------- (7) 13253 * \Sum_i w_i 13254 * 13255 * Which shows that S and s_i transform alike (which makes perfect sense 13256 * given that S is basically the (weighted) average of s_i). 13257 * 13258 * So the thing to remember is that the above is strictly UP. It is 13259 * possible to generalize to multiple runqueues -- however it gets really 13260 * yuck when you have to add affinity support, as illustrated by our very 13261 * first counter-example. 13262 * 13263 * Luckily I think we can avoid needing a full multi-queue variant for 13264 * core-scheduling (or load-balancing). The crucial observation is that we 13265 * only actually need this comparison in the presence of forced-idle; only 13266 * then do we need to tell if the stalled rq has higher priority over the 13267 * other. 13268 * 13269 * [XXX assumes SMT2; better consider the more general case, I suspect 13270 * it'll work out because our comparison is always between 2 rqs and the 13271 * answer is only interesting if one of them is forced-idle] 13272 * 13273 * And (under assumption of SMT2) when there is forced-idle, there is only 13274 * a single queue, so everything works like normal. 13275 * 13276 * Let, for our runqueue 'k': 13277 * 13278 * T_k = \Sum_i w_i s_i 13279 * W_k = \Sum_i w_i ; for all i of k (8) 13280 * 13281 * Then we can write (6) like: 13282 * 13283 * T_k 13284 * S_k = --- (9) 13285 * W_k 13286 * 13287 * From which immediately follows that: 13288 * 13289 * T_k + T_l 13290 * S_k+l = --------- (10) 13291 * W_k + W_l 13292 * 13293 * On which we can define a combined lag: 13294 * 13295 * lag_k+l(i) := S_k+l - s_i (11) 13296 * 13297 * And that gives us the tools to compare tasks across a combined runqueue. 13298 * 13299 * 13300 * Combined this gives the following: 13301 * 13302 * a) when a runqueue enters force-idle, sync it against it's sibling rq(s) 13303 * using (7); this only requires storing single 'time'-stamps. 13304 * 13305 * b) when comparing tasks between 2 runqueues of which one is forced-idle, 13306 * compare the combined lag, per (11). 13307 * 13308 * Now, of course cgroups (I so hate them) make this more interesting in 13309 * that a) seems to suggest we need to iterate all cgroup on a CPU at such 13310 * boundaries, but I think we can avoid that. The force-idle is for the 13311 * whole CPU, all it's rqs. So we can mark it in the root and lazily 13312 * propagate downward on demand. 13313 */ 13314 13315/* 13316 * So this sync is basically a relative reset of S to 0. 13317 * 13318 * So with 2 queues, when one goes idle, we drop them both to 0 and one 13319 * then increases due to not being idle, and the idle one builds up lag to 13320 * get re-elected. So far so simple, right? 13321 * 13322 * When there's 3, we can have the situation where 2 run and one is idle, 13323 * we sync to 0 and let the idle one build up lag to get re-election. Now 13324 * suppose another one also drops idle. At this point dropping all to 0 13325 * again would destroy the built-up lag from the queue that was already 13326 * idle, not good. 13327 * 13328 * So instead of syncing everything, we can: 13329 * 13330 * less := !((s64)(s_a - s_b) <= 0) 13331 * 13332 * (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b 13333 * == v_a - (v_b - S_a + S_b) 13334 * 13335 * IOW, we can recast the (lag) comparison to a one-sided difference. 13336 * So if then, instead of syncing the whole queue, sync the idle queue 13337 * against the active queue with S_a + S_b at the point where we sync. 13338 * 13339 * (XXX consider the implication of living in a cyclic group: N / 2^n N) 13340 * 13341 * This gives us means of syncing single queues against the active queue, 13342 * and for already idle queues to preserve their build-up lag. 13343 * 13344 * Of course, then we get the situation where there's 2 active and one 13345 * going idle, who do we pick to sync against? Theory would have us sync 13346 * against the combined S, but as we've already demonstrated, there is no 13347 * such thing in infeasible weight scenarios. 13348 * 13349 * One thing I've considered; and this is where that core_active rudiment 13350 * came from, is having active queues sync up between themselves after 13351 * every tick. This limits the observed divergence due to the work 13352 * conservancy. 13353 * 13354 * On top of that, we can improve upon things by employing (10) here. 13355 */ 13356 13357/* 13358 * se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed. 13359 */ 13360static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 13361 bool forceidle) 13362{ 13363 for_each_sched_entity(se) { 13364 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13365 13366 if (forceidle) { 13367 if (cfs_rq->forceidle_seq == fi_seq) 13368 break; 13369 cfs_rq->forceidle_seq = fi_seq; 13370 } 13371 13372 cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime; 13373 } 13374} 13375 13376void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 13377{ 13378 struct sched_entity *se = &p->se; 13379 13380 if (p->sched_class != &fair_sched_class) 13381 return; 13382 13383 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 13384} 13385 13386bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 13387 bool in_fi) 13388{ 13389 struct rq *rq = task_rq(a); 13390 const struct sched_entity *sea = &a->se; 13391 const struct sched_entity *seb = &b->se; 13392 struct cfs_rq *cfs_rqa; 13393 struct cfs_rq *cfs_rqb; 13394 s64 delta; 13395 13396 WARN_ON_ONCE(task_rq(b)->core != rq->core); 13397 13398#ifdef CONFIG_FAIR_GROUP_SCHED 13399 /* 13400 * Find an se in the hierarchy for tasks a and b, such that the se's 13401 * are immediate siblings. 13402 */ 13403 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { 13404 int sea_depth = sea->depth; 13405 int seb_depth = seb->depth; 13406 13407 if (sea_depth >= seb_depth) 13408 sea = parent_entity(sea); 13409 if (sea_depth <= seb_depth) 13410 seb = parent_entity(seb); 13411 } 13412 13413 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); 13414 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); 13415 13416 cfs_rqa = sea->cfs_rq; 13417 cfs_rqb = seb->cfs_rq; 13418#else /* !CONFIG_FAIR_GROUP_SCHED: */ 13419 cfs_rqa = &task_rq(a)->cfs; 13420 cfs_rqb = &task_rq(b)->cfs; 13421#endif /* !CONFIG_FAIR_GROUP_SCHED */ 13422 13423 /* 13424 * Find delta after normalizing se's vruntime with its cfs_rq's 13425 * zero_vruntime_fi, which would have been updated in prior calls 13426 * to se_fi_update(). 13427 */ 13428 delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + 13429 vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); 13430 13431 return delta > 0; 13432} 13433 13434static int task_is_throttled_fair(struct task_struct *p, int cpu) 13435{ 13436 struct cfs_rq *cfs_rq; 13437 13438#ifdef CONFIG_FAIR_GROUP_SCHED 13439 cfs_rq = task_group(p)->cfs_rq[cpu]; 13440#else 13441 cfs_rq = &cpu_rq(cpu)->cfs; 13442#endif 13443 return throttled_hierarchy(cfs_rq); 13444} 13445#else /* !CONFIG_SCHED_CORE: */ 13446static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 13447#endif /* !CONFIG_SCHED_CORE */ 13448 13449/* 13450 * scheduler tick hitting a task of our scheduling class. 13451 * 13452 * NOTE: This function can be called remotely by the tick offload that 13453 * goes along full dynticks. Therefore no local assumption can be made 13454 * and everything must be accessed through the @rq and @curr passed in 13455 * parameters. 13456 */ 13457static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 13458{ 13459 struct cfs_rq *cfs_rq; 13460 struct sched_entity *se = &curr->se; 13461 13462 for_each_sched_entity(se) { 13463 cfs_rq = cfs_rq_of(se); 13464 entity_tick(cfs_rq, se, queued); 13465 } 13466 13467 if (static_branch_unlikely(&sched_numa_balancing)) 13468 task_tick_numa(rq, curr); 13469 13470 update_misfit_status(curr, rq); 13471 check_update_overutilized_status(task_rq(curr)); 13472 13473 task_tick_core(rq, curr); 13474} 13475 13476/* 13477 * called on fork with the child task as argument from the parent's context 13478 * - child not yet on the tasklist 13479 * - preemption disabled 13480 */ 13481static void task_fork_fair(struct task_struct *p) 13482{ 13483 set_task_max_allowed_capacity(p); 13484} 13485 13486/* 13487 * Priority of the task has changed. Check to see if we preempt 13488 * the current task. 13489 */ 13490static void 13491prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) 13492{ 13493 if (!task_on_rq_queued(p)) 13494 return; 13495 13496 if (p->prio == oldprio) 13497 return; 13498 13499 if (rq->cfs.nr_queued == 1) 13500 return; 13501 13502 /* 13503 * Reschedule if we are currently running on this runqueue and 13504 * our priority decreased, or if we are not currently running on 13505 * this runqueue and our priority is higher than the current's 13506 */ 13507 if (task_current_donor(rq, p)) { 13508 if (p->prio > oldprio) 13509 resched_curr(rq); 13510 } else { 13511 wakeup_preempt(rq, p, 0); 13512 } 13513} 13514 13515#ifdef CONFIG_FAIR_GROUP_SCHED 13516/* 13517 * Propagate the changes of the sched_entity across the tg tree to make it 13518 * visible to the root 13519 */ 13520static void propagate_entity_cfs_rq(struct sched_entity *se) 13521{ 13522 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13523 13524 /* 13525 * If a task gets attached to this cfs_rq and before being queued, 13526 * it gets migrated to another CPU due to reasons like affinity 13527 * change, make sure this cfs_rq stays on leaf cfs_rq list to have 13528 * that removed load decayed or it can cause faireness problem. 13529 */ 13530 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 13531 list_add_leaf_cfs_rq(cfs_rq); 13532 13533 /* Start to propagate at parent */ 13534 se = se->parent; 13535 13536 for_each_sched_entity(se) { 13537 cfs_rq = cfs_rq_of(se); 13538 13539 update_load_avg(cfs_rq, se, UPDATE_TG); 13540 13541 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 13542 list_add_leaf_cfs_rq(cfs_rq); 13543 } 13544 13545 assert_list_leaf_cfs_rq(rq_of(cfs_rq)); 13546} 13547#else /* !CONFIG_FAIR_GROUP_SCHED: */ 13548static void propagate_entity_cfs_rq(struct sched_entity *se) { } 13549#endif /* !CONFIG_FAIR_GROUP_SCHED */ 13550 13551static void detach_entity_cfs_rq(struct sched_entity *se) 13552{ 13553 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13554 13555 /* 13556 * In case the task sched_avg hasn't been attached: 13557 * - A forked task which hasn't been woken up by wake_up_new_task(). 13558 * - A task which has been woken up by try_to_wake_up() but is 13559 * waiting for actually being woken up by sched_ttwu_pending(). 13560 */ 13561 if (!se->avg.last_update_time) 13562 return; 13563 13564 /* Catch up with the cfs_rq and remove our load when we leave */ 13565 update_load_avg(cfs_rq, se, 0); 13566 detach_entity_load_avg(cfs_rq, se); 13567 update_tg_load_avg(cfs_rq); 13568 propagate_entity_cfs_rq(se); 13569} 13570 13571static void attach_entity_cfs_rq(struct sched_entity *se) 13572{ 13573 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13574 13575 /* Synchronize entity with its cfs_rq */ 13576 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 13577 attach_entity_load_avg(cfs_rq, se); 13578 update_tg_load_avg(cfs_rq); 13579 propagate_entity_cfs_rq(se); 13580} 13581 13582static void detach_task_cfs_rq(struct task_struct *p) 13583{ 13584 struct sched_entity *se = &p->se; 13585 13586 detach_entity_cfs_rq(se); 13587} 13588 13589static void attach_task_cfs_rq(struct task_struct *p) 13590{ 13591 struct sched_entity *se = &p->se; 13592 13593 attach_entity_cfs_rq(se); 13594} 13595 13596static void switching_from_fair(struct rq *rq, struct task_struct *p) 13597{ 13598 if (p->se.sched_delayed) 13599 dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED | DEQUEUE_NOCLOCK); 13600} 13601 13602static void switched_from_fair(struct rq *rq, struct task_struct *p) 13603{ 13604 detach_task_cfs_rq(p); 13605} 13606 13607static void switched_to_fair(struct rq *rq, struct task_struct *p) 13608{ 13609 WARN_ON_ONCE(p->se.sched_delayed); 13610 13611 attach_task_cfs_rq(p); 13612 13613 set_task_max_allowed_capacity(p); 13614 13615 if (task_on_rq_queued(p)) { 13616 /* 13617 * We were most likely switched from sched_rt, so 13618 * kick off the schedule if running, otherwise just see 13619 * if we can still preempt the current task. 13620 */ 13621 if (task_current_donor(rq, p)) 13622 resched_curr(rq); 13623 else 13624 wakeup_preempt(rq, p, 0); 13625 } 13626} 13627 13628static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13629{ 13630 struct sched_entity *se = &p->se; 13631 13632 if (task_on_rq_queued(p)) { 13633 /* 13634 * Move the next running task to the front of the list, so our 13635 * cfs_tasks list becomes MRU one. 13636 */ 13637 list_move(&se->group_node, &rq->cfs_tasks); 13638 } 13639 if (!first) 13640 return; 13641 13642 WARN_ON_ONCE(se->sched_delayed); 13643 13644 if (hrtick_enabled_fair(rq)) 13645 hrtick_start_fair(rq, p); 13646 13647 update_misfit_status(p, rq); 13648 sched_fair_update_stop_tick(rq, p); 13649} 13650 13651/* 13652 * Account for a task changing its policy or group. 13653 * 13654 * This routine is mostly called to set cfs_rq->curr field when a task 13655 * migrates between groups/classes. 13656 */ 13657static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13658{ 13659 struct sched_entity *se = &p->se; 13660 13661 for_each_sched_entity(se) { 13662 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13663 13664 set_next_entity(cfs_rq, se, first); 13665 /* ensure bandwidth has been allocated on our new cfs_rq */ 13666 account_cfs_rq_runtime(cfs_rq, 0); 13667 } 13668 13669 __set_next_task_fair(rq, p, first); 13670} 13671 13672void init_cfs_rq(struct cfs_rq *cfs_rq) 13673{ 13674 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 13675 cfs_rq->zero_vruntime = (u64)(-(1LL << 20)); 13676 raw_spin_lock_init(&cfs_rq->removed.lock); 13677} 13678 13679#ifdef CONFIG_FAIR_GROUP_SCHED 13680static void task_change_group_fair(struct task_struct *p) 13681{ 13682 /* 13683 * We couldn't detach or attach a forked task which 13684 * hasn't been woken up by wake_up_new_task(). 13685 */ 13686 if (READ_ONCE(p->__state) == TASK_NEW) 13687 return; 13688 13689 detach_task_cfs_rq(p); 13690 13691 /* Tell se's cfs_rq has been changed -- migrated */ 13692 p->se.avg.last_update_time = 0; 13693 set_task_rq(p, task_cpu(p)); 13694 attach_task_cfs_rq(p); 13695} 13696 13697void free_fair_sched_group(struct task_group *tg) 13698{ 13699 int i; 13700 13701 for_each_possible_cpu(i) { 13702 if (tg->cfs_rq) 13703 kfree(tg->cfs_rq[i]); 13704 if (tg->se) 13705 kfree(tg->se[i]); 13706 } 13707 13708 kfree(tg->cfs_rq); 13709 kfree(tg->se); 13710} 13711 13712int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 13713{ 13714 struct sched_entity *se; 13715 struct cfs_rq *cfs_rq; 13716 int i; 13717 13718 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 13719 if (!tg->cfs_rq) 13720 goto err; 13721 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 13722 if (!tg->se) 13723 goto err; 13724 13725 tg->shares = NICE_0_LOAD; 13726 13727 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); 13728 13729 for_each_possible_cpu(i) { 13730 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 13731 GFP_KERNEL, cpu_to_node(i)); 13732 if (!cfs_rq) 13733 goto err; 13734 13735 se = kzalloc_node(sizeof(struct sched_entity_stats), 13736 GFP_KERNEL, cpu_to_node(i)); 13737 if (!se) 13738 goto err_free_rq; 13739 13740 init_cfs_rq(cfs_rq); 13741 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 13742 init_entity_runnable_average(se); 13743 } 13744 13745 return 1; 13746 13747err_free_rq: 13748 kfree(cfs_rq); 13749err: 13750 return 0; 13751} 13752 13753void online_fair_sched_group(struct task_group *tg) 13754{ 13755 struct sched_entity *se; 13756 struct rq_flags rf; 13757 struct rq *rq; 13758 int i; 13759 13760 for_each_possible_cpu(i) { 13761 rq = cpu_rq(i); 13762 se = tg->se[i]; 13763 rq_lock_irq(rq, &rf); 13764 update_rq_clock(rq); 13765 attach_entity_cfs_rq(se); 13766 sync_throttle(tg, i); 13767 rq_unlock_irq(rq, &rf); 13768 } 13769} 13770 13771void unregister_fair_sched_group(struct task_group *tg) 13772{ 13773 int cpu; 13774 13775 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 13776 13777 for_each_possible_cpu(cpu) { 13778 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu]; 13779 struct sched_entity *se = tg->se[cpu]; 13780 struct rq *rq = cpu_rq(cpu); 13781 13782 if (se) { 13783 if (se->sched_delayed) { 13784 guard(rq_lock_irqsave)(rq); 13785 if (se->sched_delayed) { 13786 update_rq_clock(rq); 13787 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 13788 } 13789 list_del_leaf_cfs_rq(cfs_rq); 13790 } 13791 remove_entity_load_avg(se); 13792 } 13793 13794 /* 13795 * Only empty task groups can be destroyed; so we can speculatively 13796 * check on_list without danger of it being re-added. 13797 */ 13798 if (cfs_rq->on_list) { 13799 guard(rq_lock_irqsave)(rq); 13800 list_del_leaf_cfs_rq(cfs_rq); 13801 } 13802 } 13803} 13804 13805void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 13806 struct sched_entity *se, int cpu, 13807 struct sched_entity *parent) 13808{ 13809 struct rq *rq = cpu_rq(cpu); 13810 13811 cfs_rq->tg = tg; 13812 cfs_rq->rq = rq; 13813 init_cfs_rq_runtime(cfs_rq); 13814 13815 tg->cfs_rq[cpu] = cfs_rq; 13816 tg->se[cpu] = se; 13817 13818 /* se could be NULL for root_task_group */ 13819 if (!se) 13820 return; 13821 13822 if (!parent) { 13823 se->cfs_rq = &rq->cfs; 13824 se->depth = 0; 13825 } else { 13826 se->cfs_rq = parent->my_q; 13827 se->depth = parent->depth + 1; 13828 } 13829 13830 se->my_q = cfs_rq; 13831 /* guarantee group entities always have weight */ 13832 update_load_set(&se->load, NICE_0_LOAD); 13833 se->parent = parent; 13834} 13835 13836static DEFINE_MUTEX(shares_mutex); 13837 13838static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 13839{ 13840 int i; 13841 13842 lockdep_assert_held(&shares_mutex); 13843 13844 /* 13845 * We can't change the weight of the root cgroup. 13846 */ 13847 if (!tg->se[0]) 13848 return -EINVAL; 13849 13850 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 13851 13852 if (tg->shares == shares) 13853 return 0; 13854 13855 tg->shares = shares; 13856 for_each_possible_cpu(i) { 13857 struct rq *rq = cpu_rq(i); 13858 struct sched_entity *se = tg->se[i]; 13859 struct rq_flags rf; 13860 13861 /* Propagate contribution to hierarchy */ 13862 rq_lock_irqsave(rq, &rf); 13863 update_rq_clock(rq); 13864 for_each_sched_entity(se) { 13865 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 13866 update_cfs_group(se); 13867 } 13868 rq_unlock_irqrestore(rq, &rf); 13869 } 13870 13871 return 0; 13872} 13873 13874int sched_group_set_shares(struct task_group *tg, unsigned long shares) 13875{ 13876 int ret; 13877 13878 mutex_lock(&shares_mutex); 13879 if (tg_is_idle(tg)) 13880 ret = -EINVAL; 13881 else 13882 ret = __sched_group_set_shares(tg, shares); 13883 mutex_unlock(&shares_mutex); 13884 13885 return ret; 13886} 13887 13888int sched_group_set_idle(struct task_group *tg, long idle) 13889{ 13890 int i; 13891 13892 if (tg == &root_task_group) 13893 return -EINVAL; 13894 13895 if (idle < 0 || idle > 1) 13896 return -EINVAL; 13897 13898 mutex_lock(&shares_mutex); 13899 13900 if (tg->idle == idle) { 13901 mutex_unlock(&shares_mutex); 13902 return 0; 13903 } 13904 13905 tg->idle = idle; 13906 13907 for_each_possible_cpu(i) { 13908 struct rq *rq = cpu_rq(i); 13909 struct sched_entity *se = tg->se[i]; 13910 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i]; 13911 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 13912 long idle_task_delta; 13913 struct rq_flags rf; 13914 13915 rq_lock_irqsave(rq, &rf); 13916 13917 grp_cfs_rq->idle = idle; 13918 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 13919 goto next_cpu; 13920 13921 idle_task_delta = grp_cfs_rq->h_nr_queued - 13922 grp_cfs_rq->h_nr_idle; 13923 if (!cfs_rq_is_idle(grp_cfs_rq)) 13924 idle_task_delta *= -1; 13925 13926 for_each_sched_entity(se) { 13927 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13928 13929 if (!se->on_rq) 13930 break; 13931 13932 cfs_rq->h_nr_idle += idle_task_delta; 13933 13934 /* Already accounted at parent level and above. */ 13935 if (cfs_rq_is_idle(cfs_rq)) 13936 break; 13937 } 13938 13939next_cpu: 13940 rq_unlock_irqrestore(rq, &rf); 13941 } 13942 13943 /* Idle groups have minimum weight. */ 13944 if (tg_is_idle(tg)) 13945 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 13946 else 13947 __sched_group_set_shares(tg, NICE_0_LOAD); 13948 13949 mutex_unlock(&shares_mutex); 13950 return 0; 13951} 13952 13953#endif /* CONFIG_FAIR_GROUP_SCHED */ 13954 13955 13956static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 13957{ 13958 struct sched_entity *se = &task->se; 13959 unsigned int rr_interval = 0; 13960 13961 /* 13962 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 13963 * idle runqueue: 13964 */ 13965 if (rq->cfs.load.weight) 13966 rr_interval = NS_TO_JIFFIES(se->slice); 13967 13968 return rr_interval; 13969} 13970 13971/* 13972 * All the scheduling class methods: 13973 */ 13974DEFINE_SCHED_CLASS(fair) = { 13975 13976 .queue_mask = 2, 13977 13978 .enqueue_task = enqueue_task_fair, 13979 .dequeue_task = dequeue_task_fair, 13980 .yield_task = yield_task_fair, 13981 .yield_to_task = yield_to_task_fair, 13982 13983 .wakeup_preempt = check_preempt_wakeup_fair, 13984 13985 .pick_task = pick_task_fair, 13986 .pick_next_task = pick_next_task_fair, 13987 .put_prev_task = put_prev_task_fair, 13988 .set_next_task = set_next_task_fair, 13989 13990 .select_task_rq = select_task_rq_fair, 13991 .migrate_task_rq = migrate_task_rq_fair, 13992 13993 .rq_online = rq_online_fair, 13994 .rq_offline = rq_offline_fair, 13995 13996 .task_dead = task_dead_fair, 13997 .set_cpus_allowed = set_cpus_allowed_fair, 13998 13999 .task_tick = task_tick_fair, 14000 .task_fork = task_fork_fair, 14001 14002 .reweight_task = reweight_task_fair, 14003 .prio_changed = prio_changed_fair, 14004 .switching_from = switching_from_fair, 14005 .switched_from = switched_from_fair, 14006 .switched_to = switched_to_fair, 14007 14008 .get_rr_interval = get_rr_interval_fair, 14009 14010 .update_curr = update_curr_fair, 14011 14012#ifdef CONFIG_FAIR_GROUP_SCHED 14013 .task_change_group = task_change_group_fair, 14014#endif 14015 14016#ifdef CONFIG_SCHED_CORE 14017 .task_is_throttled = task_is_throttled_fair, 14018#endif 14019 14020#ifdef CONFIG_UCLAMP_TASK 14021 .uclamp_enabled = 1, 14022#endif 14023}; 14024 14025void print_cfs_stats(struct seq_file *m, int cpu) 14026{ 14027 struct cfs_rq *cfs_rq, *pos; 14028 14029 rcu_read_lock(); 14030 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 14031 print_cfs_rq(m, cpu, cfs_rq); 14032 rcu_read_unlock(); 14033} 14034 14035#ifdef CONFIG_NUMA_BALANCING 14036void show_numa_stats(struct task_struct *p, struct seq_file *m) 14037{ 14038 int node; 14039 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 14040 struct numa_group *ng; 14041 14042 rcu_read_lock(); 14043 ng = rcu_dereference(p->numa_group); 14044 for_each_online_node(node) { 14045 if (p->numa_faults) { 14046 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 14047 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 14048 } 14049 if (ng) { 14050 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 14051 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 14052 } 14053 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 14054 } 14055 rcu_read_unlock(); 14056} 14057#endif /* CONFIG_NUMA_BALANCING */ 14058 14059__init void init_sched_fair_class(void) 14060{ 14061 int i; 14062 14063 for_each_possible_cpu(i) { 14064 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 14065 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 14066 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), 14067 GFP_KERNEL, cpu_to_node(i)); 14068 14069#ifdef CONFIG_CFS_BANDWIDTH 14070 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 14071 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 14072#endif 14073 } 14074 14075 open_softirq(SCHED_SOFTIRQ, sched_balance_softirq); 14076 14077#ifdef CONFIG_NO_HZ_COMMON 14078 nohz.next_balance = jiffies; 14079 nohz.next_blocked = jiffies; 14080 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 14081#endif 14082}
~/
Read-only file preview.