개념 설명 전체 · v6.18.37 / kernel/sched/fair.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <[email protected]> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <[email protected]> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <[email protected]> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <[email protected]> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <[email protected]> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/hugetlb_inline.h> 26 #include <linux/jiffies.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/spinlock_api.h> 30 #include <linux/cpumask_api.h> 31 #include <linux/lockdep_api.h> 32 #include <linux/softirq.h> 33 #include <linux/refcount_api.h> 34 #include <linux/topology.h> 35 #include <linux/sched/clock.h> 36 #include <linux/sched/cond_resched.h> 37 #include <linux/sched/cputime.h> 38 #include <linux/sched/isolation.h> 39 #include <linux/sched/nohz.h> 40 #include <linux/sched/prio.h> 41 42 #include <linux/cpuidle.h> 43 #include <linux/interrupt.h> 44 #include <linux/memory-tiers.h> 45 #include <linux/mempolicy.h> 46 #include <linux/mutex_api.h> 47 #include <linux/profile.h> 48 #include <linux/psi.h> 49 #include <linux/ratelimit.h> 50 #include <linux/task_work.h> 51 #include <linux/rbtree_augmented.h> 52 53 #include <asm/switch_to.h> 54 55 #include <uapi/linux/sched/types.h> 56 57 #include "sched.h" 58 #include "stats.h" 59 #include "autogroup.h" 60 61 /* 62 * The initial- and re-scaling of tunables is configurable 63 * 64 * Options are: 65 * 66 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 67 * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) 68 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 69 * 70 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 71 */ 72 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 73 74 /* 75 * Minimal preemption granularity for CPU-bound tasks: 76 * 77 * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) 78 */ 79 unsigned int sysctl_sched_base_slice = 700000ULL; 80 static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; 81 82 __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; 83 84 static int __init setup_sched_thermal_decay_shift(char *str) 85 { 86 pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n"); 87 return 1; 88 } 89 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 90 91 /* 92 * For asym packing, by default the lower numbered CPU has higher priority. 93 */ 94 int __weak arch_asym_cpu_priority(int cpu) 95 { 96 return -cpu; 97 } 98 99 /* 100 * The margin used when comparing utilization with CPU capacity. 101 * 102 * (default: ~20%) 103 */ 104 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 105 106 /* 107 * The margin used when comparing CPU capacities. 108 * is 'cap1' noticeably greater than 'cap2' 109 * 110 * (default: ~5%) 111 */ 112 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 113 114 #ifdef CONFIG_CFS_BANDWIDTH 115 /* 116 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 117 * each time a cfs_rq requests quota. 118 * 119 * Note: in the case that the slice exceeds the runtime remaining (either due 120 * to consumption or the quota being specified to be smaller than the slice) 121 * we will always only issue the remaining available time. 122 * 123 * (default: 5 msec, units: microseconds) 124 */ 125 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 126 #endif 127 128 #ifdef CONFIG_NUMA_BALANCING 129 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 130 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 131 #endif 132 133 #ifdef CONFIG_SYSCTL 134 static const struct ctl_table sched_fair_sysctls[] = { 135 #ifdef CONFIG_CFS_BANDWIDTH 136 { 137 .procname = "sched_cfs_bandwidth_slice_us", 138 .data = &sysctl_sched_cfs_bandwidth_slice, 139 .maxlen = sizeof(unsigned int), 140 .mode = 0644, 141 .proc_handler = proc_dointvec_minmax, 142 .extra1 = SYSCTL_ONE, 143 }, 144 #endif 145 #ifdef CONFIG_NUMA_BALANCING 146 { 147 .procname = "numa_balancing_promote_rate_limit_MBps", 148 .data = &sysctl_numa_balancing_promote_rate_limit, 149 .maxlen = sizeof(unsigned int), 150 .mode = 0644, 151 .proc_handler = proc_dointvec_minmax, 152 .extra1 = SYSCTL_ZERO, 153 }, 154 #endif /* CONFIG_NUMA_BALANCING */ 155 }; 156 157 static int __init sched_fair_sysctl_init(void) 158 { 159 register_sysctl_init("kernel", sched_fair_sysctls); 160 return 0; 161 } 162 late_initcall(sched_fair_sysctl_init); 163 #endif /* CONFIG_SYSCTL */ 164 165 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 166 { 167 lw->weight += inc; 168 lw->inv_weight = 0; 169 } 170 171 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 172 { 173 lw->weight -= dec; 174 lw->inv_weight = 0; 175 } 176 177 static inline void update_load_set(struct load_weight *lw, unsigned long w) 178 { 179 lw->weight = w; 180 lw->inv_weight = 0; 181 } 182 183 /* 184 * Increase the granularity value when there are more CPUs, 185 * because with more CPUs the 'effective latency' as visible 186 * to users decreases. But the relationship is not linear, 187 * so pick a second-best guess by going with the log2 of the 188 * number of CPUs. 189 * 190 * This idea comes from the SD scheduler of Con Kolivas: 191 */ 192 static unsigned int get_update_sysctl_factor(void) 193 { 194 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 195 unsigned int factor; 196 197 switch (sysctl_sched_tunable_scaling) { 198 case SCHED_TUNABLESCALING_NONE: 199 factor = 1; 200 break; 201 case SCHED_TUNABLESCALING_LINEAR: 202 factor = cpus; 203 break; 204 case SCHED_TUNABLESCALING_LOG: 205 default: 206 factor = 1 + ilog2(cpus); 207 break; 208 } 209 210 return factor; 211 } 212 213 static void update_sysctl(void) 214 { 215 unsigned int factor = get_update_sysctl_factor(); 216 217 #define SET_SYSCTL(name) \ 218 (sysctl_##name = (factor) * normalized_sysctl_##name) 219 SET_SYSCTL(sched_base_slice); 220 #undef SET_SYSCTL 221 } 222 223 void __init sched_init_granularity(void) 224 { 225 update_sysctl(); 226 } 227 228 #define WMULT_CONST (~0U) 229 #define WMULT_SHIFT 32 230 231 static void __update_inv_weight(struct load_weight *lw) 232 { 233 unsigned long w; 234 235 if (likely(lw->inv_weight)) 236 return; 237 238 w = scale_load_down(lw->weight); 239 240 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 241 lw->inv_weight = 1; 242 else if (unlikely(!w)) 243 lw->inv_weight = WMULT_CONST; 244 else 245 lw->inv_weight = WMULT_CONST / w; 246 } 247 248 /* 249 * delta_exec * weight / lw.weight 250 * OR 251 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 252 * 253 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 254 * we're guaranteed shift stays positive because inv_weight is guaranteed to 255 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 256 * 257 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 258 * weight/lw.weight <= 1, and therefore our shift will also be positive. 259 */ 260 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 261 { 262 u64 fact = scale_load_down(weight); 263 u32 fact_hi = (u32)(fact >> 32); 264 int shift = WMULT_SHIFT; 265 int fs; 266 267 __update_inv_weight(lw); 268 269 if (unlikely(fact_hi)) { 270 fs = fls(fact_hi); 271 shift -= fs; 272 fact >>= fs; 273 } 274 275 fact = mul_u32_u32(fact, lw->inv_weight); 276 277 fact_hi = (u32)(fact >> 32); 278 if (fact_hi) { 279 fs = fls(fact_hi); 280 shift -= fs; 281 fact >>= fs; 282 } 283 284 return mul_u64_u32_shr(delta_exec, fact, shift); 285 } 286 287 /* 288 * delta /= w 289 */ 290 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 291 { 292 if (unlikely(se->load.weight != NICE_0_LOAD)) 293 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 294 295 return delta; 296 } 297 298 const struct sched_class fair_sched_class; 299 300 /************************************************************** 301 * CFS operations on generic schedulable entities: 302 */ 303 304 #ifdef CONFIG_FAIR_GROUP_SCHED 305 306 /* Walk up scheduling entities hierarchy */ 307 #define for_each_sched_entity(se) \ 308 for (; se; se = se->parent) 309 310 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 311 { 312 struct rq *rq = rq_of(cfs_rq); 313 int cpu = cpu_of(rq); 314 315 if (cfs_rq->on_list) 316 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 317 318 cfs_rq->on_list = 1; 319 320 /* 321 * Ensure we either appear before our parent (if already 322 * enqueued) or force our parent to appear after us when it is 323 * enqueued. The fact that we always enqueue bottom-up 324 * reduces this to two cases and a special case for the root 325 * cfs_rq. Furthermore, it also means that we will always reset 326 * tmp_alone_branch either when the branch is connected 327 * to a tree or when we reach the top of the tree 328 */ 329 if (cfs_rq->tg->parent && 330 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 331 /* 332 * If parent is already on the list, we add the child 333 * just before. Thanks to circular linked property of 334 * the list, this means to put the child at the tail 335 * of the list that starts by parent. 336 */ 337 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 338 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 339 /* 340 * The branch is now connected to its tree so we can 341 * reset tmp_alone_branch to the beginning of the 342 * list. 343 */ 344 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 345 return true; 346 } 347 348 if (!cfs_rq->tg->parent) { 349 /* 350 * cfs rq without parent should be put 351 * at the tail of the list. 352 */ 353 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 354 &rq->leaf_cfs_rq_list); 355 /* 356 * We have reach the top of a tree so we can reset 357 * tmp_alone_branch to the beginning of the list. 358 */ 359 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 360 return true; 361 } 362 363 /* 364 * The parent has not already been added so we want to 365 * make sure that it will be put after us. 366 * tmp_alone_branch points to the begin of the branch 367 * where we will add parent. 368 */ 369 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 370 /* 371 * update tmp_alone_branch to points to the new begin 372 * of the branch 373 */ 374 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 375 return false; 376 } 377 378 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 379 { 380 if (cfs_rq->on_list) { 381 struct rq *rq = rq_of(cfs_rq); 382 383 /* 384 * With cfs_rq being unthrottled/throttled during an enqueue, 385 * it can happen the tmp_alone_branch points to the leaf that 386 * we finally want to delete. In this case, tmp_alone_branch moves 387 * to the prev element but it will point to rq->leaf_cfs_rq_list 388 * at the end of the enqueue. 389 */ 390 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 391 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 392 393 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 394 cfs_rq->on_list = 0; 395 } 396 } 397 398 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 399 { 400 WARN_ON_ONCE(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 401 } 402 403 /* Iterate through all leaf cfs_rq's on a runqueue */ 404 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 405 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 406 leaf_cfs_rq_list) 407 408 /* Do the two (enqueued) entities belong to the same group ? */ 409 static inline struct cfs_rq * 410 is_same_group(struct sched_entity *se, struct sched_entity *pse) 411 { 412 if (se->cfs_rq == pse->cfs_rq) 413 return se->cfs_rq; 414 415 return NULL; 416 } 417 418 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 419 { 420 return se->parent; 421 } 422 423 static void 424 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 425 { 426 int se_depth, pse_depth; 427 428 /* 429 * preemption test can be made between sibling entities who are in the 430 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 431 * both tasks until we find their ancestors who are siblings of common 432 * parent. 433 */ 434 435 /* First walk up until both entities are at same depth */ 436 se_depth = (*se)->depth; 437 pse_depth = (*pse)->depth; 438 439 while (se_depth > pse_depth) { 440 se_depth--; 441 *se = parent_entity(*se); 442 } 443 444 while (pse_depth > se_depth) { 445 pse_depth--; 446 *pse = parent_entity(*pse); 447 } 448 449 while (!is_same_group(*se, *pse)) { 450 *se = parent_entity(*se); 451 *pse = parent_entity(*pse); 452 } 453 } 454 455 static int tg_is_idle(struct task_group *tg) 456 { 457 return tg->idle > 0; 458 } 459 460 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 461 { 462 return cfs_rq->idle > 0; 463 } 464 465 static int se_is_idle(struct sched_entity *se) 466 { 467 if (entity_is_task(se)) 468 return task_has_idle_policy(task_of(se)); 469 return cfs_rq_is_idle(group_cfs_rq(se)); 470 } 471 472 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 473 474 #define for_each_sched_entity(se) \ 475 for (; se; se = NULL) 476 477 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 478 { 479 return true; 480 } 481 482 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 483 { 484 } 485 486 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 487 { 488 } 489 490 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 491 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 492 493 static inline struct sched_entity *parent_entity(struct sched_entity *se) 494 { 495 return NULL; 496 } 497 498 static inline void 499 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 500 { 501 } 502 503 static inline int tg_is_idle(struct task_group *tg) 504 { 505 return 0; 506 } 507 508 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 509 { 510 return 0; 511 } 512 513 static int se_is_idle(struct sched_entity *se) 514 { 515 return task_has_idle_policy(task_of(se)); 516 } 517 518 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 519 520 static __always_inline 521 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 522 523 /************************************************************** 524 * Scheduling class tree data structure manipulation methods: 525 */ 526 527 extern void __BUILD_BUG_vruntime_cmp(void); 528 529 /* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ 530 531 #define vruntime_cmp(A, CMP_STR, B) ({ \ 532 int __res = 0; \ 533 \ 534 if (!__builtin_strcmp(CMP_STR, "<")) { \ 535 __res = ((s64)((A)-(B)) < 0); \ 536 } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ 537 __res = ((s64)((A)-(B)) <= 0); \ 538 } else if (!__builtin_strcmp(CMP_STR, ">")) { \ 539 __res = ((s64)((A)-(B)) > 0); \ 540 } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ 541 __res = ((s64)((A)-(B)) >= 0); \ 542 } else { \ 543 /* Unknown operator throws linker error: */ \ 544 __BUILD_BUG_vruntime_cmp(); \ 545 } \ 546 \ 547 __res; \ 548 }) 549 550 extern void __BUILD_BUG_vruntime_op(void); 551 552 #define vruntime_op(A, OP_STR, B) ({ \ 553 s64 __res = 0; \ 554 \ 555 if (!__builtin_strcmp(OP_STR, "-")) { \ 556 __res = (s64)((A)-(B)); \ 557 } else { \ 558 /* Unknown operator throws linker error: */ \ 559 __BUILD_BUG_vruntime_op(); \ 560 } \ 561 \ 562 __res; \ 563 }) 564 565 566 static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) 567 { 568 if (vruntime_cmp(vruntime, ">", max_vruntime)) 569 max_vruntime = vruntime; 570 571 return max_vruntime; 572 } 573 574 static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) 575 { 576 if (vruntime_cmp(vruntime, "<", min_vruntime)) 577 min_vruntime = vruntime; 578 579 return min_vruntime; 580 } 581 582 static inline bool entity_before(const struct sched_entity *a, 583 const struct sched_entity *b) 584 { 585 /* 586 * Tiebreak on vruntime seems unnecessary since it can 587 * hardly happen. 588 */ 589 return vruntime_cmp(a->deadline, "<", b->deadline); 590 } 591 592 /* 593 * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale 594 * and this value should be no more than two lag bounds. Which puts it in the 595 * general order of: 596 * 597 * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT 598 * 599 * which is around 44 bits in size (on 64bit); that is 20 for 600 * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for 601 * however many msec the actual slice+tick ends up begin. 602 * 603 * (disregarding the actual divide-by-weight part makes for the worst case 604 * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually 605 * being the zero-lag point). 606 */ 607 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 608 { 609 return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); 610 } 611 612 #define __node_2_se(node) \ 613 rb_entry((node), struct sched_entity, run_node) 614 615 /* 616 * Compute virtual time from the per-task service numbers: 617 * 618 * Fair schedulers conserve lag: 619 * 620 * \Sum lag_i = 0 621 * 622 * Where lag_i is given by: 623 * 624 * lag_i = S - s_i = w_i * (V - v_i) 625 * 626 * Where S is the ideal service time and V is it's virtual time counterpart. 627 * Therefore: 628 * 629 * \Sum lag_i = 0 630 * \Sum w_i * (V - v_i) = 0 631 * \Sum w_i * V - w_i * v_i = 0 632 * 633 * From which we can solve an expression for V in v_i (which we have in 634 * se->vruntime): 635 * 636 * \Sum v_i * w_i \Sum v_i * w_i 637 * V = -------------- = -------------- 638 * \Sum w_i W 639 * 640 * Specifically, this is the weighted average of all entity virtual runtimes. 641 * 642 * [[ NOTE: this is only equal to the ideal scheduler under the condition 643 * that join/leave operations happen at lag_i = 0, otherwise the 644 * virtual time has non-contiguous motion equivalent to: 645 * 646 * V +-= lag_i / W 647 * 648 * Also see the comment in place_entity() that deals with this. ]] 649 * 650 * However, since v_i is u64, and the multiplication could easily overflow 651 * transform it into a relative form that uses smaller quantities: 652 * 653 * Substitute: v_i == (v_i - v0) + v0 654 * 655 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 656 * V = ---------------------------- = --------------------- + v0 657 * W W 658 * 659 * Which we track using: 660 * 661 * v0 := cfs_rq->zero_vruntime 662 * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime 663 * \Sum w_i := cfs_rq->sum_weight 664 * 665 * Since zero_vruntime closely tracks the per-task service, these 666 * deltas: (v_i - v), will be in the order of the maximal (virtual) lag 667 * induced in the system due to quantisation. 668 * 669 * Also, we use scale_load_down() to reduce the size. 670 * 671 * As measured, the max (key * weight) value was ~44 bits for a kernel build. 672 */ 673 static void 674 sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 675 { 676 unsigned long weight = scale_load_down(se->load.weight); 677 s64 key = entity_key(cfs_rq, se); 678 679 cfs_rq->sum_w_vruntime += key * weight; 680 cfs_rq->sum_weight += weight; 681 } 682 683 static void 684 sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 685 { 686 unsigned long weight = scale_load_down(se->load.weight); 687 s64 key = entity_key(cfs_rq, se); 688 689 cfs_rq->sum_w_vruntime -= key * weight; 690 cfs_rq->sum_weight -= weight; 691 } 692 693 static inline 694 void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) 695 { 696 /* 697 * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight 698 */ 699 cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; 700 cfs_rq->zero_vruntime += delta; 701 } 702 703 /* 704 * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true 705 * For this to be so, the result of this function must have a left bias. 706 * 707 * Called in: 708 * - place_entity() -- before enqueue 709 * - update_entity_lag() -- before dequeue 710 * - update_deadline() -- slice expiration 711 * 712 * This means it is one entry 'behind' but that puts it close enough to where 713 * the bound on entity_key() is at most two lag bounds. 714 */ 715 u64 avg_vruntime(struct cfs_rq *cfs_rq) 716 { 717 struct sched_entity *curr = cfs_rq->curr; 718 long weight = cfs_rq->sum_weight; 719 s64 delta = 0; 720 721 if (curr && !curr->on_rq) 722 curr = NULL; 723 724 if (weight) { 725 s64 runtime = cfs_rq->sum_w_vruntime; 726 727 if (curr) { 728 unsigned long w = scale_load_down(curr->load.weight); 729 730 runtime += entity_key(cfs_rq, curr) * w; 731 weight += w; 732 } 733 734 /* sign flips effective floor / ceiling */ 735 if (runtime < 0) 736 runtime -= (weight - 1); 737 738 delta = div_s64(runtime, weight); 739 } else if (curr) { 740 /* 741 * When there is but one element, it is the average. 742 */ 743 delta = curr->vruntime - cfs_rq->zero_vruntime; 744 } 745 746 update_zero_vruntime(cfs_rq, delta); 747 748 return cfs_rq->zero_vruntime; 749 } 750 751 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); 752 753 /* 754 * lag_i = S - s_i = w_i * (V - v_i) 755 * 756 * However, since V is approximated by the weighted average of all entities it 757 * is possible -- by addition/removal/reweight to the tree -- to move V around 758 * and end up with a larger lag than we started with. 759 * 760 * Limit this to either double the slice length with a minimum of TICK_NSEC 761 * since that is the timing granularity. 762 * 763 * EEVDF gives the following limit for a steady state system: 764 * 765 * -r_max < lag < max(r_max, q) 766 */ 767 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 768 { 769 u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; 770 s64 vlag, limit; 771 772 WARN_ON_ONCE(!se->on_rq); 773 774 vlag = avg_vruntime(cfs_rq) - se->vruntime; 775 limit = calc_delta_fair(max_slice, se); 776 777 se->vlag = clamp(vlag, -limit, limit); 778 } 779 780 /* 781 * Entity is eligible once it received less service than it ought to have, 782 * eg. lag >= 0. 783 * 784 * lag_i = S - s_i = w_i*(V - v_i) 785 * 786 * lag_i >= 0 -> V >= v_i 787 * 788 * \Sum (v_i - v)*w_i 789 * V = ------------------ + v 790 * \Sum w_i 791 * 792 * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i) 793 * 794 * Note: using 'avg_vruntime() > se->vruntime' is inaccurate due 795 * to the loss in precision caused by the division. 796 */ 797 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) 798 { 799 struct sched_entity *curr = cfs_rq->curr; 800 s64 avg = cfs_rq->sum_w_vruntime; 801 long load = cfs_rq->sum_weight; 802 803 if (curr && curr->on_rq) { 804 unsigned long weight = scale_load_down(curr->load.weight); 805 806 avg += entity_key(cfs_rq, curr) * weight; 807 load += weight; 808 } 809 810 return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; 811 } 812 813 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 814 { 815 return vruntime_eligible(cfs_rq, se->vruntime); 816 } 817 818 static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) 819 { 820 struct sched_entity *root = __pick_root_entity(cfs_rq); 821 struct sched_entity *curr = cfs_rq->curr; 822 u64 min_slice = ~0ULL; 823 824 if (curr && curr->on_rq) 825 min_slice = curr->slice; 826 827 if (root) 828 min_slice = min(min_slice, root->min_slice); 829 830 return min_slice; 831 } 832 833 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) 834 { 835 struct sched_entity *root = __pick_root_entity(cfs_rq); 836 struct sched_entity *curr = cfs_rq->curr; 837 u64 max_slice = 0ULL; 838 839 if (curr && curr->on_rq) 840 max_slice = curr->slice; 841 842 if (root) 843 max_slice = max(max_slice, root->max_slice); 844 845 return max_slice; 846 } 847 848 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 849 { 850 return entity_before(__node_2_se(a), __node_2_se(b)); 851 } 852 853 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) 854 { 855 if (node) { 856 struct sched_entity *rse = __node_2_se(node); 857 858 if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) 859 se->min_vruntime = rse->min_vruntime; 860 } 861 } 862 863 static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) 864 { 865 if (node) { 866 struct sched_entity *rse = __node_2_se(node); 867 if (rse->min_slice < se->min_slice) 868 se->min_slice = rse->min_slice; 869 } 870 } 871 872 static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) 873 { 874 if (node) { 875 struct sched_entity *rse = __node_2_se(node); 876 if (rse->max_slice > se->max_slice) 877 se->max_slice = rse->max_slice; 878 } 879 } 880 881 /* 882 * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) 883 */ 884 static inline bool min_vruntime_update(struct sched_entity *se, bool exit) 885 { 886 u64 old_min_vruntime = se->min_vruntime; 887 u64 old_min_slice = se->min_slice; 888 u64 old_max_slice = se->max_slice; 889 struct rb_node *node = &se->run_node; 890 891 se->min_vruntime = se->vruntime; 892 __min_vruntime_update(se, node->rb_right); 893 __min_vruntime_update(se, node->rb_left); 894 895 se->min_slice = se->slice; 896 __min_slice_update(se, node->rb_right); 897 __min_slice_update(se, node->rb_left); 898 899 se->max_slice = se->slice; 900 __max_slice_update(se, node->rb_right); 901 __max_slice_update(se, node->rb_left); 902 903 return se->min_vruntime == old_min_vruntime && 904 se->min_slice == old_min_slice && 905 se->max_slice == old_max_slice; 906 } 907 908 RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, 909 run_node, min_vruntime, min_vruntime_update); 910 911 /* 912 * Enqueue an entity into the rb-tree: 913 */ 914 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 915 { 916 sum_w_vruntime_add(cfs_rq, se); 917 se->min_vruntime = se->vruntime; 918 se->min_slice = se->slice; 919 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 920 __entity_less, &min_vruntime_cb); 921 } 922 923 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 924 { 925 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 926 &min_vruntime_cb); 927 sum_w_vruntime_sub(cfs_rq, se); 928 } 929 930 struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) 931 { 932 struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node; 933 934 if (!root) 935 return NULL; 936 937 return __node_2_se(root); 938 } 939 940 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 941 { 942 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 943 944 if (!left) 945 return NULL; 946 947 return __node_2_se(left); 948 } 949 950 /* 951 * Set the vruntime up to which an entity can run before looking 952 * for another entity to pick. 953 * In case of run to parity, we use the shortest slice of the enqueued 954 * entities to set the protected period. 955 * When run to parity is disabled, we give a minimum quantum to the running 956 * entity to ensure progress. 957 */ 958 static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 959 { 960 u64 slice = normalized_sysctl_sched_base_slice; 961 u64 vprot = se->deadline; 962 963 if (sched_feat(RUN_TO_PARITY)) 964 slice = cfs_rq_min_slice(cfs_rq); 965 966 slice = min(slice, se->slice); 967 if (slice != se->slice) 968 vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); 969 970 se->vprot = vprot; 971 } 972 973 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 974 { 975 u64 slice = cfs_rq_min_slice(cfs_rq); 976 977 se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); 978 } 979 980 static inline bool protect_slice(struct sched_entity *se) 981 { 982 return vruntime_cmp(se->vruntime, "<", se->vprot); 983 } 984 985 static inline void cancel_protect_slice(struct sched_entity *se) 986 { 987 if (protect_slice(se)) 988 se->vprot = se->vruntime; 989 } 990 991 /* 992 * Earliest Eligible Virtual Deadline First 993 * 994 * In order to provide latency guarantees for different request sizes 995 * EEVDF selects the best runnable task from two criteria: 996 * 997 * 1) the task must be eligible (must be owed service) 998 * 999 * 2) from those tasks that meet 1), we select the one 1000 * with the earliest virtual deadline. 1001 * 1002 * We can do this in O(log n) time due to an augmented RB-tree. The 1003 * tree keeps the entries sorted on deadline, but also functions as a 1004 * heap based on the vruntime by keeping: 1005 * 1006 * se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime) 1007 * 1008 * Which allows tree pruning through eligibility. 1009 */ 1010 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) 1011 { 1012 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 1013 struct sched_entity *se = __pick_first_entity(cfs_rq); 1014 struct sched_entity *curr = cfs_rq->curr; 1015 struct sched_entity *best = NULL; 1016 1017 /* 1018 * We can safely skip eligibility check if there is only one entity 1019 * in this cfs_rq, saving some cycles. 1020 */ 1021 if (cfs_rq->nr_queued == 1) 1022 return curr && curr->on_rq ? curr : se; 1023 1024 /* 1025 * Picking the ->next buddy will affect latency but not fairness. 1026 */ 1027 if (sched_feat(PICK_BUDDY) && 1028 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) { 1029 /* ->next will never be delayed */ 1030 WARN_ON_ONCE(cfs_rq->next->sched_delayed); 1031 return cfs_rq->next; 1032 } 1033 1034 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 1035 curr = NULL; 1036 1037 if (curr && protect && protect_slice(curr)) 1038 return curr; 1039 1040 /* Pick the leftmost entity if it's eligible */ 1041 if (se && entity_eligible(cfs_rq, se)) { 1042 best = se; 1043 goto found; 1044 } 1045 1046 /* Heap search for the EEVD entity */ 1047 while (node) { 1048 struct rb_node *left = node->rb_left; 1049 1050 /* 1051 * Eligible entities in left subtree are always better 1052 * choices, since they have earlier deadlines. 1053 */ 1054 if (left && vruntime_eligible(cfs_rq, 1055 __node_2_se(left)->min_vruntime)) { 1056 node = left; 1057 continue; 1058 } 1059 1060 se = __node_2_se(node); 1061 1062 /* 1063 * The left subtree either is empty or has no eligible 1064 * entity, so check the current node since it is the one 1065 * with earliest deadline that might be eligible. 1066 */ 1067 if (entity_eligible(cfs_rq, se)) { 1068 best = se; 1069 break; 1070 } 1071 1072 node = node->rb_right; 1073 } 1074 found: 1075 if (!best || (curr && entity_before(curr, best))) 1076 best = curr; 1077 1078 return best; 1079 } 1080 1081 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 1082 { 1083 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 1084 1085 if (!last) 1086 return NULL; 1087 1088 return __node_2_se(last); 1089 } 1090 1091 /************************************************************** 1092 * Scheduling class statistics methods: 1093 */ 1094 int sched_update_scaling(void) 1095 { 1096 unsigned int factor = get_update_sysctl_factor(); 1097 1098 #define WRT_SYSCTL(name) \ 1099 (normalized_sysctl_##name = sysctl_##name / (factor)) 1100 WRT_SYSCTL(sched_base_slice); 1101 #undef WRT_SYSCTL 1102 1103 return 0; 1104 } 1105 1106 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1107 1108 /* 1109 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1110 * this is probably good enough. 1111 */ 1112 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1113 { 1114 if (vruntime_cmp(se->vruntime, "<", se->deadline)) 1115 return false; 1116 1117 /* 1118 * For EEVDF the virtual time slope is determined by w_i (iow. 1119 * nice) while the request time r_i is determined by 1120 * sysctl_sched_base_slice. 1121 */ 1122 if (!se->custom_slice) 1123 se->slice = sysctl_sched_base_slice; 1124 1125 /* 1126 * EEVDF: vd_i = ve_i + r_i / w_i 1127 */ 1128 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1129 avg_vruntime(cfs_rq); 1130 1131 /* 1132 * The task has consumed its request, reschedule. 1133 */ 1134 return true; 1135 } 1136 1137 #include "pelt.h" 1138 1139 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1140 static unsigned long task_h_load(struct task_struct *p); 1141 static unsigned long capacity_of(int cpu); 1142 1143 /* Give new sched_entity start runnable values to heavy its load in infant time */ 1144 void init_entity_runnable_average(struct sched_entity *se) 1145 { 1146 struct sched_avg *sa = &se->avg; 1147 1148 memset(sa, 0, sizeof(*sa)); 1149 1150 /* 1151 * Tasks are initialized with full load to be seen as heavy tasks until 1152 * they get a chance to stabilize to their real load level. 1153 * Group entities are initialized with zero load to reflect the fact that 1154 * nothing has been attached to the task group yet. 1155 */ 1156 if (entity_is_task(se)) 1157 sa->load_avg = scale_load_down(se->load.weight); 1158 1159 /* when this task is enqueued, it will contribute to its cfs_rq's load_avg */ 1160 } 1161 1162 /* 1163 * With new tasks being created, their initial util_avgs are extrapolated 1164 * based on the cfs_rq's current util_avg: 1165 * 1166 * util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1) 1167 * * se_weight(se) 1168 * 1169 * However, in many cases, the above util_avg does not give a desired 1170 * value. Moreover, the sum of the util_avgs may be divergent, such 1171 * as when the series is a harmonic series. 1172 * 1173 * To solve this problem, we also cap the util_avg of successive tasks to 1174 * only 1/2 of the left utilization budget: 1175 * 1176 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1177 * 1178 * where n denotes the nth task and cpu_scale the CPU capacity. 1179 * 1180 * For example, for a CPU with 1024 of capacity, a simplest series from 1181 * the beginning would be like: 1182 * 1183 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1184 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1185 * 1186 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1187 * if util_avg > util_avg_cap. 1188 */ 1189 void post_init_entity_util_avg(struct task_struct *p) 1190 { 1191 struct sched_entity *se = &p->se; 1192 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1193 struct sched_avg *sa = &se->avg; 1194 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1195 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1196 1197 if (p->sched_class != &fair_sched_class) { 1198 /* 1199 * For !fair tasks do: 1200 * 1201 update_cfs_rq_load_avg(now, cfs_rq); 1202 attach_entity_load_avg(cfs_rq, se); 1203 switched_from_fair(rq, p); 1204 * 1205 * such that the next switched_to_fair() has the 1206 * expected state. 1207 */ 1208 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1209 return; 1210 } 1211 1212 if (cap > 0) { 1213 if (cfs_rq->avg.util_avg != 0) { 1214 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); 1215 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1216 1217 if (sa->util_avg > cap) 1218 sa->util_avg = cap; 1219 } else { 1220 sa->util_avg = cap; 1221 } 1222 } 1223 1224 sa->runnable_avg = sa->util_avg; 1225 } 1226 1227 static s64 update_se(struct rq *rq, struct sched_entity *se) 1228 { 1229 u64 now = rq_clock_task(rq); 1230 s64 delta_exec; 1231 1232 delta_exec = now - se->exec_start; 1233 if (unlikely(delta_exec <= 0)) 1234 return delta_exec; 1235 1236 se->exec_start = now; 1237 if (entity_is_task(se)) { 1238 struct task_struct *donor = task_of(se); 1239 struct task_struct *running = rq->curr; 1240 /* 1241 * If se is a task, we account the time against the running 1242 * task, as w/ proxy-exec they may not be the same. 1243 */ 1244 running->se.exec_start = now; 1245 running->se.sum_exec_runtime += delta_exec; 1246 1247 trace_sched_stat_runtime(running, delta_exec); 1248 account_group_exec_runtime(running, delta_exec); 1249 1250 /* cgroup time is always accounted against the donor */ 1251 cgroup_account_cputime(donor, delta_exec); 1252 } else { 1253 /* If not task, account the time against donor se */ 1254 se->sum_exec_runtime += delta_exec; 1255 } 1256 1257 if (schedstat_enabled()) { 1258 struct sched_statistics *stats; 1259 1260 stats = __schedstats_from_se(se); 1261 __schedstat_set(stats->exec_max, 1262 max(delta_exec, stats->exec_max)); 1263 } 1264 1265 return delta_exec; 1266 } 1267 1268 static void set_next_buddy(struct sched_entity *se); 1269 1270 /* 1271 * Used by other classes to account runtime. 1272 */ 1273 s64 update_curr_common(struct rq *rq) 1274 { 1275 return update_se(rq, &rq->donor->se); 1276 } 1277 1278 /* 1279 * Update the current task's runtime statistics. 1280 */ 1281 static void update_curr(struct cfs_rq *cfs_rq) 1282 { 1283 /* 1284 * Note: cfs_rq->curr corresponds to the task picked to 1285 * run (ie: rq->donor.se) which due to proxy-exec may 1286 * not necessarily be the actual task running 1287 * (rq->curr.se). This is easy to confuse! 1288 */ 1289 struct sched_entity *curr = cfs_rq->curr; 1290 struct rq *rq = rq_of(cfs_rq); 1291 s64 delta_exec; 1292 bool resched; 1293 1294 if (unlikely(!curr)) 1295 return; 1296 1297 delta_exec = update_se(rq, curr); 1298 if (unlikely(delta_exec <= 0)) 1299 return; 1300 1301 curr->vruntime += calc_delta_fair(delta_exec, curr); 1302 resched = update_deadline(cfs_rq, curr); 1303 1304 if (entity_is_task(curr)) { 1305 /* 1306 * If the fair_server is active, we need to account for the 1307 * fair_server time whether or not the task is running on 1308 * behalf of fair_server or not: 1309 * - If the task is running on behalf of fair_server, we need 1310 * to limit its time based on the assigned runtime. 1311 * - Fair task that runs outside of fair_server should account 1312 * against fair_server such that it can account for this time 1313 * and possibly avoid running this period. 1314 */ 1315 if (dl_server_active(&rq->fair_server)) 1316 dl_server_update(&rq->fair_server, delta_exec); 1317 } 1318 1319 account_cfs_rq_runtime(cfs_rq, delta_exec); 1320 1321 if (cfs_rq->nr_queued == 1) 1322 return; 1323 1324 if (resched || !protect_slice(curr)) { 1325 resched_curr_lazy(rq); 1326 clear_buddies(cfs_rq, curr); 1327 } 1328 } 1329 1330 static void update_curr_fair(struct rq *rq) 1331 { 1332 update_curr(cfs_rq_of(&rq->donor->se)); 1333 } 1334 1335 static inline void 1336 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1337 { 1338 struct sched_statistics *stats; 1339 struct task_struct *p = NULL; 1340 1341 if (!schedstat_enabled()) 1342 return; 1343 1344 stats = __schedstats_from_se(se); 1345 1346 if (entity_is_task(se)) 1347 p = task_of(se); 1348 1349 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 1350 } 1351 1352 static inline void 1353 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1354 { 1355 struct sched_statistics *stats; 1356 struct task_struct *p = NULL; 1357 1358 if (!schedstat_enabled()) 1359 return; 1360 1361 stats = __schedstats_from_se(se); 1362 1363 /* 1364 * When the sched_schedstat changes from 0 to 1, some sched se 1365 * maybe already in the runqueue, the se->statistics.wait_start 1366 * will be 0.So it will let the delta wrong. We need to avoid this 1367 * scenario. 1368 */ 1369 if (unlikely(!schedstat_val(stats->wait_start))) 1370 return; 1371 1372 if (entity_is_task(se)) 1373 p = task_of(se); 1374 1375 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 1376 } 1377 1378 static inline void 1379 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1380 { 1381 struct sched_statistics *stats; 1382 struct task_struct *tsk = NULL; 1383 1384 if (!schedstat_enabled()) 1385 return; 1386 1387 stats = __schedstats_from_se(se); 1388 1389 if (entity_is_task(se)) 1390 tsk = task_of(se); 1391 1392 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1393 } 1394 1395 /* 1396 * Task is being enqueued - update stats: 1397 */ 1398 static inline void 1399 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1400 { 1401 if (!schedstat_enabled()) 1402 return; 1403 1404 /* 1405 * Are we enqueueing a waiting task? (for current tasks 1406 * a dequeue/enqueue event is a NOP) 1407 */ 1408 if (se != cfs_rq->curr) 1409 update_stats_wait_start_fair(cfs_rq, se); 1410 1411 if (flags & ENQUEUE_WAKEUP) 1412 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1413 } 1414 1415 static inline void 1416 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1417 { 1418 1419 if (!schedstat_enabled()) 1420 return; 1421 1422 /* 1423 * Mark the end of the wait period if dequeueing a 1424 * waiting task: 1425 */ 1426 if (se != cfs_rq->curr) 1427 update_stats_wait_end_fair(cfs_rq, se); 1428 1429 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1430 struct task_struct *tsk = task_of(se); 1431 unsigned int state; 1432 1433 /* XXX racy against TTWU */ 1434 state = READ_ONCE(tsk->__state); 1435 if (state & TASK_INTERRUPTIBLE) 1436 __schedstat_set(tsk->stats.sleep_start, 1437 rq_clock(rq_of(cfs_rq))); 1438 if (state & TASK_UNINTERRUPTIBLE) 1439 __schedstat_set(tsk->stats.block_start, 1440 rq_clock(rq_of(cfs_rq))); 1441 } 1442 } 1443 1444 /* 1445 * We are picking a new current task - update its stats: 1446 */ 1447 static inline void 1448 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1449 { 1450 /* 1451 * We are starting a new run period: 1452 */ 1453 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1454 } 1455 1456 /************************************************** 1457 * Scheduling class queueing methods: 1458 */ 1459 1460 static inline bool is_core_idle(int cpu) 1461 { 1462 #ifdef CONFIG_SCHED_SMT 1463 int sibling; 1464 1465 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1466 if (cpu == sibling) 1467 continue; 1468 1469 if (!idle_cpu(sibling)) 1470 return false; 1471 } 1472 #endif 1473 1474 return true; 1475 } 1476 1477 #ifdef CONFIG_NUMA 1478 #define NUMA_IMBALANCE_MIN 2 1479 1480 static inline long 1481 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1482 { 1483 /* 1484 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1485 * threshold. Above this threshold, individual tasks may be contending 1486 * for both memory bandwidth and any shared HT resources. This is an 1487 * approximation as the number of running tasks may not be related to 1488 * the number of busy CPUs due to sched_setaffinity. 1489 */ 1490 if (dst_running > imb_numa_nr) 1491 return imbalance; 1492 1493 /* 1494 * Allow a small imbalance based on a simple pair of communicating 1495 * tasks that remain local when the destination is lightly loaded. 1496 */ 1497 if (imbalance <= NUMA_IMBALANCE_MIN) 1498 return 0; 1499 1500 return imbalance; 1501 } 1502 #endif /* CONFIG_NUMA */ 1503 1504 #ifdef CONFIG_NUMA_BALANCING 1505 /* 1506 * Approximate time to scan a full NUMA task in ms. The task scan period is 1507 * calculated based on the tasks virtual memory size and 1508 * numa_balancing_scan_size. 1509 */ 1510 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1511 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1512 1513 /* Portion of address space to scan in MB */ 1514 unsigned int sysctl_numa_balancing_scan_size = 256; 1515 1516 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1517 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1518 1519 /* The page with hint page fault latency < threshold in ms is considered hot */ 1520 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1521 1522 struct numa_group { 1523 refcount_t refcount; 1524 1525 spinlock_t lock; /* nr_tasks, tasks */ 1526 int nr_tasks; 1527 pid_t gid; 1528 int active_nodes; 1529 1530 struct rcu_head rcu; 1531 unsigned long total_faults; 1532 unsigned long max_faults_cpu; 1533 /* 1534 * faults[] array is split into two regions: faults_mem and faults_cpu. 1535 * 1536 * Faults_cpu is used to decide whether memory should move 1537 * towards the CPU. As a consequence, these stats are weighted 1538 * more by CPU use than by memory faults. 1539 */ 1540 unsigned long faults[]; 1541 }; 1542 1543 /* 1544 * For functions that can be called in multiple contexts that permit reading 1545 * ->numa_group (see struct task_struct for locking rules). 1546 */ 1547 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1548 { 1549 return rcu_dereference_check(p->numa_group, p == current || 1550 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1551 } 1552 1553 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1554 { 1555 return rcu_dereference_protected(p->numa_group, p == current); 1556 } 1557 1558 static inline unsigned long group_faults_priv(struct numa_group *ng); 1559 static inline unsigned long group_faults_shared(struct numa_group *ng); 1560 1561 static unsigned int task_nr_scan_windows(struct task_struct *p) 1562 { 1563 unsigned long rss = 0; 1564 unsigned long nr_scan_pages; 1565 1566 /* 1567 * Calculations based on RSS as non-present and empty pages are skipped 1568 * by the PTE scanner and NUMA hinting faults should be trapped based 1569 * on resident pages 1570 */ 1571 nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); 1572 rss = get_mm_rss(p->mm); 1573 if (!rss) 1574 rss = nr_scan_pages; 1575 1576 rss = round_up(rss, nr_scan_pages); 1577 return rss / nr_scan_pages; 1578 } 1579 1580 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1581 #define MAX_SCAN_WINDOW 2560 1582 1583 static unsigned int task_scan_min(struct task_struct *p) 1584 { 1585 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1586 unsigned int scan, floor; 1587 unsigned int windows = 1; 1588 1589 if (scan_size < MAX_SCAN_WINDOW) 1590 windows = MAX_SCAN_WINDOW / scan_size; 1591 floor = 1000 / windows; 1592 1593 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1594 return max_t(unsigned int, floor, scan); 1595 } 1596 1597 static unsigned int task_scan_start(struct task_struct *p) 1598 { 1599 unsigned long smin = task_scan_min(p); 1600 unsigned long period = smin; 1601 struct numa_group *ng; 1602 1603 /* Scale the maximum scan period with the amount of shared memory. */ 1604 rcu_read_lock(); 1605 ng = rcu_dereference(p->numa_group); 1606 if (ng) { 1607 unsigned long shared = group_faults_shared(ng); 1608 unsigned long private = group_faults_priv(ng); 1609 1610 period *= refcount_read(&ng->refcount); 1611 period *= shared + 1; 1612 period /= private + shared + 1; 1613 } 1614 rcu_read_unlock(); 1615 1616 return max(smin, period); 1617 } 1618 1619 static unsigned int task_scan_max(struct task_struct *p) 1620 { 1621 unsigned long smin = task_scan_min(p); 1622 unsigned long smax; 1623 struct numa_group *ng; 1624 1625 /* Watch for min being lower than max due to floor calculations */ 1626 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1627 1628 /* Scale the maximum scan period with the amount of shared memory. */ 1629 ng = deref_curr_numa_group(p); 1630 if (ng) { 1631 unsigned long shared = group_faults_shared(ng); 1632 unsigned long private = group_faults_priv(ng); 1633 unsigned long period = smax; 1634 1635 period *= refcount_read(&ng->refcount); 1636 period *= shared + 1; 1637 period /= private + shared + 1; 1638 1639 smax = max(smax, period); 1640 } 1641 1642 return max(smin, smax); 1643 } 1644 1645 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1646 { 1647 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1648 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1649 } 1650 1651 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1652 { 1653 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1654 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1655 } 1656 1657 /* Shared or private faults. */ 1658 #define NR_NUMA_HINT_FAULT_TYPES 2 1659 1660 /* Memory and CPU locality */ 1661 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1662 1663 /* Averaged statistics, and temporary buffers. */ 1664 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1665 1666 pid_t task_numa_group_id(struct task_struct *p) 1667 { 1668 struct numa_group *ng; 1669 pid_t gid = 0; 1670 1671 rcu_read_lock(); 1672 ng = rcu_dereference(p->numa_group); 1673 if (ng) 1674 gid = ng->gid; 1675 rcu_read_unlock(); 1676 1677 return gid; 1678 } 1679 1680 /* 1681 * The averaged statistics, shared & private, memory & CPU, 1682 * occupy the first half of the array. The second half of the 1683 * array is for current counters, which are averaged into the 1684 * first set by task_numa_placement. 1685 */ 1686 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1687 { 1688 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1689 } 1690 1691 static inline unsigned long task_faults(struct task_struct *p, int nid) 1692 { 1693 if (!p->numa_faults) 1694 return 0; 1695 1696 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1697 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1698 } 1699 1700 static inline unsigned long group_faults(struct task_struct *p, int nid) 1701 { 1702 struct numa_group *ng = deref_task_numa_group(p); 1703 1704 if (!ng) 1705 return 0; 1706 1707 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1708 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1709 } 1710 1711 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1712 { 1713 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1714 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1715 } 1716 1717 static inline unsigned long group_faults_priv(struct numa_group *ng) 1718 { 1719 unsigned long faults = 0; 1720 int node; 1721 1722 for_each_online_node(node) { 1723 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1724 } 1725 1726 return faults; 1727 } 1728 1729 static inline unsigned long group_faults_shared(struct numa_group *ng) 1730 { 1731 unsigned long faults = 0; 1732 int node; 1733 1734 for_each_online_node(node) { 1735 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1736 } 1737 1738 return faults; 1739 } 1740 1741 /* 1742 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1743 * considered part of a numa group's pseudo-interleaving set. Migrations 1744 * between these nodes are slowed down, to allow things to settle down. 1745 */ 1746 #define ACTIVE_NODE_FRACTION 3 1747 1748 static bool numa_is_active_node(int nid, struct numa_group *ng) 1749 { 1750 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1751 } 1752 1753 /* Handle placement on systems where not all nodes are directly connected. */ 1754 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1755 int lim_dist, bool task) 1756 { 1757 unsigned long score = 0; 1758 int node, max_dist; 1759 1760 /* 1761 * All nodes are directly connected, and the same distance 1762 * from each other. No need for fancy placement algorithms. 1763 */ 1764 if (sched_numa_topology_type == NUMA_DIRECT) 1765 return 0; 1766 1767 /* sched_max_numa_distance may be changed in parallel. */ 1768 max_dist = READ_ONCE(sched_max_numa_distance); 1769 /* 1770 * This code is called for each node, introducing N^2 complexity, 1771 * which should be OK given the number of nodes rarely exceeds 8. 1772 */ 1773 for_each_online_node(node) { 1774 unsigned long faults; 1775 int dist = node_distance(nid, node); 1776 1777 /* 1778 * The furthest away nodes in the system are not interesting 1779 * for placement; nid was already counted. 1780 */ 1781 if (dist >= max_dist || node == nid) 1782 continue; 1783 1784 /* 1785 * On systems with a backplane NUMA topology, compare groups 1786 * of nodes, and move tasks towards the group with the most 1787 * memory accesses. When comparing two nodes at distance 1788 * "hoplimit", only nodes closer by than "hoplimit" are part 1789 * of each group. Skip other nodes. 1790 */ 1791 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1792 continue; 1793 1794 /* Add up the faults from nearby nodes. */ 1795 if (task) 1796 faults = task_faults(p, node); 1797 else 1798 faults = group_faults(p, node); 1799 1800 /* 1801 * On systems with a glueless mesh NUMA topology, there are 1802 * no fixed "groups of nodes". Instead, nodes that are not 1803 * directly connected bounce traffic through intermediate 1804 * nodes; a numa_group can occupy any set of nodes. 1805 * The further away a node is, the less the faults count. 1806 * This seems to result in good task placement. 1807 */ 1808 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1809 faults *= (max_dist - dist); 1810 faults /= (max_dist - LOCAL_DISTANCE); 1811 } 1812 1813 score += faults; 1814 } 1815 1816 return score; 1817 } 1818 1819 /* 1820 * These return the fraction of accesses done by a particular task, or 1821 * task group, on a particular numa node. The group weight is given a 1822 * larger multiplier, in order to group tasks together that are almost 1823 * evenly spread out between numa nodes. 1824 */ 1825 static inline unsigned long task_weight(struct task_struct *p, int nid, 1826 int dist) 1827 { 1828 unsigned long faults, total_faults; 1829 1830 if (!p->numa_faults) 1831 return 0; 1832 1833 total_faults = p->total_numa_faults; 1834 1835 if (!total_faults) 1836 return 0; 1837 1838 faults = task_faults(p, nid); 1839 faults += score_nearby_nodes(p, nid, dist, true); 1840 1841 return 1000 * faults / total_faults; 1842 } 1843 1844 static inline unsigned long group_weight(struct task_struct *p, int nid, 1845 int dist) 1846 { 1847 struct numa_group *ng = deref_task_numa_group(p); 1848 unsigned long faults, total_faults; 1849 1850 if (!ng) 1851 return 0; 1852 1853 total_faults = ng->total_faults; 1854 1855 if (!total_faults) 1856 return 0; 1857 1858 faults = group_faults(p, nid); 1859 faults += score_nearby_nodes(p, nid, dist, false); 1860 1861 return 1000 * faults / total_faults; 1862 } 1863 1864 /* 1865 * If memory tiering mode is enabled, cpupid of slow memory page is 1866 * used to record scan time instead of CPU and PID. When tiering mode 1867 * is disabled at run time, the scan time (in cpupid) will be 1868 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1869 * access out of array bound. 1870 */ 1871 static inline bool cpupid_valid(int cpupid) 1872 { 1873 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1874 } 1875 1876 /* 1877 * For memory tiering mode, if there are enough free pages (more than 1878 * enough watermark defined here) in fast memory node, to take full 1879 * advantage of fast memory capacity, all recently accessed slow 1880 * memory pages will be migrated to fast memory node without 1881 * considering hot threshold. 1882 */ 1883 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1884 { 1885 int z; 1886 unsigned long enough_wmark; 1887 1888 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1889 pgdat->node_present_pages >> 4); 1890 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1891 struct zone *zone = pgdat->node_zones + z; 1892 1893 if (!populated_zone(zone)) 1894 continue; 1895 1896 if (zone_watermark_ok(zone, 0, 1897 promo_wmark_pages(zone) + enough_wmark, 1898 ZONE_MOVABLE, 0)) 1899 return true; 1900 } 1901 return false; 1902 } 1903 1904 /* 1905 * For memory tiering mode, when page tables are scanned, the scan 1906 * time will be recorded in struct page in addition to make page 1907 * PROT_NONE for slow memory page. So when the page is accessed, in 1908 * hint page fault handler, the hint page fault latency is calculated 1909 * via, 1910 * 1911 * hint page fault latency = hint page fault time - scan time 1912 * 1913 * The smaller the hint page fault latency, the higher the possibility 1914 * for the page to be hot. 1915 */ 1916 static int numa_hint_fault_latency(struct folio *folio) 1917 { 1918 int last_time, time; 1919 1920 time = jiffies_to_msecs(jiffies); 1921 last_time = folio_xchg_access_time(folio, time); 1922 1923 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1924 } 1925 1926 /* 1927 * For memory tiering mode, too high promotion/demotion throughput may 1928 * hurt application latency. So we provide a mechanism to rate limit 1929 * the number of pages that are tried to be promoted. 1930 */ 1931 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1932 unsigned long rate_limit, int nr) 1933 { 1934 unsigned long nr_cand; 1935 unsigned int now, start; 1936 1937 now = jiffies_to_msecs(jiffies); 1938 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1939 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1940 start = pgdat->nbp_rl_start; 1941 if (now - start > MSEC_PER_SEC && 1942 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1943 pgdat->nbp_rl_nr_cand = nr_cand; 1944 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1945 return true; 1946 return false; 1947 } 1948 1949 #define NUMA_MIGRATION_ADJUST_STEPS 16 1950 1951 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1952 unsigned long rate_limit, 1953 unsigned int ref_th) 1954 { 1955 unsigned int now, start, th_period, unit_th, th; 1956 unsigned long nr_cand, ref_cand, diff_cand; 1957 1958 now = jiffies_to_msecs(jiffies); 1959 th_period = sysctl_numa_balancing_scan_period_max; 1960 start = pgdat->nbp_th_start; 1961 if (now - start > th_period && 1962 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1963 ref_cand = rate_limit * 1964 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1965 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1966 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1967 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1968 th = pgdat->nbp_threshold ? : ref_th; 1969 if (diff_cand > ref_cand * 11 / 10) 1970 th = max(th - unit_th, unit_th); 1971 else if (diff_cand < ref_cand * 9 / 10) 1972 th = min(th + unit_th, ref_th * 2); 1973 pgdat->nbp_th_nr_cand = nr_cand; 1974 pgdat->nbp_threshold = th; 1975 } 1976 } 1977 1978 bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, 1979 int src_nid, int dst_cpu) 1980 { 1981 struct numa_group *ng = deref_curr_numa_group(p); 1982 int dst_nid = cpu_to_node(dst_cpu); 1983 int last_cpupid, this_cpupid; 1984 1985 /* 1986 * Cannot migrate to memoryless nodes. 1987 */ 1988 if (!node_state(dst_nid, N_MEMORY)) 1989 return false; 1990 1991 /* 1992 * The pages in slow memory node should be migrated according 1993 * to hot/cold instead of private/shared. 1994 */ 1995 if (folio_use_access_time(folio)) { 1996 struct pglist_data *pgdat; 1997 unsigned long rate_limit; 1998 unsigned int latency, th, def_th; 1999 long nr = folio_nr_pages(folio); 2000 2001 pgdat = NODE_DATA(dst_nid); 2002 if (pgdat_free_space_enough(pgdat)) { 2003 /* workload changed, reset hot threshold */ 2004 pgdat->nbp_threshold = 0; 2005 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr); 2006 return true; 2007 } 2008 2009 def_th = sysctl_numa_balancing_hot_threshold; 2010 rate_limit = MB_TO_PAGES(sysctl_numa_balancing_promote_rate_limit); 2011 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 2012 2013 th = pgdat->nbp_threshold ? : def_th; 2014 latency = numa_hint_fault_latency(folio); 2015 if (latency >= th) 2016 return false; 2017 2018 return !numa_promotion_rate_limit(pgdat, rate_limit, nr); 2019 } 2020 2021 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 2022 last_cpupid = folio_xchg_last_cpupid(folio, this_cpupid); 2023 2024 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 2025 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 2026 return false; 2027 2028 /* 2029 * Allow first faults or private faults to migrate immediately early in 2030 * the lifetime of a task. The magic number 4 is based on waiting for 2031 * two full passes of the "multi-stage node selection" test that is 2032 * executed below. 2033 */ 2034 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 2035 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 2036 return true; 2037 2038 /* 2039 * Multi-stage node selection is used in conjunction with a periodic 2040 * migration fault to build a temporal task<->page relation. By using 2041 * a two-stage filter we remove short/unlikely relations. 2042 * 2043 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 2044 * a task's usage of a particular page (n_p) per total usage of this 2045 * page (n_t) (in a given time-span) to a probability. 2046 * 2047 * Our periodic faults will sample this probability and getting the 2048 * same result twice in a row, given these samples are fully 2049 * independent, is then given by P(n)^2, provided our sample period 2050 * is sufficiently short compared to the usage pattern. 2051 * 2052 * This quadric squishes small probabilities, making it less likely we 2053 * act on an unlikely task<->page relation. 2054 */ 2055 if (!cpupid_pid_unset(last_cpupid) && 2056 cpupid_to_nid(last_cpupid) != dst_nid) 2057 return false; 2058 2059 /* Always allow migrate on private faults */ 2060 if (cpupid_match_pid(p, last_cpupid)) 2061 return true; 2062 2063 /* A shared fault, but p->numa_group has not been set up yet. */ 2064 if (!ng) 2065 return true; 2066 2067 /* 2068 * Destination node is much more heavily used than the source 2069 * node? Allow migration. 2070 */ 2071 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 2072 ACTIVE_NODE_FRACTION) 2073 return true; 2074 2075 /* 2076 * Distribute memory according to CPU & memory use on each node, 2077 * with 3/4 hysteresis to avoid unnecessary memory migrations: 2078 * 2079 * faults_cpu(dst) 3 faults_cpu(src) 2080 * --------------- * - > --------------- 2081 * faults_mem(dst) 4 faults_mem(src) 2082 */ 2083 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 2084 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 2085 } 2086 2087 /* 2088 * 'numa_type' describes the node at the moment of load balancing. 2089 */ 2090 enum numa_type { 2091 /* The node has spare capacity that can be used to run more tasks. */ 2092 node_has_spare = 0, 2093 /* 2094 * The node is fully used and the tasks don't compete for more CPU 2095 * cycles. Nevertheless, some tasks might wait before running. 2096 */ 2097 node_fully_busy, 2098 /* 2099 * The node is overloaded and can't provide expected CPU cycles to all 2100 * tasks. 2101 */ 2102 node_overloaded 2103 }; 2104 2105 /* Cached statistics for all CPUs within a node */ 2106 struct numa_stats { 2107 unsigned long load; 2108 unsigned long runnable; 2109 unsigned long util; 2110 /* Total compute capacity of CPUs on a node */ 2111 unsigned long compute_capacity; 2112 unsigned int nr_running; 2113 unsigned int weight; 2114 enum numa_type node_type; 2115 int idle_cpu; 2116 }; 2117 2118 struct task_numa_env { 2119 struct task_struct *p; 2120 2121 int src_cpu, src_nid; 2122 int dst_cpu, dst_nid; 2123 int imb_numa_nr; 2124 2125 struct numa_stats src_stats, dst_stats; 2126 2127 int imbalance_pct; 2128 int dist; 2129 2130 struct task_struct *best_task; 2131 long best_imp; 2132 int best_cpu; 2133 }; 2134 2135 static unsigned long cpu_load(struct rq *rq); 2136 static unsigned long cpu_runnable(struct rq *rq); 2137 2138 static inline enum 2139 numa_type numa_classify(unsigned int imbalance_pct, 2140 struct numa_stats *ns) 2141 { 2142 if ((ns->nr_running > ns->weight) && 2143 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 2144 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 2145 return node_overloaded; 2146 2147 if ((ns->nr_running < ns->weight) || 2148 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 2149 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 2150 return node_has_spare; 2151 2152 return node_fully_busy; 2153 } 2154 2155 #ifdef CONFIG_SCHED_SMT 2156 /* Forward declarations of select_idle_sibling helpers */ 2157 static inline bool test_idle_cores(int cpu); 2158 static inline int numa_idle_core(int idle_core, int cpu) 2159 { 2160 if (!static_branch_likely(&sched_smt_present) || 2161 idle_core >= 0 || !test_idle_cores(cpu)) 2162 return idle_core; 2163 2164 /* 2165 * Prefer cores instead of packing HT siblings 2166 * and triggering future load balancing. 2167 */ 2168 if (is_core_idle(cpu)) 2169 idle_core = cpu; 2170 2171 return idle_core; 2172 } 2173 #else /* !CONFIG_SCHED_SMT: */ 2174 static inline int numa_idle_core(int idle_core, int cpu) 2175 { 2176 return idle_core; 2177 } 2178 #endif /* !CONFIG_SCHED_SMT */ 2179 2180 /* 2181 * Gather all necessary information to make NUMA balancing placement 2182 * decisions that are compatible with standard load balancer. This 2183 * borrows code and logic from update_sg_lb_stats but sharing a 2184 * common implementation is impractical. 2185 */ 2186 static void update_numa_stats(struct task_numa_env *env, 2187 struct numa_stats *ns, int nid, 2188 bool find_idle) 2189 { 2190 int cpu, idle_core = -1; 2191 2192 memset(ns, 0, sizeof(*ns)); 2193 ns->idle_cpu = -1; 2194 2195 rcu_read_lock(); 2196 for_each_cpu(cpu, cpumask_of_node(nid)) { 2197 struct rq *rq = cpu_rq(cpu); 2198 2199 ns->load += cpu_load(rq); 2200 ns->runnable += cpu_runnable(rq); 2201 ns->util += cpu_util_cfs(cpu); 2202 ns->nr_running += rq->cfs.h_nr_runnable; 2203 ns->compute_capacity += capacity_of(cpu); 2204 2205 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 2206 if (READ_ONCE(rq->numa_migrate_on) || 2207 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2208 continue; 2209 2210 if (ns->idle_cpu == -1) 2211 ns->idle_cpu = cpu; 2212 2213 idle_core = numa_idle_core(idle_core, cpu); 2214 } 2215 } 2216 rcu_read_unlock(); 2217 2218 ns->weight = cpumask_weight(cpumask_of_node(nid)); 2219 2220 ns->node_type = numa_classify(env->imbalance_pct, ns); 2221 2222 if (idle_core >= 0) 2223 ns->idle_cpu = idle_core; 2224 } 2225 2226 static void task_numa_assign(struct task_numa_env *env, 2227 struct task_struct *p, long imp) 2228 { 2229 struct rq *rq = cpu_rq(env->dst_cpu); 2230 2231 /* Check if run-queue part of active NUMA balance. */ 2232 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 2233 int cpu; 2234 int start = env->dst_cpu; 2235 2236 /* Find alternative idle CPU. */ 2237 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 2238 if (cpu == env->best_cpu || !idle_cpu(cpu) || 2239 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 2240 continue; 2241 } 2242 2243 env->dst_cpu = cpu; 2244 rq = cpu_rq(env->dst_cpu); 2245 if (!xchg(&rq->numa_migrate_on, 1)) 2246 goto assign; 2247 } 2248 2249 /* Failed to find an alternative idle CPU */ 2250 return; 2251 } 2252 2253 assign: 2254 /* 2255 * Clear previous best_cpu/rq numa-migrate flag, since task now 2256 * found a better CPU to move/swap. 2257 */ 2258 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 2259 rq = cpu_rq(env->best_cpu); 2260 WRITE_ONCE(rq->numa_migrate_on, 0); 2261 } 2262 2263 if (env->best_task) 2264 put_task_struct(env->best_task); 2265 if (p) 2266 get_task_struct(p); 2267 2268 env->best_task = p; 2269 env->best_imp = imp; 2270 env->best_cpu = env->dst_cpu; 2271 } 2272 2273 static bool load_too_imbalanced(long src_load, long dst_load, 2274 struct task_numa_env *env) 2275 { 2276 long imb, old_imb; 2277 long orig_src_load, orig_dst_load; 2278 long src_capacity, dst_capacity; 2279 2280 /* 2281 * The load is corrected for the CPU capacity available on each node. 2282 * 2283 * src_load dst_load 2284 * ------------ vs --------- 2285 * src_capacity dst_capacity 2286 */ 2287 src_capacity = env->src_stats.compute_capacity; 2288 dst_capacity = env->dst_stats.compute_capacity; 2289 2290 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 2291 2292 orig_src_load = env->src_stats.load; 2293 orig_dst_load = env->dst_stats.load; 2294 2295 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 2296 2297 /* Would this change make things worse? */ 2298 return (imb > old_imb); 2299 } 2300 2301 /* 2302 * Maximum NUMA importance can be 1998 (2*999); 2303 * SMALLIMP @ 30 would be close to 1998/64. 2304 * Used to deter task migration. 2305 */ 2306 #define SMALLIMP 30 2307 2308 /* 2309 * This checks if the overall compute and NUMA accesses of the system would 2310 * be improved if the source tasks was migrated to the target dst_cpu taking 2311 * into account that it might be best if task running on the dst_cpu should 2312 * be exchanged with the source task 2313 */ 2314 static bool task_numa_compare(struct task_numa_env *env, 2315 long taskimp, long groupimp, bool maymove) 2316 { 2317 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 2318 struct rq *dst_rq = cpu_rq(env->dst_cpu); 2319 long imp = p_ng ? groupimp : taskimp; 2320 struct task_struct *cur; 2321 long src_load, dst_load; 2322 int dist = env->dist; 2323 long moveimp = imp; 2324 long load; 2325 bool stopsearch = false; 2326 2327 if (READ_ONCE(dst_rq->numa_migrate_on)) 2328 return false; 2329 2330 rcu_read_lock(); 2331 cur = rcu_dereference(dst_rq->curr); 2332 if (cur && ((cur->flags & (PF_EXITING | PF_KTHREAD)) || 2333 !cur->mm)) 2334 cur = NULL; 2335 2336 /* 2337 * Because we have preemption enabled we can get migrated around and 2338 * end try selecting ourselves (current == env->p) as a swap candidate. 2339 */ 2340 if (cur == env->p) { 2341 stopsearch = true; 2342 goto unlock; 2343 } 2344 2345 if (!cur) { 2346 if (maymove && moveimp >= env->best_imp) 2347 goto assign; 2348 else 2349 goto unlock; 2350 } 2351 2352 /* Skip this swap candidate if cannot move to the source cpu. */ 2353 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 2354 goto unlock; 2355 2356 /* 2357 * Skip this swap candidate if it is not moving to its preferred 2358 * node and the best task is. 2359 */ 2360 if (env->best_task && 2361 env->best_task->numa_preferred_nid == env->src_nid && 2362 cur->numa_preferred_nid != env->src_nid) { 2363 goto unlock; 2364 } 2365 2366 /* 2367 * "imp" is the fault differential for the source task between the 2368 * source and destination node. Calculate the total differential for 2369 * the source task and potential destination task. The more negative 2370 * the value is, the more remote accesses that would be expected to 2371 * be incurred if the tasks were swapped. 2372 * 2373 * If dst and source tasks are in the same NUMA group, or not 2374 * in any group then look only at task weights. 2375 */ 2376 cur_ng = rcu_dereference(cur->numa_group); 2377 if (cur_ng == p_ng) { 2378 /* 2379 * Do not swap within a group or between tasks that have 2380 * no group if there is spare capacity. Swapping does 2381 * not address the load imbalance and helps one task at 2382 * the cost of punishing another. 2383 */ 2384 if (env->dst_stats.node_type == node_has_spare) 2385 goto unlock; 2386 2387 imp = taskimp + task_weight(cur, env->src_nid, dist) - 2388 task_weight(cur, env->dst_nid, dist); 2389 /* 2390 * Add some hysteresis to prevent swapping the 2391 * tasks within a group over tiny differences. 2392 */ 2393 if (cur_ng) 2394 imp -= imp / 16; 2395 } else { 2396 /* 2397 * Compare the group weights. If a task is all by itself 2398 * (not part of a group), use the task weight instead. 2399 */ 2400 if (cur_ng && p_ng) 2401 imp += group_weight(cur, env->src_nid, dist) - 2402 group_weight(cur, env->dst_nid, dist); 2403 else 2404 imp += task_weight(cur, env->src_nid, dist) - 2405 task_weight(cur, env->dst_nid, dist); 2406 } 2407 2408 /* Discourage picking a task already on its preferred node */ 2409 if (cur->numa_preferred_nid == env->dst_nid) 2410 imp -= imp / 16; 2411 2412 /* 2413 * Encourage picking a task that moves to its preferred node. 2414 * This potentially makes imp larger than it's maximum of 2415 * 1998 (see SMALLIMP and task_weight for why) but in this 2416 * case, it does not matter. 2417 */ 2418 if (cur->numa_preferred_nid == env->src_nid) 2419 imp += imp / 8; 2420 2421 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2422 imp = moveimp; 2423 cur = NULL; 2424 goto assign; 2425 } 2426 2427 /* 2428 * Prefer swapping with a task moving to its preferred node over a 2429 * task that is not. 2430 */ 2431 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2432 env->best_task->numa_preferred_nid != env->src_nid) { 2433 goto assign; 2434 } 2435 2436 /* 2437 * If the NUMA importance is less than SMALLIMP, 2438 * task migration might only result in ping pong 2439 * of tasks and also hurt performance due to cache 2440 * misses. 2441 */ 2442 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2443 goto unlock; 2444 2445 /* 2446 * In the overloaded case, try and keep the load balanced. 2447 */ 2448 load = task_h_load(env->p) - task_h_load(cur); 2449 if (!load) 2450 goto assign; 2451 2452 dst_load = env->dst_stats.load + load; 2453 src_load = env->src_stats.load - load; 2454 2455 if (load_too_imbalanced(src_load, dst_load, env)) 2456 goto unlock; 2457 2458 assign: 2459 /* Evaluate an idle CPU for a task numa move. */ 2460 if (!cur) { 2461 int cpu = env->dst_stats.idle_cpu; 2462 2463 /* Nothing cached so current CPU went idle since the search. */ 2464 if (cpu < 0) 2465 cpu = env->dst_cpu; 2466 2467 /* 2468 * If the CPU is no longer truly idle and the previous best CPU 2469 * is, keep using it. 2470 */ 2471 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2472 idle_cpu(env->best_cpu)) { 2473 cpu = env->best_cpu; 2474 } 2475 2476 env->dst_cpu = cpu; 2477 } 2478 2479 task_numa_assign(env, cur, imp); 2480 2481 /* 2482 * If a move to idle is allowed because there is capacity or load 2483 * balance improves then stop the search. While a better swap 2484 * candidate may exist, a search is not free. 2485 */ 2486 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2487 stopsearch = true; 2488 2489 /* 2490 * If a swap candidate must be identified and the current best task 2491 * moves its preferred node then stop the search. 2492 */ 2493 if (!maymove && env->best_task && 2494 env->best_task->numa_preferred_nid == env->src_nid) { 2495 stopsearch = true; 2496 } 2497 unlock: 2498 rcu_read_unlock(); 2499 2500 return stopsearch; 2501 } 2502 2503 static void task_numa_find_cpu(struct task_numa_env *env, 2504 long taskimp, long groupimp) 2505 { 2506 bool maymove = false; 2507 int cpu; 2508 2509 /* 2510 * If dst node has spare capacity, then check if there is an 2511 * imbalance that would be overruled by the load balancer. 2512 */ 2513 if (env->dst_stats.node_type == node_has_spare) { 2514 unsigned int imbalance; 2515 int src_running, dst_running; 2516 2517 /* 2518 * Would movement cause an imbalance? Note that if src has 2519 * more running tasks that the imbalance is ignored as the 2520 * move improves the imbalance from the perspective of the 2521 * CPU load balancer. 2522 * */ 2523 src_running = env->src_stats.nr_running - 1; 2524 dst_running = env->dst_stats.nr_running + 1; 2525 imbalance = max(0, dst_running - src_running); 2526 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2527 env->imb_numa_nr); 2528 2529 /* Use idle CPU if there is no imbalance */ 2530 if (!imbalance) { 2531 maymove = true; 2532 if (env->dst_stats.idle_cpu >= 0) { 2533 env->dst_cpu = env->dst_stats.idle_cpu; 2534 task_numa_assign(env, NULL, 0); 2535 return; 2536 } 2537 } 2538 } else { 2539 long src_load, dst_load, load; 2540 /* 2541 * If the improvement from just moving env->p direction is better 2542 * than swapping tasks around, check if a move is possible. 2543 */ 2544 load = task_h_load(env->p); 2545 dst_load = env->dst_stats.load + load; 2546 src_load = env->src_stats.load - load; 2547 maymove = !load_too_imbalanced(src_load, dst_load, env); 2548 } 2549 2550 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 2551 /* Skip this CPU if the source task cannot migrate */ 2552 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2553 continue; 2554 2555 env->dst_cpu = cpu; 2556 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2557 break; 2558 } 2559 } 2560 2561 static int task_numa_migrate(struct task_struct *p) 2562 { 2563 struct task_numa_env env = { 2564 .p = p, 2565 2566 .src_cpu = task_cpu(p), 2567 .src_nid = task_node(p), 2568 2569 .imbalance_pct = 112, 2570 2571 .best_task = NULL, 2572 .best_imp = 0, 2573 .best_cpu = -1, 2574 }; 2575 unsigned long taskweight, groupweight; 2576 struct sched_domain *sd; 2577 long taskimp, groupimp; 2578 struct numa_group *ng; 2579 struct rq *best_rq; 2580 int nid, ret, dist; 2581 2582 /* 2583 * Pick the lowest SD_NUMA domain, as that would have the smallest 2584 * imbalance and would be the first to start moving tasks about. 2585 * 2586 * And we want to avoid any moving of tasks about, as that would create 2587 * random movement of tasks -- counter the numa conditions we're trying 2588 * to satisfy here. 2589 */ 2590 rcu_read_lock(); 2591 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 2592 if (sd) { 2593 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2594 env.imb_numa_nr = sd->imb_numa_nr; 2595 } 2596 rcu_read_unlock(); 2597 2598 /* 2599 * Cpusets can break the scheduler domain tree into smaller 2600 * balance domains, some of which do not cross NUMA boundaries. 2601 * Tasks that are "trapped" in such domains cannot be migrated 2602 * elsewhere, so there is no point in (re)trying. 2603 */ 2604 if (unlikely(!sd)) { 2605 sched_setnuma(p, task_node(p)); 2606 return -EINVAL; 2607 } 2608 2609 env.dst_nid = p->numa_preferred_nid; 2610 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2611 taskweight = task_weight(p, env.src_nid, dist); 2612 groupweight = group_weight(p, env.src_nid, dist); 2613 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2614 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2615 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2616 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2617 2618 /* Try to find a spot on the preferred nid. */ 2619 task_numa_find_cpu(&env, taskimp, groupimp); 2620 2621 /* 2622 * Look at other nodes in these cases: 2623 * - there is no space available on the preferred_nid 2624 * - the task is part of a numa_group that is interleaved across 2625 * multiple NUMA nodes; in order to better consolidate the group, 2626 * we need to check other locations. 2627 */ 2628 ng = deref_curr_numa_group(p); 2629 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2630 for_each_node_state(nid, N_CPU) { 2631 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2632 continue; 2633 2634 dist = node_distance(env.src_nid, env.dst_nid); 2635 if (sched_numa_topology_type == NUMA_BACKPLANE && 2636 dist != env.dist) { 2637 taskweight = task_weight(p, env.src_nid, dist); 2638 groupweight = group_weight(p, env.src_nid, dist); 2639 } 2640 2641 /* Only consider nodes where both task and groups benefit */ 2642 taskimp = task_weight(p, nid, dist) - taskweight; 2643 groupimp = group_weight(p, nid, dist) - groupweight; 2644 if (taskimp < 0 && groupimp < 0) 2645 continue; 2646 2647 env.dist = dist; 2648 env.dst_nid = nid; 2649 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2650 task_numa_find_cpu(&env, taskimp, groupimp); 2651 } 2652 } 2653 2654 /* 2655 * If the task is part of a workload that spans multiple NUMA nodes, 2656 * and is migrating into one of the workload's active nodes, remember 2657 * this node as the task's preferred numa node, so the workload can 2658 * settle down. 2659 * A task that migrated to a second choice node will be better off 2660 * trying for a better one later. Do not set the preferred node here. 2661 */ 2662 if (ng) { 2663 if (env.best_cpu == -1) 2664 nid = env.src_nid; 2665 else 2666 nid = cpu_to_node(env.best_cpu); 2667 2668 if (nid != p->numa_preferred_nid) 2669 sched_setnuma(p, nid); 2670 } 2671 2672 /* No better CPU than the current one was found. */ 2673 if (env.best_cpu == -1) { 2674 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2675 return -EAGAIN; 2676 } 2677 2678 best_rq = cpu_rq(env.best_cpu); 2679 if (env.best_task == NULL) { 2680 ret = migrate_task_to(p, env.best_cpu); 2681 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2682 if (ret != 0) 2683 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2684 return ret; 2685 } 2686 2687 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2688 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2689 2690 if (ret != 0) 2691 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2692 put_task_struct(env.best_task); 2693 return ret; 2694 } 2695 2696 /* Attempt to migrate a task to a CPU on the preferred node. */ 2697 static void numa_migrate_preferred(struct task_struct *p) 2698 { 2699 unsigned long interval = HZ; 2700 2701 /* This task has no NUMA fault statistics yet */ 2702 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2703 return; 2704 2705 /* Periodically retry migrating the task to the preferred node */ 2706 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2707 p->numa_migrate_retry = jiffies + interval; 2708 2709 /* Success if task is already running on preferred CPU */ 2710 if (task_node(p) == p->numa_preferred_nid) 2711 return; 2712 2713 /* Otherwise, try migrate to a CPU on the preferred node */ 2714 task_numa_migrate(p); 2715 } 2716 2717 /* 2718 * Find out how many nodes the workload is actively running on. Do this by 2719 * tracking the nodes from which NUMA hinting faults are triggered. This can 2720 * be different from the set of nodes where the workload's memory is currently 2721 * located. 2722 */ 2723 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2724 { 2725 unsigned long faults, max_faults = 0; 2726 int nid, active_nodes = 0; 2727 2728 for_each_node_state(nid, N_CPU) { 2729 faults = group_faults_cpu(numa_group, nid); 2730 if (faults > max_faults) 2731 max_faults = faults; 2732 } 2733 2734 for_each_node_state(nid, N_CPU) { 2735 faults = group_faults_cpu(numa_group, nid); 2736 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2737 active_nodes++; 2738 } 2739 2740 numa_group->max_faults_cpu = max_faults; 2741 numa_group->active_nodes = active_nodes; 2742 } 2743 2744 /* 2745 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2746 * increments. The more local the fault statistics are, the higher the scan 2747 * period will be for the next scan window. If local/(local+remote) ratio is 2748 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2749 * the scan period will decrease. Aim for 70% local accesses. 2750 */ 2751 #define NUMA_PERIOD_SLOTS 10 2752 #define NUMA_PERIOD_THRESHOLD 7 2753 2754 /* 2755 * Increase the scan period (slow down scanning) if the majority of 2756 * our memory is already on our local node, or if the majority of 2757 * the page accesses are shared with other processes. 2758 * Otherwise, decrease the scan period. 2759 */ 2760 static void update_task_scan_period(struct task_struct *p, 2761 unsigned long shared, unsigned long private) 2762 { 2763 unsigned int period_slot; 2764 int lr_ratio, ps_ratio; 2765 int diff; 2766 2767 unsigned long remote = p->numa_faults_locality[0]; 2768 unsigned long local = p->numa_faults_locality[1]; 2769 2770 /* 2771 * If there were no record hinting faults then either the task is 2772 * completely idle or all activity is in areas that are not of interest 2773 * to automatic numa balancing. Related to that, if there were failed 2774 * migration then it implies we are migrating too quickly or the local 2775 * node is overloaded. In either case, scan slower 2776 */ 2777 if (local + shared == 0 || p->numa_faults_locality[2]) { 2778 p->numa_scan_period = min(p->numa_scan_period_max, 2779 p->numa_scan_period << 1); 2780 2781 p->mm->numa_next_scan = jiffies + 2782 msecs_to_jiffies(p->numa_scan_period); 2783 2784 return; 2785 } 2786 2787 /* 2788 * Prepare to scale scan period relative to the current period. 2789 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2790 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2791 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2792 */ 2793 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2794 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2795 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2796 2797 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2798 /* 2799 * Most memory accesses are local. There is no need to 2800 * do fast NUMA scanning, since memory is already local. 2801 */ 2802 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2803 if (!slot) 2804 slot = 1; 2805 diff = slot * period_slot; 2806 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2807 /* 2808 * Most memory accesses are shared with other tasks. 2809 * There is no point in continuing fast NUMA scanning, 2810 * since other tasks may just move the memory elsewhere. 2811 */ 2812 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2813 if (!slot) 2814 slot = 1; 2815 diff = slot * period_slot; 2816 } else { 2817 /* 2818 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2819 * yet they are not on the local NUMA node. Speed up 2820 * NUMA scanning to get the memory moved over. 2821 */ 2822 int ratio = max(lr_ratio, ps_ratio); 2823 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2824 } 2825 2826 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2827 task_scan_min(p), task_scan_max(p)); 2828 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2829 } 2830 2831 /* 2832 * Get the fraction of time the task has been running since the last 2833 * NUMA placement cycle. The scheduler keeps similar statistics, but 2834 * decays those on a 32ms period, which is orders of magnitude off 2835 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2836 * stats only if the task is so new there are no NUMA statistics yet. 2837 */ 2838 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2839 { 2840 u64 runtime, delta, now; 2841 /* Use the start of this time slice to avoid calculations. */ 2842 now = p->se.exec_start; 2843 runtime = p->se.sum_exec_runtime; 2844 2845 if (p->last_task_numa_placement) { 2846 delta = runtime - p->last_sum_exec_runtime; 2847 *period = now - p->last_task_numa_placement; 2848 2849 /* Avoid time going backwards, prevent potential divide error: */ 2850 if (unlikely((s64)*period < 0)) 2851 *period = 0; 2852 } else { 2853 delta = p->se.avg.load_sum; 2854 *period = LOAD_AVG_MAX; 2855 } 2856 2857 p->last_sum_exec_runtime = runtime; 2858 p->last_task_numa_placement = now; 2859 2860 return delta; 2861 } 2862 2863 /* 2864 * Determine the preferred nid for a task in a numa_group. This needs to 2865 * be done in a way that produces consistent results with group_weight, 2866 * otherwise workloads might not converge. 2867 */ 2868 static int preferred_group_nid(struct task_struct *p, int nid) 2869 { 2870 nodemask_t nodes; 2871 int dist; 2872 2873 /* Direct connections between all NUMA nodes. */ 2874 if (sched_numa_topology_type == NUMA_DIRECT) 2875 return nid; 2876 2877 /* 2878 * On a system with glueless mesh NUMA topology, group_weight 2879 * scores nodes according to the number of NUMA hinting faults on 2880 * both the node itself, and on nearby nodes. 2881 */ 2882 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2883 unsigned long score, max_score = 0; 2884 int node, max_node = nid; 2885 2886 dist = sched_max_numa_distance; 2887 2888 for_each_node_state(node, N_CPU) { 2889 score = group_weight(p, node, dist); 2890 if (score > max_score) { 2891 max_score = score; 2892 max_node = node; 2893 } 2894 } 2895 return max_node; 2896 } 2897 2898 /* 2899 * Finding the preferred nid in a system with NUMA backplane 2900 * interconnect topology is more involved. The goal is to locate 2901 * tasks from numa_groups near each other in the system, and 2902 * untangle workloads from different sides of the system. This requires 2903 * searching down the hierarchy of node groups, recursively searching 2904 * inside the highest scoring group of nodes. The nodemask tricks 2905 * keep the complexity of the search down. 2906 */ 2907 nodes = node_states[N_CPU]; 2908 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2909 unsigned long max_faults = 0; 2910 nodemask_t max_group = NODE_MASK_NONE; 2911 int a, b; 2912 2913 /* Are there nodes at this distance from each other? */ 2914 if (!find_numa_distance(dist)) 2915 continue; 2916 2917 for_each_node_mask(a, nodes) { 2918 unsigned long faults = 0; 2919 nodemask_t this_group; 2920 nodes_clear(this_group); 2921 2922 /* Sum group's NUMA faults; includes a==b case. */ 2923 for_each_node_mask(b, nodes) { 2924 if (node_distance(a, b) < dist) { 2925 faults += group_faults(p, b); 2926 node_set(b, this_group); 2927 node_clear(b, nodes); 2928 } 2929 } 2930 2931 /* Remember the top group. */ 2932 if (faults > max_faults) { 2933 max_faults = faults; 2934 max_group = this_group; 2935 /* 2936 * subtle: at the smallest distance there is 2937 * just one node left in each "group", the 2938 * winner is the preferred nid. 2939 */ 2940 nid = a; 2941 } 2942 } 2943 /* Next round, evaluate the nodes within max_group. */ 2944 if (!max_faults) 2945 break; 2946 nodes = max_group; 2947 } 2948 return nid; 2949 } 2950 2951 static void task_numa_placement(struct task_struct *p) 2952 { 2953 int seq, nid, max_nid = NUMA_NO_NODE; 2954 unsigned long max_faults = 0; 2955 unsigned long fault_types[2] = { 0, 0 }; 2956 unsigned long total_faults; 2957 u64 runtime, period; 2958 spinlock_t *group_lock = NULL; 2959 struct numa_group *ng; 2960 2961 /* 2962 * The p->mm->numa_scan_seq field gets updated without 2963 * exclusive access. Use READ_ONCE() here to ensure 2964 * that the field is read in a single access: 2965 */ 2966 seq = READ_ONCE(p->mm->numa_scan_seq); 2967 if (p->numa_scan_seq == seq) 2968 return; 2969 p->numa_scan_seq = seq; 2970 p->numa_scan_period_max = task_scan_max(p); 2971 2972 total_faults = p->numa_faults_locality[0] + 2973 p->numa_faults_locality[1]; 2974 runtime = numa_get_avg_runtime(p, &period); 2975 2976 /* If the task is part of a group prevent parallel updates to group stats */ 2977 ng = deref_curr_numa_group(p); 2978 if (ng) { 2979 group_lock = &ng->lock; 2980 spin_lock_irq(group_lock); 2981 } 2982 2983 /* Find the node with the highest number of faults */ 2984 for_each_online_node(nid) { 2985 /* Keep track of the offsets in numa_faults array */ 2986 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2987 unsigned long faults = 0, group_faults = 0; 2988 int priv; 2989 2990 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2991 long diff, f_diff, f_weight; 2992 2993 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2994 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2995 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2996 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2997 2998 /* Decay existing window, copy faults since last scan */ 2999 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 3000 fault_types[priv] += p->numa_faults[membuf_idx]; 3001 p->numa_faults[membuf_idx] = 0; 3002 3003 /* 3004 * Normalize the faults_from, so all tasks in a group 3005 * count according to CPU use, instead of by the raw 3006 * number of faults. Tasks with little runtime have 3007 * little over-all impact on throughput, and thus their 3008 * faults are less important. 3009 */ 3010 f_weight = div64_u64(runtime << 16, period + 1); 3011 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 3012 (total_faults + 1); 3013 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 3014 p->numa_faults[cpubuf_idx] = 0; 3015 3016 p->numa_faults[mem_idx] += diff; 3017 p->numa_faults[cpu_idx] += f_diff; 3018 faults += p->numa_faults[mem_idx]; 3019 p->total_numa_faults += diff; 3020 if (ng) { 3021 /* 3022 * safe because we can only change our own group 3023 * 3024 * mem_idx represents the offset for a given 3025 * nid and priv in a specific region because it 3026 * is at the beginning of the numa_faults array. 3027 */ 3028 ng->faults[mem_idx] += diff; 3029 ng->faults[cpu_idx] += f_diff; 3030 ng->total_faults += diff; 3031 group_faults += ng->faults[mem_idx]; 3032 } 3033 } 3034 3035 if (!ng) { 3036 if (faults > max_faults) { 3037 max_faults = faults; 3038 max_nid = nid; 3039 } 3040 } else if (group_faults > max_faults) { 3041 max_faults = group_faults; 3042 max_nid = nid; 3043 } 3044 } 3045 3046 /* Cannot migrate task to CPU-less node */ 3047 max_nid = numa_nearest_node(max_nid, N_CPU); 3048 3049 if (ng) { 3050 numa_group_count_active_nodes(ng); 3051 spin_unlock_irq(group_lock); 3052 max_nid = preferred_group_nid(p, max_nid); 3053 } 3054 3055 if (max_faults) { 3056 /* Set the new preferred node */ 3057 if (max_nid != p->numa_preferred_nid) 3058 sched_setnuma(p, max_nid); 3059 } 3060 3061 update_task_scan_period(p, fault_types[0], fault_types[1]); 3062 } 3063 3064 static inline int get_numa_group(struct numa_group *grp) 3065 { 3066 return refcount_inc_not_zero(&grp->refcount); 3067 } 3068 3069 static inline void put_numa_group(struct numa_group *grp) 3070 { 3071 if (refcount_dec_and_test(&grp->refcount)) 3072 kfree_rcu(grp, rcu); 3073 } 3074 3075 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 3076 int *priv) 3077 { 3078 struct numa_group *grp, *my_grp; 3079 struct task_struct *tsk; 3080 bool join = false; 3081 int cpu = cpupid_to_cpu(cpupid); 3082 int i; 3083 3084 if (unlikely(!deref_curr_numa_group(p))) { 3085 unsigned int size = sizeof(struct numa_group) + 3086 NR_NUMA_HINT_FAULT_STATS * 3087 nr_node_ids * sizeof(unsigned long); 3088 3089 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 3090 if (!grp) 3091 return; 3092 3093 refcount_set(&grp->refcount, 1); 3094 grp->active_nodes = 1; 3095 grp->max_faults_cpu = 0; 3096 spin_lock_init(&grp->lock); 3097 grp->gid = p->pid; 3098 3099 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3100 grp->faults[i] = p->numa_faults[i]; 3101 3102 grp->total_faults = p->total_numa_faults; 3103 3104 grp->nr_tasks++; 3105 rcu_assign_pointer(p->numa_group, grp); 3106 } 3107 3108 rcu_read_lock(); 3109 tsk = READ_ONCE(cpu_rq(cpu)->curr); 3110 3111 if (!cpupid_match_pid(tsk, cpupid)) 3112 goto no_join; 3113 3114 grp = rcu_dereference(tsk->numa_group); 3115 if (!grp) 3116 goto no_join; 3117 3118 my_grp = deref_curr_numa_group(p); 3119 if (grp == my_grp) 3120 goto no_join; 3121 3122 /* 3123 * Only join the other group if its bigger; if we're the bigger group, 3124 * the other task will join us. 3125 */ 3126 if (my_grp->nr_tasks > grp->nr_tasks) 3127 goto no_join; 3128 3129 /* 3130 * Tie-break on the grp address. 3131 */ 3132 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 3133 goto no_join; 3134 3135 /* Always join threads in the same process. */ 3136 if (tsk->mm == current->mm) 3137 join = true; 3138 3139 /* Simple filter to avoid false positives due to PID collisions */ 3140 if (flags & TNF_SHARED) 3141 join = true; 3142 3143 /* Update priv based on whether false sharing was detected */ 3144 *priv = !join; 3145 3146 if (join && !get_numa_group(grp)) 3147 goto no_join; 3148 3149 rcu_read_unlock(); 3150 3151 if (!join) 3152 return; 3153 3154 WARN_ON_ONCE(irqs_disabled()); 3155 double_lock_irq(&my_grp->lock, &grp->lock); 3156 3157 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 3158 my_grp->faults[i] -= p->numa_faults[i]; 3159 grp->faults[i] += p->numa_faults[i]; 3160 } 3161 my_grp->total_faults -= p->total_numa_faults; 3162 grp->total_faults += p->total_numa_faults; 3163 3164 my_grp->nr_tasks--; 3165 grp->nr_tasks++; 3166 3167 spin_unlock(&my_grp->lock); 3168 spin_unlock_irq(&grp->lock); 3169 3170 rcu_assign_pointer(p->numa_group, grp); 3171 3172 put_numa_group(my_grp); 3173 return; 3174 3175 no_join: 3176 rcu_read_unlock(); 3177 return; 3178 } 3179 3180 /* 3181 * Get rid of NUMA statistics associated with a task (either current or dead). 3182 * If @final is set, the task is dead and has reached refcount zero, so we can 3183 * safely free all relevant data structures. Otherwise, there might be 3184 * concurrent reads from places like load balancing and procfs, and we should 3185 * reset the data back to default state without freeing ->numa_faults. 3186 */ 3187 void task_numa_free(struct task_struct *p, bool final) 3188 { 3189 /* safe: p either is current or is being freed by current */ 3190 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 3191 unsigned long *numa_faults = p->numa_faults; 3192 unsigned long flags; 3193 int i; 3194 3195 if (!numa_faults) 3196 return; 3197 3198 if (grp) { 3199 spin_lock_irqsave(&grp->lock, flags); 3200 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3201 grp->faults[i] -= p->numa_faults[i]; 3202 grp->total_faults -= p->total_numa_faults; 3203 3204 grp->nr_tasks--; 3205 spin_unlock_irqrestore(&grp->lock, flags); 3206 RCU_INIT_POINTER(p->numa_group, NULL); 3207 put_numa_group(grp); 3208 } 3209 3210 if (final) { 3211 p->numa_faults = NULL; 3212 kfree(numa_faults); 3213 } else { 3214 p->total_numa_faults = 0; 3215 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3216 numa_faults[i] = 0; 3217 } 3218 } 3219 3220 /* 3221 * Got a PROT_NONE fault for a page on @node. 3222 */ 3223 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 3224 { 3225 struct task_struct *p = current; 3226 bool migrated = flags & TNF_MIGRATED; 3227 int cpu_node = task_node(current); 3228 int local = !!(flags & TNF_FAULT_LOCAL); 3229 struct numa_group *ng; 3230 int priv; 3231 3232 if (!static_branch_likely(&sched_numa_balancing)) 3233 return; 3234 3235 /* for example, ksmd faulting in a user's mm */ 3236 if (!p->mm) 3237 return; 3238 3239 /* 3240 * NUMA faults statistics are unnecessary for the slow memory 3241 * node for memory tiering mode. 3242 */ 3243 if (!node_is_toptier(mem_node) && 3244 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 3245 !cpupid_valid(last_cpupid))) 3246 return; 3247 3248 /* Allocate buffer to track faults on a per-node basis */ 3249 if (unlikely(!p->numa_faults)) { 3250 int size = sizeof(*p->numa_faults) * 3251 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 3252 3253 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 3254 if (!p->numa_faults) 3255 return; 3256 3257 p->total_numa_faults = 0; 3258 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 3259 } 3260 3261 /* 3262 * First accesses are treated as private, otherwise consider accesses 3263 * to be private if the accessing pid has not changed 3264 */ 3265 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 3266 priv = 1; 3267 } else { 3268 priv = cpupid_match_pid(p, last_cpupid); 3269 if (!priv && !(flags & TNF_NO_GROUP)) 3270 task_numa_group(p, last_cpupid, flags, &priv); 3271 } 3272 3273 /* 3274 * If a workload spans multiple NUMA nodes, a shared fault that 3275 * occurs wholly within the set of nodes that the workload is 3276 * actively using should be counted as local. This allows the 3277 * scan rate to slow down when a workload has settled down. 3278 */ 3279 ng = deref_curr_numa_group(p); 3280 if (!priv && !local && ng && ng->active_nodes > 1 && 3281 numa_is_active_node(cpu_node, ng) && 3282 numa_is_active_node(mem_node, ng)) 3283 local = 1; 3284 3285 /* 3286 * Retry to migrate task to preferred node periodically, in case it 3287 * previously failed, or the scheduler moved us. 3288 */ 3289 if (time_after(jiffies, p->numa_migrate_retry)) { 3290 task_numa_placement(p); 3291 numa_migrate_preferred(p); 3292 } 3293 3294 if (migrated) 3295 p->numa_pages_migrated += pages; 3296 if (flags & TNF_MIGRATE_FAIL) 3297 p->numa_faults_locality[2] += pages; 3298 3299 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 3300 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 3301 p->numa_faults_locality[local] += pages; 3302 } 3303 3304 static void reset_ptenuma_scan(struct task_struct *p) 3305 { 3306 /* 3307 * We only did a read acquisition of the mmap sem, so 3308 * p->mm->numa_scan_seq is written to without exclusive access 3309 * and the update is not guaranteed to be atomic. That's not 3310 * much of an issue though, since this is just used for 3311 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 3312 * expensive, to avoid any form of compiler optimizations: 3313 */ 3314 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 3315 p->mm->numa_scan_offset = 0; 3316 } 3317 3318 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) 3319 { 3320 unsigned long pids; 3321 /* 3322 * Allow unconditional access first two times, so that all the (pages) 3323 * of VMAs get prot_none fault introduced irrespective of accesses. 3324 * This is also done to avoid any side effect of task scanning 3325 * amplifying the unfairness of disjoint set of VMAs' access. 3326 */ 3327 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) 3328 return true; 3329 3330 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; 3331 if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids)) 3332 return true; 3333 3334 /* 3335 * Complete a scan that has already started regardless of PID access, or 3336 * some VMAs may never be scanned in multi-threaded applications: 3337 */ 3338 if (mm->numa_scan_offset > vma->vm_start) { 3339 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); 3340 return true; 3341 } 3342 3343 /* 3344 * This vma has not been accessed for a while, and if the number 3345 * the threads in the same process is low, which means no other 3346 * threads can help scan this vma, force a vma scan. 3347 */ 3348 if (READ_ONCE(mm->numa_scan_seq) > 3349 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) 3350 return true; 3351 3352 return false; 3353 } 3354 3355 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 3356 3357 /* 3358 * The expensive part of numa migration is done from task_work context. 3359 * Triggered from task_tick_numa(). 3360 */ 3361 static void task_numa_work(struct callback_head *work) 3362 { 3363 unsigned long migrate, next_scan, now = jiffies; 3364 struct task_struct *p = current; 3365 struct mm_struct *mm = p->mm; 3366 u64 runtime = p->se.sum_exec_runtime; 3367 struct vm_area_struct *vma; 3368 unsigned long start, end; 3369 unsigned long nr_pte_updates = 0; 3370 long pages, virtpages; 3371 struct vma_iterator vmi; 3372 bool vma_pids_skipped; 3373 bool vma_pids_forced = false; 3374 3375 WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); 3376 3377 work->next = work; 3378 /* 3379 * Who cares about NUMA placement when they're dying. 3380 * 3381 * NOTE: make sure not to dereference p->mm before this check, 3382 * exit_task_work() happens _after_ exit_mm() so we could be called 3383 * without p->mm even though we still had it when we enqueued this 3384 * work. 3385 */ 3386 if (p->flags & PF_EXITING) 3387 return; 3388 3389 /* 3390 * Memory is pinned to only one NUMA node via cpuset.mems, naturally 3391 * no page can be migrated. 3392 */ 3393 if (cpusets_enabled() && nodes_weight(cpuset_current_mems_allowed) == 1) { 3394 trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed); 3395 return; 3396 } 3397 3398 if (!mm->numa_next_scan) { 3399 mm->numa_next_scan = now + 3400 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3401 } 3402 3403 /* 3404 * Enforce maximal scan/migration frequency.. 3405 */ 3406 migrate = mm->numa_next_scan; 3407 if (time_before(now, migrate)) 3408 return; 3409 3410 if (p->numa_scan_period == 0) { 3411 p->numa_scan_period_max = task_scan_max(p); 3412 p->numa_scan_period = task_scan_start(p); 3413 } 3414 3415 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 3416 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 3417 return; 3418 3419 /* 3420 * Delay this task enough that another task of this mm will likely win 3421 * the next time around. 3422 */ 3423 p->node_stamp += 2 * TICK_NSEC; 3424 3425 pages = sysctl_numa_balancing_scan_size; 3426 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3427 virtpages = pages * 8; /* Scan up to this much virtual space */ 3428 if (!pages) 3429 return; 3430 3431 3432 if (!mmap_read_trylock(mm)) 3433 return; 3434 3435 /* 3436 * VMAs are skipped if the current PID has not trapped a fault within 3437 * the VMA recently. Allow scanning to be forced if there is no 3438 * suitable VMA remaining. 3439 */ 3440 vma_pids_skipped = false; 3441 3442 retry_pids: 3443 start = mm->numa_scan_offset; 3444 vma_iter_init(&vmi, mm, start); 3445 vma = vma_next(&vmi); 3446 if (!vma) { 3447 reset_ptenuma_scan(p); 3448 start = 0; 3449 vma_iter_set(&vmi, start); 3450 vma = vma_next(&vmi); 3451 } 3452 3453 for (; vma; vma = vma_next(&vmi)) { 3454 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3455 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3456 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); 3457 continue; 3458 } 3459 3460 /* 3461 * Shared library pages mapped by multiple processes are not 3462 * migrated as it is expected they are cache replicated. Avoid 3463 * hinting faults in read-only file-backed mappings or the vDSO 3464 * as migrating the pages will be of marginal benefit. 3465 */ 3466 if (!vma->vm_mm || 3467 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { 3468 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); 3469 continue; 3470 } 3471 3472 /* 3473 * Skip inaccessible VMAs to avoid any confusion between 3474 * PROT_NONE and NUMA hinting PTEs 3475 */ 3476 if (!vma_is_accessible(vma)) { 3477 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); 3478 continue; 3479 } 3480 3481 /* Initialise new per-VMA NUMAB state. */ 3482 if (!vma->numab_state) { 3483 struct vma_numab_state *ptr; 3484 3485 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 3486 if (!ptr) 3487 continue; 3488 3489 if (cmpxchg(&vma->numab_state, NULL, ptr)) { 3490 kfree(ptr); 3491 continue; 3492 } 3493 3494 vma->numab_state->start_scan_seq = mm->numa_scan_seq; 3495 3496 vma->numab_state->next_scan = now + 3497 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3498 3499 /* Reset happens after 4 times scan delay of scan start */ 3500 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + 3501 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3502 3503 /* 3504 * Ensure prev_scan_seq does not match numa_scan_seq, 3505 * to prevent VMAs being skipped prematurely on the 3506 * first scan: 3507 */ 3508 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; 3509 } 3510 3511 /* 3512 * Scanning the VMAs of short lived tasks add more overhead. So 3513 * delay the scan for new VMAs. 3514 */ 3515 if (mm->numa_scan_seq && time_before(jiffies, 3516 vma->numab_state->next_scan)) { 3517 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); 3518 continue; 3519 } 3520 3521 /* RESET access PIDs regularly for old VMAs. */ 3522 if (mm->numa_scan_seq && 3523 time_after(jiffies, vma->numab_state->pids_active_reset)) { 3524 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + 3525 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3526 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); 3527 vma->numab_state->pids_active[1] = 0; 3528 } 3529 3530 /* Do not rescan VMAs twice within the same sequence. */ 3531 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { 3532 mm->numa_scan_offset = vma->vm_end; 3533 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); 3534 continue; 3535 } 3536 3537 /* 3538 * Do not scan the VMA if task has not accessed it, unless no other 3539 * VMA candidate exists. 3540 */ 3541 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { 3542 vma_pids_skipped = true; 3543 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); 3544 continue; 3545 } 3546 3547 do { 3548 start = max(start, vma->vm_start); 3549 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3550 end = min(end, vma->vm_end); 3551 nr_pte_updates = change_prot_numa(vma, start, end); 3552 3553 /* 3554 * Try to scan sysctl_numa_balancing_size worth of 3555 * hpages that have at least one present PTE that 3556 * is not already PTE-numa. If the VMA contains 3557 * areas that are unused or already full of prot_numa 3558 * PTEs, scan up to virtpages, to skip through those 3559 * areas faster. 3560 */ 3561 if (nr_pte_updates) 3562 pages -= (end - start) >> PAGE_SHIFT; 3563 virtpages -= (end - start) >> PAGE_SHIFT; 3564 3565 start = end; 3566 if (pages <= 0 || virtpages <= 0) 3567 goto out; 3568 3569 cond_resched(); 3570 } while (end != vma->vm_end); 3571 3572 /* VMA scan is complete, do not scan until next sequence. */ 3573 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; 3574 3575 /* 3576 * Only force scan within one VMA at a time, to limit the 3577 * cost of scanning a potentially uninteresting VMA. 3578 */ 3579 if (vma_pids_forced) 3580 break; 3581 } 3582 3583 /* 3584 * If no VMAs are remaining and VMAs were skipped due to the PID 3585 * not accessing the VMA previously, then force a scan to ensure 3586 * forward progress: 3587 */ 3588 if (!vma && !vma_pids_forced && vma_pids_skipped) { 3589 vma_pids_forced = true; 3590 goto retry_pids; 3591 } 3592 3593 out: 3594 /* 3595 * It is possible to reach the end of the VMA list but the last few 3596 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3597 * would find the !migratable VMA on the next scan but not reset the 3598 * scanner to the start so check it now. 3599 */ 3600 if (vma) 3601 mm->numa_scan_offset = start; 3602 else 3603 reset_ptenuma_scan(p); 3604 mmap_read_unlock(mm); 3605 3606 /* 3607 * Make sure tasks use at least 32x as much time to run other code 3608 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3609 * Usually update_task_scan_period slows down scanning enough; on an 3610 * overloaded system we need to limit overhead on a per task basis. 3611 */ 3612 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3613 u64 diff = p->se.sum_exec_runtime - runtime; 3614 p->node_stamp += 32 * diff; 3615 } 3616 } 3617 3618 void init_numa_balancing(u64 clone_flags, struct task_struct *p) 3619 { 3620 int mm_users = 0; 3621 struct mm_struct *mm = p->mm; 3622 3623 if (mm) { 3624 mm_users = atomic_read(&mm->mm_users); 3625 if (mm_users == 1) { 3626 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3627 mm->numa_scan_seq = 0; 3628 } 3629 } 3630 p->node_stamp = 0; 3631 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3632 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3633 p->numa_migrate_retry = 0; 3634 /* Protect against double add, see task_tick_numa and task_numa_work */ 3635 p->numa_work.next = &p->numa_work; 3636 p->numa_faults = NULL; 3637 p->numa_pages_migrated = 0; 3638 p->total_numa_faults = 0; 3639 RCU_INIT_POINTER(p->numa_group, NULL); 3640 p->last_task_numa_placement = 0; 3641 p->last_sum_exec_runtime = 0; 3642 3643 init_task_work(&p->numa_work, task_numa_work); 3644 3645 /* New address space, reset the preferred nid */ 3646 if (!(clone_flags & CLONE_VM)) { 3647 p->numa_preferred_nid = NUMA_NO_NODE; 3648 return; 3649 } 3650 3651 /* 3652 * New thread, keep existing numa_preferred_nid which should be copied 3653 * already by arch_dup_task_struct but stagger when scans start. 3654 */ 3655 if (mm) { 3656 unsigned int delay; 3657 3658 delay = min_t(unsigned int, task_scan_max(current), 3659 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3660 delay += 2 * TICK_NSEC; 3661 p->node_stamp = delay; 3662 } 3663 } 3664 3665 /* 3666 * Drive the periodic memory faults.. 3667 */ 3668 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3669 { 3670 struct callback_head *work = &curr->numa_work; 3671 u64 period, now; 3672 3673 /* 3674 * We don't care about NUMA placement if we don't have memory. 3675 */ 3676 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3677 return; 3678 3679 /* 3680 * Using runtime rather than walltime has the dual advantage that 3681 * we (mostly) drive the selection from busy threads and that the 3682 * task needs to have done some actual work before we bother with 3683 * NUMA placement. 3684 */ 3685 now = curr->se.sum_exec_runtime; 3686 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3687 3688 if (now > curr->node_stamp + period) { 3689 if (!curr->node_stamp) 3690 curr->numa_scan_period = task_scan_start(curr); 3691 curr->node_stamp += period; 3692 3693 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3694 task_work_add(curr, work, TWA_RESUME); 3695 } 3696 } 3697 3698 static void update_scan_period(struct task_struct *p, int new_cpu) 3699 { 3700 int src_nid = cpu_to_node(task_cpu(p)); 3701 int dst_nid = cpu_to_node(new_cpu); 3702 3703 if (!static_branch_likely(&sched_numa_balancing)) 3704 return; 3705 3706 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3707 return; 3708 3709 if (src_nid == dst_nid) 3710 return; 3711 3712 /* 3713 * Allow resets if faults have been trapped before one scan 3714 * has completed. This is most likely due to a new task that 3715 * is pulled cross-node due to wakeups or load balancing. 3716 */ 3717 if (p->numa_scan_seq) { 3718 /* 3719 * Avoid scan adjustments if moving to the preferred 3720 * node or if the task was not previously running on 3721 * the preferred node. 3722 */ 3723 if (dst_nid == p->numa_preferred_nid || 3724 (p->numa_preferred_nid != NUMA_NO_NODE && 3725 src_nid != p->numa_preferred_nid)) 3726 return; 3727 } 3728 3729 p->numa_scan_period = task_scan_start(p); 3730 } 3731 3732 #else /* !CONFIG_NUMA_BALANCING: */ 3733 3734 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3735 { 3736 } 3737 3738 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3739 { 3740 } 3741 3742 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3743 { 3744 } 3745 3746 static inline void update_scan_period(struct task_struct *p, int new_cpu) 3747 { 3748 } 3749 3750 #endif /* !CONFIG_NUMA_BALANCING */ 3751 3752 static void 3753 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3754 { 3755 update_load_add(&cfs_rq->load, se->load.weight); 3756 if (entity_is_task(se)) { 3757 struct rq *rq = rq_of(cfs_rq); 3758 3759 account_numa_enqueue(rq, task_of(se)); 3760 list_add(&se->group_node, &rq->cfs_tasks); 3761 } 3762 cfs_rq->nr_queued++; 3763 } 3764 3765 static void 3766 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3767 { 3768 update_load_sub(&cfs_rq->load, se->load.weight); 3769 if (entity_is_task(se)) { 3770 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3771 list_del_init(&se->group_node); 3772 } 3773 cfs_rq->nr_queued--; 3774 } 3775 3776 /* 3777 * Signed add and clamp on underflow. 3778 * 3779 * Explicitly do a load-store to ensure the intermediate value never hits 3780 * memory. This allows lockless observations without ever seeing the negative 3781 * values. 3782 */ 3783 #define add_positive(_ptr, _val) do { \ 3784 typeof(_ptr) ptr = (_ptr); \ 3785 typeof(_val) val = (_val); \ 3786 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3787 \ 3788 res = var + val; \ 3789 \ 3790 if (val < 0 && res > var) \ 3791 res = 0; \ 3792 \ 3793 WRITE_ONCE(*ptr, res); \ 3794 } while (0) 3795 3796 /* 3797 * Unsigned subtract and clamp on underflow. 3798 * 3799 * Explicitly do a load-store to ensure the intermediate value never hits 3800 * memory. This allows lockless observations without ever seeing the negative 3801 * values. 3802 */ 3803 #define sub_positive(_ptr, _val) do { \ 3804 typeof(_ptr) ptr = (_ptr); \ 3805 typeof(*ptr) val = (_val); \ 3806 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3807 res = var - val; \ 3808 if (res > var) \ 3809 res = 0; \ 3810 WRITE_ONCE(*ptr, res); \ 3811 } while (0) 3812 3813 /* 3814 * Remove and clamp on negative, from a local variable. 3815 * 3816 * A variant of sub_positive(), which does not use explicit load-store 3817 * and is thus optimized for local variable updates. 3818 */ 3819 #define lsub_positive(_ptr, _val) do { \ 3820 typeof(_ptr) ptr = (_ptr); \ 3821 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3822 } while (0) 3823 3824 static inline void 3825 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3826 { 3827 cfs_rq->avg.load_avg += se->avg.load_avg; 3828 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3829 } 3830 3831 static inline void 3832 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3833 { 3834 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3835 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3836 /* See update_cfs_rq_load_avg() */ 3837 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3838 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3839 } 3840 3841 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags); 3842 3843 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3844 unsigned long weight) 3845 { 3846 bool curr = cfs_rq->curr == se; 3847 bool rel_vprot = false; 3848 u64 vprot; 3849 3850 if (se->on_rq) { 3851 /* commit outstanding execution time */ 3852 update_curr(cfs_rq); 3853 update_entity_lag(cfs_rq, se); 3854 se->deadline -= se->vruntime; 3855 se->rel_deadline = 1; 3856 if (curr && protect_slice(se)) { 3857 vprot = se->vprot - se->vruntime; 3858 rel_vprot = true; 3859 } 3860 3861 cfs_rq->nr_queued--; 3862 if (!curr) 3863 __dequeue_entity(cfs_rq, se); 3864 update_load_sub(&cfs_rq->load, se->load.weight); 3865 } 3866 dequeue_load_avg(cfs_rq, se); 3867 3868 /* 3869 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), 3870 * we need to scale se->vlag when w_i changes. 3871 */ 3872 se->vlag = div_s64(se->vlag * se->load.weight, weight); 3873 if (se->rel_deadline) 3874 se->deadline = div_s64(se->deadline * se->load.weight, weight); 3875 3876 if (rel_vprot) 3877 vprot = div_s64(vprot * se->load.weight, weight); 3878 3879 update_load_set(&se->load, weight); 3880 3881 do { 3882 u32 divider = get_pelt_divider(&se->avg); 3883 3884 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3885 } while (0); 3886 3887 enqueue_load_avg(cfs_rq, se); 3888 if (se->on_rq) { 3889 place_entity(cfs_rq, se, 0); 3890 if (rel_vprot) 3891 se->vprot = se->vruntime + vprot; 3892 update_load_add(&cfs_rq->load, se->load.weight); 3893 if (!curr) 3894 __enqueue_entity(cfs_rq, se); 3895 cfs_rq->nr_queued++; 3896 } 3897 } 3898 3899 static void reweight_task_fair(struct rq *rq, struct task_struct *p, 3900 const struct load_weight *lw) 3901 { 3902 struct sched_entity *se = &p->se; 3903 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3904 struct load_weight *load = &se->load; 3905 3906 reweight_entity(cfs_rq, se, lw->weight); 3907 load->inv_weight = lw->inv_weight; 3908 } 3909 3910 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3911 3912 #ifdef CONFIG_FAIR_GROUP_SCHED 3913 /* 3914 * All this does is approximate the hierarchical proportion which includes that 3915 * global sum we all love to hate. 3916 * 3917 * That is, the weight of a group entity, is the proportional share of the 3918 * group weight based on the group runqueue weights. That is: 3919 * 3920 * tg->weight * grq->load.weight 3921 * ge->load.weight = ----------------------------- (1) 3922 * \Sum grq->load.weight 3923 * 3924 * Now, because computing that sum is prohibitively expensive to compute (been 3925 * there, done that) we approximate it with this average stuff. The average 3926 * moves slower and therefore the approximation is cheaper and more stable. 3927 * 3928 * So instead of the above, we substitute: 3929 * 3930 * grq->load.weight -> grq->avg.load_avg (2) 3931 * 3932 * which yields the following: 3933 * 3934 * tg->weight * grq->avg.load_avg 3935 * ge->load.weight = ------------------------------ (3) 3936 * tg->load_avg 3937 * 3938 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3939 * 3940 * That is shares_avg, and it is right (given the approximation (2)). 3941 * 3942 * The problem with it is that because the average is slow -- it was designed 3943 * to be exactly that of course -- this leads to transients in boundary 3944 * conditions. In specific, the case where the group was idle and we start the 3945 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3946 * yielding bad latency etc.. 3947 * 3948 * Now, in that special case (1) reduces to: 3949 * 3950 * tg->weight * grq->load.weight 3951 * ge->load.weight = ----------------------------- = tg->weight (4) 3952 * grp->load.weight 3953 * 3954 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3955 * 3956 * So what we do is modify our approximation (3) to approach (4) in the (near) 3957 * UP case, like: 3958 * 3959 * ge->load.weight = 3960 * 3961 * tg->weight * grq->load.weight 3962 * --------------------------------------------------- (5) 3963 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3964 * 3965 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3966 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3967 * 3968 * 3969 * tg->weight * grq->load.weight 3970 * ge->load.weight = ----------------------------- (6) 3971 * tg_load_avg' 3972 * 3973 * Where: 3974 * 3975 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3976 * max(grq->load.weight, grq->avg.load_avg) 3977 * 3978 * And that is shares_weight and is icky. In the (near) UP case it approaches 3979 * (4) while in the normal case it approaches (3). It consistently 3980 * overestimates the ge->load.weight and therefore: 3981 * 3982 * \Sum ge->load.weight >= tg->weight 3983 * 3984 * hence icky! 3985 */ 3986 static long calc_group_shares(struct cfs_rq *cfs_rq) 3987 { 3988 long tg_weight, tg_shares, load, shares; 3989 struct task_group *tg = cfs_rq->tg; 3990 3991 tg_shares = READ_ONCE(tg->shares); 3992 3993 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3994 3995 tg_weight = atomic_long_read(&tg->load_avg); 3996 3997 /* Ensure tg_weight >= load */ 3998 tg_weight -= cfs_rq->tg_load_avg_contrib; 3999 tg_weight += load; 4000 4001 shares = (tg_shares * load); 4002 if (tg_weight) 4003 shares /= tg_weight; 4004 4005 /* 4006 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 4007 * of a group with small tg->shares value. It is a floor value which is 4008 * assigned as a minimum load.weight to the sched_entity representing 4009 * the group on a CPU. 4010 * 4011 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 4012 * on an 8-core system with 8 tasks each runnable on one CPU shares has 4013 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 4014 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 4015 * instead of 0. 4016 */ 4017 return clamp_t(long, shares, MIN_SHARES, tg_shares); 4018 } 4019 4020 /* 4021 * Recomputes the group entity based on the current state of its group 4022 * runqueue. 4023 */ 4024 static void update_cfs_group(struct sched_entity *se) 4025 { 4026 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4027 long shares; 4028 4029 /* 4030 * When a group becomes empty, preserve its weight. This matters for 4031 * DELAY_DEQUEUE. 4032 */ 4033 if (!gcfs_rq || !gcfs_rq->load.weight) 4034 return; 4035 4036 shares = calc_group_shares(gcfs_rq); 4037 if (unlikely(se->load.weight != shares)) 4038 reweight_entity(cfs_rq_of(se), se, shares); 4039 } 4040 4041 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 4042 static inline void update_cfs_group(struct sched_entity *se) 4043 { 4044 } 4045 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 4046 4047 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 4048 { 4049 struct rq *rq = rq_of(cfs_rq); 4050 4051 if (&rq->cfs == cfs_rq) { 4052 /* 4053 * There are a few boundary cases this might miss but it should 4054 * get called often enough that that should (hopefully) not be 4055 * a real problem. 4056 * 4057 * It will not get called when we go idle, because the idle 4058 * thread is a different class (!fair), nor will the utilization 4059 * number include things like RT tasks. 4060 * 4061 * As is, the util number is not freq-invariant (we'd have to 4062 * implement arch_scale_freq_capacity() for that). 4063 * 4064 * See cpu_util_cfs(). 4065 */ 4066 cpufreq_update_util(rq, flags); 4067 } 4068 } 4069 4070 static inline bool load_avg_is_decayed(struct sched_avg *sa) 4071 { 4072 if (sa->load_sum) 4073 return false; 4074 4075 if (sa->util_sum) 4076 return false; 4077 4078 if (sa->runnable_sum) 4079 return false; 4080 4081 /* 4082 * _avg must be null when _sum are null because _avg = _sum / divider 4083 * Make sure that rounding and/or propagation of PELT values never 4084 * break this. 4085 */ 4086 WARN_ON_ONCE(sa->load_avg || 4087 sa->util_avg || 4088 sa->runnable_avg); 4089 4090 return true; 4091 } 4092 4093 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 4094 { 4095 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 4096 cfs_rq->last_update_time_copy); 4097 } 4098 #ifdef CONFIG_FAIR_GROUP_SCHED 4099 /* 4100 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 4101 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 4102 * bottom-up, we only have to test whether the cfs_rq before us on the list 4103 * is our child. 4104 * If cfs_rq is not on the list, test whether a child needs its to be added to 4105 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 4106 */ 4107 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 4108 { 4109 struct cfs_rq *prev_cfs_rq; 4110 struct list_head *prev; 4111 struct rq *rq = rq_of(cfs_rq); 4112 4113 if (cfs_rq->on_list) { 4114 prev = cfs_rq->leaf_cfs_rq_list.prev; 4115 } else { 4116 prev = rq->tmp_alone_branch; 4117 } 4118 4119 if (prev == &rq->leaf_cfs_rq_list) 4120 return false; 4121 4122 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 4123 4124 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 4125 } 4126 4127 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4128 { 4129 if (cfs_rq->load.weight) 4130 return false; 4131 4132 if (!load_avg_is_decayed(&cfs_rq->avg)) 4133 return false; 4134 4135 if (child_cfs_rq_on_list(cfs_rq)) 4136 return false; 4137 4138 if (cfs_rq->tg_load_avg_contrib) 4139 return false; 4140 4141 return true; 4142 } 4143 4144 /** 4145 * update_tg_load_avg - update the tg's load avg 4146 * @cfs_rq: the cfs_rq whose avg changed 4147 * 4148 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 4149 * However, because tg->load_avg is a global value there are performance 4150 * considerations. 4151 * 4152 * In order to avoid having to look at the other cfs_rq's, we use a 4153 * differential update where we store the last value we propagated. This in 4154 * turn allows skipping updates if the differential is 'small'. 4155 * 4156 * Updating tg's load_avg is necessary before update_cfs_share(). 4157 */ 4158 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 4159 { 4160 long delta; 4161 u64 now; 4162 4163 /* 4164 * No need to update load_avg for root_task_group as it is not used. 4165 */ 4166 if (cfs_rq->tg == &root_task_group) 4167 return; 4168 4169 /* rq has been offline and doesn't contribute to the share anymore: */ 4170 if (!cpu_active(cpu_of(rq_of(cfs_rq)))) 4171 return; 4172 4173 /* 4174 * For migration heavy workloads, access to tg->load_avg can be 4175 * unbound. Limit the update rate to at most once per ms. 4176 */ 4177 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4178 if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) 4179 return; 4180 4181 delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 4182 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 4183 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4184 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 4185 cfs_rq->last_update_tg_load_avg = now; 4186 } 4187 } 4188 4189 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) 4190 { 4191 long delta; 4192 u64 now; 4193 4194 /* 4195 * No need to update load_avg for root_task_group, as it is not used. 4196 */ 4197 if (cfs_rq->tg == &root_task_group) 4198 return; 4199 4200 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4201 delta = 0 - cfs_rq->tg_load_avg_contrib; 4202 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4203 cfs_rq->tg_load_avg_contrib = 0; 4204 cfs_rq->last_update_tg_load_avg = now; 4205 } 4206 4207 /* CPU offline callback: */ 4208 static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) 4209 { 4210 struct task_group *tg; 4211 4212 lockdep_assert_rq_held(rq); 4213 4214 /* 4215 * The rq clock has already been updated in 4216 * set_rq_offline(), so we should skip updating 4217 * the rq clock again in unthrottle_cfs_rq(). 4218 */ 4219 rq_clock_start_loop_update(rq); 4220 4221 rcu_read_lock(); 4222 list_for_each_entry_rcu(tg, &task_groups, list) { 4223 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4224 4225 clear_tg_load_avg(cfs_rq); 4226 } 4227 rcu_read_unlock(); 4228 4229 rq_clock_stop_loop_update(rq); 4230 } 4231 4232 /* 4233 * Called within set_task_rq() right before setting a task's CPU. The 4234 * caller only guarantees p->pi_lock is held; no other assumptions, 4235 * including the state of rq->lock, should be made. 4236 */ 4237 void set_task_rq_fair(struct sched_entity *se, 4238 struct cfs_rq *prev, struct cfs_rq *next) 4239 { 4240 u64 p_last_update_time; 4241 u64 n_last_update_time; 4242 4243 if (!sched_feat(ATTACH_AGE_LOAD)) 4244 return; 4245 4246 /* 4247 * We are supposed to update the task to "current" time, then its up to 4248 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 4249 * getting what current time is, so simply throw away the out-of-date 4250 * time. This will result in the wakee task is less decayed, but giving 4251 * the wakee more load sounds not bad. 4252 */ 4253 if (!(se->avg.last_update_time && prev)) 4254 return; 4255 4256 p_last_update_time = cfs_rq_last_update_time(prev); 4257 n_last_update_time = cfs_rq_last_update_time(next); 4258 4259 __update_load_avg_blocked_se(p_last_update_time, se); 4260 se->avg.last_update_time = n_last_update_time; 4261 } 4262 4263 /* 4264 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 4265 * propagate its contribution. The key to this propagation is the invariant 4266 * that for each group: 4267 * 4268 * ge->avg == grq->avg (1) 4269 * 4270 * _IFF_ we look at the pure running and runnable sums. Because they 4271 * represent the very same entity, just at different points in the hierarchy. 4272 * 4273 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 4274 * and simply copies the running/runnable sum over (but still wrong, because 4275 * the group entity and group rq do not have their PELT windows aligned). 4276 * 4277 * However, update_tg_cfs_load() is more complex. So we have: 4278 * 4279 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 4280 * 4281 * And since, like util, the runnable part should be directly transferable, 4282 * the following would _appear_ to be the straight forward approach: 4283 * 4284 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 4285 * 4286 * And per (1) we have: 4287 * 4288 * ge->avg.runnable_avg == grq->avg.runnable_avg 4289 * 4290 * Which gives: 4291 * 4292 * ge->load.weight * grq->avg.load_avg 4293 * ge->avg.load_avg = ----------------------------------- (4) 4294 * grq->load.weight 4295 * 4296 * Except that is wrong! 4297 * 4298 * Because while for entities historical weight is not important and we 4299 * really only care about our future and therefore can consider a pure 4300 * runnable sum, runqueues can NOT do this. 4301 * 4302 * We specifically want runqueues to have a load_avg that includes 4303 * historical weights. Those represent the blocked load, the load we expect 4304 * to (shortly) return to us. This only works by keeping the weights as 4305 * integral part of the sum. We therefore cannot decompose as per (3). 4306 * 4307 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 4308 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 4309 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 4310 * runnable section of these tasks overlap (or not). If they were to perfectly 4311 * align the rq as a whole would be runnable 2/3 of the time. If however we 4312 * always have at least 1 runnable task, the rq as a whole is always runnable. 4313 * 4314 * So we'll have to approximate.. :/ 4315 * 4316 * Given the constraint: 4317 * 4318 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 4319 * 4320 * We can construct a rule that adds runnable to a rq by assuming minimal 4321 * overlap. 4322 * 4323 * On removal, we'll assume each task is equally runnable; which yields: 4324 * 4325 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 4326 * 4327 * XXX: only do this for the part of runnable > running ? 4328 * 4329 */ 4330 static inline void 4331 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4332 { 4333 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 4334 u32 new_sum, divider; 4335 4336 /* Nothing to update */ 4337 if (!delta_avg) 4338 return; 4339 4340 /* 4341 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4342 * See ___update_load_avg() for details. 4343 */ 4344 divider = get_pelt_divider(&cfs_rq->avg); 4345 4346 4347 /* Set new sched_entity's utilization */ 4348 se->avg.util_avg = gcfs_rq->avg.util_avg; 4349 new_sum = se->avg.util_avg * divider; 4350 delta_sum = (long)new_sum - (long)se->avg.util_sum; 4351 se->avg.util_sum = new_sum; 4352 4353 /* Update parent cfs_rq utilization */ 4354 add_positive(&cfs_rq->avg.util_avg, delta_avg); 4355 add_positive(&cfs_rq->avg.util_sum, delta_sum); 4356 4357 /* See update_cfs_rq_load_avg() */ 4358 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4359 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4360 } 4361 4362 static inline void 4363 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4364 { 4365 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 4366 u32 new_sum, divider; 4367 4368 /* Nothing to update */ 4369 if (!delta_avg) 4370 return; 4371 4372 /* 4373 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4374 * See ___update_load_avg() for details. 4375 */ 4376 divider = get_pelt_divider(&cfs_rq->avg); 4377 4378 /* Set new sched_entity's runnable */ 4379 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 4380 new_sum = se->avg.runnable_avg * divider; 4381 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 4382 se->avg.runnable_sum = new_sum; 4383 4384 /* Update parent cfs_rq runnable */ 4385 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); 4386 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); 4387 /* See update_cfs_rq_load_avg() */ 4388 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4389 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4390 } 4391 4392 static inline void 4393 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4394 { 4395 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 4396 unsigned long load_avg; 4397 u64 load_sum = 0; 4398 s64 delta_sum; 4399 u32 divider; 4400 4401 if (!runnable_sum) 4402 return; 4403 4404 gcfs_rq->prop_runnable_sum = 0; 4405 4406 /* 4407 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4408 * See ___update_load_avg() for details. 4409 */ 4410 divider = get_pelt_divider(&cfs_rq->avg); 4411 4412 if (runnable_sum >= 0) { 4413 /* 4414 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 4415 * the CPU is saturated running == runnable. 4416 */ 4417 runnable_sum += se->avg.load_sum; 4418 runnable_sum = min_t(long, runnable_sum, divider); 4419 } else { 4420 /* 4421 * Estimate the new unweighted runnable_sum of the gcfs_rq by 4422 * assuming all tasks are equally runnable. 4423 */ 4424 if (scale_load_down(gcfs_rq->load.weight)) { 4425 load_sum = div_u64(gcfs_rq->avg.load_sum, 4426 scale_load_down(gcfs_rq->load.weight)); 4427 } 4428 4429 /* But make sure to not inflate se's runnable */ 4430 runnable_sum = min(se->avg.load_sum, load_sum); 4431 } 4432 4433 /* 4434 * runnable_sum can't be lower than running_sum 4435 * Rescale running sum to be in the same range as runnable sum 4436 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 4437 * runnable_sum is in [0 : LOAD_AVG_MAX] 4438 */ 4439 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 4440 runnable_sum = max(runnable_sum, running_sum); 4441 4442 load_sum = se_weight(se) * runnable_sum; 4443 load_avg = div_u64(load_sum, divider); 4444 4445 delta_avg = load_avg - se->avg.load_avg; 4446 if (!delta_avg) 4447 return; 4448 4449 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 4450 4451 se->avg.load_sum = runnable_sum; 4452 se->avg.load_avg = load_avg; 4453 add_positive(&cfs_rq->avg.load_avg, delta_avg); 4454 add_positive(&cfs_rq->avg.load_sum, delta_sum); 4455 /* See update_cfs_rq_load_avg() */ 4456 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 4457 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 4458 } 4459 4460 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 4461 { 4462 cfs_rq->propagate = 1; 4463 cfs_rq->prop_runnable_sum += runnable_sum; 4464 } 4465 4466 /* Update task and its cfs_rq load average */ 4467 static inline int propagate_entity_load_avg(struct sched_entity *se) 4468 { 4469 struct cfs_rq *cfs_rq, *gcfs_rq; 4470 4471 if (entity_is_task(se)) 4472 return 0; 4473 4474 gcfs_rq = group_cfs_rq(se); 4475 if (!gcfs_rq->propagate) 4476 return 0; 4477 4478 gcfs_rq->propagate = 0; 4479 4480 cfs_rq = cfs_rq_of(se); 4481 4482 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 4483 4484 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 4485 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 4486 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 4487 4488 trace_pelt_cfs_tp(cfs_rq); 4489 trace_pelt_se_tp(se); 4490 4491 return 1; 4492 } 4493 4494 /* 4495 * Check if we need to update the load and the utilization of a blocked 4496 * group_entity: 4497 */ 4498 static inline bool skip_blocked_update(struct sched_entity *se) 4499 { 4500 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4501 4502 /* 4503 * If sched_entity still have not zero load or utilization, we have to 4504 * decay it: 4505 */ 4506 if (se->avg.load_avg || se->avg.util_avg) 4507 return false; 4508 4509 /* 4510 * If there is a pending propagation, we have to update the load and 4511 * the utilization of the sched_entity: 4512 */ 4513 if (gcfs_rq->propagate) 4514 return false; 4515 4516 /* 4517 * Otherwise, the load and the utilization of the sched_entity is 4518 * already zero and there is no pending propagation, so it will be a 4519 * waste of time to try to decay it: 4520 */ 4521 return true; 4522 } 4523 4524 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 4525 4526 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 4527 4528 static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {} 4529 4530 static inline int propagate_entity_load_avg(struct sched_entity *se) 4531 { 4532 return 0; 4533 } 4534 4535 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 4536 4537 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 4538 4539 #ifdef CONFIG_NO_HZ_COMMON 4540 static inline void migrate_se_pelt_lag(struct sched_entity *se) 4541 { 4542 u64 throttled = 0, now, lut; 4543 struct cfs_rq *cfs_rq; 4544 struct rq *rq; 4545 bool is_idle; 4546 4547 if (load_avg_is_decayed(&se->avg)) 4548 return; 4549 4550 cfs_rq = cfs_rq_of(se); 4551 rq = rq_of(cfs_rq); 4552 4553 rcu_read_lock(); 4554 is_idle = is_idle_task(rcu_dereference(rq->curr)); 4555 rcu_read_unlock(); 4556 4557 /* 4558 * The lag estimation comes with a cost we don't want to pay all the 4559 * time. Hence, limiting to the case where the source CPU is idle and 4560 * we know we are at the greatest risk to have an outdated clock. 4561 */ 4562 if (!is_idle) 4563 return; 4564 4565 /* 4566 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4567 * 4568 * last_update_time (the cfs_rq's last_update_time) 4569 * = cfs_rq_clock_pelt()@cfs_rq_idle 4570 * = rq_clock_pelt()@cfs_rq_idle 4571 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4572 * 4573 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4574 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4575 * 4576 * rq_idle_lag (delta between now and rq's update) 4577 * = sched_clock_cpu() - rq_clock()@rq_idle 4578 * 4579 * We can then write: 4580 * 4581 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4582 * sched_clock_cpu() - rq_clock()@rq_idle 4583 * Where: 4584 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4585 * rq_clock()@rq_idle is rq->clock_idle 4586 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4587 * is cfs_rq->throttled_pelt_idle 4588 */ 4589 4590 #ifdef CONFIG_CFS_BANDWIDTH 4591 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4592 /* The clock has been stopped for throttling */ 4593 if (throttled == U64_MAX) 4594 return; 4595 #endif 4596 now = u64_u32_load(rq->clock_pelt_idle); 4597 /* 4598 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4599 * is observed the old clock_pelt_idle value and the new clock_idle, 4600 * which lead to an underestimation. The opposite would lead to an 4601 * overestimation. 4602 */ 4603 smp_rmb(); 4604 lut = cfs_rq_last_update_time(cfs_rq); 4605 4606 now -= throttled; 4607 if (now < lut) 4608 /* 4609 * cfs_rq->avg.last_update_time is more recent than our 4610 * estimation, let's use it. 4611 */ 4612 now = lut; 4613 else 4614 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4615 4616 __update_load_avg_blocked_se(now, se); 4617 } 4618 #else /* !CONFIG_NO_HZ_COMMON: */ 4619 static void migrate_se_pelt_lag(struct sched_entity *se) {} 4620 #endif /* !CONFIG_NO_HZ_COMMON */ 4621 4622 /** 4623 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4624 * @now: current time, as per cfs_rq_clock_pelt() 4625 * @cfs_rq: cfs_rq to update 4626 * 4627 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4628 * avg. The immediate corollary is that all (fair) tasks must be attached. 4629 * 4630 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4631 * 4632 * Return: true if the load decayed or we removed load. 4633 * 4634 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4635 * call update_tg_load_avg() when this function returns true. 4636 */ 4637 static inline int 4638 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4639 { 4640 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4641 struct sched_avg *sa = &cfs_rq->avg; 4642 int decayed = 0; 4643 4644 if (cfs_rq->removed.nr) { 4645 unsigned long r; 4646 u32 divider = get_pelt_divider(&cfs_rq->avg); 4647 4648 raw_spin_lock(&cfs_rq->removed.lock); 4649 swap(cfs_rq->removed.util_avg, removed_util); 4650 swap(cfs_rq->removed.load_avg, removed_load); 4651 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4652 cfs_rq->removed.nr = 0; 4653 raw_spin_unlock(&cfs_rq->removed.lock); 4654 4655 r = removed_load; 4656 sub_positive(&sa->load_avg, r); 4657 sub_positive(&sa->load_sum, r * divider); 4658 /* See sa->util_sum below */ 4659 sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER); 4660 4661 r = removed_util; 4662 sub_positive(&sa->util_avg, r); 4663 sub_positive(&sa->util_sum, r * divider); 4664 /* 4665 * Because of rounding, se->util_sum might ends up being +1 more than 4666 * cfs->util_sum. Although this is not a problem by itself, detaching 4667 * a lot of tasks with the rounding problem between 2 updates of 4668 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4669 * cfs_util_avg is not. 4670 * Check that util_sum is still above its lower bound for the new 4671 * util_avg. Given that period_contrib might have moved since the last 4672 * sync, we are only sure that util_sum must be above or equal to 4673 * util_avg * minimum possible divider 4674 */ 4675 sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER); 4676 4677 r = removed_runnable; 4678 sub_positive(&sa->runnable_avg, r); 4679 sub_positive(&sa->runnable_sum, r * divider); 4680 /* See sa->util_sum above */ 4681 sa->runnable_sum = max_t(u32, sa->runnable_sum, 4682 sa->runnable_avg * PELT_MIN_DIVIDER); 4683 4684 /* 4685 * removed_runnable is the unweighted version of removed_load so we 4686 * can use it to estimate removed_load_sum. 4687 */ 4688 add_tg_cfs_propagate(cfs_rq, 4689 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4690 4691 decayed = 1; 4692 } 4693 4694 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4695 u64_u32_store_copy(sa->last_update_time, 4696 cfs_rq->last_update_time_copy, 4697 sa->last_update_time); 4698 return decayed; 4699 } 4700 4701 /** 4702 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4703 * @cfs_rq: cfs_rq to attach to 4704 * @se: sched_entity to attach 4705 * 4706 * Must call update_cfs_rq_load_avg() before this, since we rely on 4707 * cfs_rq->avg.last_update_time being current. 4708 */ 4709 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4710 { 4711 /* 4712 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4713 * See ___update_load_avg() for details. 4714 */ 4715 u32 divider = get_pelt_divider(&cfs_rq->avg); 4716 4717 /* 4718 * When we attach the @se to the @cfs_rq, we must align the decay 4719 * window because without that, really weird and wonderful things can 4720 * happen. 4721 * 4722 * XXX illustrate 4723 */ 4724 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4725 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4726 4727 /* 4728 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4729 * period_contrib. This isn't strictly correct, but since we're 4730 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4731 * _sum a little. 4732 */ 4733 se->avg.util_sum = se->avg.util_avg * divider; 4734 4735 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4736 4737 se->avg.load_sum = se->avg.load_avg * divider; 4738 if (se_weight(se) < se->avg.load_sum) 4739 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4740 else 4741 se->avg.load_sum = 1; 4742 4743 enqueue_load_avg(cfs_rq, se); 4744 cfs_rq->avg.util_avg += se->avg.util_avg; 4745 cfs_rq->avg.util_sum += se->avg.util_sum; 4746 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4747 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4748 4749 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4750 4751 cfs_rq_util_change(cfs_rq, 0); 4752 4753 trace_pelt_cfs_tp(cfs_rq); 4754 } 4755 4756 /** 4757 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4758 * @cfs_rq: cfs_rq to detach from 4759 * @se: sched_entity to detach 4760 * 4761 * Must call update_cfs_rq_load_avg() before this, since we rely on 4762 * cfs_rq->avg.last_update_time being current. 4763 */ 4764 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4765 { 4766 dequeue_load_avg(cfs_rq, se); 4767 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 4768 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 4769 /* See update_cfs_rq_load_avg() */ 4770 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4771 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4772 4773 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 4774 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 4775 /* See update_cfs_rq_load_avg() */ 4776 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4777 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4778 4779 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4780 4781 cfs_rq_util_change(cfs_rq, 0); 4782 4783 trace_pelt_cfs_tp(cfs_rq); 4784 } 4785 4786 /* 4787 * Optional action to be done while updating the load average 4788 */ 4789 #define UPDATE_TG 0x1 4790 #define SKIP_AGE_LOAD 0x2 4791 #define DO_ATTACH 0x4 4792 #define DO_DETACH 0x8 4793 4794 /* Update task and its cfs_rq load average */ 4795 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4796 { 4797 u64 now = cfs_rq_clock_pelt(cfs_rq); 4798 int decayed; 4799 4800 /* 4801 * Track task load average for carrying it to new CPU after migrated, and 4802 * track group sched_entity load average for task_h_load calculation in migration 4803 */ 4804 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4805 __update_load_avg_se(now, cfs_rq, se); 4806 4807 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4808 decayed |= propagate_entity_load_avg(se); 4809 4810 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4811 4812 /* 4813 * DO_ATTACH means we're here from enqueue_entity(). 4814 * !last_update_time means we've passed through 4815 * migrate_task_rq_fair() indicating we migrated. 4816 * 4817 * IOW we're enqueueing a task on a new CPU. 4818 */ 4819 attach_entity_load_avg(cfs_rq, se); 4820 update_tg_load_avg(cfs_rq); 4821 4822 } else if (flags & DO_DETACH) { 4823 /* 4824 * DO_DETACH means we're here from dequeue_entity() 4825 * and we are migrating task out of the CPU. 4826 */ 4827 detach_entity_load_avg(cfs_rq, se); 4828 update_tg_load_avg(cfs_rq); 4829 } else if (decayed) { 4830 cfs_rq_util_change(cfs_rq, 0); 4831 4832 if (flags & UPDATE_TG) 4833 update_tg_load_avg(cfs_rq); 4834 } 4835 } 4836 4837 /* 4838 * Synchronize entity load avg of dequeued entity without locking 4839 * the previous rq. 4840 */ 4841 static void sync_entity_load_avg(struct sched_entity *se) 4842 { 4843 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4844 u64 last_update_time; 4845 4846 last_update_time = cfs_rq_last_update_time(cfs_rq); 4847 __update_load_avg_blocked_se(last_update_time, se); 4848 } 4849 4850 /* 4851 * Task first catches up with cfs_rq, and then subtract 4852 * itself from the cfs_rq (task must be off the queue now). 4853 */ 4854 static void remove_entity_load_avg(struct sched_entity *se) 4855 { 4856 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4857 unsigned long flags; 4858 4859 /* 4860 * tasks cannot exit without having gone through wake_up_new_task() -> 4861 * enqueue_task_fair() which will have added things to the cfs_rq, 4862 * so we can remove unconditionally. 4863 */ 4864 4865 sync_entity_load_avg(se); 4866 4867 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4868 ++cfs_rq->removed.nr; 4869 cfs_rq->removed.util_avg += se->avg.util_avg; 4870 cfs_rq->removed.load_avg += se->avg.load_avg; 4871 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4872 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4873 } 4874 4875 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4876 { 4877 return cfs_rq->avg.runnable_avg; 4878 } 4879 4880 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4881 { 4882 return cfs_rq->avg.load_avg; 4883 } 4884 4885 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf); 4886 4887 static inline unsigned long task_util(struct task_struct *p) 4888 { 4889 return READ_ONCE(p->se.avg.util_avg); 4890 } 4891 4892 static inline unsigned long task_runnable(struct task_struct *p) 4893 { 4894 return READ_ONCE(p->se.avg.runnable_avg); 4895 } 4896 4897 static inline unsigned long _task_util_est(struct task_struct *p) 4898 { 4899 return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; 4900 } 4901 4902 static inline unsigned long task_util_est(struct task_struct *p) 4903 { 4904 return max(task_util(p), _task_util_est(p)); 4905 } 4906 4907 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4908 struct task_struct *p) 4909 { 4910 unsigned int enqueued; 4911 4912 if (!sched_feat(UTIL_EST)) 4913 return; 4914 4915 /* Update root cfs_rq's estimated utilization */ 4916 enqueued = cfs_rq->avg.util_est; 4917 enqueued += _task_util_est(p); 4918 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4919 4920 trace_sched_util_est_cfs_tp(cfs_rq); 4921 } 4922 4923 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4924 struct task_struct *p) 4925 { 4926 unsigned int enqueued; 4927 4928 if (!sched_feat(UTIL_EST)) 4929 return; 4930 4931 /* Update root cfs_rq's estimated utilization */ 4932 enqueued = cfs_rq->avg.util_est; 4933 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4934 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4935 4936 trace_sched_util_est_cfs_tp(cfs_rq); 4937 } 4938 4939 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4940 4941 static inline void util_est_update(struct cfs_rq *cfs_rq, 4942 struct task_struct *p, 4943 bool task_sleep) 4944 { 4945 unsigned int ewma, dequeued, last_ewma_diff; 4946 4947 if (!sched_feat(UTIL_EST)) 4948 return; 4949 4950 /* 4951 * Skip update of task's estimated utilization when the task has not 4952 * yet completed an activation, e.g. being migrated. 4953 */ 4954 if (!task_sleep) 4955 return; 4956 4957 /* Get current estimate of utilization */ 4958 ewma = READ_ONCE(p->se.avg.util_est); 4959 4960 /* 4961 * If the PELT values haven't changed since enqueue time, 4962 * skip the util_est update. 4963 */ 4964 if (ewma & UTIL_AVG_UNCHANGED) 4965 return; 4966 4967 /* Get utilization at dequeue */ 4968 dequeued = task_util(p); 4969 4970 /* 4971 * Reset EWMA on utilization increases, the moving average is used only 4972 * to smooth utilization decreases. 4973 */ 4974 if (ewma <= dequeued) { 4975 ewma = dequeued; 4976 goto done; 4977 } 4978 4979 /* 4980 * Skip update of task's estimated utilization when its members are 4981 * already ~1% close to its last activation value. 4982 */ 4983 last_ewma_diff = ewma - dequeued; 4984 if (last_ewma_diff < UTIL_EST_MARGIN) 4985 goto done; 4986 4987 /* 4988 * To avoid underestimate of task utilization, skip updates of EWMA if 4989 * we cannot grant that thread got all CPU time it wanted. 4990 */ 4991 if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p)) 4992 goto done; 4993 4994 4995 /* 4996 * Update Task's estimated utilization 4997 * 4998 * When *p completes an activation we can consolidate another sample 4999 * of the task size. This is done by using this value to update the 5000 * Exponential Weighted Moving Average (EWMA): 5001 * 5002 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 5003 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 5004 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 5005 * = w * ( -last_ewma_diff ) + ewma(t-1) 5006 * = w * (-last_ewma_diff + ewma(t-1) / w) 5007 * 5008 * Where 'w' is the weight of new samples, which is configured to be 5009 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 5010 */ 5011 ewma <<= UTIL_EST_WEIGHT_SHIFT; 5012 ewma -= last_ewma_diff; 5013 ewma >>= UTIL_EST_WEIGHT_SHIFT; 5014 done: 5015 ewma |= UTIL_AVG_UNCHANGED; 5016 WRITE_ONCE(p->se.avg.util_est, ewma); 5017 5018 trace_sched_util_est_se_tp(&p->se); 5019 } 5020 5021 static inline unsigned long get_actual_cpu_capacity(int cpu) 5022 { 5023 unsigned long capacity = arch_scale_cpu_capacity(cpu); 5024 5025 capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu)); 5026 5027 return capacity; 5028 } 5029 5030 static inline int util_fits_cpu(unsigned long util, 5031 unsigned long uclamp_min, 5032 unsigned long uclamp_max, 5033 int cpu) 5034 { 5035 unsigned long capacity = capacity_of(cpu); 5036 unsigned long capacity_orig; 5037 bool fits, uclamp_max_fits; 5038 5039 /* 5040 * Check if the real util fits without any uclamp boost/cap applied. 5041 */ 5042 fits = fits_capacity(util, capacity); 5043 5044 if (!uclamp_is_used()) 5045 return fits; 5046 5047 /* 5048 * We must use arch_scale_cpu_capacity() for comparing against uclamp_min and 5049 * uclamp_max. We only care about capacity pressure (by using 5050 * capacity_of()) for comparing against the real util. 5051 * 5052 * If a task is boosted to 1024 for example, we don't want a tiny 5053 * pressure to skew the check whether it fits a CPU or not. 5054 * 5055 * Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it 5056 * should fit a little cpu even if there's some pressure. 5057 * 5058 * Only exception is for HW or cpufreq pressure since it has a direct impact 5059 * on available OPP of the system. 5060 * 5061 * We honour it for uclamp_min only as a drop in performance level 5062 * could result in not getting the requested minimum performance level. 5063 * 5064 * For uclamp_max, we can tolerate a drop in performance level as the 5065 * goal is to cap the task. So it's okay if it's getting less. 5066 */ 5067 capacity_orig = arch_scale_cpu_capacity(cpu); 5068 5069 /* 5070 * We want to force a task to fit a cpu as implied by uclamp_max. 5071 * But we do have some corner cases to cater for.. 5072 * 5073 * 5074 * C=z 5075 * | ___ 5076 * | C=y | | 5077 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5078 * | C=x | | | | 5079 * | ___ | | | | 5080 * | | | | | | | (util somewhere in this region) 5081 * | | | | | | | 5082 * | | | | | | | 5083 * +---------------------------------------- 5084 * CPU0 CPU1 CPU2 5085 * 5086 * In the above example if a task is capped to a specific performance 5087 * point, y, then when: 5088 * 5089 * * util = 80% of x then it does not fit on CPU0 and should migrate 5090 * to CPU1 5091 * * util = 80% of y then it is forced to fit on CPU1 to honour 5092 * uclamp_max request. 5093 * 5094 * which is what we're enforcing here. A task always fits if 5095 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 5096 * the normal upmigration rules should withhold still. 5097 * 5098 * Only exception is when we are on max capacity, then we need to be 5099 * careful not to block overutilized state. This is so because: 5100 * 5101 * 1. There's no concept of capping at max_capacity! We can't go 5102 * beyond this performance level anyway. 5103 * 2. The system is being saturated when we're operating near 5104 * max capacity, it doesn't make sense to block overutilized. 5105 */ 5106 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 5107 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 5108 fits = fits || uclamp_max_fits; 5109 5110 /* 5111 * 5112 * C=z 5113 * | ___ (region a, capped, util >= uclamp_max) 5114 * | C=y | | 5115 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5116 * | C=x | | | | 5117 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 5118 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 5119 * | | | | | | | 5120 * | | | | | | | (region c, boosted, util < uclamp_min) 5121 * +---------------------------------------- 5122 * CPU0 CPU1 CPU2 5123 * 5124 * a) If util > uclamp_max, then we're capped, we don't care about 5125 * actual fitness value here. We only care if uclamp_max fits 5126 * capacity without taking margin/pressure into account. 5127 * See comment above. 5128 * 5129 * b) If uclamp_min <= util <= uclamp_max, then the normal 5130 * fits_capacity() rules apply. Except we need to ensure that we 5131 * enforce we remain within uclamp_max, see comment above. 5132 * 5133 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 5134 * need to take into account the boosted value fits the CPU without 5135 * taking margin/pressure into account. 5136 * 5137 * Cases (a) and (b) are handled in the 'fits' variable already. We 5138 * just need to consider an extra check for case (c) after ensuring we 5139 * handle the case uclamp_min > uclamp_max. 5140 */ 5141 uclamp_min = min(uclamp_min, uclamp_max); 5142 if (fits && (util < uclamp_min) && 5143 (uclamp_min > get_actual_cpu_capacity(cpu))) 5144 return -1; 5145 5146 return fits; 5147 } 5148 5149 static inline int task_fits_cpu(struct task_struct *p, int cpu) 5150 { 5151 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 5152 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 5153 unsigned long util = task_util_est(p); 5154 /* 5155 * Return true only if the cpu fully fits the task requirements, which 5156 * include the utilization but also the performance hints. 5157 */ 5158 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 5159 } 5160 5161 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 5162 { 5163 int cpu = cpu_of(rq); 5164 5165 if (!sched_asym_cpucap_active()) 5166 return; 5167 5168 /* 5169 * Affinity allows us to go somewhere higher? Or are we on biggest 5170 * available CPU already? Or do we fit into this CPU ? 5171 */ 5172 if (!p || (p->nr_cpus_allowed == 1) || 5173 (arch_scale_cpu_capacity(cpu) == p->max_allowed_capacity) || 5174 task_fits_cpu(p, cpu)) { 5175 5176 rq->misfit_task_load = 0; 5177 return; 5178 } 5179 5180 /* 5181 * Make sure that misfit_task_load will not be null even if 5182 * task_h_load() returns 0. 5183 */ 5184 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 5185 } 5186 5187 void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) 5188 { 5189 struct sched_entity *se = &p->se; 5190 5191 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 5192 if (attr->sched_runtime) { 5193 se->custom_slice = 1; 5194 se->slice = clamp_t(u64, attr->sched_runtime, 5195 NSEC_PER_MSEC/10, /* HZ=1000 * 10 */ 5196 NSEC_PER_MSEC*100); /* HZ=100 / 10 */ 5197 } else { 5198 se->custom_slice = 0; 5199 se->slice = sysctl_sched_base_slice; 5200 } 5201 } 5202 5203 static void 5204 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5205 { 5206 u64 vslice, vruntime = avg_vruntime(cfs_rq); 5207 s64 lag = 0; 5208 5209 if (!se->custom_slice) 5210 se->slice = sysctl_sched_base_slice; 5211 vslice = calc_delta_fair(se->slice, se); 5212 5213 /* 5214 * Due to how V is constructed as the weighted average of entities, 5215 * adding tasks with positive lag, or removing tasks with negative lag 5216 * will move 'time' backwards, this can screw around with the lag of 5217 * other tasks. 5218 * 5219 * EEVDF: placement strategy #1 / #2 5220 */ 5221 if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { 5222 struct sched_entity *curr = cfs_rq->curr; 5223 unsigned long load; 5224 5225 lag = se->vlag; 5226 5227 /* 5228 * If we want to place a task and preserve lag, we have to 5229 * consider the effect of the new entity on the weighted 5230 * average and compensate for this, otherwise lag can quickly 5231 * evaporate. 5232 * 5233 * Lag is defined as: 5234 * 5235 * lag_i = S - s_i = w_i * (V - v_i) 5236 * 5237 * To avoid the 'w_i' term all over the place, we only track 5238 * the virtual lag: 5239 * 5240 * vl_i = V - v_i <=> v_i = V - vl_i 5241 * 5242 * And we take V to be the weighted average of all v: 5243 * 5244 * V = (\Sum w_j*v_j) / W 5245 * 5246 * Where W is: \Sum w_j 5247 * 5248 * Then, the weighted average after adding an entity with lag 5249 * vl_i is given by: 5250 * 5251 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 5252 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 5253 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 5254 * = (V*(W + w_i) - w_i*vl_i) / (W + w_i) 5255 * = V - w_i*vl_i / (W + w_i) 5256 * 5257 * And the actual lag after adding an entity with vl_i is: 5258 * 5259 * vl'_i = V' - v_i 5260 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 5261 * = vl_i - w_i*vl_i / (W + w_i) 5262 * 5263 * Which is strictly less than vl_i. So in order to preserve lag 5264 * we should inflate the lag before placement such that the 5265 * effective lag after placement comes out right. 5266 * 5267 * As such, invert the above relation for vl'_i to get the vl_i 5268 * we need to use such that the lag after placement is the lag 5269 * we computed before dequeue. 5270 * 5271 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 5272 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 5273 * 5274 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 5275 * = W*vl_i 5276 * 5277 * vl_i = (W + w_i)*vl'_i / W 5278 */ 5279 load = cfs_rq->sum_weight; 5280 if (curr && curr->on_rq) 5281 load += scale_load_down(curr->load.weight); 5282 5283 lag *= load + scale_load_down(se->load.weight); 5284 if (WARN_ON_ONCE(!load)) 5285 load = 1; 5286 lag = div_s64(lag, load); 5287 } 5288 5289 se->vruntime = vruntime - lag; 5290 5291 if (se->rel_deadline) { 5292 se->deadline += se->vruntime; 5293 se->rel_deadline = 0; 5294 return; 5295 } 5296 5297 /* 5298 * When joining the competition; the existing tasks will be, 5299 * on average, halfway through their slice, as such start tasks 5300 * off with half a slice to ease into the competition. 5301 */ 5302 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 5303 vslice /= 2; 5304 5305 /* 5306 * EEVDF: vd_i = ve_i + r_i/w_i 5307 */ 5308 se->deadline = se->vruntime + vslice; 5309 } 5310 5311 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 5312 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 5313 5314 static void 5315 requeue_delayed_entity(struct sched_entity *se); 5316 5317 static void 5318 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5319 { 5320 bool curr = cfs_rq->curr == se; 5321 5322 /* 5323 * If we're the current task, we must renormalise before calling 5324 * update_curr(). 5325 */ 5326 if (curr) 5327 place_entity(cfs_rq, se, flags); 5328 5329 update_curr(cfs_rq); 5330 5331 /* 5332 * When enqueuing a sched_entity, we must: 5333 * - Update loads to have both entity and cfs_rq synced with now. 5334 * - For group_entity, update its runnable_weight to reflect the new 5335 * h_nr_runnable of its group cfs_rq. 5336 * - For group_entity, update its weight to reflect the new share of 5337 * its group cfs_rq 5338 * - Add its new weight to cfs_rq->load.weight 5339 */ 5340 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 5341 se_update_runnable(se); 5342 /* 5343 * XXX update_load_avg() above will have attached us to the pelt sum; 5344 * but update_cfs_group() here will re-adjust the weight and have to 5345 * undo/redo all that. Seems wasteful. 5346 */ 5347 update_cfs_group(se); 5348 5349 /* 5350 * XXX now that the entity has been re-weighted, and it's lag adjusted, 5351 * we can place the entity. 5352 */ 5353 if (!curr) 5354 place_entity(cfs_rq, se, flags); 5355 5356 account_entity_enqueue(cfs_rq, se); 5357 5358 /* Entity has migrated, no longer consider this task hot */ 5359 if (flags & ENQUEUE_MIGRATED) 5360 se->exec_start = 0; 5361 5362 check_schedstat_required(); 5363 update_stats_enqueue_fair(cfs_rq, se, flags); 5364 if (!curr) 5365 __enqueue_entity(cfs_rq, se); 5366 se->on_rq = 1; 5367 5368 if (cfs_rq->nr_queued == 1) { 5369 check_enqueue_throttle(cfs_rq); 5370 list_add_leaf_cfs_rq(cfs_rq); 5371 #ifdef CONFIG_CFS_BANDWIDTH 5372 if (cfs_rq->pelt_clock_throttled) { 5373 struct rq *rq = rq_of(cfs_rq); 5374 5375 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5376 cfs_rq->throttled_clock_pelt; 5377 cfs_rq->pelt_clock_throttled = 0; 5378 } 5379 #endif 5380 } 5381 } 5382 5383 static void __clear_buddies_next(struct sched_entity *se) 5384 { 5385 for_each_sched_entity(se) { 5386 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5387 if (cfs_rq->next != se) 5388 break; 5389 5390 cfs_rq->next = NULL; 5391 } 5392 } 5393 5394 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 5395 { 5396 if (cfs_rq->next == se) 5397 __clear_buddies_next(se); 5398 } 5399 5400 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5401 5402 static void set_delayed(struct sched_entity *se) 5403 { 5404 se->sched_delayed = 1; 5405 5406 /* 5407 * Delayed se of cfs_rq have no tasks queued on them. 5408 * Do not adjust h_nr_runnable since dequeue_entities() 5409 * will account it for blocked tasks. 5410 */ 5411 if (!entity_is_task(se)) 5412 return; 5413 5414 for_each_sched_entity(se) { 5415 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5416 5417 cfs_rq->h_nr_runnable--; 5418 } 5419 } 5420 5421 static void clear_delayed(struct sched_entity *se) 5422 { 5423 se->sched_delayed = 0; 5424 5425 /* 5426 * Delayed se of cfs_rq have no tasks queued on them. 5427 * Do not adjust h_nr_runnable since a dequeue has 5428 * already accounted for it or an enqueue of a task 5429 * below it will account for it in enqueue_task_fair(). 5430 */ 5431 if (!entity_is_task(se)) 5432 return; 5433 5434 for_each_sched_entity(se) { 5435 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5436 5437 cfs_rq->h_nr_runnable++; 5438 } 5439 } 5440 5441 static inline void finish_delayed_dequeue_entity(struct sched_entity *se) 5442 { 5443 clear_delayed(se); 5444 if (sched_feat(DELAY_ZERO) && se->vlag > 0) 5445 se->vlag = 0; 5446 } 5447 5448 static bool 5449 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5450 { 5451 bool sleep = flags & DEQUEUE_SLEEP; 5452 int action = UPDATE_TG; 5453 5454 update_curr(cfs_rq); 5455 clear_buddies(cfs_rq, se); 5456 5457 if (flags & DEQUEUE_DELAYED) { 5458 WARN_ON_ONCE(!se->sched_delayed); 5459 } else { 5460 bool delay = sleep; 5461 /* 5462 * DELAY_DEQUEUE relies on spurious wakeups, special task 5463 * states must not suffer spurious wakeups, excempt them. 5464 */ 5465 if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) 5466 delay = false; 5467 5468 WARN_ON_ONCE(delay && se->sched_delayed); 5469 5470 if (sched_feat(DELAY_DEQUEUE) && delay && 5471 !entity_eligible(cfs_rq, se)) { 5472 update_load_avg(cfs_rq, se, 0); 5473 set_delayed(se); 5474 return false; 5475 } 5476 } 5477 5478 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 5479 action |= DO_DETACH; 5480 5481 /* 5482 * When dequeuing a sched_entity, we must: 5483 * - Update loads to have both entity and cfs_rq synced with now. 5484 * - For group_entity, update its runnable_weight to reflect the new 5485 * h_nr_runnable of its group cfs_rq. 5486 * - Subtract its previous weight from cfs_rq->load.weight. 5487 * - For group entity, update its weight to reflect the new share 5488 * of its group cfs_rq. 5489 */ 5490 update_load_avg(cfs_rq, se, action); 5491 se_update_runnable(se); 5492 5493 update_stats_dequeue_fair(cfs_rq, se, flags); 5494 5495 update_entity_lag(cfs_rq, se); 5496 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { 5497 se->deadline -= se->vruntime; 5498 se->rel_deadline = 1; 5499 } 5500 5501 if (se != cfs_rq->curr) 5502 __dequeue_entity(cfs_rq, se); 5503 se->on_rq = 0; 5504 account_entity_dequeue(cfs_rq, se); 5505 5506 /* return excess runtime on last dequeue */ 5507 return_cfs_rq_runtime(cfs_rq); 5508 5509 update_cfs_group(se); 5510 5511 if (flags & DEQUEUE_DELAYED) 5512 finish_delayed_dequeue_entity(se); 5513 5514 if (cfs_rq->nr_queued == 0) { 5515 update_idle_cfs_rq_clock_pelt(cfs_rq); 5516 #ifdef CONFIG_CFS_BANDWIDTH 5517 if (throttled_hierarchy(cfs_rq)) { 5518 struct rq *rq = rq_of(cfs_rq); 5519 5520 list_del_leaf_cfs_rq(cfs_rq); 5521 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5522 cfs_rq->pelt_clock_throttled = 1; 5523 } 5524 #endif 5525 } 5526 5527 return true; 5528 } 5529 5530 static void 5531 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) 5532 { 5533 clear_buddies(cfs_rq, se); 5534 5535 /* 'current' is not kept within the tree. */ 5536 if (se->on_rq) { 5537 /* 5538 * Any task has to be enqueued before it get to execute on 5539 * a CPU. So account for the time it spent waiting on the 5540 * runqueue. 5541 */ 5542 update_stats_wait_end_fair(cfs_rq, se); 5543 __dequeue_entity(cfs_rq, se); 5544 update_load_avg(cfs_rq, se, UPDATE_TG); 5545 5546 if (first) 5547 set_protect_slice(cfs_rq, se); 5548 } 5549 5550 update_stats_curr_start(cfs_rq, se); 5551 WARN_ON_ONCE(cfs_rq->curr); 5552 cfs_rq->curr = se; 5553 5554 /* 5555 * Track our maximum slice length, if the CPU's load is at 5556 * least twice that of our own weight (i.e. don't track it 5557 * when there are only lesser-weight tasks around): 5558 */ 5559 if (schedstat_enabled() && 5560 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5561 struct sched_statistics *stats; 5562 5563 stats = __schedstats_from_se(se); 5564 __schedstat_set(stats->slice_max, 5565 max((u64)stats->slice_max, 5566 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5567 } 5568 5569 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5570 } 5571 5572 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); 5573 5574 /* 5575 * Pick the next process, keeping these things in mind, in this order: 5576 * 1) keep things fair between processes/task groups 5577 * 2) pick the "next" process, since someone really wants that to run 5578 * 3) pick the "last" process, for cache locality 5579 * 4) do not run the "skip" process, if something else is available 5580 */ 5581 static struct sched_entity * 5582 pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) 5583 { 5584 struct sched_entity *se; 5585 5586 se = pick_eevdf(cfs_rq, protect); 5587 if (se->sched_delayed) { 5588 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 5589 /* 5590 * Must not reference @se again, see __block_task(). 5591 */ 5592 return NULL; 5593 } 5594 return se; 5595 } 5596 5597 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5598 5599 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5600 { 5601 /* 5602 * If still on the runqueue then deactivate_task() 5603 * was not called and update_curr() has to be done: 5604 */ 5605 if (prev->on_rq) 5606 update_curr(cfs_rq); 5607 5608 /* throttle cfs_rqs exceeding runtime */ 5609 check_cfs_rq_runtime(cfs_rq); 5610 5611 if (prev->on_rq) { 5612 update_stats_wait_start_fair(cfs_rq, prev); 5613 /* Put 'current' back into the tree. */ 5614 __enqueue_entity(cfs_rq, prev); 5615 /* in !on_rq case, update occurred at dequeue */ 5616 update_load_avg(cfs_rq, prev, 0); 5617 } 5618 WARN_ON_ONCE(cfs_rq->curr != prev); 5619 cfs_rq->curr = NULL; 5620 } 5621 5622 static void 5623 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5624 { 5625 /* 5626 * Update run-time statistics of the 'current'. 5627 */ 5628 update_curr(cfs_rq); 5629 5630 /* 5631 * Ensure that runnable average is periodically updated. 5632 */ 5633 update_load_avg(cfs_rq, curr, UPDATE_TG); 5634 update_cfs_group(curr); 5635 5636 #ifdef CONFIG_SCHED_HRTICK 5637 /* 5638 * queued ticks are scheduled to match the slice, so don't bother 5639 * validating it and just reschedule. 5640 */ 5641 if (queued) { 5642 resched_curr_lazy(rq_of(cfs_rq)); 5643 return; 5644 } 5645 #endif 5646 } 5647 5648 5649 /************************************************** 5650 * CFS bandwidth control machinery 5651 */ 5652 5653 #ifdef CONFIG_CFS_BANDWIDTH 5654 5655 #ifdef CONFIG_JUMP_LABEL 5656 static struct static_key __cfs_bandwidth_used; 5657 5658 static inline bool cfs_bandwidth_used(void) 5659 { 5660 return static_key_false(&__cfs_bandwidth_used); 5661 } 5662 5663 void cfs_bandwidth_usage_inc(void) 5664 { 5665 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5666 } 5667 5668 void cfs_bandwidth_usage_dec(void) 5669 { 5670 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5671 } 5672 #else /* !CONFIG_JUMP_LABEL: */ 5673 static bool cfs_bandwidth_used(void) 5674 { 5675 return true; 5676 } 5677 5678 void cfs_bandwidth_usage_inc(void) {} 5679 void cfs_bandwidth_usage_dec(void) {} 5680 #endif /* !CONFIG_JUMP_LABEL */ 5681 5682 static inline u64 sched_cfs_bandwidth_slice(void) 5683 { 5684 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5685 } 5686 5687 /* 5688 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5689 * directly instead of rq->clock to avoid adding additional synchronization 5690 * around rq->lock. 5691 * 5692 * requires cfs_b->lock 5693 */ 5694 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5695 { 5696 s64 runtime; 5697 5698 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5699 return; 5700 5701 cfs_b->runtime += cfs_b->quota; 5702 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5703 if (runtime > 0) { 5704 cfs_b->burst_time += runtime; 5705 cfs_b->nr_burst++; 5706 } 5707 5708 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5709 cfs_b->runtime_snap = cfs_b->runtime; 5710 } 5711 5712 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5713 { 5714 return &tg->cfs_bandwidth; 5715 } 5716 5717 /* returns 0 on failure to allocate runtime */ 5718 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5719 struct cfs_rq *cfs_rq, u64 target_runtime) 5720 { 5721 u64 min_amount, amount = 0; 5722 5723 lockdep_assert_held(&cfs_b->lock); 5724 5725 /* note: this is a positive sum as runtime_remaining <= 0 */ 5726 min_amount = target_runtime - cfs_rq->runtime_remaining; 5727 5728 if (cfs_b->quota == RUNTIME_INF) 5729 amount = min_amount; 5730 else { 5731 start_cfs_bandwidth(cfs_b); 5732 5733 if (cfs_b->runtime > 0) { 5734 amount = min(cfs_b->runtime, min_amount); 5735 cfs_b->runtime -= amount; 5736 cfs_b->idle = 0; 5737 } 5738 } 5739 5740 cfs_rq->runtime_remaining += amount; 5741 5742 return cfs_rq->runtime_remaining > 0; 5743 } 5744 5745 /* returns 0 on failure to allocate runtime */ 5746 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5747 { 5748 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5749 int ret; 5750 5751 raw_spin_lock(&cfs_b->lock); 5752 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5753 raw_spin_unlock(&cfs_b->lock); 5754 5755 return ret; 5756 } 5757 5758 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5759 { 5760 /* dock delta_exec before expiring quota (as it could span periods) */ 5761 cfs_rq->runtime_remaining -= delta_exec; 5762 5763 if (likely(cfs_rq->runtime_remaining > 0)) 5764 return; 5765 5766 if (cfs_rq->throttled) 5767 return; 5768 /* 5769 * if we're unable to extend our runtime we resched so that the active 5770 * hierarchy can be throttled 5771 */ 5772 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5773 resched_curr(rq_of(cfs_rq)); 5774 } 5775 5776 static __always_inline 5777 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5778 { 5779 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5780 return; 5781 5782 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5783 } 5784 5785 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5786 { 5787 return cfs_bandwidth_used() && cfs_rq->throttled; 5788 } 5789 5790 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 5791 { 5792 return cfs_bandwidth_used() && cfs_rq->pelt_clock_throttled; 5793 } 5794 5795 /* check whether cfs_rq, or any parent, is throttled */ 5796 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5797 { 5798 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5799 } 5800 5801 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 5802 { 5803 return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]); 5804 } 5805 5806 static inline bool task_is_throttled(struct task_struct *p) 5807 { 5808 return cfs_bandwidth_used() && p->throttled; 5809 } 5810 5811 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5812 static void throttle_cfs_rq_work(struct callback_head *work) 5813 { 5814 struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work); 5815 struct sched_entity *se; 5816 struct cfs_rq *cfs_rq; 5817 struct rq *rq; 5818 5819 WARN_ON_ONCE(p != current); 5820 p->sched_throttle_work.next = &p->sched_throttle_work; 5821 5822 /* 5823 * If task is exiting, then there won't be a return to userspace, so we 5824 * don't have to bother with any of this. 5825 */ 5826 if ((p->flags & PF_EXITING)) 5827 return; 5828 5829 scoped_guard(task_rq_lock, p) { 5830 se = &p->se; 5831 cfs_rq = cfs_rq_of(se); 5832 5833 /* Raced, forget */ 5834 if (p->sched_class != &fair_sched_class) 5835 return; 5836 5837 /* 5838 * If not in limbo, then either replenish has happened or this 5839 * task got migrated out of the throttled cfs_rq, move along. 5840 */ 5841 if (!cfs_rq->throttle_count) 5842 return; 5843 rq = scope.rq; 5844 update_rq_clock(rq); 5845 WARN_ON_ONCE(p->throttled || !list_empty(&p->throttle_node)); 5846 dequeue_task_fair(rq, p, DEQUEUE_SLEEP | DEQUEUE_THROTTLE); 5847 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5848 /* 5849 * Must not set throttled before dequeue or dequeue will 5850 * mistakenly regard this task as an already throttled one. 5851 */ 5852 p->throttled = true; 5853 resched_curr(rq); 5854 } 5855 } 5856 5857 void init_cfs_throttle_work(struct task_struct *p) 5858 { 5859 init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work); 5860 /* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */ 5861 p->sched_throttle_work.next = &p->sched_throttle_work; 5862 INIT_LIST_HEAD(&p->throttle_node); 5863 } 5864 5865 /* 5866 * Task is throttled and someone wants to dequeue it again: 5867 * it could be sched/core when core needs to do things like 5868 * task affinity change, task group change, task sched class 5869 * change etc. and in these cases, DEQUEUE_SLEEP is not set; 5870 * or the task is blocked after throttled due to freezer etc. 5871 * and in these cases, DEQUEUE_SLEEP is set. 5872 */ 5873 static void detach_task_cfs_rq(struct task_struct *p); 5874 static void dequeue_throttled_task(struct task_struct *p, int flags) 5875 { 5876 WARN_ON_ONCE(p->se.on_rq); 5877 list_del_init(&p->throttle_node); 5878 5879 /* task blocked after throttled */ 5880 if (flags & DEQUEUE_SLEEP) { 5881 p->throttled = false; 5882 return; 5883 } 5884 5885 /* 5886 * task is migrating off its old cfs_rq, detach 5887 * the task's load from its old cfs_rq. 5888 */ 5889 if (task_on_rq_migrating(p)) 5890 detach_task_cfs_rq(p); 5891 } 5892 5893 static bool enqueue_throttled_task(struct task_struct *p) 5894 { 5895 struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); 5896 5897 /* @p should have gone through dequeue_throttled_task() first */ 5898 WARN_ON_ONCE(!list_empty(&p->throttle_node)); 5899 5900 /* 5901 * If the throttled task @p is enqueued to a throttled cfs_rq, 5902 * take the fast path by directly putting the task on the 5903 * target cfs_rq's limbo list. 5904 * 5905 * Do not do that when @p is current because the following race can 5906 * cause @p's group_node to be incorectly re-insterted in its rq's 5907 * cfs_tasks list, despite being throttled: 5908 * 5909 * cpuX cpuY 5910 * p ret2user 5911 * throttle_cfs_rq_work() sched_move_task(p) 5912 * LOCK task_rq_lock 5913 * dequeue_task_fair(p) 5914 * UNLOCK task_rq_lock 5915 * LOCK task_rq_lock 5916 * task_current_donor(p) == true 5917 * task_on_rq_queued(p) == true 5918 * dequeue_task(p) 5919 * put_prev_task(p) 5920 * sched_change_group() 5921 * enqueue_task(p) -> p's new cfs_rq 5922 * is throttled, go 5923 * fast path and skip 5924 * actual enqueue 5925 * set_next_task(p) 5926 * list_move(&se->group_node, &rq->cfs_tasks); // bug 5927 * schedule() 5928 * 5929 * In the above race case, @p current cfs_rq is in the same rq as 5930 * its previous cfs_rq because sched_move_task() only moves a task 5931 * to a different group from the same rq, so we can use its current 5932 * cfs_rq to derive rq and test if the task is current. 5933 */ 5934 if (throttled_hierarchy(cfs_rq) && 5935 !task_current_donor(rq_of(cfs_rq), p)) { 5936 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5937 return true; 5938 } 5939 5940 /* we can't take the fast path, do an actual enqueue*/ 5941 p->throttled = false; 5942 return false; 5943 } 5944 5945 static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5946 static int tg_unthrottle_up(struct task_group *tg, void *data) 5947 { 5948 struct rq *rq = data; 5949 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5950 struct task_struct *p, *tmp; 5951 5952 if (--cfs_rq->throttle_count) 5953 return 0; 5954 5955 if (cfs_rq->pelt_clock_throttled) { 5956 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5957 cfs_rq->throttled_clock_pelt; 5958 cfs_rq->pelt_clock_throttled = 0; 5959 } 5960 5961 if (cfs_rq->throttled_clock_self) { 5962 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 5963 5964 cfs_rq->throttled_clock_self = 0; 5965 5966 if (WARN_ON_ONCE((s64)delta < 0)) 5967 delta = 0; 5968 5969 cfs_rq->throttled_clock_self_time += delta; 5970 } 5971 5972 /* Re-enqueue the tasks that have been throttled at this level. */ 5973 list_for_each_entry_safe(p, tmp, &cfs_rq->throttled_limbo_list, throttle_node) { 5974 list_del_init(&p->throttle_node); 5975 p->throttled = false; 5976 enqueue_task_fair(rq_of(cfs_rq), p, ENQUEUE_WAKEUP); 5977 } 5978 5979 /* Add cfs_rq with load or one or more already running entities to the list */ 5980 if (!cfs_rq_is_decayed(cfs_rq)) 5981 list_add_leaf_cfs_rq(cfs_rq); 5982 5983 return 0; 5984 } 5985 5986 static inline bool task_has_throttle_work(struct task_struct *p) 5987 { 5988 return p->sched_throttle_work.next != &p->sched_throttle_work; 5989 } 5990 5991 static inline void task_throttle_setup_work(struct task_struct *p) 5992 { 5993 if (task_has_throttle_work(p)) 5994 return; 5995 5996 /* 5997 * Kthreads and exiting tasks don't return to userspace, so adding the 5998 * work is pointless 5999 */ 6000 if ((p->flags & (PF_EXITING | PF_KTHREAD))) 6001 return; 6002 6003 task_work_add(p, &p->sched_throttle_work, TWA_RESUME); 6004 } 6005 6006 static void record_throttle_clock(struct cfs_rq *cfs_rq) 6007 { 6008 struct rq *rq = rq_of(cfs_rq); 6009 6010 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 6011 cfs_rq->throttled_clock = rq_clock(rq); 6012 6013 if (!cfs_rq->throttled_clock_self) 6014 cfs_rq->throttled_clock_self = rq_clock(rq); 6015 } 6016 6017 static int tg_throttle_down(struct task_group *tg, void *data) 6018 { 6019 struct rq *rq = data; 6020 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6021 6022 if (cfs_rq->throttle_count++) 6023 return 0; 6024 6025 /* 6026 * For cfs_rqs that still have entities enqueued, PELT clock 6027 * stop happens at dequeue time when all entities are dequeued. 6028 */ 6029 if (!cfs_rq->nr_queued) { 6030 list_del_leaf_cfs_rq(cfs_rq); 6031 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 6032 cfs_rq->pelt_clock_throttled = 1; 6033 } 6034 6035 WARN_ON_ONCE(cfs_rq->throttled_clock_self); 6036 WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_limbo_list)); 6037 return 0; 6038 } 6039 6040 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 6041 { 6042 struct rq *rq = rq_of(cfs_rq); 6043 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6044 int dequeue = 1; 6045 6046 raw_spin_lock(&cfs_b->lock); 6047 /* This will start the period timer if necessary */ 6048 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 6049 /* 6050 * We have raced with bandwidth becoming available, and if we 6051 * actually throttled the timer might not unthrottle us for an 6052 * entire period. We additionally needed to make sure that any 6053 * subsequent check_cfs_rq_runtime calls agree not to throttle 6054 * us, as we may commit to do cfs put_prev+pick_next, so we ask 6055 * for 1ns of runtime rather than just check cfs_b. 6056 */ 6057 dequeue = 0; 6058 } else { 6059 list_add_tail_rcu(&cfs_rq->throttled_list, 6060 &cfs_b->throttled_cfs_rq); 6061 } 6062 raw_spin_unlock(&cfs_b->lock); 6063 6064 if (!dequeue) 6065 return false; /* Throttle no longer required. */ 6066 6067 /* freeze hierarchy runnable averages while throttled */ 6068 rcu_read_lock(); 6069 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 6070 rcu_read_unlock(); 6071 6072 /* 6073 * Note: distribution will already see us throttled via the 6074 * throttled-list. rq->lock protects completion. 6075 */ 6076 cfs_rq->throttled = 1; 6077 WARN_ON_ONCE(cfs_rq->throttled_clock); 6078 return true; 6079 } 6080 6081 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 6082 { 6083 struct rq *rq = rq_of(cfs_rq); 6084 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6085 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 6086 6087 /* 6088 * It's possible we are called with runtime_remaining < 0 due to things 6089 * like async unthrottled us with a positive runtime_remaining but other 6090 * still running entities consumed those runtime before we reached here. 6091 * 6092 * We can't unthrottle this cfs_rq without any runtime remaining because 6093 * any enqueue in tg_unthrottle_up() will immediately trigger a throttle, 6094 * which is not supposed to happen on unthrottle path. 6095 */ 6096 if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0) 6097 return; 6098 6099 cfs_rq->throttled = 0; 6100 6101 update_rq_clock(rq); 6102 6103 raw_spin_lock(&cfs_b->lock); 6104 if (cfs_rq->throttled_clock) { 6105 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 6106 cfs_rq->throttled_clock = 0; 6107 } 6108 list_del_rcu(&cfs_rq->throttled_list); 6109 raw_spin_unlock(&cfs_b->lock); 6110 6111 /* update hierarchical throttle state */ 6112 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 6113 6114 if (!cfs_rq->load.weight) { 6115 if (!cfs_rq->on_list) 6116 return; 6117 /* 6118 * Nothing to run but something to decay (on_list)? 6119 * Complete the branch. 6120 */ 6121 for_each_sched_entity(se) { 6122 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 6123 break; 6124 } 6125 } 6126 6127 assert_list_leaf_cfs_rq(rq); 6128 6129 /* Determine whether we need to wake up potentially idle CPU: */ 6130 if (rq->curr == rq->idle && rq->cfs.nr_queued) 6131 resched_curr(rq); 6132 } 6133 6134 static void __cfsb_csd_unthrottle(void *arg) 6135 { 6136 struct cfs_rq *cursor, *tmp; 6137 struct rq *rq = arg; 6138 struct rq_flags rf; 6139 6140 rq_lock(rq, &rf); 6141 6142 /* 6143 * Iterating over the list can trigger several call to 6144 * update_rq_clock() in unthrottle_cfs_rq(). 6145 * Do it once and skip the potential next ones. 6146 */ 6147 update_rq_clock(rq); 6148 rq_clock_start_loop_update(rq); 6149 6150 /* 6151 * Since we hold rq lock we're safe from concurrent manipulation of 6152 * the CSD list. However, this RCU critical section annotates the 6153 * fact that we pair with sched_free_group_rcu(), so that we cannot 6154 * race with group being freed in the window between removing it 6155 * from the list and advancing to the next entry in the list. 6156 */ 6157 rcu_read_lock(); 6158 6159 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 6160 throttled_csd_list) { 6161 list_del_init(&cursor->throttled_csd_list); 6162 6163 if (cfs_rq_throttled(cursor)) 6164 unthrottle_cfs_rq(cursor); 6165 } 6166 6167 rcu_read_unlock(); 6168 6169 rq_clock_stop_loop_update(rq); 6170 rq_unlock(rq, &rf); 6171 } 6172 6173 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6174 { 6175 struct rq *rq = rq_of(cfs_rq); 6176 bool first; 6177 6178 if (rq == this_rq()) { 6179 unthrottle_cfs_rq(cfs_rq); 6180 return; 6181 } 6182 6183 /* Already enqueued */ 6184 if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list))) 6185 return; 6186 6187 first = list_empty(&rq->cfsb_csd_list); 6188 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 6189 if (first) 6190 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 6191 } 6192 6193 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6194 { 6195 lockdep_assert_rq_held(rq_of(cfs_rq)); 6196 6197 if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) || 6198 cfs_rq->runtime_remaining <= 0)) 6199 return; 6200 6201 __unthrottle_cfs_rq_async(cfs_rq); 6202 } 6203 6204 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 6205 { 6206 int this_cpu = smp_processor_id(); 6207 u64 runtime, remaining = 1; 6208 bool throttled = false; 6209 struct cfs_rq *cfs_rq, *tmp; 6210 struct rq_flags rf; 6211 struct rq *rq; 6212 LIST_HEAD(local_unthrottle); 6213 6214 rcu_read_lock(); 6215 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 6216 throttled_list) { 6217 rq = rq_of(cfs_rq); 6218 6219 if (!remaining) { 6220 throttled = true; 6221 break; 6222 } 6223 6224 rq_lock_irqsave(rq, &rf); 6225 if (!cfs_rq_throttled(cfs_rq)) 6226 goto next; 6227 6228 /* Already queued for async unthrottle */ 6229 if (!list_empty(&cfs_rq->throttled_csd_list)) 6230 goto next; 6231 6232 /* By the above checks, this should never be true */ 6233 WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); 6234 6235 raw_spin_lock(&cfs_b->lock); 6236 runtime = -cfs_rq->runtime_remaining + 1; 6237 if (runtime > cfs_b->runtime) 6238 runtime = cfs_b->runtime; 6239 cfs_b->runtime -= runtime; 6240 remaining = cfs_b->runtime; 6241 raw_spin_unlock(&cfs_b->lock); 6242 6243 cfs_rq->runtime_remaining += runtime; 6244 6245 /* we check whether we're throttled above */ 6246 if (cfs_rq->runtime_remaining > 0) { 6247 if (cpu_of(rq) != this_cpu) { 6248 unthrottle_cfs_rq_async(cfs_rq); 6249 } else { 6250 /* 6251 * We currently only expect to be unthrottling 6252 * a single cfs_rq locally. 6253 */ 6254 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6255 list_add_tail(&cfs_rq->throttled_csd_list, 6256 &local_unthrottle); 6257 } 6258 } else { 6259 throttled = true; 6260 } 6261 6262 next: 6263 rq_unlock_irqrestore(rq, &rf); 6264 } 6265 6266 list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, 6267 throttled_csd_list) { 6268 struct rq *rq = rq_of(cfs_rq); 6269 6270 rq_lock_irqsave(rq, &rf); 6271 6272 list_del_init(&cfs_rq->throttled_csd_list); 6273 6274 if (cfs_rq_throttled(cfs_rq)) 6275 unthrottle_cfs_rq(cfs_rq); 6276 6277 rq_unlock_irqrestore(rq, &rf); 6278 } 6279 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6280 6281 rcu_read_unlock(); 6282 6283 return throttled; 6284 } 6285 6286 /* 6287 * Responsible for refilling a task_group's bandwidth and unthrottling its 6288 * cfs_rqs as appropriate. If there has been no activity within the last 6289 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 6290 * used to track this state. 6291 */ 6292 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 6293 { 6294 int throttled; 6295 6296 /* no need to continue the timer with no bandwidth constraint */ 6297 if (cfs_b->quota == RUNTIME_INF) 6298 goto out_deactivate; 6299 6300 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 6301 cfs_b->nr_periods += overrun; 6302 6303 /* Refill extra burst quota even if cfs_b->idle */ 6304 __refill_cfs_bandwidth_runtime(cfs_b); 6305 6306 /* 6307 * idle depends on !throttled (for the case of a large deficit), and if 6308 * we're going inactive then everything else can be deferred 6309 */ 6310 if (cfs_b->idle && !throttled) 6311 goto out_deactivate; 6312 6313 if (!throttled) { 6314 /* mark as potentially idle for the upcoming period */ 6315 cfs_b->idle = 1; 6316 return 0; 6317 } 6318 6319 /* account preceding periods in which throttling occurred */ 6320 cfs_b->nr_throttled += overrun; 6321 6322 /* 6323 * This check is repeated as we release cfs_b->lock while we unthrottle. 6324 */ 6325 while (throttled && cfs_b->runtime > 0) { 6326 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6327 /* we can't nest cfs_b->lock while distributing bandwidth */ 6328 throttled = distribute_cfs_runtime(cfs_b); 6329 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6330 } 6331 6332 /* 6333 * While we are ensured activity in the period following an 6334 * unthrottle, this also covers the case in which the new bandwidth is 6335 * insufficient to cover the existing bandwidth deficit. (Forcing the 6336 * timer to remain active while there are any throttled entities.) 6337 */ 6338 cfs_b->idle = 0; 6339 6340 return 0; 6341 6342 out_deactivate: 6343 return 1; 6344 } 6345 6346 /* a cfs_rq won't donate quota below this amount */ 6347 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 6348 /* minimum remaining period time to redistribute slack quota */ 6349 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 6350 /* how long we wait to gather additional slack before distributing */ 6351 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 6352 6353 /* 6354 * Are we near the end of the current quota period? 6355 * 6356 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 6357 * hrtimer base being cleared by hrtimer_start. In the case of 6358 * migrate_hrtimers, base is never cleared, so we are fine. 6359 */ 6360 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 6361 { 6362 struct hrtimer *refresh_timer = &cfs_b->period_timer; 6363 s64 remaining; 6364 6365 /* if the call-back is running a quota refresh is already occurring */ 6366 if (hrtimer_callback_running(refresh_timer)) 6367 return 1; 6368 6369 /* is a quota refresh about to occur? */ 6370 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 6371 if (remaining < (s64)min_expire) 6372 return 1; 6373 6374 return 0; 6375 } 6376 6377 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 6378 { 6379 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 6380 6381 /* if there's a quota refresh soon don't bother with slack */ 6382 if (runtime_refresh_within(cfs_b, min_left)) 6383 return; 6384 6385 /* don't push forwards an existing deferred unthrottle */ 6386 if (cfs_b->slack_started) 6387 return; 6388 cfs_b->slack_started = true; 6389 6390 hrtimer_start(&cfs_b->slack_timer, 6391 ns_to_ktime(cfs_bandwidth_slack_period), 6392 HRTIMER_MODE_REL); 6393 } 6394 6395 /* we know any runtime found here is valid as update_curr() precedes return */ 6396 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6397 { 6398 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6399 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 6400 6401 if (slack_runtime <= 0) 6402 return; 6403 6404 raw_spin_lock(&cfs_b->lock); 6405 if (cfs_b->quota != RUNTIME_INF) { 6406 cfs_b->runtime += slack_runtime; 6407 6408 /* we are under rq->lock, defer unthrottling using a timer */ 6409 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 6410 !list_empty(&cfs_b->throttled_cfs_rq)) 6411 start_cfs_slack_bandwidth(cfs_b); 6412 } 6413 raw_spin_unlock(&cfs_b->lock); 6414 6415 /* even if it's not valid for return we don't want to try again */ 6416 cfs_rq->runtime_remaining -= slack_runtime; 6417 } 6418 6419 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6420 { 6421 if (!cfs_bandwidth_used()) 6422 return; 6423 6424 if (!cfs_rq->runtime_enabled || cfs_rq->nr_queued) 6425 return; 6426 6427 __return_cfs_rq_runtime(cfs_rq); 6428 } 6429 6430 /* 6431 * This is done with a timer (instead of inline with bandwidth return) since 6432 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 6433 */ 6434 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 6435 { 6436 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 6437 unsigned long flags; 6438 6439 /* confirm we're still not at a refresh boundary */ 6440 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6441 cfs_b->slack_started = false; 6442 6443 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 6444 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6445 return; 6446 } 6447 6448 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 6449 runtime = cfs_b->runtime; 6450 6451 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6452 6453 if (!runtime) 6454 return; 6455 6456 distribute_cfs_runtime(cfs_b); 6457 } 6458 6459 /* 6460 * When a group wakes up we want to make sure that its quota is not already 6461 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 6462 * runtime as update_curr() throttling can not trigger until it's on-rq. 6463 */ 6464 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 6465 { 6466 if (!cfs_bandwidth_used()) 6467 return; 6468 6469 /* an active group must be handled by the update_curr()->put() path */ 6470 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 6471 return; 6472 6473 /* ensure the group is not already throttled */ 6474 if (cfs_rq_throttled(cfs_rq)) 6475 return; 6476 6477 /* update runtime allocation */ 6478 account_cfs_rq_runtime(cfs_rq, 0); 6479 if (cfs_rq->runtime_remaining <= 0) 6480 throttle_cfs_rq(cfs_rq); 6481 } 6482 6483 static void sync_throttle(struct task_group *tg, int cpu) 6484 { 6485 struct cfs_rq *pcfs_rq, *cfs_rq; 6486 6487 if (!cfs_bandwidth_used()) 6488 return; 6489 6490 if (!tg->parent) 6491 return; 6492 6493 cfs_rq = tg->cfs_rq[cpu]; 6494 pcfs_rq = tg->parent->cfs_rq[cpu]; 6495 6496 cfs_rq->throttle_count = pcfs_rq->throttle_count; 6497 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 6498 6499 /* 6500 * It is not enough to sync the "pelt_clock_throttled" indicator 6501 * with the parent cfs_rq when the hierarchy is not queued. 6502 * Always join a throttled hierarchy with PELT clock throttled 6503 * and leaf it to the first enqueue, or distribution to 6504 * unthrottle the PELT clock. 6505 */ 6506 if (cfs_rq->throttle_count) 6507 cfs_rq->pelt_clock_throttled = 1; 6508 } 6509 6510 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 6511 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6512 { 6513 if (!cfs_bandwidth_used()) 6514 return false; 6515 6516 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 6517 return false; 6518 6519 /* 6520 * it's possible for a throttled entity to be forced into a running 6521 * state (e.g. set_curr_task), in this case we're finished. 6522 */ 6523 if (cfs_rq_throttled(cfs_rq)) 6524 return true; 6525 6526 return throttle_cfs_rq(cfs_rq); 6527 } 6528 6529 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 6530 { 6531 struct cfs_bandwidth *cfs_b = 6532 container_of(timer, struct cfs_bandwidth, slack_timer); 6533 6534 do_sched_cfs_slack_timer(cfs_b); 6535 6536 return HRTIMER_NORESTART; 6537 } 6538 6539 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 6540 { 6541 struct cfs_bandwidth *cfs_b = 6542 container_of(timer, struct cfs_bandwidth, period_timer); 6543 unsigned long flags; 6544 int overrun; 6545 int idle = 0; 6546 int count = 0; 6547 6548 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6549 for (;;) { 6550 overrun = hrtimer_forward_now(timer, cfs_b->period); 6551 if (!overrun) 6552 break; 6553 6554 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 6555 6556 if (++count > 3) { 6557 u64 new, old = ktime_to_ns(cfs_b->period); 6558 6559 /* 6560 * Grow period by a factor of 2 to avoid losing precision. 6561 * Precision loss in the quota/period ratio can cause __cfs_schedulable 6562 * to fail. 6563 */ 6564 new = old * 2; 6565 if (new < max_bw_quota_period_us * NSEC_PER_USEC) { 6566 cfs_b->period = ns_to_ktime(new); 6567 cfs_b->quota *= 2; 6568 cfs_b->burst *= 2; 6569 6570 pr_warn_ratelimited( 6571 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6572 smp_processor_id(), 6573 div_u64(new, NSEC_PER_USEC), 6574 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6575 } else { 6576 pr_warn_ratelimited( 6577 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6578 smp_processor_id(), 6579 div_u64(old, NSEC_PER_USEC), 6580 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6581 } 6582 6583 /* reset count so we don't come right back in here */ 6584 count = 0; 6585 } 6586 } 6587 if (idle) 6588 cfs_b->period_active = 0; 6589 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6590 6591 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6592 } 6593 6594 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 6595 { 6596 raw_spin_lock_init(&cfs_b->lock); 6597 cfs_b->runtime = 0; 6598 cfs_b->quota = RUNTIME_INF; 6599 cfs_b->period = us_to_ktime(default_bw_period_us()); 6600 cfs_b->burst = 0; 6601 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 6602 6603 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6604 hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC, 6605 HRTIMER_MODE_ABS_PINNED); 6606 6607 /* Add a random offset so that timers interleave */ 6608 hrtimer_set_expires(&cfs_b->period_timer, 6609 get_random_u32_below(cfs_b->period)); 6610 hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC, 6611 HRTIMER_MODE_REL); 6612 cfs_b->slack_started = false; 6613 } 6614 6615 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6616 { 6617 cfs_rq->runtime_enabled = 0; 6618 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6619 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6620 INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list); 6621 } 6622 6623 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6624 { 6625 lockdep_assert_held(&cfs_b->lock); 6626 6627 if (cfs_b->period_active) 6628 return; 6629 6630 cfs_b->period_active = 1; 6631 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6632 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6633 } 6634 6635 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6636 { 6637 int __maybe_unused i; 6638 6639 /* init_cfs_bandwidth() was not called */ 6640 if (!cfs_b->throttled_cfs_rq.next) 6641 return; 6642 6643 hrtimer_cancel(&cfs_b->period_timer); 6644 hrtimer_cancel(&cfs_b->slack_timer); 6645 6646 /* 6647 * It is possible that we still have some cfs_rq's pending on a CSD 6648 * list, though this race is very rare. In order for this to occur, we 6649 * must have raced with the last task leaving the group while there 6650 * exist throttled cfs_rq(s), and the period_timer must have queued the 6651 * CSD item but the remote cpu has not yet processed it. To handle this, 6652 * we can simply flush all pending CSD work inline here. We're 6653 * guaranteed at this point that no additional cfs_rq of this group can 6654 * join a CSD list. 6655 */ 6656 for_each_possible_cpu(i) { 6657 struct rq *rq = cpu_rq(i); 6658 unsigned long flags; 6659 6660 if (list_empty(&rq->cfsb_csd_list)) 6661 continue; 6662 6663 local_irq_save(flags); 6664 __cfsb_csd_unthrottle(rq); 6665 local_irq_restore(flags); 6666 } 6667 } 6668 6669 /* 6670 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6671 * 6672 * The race is harmless, since modifying bandwidth settings of unhooked group 6673 * bits doesn't do much. 6674 */ 6675 6676 /* cpu online callback */ 6677 static void __maybe_unused update_runtime_enabled(struct rq *rq) 6678 { 6679 struct task_group *tg; 6680 6681 lockdep_assert_rq_held(rq); 6682 6683 rcu_read_lock(); 6684 list_for_each_entry_rcu(tg, &task_groups, list) { 6685 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6686 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6687 6688 raw_spin_lock(&cfs_b->lock); 6689 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6690 raw_spin_unlock(&cfs_b->lock); 6691 } 6692 rcu_read_unlock(); 6693 } 6694 6695 /* cpu offline callback */ 6696 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6697 { 6698 struct task_group *tg; 6699 6700 lockdep_assert_rq_held(rq); 6701 6702 // Do not unthrottle for an active CPU 6703 if (cpumask_test_cpu(cpu_of(rq), cpu_active_mask)) 6704 return; 6705 6706 /* 6707 * The rq clock has already been updated in the 6708 * set_rq_offline(), so we should skip updating 6709 * the rq clock again in unthrottle_cfs_rq(). 6710 */ 6711 rq_clock_start_loop_update(rq); 6712 6713 rcu_read_lock(); 6714 list_for_each_entry_rcu(tg, &task_groups, list) { 6715 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6716 6717 if (!cfs_rq->runtime_enabled) 6718 continue; 6719 6720 /* 6721 * Offline rq is schedulable till CPU is completely disabled 6722 * in take_cpu_down(), so we prevent new cfs throttling here. 6723 */ 6724 cfs_rq->runtime_enabled = 0; 6725 6726 if (!cfs_rq_throttled(cfs_rq)) 6727 continue; 6728 6729 /* 6730 * clock_task is not advancing so we just need to make sure 6731 * there's some valid quota amount 6732 */ 6733 cfs_rq->runtime_remaining = 1; 6734 unthrottle_cfs_rq(cfs_rq); 6735 } 6736 rcu_read_unlock(); 6737 6738 rq_clock_stop_loop_update(rq); 6739 } 6740 6741 bool cfs_task_bw_constrained(struct task_struct *p) 6742 { 6743 struct cfs_rq *cfs_rq = task_cfs_rq(p); 6744 6745 if (!cfs_bandwidth_used()) 6746 return false; 6747 6748 if (cfs_rq->runtime_enabled || 6749 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 6750 return true; 6751 6752 return false; 6753 } 6754 6755 #ifdef CONFIG_NO_HZ_FULL 6756 /* called from pick_next_task_fair() */ 6757 static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 6758 { 6759 int cpu = cpu_of(rq); 6760 6761 if (!cfs_bandwidth_used()) 6762 return; 6763 6764 if (!tick_nohz_full_cpu(cpu)) 6765 return; 6766 6767 if (rq->nr_running != 1) 6768 return; 6769 6770 /* 6771 * We know there is only one task runnable and we've just picked it. The 6772 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 6773 * be otherwise able to stop the tick. Just need to check if we are using 6774 * bandwidth control. 6775 */ 6776 if (cfs_task_bw_constrained(p)) 6777 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 6778 } 6779 #endif /* CONFIG_NO_HZ_FULL */ 6780 6781 #else /* !CONFIG_CFS_BANDWIDTH: */ 6782 6783 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6784 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6785 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6786 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6787 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6788 static void task_throttle_setup_work(struct task_struct *p) {} 6789 static bool task_is_throttled(struct task_struct *p) { return false; } 6790 static void dequeue_throttled_task(struct task_struct *p, int flags) {} 6791 static bool enqueue_throttled_task(struct task_struct *p) { return false; } 6792 static void record_throttle_clock(struct cfs_rq *cfs_rq) {} 6793 6794 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6795 { 6796 return 0; 6797 } 6798 6799 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 6800 { 6801 return false; 6802 } 6803 6804 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6805 { 6806 return 0; 6807 } 6808 6809 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 6810 { 6811 return 0; 6812 } 6813 6814 #ifdef CONFIG_FAIR_GROUP_SCHED 6815 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 6816 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6817 #endif 6818 6819 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6820 { 6821 return NULL; 6822 } 6823 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6824 static inline void update_runtime_enabled(struct rq *rq) {} 6825 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6826 #ifdef CONFIG_CGROUP_SCHED 6827 bool cfs_task_bw_constrained(struct task_struct *p) 6828 { 6829 return false; 6830 } 6831 #endif 6832 #endif /* !CONFIG_CFS_BANDWIDTH */ 6833 6834 #if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 6835 static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 6836 #endif 6837 6838 /************************************************** 6839 * CFS operations on tasks: 6840 */ 6841 6842 #ifdef CONFIG_SCHED_HRTICK 6843 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6844 { 6845 struct sched_entity *se = &p->se; 6846 6847 WARN_ON_ONCE(task_rq(p) != rq); 6848 6849 if (rq->cfs.h_nr_queued > 1) { 6850 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6851 u64 slice = se->slice; 6852 s64 delta = slice - ran; 6853 6854 if (delta < 0) { 6855 if (task_current_donor(rq, p)) 6856 resched_curr(rq); 6857 return; 6858 } 6859 hrtick_start(rq, delta); 6860 } 6861 } 6862 6863 /* 6864 * called from enqueue/dequeue and updates the hrtick when the 6865 * current task is from our class and nr_running is low enough 6866 * to matter. 6867 */ 6868 static void hrtick_update(struct rq *rq) 6869 { 6870 struct task_struct *donor = rq->donor; 6871 6872 if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class) 6873 return; 6874 6875 hrtick_start_fair(rq, donor); 6876 } 6877 #else /* !CONFIG_SCHED_HRTICK: */ 6878 static inline void 6879 hrtick_start_fair(struct rq *rq, struct task_struct *p) 6880 { 6881 } 6882 6883 static inline void hrtick_update(struct rq *rq) 6884 { 6885 } 6886 #endif /* !CONFIG_SCHED_HRTICK */ 6887 6888 static inline bool cpu_overutilized(int cpu) 6889 { 6890 unsigned long rq_util_min, rq_util_max; 6891 6892 if (!sched_energy_enabled()) 6893 return false; 6894 6895 rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6896 rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6897 6898 /* Return true only if the utilization doesn't fit CPU's capacity */ 6899 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6900 } 6901 6902 /* 6903 * overutilized value make sense only if EAS is enabled 6904 */ 6905 static inline bool is_rd_overutilized(struct root_domain *rd) 6906 { 6907 return !sched_energy_enabled() || READ_ONCE(rd->overutilized); 6908 } 6909 6910 static inline void set_rd_overutilized(struct root_domain *rd, bool flag) 6911 { 6912 if (!sched_energy_enabled()) 6913 return; 6914 6915 WRITE_ONCE(rd->overutilized, flag); 6916 trace_sched_overutilized_tp(rd, flag); 6917 } 6918 6919 static inline void check_update_overutilized_status(struct rq *rq) 6920 { 6921 /* 6922 * overutilized field is used for load balancing decisions only 6923 * if energy aware scheduler is being used 6924 */ 6925 6926 if (!is_rd_overutilized(rq->rd) && cpu_overutilized(rq->cpu)) 6927 set_rd_overutilized(rq->rd, 1); 6928 } 6929 6930 /* Runqueue only has SCHED_IDLE tasks enqueued */ 6931 static int sched_idle_rq(struct rq *rq) 6932 { 6933 return unlikely(rq->nr_running == rq->cfs.h_nr_idle && 6934 rq->nr_running); 6935 } 6936 6937 static int sched_idle_cpu(int cpu) 6938 { 6939 return sched_idle_rq(cpu_rq(cpu)); 6940 } 6941 6942 static void 6943 requeue_delayed_entity(struct sched_entity *se) 6944 { 6945 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6946 6947 /* 6948 * se->sched_delayed should imply: se->on_rq == 1. 6949 * Because a delayed entity is one that is still on 6950 * the runqueue competing until elegibility. 6951 */ 6952 WARN_ON_ONCE(!se->sched_delayed); 6953 WARN_ON_ONCE(!se->on_rq); 6954 6955 if (sched_feat(DELAY_ZERO)) { 6956 update_entity_lag(cfs_rq, se); 6957 if (se->vlag > 0) { 6958 cfs_rq->nr_queued--; 6959 if (se != cfs_rq->curr) 6960 __dequeue_entity(cfs_rq, se); 6961 se->vlag = 0; 6962 place_entity(cfs_rq, se, 0); 6963 if (se != cfs_rq->curr) 6964 __enqueue_entity(cfs_rq, se); 6965 cfs_rq->nr_queued++; 6966 } 6967 } 6968 6969 update_load_avg(cfs_rq, se, 0); 6970 clear_delayed(se); 6971 } 6972 6973 /* 6974 * The enqueue_task method is called before nr_running is 6975 * increased. Here we update the fair scheduling stats and 6976 * then put the task into the rbtree: 6977 */ 6978 static void 6979 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6980 { 6981 struct cfs_rq *cfs_rq; 6982 struct sched_entity *se = &p->se; 6983 int h_nr_idle = task_has_idle_policy(p); 6984 int h_nr_runnable = 1; 6985 int task_new = !(flags & ENQUEUE_WAKEUP); 6986 int rq_h_nr_queued = rq->cfs.h_nr_queued; 6987 u64 slice = 0; 6988 6989 if (task_is_throttled(p) && enqueue_throttled_task(p)) 6990 return; 6991 6992 /* 6993 * The code below (indirectly) updates schedutil which looks at 6994 * the cfs_rq utilization to select a frequency. 6995 * Let's add the task's estimated utilization to the cfs_rq's 6996 * estimated utilization, before we update schedutil. 6997 */ 6998 if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) 6999 util_est_enqueue(&rq->cfs, p); 7000 7001 if (flags & ENQUEUE_DELAYED) { 7002 requeue_delayed_entity(se); 7003 return; 7004 } 7005 7006 /* 7007 * If in_iowait is set, the code below may not trigger any cpufreq 7008 * utilization updates, so do it here explicitly with the IOWAIT flag 7009 * passed. 7010 */ 7011 if (p->in_iowait) 7012 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 7013 7014 if (task_new && se->sched_delayed) 7015 h_nr_runnable = 0; 7016 7017 for_each_sched_entity(se) { 7018 if (se->on_rq) { 7019 if (se->sched_delayed) 7020 requeue_delayed_entity(se); 7021 break; 7022 } 7023 cfs_rq = cfs_rq_of(se); 7024 7025 /* 7026 * Basically set the slice of group entries to the min_slice of 7027 * their respective cfs_rq. This ensures the group can service 7028 * its entities in the desired time-frame. 7029 */ 7030 if (slice) { 7031 se->slice = slice; 7032 se->custom_slice = 1; 7033 } 7034 enqueue_entity(cfs_rq, se, flags); 7035 slice = cfs_rq_min_slice(cfs_rq); 7036 7037 cfs_rq->h_nr_runnable += h_nr_runnable; 7038 cfs_rq->h_nr_queued++; 7039 cfs_rq->h_nr_idle += h_nr_idle; 7040 7041 if (cfs_rq_is_idle(cfs_rq)) 7042 h_nr_idle = 1; 7043 7044 flags = ENQUEUE_WAKEUP; 7045 } 7046 7047 for_each_sched_entity(se) { 7048 cfs_rq = cfs_rq_of(se); 7049 7050 update_load_avg(cfs_rq, se, UPDATE_TG); 7051 se_update_runnable(se); 7052 update_cfs_group(se); 7053 7054 se->slice = slice; 7055 if (se != cfs_rq->curr) 7056 min_vruntime_cb_propagate(&se->run_node, NULL); 7057 slice = cfs_rq_min_slice(cfs_rq); 7058 7059 cfs_rq->h_nr_runnable += h_nr_runnable; 7060 cfs_rq->h_nr_queued++; 7061 cfs_rq->h_nr_idle += h_nr_idle; 7062 7063 if (cfs_rq_is_idle(cfs_rq)) 7064 h_nr_idle = 1; 7065 } 7066 7067 if (!rq_h_nr_queued && rq->cfs.h_nr_queued) { 7068 /* Account for idle runtime */ 7069 if (!rq->nr_running) 7070 dl_server_update_idle_time(rq, rq->curr); 7071 dl_server_start(&rq->fair_server); 7072 } 7073 7074 /* At this point se is NULL and we are at root level*/ 7075 add_nr_running(rq, 1); 7076 7077 /* 7078 * Since new tasks are assigned an initial util_avg equal to 7079 * half of the spare capacity of their CPU, tiny tasks have the 7080 * ability to cross the overutilized threshold, which will 7081 * result in the load balancer ruining all the task placement 7082 * done by EAS. As a way to mitigate that effect, do not account 7083 * for the first enqueue operation of new tasks during the 7084 * overutilized flag detection. 7085 * 7086 * A better way of solving this problem would be to wait for 7087 * the PELT signals of tasks to converge before taking them 7088 * into account, but that is not straightforward to implement, 7089 * and the following generally works well enough in practice. 7090 */ 7091 if (!task_new) 7092 check_update_overutilized_status(rq); 7093 7094 assert_list_leaf_cfs_rq(rq); 7095 7096 hrtick_update(rq); 7097 } 7098 7099 /* 7100 * Basically dequeue_task_fair(), except it can deal with dequeue_entity() 7101 * failing half-way through and resume the dequeue later. 7102 * 7103 * Returns: 7104 * -1 - dequeue delayed 7105 * 0 - dequeue throttled 7106 * 1 - dequeue complete 7107 */ 7108 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) 7109 { 7110 bool was_sched_idle = sched_idle_rq(rq); 7111 bool task_sleep = flags & DEQUEUE_SLEEP; 7112 bool task_delayed = flags & DEQUEUE_DELAYED; 7113 bool task_throttled = flags & DEQUEUE_THROTTLE; 7114 struct task_struct *p = NULL; 7115 int h_nr_idle = 0; 7116 int h_nr_queued = 0; 7117 int h_nr_runnable = 0; 7118 struct cfs_rq *cfs_rq; 7119 u64 slice = 0; 7120 7121 if (entity_is_task(se)) { 7122 p = task_of(se); 7123 h_nr_queued = 1; 7124 h_nr_idle = task_has_idle_policy(p); 7125 if (task_sleep || task_delayed || !se->sched_delayed) 7126 h_nr_runnable = 1; 7127 } 7128 7129 for_each_sched_entity(se) { 7130 cfs_rq = cfs_rq_of(se); 7131 7132 if (!dequeue_entity(cfs_rq, se, flags)) { 7133 if (p && &p->se == se) 7134 return -1; 7135 7136 slice = cfs_rq_min_slice(cfs_rq); 7137 break; 7138 } 7139 7140 cfs_rq->h_nr_runnable -= h_nr_runnable; 7141 cfs_rq->h_nr_queued -= h_nr_queued; 7142 cfs_rq->h_nr_idle -= h_nr_idle; 7143 7144 if (cfs_rq_is_idle(cfs_rq)) 7145 h_nr_idle = h_nr_queued; 7146 7147 if (throttled_hierarchy(cfs_rq) && task_throttled) 7148 record_throttle_clock(cfs_rq); 7149 7150 /* Don't dequeue parent if it has other entities besides us */ 7151 if (cfs_rq->load.weight) { 7152 slice = cfs_rq_min_slice(cfs_rq); 7153 7154 /* Avoid re-evaluating load for this entity: */ 7155 se = parent_entity(se); 7156 /* 7157 * Bias pick_next to pick a task from this cfs_rq, as 7158 * p is sleeping when it is within its sched_slice. 7159 */ 7160 if (task_sleep && se) 7161 set_next_buddy(se); 7162 break; 7163 } 7164 flags |= DEQUEUE_SLEEP; 7165 flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); 7166 } 7167 7168 for_each_sched_entity(se) { 7169 cfs_rq = cfs_rq_of(se); 7170 7171 update_load_avg(cfs_rq, se, UPDATE_TG); 7172 se_update_runnable(se); 7173 update_cfs_group(se); 7174 7175 se->slice = slice; 7176 if (se != cfs_rq->curr) 7177 min_vruntime_cb_propagate(&se->run_node, NULL); 7178 slice = cfs_rq_min_slice(cfs_rq); 7179 7180 cfs_rq->h_nr_runnable -= h_nr_runnable; 7181 cfs_rq->h_nr_queued -= h_nr_queued; 7182 cfs_rq->h_nr_idle -= h_nr_idle; 7183 7184 if (cfs_rq_is_idle(cfs_rq)) 7185 h_nr_idle = h_nr_queued; 7186 7187 if (throttled_hierarchy(cfs_rq) && task_throttled) 7188 record_throttle_clock(cfs_rq); 7189 } 7190 7191 sub_nr_running(rq, h_nr_queued); 7192 7193 /* balance early to pull high priority tasks */ 7194 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 7195 rq->next_balance = jiffies; 7196 7197 if (p && task_delayed) { 7198 WARN_ON_ONCE(!task_sleep); 7199 WARN_ON_ONCE(p->on_rq != 1); 7200 7201 /* Fix-up what dequeue_task_fair() skipped */ 7202 hrtick_update(rq); 7203 7204 /* 7205 * Fix-up what block_task() skipped. 7206 * 7207 * Must be last, @p might not be valid after this. 7208 */ 7209 __block_task(rq, p); 7210 } 7211 7212 return 1; 7213 } 7214 7215 /* 7216 * The dequeue_task method is called before nr_running is 7217 * decreased. We remove the task from the rbtree and 7218 * update the fair scheduling stats: 7219 */ 7220 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 7221 { 7222 if (task_is_throttled(p)) { 7223 dequeue_throttled_task(p, flags); 7224 return true; 7225 } 7226 7227 if (!p->se.sched_delayed) 7228 util_est_dequeue(&rq->cfs, p); 7229 7230 util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP); 7231 if (dequeue_entities(rq, &p->se, flags) < 0) 7232 return false; 7233 7234 /* 7235 * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). 7236 */ 7237 7238 hrtick_update(rq); 7239 return true; 7240 } 7241 7242 static inline unsigned int cfs_h_nr_delayed(struct rq *rq) 7243 { 7244 return (rq->cfs.h_nr_queued - rq->cfs.h_nr_runnable); 7245 } 7246 7247 /* Working cpumask for: sched_balance_rq(), sched_balance_newidle(). */ 7248 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 7249 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 7250 static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 7251 7252 #ifdef CONFIG_NO_HZ_COMMON 7253 7254 static struct { 7255 cpumask_var_t idle_cpus_mask; 7256 atomic_t nr_cpus; 7257 int has_blocked; /* Idle CPUS has blocked load */ 7258 int needs_update; /* Newly idle CPUs need their next_balance collated */ 7259 unsigned long next_balance; /* in jiffy units */ 7260 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 7261 } nohz ____cacheline_aligned; 7262 7263 #endif /* CONFIG_NO_HZ_COMMON */ 7264 7265 static unsigned long cpu_load(struct rq *rq) 7266 { 7267 return cfs_rq_load_avg(&rq->cfs); 7268 } 7269 7270 /* 7271 * cpu_load_without - compute CPU load without any contributions from *p 7272 * @cpu: the CPU which load is requested 7273 * @p: the task which load should be discounted 7274 * 7275 * The load of a CPU is defined by the load of tasks currently enqueued on that 7276 * CPU as well as tasks which are currently sleeping after an execution on that 7277 * CPU. 7278 * 7279 * This method returns the load of the specified CPU by discounting the load of 7280 * the specified task, whenever the task is currently contributing to the CPU 7281 * load. 7282 */ 7283 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 7284 { 7285 struct cfs_rq *cfs_rq; 7286 unsigned int load; 7287 7288 /* Task has no contribution or is new */ 7289 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7290 return cpu_load(rq); 7291 7292 cfs_rq = &rq->cfs; 7293 load = READ_ONCE(cfs_rq->avg.load_avg); 7294 7295 /* Discount task's util from CPU's util */ 7296 lsub_positive(&load, task_h_load(p)); 7297 7298 return load; 7299 } 7300 7301 static unsigned long cpu_runnable(struct rq *rq) 7302 { 7303 return cfs_rq_runnable_avg(&rq->cfs); 7304 } 7305 7306 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 7307 { 7308 struct cfs_rq *cfs_rq; 7309 unsigned int runnable; 7310 7311 /* Task has no contribution or is new */ 7312 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7313 return cpu_runnable(rq); 7314 7315 cfs_rq = &rq->cfs; 7316 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7317 7318 /* Discount task's runnable from CPU's runnable */ 7319 lsub_positive(&runnable, p->se.avg.runnable_avg); 7320 7321 return runnable; 7322 } 7323 7324 static unsigned long capacity_of(int cpu) 7325 { 7326 return cpu_rq(cpu)->cpu_capacity; 7327 } 7328 7329 static void record_wakee(struct task_struct *p) 7330 { 7331 /* 7332 * Only decay a single time; tasks that have less then 1 wakeup per 7333 * jiffy will not have built up many flips. 7334 */ 7335 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 7336 current->wakee_flips >>= 1; 7337 current->wakee_flip_decay_ts = jiffies; 7338 } 7339 7340 if (current->last_wakee != p) { 7341 current->last_wakee = p; 7342 current->wakee_flips++; 7343 } 7344 } 7345 7346 /* 7347 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 7348 * 7349 * A waker of many should wake a different task than the one last awakened 7350 * at a frequency roughly N times higher than one of its wakees. 7351 * 7352 * In order to determine whether we should let the load spread vs consolidating 7353 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 7354 * partner, and a factor of lls_size higher frequency in the other. 7355 * 7356 * With both conditions met, we can be relatively sure that the relationship is 7357 * non-monogamous, with partner count exceeding socket size. 7358 * 7359 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 7360 * whatever is irrelevant, spread criteria is apparent partner count exceeds 7361 * socket size. 7362 */ 7363 static int wake_wide(struct task_struct *p) 7364 { 7365 unsigned int master = current->wakee_flips; 7366 unsigned int slave = p->wakee_flips; 7367 int factor = __this_cpu_read(sd_llc_size); 7368 7369 if (master < slave) 7370 swap(master, slave); 7371 if (slave < factor || master < slave * factor) 7372 return 0; 7373 return 1; 7374 } 7375 7376 /* 7377 * The purpose of wake_affine() is to quickly determine on which CPU we can run 7378 * soonest. For the purpose of speed we only consider the waking and previous 7379 * CPU. 7380 * 7381 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 7382 * cache-affine and is (or will be) idle. 7383 * 7384 * wake_affine_weight() - considers the weight to reflect the average 7385 * scheduling latency of the CPUs. This seems to work 7386 * for the overloaded case. 7387 */ 7388 static int 7389 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 7390 { 7391 /* 7392 * If this_cpu is idle, it implies the wakeup is from interrupt 7393 * context. Only allow the move if cache is shared. Otherwise an 7394 * interrupt intensive workload could force all tasks onto one 7395 * node depending on the IO topology or IRQ affinity settings. 7396 * 7397 * If the prev_cpu is idle and cache affine then avoid a migration. 7398 * There is no guarantee that the cache hot data from an interrupt 7399 * is more important than cache hot data on the prev_cpu and from 7400 * a cpufreq perspective, it's better to have higher utilisation 7401 * on one CPU. 7402 */ 7403 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 7404 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 7405 7406 if (sync) { 7407 struct rq *rq = cpu_rq(this_cpu); 7408 7409 if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) 7410 return this_cpu; 7411 } 7412 7413 if (available_idle_cpu(prev_cpu)) 7414 return prev_cpu; 7415 7416 return nr_cpumask_bits; 7417 } 7418 7419 static int 7420 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 7421 int this_cpu, int prev_cpu, int sync) 7422 { 7423 s64 this_eff_load, prev_eff_load; 7424 unsigned long task_load; 7425 7426 this_eff_load = cpu_load(cpu_rq(this_cpu)); 7427 7428 if (sync) { 7429 unsigned long current_load = task_h_load(current); 7430 7431 if (current_load > this_eff_load) 7432 return this_cpu; 7433 7434 this_eff_load -= current_load; 7435 } 7436 7437 task_load = task_h_load(p); 7438 7439 this_eff_load += task_load; 7440 if (sched_feat(WA_BIAS)) 7441 this_eff_load *= 100; 7442 this_eff_load *= capacity_of(prev_cpu); 7443 7444 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 7445 prev_eff_load -= task_load; 7446 if (sched_feat(WA_BIAS)) 7447 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 7448 prev_eff_load *= capacity_of(this_cpu); 7449 7450 /* 7451 * If sync, adjust the weight of prev_eff_load such that if 7452 * prev_eff == this_eff that select_idle_sibling() will consider 7453 * stacking the wakee on top of the waker if no other CPU is 7454 * idle. 7455 */ 7456 if (sync) 7457 prev_eff_load += 1; 7458 7459 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 7460 } 7461 7462 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 7463 int this_cpu, int prev_cpu, int sync) 7464 { 7465 int target = nr_cpumask_bits; 7466 7467 if (sched_feat(WA_IDLE)) 7468 target = wake_affine_idle(this_cpu, prev_cpu, sync); 7469 7470 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 7471 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 7472 7473 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 7474 if (target != this_cpu) 7475 return prev_cpu; 7476 7477 schedstat_inc(sd->ttwu_move_affine); 7478 schedstat_inc(p->stats.nr_wakeups_affine); 7479 return target; 7480 } 7481 7482 static struct sched_group * 7483 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 7484 7485 /* 7486 * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group. 7487 */ 7488 static int 7489 sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 7490 { 7491 unsigned long load, min_load = ULONG_MAX; 7492 unsigned int min_exit_latency = UINT_MAX; 7493 u64 latest_idle_timestamp = 0; 7494 int least_loaded_cpu = this_cpu; 7495 int shallowest_idle_cpu = -1; 7496 int i; 7497 7498 /* Check if we have any choice: */ 7499 if (group->group_weight == 1) 7500 return cpumask_first(sched_group_span(group)); 7501 7502 /* Traverse only the allowed CPUs */ 7503 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 7504 struct rq *rq = cpu_rq(i); 7505 7506 if (!sched_core_cookie_match(rq, p)) 7507 continue; 7508 7509 if (sched_idle_cpu(i)) 7510 return i; 7511 7512 if (available_idle_cpu(i)) { 7513 struct cpuidle_state *idle = idle_get_state(rq); 7514 if (idle && idle->exit_latency < min_exit_latency) { 7515 /* 7516 * We give priority to a CPU whose idle state 7517 * has the smallest exit latency irrespective 7518 * of any idle timestamp. 7519 */ 7520 min_exit_latency = idle->exit_latency; 7521 latest_idle_timestamp = rq->idle_stamp; 7522 shallowest_idle_cpu = i; 7523 } else if ((!idle || idle->exit_latency == min_exit_latency) && 7524 rq->idle_stamp > latest_idle_timestamp) { 7525 /* 7526 * If equal or no active idle state, then 7527 * the most recently idled CPU might have 7528 * a warmer cache. 7529 */ 7530 latest_idle_timestamp = rq->idle_stamp; 7531 shallowest_idle_cpu = i; 7532 } 7533 } else if (shallowest_idle_cpu == -1) { 7534 load = cpu_load(cpu_rq(i)); 7535 if (load < min_load) { 7536 min_load = load; 7537 least_loaded_cpu = i; 7538 } 7539 } 7540 } 7541 7542 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 7543 } 7544 7545 static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p, 7546 int cpu, int prev_cpu, int sd_flag) 7547 { 7548 int new_cpu = cpu; 7549 7550 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 7551 return prev_cpu; 7552 7553 /* 7554 * We need task's util for cpu_util_without, sync it up to 7555 * prev_cpu's last_update_time. 7556 */ 7557 if (!(sd_flag & SD_BALANCE_FORK)) 7558 sync_entity_load_avg(&p->se); 7559 7560 while (sd) { 7561 struct sched_group *group; 7562 struct sched_domain *tmp; 7563 int weight; 7564 7565 if (!(sd->flags & sd_flag)) { 7566 sd = sd->child; 7567 continue; 7568 } 7569 7570 group = sched_balance_find_dst_group(sd, p, cpu); 7571 if (!group) { 7572 sd = sd->child; 7573 continue; 7574 } 7575 7576 new_cpu = sched_balance_find_dst_group_cpu(group, p, cpu); 7577 if (new_cpu == cpu) { 7578 /* Now try balancing at a lower domain level of 'cpu': */ 7579 sd = sd->child; 7580 continue; 7581 } 7582 7583 /* Now try balancing at a lower domain level of 'new_cpu': */ 7584 cpu = new_cpu; 7585 weight = sd->span_weight; 7586 sd = NULL; 7587 for_each_domain(cpu, tmp) { 7588 if (weight <= tmp->span_weight) 7589 break; 7590 if (tmp->flags & sd_flag) 7591 sd = tmp; 7592 } 7593 } 7594 7595 return new_cpu; 7596 } 7597 7598 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 7599 { 7600 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 7601 sched_cpu_cookie_match(cpu_rq(cpu), p)) 7602 return cpu; 7603 7604 return -1; 7605 } 7606 7607 #ifdef CONFIG_SCHED_SMT 7608 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 7609 EXPORT_SYMBOL_GPL(sched_smt_present); 7610 7611 static inline void set_idle_cores(int cpu, int val) 7612 { 7613 struct sched_domain_shared *sds; 7614 7615 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7616 if (sds) 7617 WRITE_ONCE(sds->has_idle_cores, val); 7618 } 7619 7620 static inline bool test_idle_cores(int cpu) 7621 { 7622 struct sched_domain_shared *sds; 7623 7624 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7625 if (sds) 7626 return READ_ONCE(sds->has_idle_cores); 7627 7628 return false; 7629 } 7630 7631 /* 7632 * Scans the local SMT mask to see if the entire core is idle, and records this 7633 * information in sd_llc_shared->has_idle_cores. 7634 * 7635 * Since SMT siblings share all cache levels, inspecting this limited remote 7636 * state should be fairly cheap. 7637 */ 7638 void __update_idle_core(struct rq *rq) 7639 { 7640 int core = cpu_of(rq); 7641 int cpu; 7642 7643 rcu_read_lock(); 7644 if (test_idle_cores(core)) 7645 goto unlock; 7646 7647 for_each_cpu(cpu, cpu_smt_mask(core)) { 7648 if (cpu == core) 7649 continue; 7650 7651 if (!available_idle_cpu(cpu)) 7652 goto unlock; 7653 } 7654 7655 set_idle_cores(core, 1); 7656 unlock: 7657 rcu_read_unlock(); 7658 } 7659 7660 /* 7661 * Scan the entire LLC domain for idle cores; this dynamically switches off if 7662 * there are no idle cores left in the system; tracked through 7663 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 7664 */ 7665 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7666 { 7667 bool idle = true; 7668 int cpu; 7669 7670 for_each_cpu(cpu, cpu_smt_mask(core)) { 7671 if (!available_idle_cpu(cpu)) { 7672 idle = false; 7673 if (*idle_cpu == -1) { 7674 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpus)) { 7675 *idle_cpu = cpu; 7676 break; 7677 } 7678 continue; 7679 } 7680 break; 7681 } 7682 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus)) 7683 *idle_cpu = cpu; 7684 } 7685 7686 if (idle) 7687 return core; 7688 7689 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 7690 return -1; 7691 } 7692 7693 /* 7694 * Scan the local SMT mask for idle CPUs. 7695 */ 7696 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7697 { 7698 int cpu; 7699 7700 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 7701 if (cpu == target) 7702 continue; 7703 /* 7704 * Check if the CPU is in the LLC scheduling domain of @target. 7705 * Due to isolcpus, there is no guarantee that all the siblings are in the domain. 7706 */ 7707 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7708 continue; 7709 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 7710 return cpu; 7711 } 7712 7713 return -1; 7714 } 7715 7716 #else /* !CONFIG_SCHED_SMT: */ 7717 7718 static inline void set_idle_cores(int cpu, int val) 7719 { 7720 } 7721 7722 static inline bool test_idle_cores(int cpu) 7723 { 7724 return false; 7725 } 7726 7727 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7728 { 7729 return __select_idle_cpu(core, p); 7730 } 7731 7732 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7733 { 7734 return -1; 7735 } 7736 7737 #endif /* !CONFIG_SCHED_SMT */ 7738 7739 /* 7740 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 7741 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 7742 * average idle time for this rq (as found in rq->avg_idle). 7743 */ 7744 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 7745 { 7746 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7747 int i, cpu, idle_cpu = -1, nr = INT_MAX; 7748 struct sched_domain_shared *sd_share; 7749 7750 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7751 7752 if (sched_feat(SIS_UTIL)) { 7753 sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); 7754 if (sd_share) { 7755 /* because !--nr is the condition to stop scan */ 7756 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7757 /* overloaded LLC is unlikely to have idle cpu/core */ 7758 if (nr == 1) 7759 return -1; 7760 } 7761 } 7762 7763 if (static_branch_unlikely(&sched_cluster_active)) { 7764 struct sched_group *sg = sd->groups; 7765 7766 if (sg->flags & SD_CLUSTER) { 7767 for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) { 7768 if (!cpumask_test_cpu(cpu, cpus)) 7769 continue; 7770 7771 if (has_idle_core) { 7772 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7773 if ((unsigned int)i < nr_cpumask_bits) 7774 return i; 7775 } else { 7776 if (--nr <= 0) 7777 return -1; 7778 idle_cpu = __select_idle_cpu(cpu, p); 7779 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7780 return idle_cpu; 7781 } 7782 } 7783 cpumask_andnot(cpus, cpus, sched_group_span(sg)); 7784 } 7785 } 7786 7787 for_each_cpu_wrap(cpu, cpus, target + 1) { 7788 if (has_idle_core) { 7789 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7790 if ((unsigned int)i < nr_cpumask_bits) 7791 return i; 7792 7793 } else { 7794 if (--nr <= 0) 7795 return -1; 7796 idle_cpu = __select_idle_cpu(cpu, p); 7797 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7798 break; 7799 } 7800 } 7801 7802 if (has_idle_core) 7803 set_idle_cores(target, false); 7804 7805 return idle_cpu; 7806 } 7807 7808 /* 7809 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7810 * the task fits. If no CPU is big enough, but there are idle ones, try to 7811 * maximize capacity. 7812 */ 7813 static int 7814 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7815 { 7816 unsigned long task_util, util_min, util_max, best_cap = 0; 7817 int fits, best_fits = 0; 7818 int cpu, best_cpu = -1; 7819 struct cpumask *cpus; 7820 7821 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7822 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7823 7824 task_util = task_util_est(p); 7825 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7826 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7827 7828 for_each_cpu_wrap(cpu, cpus, target) { 7829 unsigned long cpu_cap = capacity_of(cpu); 7830 7831 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7832 continue; 7833 7834 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7835 7836 /* This CPU fits with all requirements */ 7837 if (fits > 0) 7838 return cpu; 7839 /* 7840 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7841 * Look for the CPU with best capacity. 7842 */ 7843 else if (fits < 0) 7844 cpu_cap = get_actual_cpu_capacity(cpu); 7845 7846 /* 7847 * First, select CPU which fits better (-1 being better than 0). 7848 * Then, select the one with best capacity at same level. 7849 */ 7850 if ((fits < best_fits) || 7851 ((fits == best_fits) && (cpu_cap > best_cap))) { 7852 best_cap = cpu_cap; 7853 best_cpu = cpu; 7854 best_fits = fits; 7855 } 7856 } 7857 7858 return best_cpu; 7859 } 7860 7861 static inline bool asym_fits_cpu(unsigned long util, 7862 unsigned long util_min, 7863 unsigned long util_max, 7864 int cpu) 7865 { 7866 if (sched_asym_cpucap_active()) 7867 /* 7868 * Return true only if the cpu fully fits the task requirements 7869 * which include the utilization and the performance hints. 7870 */ 7871 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7872 7873 return true; 7874 } 7875 7876 /* 7877 * Try and locate an idle core/thread in the LLC cache domain. 7878 */ 7879 static int select_idle_sibling(struct task_struct *p, int prev, int target) 7880 { 7881 bool has_idle_core = false; 7882 struct sched_domain *sd; 7883 unsigned long task_util, util_min, util_max; 7884 int i, recent_used_cpu, prev_aff = -1; 7885 7886 /* 7887 * On asymmetric system, update task utilization because we will check 7888 * that the task fits with CPU's capacity. 7889 */ 7890 if (sched_asym_cpucap_active()) { 7891 sync_entity_load_avg(&p->se); 7892 task_util = task_util_est(p); 7893 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7894 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7895 } 7896 7897 /* 7898 * per-cpu select_rq_mask usage 7899 */ 7900 lockdep_assert_irqs_disabled(); 7901 7902 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7903 asym_fits_cpu(task_util, util_min, util_max, target)) 7904 return target; 7905 7906 /* 7907 * If the previous CPU is cache affine and idle, don't be stupid: 7908 */ 7909 if (prev != target && cpus_share_cache(prev, target) && 7910 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7911 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7912 7913 if (!static_branch_unlikely(&sched_cluster_active) || 7914 cpus_share_resources(prev, target)) 7915 return prev; 7916 7917 prev_aff = prev; 7918 } 7919 7920 /* 7921 * Allow a per-cpu kthread to stack with the wakee if the 7922 * kworker thread and the tasks previous CPUs are the same. 7923 * The assumption is that the wakee queued work for the 7924 * per-cpu kthread that is now complete and the wakeup is 7925 * essentially a sync wakeup. An obvious example of this 7926 * pattern is IO completions. 7927 */ 7928 if (is_per_cpu_kthread(current) && 7929 in_task() && 7930 prev == smp_processor_id() && 7931 this_rq()->nr_running <= 1 && 7932 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7933 return prev; 7934 } 7935 7936 /* Check a recently used CPU as a potential idle candidate: */ 7937 recent_used_cpu = p->recent_used_cpu; 7938 p->recent_used_cpu = prev; 7939 if (recent_used_cpu != prev && 7940 recent_used_cpu != target && 7941 cpus_share_cache(recent_used_cpu, target) && 7942 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7943 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7944 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7945 7946 if (!static_branch_unlikely(&sched_cluster_active) || 7947 cpus_share_resources(recent_used_cpu, target)) 7948 return recent_used_cpu; 7949 7950 } else { 7951 recent_used_cpu = -1; 7952 } 7953 7954 /* 7955 * For asymmetric CPU capacity systems, our domain of interest is 7956 * sd_asym_cpucapacity rather than sd_llc. 7957 */ 7958 if (sched_asym_cpucap_active()) { 7959 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 7960 /* 7961 * On an asymmetric CPU capacity system where an exclusive 7962 * cpuset defines a symmetric island (i.e. one unique 7963 * capacity_orig value through the cpuset), the key will be set 7964 * but the CPUs within that cpuset will not have a domain with 7965 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7966 * capacity path. 7967 */ 7968 if (sd) { 7969 i = select_idle_capacity(p, sd, target); 7970 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7971 } 7972 } 7973 7974 sd = rcu_dereference(per_cpu(sd_llc, target)); 7975 if (!sd) 7976 return target; 7977 7978 if (sched_smt_active()) { 7979 has_idle_core = test_idle_cores(target); 7980 7981 if (!has_idle_core && cpus_share_cache(prev, target)) { 7982 i = select_idle_smt(p, sd, prev); 7983 if ((unsigned int)i < nr_cpumask_bits) 7984 return i; 7985 } 7986 } 7987 7988 i = select_idle_cpu(p, sd, has_idle_core, target); 7989 if ((unsigned)i < nr_cpumask_bits) 7990 return i; 7991 7992 /* 7993 * For cluster machines which have lower sharing cache like L2 or 7994 * LLC Tag, we tend to find an idle CPU in the target's cluster 7995 * first. But prev_cpu or recent_used_cpu may also be a good candidate, 7996 * use them if possible when no idle CPU found in select_idle_cpu(). 7997 */ 7998 if ((unsigned int)prev_aff < nr_cpumask_bits) 7999 return prev_aff; 8000 if ((unsigned int)recent_used_cpu < nr_cpumask_bits) 8001 return recent_used_cpu; 8002 8003 return target; 8004 } 8005 8006 /** 8007 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 8008 * @cpu: the CPU to get the utilization for 8009 * @p: task for which the CPU utilization should be predicted or NULL 8010 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 8011 * @boost: 1 to enable boosting, otherwise 0 8012 * 8013 * The unit of the return value must be the same as the one of CPU capacity 8014 * so that CPU utilization can be compared with CPU capacity. 8015 * 8016 * CPU utilization is the sum of running time of runnable tasks plus the 8017 * recent utilization of currently non-runnable tasks on that CPU. 8018 * It represents the amount of CPU capacity currently used by CFS tasks in 8019 * the range [0..max CPU capacity] with max CPU capacity being the CPU 8020 * capacity at f_max. 8021 * 8022 * The estimated CPU utilization is defined as the maximum between CPU 8023 * utilization and sum of the estimated utilization of the currently 8024 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 8025 * previously-executed tasks, which helps better deduce how busy a CPU will 8026 * be when a long-sleeping task wakes up. The contribution to CPU utilization 8027 * of such a task would be significantly decayed at this point of time. 8028 * 8029 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 8030 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 8031 * utilization. Boosting is implemented in cpu_util() so that internal 8032 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 8033 * latter via cpu_util_cfs_boost(). 8034 * 8035 * CPU utilization can be higher than the current CPU capacity 8036 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 8037 * of rounding errors as well as task migrations or wakeups of new tasks. 8038 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 8039 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 8040 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 8041 * capacity. CPU utilization is allowed to overshoot current CPU capacity 8042 * though since this is useful for predicting the CPU capacity required 8043 * after task migrations (scheduler-driven DVFS). 8044 * 8045 * Return: (Boosted) (estimated) utilization for the specified CPU. 8046 */ 8047 static unsigned long 8048 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 8049 { 8050 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 8051 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 8052 unsigned long runnable; 8053 8054 if (boost) { 8055 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 8056 util = max(util, runnable); 8057 } 8058 8059 /* 8060 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 8061 * contribution. If @p migrates from another CPU to @cpu add its 8062 * contribution. In all the other cases @cpu is not impacted by the 8063 * migration so its util_avg is already correct. 8064 */ 8065 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 8066 lsub_positive(&util, task_util(p)); 8067 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 8068 util += task_util(p); 8069 8070 if (sched_feat(UTIL_EST)) { 8071 unsigned long util_est; 8072 8073 util_est = READ_ONCE(cfs_rq->avg.util_est); 8074 8075 /* 8076 * During wake-up @p isn't enqueued yet and doesn't contribute 8077 * to any cpu_rq(cpu)->cfs.avg.util_est. 8078 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 8079 * has been enqueued. 8080 * 8081 * During exec (@dst_cpu = -1) @p is enqueued and does 8082 * contribute to cpu_rq(cpu)->cfs.util_est. 8083 * Remove it to "simulate" cpu_util without @p's contribution. 8084 * 8085 * Despite the task_on_rq_queued(@p) check there is still a 8086 * small window for a possible race when an exec 8087 * select_task_rq_fair() races with LB's detach_task(). 8088 * 8089 * detach_task() 8090 * deactivate_task() 8091 * p->on_rq = TASK_ON_RQ_MIGRATING; 8092 * -------------------------------- A 8093 * dequeue_task() \ 8094 * dequeue_task_fair() + Race Time 8095 * util_est_dequeue() / 8096 * -------------------------------- B 8097 * 8098 * The additional check "current == p" is required to further 8099 * reduce the race window. 8100 */ 8101 if (dst_cpu == cpu) 8102 util_est += _task_util_est(p); 8103 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 8104 lsub_positive(&util_est, _task_util_est(p)); 8105 8106 util = max(util, util_est); 8107 } 8108 8109 return min(util, arch_scale_cpu_capacity(cpu)); 8110 } 8111 8112 unsigned long cpu_util_cfs(int cpu) 8113 { 8114 return cpu_util(cpu, NULL, -1, 0); 8115 } 8116 8117 unsigned long cpu_util_cfs_boost(int cpu) 8118 { 8119 return cpu_util(cpu, NULL, -1, 1); 8120 } 8121 8122 /* 8123 * cpu_util_without: compute cpu utilization without any contributions from *p 8124 * @cpu: the CPU which utilization is requested 8125 * @p: the task which utilization should be discounted 8126 * 8127 * The utilization of a CPU is defined by the utilization of tasks currently 8128 * enqueued on that CPU as well as tasks which are currently sleeping after an 8129 * execution on that CPU. 8130 * 8131 * This method returns the utilization of the specified CPU by discounting the 8132 * utilization of the specified task, whenever the task is currently 8133 * contributing to the CPU utilization. 8134 */ 8135 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 8136 { 8137 /* Task has no contribution or is new */ 8138 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8139 p = NULL; 8140 8141 return cpu_util(cpu, p, -1, 0); 8142 } 8143 8144 /* 8145 * This function computes an effective utilization for the given CPU, to be 8146 * used for frequency selection given the linear relation: f = u * f_max. 8147 * 8148 * The scheduler tracks the following metrics: 8149 * 8150 * cpu_util_{cfs,rt,dl,irq}() 8151 * cpu_bw_dl() 8152 * 8153 * Where the cfs,rt and dl util numbers are tracked with the same metric and 8154 * synchronized windows and are thus directly comparable. 8155 * 8156 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 8157 * which excludes things like IRQ and steal-time. These latter are then accrued 8158 * in the IRQ utilization. 8159 * 8160 * The DL bandwidth number OTOH is not a measured metric but a value computed 8161 * based on the task model parameters and gives the minimal utilization 8162 * required to meet deadlines. 8163 */ 8164 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 8165 unsigned long *min, 8166 unsigned long *max) 8167 { 8168 unsigned long util, irq, scale; 8169 struct rq *rq = cpu_rq(cpu); 8170 8171 scale = arch_scale_cpu_capacity(cpu); 8172 8173 /* 8174 * Early check to see if IRQ/steal time saturates the CPU, can be 8175 * because of inaccuracies in how we track these -- see 8176 * update_irq_load_avg(). 8177 */ 8178 irq = cpu_util_irq(rq); 8179 if (unlikely(irq >= scale)) { 8180 if (min) 8181 *min = scale; 8182 if (max) 8183 *max = scale; 8184 return scale; 8185 } 8186 8187 if (min) { 8188 /* 8189 * The minimum utilization returns the highest level between: 8190 * - the computed DL bandwidth needed with the IRQ pressure which 8191 * steals time to the deadline task. 8192 * - The minimum performance requirement for CFS and/or RT. 8193 */ 8194 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); 8195 8196 /* 8197 * When an RT task is runnable and uclamp is not used, we must 8198 * ensure that the task will run at maximum compute capacity. 8199 */ 8200 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) 8201 *min = max(*min, scale); 8202 } 8203 8204 /* 8205 * Because the time spend on RT/DL tasks is visible as 'lost' time to 8206 * CFS tasks and we use the same metric to track the effective 8207 * utilization (PELT windows are synchronized) we can directly add them 8208 * to obtain the CPU's actual utilization. 8209 */ 8210 util = util_cfs + cpu_util_rt(rq); 8211 util += cpu_util_dl(rq); 8212 8213 /* 8214 * The maximum hint is a soft bandwidth requirement, which can be lower 8215 * than the actual utilization because of uclamp_max requirements. 8216 */ 8217 if (max) 8218 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); 8219 8220 if (util >= scale) 8221 return scale; 8222 8223 /* 8224 * There is still idle time; further improve the number by using the 8225 * IRQ metric. Because IRQ/steal time is hidden from the task clock we 8226 * need to scale the task numbers: 8227 * 8228 * max - irq 8229 * U' = irq + --------- * U 8230 * max 8231 */ 8232 util = scale_irq_capacity(util, irq, scale); 8233 util += irq; 8234 8235 return min(scale, util); 8236 } 8237 8238 unsigned long sched_cpu_util(int cpu) 8239 { 8240 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); 8241 } 8242 8243 /* 8244 * energy_env - Utilization landscape for energy estimation. 8245 * @task_busy_time: Utilization contribution by the task for which we test the 8246 * placement. Given by eenv_task_busy_time(). 8247 * @pd_busy_time: Utilization of the whole perf domain without the task 8248 * contribution. Given by eenv_pd_busy_time(). 8249 * @cpu_cap: Maximum CPU capacity for the perf domain. 8250 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 8251 */ 8252 struct energy_env { 8253 unsigned long task_busy_time; 8254 unsigned long pd_busy_time; 8255 unsigned long cpu_cap; 8256 unsigned long pd_cap; 8257 }; 8258 8259 /* 8260 * Compute the task busy time for compute_energy(). This time cannot be 8261 * injected directly into effective_cpu_util() because of the IRQ scaling. 8262 * The latter only makes sense with the most recent CPUs where the task has 8263 * run. 8264 */ 8265 static inline void eenv_task_busy_time(struct energy_env *eenv, 8266 struct task_struct *p, int prev_cpu) 8267 { 8268 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 8269 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 8270 8271 if (unlikely(irq >= max_cap)) 8272 busy_time = max_cap; 8273 else 8274 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 8275 8276 eenv->task_busy_time = busy_time; 8277 } 8278 8279 /* 8280 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 8281 * utilization for each @pd_cpus, it however doesn't take into account 8282 * clamping since the ratio (utilization / cpu_capacity) is already enough to 8283 * scale the EM reported power consumption at the (eventually clamped) 8284 * cpu_capacity. 8285 * 8286 * The contribution of the task @p for which we want to estimate the 8287 * energy cost is removed (by cpu_util()) and must be calculated 8288 * separately (see eenv_task_busy_time). This ensures: 8289 * 8290 * - A stable PD utilization, no matter which CPU of that PD we want to place 8291 * the task on. 8292 * 8293 * - A fair comparison between CPUs as the task contribution (task_util()) 8294 * will always be the same no matter which CPU utilization we rely on 8295 * (util_avg or util_est). 8296 * 8297 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 8298 * exceed @eenv->pd_cap. 8299 */ 8300 static inline void eenv_pd_busy_time(struct energy_env *eenv, 8301 struct cpumask *pd_cpus, 8302 struct task_struct *p) 8303 { 8304 unsigned long busy_time = 0; 8305 int cpu; 8306 8307 for_each_cpu(cpu, pd_cpus) { 8308 unsigned long util = cpu_util(cpu, p, -1, 0); 8309 8310 busy_time += effective_cpu_util(cpu, util, NULL, NULL); 8311 } 8312 8313 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 8314 } 8315 8316 /* 8317 * Compute the maximum utilization for compute_energy() when the task @p 8318 * is placed on the cpu @dst_cpu. 8319 * 8320 * Returns the maximum utilization among @eenv->cpus. This utilization can't 8321 * exceed @eenv->cpu_cap. 8322 */ 8323 static inline unsigned long 8324 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 8325 struct task_struct *p, int dst_cpu) 8326 { 8327 unsigned long max_util = 0; 8328 int cpu; 8329 8330 for_each_cpu(cpu, pd_cpus) { 8331 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 8332 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 8333 unsigned long eff_util, min, max; 8334 8335 /* 8336 * Performance domain frequency: utilization clamping 8337 * must be considered since it affects the selection 8338 * of the performance domain frequency. 8339 * NOTE: in case RT tasks are running, by default the min 8340 * utilization can be max OPP. 8341 */ 8342 eff_util = effective_cpu_util(cpu, util, &min, &max); 8343 8344 /* Task's uclamp can modify min and max value */ 8345 if (tsk && uclamp_is_used()) { 8346 min = max(min, uclamp_eff_value(p, UCLAMP_MIN)); 8347 8348 /* 8349 * If there is no active max uclamp constraint, 8350 * directly use task's one, otherwise keep max. 8351 */ 8352 if (uclamp_rq_is_idle(cpu_rq(cpu))) 8353 max = uclamp_eff_value(p, UCLAMP_MAX); 8354 else 8355 max = max(max, uclamp_eff_value(p, UCLAMP_MAX)); 8356 } 8357 8358 eff_util = sugov_effective_cpu_perf(cpu, eff_util, min, max); 8359 max_util = max(max_util, eff_util); 8360 } 8361 8362 return min(max_util, eenv->cpu_cap); 8363 } 8364 8365 /* 8366 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 8367 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 8368 * contribution is ignored. 8369 */ 8370 static inline unsigned long 8371 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 8372 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 8373 { 8374 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 8375 unsigned long busy_time = eenv->pd_busy_time; 8376 unsigned long energy; 8377 8378 if (dst_cpu >= 0) 8379 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 8380 8381 energy = em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 8382 8383 trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time); 8384 8385 return energy; 8386 } 8387 8388 /* 8389 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 8390 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 8391 * spare capacity in each performance domain and uses it as a potential 8392 * candidate to execute the task. Then, it uses the Energy Model to figure 8393 * out which of the CPU candidates is the most energy-efficient. 8394 * 8395 * The rationale for this heuristic is as follows. In a performance domain, 8396 * all the most energy efficient CPU candidates (according to the Energy 8397 * Model) are those for which we'll request a low frequency. When there are 8398 * several CPUs for which the frequency request will be the same, we don't 8399 * have enough data to break the tie between them, because the Energy Model 8400 * only includes active power costs. With this model, if we assume that 8401 * frequency requests follow utilization (e.g. using schedutil), the CPU with 8402 * the maximum spare capacity in a performance domain is guaranteed to be among 8403 * the best candidates of the performance domain. 8404 * 8405 * In practice, it could be preferable from an energy standpoint to pack 8406 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 8407 * but that could also hurt our chances to go cluster idle, and we have no 8408 * ways to tell with the current Energy Model if this is actually a good 8409 * idea or not. So, find_energy_efficient_cpu() basically favors 8410 * cluster-packing, and spreading inside a cluster. That should at least be 8411 * a good thing for latency, and this is consistent with the idea that most 8412 * of the energy savings of EAS come from the asymmetry of the system, and 8413 * not so much from breaking the tie between identical CPUs. That's also the 8414 * reason why EAS is enabled in the topology code only for systems where 8415 * SD_ASYM_CPUCAPACITY is set. 8416 * 8417 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 8418 * they don't have any useful utilization data yet and it's not possible to 8419 * forecast their impact on energy consumption. Consequently, they will be 8420 * placed by sched_balance_find_dst_cpu() on the least loaded CPU, which might turn out 8421 * to be energy-inefficient in some use-cases. The alternative would be to 8422 * bias new tasks towards specific types of CPUs first, or to try to infer 8423 * their util_avg from the parent task, but those heuristics could hurt 8424 * other use-cases too. So, until someone finds a better way to solve this, 8425 * let's keep things simple by re-using the existing slow path. 8426 */ 8427 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 8428 { 8429 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 8430 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 8431 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 8432 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 8433 struct root_domain *rd = this_rq()->rd; 8434 int cpu, best_energy_cpu, target = -1; 8435 int prev_fits = -1, best_fits = -1; 8436 unsigned long best_actual_cap = 0; 8437 unsigned long prev_actual_cap = 0; 8438 struct sched_domain *sd; 8439 struct perf_domain *pd; 8440 struct energy_env eenv; 8441 8442 rcu_read_lock(); 8443 pd = rcu_dereference(rd->pd); 8444 if (!pd) 8445 goto unlock; 8446 8447 /* 8448 * Energy-aware wake-up happens on the lowest sched_domain starting 8449 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 8450 */ 8451 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 8452 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 8453 sd = sd->parent; 8454 if (!sd) 8455 goto unlock; 8456 8457 target = prev_cpu; 8458 8459 sync_entity_load_avg(&p->se); 8460 if (!task_util_est(p) && p_util_min == 0) 8461 goto unlock; 8462 8463 eenv_task_busy_time(&eenv, p, prev_cpu); 8464 8465 for (; pd; pd = pd->next) { 8466 unsigned long util_min = p_util_min, util_max = p_util_max; 8467 unsigned long cpu_cap, cpu_actual_cap, util; 8468 long prev_spare_cap = -1, max_spare_cap = -1; 8469 unsigned long rq_util_min, rq_util_max; 8470 unsigned long cur_delta, base_energy; 8471 int max_spare_cap_cpu = -1; 8472 int fits, max_fits = -1; 8473 8474 cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); 8475 8476 if (cpumask_empty(cpus)) 8477 continue; 8478 8479 /* Account external pressure for the energy estimation */ 8480 cpu = cpumask_first(cpus); 8481 cpu_actual_cap = get_actual_cpu_capacity(cpu); 8482 8483 eenv.cpu_cap = cpu_actual_cap; 8484 eenv.pd_cap = 0; 8485 8486 for_each_cpu(cpu, cpus) { 8487 struct rq *rq = cpu_rq(cpu); 8488 8489 eenv.pd_cap += cpu_actual_cap; 8490 8491 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 8492 continue; 8493 8494 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 8495 continue; 8496 8497 util = cpu_util(cpu, p, cpu, 0); 8498 cpu_cap = capacity_of(cpu); 8499 8500 /* 8501 * Skip CPUs that cannot satisfy the capacity request. 8502 * IOW, placing the task there would make the CPU 8503 * overutilized. Take uclamp into account to see how 8504 * much capacity we can get out of the CPU; this is 8505 * aligned with sched_cpu_util(). 8506 */ 8507 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 8508 /* 8509 * Open code uclamp_rq_util_with() except for 8510 * the clamp() part. I.e.: apply max aggregation 8511 * only. util_fits_cpu() logic requires to 8512 * operate on non clamped util but must use the 8513 * max-aggregated uclamp_{min, max}. 8514 */ 8515 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 8516 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 8517 8518 util_min = max(rq_util_min, p_util_min); 8519 util_max = max(rq_util_max, p_util_max); 8520 } 8521 8522 fits = util_fits_cpu(util, util_min, util_max, cpu); 8523 if (!fits) 8524 continue; 8525 8526 lsub_positive(&cpu_cap, util); 8527 8528 if (cpu == prev_cpu) { 8529 /* Always use prev_cpu as a candidate. */ 8530 prev_spare_cap = cpu_cap; 8531 prev_fits = fits; 8532 } else if ((fits > max_fits) || 8533 ((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) { 8534 /* 8535 * Find the CPU with the maximum spare capacity 8536 * among the remaining CPUs in the performance 8537 * domain. 8538 */ 8539 max_spare_cap = cpu_cap; 8540 max_spare_cap_cpu = cpu; 8541 max_fits = fits; 8542 } 8543 } 8544 8545 if (max_spare_cap_cpu < 0 && prev_spare_cap < 0) 8546 continue; 8547 8548 eenv_pd_busy_time(&eenv, cpus, p); 8549 /* Compute the 'base' energy of the pd, without @p */ 8550 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 8551 8552 /* Evaluate the energy impact of using prev_cpu. */ 8553 if (prev_spare_cap > -1) { 8554 prev_delta = compute_energy(&eenv, pd, cpus, p, 8555 prev_cpu); 8556 /* CPU utilization has changed */ 8557 if (prev_delta < base_energy) 8558 goto unlock; 8559 prev_delta -= base_energy; 8560 prev_actual_cap = cpu_actual_cap; 8561 best_delta = min(best_delta, prev_delta); 8562 } 8563 8564 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 8565 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 8566 /* Current best energy cpu fits better */ 8567 if (max_fits < best_fits) 8568 continue; 8569 8570 /* 8571 * Both don't fit performance hint (i.e. uclamp_min) 8572 * but best energy cpu has better capacity. 8573 */ 8574 if ((max_fits < 0) && 8575 (cpu_actual_cap <= best_actual_cap)) 8576 continue; 8577 8578 cur_delta = compute_energy(&eenv, pd, cpus, p, 8579 max_spare_cap_cpu); 8580 /* CPU utilization has changed */ 8581 if (cur_delta < base_energy) 8582 goto unlock; 8583 cur_delta -= base_energy; 8584 8585 /* 8586 * Both fit for the task but best energy cpu has lower 8587 * energy impact. 8588 */ 8589 if ((max_fits > 0) && (best_fits > 0) && 8590 (cur_delta >= best_delta)) 8591 continue; 8592 8593 best_delta = cur_delta; 8594 best_energy_cpu = max_spare_cap_cpu; 8595 best_fits = max_fits; 8596 best_actual_cap = cpu_actual_cap; 8597 } 8598 } 8599 rcu_read_unlock(); 8600 8601 if ((best_fits > prev_fits) || 8602 ((best_fits > 0) && (best_delta < prev_delta)) || 8603 ((best_fits < 0) && (best_actual_cap > prev_actual_cap))) 8604 target = best_energy_cpu; 8605 8606 return target; 8607 8608 unlock: 8609 rcu_read_unlock(); 8610 8611 return target; 8612 } 8613 8614 /* 8615 * select_task_rq_fair: Select target runqueue for the waking task in domains 8616 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 8617 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 8618 * 8619 * Balances load by selecting the idlest CPU in the idlest group, or under 8620 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 8621 * 8622 * Returns the target CPU number. 8623 */ 8624 static int 8625 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 8626 { 8627 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 8628 struct sched_domain *tmp, *sd = NULL; 8629 int cpu = smp_processor_id(); 8630 int new_cpu = prev_cpu; 8631 int want_affine = 0; 8632 /* SD_flags and WF_flags share the first nibble */ 8633 int sd_flag = wake_flags & 0xF; 8634 8635 /* 8636 * required for stable ->cpus_allowed 8637 */ 8638 lockdep_assert_held(&p->pi_lock); 8639 if (wake_flags & WF_TTWU) { 8640 record_wakee(p); 8641 8642 if ((wake_flags & WF_CURRENT_CPU) && 8643 cpumask_test_cpu(cpu, p->cpus_ptr)) 8644 return cpu; 8645 8646 if (!is_rd_overutilized(this_rq()->rd)) { 8647 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 8648 if (new_cpu >= 0) 8649 return new_cpu; 8650 new_cpu = prev_cpu; 8651 } 8652 8653 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 8654 } 8655 8656 rcu_read_lock(); 8657 for_each_domain(cpu, tmp) { 8658 /* 8659 * If both 'cpu' and 'prev_cpu' are part of this domain, 8660 * cpu is a valid SD_WAKE_AFFINE target. 8661 */ 8662 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 8663 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 8664 if (cpu != prev_cpu) 8665 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 8666 8667 sd = NULL; /* Prefer wake_affine over balance flags */ 8668 break; 8669 } 8670 8671 /* 8672 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 8673 * usually do not have SD_BALANCE_WAKE set. That means wakeup 8674 * will usually go to the fast path. 8675 */ 8676 if (tmp->flags & sd_flag) 8677 sd = tmp; 8678 else if (!want_affine) 8679 break; 8680 } 8681 8682 if (unlikely(sd)) { 8683 /* Slow path */ 8684 new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); 8685 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 8686 /* Fast path */ 8687 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 8688 } 8689 rcu_read_unlock(); 8690 8691 return new_cpu; 8692 } 8693 8694 /* 8695 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 8696 * cfs_rq_of(p) references at time of call are still valid and identify the 8697 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 8698 */ 8699 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 8700 { 8701 struct sched_entity *se = &p->se; 8702 8703 if (!task_on_rq_migrating(p)) { 8704 remove_entity_load_avg(se); 8705 8706 /* 8707 * Here, the task's PELT values have been updated according to 8708 * the current rq's clock. But if that clock hasn't been 8709 * updated in a while, a substantial idle time will be missed, 8710 * leading to an inflation after wake-up on the new rq. 8711 * 8712 * Estimate the missing time from the cfs_rq last_update_time 8713 * and update sched_avg to improve the PELT continuity after 8714 * migration. 8715 */ 8716 migrate_se_pelt_lag(se); 8717 } 8718 8719 /* Tell new CPU we are migrated */ 8720 se->avg.last_update_time = 0; 8721 8722 update_scan_period(p, new_cpu); 8723 } 8724 8725 static void task_dead_fair(struct task_struct *p) 8726 { 8727 struct sched_entity *se = &p->se; 8728 8729 if (se->sched_delayed) { 8730 struct rq_flags rf; 8731 struct rq *rq; 8732 8733 rq = task_rq_lock(p, &rf); 8734 if (se->sched_delayed) { 8735 update_rq_clock(rq); 8736 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 8737 } 8738 task_rq_unlock(rq, p, &rf); 8739 } 8740 8741 remove_entity_load_avg(se); 8742 } 8743 8744 /* 8745 * Set the max capacity the task is allowed to run at for misfit detection. 8746 */ 8747 static void set_task_max_allowed_capacity(struct task_struct *p) 8748 { 8749 struct asym_cap_data *entry; 8750 8751 if (!sched_asym_cpucap_active()) 8752 return; 8753 8754 rcu_read_lock(); 8755 list_for_each_entry_rcu(entry, &asym_cap_list, link) { 8756 cpumask_t *cpumask; 8757 8758 cpumask = cpu_capacity_span(entry); 8759 if (!cpumask_intersects(p->cpus_ptr, cpumask)) 8760 continue; 8761 8762 p->max_allowed_capacity = entry->capacity; 8763 break; 8764 } 8765 rcu_read_unlock(); 8766 } 8767 8768 static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context *ctx) 8769 { 8770 set_cpus_allowed_common(p, ctx); 8771 set_task_max_allowed_capacity(p); 8772 } 8773 8774 static int 8775 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8776 { 8777 if (sched_fair_runnable(rq)) 8778 return 1; 8779 8780 return sched_balance_newidle(rq, rf) != 0; 8781 } 8782 8783 static void set_next_buddy(struct sched_entity *se) 8784 { 8785 for_each_sched_entity(se) { 8786 if (WARN_ON_ONCE(!se->on_rq)) 8787 return; 8788 if (se_is_idle(se)) 8789 return; 8790 cfs_rq_of(se)->next = se; 8791 } 8792 } 8793 8794 enum preempt_wakeup_action { 8795 PREEMPT_WAKEUP_NONE, /* No preemption. */ 8796 PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ 8797 PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ 8798 PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ 8799 }; 8800 8801 static inline bool 8802 set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, 8803 struct sched_entity *pse, struct sched_entity *se) 8804 { 8805 /* 8806 * Keep existing buddy if the deadline is sooner than pse. 8807 * The older buddy may be cache cold and completely unrelated 8808 * to the current wakeup but that is unpredictable where as 8809 * obeying the deadline is more in line with EEVDF objectives. 8810 */ 8811 if (cfs_rq->next && entity_before(cfs_rq->next, pse)) 8812 return false; 8813 8814 set_next_buddy(pse); 8815 return true; 8816 } 8817 8818 /* 8819 * WF_SYNC|WF_TTWU indicates the waker expects to sleep but it is not 8820 * strictly enforced because the hint is either misunderstood or 8821 * multiple tasks must be woken up. 8822 */ 8823 static inline enum preempt_wakeup_action 8824 preempt_sync(struct rq *rq, int wake_flags, 8825 struct sched_entity *pse, struct sched_entity *se) 8826 { 8827 u64 threshold, delta; 8828 8829 /* 8830 * WF_SYNC without WF_TTWU is not expected so warn if it happens even 8831 * though it is likely harmless. 8832 */ 8833 WARN_ON_ONCE(!(wake_flags & WF_TTWU)); 8834 8835 threshold = sysctl_sched_migration_cost; 8836 delta = rq_clock_task(rq) - se->exec_start; 8837 if ((s64)delta < 0) 8838 delta = 0; 8839 8840 /* 8841 * WF_RQ_SELECTED implies the tasks are stacking on a CPU when they 8842 * could run on other CPUs. Reduce the threshold before preemption is 8843 * allowed to an arbitrary lower value as it is more likely (but not 8844 * guaranteed) the waker requires the wakee to finish. 8845 */ 8846 if (wake_flags & WF_RQ_SELECTED) 8847 threshold >>= 2; 8848 8849 /* 8850 * As WF_SYNC is not strictly obeyed, allow some runtime for batch 8851 * wakeups to be issued. 8852 */ 8853 if (entity_before(pse, se) && delta >= threshold) 8854 return PREEMPT_WAKEUP_RESCHED; 8855 8856 return PREEMPT_WAKEUP_NONE; 8857 } 8858 8859 /* 8860 * Preempt the current task with a newly woken task if needed: 8861 */ 8862 static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int wake_flags) 8863 { 8864 enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; 8865 struct task_struct *donor = rq->donor; 8866 struct sched_entity *nse, *se = &donor->se, *pse = &p->se; 8867 struct cfs_rq *cfs_rq = task_cfs_rq(donor); 8868 int cse_is_idle, pse_is_idle; 8869 8870 if (unlikely(se == pse)) 8871 return; 8872 8873 /* 8874 * This is possible from callers such as attach_tasks(), in which we 8875 * unconditionally wakeup_preempt() after an enqueue (which may have 8876 * lead to a throttle). This both saves work and prevents false 8877 * next-buddy nomination below. 8878 */ 8879 if (task_is_throttled(p)) 8880 return; 8881 8882 /* 8883 * We can come here with TIF_NEED_RESCHED already set from new task 8884 * wake up path. 8885 * 8886 * Note: this also catches the edge-case of curr being in a throttled 8887 * group (e.g. via set_curr_task), since update_curr() (in the 8888 * enqueue of curr) will have resulted in resched being set. This 8889 * prevents us from potentially nominating it as a false LAST_BUDDY 8890 * below. 8891 */ 8892 if (test_tsk_need_resched(rq->curr)) 8893 return; 8894 8895 if (!sched_feat(WAKEUP_PREEMPTION)) 8896 return; 8897 8898 find_matching_se(&se, &pse); 8899 WARN_ON_ONCE(!pse); 8900 8901 cse_is_idle = se_is_idle(se); 8902 pse_is_idle = se_is_idle(pse); 8903 8904 /* 8905 * Preempt an idle entity in favor of a non-idle entity (and don't preempt 8906 * in the inverse case). 8907 */ 8908 if (cse_is_idle && !pse_is_idle) { 8909 /* 8910 * When non-idle entity preempt an idle entity, 8911 * don't give idle entity slice protection. 8912 */ 8913 preempt_action = PREEMPT_WAKEUP_SHORT; 8914 goto preempt; 8915 } 8916 8917 if (cse_is_idle != pse_is_idle) 8918 return; 8919 8920 /* 8921 * BATCH and IDLE tasks do not preempt others. 8922 */ 8923 if (unlikely(!normal_policy(p->policy))) 8924 return; 8925 8926 cfs_rq = cfs_rq_of(se); 8927 update_curr(cfs_rq); 8928 /* 8929 * If @p has a shorter slice than current and @p is eligible, override 8930 * current's slice protection in order to allow preemption. 8931 */ 8932 if (sched_feat(PREEMPT_SHORT) && (pse->slice < se->slice)) { 8933 preempt_action = PREEMPT_WAKEUP_SHORT; 8934 goto pick; 8935 } 8936 8937 /* 8938 * Ignore wakee preemption on WF_FORK as it is less likely that 8939 * there is shared data as exec often follow fork. Do not 8940 * preempt for tasks that are sched_delayed as it would violate 8941 * EEVDF to forcibly queue an ineligible task. 8942 */ 8943 if ((wake_flags & WF_FORK) || pse->sched_delayed) 8944 return; 8945 8946 /* Prefer picking wakee soon if appropriate. */ 8947 if (sched_feat(NEXT_BUDDY) && 8948 set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { 8949 8950 /* 8951 * Decide whether to obey WF_SYNC hint for a new buddy. Old 8952 * buddies are ignored as they may not be relevant to the 8953 * waker and less likely to be cache hot. 8954 */ 8955 if (wake_flags & WF_SYNC) 8956 preempt_action = preempt_sync(rq, wake_flags, pse, se); 8957 } 8958 8959 switch (preempt_action) { 8960 case PREEMPT_WAKEUP_NONE: 8961 return; 8962 case PREEMPT_WAKEUP_RESCHED: 8963 goto preempt; 8964 case PREEMPT_WAKEUP_SHORT: 8965 fallthrough; 8966 case PREEMPT_WAKEUP_PICK: 8967 break; 8968 } 8969 8970 pick: 8971 nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); 8972 /* If @p has become the most eligible task, force preemption */ 8973 if (nse == pse) 8974 goto preempt; 8975 8976 /* 8977 * Because p is enqueued, nse being null can only mean that we 8978 * dequeued a delayed task. If there are still entities queued in 8979 * cfs, check if the next one will be p. 8980 */ 8981 if (!nse && cfs_rq->nr_queued) 8982 goto pick; 8983 8984 if (sched_feat(RUN_TO_PARITY)) 8985 update_protect_slice(cfs_rq, se); 8986 8987 return; 8988 8989 preempt: 8990 if (preempt_action == PREEMPT_WAKEUP_SHORT) 8991 cancel_protect_slice(se); 8992 8993 resched_curr_lazy(rq); 8994 } 8995 8996 static struct task_struct *pick_task_fair(struct rq *rq) 8997 { 8998 struct sched_entity *se; 8999 struct cfs_rq *cfs_rq; 9000 struct task_struct *p; 9001 bool throttled; 9002 9003 again: 9004 cfs_rq = &rq->cfs; 9005 if (!cfs_rq->nr_queued) 9006 return NULL; 9007 9008 throttled = false; 9009 9010 do { 9011 /* Might not have done put_prev_entity() */ 9012 if (cfs_rq->curr && cfs_rq->curr->on_rq) 9013 update_curr(cfs_rq); 9014 9015 throttled |= check_cfs_rq_runtime(cfs_rq); 9016 9017 se = pick_next_entity(rq, cfs_rq, true); 9018 if (!se) 9019 goto again; 9020 cfs_rq = group_cfs_rq(se); 9021 } while (cfs_rq); 9022 9023 p = task_of(se); 9024 if (unlikely(throttled)) 9025 task_throttle_setup_work(p); 9026 return p; 9027 } 9028 9029 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 9030 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 9031 9032 struct task_struct * 9033 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 9034 { 9035 struct sched_entity *se; 9036 struct task_struct *p; 9037 int new_tasks; 9038 9039 again: 9040 p = pick_task_fair(rq); 9041 if (!p) 9042 goto idle; 9043 se = &p->se; 9044 9045 #ifdef CONFIG_FAIR_GROUP_SCHED 9046 if (prev->sched_class != &fair_sched_class) 9047 goto simple; 9048 9049 __put_prev_set_next_dl_server(rq, prev, p); 9050 9051 /* 9052 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 9053 * likely that a next task is from the same cgroup as the current. 9054 * 9055 * Therefore attempt to avoid putting and setting the entire cgroup 9056 * hierarchy, only change the part that actually changes. 9057 * 9058 * Since we haven't yet done put_prev_entity and if the selected task 9059 * is a different task than we started out with, try and touch the 9060 * least amount of cfs_rqs. 9061 */ 9062 if (prev != p) { 9063 struct sched_entity *pse = &prev->se; 9064 struct cfs_rq *cfs_rq; 9065 9066 while (!(cfs_rq = is_same_group(se, pse))) { 9067 int se_depth = se->depth; 9068 int pse_depth = pse->depth; 9069 9070 if (se_depth <= pse_depth) { 9071 put_prev_entity(cfs_rq_of(pse), pse); 9072 pse = parent_entity(pse); 9073 } 9074 if (se_depth >= pse_depth) { 9075 set_next_entity(cfs_rq_of(se), se, true); 9076 se = parent_entity(se); 9077 } 9078 } 9079 9080 put_prev_entity(cfs_rq, pse); 9081 set_next_entity(cfs_rq, se, true); 9082 9083 __set_next_task_fair(rq, p, true); 9084 } 9085 9086 return p; 9087 9088 simple: 9089 #endif /* CONFIG_FAIR_GROUP_SCHED */ 9090 put_prev_set_next_task(rq, prev, p); 9091 return p; 9092 9093 idle: 9094 if (rf) { 9095 new_tasks = sched_balance_newidle(rq, rf); 9096 9097 /* 9098 * Because sched_balance_newidle() releases (and re-acquires) 9099 * rq->lock, it is possible for any higher priority task to 9100 * appear. In that case we must re-start the pick_next_entity() 9101 * loop. 9102 */ 9103 if (new_tasks < 0) 9104 return RETRY_TASK; 9105 9106 if (new_tasks > 0) 9107 goto again; 9108 } 9109 9110 return NULL; 9111 } 9112 9113 static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev) 9114 { 9115 return pick_next_task_fair(rq, prev, NULL); 9116 } 9117 9118 static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se) 9119 { 9120 return pick_task_fair(dl_se->rq); 9121 } 9122 9123 void fair_server_init(struct rq *rq) 9124 { 9125 struct sched_dl_entity *dl_se = &rq->fair_server; 9126 9127 init_dl_entity(dl_se); 9128 9129 dl_server_init(dl_se, rq, fair_server_pick_task); 9130 } 9131 9132 /* 9133 * Account for a descheduled task: 9134 */ 9135 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) 9136 { 9137 struct sched_entity *se = &prev->se; 9138 struct cfs_rq *cfs_rq; 9139 9140 for_each_sched_entity(se) { 9141 cfs_rq = cfs_rq_of(se); 9142 put_prev_entity(cfs_rq, se); 9143 } 9144 } 9145 9146 /* 9147 * sched_yield() is very simple 9148 */ 9149 static void yield_task_fair(struct rq *rq) 9150 { 9151 struct task_struct *curr = rq->donor; 9152 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 9153 struct sched_entity *se = &curr->se; 9154 9155 /* 9156 * Are we the only task in the tree? 9157 */ 9158 if (unlikely(rq->nr_running == 1)) 9159 return; 9160 9161 clear_buddies(cfs_rq, se); 9162 9163 update_rq_clock(rq); 9164 /* 9165 * Update run-time statistics of the 'current'. 9166 */ 9167 update_curr(cfs_rq); 9168 /* 9169 * Tell update_rq_clock() that we've just updated, 9170 * so we don't do microscopic update in schedule() 9171 * and double the fastpath cost. 9172 */ 9173 rq_clock_skip_update(rq); 9174 9175 /* 9176 * Forfeit the remaining vruntime, only if the entity is eligible. This 9177 * condition is necessary because in core scheduling we prefer to run 9178 * ineligible tasks rather than force idling. If this happens we may 9179 * end up in a loop where the core scheduler picks the yielding task, 9180 * which yields immediately again; without the condition the vruntime 9181 * ends up quickly running away. 9182 */ 9183 if (entity_eligible(cfs_rq, se)) { 9184 se->vruntime = se->deadline; 9185 update_deadline(cfs_rq, se); 9186 } 9187 } 9188 9189 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 9190 { 9191 struct sched_entity *se = &p->se; 9192 9193 /* !se->on_rq also covers throttled task */ 9194 if (!se->on_rq) 9195 return false; 9196 9197 /* Tell the scheduler that we'd really like se to run next. */ 9198 set_next_buddy(se); 9199 9200 yield_task_fair(rq); 9201 9202 return true; 9203 } 9204 9205 /************************************************** 9206 * Fair scheduling class load-balancing methods. 9207 * 9208 * BASICS 9209 * 9210 * The purpose of load-balancing is to achieve the same basic fairness the 9211 * per-CPU scheduler provides, namely provide a proportional amount of compute 9212 * time to each task. This is expressed in the following equation: 9213 * 9214 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 9215 * 9216 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 9217 * W_i,0 is defined as: 9218 * 9219 * W_i,0 = \Sum_j w_i,j (2) 9220 * 9221 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 9222 * is derived from the nice value as per sched_prio_to_weight[]. 9223 * 9224 * The weight average is an exponential decay average of the instantaneous 9225 * weight: 9226 * 9227 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 9228 * 9229 * C_i is the compute capacity of CPU i, typically it is the 9230 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 9231 * can also include other factors [XXX]. 9232 * 9233 * To achieve this balance we define a measure of imbalance which follows 9234 * directly from (1): 9235 * 9236 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 9237 * 9238 * We them move tasks around to minimize the imbalance. In the continuous 9239 * function space it is obvious this converges, in the discrete case we get 9240 * a few fun cases generally called infeasible weight scenarios. 9241 * 9242 * [XXX expand on: 9243 * - infeasible weights; 9244 * - local vs global optima in the discrete case. ] 9245 * 9246 * 9247 * SCHED DOMAINS 9248 * 9249 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 9250 * for all i,j solution, we create a tree of CPUs that follows the hardware 9251 * topology where each level pairs two lower groups (or better). This results 9252 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 9253 * tree to only the first of the previous level and we decrease the frequency 9254 * of load-balance at each level inversely proportional to the number of CPUs in 9255 * the groups. 9256 * 9257 * This yields: 9258 * 9259 * log_2 n 1 n 9260 * \Sum { --- * --- * 2^i } = O(n) (5) 9261 * i = 0 2^i 2^i 9262 * `- size of each group 9263 * | | `- number of CPUs doing load-balance 9264 * | `- freq 9265 * `- sum over all levels 9266 * 9267 * Coupled with a limit on how many tasks we can migrate every balance pass, 9268 * this makes (5) the runtime complexity of the balancer. 9269 * 9270 * An important property here is that each CPU is still (indirectly) connected 9271 * to every other CPU in at most O(log n) steps: 9272 * 9273 * The adjacency matrix of the resulting graph is given by: 9274 * 9275 * log_2 n 9276 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 9277 * k = 0 9278 * 9279 * And you'll find that: 9280 * 9281 * A^(log_2 n)_i,j != 0 for all i,j (7) 9282 * 9283 * Showing there's indeed a path between every CPU in at most O(log n) steps. 9284 * The task movement gives a factor of O(m), giving a convergence complexity 9285 * of: 9286 * 9287 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 9288 * 9289 * 9290 * WORK CONSERVING 9291 * 9292 * In order to avoid CPUs going idle while there's still work to do, new idle 9293 * balancing is more aggressive and has the newly idle CPU iterate up the domain 9294 * tree itself instead of relying on other CPUs to bring it work. 9295 * 9296 * This adds some complexity to both (5) and (8) but it reduces the total idle 9297 * time. 9298 * 9299 * [XXX more?] 9300 * 9301 * 9302 * CGROUPS 9303 * 9304 * Cgroups make a horror show out of (2), instead of a simple sum we get: 9305 * 9306 * s_k,i 9307 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 9308 * S_k 9309 * 9310 * Where 9311 * 9312 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 9313 * 9314 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 9315 * 9316 * The big problem is S_k, its a global sum needed to compute a local (W_i) 9317 * property. 9318 * 9319 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 9320 * rewrite all of this once again.] 9321 */ 9322 9323 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 9324 9325 enum fbq_type { regular, remote, all }; 9326 9327 /* 9328 * 'group_type' describes the group of CPUs at the moment of load balancing. 9329 * 9330 * The enum is ordered by pulling priority, with the group with lowest priority 9331 * first so the group_type can simply be compared when selecting the busiest 9332 * group. See update_sd_pick_busiest(). 9333 */ 9334 enum group_type { 9335 /* The group has spare capacity that can be used to run more tasks. */ 9336 group_has_spare = 0, 9337 /* 9338 * The group is fully used and the tasks don't compete for more CPU 9339 * cycles. Nevertheless, some tasks might wait before running. 9340 */ 9341 group_fully_busy, 9342 /* 9343 * One task doesn't fit with CPU's capacity and must be migrated to a 9344 * more powerful CPU. 9345 */ 9346 group_misfit_task, 9347 /* 9348 * Balance SMT group that's fully busy. Can benefit from migration 9349 * a task on SMT with busy sibling to another CPU on idle core. 9350 */ 9351 group_smt_balance, 9352 /* 9353 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 9354 * and the task should be migrated to it instead of running on the 9355 * current CPU. 9356 */ 9357 group_asym_packing, 9358 /* 9359 * The tasks' affinity constraints previously prevented the scheduler 9360 * from balancing the load across the system. 9361 */ 9362 group_imbalanced, 9363 /* 9364 * The CPU is overloaded and can't provide expected CPU cycles to all 9365 * tasks. 9366 */ 9367 group_overloaded 9368 }; 9369 9370 enum migration_type { 9371 migrate_load = 0, 9372 migrate_util, 9373 migrate_task, 9374 migrate_misfit 9375 }; 9376 9377 #define LBF_ALL_PINNED 0x01 9378 #define LBF_NEED_BREAK 0x02 9379 #define LBF_DST_PINNED 0x04 9380 #define LBF_SOME_PINNED 0x08 9381 #define LBF_ACTIVE_LB 0x10 9382 9383 struct lb_env { 9384 struct sched_domain *sd; 9385 9386 struct rq *src_rq; 9387 int src_cpu; 9388 9389 int dst_cpu; 9390 struct rq *dst_rq; 9391 9392 struct cpumask *dst_grpmask; 9393 int new_dst_cpu; 9394 enum cpu_idle_type idle; 9395 long imbalance; 9396 /* The set of CPUs under consideration for load-balancing */ 9397 struct cpumask *cpus; 9398 9399 unsigned int flags; 9400 9401 unsigned int loop; 9402 unsigned int loop_break; 9403 unsigned int loop_max; 9404 9405 enum fbq_type fbq_type; 9406 enum migration_type migration_type; 9407 struct list_head tasks; 9408 }; 9409 9410 /* 9411 * Is this task likely cache-hot: 9412 */ 9413 static int task_hot(struct task_struct *p, struct lb_env *env) 9414 { 9415 s64 delta; 9416 9417 lockdep_assert_rq_held(env->src_rq); 9418 9419 if (p->sched_class != &fair_sched_class) 9420 return 0; 9421 9422 if (unlikely(task_has_idle_policy(p))) 9423 return 0; 9424 9425 /* SMT siblings share cache */ 9426 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 9427 return 0; 9428 9429 /* 9430 * Buddy candidates are cache hot: 9431 */ 9432 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 9433 (&p->se == cfs_rq_of(&p->se)->next)) 9434 return 1; 9435 9436 if (sysctl_sched_migration_cost == -1) 9437 return 1; 9438 9439 /* 9440 * Don't migrate task if the task's cookie does not match 9441 * with the destination CPU's core cookie. 9442 */ 9443 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 9444 return 1; 9445 9446 if (sysctl_sched_migration_cost == 0) 9447 return 0; 9448 9449 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 9450 9451 return delta < (s64)sysctl_sched_migration_cost; 9452 } 9453 9454 #ifdef CONFIG_NUMA_BALANCING 9455 /* 9456 * Returns a positive value, if task migration degrades locality. 9457 * Returns 0, if task migration is not affected by locality. 9458 * Returns a negative value, if task migration improves locality i.e migration preferred. 9459 */ 9460 static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 9461 { 9462 struct numa_group *numa_group = rcu_dereference(p->numa_group); 9463 unsigned long src_weight, dst_weight; 9464 int src_nid, dst_nid, dist; 9465 9466 if (!static_branch_likely(&sched_numa_balancing)) 9467 return 0; 9468 9469 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 9470 return 0; 9471 9472 src_nid = cpu_to_node(env->src_cpu); 9473 dst_nid = cpu_to_node(env->dst_cpu); 9474 9475 if (src_nid == dst_nid) 9476 return 0; 9477 9478 /* Migrating away from the preferred node is always bad. */ 9479 if (src_nid == p->numa_preferred_nid) { 9480 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 9481 return 1; 9482 else 9483 return 0; 9484 } 9485 9486 /* Encourage migration to the preferred node. */ 9487 if (dst_nid == p->numa_preferred_nid) 9488 return -1; 9489 9490 /* Leaving a core idle is often worse than degrading locality. */ 9491 if (env->idle == CPU_IDLE) 9492 return 0; 9493 9494 dist = node_distance(src_nid, dst_nid); 9495 if (numa_group) { 9496 src_weight = group_weight(p, src_nid, dist); 9497 dst_weight = group_weight(p, dst_nid, dist); 9498 } else { 9499 src_weight = task_weight(p, src_nid, dist); 9500 dst_weight = task_weight(p, dst_nid, dist); 9501 } 9502 9503 return src_weight - dst_weight; 9504 } 9505 9506 #else /* !CONFIG_NUMA_BALANCING: */ 9507 static inline long migrate_degrades_locality(struct task_struct *p, 9508 struct lb_env *env) 9509 { 9510 return 0; 9511 } 9512 #endif /* !CONFIG_NUMA_BALANCING */ 9513 9514 /* 9515 * Check whether the task is ineligible on the destination cpu 9516 * 9517 * When the PLACE_LAG scheduling feature is enabled and 9518 * dst_cfs_rq->nr_queued is greater than 1, if the task 9519 * is ineligible, it will also be ineligible when 9520 * it is migrated to the destination cpu. 9521 */ 9522 static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) 9523 { 9524 struct cfs_rq *dst_cfs_rq; 9525 9526 #ifdef CONFIG_FAIR_GROUP_SCHED 9527 dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu]; 9528 #else 9529 dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; 9530 #endif 9531 if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && 9532 !entity_eligible(task_cfs_rq(p), &p->se)) 9533 return 1; 9534 9535 return 0; 9536 } 9537 9538 /* 9539 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 9540 */ 9541 static 9542 int can_migrate_task(struct task_struct *p, struct lb_env *env) 9543 { 9544 long degrades, hot; 9545 9546 lockdep_assert_rq_held(env->src_rq); 9547 if (p->sched_task_hot) 9548 p->sched_task_hot = 0; 9549 9550 /* 9551 * We do not migrate tasks that are: 9552 * 1) delayed dequeued unless we migrate load, or 9553 * 2) target cfs_rq is in throttled hierarchy, or 9554 * 3) cannot be migrated to this CPU due to cpus_ptr, or 9555 * 4) running (obviously), or 9556 * 5) are cache-hot on their current CPU, or 9557 * 6) are blocked on mutexes (if SCHED_PROXY_EXEC is enabled) 9558 */ 9559 if ((p->se.sched_delayed) && (env->migration_type != migrate_load)) 9560 return 0; 9561 9562 if (lb_throttled_hierarchy(p, env->dst_cpu)) 9563 return 0; 9564 9565 /* 9566 * We want to prioritize the migration of eligible tasks. 9567 * For ineligible tasks we soft-limit them and only allow 9568 * them to migrate when nr_balance_failed is non-zero to 9569 * avoid load-balancing trying very hard to balance the load. 9570 */ 9571 if (!env->sd->nr_balance_failed && 9572 task_is_ineligible_on_dst_cpu(p, env->dst_cpu)) 9573 return 0; 9574 9575 /* Disregard percpu kthreads; they are where they need to be. */ 9576 if (kthread_is_per_cpu(p)) 9577 return 0; 9578 9579 if (task_is_blocked(p)) 9580 return 0; 9581 9582 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 9583 int cpu; 9584 9585 schedstat_inc(p->stats.nr_failed_migrations_affine); 9586 9587 env->flags |= LBF_SOME_PINNED; 9588 9589 /* 9590 * Remember if this task can be migrated to any other CPU in 9591 * our sched_group. We may want to revisit it if we couldn't 9592 * meet load balance goals by pulling other tasks on src_cpu. 9593 * 9594 * Avoid computing new_dst_cpu 9595 * - for NEWLY_IDLE 9596 * - if we have already computed one in current iteration 9597 * - if it's an active balance 9598 */ 9599 if (env->idle == CPU_NEWLY_IDLE || 9600 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 9601 return 0; 9602 9603 /* Prevent to re-select dst_cpu via env's CPUs: */ 9604 cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr); 9605 9606 if (cpu < nr_cpu_ids) { 9607 env->flags |= LBF_DST_PINNED; 9608 env->new_dst_cpu = cpu; 9609 } 9610 9611 return 0; 9612 } 9613 9614 /* Record that we found at least one task that could run on dst_cpu */ 9615 env->flags &= ~LBF_ALL_PINNED; 9616 9617 if (task_on_cpu(env->src_rq, p) || 9618 task_current_donor(env->src_rq, p)) { 9619 schedstat_inc(p->stats.nr_failed_migrations_running); 9620 return 0; 9621 } 9622 9623 /* 9624 * Aggressive migration if: 9625 * 1) active balance 9626 * 2) destination numa is preferred 9627 * 3) task is cache cold, or 9628 * 4) too many balance attempts have failed. 9629 */ 9630 if (env->flags & LBF_ACTIVE_LB) 9631 return 1; 9632 9633 degrades = migrate_degrades_locality(p, env); 9634 if (!degrades) 9635 hot = task_hot(p, env); 9636 else 9637 hot = degrades > 0; 9638 9639 if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 9640 if (hot) 9641 p->sched_task_hot = 1; 9642 return 1; 9643 } 9644 9645 schedstat_inc(p->stats.nr_failed_migrations_hot); 9646 return 0; 9647 } 9648 9649 /* 9650 * detach_task() -- detach the task for the migration specified in env 9651 */ 9652 static void detach_task(struct task_struct *p, struct lb_env *env) 9653 { 9654 lockdep_assert_rq_held(env->src_rq); 9655 9656 if (p->sched_task_hot) { 9657 p->sched_task_hot = 0; 9658 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 9659 schedstat_inc(p->stats.nr_forced_migrations); 9660 } 9661 9662 WARN_ON(task_current(env->src_rq, p)); 9663 WARN_ON(task_current_donor(env->src_rq, p)); 9664 9665 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 9666 set_task_cpu(p, env->dst_cpu); 9667 } 9668 9669 /* 9670 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 9671 * part of active balancing operations within "domain". 9672 * 9673 * Returns a task if successful and NULL otherwise. 9674 */ 9675 static struct task_struct *detach_one_task(struct lb_env *env) 9676 { 9677 struct task_struct *p; 9678 9679 lockdep_assert_rq_held(env->src_rq); 9680 9681 list_for_each_entry_reverse(p, 9682 &env->src_rq->cfs_tasks, se.group_node) { 9683 if (!can_migrate_task(p, env)) 9684 continue; 9685 9686 detach_task(p, env); 9687 9688 /* 9689 * Right now, this is only the second place where 9690 * lb_gained[env->idle] is updated (other is detach_tasks) 9691 * so we can safely collect stats here rather than 9692 * inside detach_tasks(). 9693 */ 9694 schedstat_inc(env->sd->lb_gained[env->idle]); 9695 return p; 9696 } 9697 return NULL; 9698 } 9699 9700 /* 9701 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 9702 * busiest_rq, as part of a balancing operation within domain "sd". 9703 * 9704 * Returns number of detached tasks if successful and 0 otherwise. 9705 */ 9706 static int detach_tasks(struct lb_env *env) 9707 { 9708 struct list_head *tasks = &env->src_rq->cfs_tasks; 9709 unsigned long util, load; 9710 struct task_struct *p; 9711 int detached = 0; 9712 9713 lockdep_assert_rq_held(env->src_rq); 9714 9715 /* 9716 * Source run queue has been emptied by another CPU, clear 9717 * LBF_ALL_PINNED flag as we will not test any task. 9718 */ 9719 if (env->src_rq->nr_running <= 1) { 9720 env->flags &= ~LBF_ALL_PINNED; 9721 return 0; 9722 } 9723 9724 if (env->imbalance <= 0) 9725 return 0; 9726 9727 while (!list_empty(tasks)) { 9728 /* 9729 * We don't want to steal all, otherwise we may be treated likewise, 9730 * which could at worst lead to a livelock crash. 9731 */ 9732 if (env->idle && env->src_rq->nr_running <= 1) 9733 break; 9734 9735 env->loop++; 9736 /* We've more or less seen every task there is, call it quits */ 9737 if (env->loop > env->loop_max) 9738 break; 9739 9740 /* take a breather every nr_migrate tasks */ 9741 if (env->loop > env->loop_break) { 9742 env->loop_break += SCHED_NR_MIGRATE_BREAK; 9743 env->flags |= LBF_NEED_BREAK; 9744 break; 9745 } 9746 9747 p = list_last_entry(tasks, struct task_struct, se.group_node); 9748 9749 if (!can_migrate_task(p, env)) 9750 goto next; 9751 9752 switch (env->migration_type) { 9753 case migrate_load: 9754 /* 9755 * Depending of the number of CPUs and tasks and the 9756 * cgroup hierarchy, task_h_load() can return a null 9757 * value. Make sure that env->imbalance decreases 9758 * otherwise detach_tasks() will stop only after 9759 * detaching up to loop_max tasks. 9760 */ 9761 load = max_t(unsigned long, task_h_load(p), 1); 9762 9763 if (sched_feat(LB_MIN) && 9764 load < 16 && !env->sd->nr_balance_failed) 9765 goto next; 9766 9767 /* 9768 * Make sure that we don't migrate too much load. 9769 * Nevertheless, let relax the constraint if 9770 * scheduler fails to find a good waiting task to 9771 * migrate. 9772 */ 9773 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 9774 goto next; 9775 9776 env->imbalance -= load; 9777 break; 9778 9779 case migrate_util: 9780 util = task_util_est(p); 9781 9782 if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance) 9783 goto next; 9784 9785 env->imbalance -= util; 9786 break; 9787 9788 case migrate_task: 9789 env->imbalance--; 9790 break; 9791 9792 case migrate_misfit: 9793 /* This is not a misfit task */ 9794 if (task_fits_cpu(p, env->src_cpu)) 9795 goto next; 9796 9797 env->imbalance = 0; 9798 break; 9799 } 9800 9801 detach_task(p, env); 9802 list_add(&p->se.group_node, &env->tasks); 9803 9804 detached++; 9805 9806 #ifdef CONFIG_PREEMPTION 9807 /* 9808 * NEWIDLE balancing is a source of latency, so preemptible 9809 * kernels will stop after the first task is detached to minimize 9810 * the critical section. 9811 */ 9812 if (env->idle == CPU_NEWLY_IDLE) 9813 break; 9814 #endif 9815 9816 /* 9817 * We only want to steal up to the prescribed amount of 9818 * load/util/tasks. 9819 */ 9820 if (env->imbalance <= 0) 9821 break; 9822 9823 continue; 9824 next: 9825 if (p->sched_task_hot) 9826 schedstat_inc(p->stats.nr_failed_migrations_hot); 9827 9828 list_move(&p->se.group_node, tasks); 9829 } 9830 9831 /* 9832 * Right now, this is one of only two places we collect this stat 9833 * so we can safely collect detach_one_task() stats here rather 9834 * than inside detach_one_task(). 9835 */ 9836 schedstat_add(env->sd->lb_gained[env->idle], detached); 9837 9838 return detached; 9839 } 9840 9841 /* 9842 * attach_task() -- attach the task detached by detach_task() to its new rq. 9843 */ 9844 static void attach_task(struct rq *rq, struct task_struct *p) 9845 { 9846 lockdep_assert_rq_held(rq); 9847 9848 WARN_ON_ONCE(task_rq(p) != rq); 9849 activate_task(rq, p, ENQUEUE_NOCLOCK); 9850 wakeup_preempt(rq, p, 0); 9851 } 9852 9853 /* 9854 * attach_one_task() -- attaches the task returned from detach_one_task() to 9855 * its new rq. 9856 */ 9857 static void attach_one_task(struct rq *rq, struct task_struct *p) 9858 { 9859 struct rq_flags rf; 9860 9861 rq_lock(rq, &rf); 9862 update_rq_clock(rq); 9863 attach_task(rq, p); 9864 rq_unlock(rq, &rf); 9865 } 9866 9867 /* 9868 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 9869 * new rq. 9870 */ 9871 static void attach_tasks(struct lb_env *env) 9872 { 9873 struct list_head *tasks = &env->tasks; 9874 struct task_struct *p; 9875 struct rq_flags rf; 9876 9877 rq_lock(env->dst_rq, &rf); 9878 update_rq_clock(env->dst_rq); 9879 9880 while (!list_empty(tasks)) { 9881 p = list_first_entry(tasks, struct task_struct, se.group_node); 9882 list_del_init(&p->se.group_node); 9883 9884 attach_task(env->dst_rq, p); 9885 } 9886 9887 rq_unlock(env->dst_rq, &rf); 9888 } 9889 9890 #ifdef CONFIG_NO_HZ_COMMON 9891 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 9892 { 9893 if (cfs_rq->avg.load_avg) 9894 return true; 9895 9896 if (cfs_rq->avg.util_avg) 9897 return true; 9898 9899 return false; 9900 } 9901 9902 static inline bool others_have_blocked(struct rq *rq) 9903 { 9904 if (cpu_util_rt(rq)) 9905 return true; 9906 9907 if (cpu_util_dl(rq)) 9908 return true; 9909 9910 if (hw_load_avg(rq)) 9911 return true; 9912 9913 if (cpu_util_irq(rq)) 9914 return true; 9915 9916 return false; 9917 } 9918 9919 static inline void update_blocked_load_tick(struct rq *rq) 9920 { 9921 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 9922 } 9923 9924 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 9925 { 9926 if (!has_blocked) 9927 rq->has_blocked_load = 0; 9928 } 9929 #else /* !CONFIG_NO_HZ_COMMON: */ 9930 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 9931 static inline bool others_have_blocked(struct rq *rq) { return false; } 9932 static inline void update_blocked_load_tick(struct rq *rq) {} 9933 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 9934 #endif /* !CONFIG_NO_HZ_COMMON */ 9935 9936 static bool __update_blocked_others(struct rq *rq, bool *done) 9937 { 9938 bool updated; 9939 9940 /* 9941 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 9942 * DL and IRQ signals have been updated before updating CFS. 9943 */ 9944 updated = update_other_load_avgs(rq); 9945 9946 if (others_have_blocked(rq)) 9947 *done = false; 9948 9949 return updated; 9950 } 9951 9952 #ifdef CONFIG_FAIR_GROUP_SCHED 9953 9954 static bool __update_blocked_fair(struct rq *rq, bool *done) 9955 { 9956 struct cfs_rq *cfs_rq, *pos; 9957 bool decayed = false; 9958 int cpu = cpu_of(rq); 9959 9960 /* 9961 * Iterates the task_group tree in a bottom up fashion, see 9962 * list_add_leaf_cfs_rq() for details. 9963 */ 9964 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 9965 struct sched_entity *se; 9966 9967 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9968 update_tg_load_avg(cfs_rq); 9969 9970 if (cfs_rq->nr_queued == 0) 9971 update_idle_cfs_rq_clock_pelt(cfs_rq); 9972 9973 if (cfs_rq == &rq->cfs) 9974 decayed = true; 9975 } 9976 9977 /* Propagate pending load changes to the parent, if any: */ 9978 se = cfs_rq->tg->se[cpu]; 9979 if (se && !skip_blocked_update(se)) 9980 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9981 9982 /* 9983 * There can be a lot of idle CPU cgroups. Don't let fully 9984 * decayed cfs_rqs linger on the list. 9985 */ 9986 if (cfs_rq_is_decayed(cfs_rq)) 9987 list_del_leaf_cfs_rq(cfs_rq); 9988 9989 /* Don't need periodic decay once load/util_avg are null */ 9990 if (cfs_rq_has_blocked(cfs_rq)) 9991 *done = false; 9992 } 9993 9994 return decayed; 9995 } 9996 9997 /* 9998 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9999 * This needs to be done in a top-down fashion because the load of a child 10000 * group is a fraction of its parents load. 10001 */ 10002 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) 10003 { 10004 struct rq *rq = rq_of(cfs_rq); 10005 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 10006 unsigned long now = jiffies; 10007 unsigned long load; 10008 10009 if (cfs_rq->last_h_load_update == now) 10010 return; 10011 10012 WRITE_ONCE(cfs_rq->h_load_next, NULL); 10013 for_each_sched_entity(se) { 10014 cfs_rq = cfs_rq_of(se); 10015 WRITE_ONCE(cfs_rq->h_load_next, se); 10016 if (cfs_rq->last_h_load_update == now) 10017 break; 10018 } 10019 10020 if (!se) { 10021 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq); 10022 cfs_rq->last_h_load_update = now; 10023 } 10024 10025 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { 10026 load = cfs_rq->h_load; 10027 load = div64_ul(load * se->avg.load_avg, 10028 cfs_rq_load_avg(cfs_rq) + 1); 10029 cfs_rq = group_cfs_rq(se); 10030 cfs_rq->h_load = load; 10031 cfs_rq->last_h_load_update = now; 10032 } 10033 } 10034 10035 static unsigned long task_h_load(struct task_struct *p) 10036 { 10037 struct cfs_rq *cfs_rq = task_cfs_rq(p); 10038 10039 update_cfs_rq_h_load(cfs_rq); 10040 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load, 10041 cfs_rq_load_avg(cfs_rq) + 1); 10042 } 10043 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 10044 static bool __update_blocked_fair(struct rq *rq, bool *done) 10045 { 10046 struct cfs_rq *cfs_rq = &rq->cfs; 10047 bool decayed; 10048 10049 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq); 10050 if (cfs_rq_has_blocked(cfs_rq)) 10051 *done = false; 10052 10053 return decayed; 10054 } 10055 10056 static unsigned long task_h_load(struct task_struct *p) 10057 { 10058 return p->se.avg.load_avg; 10059 } 10060 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 10061 10062 static void sched_balance_update_blocked_averages(int cpu) 10063 { 10064 bool decayed = false, done = true; 10065 struct rq *rq = cpu_rq(cpu); 10066 struct rq_flags rf; 10067 10068 rq_lock_irqsave(rq, &rf); 10069 update_blocked_load_tick(rq); 10070 update_rq_clock(rq); 10071 10072 decayed |= __update_blocked_others(rq, &done); 10073 decayed |= __update_blocked_fair(rq, &done); 10074 10075 update_blocked_load_status(rq, !done); 10076 if (decayed) 10077 cpufreq_update_util(rq, 0); 10078 rq_unlock_irqrestore(rq, &rf); 10079 } 10080 10081 /********** Helpers for sched_balance_find_src_group ************************/ 10082 10083 /* 10084 * sg_lb_stats - stats of a sched_group required for load-balancing: 10085 */ 10086 struct sg_lb_stats { 10087 unsigned long avg_load; /* Avg load over the CPUs of the group */ 10088 unsigned long group_load; /* Total load over the CPUs of the group */ 10089 unsigned long group_capacity; /* Capacity over the CPUs of the group */ 10090 unsigned long group_util; /* Total utilization over the CPUs of the group */ 10091 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */ 10092 unsigned int sum_nr_running; /* Nr of all tasks running in the group */ 10093 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */ 10094 unsigned int idle_cpus; /* Nr of idle CPUs in the group */ 10095 unsigned int group_weight; 10096 enum group_type group_type; 10097 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */ 10098 unsigned int group_smt_balance; /* Task on busy SMT be moved */ 10099 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */ 10100 #ifdef CONFIG_NUMA_BALANCING 10101 unsigned int nr_numa_running; 10102 unsigned int nr_preferred_running; 10103 #endif 10104 }; 10105 10106 /* 10107 * sd_lb_stats - stats of a sched_domain required for load-balancing: 10108 */ 10109 struct sd_lb_stats { 10110 struct sched_group *busiest; /* Busiest group in this sd */ 10111 struct sched_group *local; /* Local group in this sd */ 10112 unsigned long total_load; /* Total load of all groups in sd */ 10113 unsigned long total_capacity; /* Total capacity of all groups in sd */ 10114 unsigned long avg_load; /* Average load across all groups in sd */ 10115 unsigned int prefer_sibling; /* Tasks should go to sibling first */ 10116 10117 struct sg_lb_stats busiest_stat; /* Statistics of the busiest group */ 10118 struct sg_lb_stats local_stat; /* Statistics of the local group */ 10119 }; 10120 10121 static inline void init_sd_lb_stats(struct sd_lb_stats *sds) 10122 { 10123 /* 10124 * Skimp on the clearing to avoid duplicate work. We can avoid clearing 10125 * local_stat because update_sg_lb_stats() does a full clear/assignment. 10126 * We must however set busiest_stat::group_type and 10127 * busiest_stat::idle_cpus to the worst busiest group because 10128 * update_sd_pick_busiest() reads these before assignment. 10129 */ 10130 *sds = (struct sd_lb_stats){ 10131 .busiest = NULL, 10132 .local = NULL, 10133 .total_load = 0UL, 10134 .total_capacity = 0UL, 10135 .busiest_stat = { 10136 .idle_cpus = UINT_MAX, 10137 .group_type = group_has_spare, 10138 }, 10139 }; 10140 } 10141 10142 static unsigned long scale_rt_capacity(int cpu) 10143 { 10144 unsigned long max = get_actual_cpu_capacity(cpu); 10145 struct rq *rq = cpu_rq(cpu); 10146 unsigned long used, free; 10147 unsigned long irq; 10148 10149 irq = cpu_util_irq(rq); 10150 10151 if (unlikely(irq >= max)) 10152 return 1; 10153 10154 /* 10155 * avg_rt.util_avg and avg_dl.util_avg track binary signals 10156 * (running and not running) with weights 0 and 1024 respectively. 10157 */ 10158 used = cpu_util_rt(rq); 10159 used += cpu_util_dl(rq); 10160 10161 if (unlikely(used >= max)) 10162 return 1; 10163 10164 free = max - used; 10165 10166 return scale_irq_capacity(free, irq, max); 10167 } 10168 10169 static void update_cpu_capacity(struct sched_domain *sd, int cpu) 10170 { 10171 unsigned long capacity = scale_rt_capacity(cpu); 10172 struct sched_group *sdg = sd->groups; 10173 10174 if (!capacity) 10175 capacity = 1; 10176 10177 cpu_rq(cpu)->cpu_capacity = capacity; 10178 trace_sched_cpu_capacity_tp(cpu_rq(cpu)); 10179 10180 sdg->sgc->capacity = capacity; 10181 sdg->sgc->min_capacity = capacity; 10182 sdg->sgc->max_capacity = capacity; 10183 } 10184 10185 void update_group_capacity(struct sched_domain *sd, int cpu) 10186 { 10187 struct sched_domain *child = sd->child; 10188 struct sched_group *group, *sdg = sd->groups; 10189 unsigned long capacity, min_capacity, max_capacity; 10190 unsigned long interval; 10191 10192 interval = msecs_to_jiffies(sd->balance_interval); 10193 interval = clamp(interval, 1UL, max_load_balance_interval); 10194 sdg->sgc->next_update = jiffies + interval; 10195 10196 if (!child) { 10197 update_cpu_capacity(sd, cpu); 10198 return; 10199 } 10200 10201 capacity = 0; 10202 min_capacity = ULONG_MAX; 10203 max_capacity = 0; 10204 10205 if (child->flags & SD_NUMA) { 10206 /* 10207 * SD_NUMA domains cannot assume that child groups 10208 * span the current group. 10209 */ 10210 10211 for_each_cpu(cpu, sched_group_span(sdg)) { 10212 unsigned long cpu_cap = capacity_of(cpu); 10213 10214 capacity += cpu_cap; 10215 min_capacity = min(cpu_cap, min_capacity); 10216 max_capacity = max(cpu_cap, max_capacity); 10217 } 10218 } else { 10219 /* 10220 * !SD_NUMA domains can assume that child groups 10221 * span the current group. 10222 */ 10223 10224 group = child->groups; 10225 do { 10226 struct sched_group_capacity *sgc = group->sgc; 10227 10228 capacity += sgc->capacity; 10229 min_capacity = min(sgc->min_capacity, min_capacity); 10230 max_capacity = max(sgc->max_capacity, max_capacity); 10231 group = group->next; 10232 } while (group != child->groups); 10233 } 10234 10235 sdg->sgc->capacity = capacity; 10236 sdg->sgc->min_capacity = min_capacity; 10237 sdg->sgc->max_capacity = max_capacity; 10238 } 10239 10240 /* 10241 * Check whether the capacity of the rq has been noticeably reduced by side 10242 * activity. The imbalance_pct is used for the threshold. 10243 * Return true is the capacity is reduced 10244 */ 10245 static inline int 10246 check_cpu_capacity(struct rq *rq, struct sched_domain *sd) 10247 { 10248 return ((rq->cpu_capacity * sd->imbalance_pct) < 10249 (arch_scale_cpu_capacity(cpu_of(rq)) * 100)); 10250 } 10251 10252 /* Check if the rq has a misfit task */ 10253 static inline bool check_misfit_status(struct rq *rq) 10254 { 10255 return rq->misfit_task_load; 10256 } 10257 10258 /* 10259 * Group imbalance indicates (and tries to solve) the problem where balancing 10260 * groups is inadequate due to ->cpus_ptr constraints. 10261 * 10262 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a 10263 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group. 10264 * Something like: 10265 * 10266 * { 0 1 2 3 } { 4 5 6 7 } 10267 * * * * * 10268 * 10269 * If we were to balance group-wise we'd place two tasks in the first group and 10270 * two tasks in the second group. Clearly this is undesired as it will overload 10271 * cpu 3 and leave one of the CPUs in the second group unused. 10272 * 10273 * The current solution to this issue is detecting the skew in the first group 10274 * by noticing the lower domain failed to reach balance and had difficulty 10275 * moving tasks due to affinity constraints. 10276 * 10277 * When this is so detected; this group becomes a candidate for busiest; see 10278 * update_sd_pick_busiest(). And calculate_imbalance() and 10279 * sched_balance_find_src_group() avoid some of the usual balance conditions to allow it 10280 * to create an effective group imbalance. 10281 * 10282 * This is a somewhat tricky proposition since the next run might not find the 10283 * group imbalance and decide the groups need to be balanced again. A most 10284 * subtle and fragile situation. 10285 */ 10286 10287 static inline int sg_imbalanced(struct sched_group *group) 10288 { 10289 return group->sgc->imbalance; 10290 } 10291 10292 /* 10293 * group_has_capacity returns true if the group has spare capacity that could 10294 * be used by some tasks. 10295 * We consider that a group has spare capacity if the number of task is 10296 * smaller than the number of CPUs or if the utilization is lower than the 10297 * available capacity for CFS tasks. 10298 * For the latter, we use a threshold to stabilize the state, to take into 10299 * account the variance of the tasks' load and to return true if the available 10300 * capacity in meaningful for the load balancer. 10301 * As an example, an available capacity of 1% can appear but it doesn't make 10302 * any benefit for the load balance. 10303 */ 10304 static inline bool 10305 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 10306 { 10307 if (sgs->sum_nr_running < sgs->group_weight) 10308 return true; 10309 10310 if ((sgs->group_capacity * imbalance_pct) < 10311 (sgs->group_runnable * 100)) 10312 return false; 10313 10314 if ((sgs->group_capacity * 100) > 10315 (sgs->group_util * imbalance_pct)) 10316 return true; 10317 10318 return false; 10319 } 10320 10321 /* 10322 * group_is_overloaded returns true if the group has more tasks than it can 10323 * handle. 10324 * group_is_overloaded is not equals to !group_has_capacity because a group 10325 * with the exact right number of tasks, has no more spare capacity but is not 10326 * overloaded so both group_has_capacity and group_is_overloaded return 10327 * false. 10328 */ 10329 static inline bool 10330 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs) 10331 { 10332 if (sgs->sum_nr_running <= sgs->group_weight) 10333 return false; 10334 10335 if ((sgs->group_capacity * 100) < 10336 (sgs->group_util * imbalance_pct)) 10337 return true; 10338 10339 if ((sgs->group_capacity * imbalance_pct) < 10340 (sgs->group_runnable * 100)) 10341 return true; 10342 10343 return false; 10344 } 10345 10346 static inline enum 10347 group_type group_classify(unsigned int imbalance_pct, 10348 struct sched_group *group, 10349 struct sg_lb_stats *sgs) 10350 { 10351 if (group_is_overloaded(imbalance_pct, sgs)) 10352 return group_overloaded; 10353 10354 if (sg_imbalanced(group)) 10355 return group_imbalanced; 10356 10357 if (sgs->group_asym_packing) 10358 return group_asym_packing; 10359 10360 if (sgs->group_smt_balance) 10361 return group_smt_balance; 10362 10363 if (sgs->group_misfit_task_load) 10364 return group_misfit_task; 10365 10366 if (!group_has_capacity(imbalance_pct, sgs)) 10367 return group_fully_busy; 10368 10369 return group_has_spare; 10370 } 10371 10372 /** 10373 * sched_use_asym_prio - Check whether asym_packing priority must be used 10374 * @sd: The scheduling domain of the load balancing 10375 * @cpu: A CPU 10376 * 10377 * Always use CPU priority when balancing load between SMT siblings. When 10378 * balancing load between cores, it is not sufficient that @cpu is idle. Only 10379 * use CPU priority if the whole core is idle. 10380 * 10381 * Returns: True if the priority of @cpu must be followed. False otherwise. 10382 */ 10383 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu) 10384 { 10385 if (!(sd->flags & SD_ASYM_PACKING)) 10386 return false; 10387 10388 if (!sched_smt_active()) 10389 return true; 10390 10391 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu); 10392 } 10393 10394 static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu) 10395 { 10396 /* 10397 * First check if @dst_cpu can do asym_packing load balance. Only do it 10398 * if it has higher priority than @src_cpu. 10399 */ 10400 return sched_use_asym_prio(sd, dst_cpu) && 10401 sched_asym_prefer(dst_cpu, src_cpu); 10402 } 10403 10404 /** 10405 * sched_group_asym - Check if the destination CPU can do asym_packing balance 10406 * @env: The load balancing environment 10407 * @sgs: Load-balancing statistics of the candidate busiest group 10408 * @group: The candidate busiest group 10409 * 10410 * @env::dst_cpu can do asym_packing if it has higher priority than the 10411 * preferred CPU of @group. 10412 * 10413 * Return: true if @env::dst_cpu can do with asym_packing load balance. False 10414 * otherwise. 10415 */ 10416 static inline bool 10417 sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) 10418 { 10419 /* 10420 * CPU priorities do not make sense for SMT cores with more than one 10421 * busy sibling. 10422 */ 10423 if ((group->flags & SD_SHARE_CPUCAPACITY) && 10424 (sgs->group_weight - sgs->idle_cpus != 1)) 10425 return false; 10426 10427 return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu)); 10428 } 10429 10430 /* One group has more than one SMT CPU while the other group does not */ 10431 static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1, 10432 struct sched_group *sg2) 10433 { 10434 if (!sg1 || !sg2) 10435 return false; 10436 10437 return (sg1->flags & SD_SHARE_CPUCAPACITY) != 10438 (sg2->flags & SD_SHARE_CPUCAPACITY); 10439 } 10440 10441 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs, 10442 struct sched_group *group) 10443 { 10444 if (!env->idle) 10445 return false; 10446 10447 /* 10448 * For SMT source group, it is better to move a task 10449 * to a CPU that doesn't have multiple tasks sharing its CPU capacity. 10450 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY 10451 * will not be on. 10452 */ 10453 if (group->flags & SD_SHARE_CPUCAPACITY && 10454 sgs->sum_h_nr_running > 1) 10455 return true; 10456 10457 return false; 10458 } 10459 10460 static inline long sibling_imbalance(struct lb_env *env, 10461 struct sd_lb_stats *sds, 10462 struct sg_lb_stats *busiest, 10463 struct sg_lb_stats *local) 10464 { 10465 int ncores_busiest, ncores_local; 10466 long imbalance; 10467 10468 if (!env->idle || !busiest->sum_nr_running) 10469 return 0; 10470 10471 ncores_busiest = sds->busiest->cores; 10472 ncores_local = sds->local->cores; 10473 10474 if (ncores_busiest == ncores_local) { 10475 imbalance = busiest->sum_nr_running; 10476 lsub_positive(&imbalance, local->sum_nr_running); 10477 return imbalance; 10478 } 10479 10480 /* Balance such that nr_running/ncores ratio are same on both groups */ 10481 imbalance = ncores_local * busiest->sum_nr_running; 10482 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running); 10483 /* Normalize imbalance and do rounding on normalization */ 10484 imbalance = 2 * imbalance + ncores_local + ncores_busiest; 10485 imbalance /= ncores_local + ncores_busiest; 10486 10487 /* Take advantage of resource in an empty sched group */ 10488 if (imbalance <= 1 && local->sum_nr_running == 0 && 10489 busiest->sum_nr_running > 1) 10490 imbalance = 2; 10491 10492 return imbalance; 10493 } 10494 10495 static inline bool 10496 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd) 10497 { 10498 /* 10499 * When there is more than 1 task, the group_overloaded case already 10500 * takes care of cpu with reduced capacity 10501 */ 10502 if (rq->cfs.h_nr_runnable != 1) 10503 return false; 10504 10505 return check_cpu_capacity(rq, sd); 10506 } 10507 10508 /** 10509 * update_sg_lb_stats - Update sched_group's statistics for load balancing. 10510 * @env: The load balancing environment. 10511 * @sds: Load-balancing data with statistics of the local group. 10512 * @group: sched_group whose statistics are to be updated. 10513 * @sgs: variable to hold the statistics for this group. 10514 * @sg_overloaded: sched_group is overloaded 10515 * @sg_overutilized: sched_group is overutilized 10516 */ 10517 static inline void update_sg_lb_stats(struct lb_env *env, 10518 struct sd_lb_stats *sds, 10519 struct sched_group *group, 10520 struct sg_lb_stats *sgs, 10521 bool *sg_overloaded, 10522 bool *sg_overutilized) 10523 { 10524 int i, nr_running, local_group, sd_flags = env->sd->flags; 10525 bool balancing_at_rd = !env->sd->parent; 10526 10527 memset(sgs, 0, sizeof(*sgs)); 10528 10529 local_group = group == sds->local; 10530 10531 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 10532 struct rq *rq = cpu_rq(i); 10533 unsigned long load = cpu_load(rq); 10534 10535 sgs->group_load += load; 10536 sgs->group_util += cpu_util_cfs(i); 10537 sgs->group_runnable += cpu_runnable(rq); 10538 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable; 10539 10540 nr_running = rq->nr_running; 10541 sgs->sum_nr_running += nr_running; 10542 10543 if (cpu_overutilized(i)) 10544 *sg_overutilized = 1; 10545 10546 /* 10547 * No need to call idle_cpu() if nr_running is not 0 10548 */ 10549 if (!nr_running && idle_cpu(i)) { 10550 sgs->idle_cpus++; 10551 /* Idle cpu can't have misfit task */ 10552 continue; 10553 } 10554 10555 /* Overload indicator is only updated at root domain */ 10556 if (balancing_at_rd && nr_running > 1) 10557 *sg_overloaded = 1; 10558 10559 #ifdef CONFIG_NUMA_BALANCING 10560 /* Only fbq_classify_group() uses this to classify NUMA groups */ 10561 if (sd_flags & SD_NUMA) { 10562 sgs->nr_numa_running += rq->nr_numa_running; 10563 sgs->nr_preferred_running += rq->nr_preferred_running; 10564 } 10565 #endif 10566 if (local_group) 10567 continue; 10568 10569 if (sd_flags & SD_ASYM_CPUCAPACITY) { 10570 /* Check for a misfit task on the cpu */ 10571 if (sgs->group_misfit_task_load < rq->misfit_task_load) { 10572 sgs->group_misfit_task_load = rq->misfit_task_load; 10573 *sg_overloaded = 1; 10574 } 10575 } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { 10576 /* Check for a task running on a CPU with reduced capacity */ 10577 if (sgs->group_misfit_task_load < load) 10578 sgs->group_misfit_task_load = load; 10579 } 10580 } 10581 10582 sgs->group_capacity = group->sgc->capacity; 10583 10584 sgs->group_weight = group->group_weight; 10585 10586 /* Check if dst CPU is idle and preferred to this group */ 10587 if (!local_group && env->idle && sgs->sum_h_nr_running && 10588 sched_group_asym(env, sgs, group)) 10589 sgs->group_asym_packing = 1; 10590 10591 /* Check for loaded SMT group to be balanced to dst CPU */ 10592 if (!local_group && smt_balance(env, sgs, group)) 10593 sgs->group_smt_balance = 1; 10594 10595 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs); 10596 10597 /* Computing avg_load makes sense only when group is overloaded */ 10598 if (sgs->group_type == group_overloaded) 10599 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10600 sgs->group_capacity; 10601 } 10602 10603 /** 10604 * update_sd_pick_busiest - return 1 on busiest group 10605 * @env: The load balancing environment. 10606 * @sds: sched_domain statistics 10607 * @sg: sched_group candidate to be checked for being the busiest 10608 * @sgs: sched_group statistics 10609 * 10610 * Determine if @sg is a busier group than the previously selected 10611 * busiest group. 10612 * 10613 * Return: %true if @sg is a busier group than the previously selected 10614 * busiest group. %false otherwise. 10615 */ 10616 static bool update_sd_pick_busiest(struct lb_env *env, 10617 struct sd_lb_stats *sds, 10618 struct sched_group *sg, 10619 struct sg_lb_stats *sgs) 10620 { 10621 struct sg_lb_stats *busiest = &sds->busiest_stat; 10622 10623 /* Make sure that there is at least one task to pull */ 10624 if (!sgs->sum_h_nr_running) 10625 return false; 10626 10627 /* 10628 * Don't try to pull misfit tasks we can't help. 10629 * We can use max_capacity here as reduction in capacity on some 10630 * CPUs in the group should either be possible to resolve 10631 * internally or be covered by avg_load imbalance (eventually). 10632 */ 10633 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10634 (sgs->group_type == group_misfit_task) && 10635 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || 10636 sds->local_stat.group_type != group_has_spare)) 10637 return false; 10638 10639 if (sgs->group_type > busiest->group_type) 10640 return true; 10641 10642 if (sgs->group_type < busiest->group_type) 10643 return false; 10644 10645 /* 10646 * The candidate and the current busiest group are the same type of 10647 * group. Let check which one is the busiest according to the type. 10648 */ 10649 10650 switch (sgs->group_type) { 10651 case group_overloaded: 10652 /* Select the overloaded group with highest avg_load. */ 10653 return sgs->avg_load > busiest->avg_load; 10654 10655 case group_imbalanced: 10656 /* 10657 * Select the 1st imbalanced group as we don't have any way to 10658 * choose one more than another. 10659 */ 10660 return false; 10661 10662 case group_asym_packing: 10663 /* Prefer to move from lowest priority CPU's work */ 10664 return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu), 10665 READ_ONCE(sg->asym_prefer_cpu)); 10666 10667 case group_misfit_task: 10668 /* 10669 * If we have more than one misfit sg go with the biggest 10670 * misfit. 10671 */ 10672 return sgs->group_misfit_task_load > busiest->group_misfit_task_load; 10673 10674 case group_smt_balance: 10675 /* 10676 * Check if we have spare CPUs on either SMT group to 10677 * choose has spare or fully busy handling. 10678 */ 10679 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0) 10680 goto has_spare; 10681 10682 fallthrough; 10683 10684 case group_fully_busy: 10685 /* 10686 * Select the fully busy group with highest avg_load. In 10687 * theory, there is no need to pull task from such kind of 10688 * group because tasks have all compute capacity that they need 10689 * but we can still improve the overall throughput by reducing 10690 * contention when accessing shared HW resources. 10691 * 10692 * XXX for now avg_load is not computed and always 0 so we 10693 * select the 1st one, except if @sg is composed of SMT 10694 * siblings. 10695 */ 10696 10697 if (sgs->avg_load < busiest->avg_load) 10698 return false; 10699 10700 if (sgs->avg_load == busiest->avg_load) { 10701 /* 10702 * SMT sched groups need more help than non-SMT groups. 10703 * If @sg happens to also be SMT, either choice is good. 10704 */ 10705 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY) 10706 return false; 10707 } 10708 10709 break; 10710 10711 case group_has_spare: 10712 /* 10713 * Do not pick sg with SMT CPUs over sg with pure CPUs, 10714 * as we do not want to pull task off SMT core with one task 10715 * and make the core idle. 10716 */ 10717 if (smt_vs_nonsmt_groups(sds->busiest, sg)) { 10718 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1) 10719 return false; 10720 else 10721 return true; 10722 } 10723 has_spare: 10724 10725 /* 10726 * Select not overloaded group with lowest number of idle CPUs 10727 * and highest number of running tasks. We could also compare 10728 * the spare capacity which is more stable but it can end up 10729 * that the group has less spare capacity but finally more idle 10730 * CPUs which means less opportunity to pull tasks. 10731 */ 10732 if (sgs->idle_cpus > busiest->idle_cpus) 10733 return false; 10734 else if ((sgs->idle_cpus == busiest->idle_cpus) && 10735 (sgs->sum_nr_running <= busiest->sum_nr_running)) 10736 return false; 10737 10738 break; 10739 } 10740 10741 /* 10742 * Candidate sg has no more than one task per CPU and has higher 10743 * per-CPU capacity. Migrating tasks to less capable CPUs may harm 10744 * throughput. Maximize throughput, power/energy consequences are not 10745 * considered. 10746 */ 10747 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && 10748 (sgs->group_type <= group_fully_busy) && 10749 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) 10750 return false; 10751 10752 return true; 10753 } 10754 10755 #ifdef CONFIG_NUMA_BALANCING 10756 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10757 { 10758 if (sgs->sum_h_nr_running > sgs->nr_numa_running) 10759 return regular; 10760 if (sgs->sum_h_nr_running > sgs->nr_preferred_running) 10761 return remote; 10762 return all; 10763 } 10764 10765 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10766 { 10767 if (rq->nr_running > rq->nr_numa_running) 10768 return regular; 10769 if (rq->nr_running > rq->nr_preferred_running) 10770 return remote; 10771 return all; 10772 } 10773 #else /* !CONFIG_NUMA_BALANCING: */ 10774 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs) 10775 { 10776 return all; 10777 } 10778 10779 static inline enum fbq_type fbq_classify_rq(struct rq *rq) 10780 { 10781 return regular; 10782 } 10783 #endif /* !CONFIG_NUMA_BALANCING */ 10784 10785 10786 struct sg_lb_stats; 10787 10788 /* 10789 * task_running_on_cpu - return 1 if @p is running on @cpu. 10790 */ 10791 10792 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p) 10793 { 10794 /* Task has no contribution or is new */ 10795 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 10796 return 0; 10797 10798 if (task_on_rq_queued(p)) 10799 return 1; 10800 10801 return 0; 10802 } 10803 10804 /** 10805 * idle_cpu_without - would a given CPU be idle without p ? 10806 * @cpu: the processor on which idleness is tested. 10807 * @p: task which should be ignored. 10808 * 10809 * Return: 1 if the CPU would be idle. 0 otherwise. 10810 */ 10811 static int idle_cpu_without(int cpu, struct task_struct *p) 10812 { 10813 struct rq *rq = cpu_rq(cpu); 10814 10815 if (rq->curr != rq->idle && rq->curr != p) 10816 return 0; 10817 10818 /* 10819 * rq->nr_running can't be used but an updated version without the 10820 * impact of p on cpu must be used instead. The updated nr_running 10821 * be computed and tested before calling idle_cpu_without(). 10822 */ 10823 10824 if (rq->ttwu_pending) 10825 return 0; 10826 10827 return 1; 10828 } 10829 10830 /* 10831 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup. 10832 * @sd: The sched_domain level to look for idlest group. 10833 * @group: sched_group whose statistics are to be updated. 10834 * @sgs: variable to hold the statistics for this group. 10835 * @p: The task for which we look for the idlest group/CPU. 10836 */ 10837 static inline void update_sg_wakeup_stats(struct sched_domain *sd, 10838 struct sched_group *group, 10839 struct sg_lb_stats *sgs, 10840 struct task_struct *p) 10841 { 10842 int i, nr_running; 10843 10844 memset(sgs, 0, sizeof(*sgs)); 10845 10846 /* Assume that task can't fit any CPU of the group */ 10847 if (sd->flags & SD_ASYM_CPUCAPACITY) 10848 sgs->group_misfit_task_load = 1; 10849 10850 for_each_cpu(i, sched_group_span(group)) { 10851 struct rq *rq = cpu_rq(i); 10852 unsigned int local; 10853 10854 sgs->group_load += cpu_load_without(rq, p); 10855 sgs->group_util += cpu_util_without(i, p); 10856 sgs->group_runnable += cpu_runnable_without(rq, p); 10857 local = task_running_on_cpu(i, p); 10858 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable - local; 10859 10860 nr_running = rq->nr_running - local; 10861 sgs->sum_nr_running += nr_running; 10862 10863 /* 10864 * No need to call idle_cpu_without() if nr_running is not 0 10865 */ 10866 if (!nr_running && idle_cpu_without(i, p)) 10867 sgs->idle_cpus++; 10868 10869 /* Check if task fits in the CPU */ 10870 if (sd->flags & SD_ASYM_CPUCAPACITY && 10871 sgs->group_misfit_task_load && 10872 task_fits_cpu(p, i)) 10873 sgs->group_misfit_task_load = 0; 10874 10875 } 10876 10877 sgs->group_capacity = group->sgc->capacity; 10878 10879 sgs->group_weight = group->group_weight; 10880 10881 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); 10882 10883 /* 10884 * Computing avg_load makes sense only when group is fully busy or 10885 * overloaded 10886 */ 10887 if (sgs->group_type == group_fully_busy || 10888 sgs->group_type == group_overloaded) 10889 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) / 10890 sgs->group_capacity; 10891 } 10892 10893 static bool update_pick_idlest(struct sched_group *idlest, 10894 struct sg_lb_stats *idlest_sgs, 10895 struct sched_group *group, 10896 struct sg_lb_stats *sgs) 10897 { 10898 if (sgs->group_type < idlest_sgs->group_type) 10899 return true; 10900 10901 if (sgs->group_type > idlest_sgs->group_type) 10902 return false; 10903 10904 /* 10905 * The candidate and the current idlest group are the same type of 10906 * group. Let check which one is the idlest according to the type. 10907 */ 10908 10909 switch (sgs->group_type) { 10910 case group_overloaded: 10911 case group_fully_busy: 10912 /* Select the group with lowest avg_load. */ 10913 if (idlest_sgs->avg_load <= sgs->avg_load) 10914 return false; 10915 break; 10916 10917 case group_imbalanced: 10918 case group_asym_packing: 10919 case group_smt_balance: 10920 /* Those types are not used in the slow wakeup path */ 10921 return false; 10922 10923 case group_misfit_task: 10924 /* Select group with the highest max capacity */ 10925 if (idlest->sgc->max_capacity >= group->sgc->max_capacity) 10926 return false; 10927 break; 10928 10929 case group_has_spare: 10930 /* Select group with most idle CPUs */ 10931 if (idlest_sgs->idle_cpus > sgs->idle_cpus) 10932 return false; 10933 10934 /* Select group with lowest group_util */ 10935 if (idlest_sgs->idle_cpus == sgs->idle_cpus && 10936 idlest_sgs->group_util <= sgs->group_util) 10937 return false; 10938 10939 break; 10940 } 10941 10942 return true; 10943 } 10944 10945 /* 10946 * sched_balance_find_dst_group() finds and returns the least busy CPU group within the 10947 * domain. 10948 * 10949 * Assumes p is allowed on at least one CPU in sd. 10950 */ 10951 static struct sched_group * 10952 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) 10953 { 10954 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; 10955 struct sg_lb_stats local_sgs, tmp_sgs; 10956 struct sg_lb_stats *sgs; 10957 unsigned long imbalance; 10958 struct sg_lb_stats idlest_sgs = { 10959 .avg_load = UINT_MAX, 10960 .group_type = group_overloaded, 10961 }; 10962 10963 do { 10964 int local_group; 10965 10966 /* Skip over this group if it has no CPUs allowed */ 10967 if (!cpumask_intersects(sched_group_span(group), 10968 p->cpus_ptr)) 10969 continue; 10970 10971 /* Skip over this group if no cookie matched */ 10972 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group)) 10973 continue; 10974 10975 local_group = cpumask_test_cpu(this_cpu, 10976 sched_group_span(group)); 10977 10978 if (local_group) { 10979 sgs = &local_sgs; 10980 local = group; 10981 } else { 10982 sgs = &tmp_sgs; 10983 } 10984 10985 update_sg_wakeup_stats(sd, group, sgs, p); 10986 10987 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) { 10988 idlest = group; 10989 idlest_sgs = *sgs; 10990 } 10991 10992 } while (group = group->next, group != sd->groups); 10993 10994 10995 /* There is no idlest group to push tasks to */ 10996 if (!idlest) 10997 return NULL; 10998 10999 /* The local group has been skipped because of CPU affinity */ 11000 if (!local) 11001 return idlest; 11002 11003 /* 11004 * If the local group is idler than the selected idlest group 11005 * don't try and push the task. 11006 */ 11007 if (local_sgs.group_type < idlest_sgs.group_type) 11008 return NULL; 11009 11010 /* 11011 * If the local group is busier than the selected idlest group 11012 * try and push the task. 11013 */ 11014 if (local_sgs.group_type > idlest_sgs.group_type) 11015 return idlest; 11016 11017 switch (local_sgs.group_type) { 11018 case group_overloaded: 11019 case group_fully_busy: 11020 11021 /* Calculate allowed imbalance based on load */ 11022 imbalance = scale_load_down(NICE_0_LOAD) * 11023 (sd->imbalance_pct-100) / 100; 11024 11025 /* 11026 * When comparing groups across NUMA domains, it's possible for 11027 * the local domain to be very lightly loaded relative to the 11028 * remote domains but "imbalance" skews the comparison making 11029 * remote CPUs look much more favourable. When considering 11030 * cross-domain, add imbalance to the load on the remote node 11031 * and consider staying local. 11032 */ 11033 11034 if ((sd->flags & SD_NUMA) && 11035 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load)) 11036 return NULL; 11037 11038 /* 11039 * If the local group is less loaded than the selected 11040 * idlest group don't try and push any tasks. 11041 */ 11042 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance)) 11043 return NULL; 11044 11045 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load) 11046 return NULL; 11047 break; 11048 11049 case group_imbalanced: 11050 case group_asym_packing: 11051 case group_smt_balance: 11052 /* Those type are not used in the slow wakeup path */ 11053 return NULL; 11054 11055 case group_misfit_task: 11056 /* Select group with the highest max capacity */ 11057 if (local->sgc->max_capacity >= idlest->sgc->max_capacity) 11058 return NULL; 11059 break; 11060 11061 case group_has_spare: 11062 #ifdef CONFIG_NUMA 11063 if (sd->flags & SD_NUMA) { 11064 int imb_numa_nr = sd->imb_numa_nr; 11065 #ifdef CONFIG_NUMA_BALANCING 11066 int idlest_cpu; 11067 /* 11068 * If there is spare capacity at NUMA, try to select 11069 * the preferred node 11070 */ 11071 if (cpu_to_node(this_cpu) == p->numa_preferred_nid) 11072 return NULL; 11073 11074 idlest_cpu = cpumask_first(sched_group_span(idlest)); 11075 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid) 11076 return idlest; 11077 #endif /* CONFIG_NUMA_BALANCING */ 11078 /* 11079 * Otherwise, keep the task close to the wakeup source 11080 * and improve locality if the number of running tasks 11081 * would remain below threshold where an imbalance is 11082 * allowed while accounting for the possibility the 11083 * task is pinned to a subset of CPUs. If there is a 11084 * real need of migration, periodic load balance will 11085 * take care of it. 11086 */ 11087 if (p->nr_cpus_allowed != NR_CPUS) { 11088 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 11089 11090 cpumask_and(cpus, sched_group_span(local), p->cpus_ptr); 11091 imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr); 11092 } 11093 11094 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus); 11095 if (!adjust_numa_imbalance(imbalance, 11096 local_sgs.sum_nr_running + 1, 11097 imb_numa_nr)) { 11098 return NULL; 11099 } 11100 } 11101 #endif /* CONFIG_NUMA */ 11102 11103 /* 11104 * Select group with highest number of idle CPUs. We could also 11105 * compare the utilization which is more stable but it can end 11106 * up that the group has less spare capacity but finally more 11107 * idle CPUs which means more opportunity to run task. 11108 */ 11109 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus) 11110 return NULL; 11111 break; 11112 } 11113 11114 return idlest; 11115 } 11116 11117 static void update_idle_cpu_scan(struct lb_env *env, 11118 unsigned long sum_util) 11119 { 11120 struct sched_domain_shared *sd_share; 11121 int llc_weight, pct; 11122 u64 x, y, tmp; 11123 /* 11124 * Update the number of CPUs to scan in LLC domain, which could 11125 * be used as a hint in select_idle_cpu(). The update of sd_share 11126 * could be expensive because it is within a shared cache line. 11127 * So the write of this hint only occurs during periodic load 11128 * balancing, rather than CPU_NEWLY_IDLE, because the latter 11129 * can fire way more frequently than the former. 11130 */ 11131 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE) 11132 return; 11133 11134 llc_weight = per_cpu(sd_llc_size, env->dst_cpu); 11135 if (env->sd->span_weight != llc_weight) 11136 return; 11137 11138 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu)); 11139 if (!sd_share) 11140 return; 11141 11142 /* 11143 * The number of CPUs to search drops as sum_util increases, when 11144 * sum_util hits 85% or above, the scan stops. 11145 * The reason to choose 85% as the threshold is because this is the 11146 * imbalance_pct(117) when a LLC sched group is overloaded. 11147 * 11148 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1] 11149 * and y'= y / SCHED_CAPACITY_SCALE 11150 * 11151 * x is the ratio of sum_util compared to the CPU capacity: 11152 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE) 11153 * y' is the ratio of CPUs to be scanned in the LLC domain, 11154 * and the number of CPUs to scan is calculated by: 11155 * 11156 * nr_scan = llc_weight * y' [2] 11157 * 11158 * When x hits the threshold of overloaded, AKA, when 11159 * x = 100 / pct, y drops to 0. According to [1], 11160 * p should be SCHED_CAPACITY_SCALE * pct^2 / 10000 11161 * 11162 * Scale x by SCHED_CAPACITY_SCALE: 11163 * x' = sum_util / llc_weight; [3] 11164 * 11165 * and finally [1] becomes: 11166 * y = SCHED_CAPACITY_SCALE - 11167 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4] 11168 * 11169 */ 11170 /* equation [3] */ 11171 x = sum_util; 11172 do_div(x, llc_weight); 11173 11174 /* equation [4] */ 11175 pct = env->sd->imbalance_pct; 11176 tmp = x * x * pct * pct; 11177 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE); 11178 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE); 11179 y = SCHED_CAPACITY_SCALE - tmp; 11180 11181 /* equation [2] */ 11182 y *= llc_weight; 11183 do_div(y, SCHED_CAPACITY_SCALE); 11184 if ((int)y != sd_share->nr_idle_scan) 11185 WRITE_ONCE(sd_share->nr_idle_scan, (int)y); 11186 } 11187 11188 /** 11189 * update_sd_lb_stats - Update sched_domain's statistics for load balancing. 11190 * @env: The load balancing environment. 11191 * @sds: variable to hold the statistics for this sched_domain. 11192 */ 11193 11194 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds) 11195 { 11196 struct sched_group *sg = env->sd->groups; 11197 struct sg_lb_stats *local = &sds->local_stat; 11198 struct sg_lb_stats tmp_sgs; 11199 unsigned long sum_util = 0; 11200 bool sg_overloaded = 0, sg_overutilized = 0; 11201 11202 do { 11203 struct sg_lb_stats *sgs = &tmp_sgs; 11204 int local_group; 11205 11206 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg)); 11207 if (local_group) { 11208 sds->local = sg; 11209 sgs = local; 11210 11211 if (env->idle != CPU_NEWLY_IDLE || 11212 time_after_eq(jiffies, sg->sgc->next_update)) 11213 update_group_capacity(env->sd, env->dst_cpu); 11214 } 11215 11216 update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized); 11217 11218 if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) { 11219 sds->busiest = sg; 11220 sds->busiest_stat = *sgs; 11221 } 11222 11223 /* Now, start updating sd_lb_stats */ 11224 sds->total_load += sgs->group_load; 11225 sds->total_capacity += sgs->group_capacity; 11226 11227 sum_util += sgs->group_util; 11228 sg = sg->next; 11229 } while (sg != env->sd->groups); 11230 11231 /* 11232 * Indicate that the child domain of the busiest group prefers tasks 11233 * go to a child's sibling domains first. NB the flags of a sched group 11234 * are those of the child domain. 11235 */ 11236 if (sds->busiest) 11237 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING); 11238 11239 11240 if (env->sd->flags & SD_NUMA) 11241 env->fbq_type = fbq_classify_group(&sds->busiest_stat); 11242 11243 if (!env->sd->parent) { 11244 /* update overload indicator if we are at root domain */ 11245 set_rd_overloaded(env->dst_rq->rd, sg_overloaded); 11246 11247 /* Update over-utilization (tipping point, U >= 0) indicator */ 11248 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 11249 } else if (sg_overutilized) { 11250 set_rd_overutilized(env->dst_rq->rd, sg_overutilized); 11251 } 11252 11253 update_idle_cpu_scan(env, sum_util); 11254 } 11255 11256 /** 11257 * calculate_imbalance - Calculate the amount of imbalance present within the 11258 * groups of a given sched_domain during load balance. 11259 * @env: load balance environment 11260 * @sds: statistics of the sched_domain whose imbalance is to be calculated. 11261 */ 11262 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds) 11263 { 11264 struct sg_lb_stats *local, *busiest; 11265 11266 local = &sds->local_stat; 11267 busiest = &sds->busiest_stat; 11268 11269 if (busiest->group_type == group_misfit_task) { 11270 if (env->sd->flags & SD_ASYM_CPUCAPACITY) { 11271 /* Set imbalance to allow misfit tasks to be balanced. */ 11272 env->migration_type = migrate_misfit; 11273 env->imbalance = 1; 11274 } else { 11275 /* 11276 * Set load imbalance to allow moving task from cpu 11277 * with reduced capacity. 11278 */ 11279 env->migration_type = migrate_load; 11280 env->imbalance = busiest->group_misfit_task_load; 11281 } 11282 return; 11283 } 11284 11285 if (busiest->group_type == group_asym_packing) { 11286 /* 11287 * In case of asym capacity, we will try to migrate all load to 11288 * the preferred CPU. 11289 */ 11290 env->migration_type = migrate_task; 11291 env->imbalance = busiest->sum_h_nr_running; 11292 return; 11293 } 11294 11295 if (busiest->group_type == group_smt_balance) { 11296 /* Reduce number of tasks sharing CPU capacity */ 11297 env->migration_type = migrate_task; 11298 env->imbalance = 1; 11299 return; 11300 } 11301 11302 if (busiest->group_type == group_imbalanced) { 11303 /* 11304 * In the group_imb case we cannot rely on group-wide averages 11305 * to ensure CPU-load equilibrium, try to move any task to fix 11306 * the imbalance. The next load balance will take care of 11307 * balancing back the system. 11308 */ 11309 env->migration_type = migrate_task; 11310 env->imbalance = 1; 11311 return; 11312 } 11313 11314 /* 11315 * Try to use spare capacity of local group without overloading it or 11316 * emptying busiest. 11317 */ 11318 if (local->group_type == group_has_spare) { 11319 if ((busiest->group_type > group_fully_busy) && 11320 !(env->sd->flags & SD_SHARE_LLC)) { 11321 /* 11322 * If busiest is overloaded, try to fill spare 11323 * capacity. This might end up creating spare capacity 11324 * in busiest or busiest still being overloaded but 11325 * there is no simple way to directly compute the 11326 * amount of load to migrate in order to balance the 11327 * system. 11328 */ 11329 env->migration_type = migrate_util; 11330 env->imbalance = max(local->group_capacity, local->group_util) - 11331 local->group_util; 11332 11333 /* 11334 * In some cases, the group's utilization is max or even 11335 * higher than capacity because of migrations but the 11336 * local CPU is (newly) idle. There is at least one 11337 * waiting task in this overloaded busiest group. Let's 11338 * try to pull it. 11339 */ 11340 if (env->idle && env->imbalance == 0) { 11341 env->migration_type = migrate_task; 11342 env->imbalance = 1; 11343 } 11344 11345 return; 11346 } 11347 11348 if (busiest->group_weight == 1 || sds->prefer_sibling) { 11349 /* 11350 * When prefer sibling, evenly spread running tasks on 11351 * groups. 11352 */ 11353 env->migration_type = migrate_task; 11354 env->imbalance = sibling_imbalance(env, sds, busiest, local); 11355 } else { 11356 11357 /* 11358 * If there is no overload, we just want to even the number of 11359 * idle CPUs. 11360 */ 11361 env->migration_type = migrate_task; 11362 env->imbalance = max_t(long, 0, 11363 (local->idle_cpus - busiest->idle_cpus)); 11364 } 11365 11366 #ifdef CONFIG_NUMA 11367 /* Consider allowing a small imbalance between NUMA groups */ 11368 if (env->sd->flags & SD_NUMA) { 11369 env->imbalance = adjust_numa_imbalance(env->imbalance, 11370 local->sum_nr_running + 1, 11371 env->sd->imb_numa_nr); 11372 } 11373 #endif 11374 11375 /* Number of tasks to move to restore balance */ 11376 env->imbalance >>= 1; 11377 11378 return; 11379 } 11380 11381 /* 11382 * Local is fully busy but has to take more load to relieve the 11383 * busiest group 11384 */ 11385 if (local->group_type < group_overloaded) { 11386 /* 11387 * Local will become overloaded so the avg_load metrics are 11388 * finally needed. 11389 */ 11390 11391 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) / 11392 local->group_capacity; 11393 11394 /* 11395 * If the local group is more loaded than the selected 11396 * busiest group don't try to pull any tasks. 11397 */ 11398 if (local->avg_load >= busiest->avg_load) { 11399 env->imbalance = 0; 11400 return; 11401 } 11402 11403 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) / 11404 sds->total_capacity; 11405 11406 /* 11407 * If the local group is more loaded than the average system 11408 * load, don't try to pull any tasks. 11409 */ 11410 if (local->avg_load >= sds->avg_load) { 11411 env->imbalance = 0; 11412 return; 11413 } 11414 11415 } 11416 11417 /* 11418 * Both group are or will become overloaded and we're trying to get all 11419 * the CPUs to the average_load, so we don't want to push ourselves 11420 * above the average load, nor do we wish to reduce the max loaded CPU 11421 * below the average load. At the same time, we also don't want to 11422 * reduce the group load below the group capacity. Thus we look for 11423 * the minimum possible imbalance. 11424 */ 11425 env->migration_type = migrate_load; 11426 env->imbalance = min( 11427 (busiest->avg_load - sds->avg_load) * busiest->group_capacity, 11428 (sds->avg_load - local->avg_load) * local->group_capacity 11429 ) / SCHED_CAPACITY_SCALE; 11430 } 11431 11432 /******* sched_balance_find_src_group() helpers end here *********************/ 11433 11434 /* 11435 * Decision matrix according to the local and busiest group type: 11436 * 11437 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded 11438 * has_spare nr_idle balanced N/A N/A balanced balanced 11439 * fully_busy nr_idle nr_idle N/A N/A balanced balanced 11440 * misfit_task force N/A N/A N/A N/A N/A 11441 * asym_packing force force N/A N/A force force 11442 * imbalanced force force N/A N/A force force 11443 * overloaded force force N/A N/A force avg_load 11444 * 11445 * N/A : Not Applicable because already filtered while updating 11446 * statistics. 11447 * balanced : The system is balanced for these 2 groups. 11448 * force : Calculate the imbalance as load migration is probably needed. 11449 * avg_load : Only if imbalance is significant enough. 11450 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite 11451 * different in groups. 11452 */ 11453 11454 /** 11455 * sched_balance_find_src_group - Returns the busiest group within the sched_domain 11456 * if there is an imbalance. 11457 * @env: The load balancing environment. 11458 * 11459 * Also calculates the amount of runnable load which should be moved 11460 * to restore balance. 11461 * 11462 * Return: - The busiest group if imbalance exists. 11463 */ 11464 static struct sched_group *sched_balance_find_src_group(struct lb_env *env) 11465 { 11466 struct sg_lb_stats *local, *busiest; 11467 struct sd_lb_stats sds; 11468 11469 init_sd_lb_stats(&sds); 11470 11471 /* 11472 * Compute the various statistics relevant for load balancing at 11473 * this level. 11474 */ 11475 update_sd_lb_stats(env, &sds); 11476 11477 /* There is no busy sibling group to pull tasks from */ 11478 if (!sds.busiest) 11479 goto out_balanced; 11480 11481 busiest = &sds.busiest_stat; 11482 11483 /* Misfit tasks should be dealt with regardless of the avg load */ 11484 if (busiest->group_type == group_misfit_task) 11485 goto force_balance; 11486 11487 if (!is_rd_overutilized(env->dst_rq->rd) && 11488 rcu_dereference(env->dst_rq->rd->pd)) 11489 goto out_balanced; 11490 11491 /* ASYM feature bypasses nice load balance check */ 11492 if (busiest->group_type == group_asym_packing) 11493 goto force_balance; 11494 11495 /* 11496 * If the busiest group is imbalanced the below checks don't 11497 * work because they assume all things are equal, which typically 11498 * isn't true due to cpus_ptr constraints and the like. 11499 */ 11500 if (busiest->group_type == group_imbalanced) 11501 goto force_balance; 11502 11503 local = &sds.local_stat; 11504 /* 11505 * If the local group is busier than the selected busiest group 11506 * don't try and pull any tasks. 11507 */ 11508 if (local->group_type > busiest->group_type) 11509 goto out_balanced; 11510 11511 /* 11512 * When groups are overloaded, use the avg_load to ensure fairness 11513 * between tasks. 11514 */ 11515 if (local->group_type == group_overloaded) { 11516 /* 11517 * If the local group is more loaded than the selected 11518 * busiest group don't try to pull any tasks. 11519 */ 11520 if (local->avg_load >= busiest->avg_load) 11521 goto out_balanced; 11522 11523 /* XXX broken for overlapping NUMA groups */ 11524 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) / 11525 sds.total_capacity; 11526 11527 /* 11528 * Don't pull any tasks if this group is already above the 11529 * domain average load. 11530 */ 11531 if (local->avg_load >= sds.avg_load) 11532 goto out_balanced; 11533 11534 /* 11535 * If the busiest group is more loaded, use imbalance_pct to be 11536 * conservative. 11537 */ 11538 if (100 * busiest->avg_load <= 11539 env->sd->imbalance_pct * local->avg_load) 11540 goto out_balanced; 11541 } 11542 11543 /* 11544 * Try to move all excess tasks to a sibling domain of the busiest 11545 * group's child domain. 11546 */ 11547 if (sds.prefer_sibling && local->group_type == group_has_spare && 11548 sibling_imbalance(env, &sds, busiest, local) > 1) 11549 goto force_balance; 11550 11551 if (busiest->group_type != group_overloaded) { 11552 if (!env->idle) { 11553 /* 11554 * If the busiest group is not overloaded (and as a 11555 * result the local one too) but this CPU is already 11556 * busy, let another idle CPU try to pull task. 11557 */ 11558 goto out_balanced; 11559 } 11560 11561 if (busiest->group_type == group_smt_balance && 11562 smt_vs_nonsmt_groups(sds.local, sds.busiest)) { 11563 /* Let non SMT CPU pull from SMT CPU sharing with sibling */ 11564 goto force_balance; 11565 } 11566 11567 if (busiest->group_weight > 1 && 11568 local->idle_cpus <= (busiest->idle_cpus + 1)) { 11569 /* 11570 * If the busiest group is not overloaded 11571 * and there is no imbalance between this and busiest 11572 * group wrt idle CPUs, it is balanced. The imbalance 11573 * becomes significant if the diff is greater than 1 11574 * otherwise we might end up to just move the imbalance 11575 * on another group. Of course this applies only if 11576 * there is more than 1 CPU per group. 11577 */ 11578 goto out_balanced; 11579 } 11580 11581 if (busiest->sum_h_nr_running == 1) { 11582 /* 11583 * busiest doesn't have any tasks waiting to run 11584 */ 11585 goto out_balanced; 11586 } 11587 } 11588 11589 force_balance: 11590 /* Looks like there is an imbalance. Compute it */ 11591 calculate_imbalance(env, &sds); 11592 return env->imbalance ? sds.busiest : NULL; 11593 11594 out_balanced: 11595 env->imbalance = 0; 11596 return NULL; 11597 } 11598 11599 /* 11600 * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group. 11601 */ 11602 static struct rq *sched_balance_find_src_rq(struct lb_env *env, 11603 struct sched_group *group) 11604 { 11605 struct rq *busiest = NULL, *rq; 11606 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; 11607 unsigned int busiest_nr = 0; 11608 int i; 11609 11610 for_each_cpu_and(i, sched_group_span(group), env->cpus) { 11611 unsigned long capacity, load, util; 11612 unsigned int nr_running; 11613 enum fbq_type rt; 11614 11615 rq = cpu_rq(i); 11616 rt = fbq_classify_rq(rq); 11617 11618 /* 11619 * We classify groups/runqueues into three groups: 11620 * - regular: there are !numa tasks 11621 * - remote: there are numa tasks that run on the 'wrong' node 11622 * - all: there is no distinction 11623 * 11624 * In order to avoid migrating ideally placed numa tasks, 11625 * ignore those when there's better options. 11626 * 11627 * If we ignore the actual busiest queue to migrate another 11628 * task, the next balance pass can still reduce the busiest 11629 * queue by moving tasks around inside the node. 11630 * 11631 * If we cannot move enough load due to this classification 11632 * the next pass will adjust the group classification and 11633 * allow migration of more tasks. 11634 * 11635 * Both cases only affect the total convergence complexity. 11636 */ 11637 if (rt > env->fbq_type) 11638 continue; 11639 11640 nr_running = rq->cfs.h_nr_runnable; 11641 if (!nr_running) 11642 continue; 11643 11644 capacity = capacity_of(i); 11645 11646 /* 11647 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could 11648 * eventually lead to active_balancing high->low capacity. 11649 * Higher per-CPU capacity is considered better than balancing 11650 * average load. 11651 */ 11652 if (env->sd->flags & SD_ASYM_CPUCAPACITY && 11653 !capacity_greater(capacity_of(env->dst_cpu), capacity) && 11654 nr_running == 1) 11655 continue; 11656 11657 /* 11658 * Make sure we only pull tasks from a CPU of lower priority 11659 * when balancing between SMT siblings. 11660 * 11661 * If balancing between cores, let lower priority CPUs help 11662 * SMT cores with more than one busy sibling. 11663 */ 11664 if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1) 11665 continue; 11666 11667 switch (env->migration_type) { 11668 case migrate_load: 11669 /* 11670 * When comparing with load imbalance, use cpu_load() 11671 * which is not scaled with the CPU capacity. 11672 */ 11673 load = cpu_load(rq); 11674 11675 if (nr_running == 1 && load > env->imbalance && 11676 !check_cpu_capacity(rq, env->sd)) 11677 break; 11678 11679 /* 11680 * For the load comparisons with the other CPUs, 11681 * consider the cpu_load() scaled with the CPU 11682 * capacity, so that the load can be moved away 11683 * from the CPU that is potentially running at a 11684 * lower capacity. 11685 * 11686 * Thus we're looking for max(load_i / capacity_i), 11687 * crosswise multiplication to rid ourselves of the 11688 * division works out to: 11689 * load_i * capacity_j > load_j * capacity_i; 11690 * where j is our previous maximum. 11691 */ 11692 if (load * busiest_capacity > busiest_load * capacity) { 11693 busiest_load = load; 11694 busiest_capacity = capacity; 11695 busiest = rq; 11696 } 11697 break; 11698 11699 case migrate_util: 11700 util = cpu_util_cfs_boost(i); 11701 11702 /* 11703 * Don't try to pull utilization from a CPU with one 11704 * running task. Whatever its utilization, we will fail 11705 * detach the task. 11706 */ 11707 if (nr_running <= 1) 11708 continue; 11709 11710 if (busiest_util < util) { 11711 busiest_util = util; 11712 busiest = rq; 11713 } 11714 break; 11715 11716 case migrate_task: 11717 if (busiest_nr < nr_running) { 11718 busiest_nr = nr_running; 11719 busiest = rq; 11720 } 11721 break; 11722 11723 case migrate_misfit: 11724 /* 11725 * For ASYM_CPUCAPACITY domains with misfit tasks we 11726 * simply seek the "biggest" misfit task. 11727 */ 11728 if (rq->misfit_task_load > busiest_load) { 11729 busiest_load = rq->misfit_task_load; 11730 busiest = rq; 11731 } 11732 11733 break; 11734 11735 } 11736 } 11737 11738 return busiest; 11739 } 11740 11741 /* 11742 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but 11743 * so long as it is large enough. 11744 */ 11745 #define MAX_PINNED_INTERVAL 512 11746 11747 static inline bool 11748 asym_active_balance(struct lb_env *env) 11749 { 11750 /* 11751 * ASYM_PACKING needs to force migrate tasks from busy but lower 11752 * priority CPUs in order to pack all tasks in the highest priority 11753 * CPUs. When done between cores, do it only if the whole core if the 11754 * whole core is idle. 11755 * 11756 * If @env::src_cpu is an SMT core with busy siblings, let 11757 * the lower priority @env::dst_cpu help it. Do not follow 11758 * CPU priority. 11759 */ 11760 return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) && 11761 (sched_asym_prefer(env->dst_cpu, env->src_cpu) || 11762 !sched_use_asym_prio(env->sd, env->src_cpu)); 11763 } 11764 11765 static inline bool 11766 imbalanced_active_balance(struct lb_env *env) 11767 { 11768 struct sched_domain *sd = env->sd; 11769 11770 /* 11771 * The imbalanced case includes the case of pinned tasks preventing a fair 11772 * distribution of the load on the system but also the even distribution of the 11773 * threads on a system with spare capacity 11774 */ 11775 if ((env->migration_type == migrate_task) && 11776 (sd->nr_balance_failed > sd->cache_nice_tries+2)) 11777 return 1; 11778 11779 return 0; 11780 } 11781 11782 static int need_active_balance(struct lb_env *env) 11783 { 11784 struct sched_domain *sd = env->sd; 11785 11786 if (asym_active_balance(env)) 11787 return 1; 11788 11789 if (imbalanced_active_balance(env)) 11790 return 1; 11791 11792 /* 11793 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task. 11794 * It's worth migrating the task if the src_cpu's capacity is reduced 11795 * because of other sched_class or IRQs if more capacity stays 11796 * available on dst_cpu. 11797 */ 11798 if (env->idle && 11799 (env->src_rq->cfs.h_nr_runnable == 1)) { 11800 if ((check_cpu_capacity(env->src_rq, sd)) && 11801 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100)) 11802 return 1; 11803 } 11804 11805 if (env->migration_type == migrate_misfit) 11806 return 1; 11807 11808 return 0; 11809 } 11810 11811 static int active_load_balance_cpu_stop(void *data); 11812 11813 static int should_we_balance(struct lb_env *env) 11814 { 11815 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask); 11816 struct sched_group *sg = env->sd->groups; 11817 int cpu, idle_smt = -1; 11818 11819 /* 11820 * Ensure the balancing environment is consistent; can happen 11821 * when the softirq triggers 'during' hotplug. 11822 */ 11823 if (!cpumask_test_cpu(env->dst_cpu, env->cpus)) 11824 return 0; 11825 11826 /* 11827 * In the newly idle case, we will allow all the CPUs 11828 * to do the newly idle load balance. 11829 * 11830 * However, we bail out if we already have tasks or a wakeup pending, 11831 * to optimize wakeup latency. 11832 */ 11833 if (env->idle == CPU_NEWLY_IDLE) { 11834 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending) 11835 return 0; 11836 return 1; 11837 } 11838 11839 cpumask_copy(swb_cpus, group_balance_mask(sg)); 11840 /* Try to find first idle CPU */ 11841 for_each_cpu_and(cpu, swb_cpus, env->cpus) { 11842 if (!idle_cpu(cpu)) 11843 continue; 11844 11845 /* 11846 * Don't balance to idle SMT in busy core right away when 11847 * balancing cores, but remember the first idle SMT CPU for 11848 * later consideration. Find CPU on an idle core first. 11849 */ 11850 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) { 11851 if (idle_smt == -1) 11852 idle_smt = cpu; 11853 /* 11854 * If the core is not idle, and first SMT sibling which is 11855 * idle has been found, then its not needed to check other 11856 * SMT siblings for idleness: 11857 */ 11858 #ifdef CONFIG_SCHED_SMT 11859 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); 11860 #endif 11861 continue; 11862 } 11863 11864 /* 11865 * Are we the first idle core in a non-SMT domain or higher, 11866 * or the first idle CPU in a SMT domain? 11867 */ 11868 return cpu == env->dst_cpu; 11869 } 11870 11871 /* Are we the first idle CPU with busy siblings? */ 11872 if (idle_smt != -1) 11873 return idle_smt == env->dst_cpu; 11874 11875 /* Are we the first CPU of this group ? */ 11876 return group_balance_cpu(sg) == env->dst_cpu; 11877 } 11878 11879 static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd, 11880 enum cpu_idle_type idle) 11881 { 11882 if (!schedstat_enabled()) 11883 return; 11884 11885 switch (env->migration_type) { 11886 case migrate_load: 11887 __schedstat_add(sd->lb_imbalance_load[idle], env->imbalance); 11888 break; 11889 case migrate_util: 11890 __schedstat_add(sd->lb_imbalance_util[idle], env->imbalance); 11891 break; 11892 case migrate_task: 11893 __schedstat_add(sd->lb_imbalance_task[idle], env->imbalance); 11894 break; 11895 case migrate_misfit: 11896 __schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance); 11897 break; 11898 } 11899 } 11900 11901 /* 11902 * This flag serializes load-balancing passes over large domains 11903 * (above the NODE topology level) - only one load-balancing instance 11904 * may run at a time, to reduce overhead on very large systems with 11905 * lots of CPUs and large NUMA distances. 11906 * 11907 * - Note that load-balancing passes triggered while another one 11908 * is executing are skipped and not re-tried. 11909 * 11910 * - Also note that this does not serialize rebalance_domains() 11911 * execution, as non-SD_SERIALIZE domains will still be 11912 * load-balanced in parallel. 11913 */ 11914 static atomic_t sched_balance_running = ATOMIC_INIT(0); 11915 11916 /* 11917 * Check this_cpu to ensure it is balanced within domain. Attempt to move 11918 * tasks if there is an imbalance. 11919 */ 11920 static int sched_balance_rq(int this_cpu, struct rq *this_rq, 11921 struct sched_domain *sd, enum cpu_idle_type idle, 11922 int *continue_balancing) 11923 { 11924 int ld_moved, cur_ld_moved, active_balance = 0; 11925 struct sched_domain *sd_parent = sd->parent; 11926 struct sched_group *group; 11927 struct rq *busiest; 11928 struct rq_flags rf; 11929 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask); 11930 struct lb_env env = { 11931 .sd = sd, 11932 .dst_cpu = this_cpu, 11933 .dst_rq = this_rq, 11934 .dst_grpmask = group_balance_mask(sd->groups), 11935 .idle = idle, 11936 .loop_break = SCHED_NR_MIGRATE_BREAK, 11937 .cpus = cpus, 11938 .fbq_type = all, 11939 .tasks = LIST_HEAD_INIT(env.tasks), 11940 }; 11941 bool need_unlock = false; 11942 11943 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask); 11944 11945 schedstat_inc(sd->lb_count[idle]); 11946 11947 redo: 11948 if (!should_we_balance(&env)) { 11949 *continue_balancing = 0; 11950 goto out_balanced; 11951 } 11952 11953 if (!need_unlock && (sd->flags & SD_SERIALIZE)) { 11954 int zero = 0; 11955 if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1)) 11956 goto out_balanced; 11957 11958 need_unlock = true; 11959 } 11960 11961 group = sched_balance_find_src_group(&env); 11962 if (!group) { 11963 schedstat_inc(sd->lb_nobusyg[idle]); 11964 goto out_balanced; 11965 } 11966 11967 busiest = sched_balance_find_src_rq(&env, group); 11968 if (!busiest) { 11969 schedstat_inc(sd->lb_nobusyq[idle]); 11970 goto out_balanced; 11971 } 11972 11973 WARN_ON_ONCE(busiest == env.dst_rq); 11974 11975 update_lb_imbalance_stat(&env, sd, idle); 11976 11977 env.src_cpu = busiest->cpu; 11978 env.src_rq = busiest; 11979 11980 ld_moved = 0; 11981 /* Clear this flag as soon as we find a pullable task */ 11982 env.flags |= LBF_ALL_PINNED; 11983 if (busiest->nr_running > 1) { 11984 /* 11985 * Attempt to move tasks. If sched_balance_find_src_group has found 11986 * an imbalance but busiest->nr_running <= 1, the group is 11987 * still unbalanced. ld_moved simply stays zero, so it is 11988 * correctly treated as an imbalance. 11989 */ 11990 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running); 11991 11992 more_balance: 11993 rq_lock_irqsave(busiest, &rf); 11994 update_rq_clock(busiest); 11995 11996 /* 11997 * cur_ld_moved - load moved in current iteration 11998 * ld_moved - cumulative load moved across iterations 11999 */ 12000 cur_ld_moved = detach_tasks(&env); 12001 12002 /* 12003 * We've detached some tasks from busiest_rq. Every 12004 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely 12005 * unlock busiest->lock, and we are able to be sure 12006 * that nobody can manipulate the tasks in parallel. 12007 * See task_rq_lock() family for the details. 12008 */ 12009 12010 rq_unlock(busiest, &rf); 12011 12012 if (cur_ld_moved) { 12013 attach_tasks(&env); 12014 ld_moved += cur_ld_moved; 12015 } 12016 12017 local_irq_restore(rf.flags); 12018 12019 if (env.flags & LBF_NEED_BREAK) { 12020 env.flags &= ~LBF_NEED_BREAK; 12021 goto more_balance; 12022 } 12023 12024 /* 12025 * Revisit (affine) tasks on src_cpu that couldn't be moved to 12026 * us and move them to an alternate dst_cpu in our sched_group 12027 * where they can run. The upper limit on how many times we 12028 * iterate on same src_cpu is dependent on number of CPUs in our 12029 * sched_group. 12030 * 12031 * This changes load balance semantics a bit on who can move 12032 * load to a given_cpu. In addition to the given_cpu itself 12033 * (or a ilb_cpu acting on its behalf where given_cpu is 12034 * nohz-idle), we now have balance_cpu in a position to move 12035 * load to given_cpu. In rare situations, this may cause 12036 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding 12037 * _independently_ and at _same_ time to move some load to 12038 * given_cpu) causing excess load to be moved to given_cpu. 12039 * This however should not happen so much in practice and 12040 * moreover subsequent load balance cycles should correct the 12041 * excess load moved. 12042 */ 12043 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) { 12044 12045 /* Prevent to re-select dst_cpu via env's CPUs */ 12046 __cpumask_clear_cpu(env.dst_cpu, env.cpus); 12047 12048 env.dst_rq = cpu_rq(env.new_dst_cpu); 12049 env.dst_cpu = env.new_dst_cpu; 12050 env.flags &= ~LBF_DST_PINNED; 12051 env.loop = 0; 12052 env.loop_break = SCHED_NR_MIGRATE_BREAK; 12053 12054 /* 12055 * Go back to "more_balance" rather than "redo" since we 12056 * need to continue with same src_cpu. 12057 */ 12058 goto more_balance; 12059 } 12060 12061 /* 12062 * We failed to reach balance because of affinity. 12063 */ 12064 if (sd_parent) { 12065 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 12066 12067 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0) 12068 *group_imbalance = 1; 12069 } 12070 12071 /* All tasks on this runqueue were pinned by CPU affinity */ 12072 if (unlikely(env.flags & LBF_ALL_PINNED)) { 12073 __cpumask_clear_cpu(cpu_of(busiest), cpus); 12074 /* 12075 * Attempting to continue load balancing at the current 12076 * sched_domain level only makes sense if there are 12077 * active CPUs remaining as possible busiest CPUs to 12078 * pull load from which are not contained within the 12079 * destination group that is receiving any migrated 12080 * load. 12081 */ 12082 if (!cpumask_subset(cpus, env.dst_grpmask)) { 12083 env.loop = 0; 12084 env.loop_break = SCHED_NR_MIGRATE_BREAK; 12085 goto redo; 12086 } 12087 goto out_all_pinned; 12088 } 12089 } 12090 12091 if (!ld_moved) { 12092 schedstat_inc(sd->lb_failed[idle]); 12093 /* 12094 * Increment the failure counter only on periodic balance. 12095 * We do not want newidle balance, which can be very 12096 * frequent, pollute the failure counter causing 12097 * excessive cache_hot migrations and active balances. 12098 * 12099 * Similarly for migration_misfit which is not related to 12100 * load/util migration, don't pollute nr_balance_failed. 12101 */ 12102 if (idle != CPU_NEWLY_IDLE && 12103 env.migration_type != migrate_misfit) 12104 sd->nr_balance_failed++; 12105 12106 if (need_active_balance(&env)) { 12107 unsigned long flags; 12108 12109 raw_spin_rq_lock_irqsave(busiest, flags); 12110 12111 /* 12112 * Don't kick the active_load_balance_cpu_stop, 12113 * if the curr task on busiest CPU can't be 12114 * moved to this_cpu: 12115 */ 12116 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { 12117 raw_spin_rq_unlock_irqrestore(busiest, flags); 12118 goto out_one_pinned; 12119 } 12120 12121 /* Record that we found at least one task that could run on this_cpu */ 12122 env.flags &= ~LBF_ALL_PINNED; 12123 12124 /* 12125 * ->active_balance synchronizes accesses to 12126 * ->active_balance_work. Once set, it's cleared 12127 * only after active load balance is finished. 12128 */ 12129 if (!busiest->active_balance) { 12130 busiest->active_balance = 1; 12131 busiest->push_cpu = this_cpu; 12132 active_balance = 1; 12133 } 12134 12135 preempt_disable(); 12136 raw_spin_rq_unlock_irqrestore(busiest, flags); 12137 if (active_balance) { 12138 stop_one_cpu_nowait(cpu_of(busiest), 12139 active_load_balance_cpu_stop, busiest, 12140 &busiest->active_balance_work); 12141 } 12142 preempt_enable(); 12143 } 12144 } else { 12145 sd->nr_balance_failed = 0; 12146 } 12147 12148 if (likely(!active_balance) || need_active_balance(&env)) { 12149 /* We were unbalanced, so reset the balancing interval */ 12150 sd->balance_interval = sd->min_interval; 12151 } 12152 12153 goto out; 12154 12155 out_balanced: 12156 /* 12157 * We reach balance although we may have faced some affinity 12158 * constraints. Clear the imbalance flag only if other tasks got 12159 * a chance to move and fix the imbalance. 12160 */ 12161 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) { 12162 int *group_imbalance = &sd_parent->groups->sgc->imbalance; 12163 12164 if (*group_imbalance) 12165 *group_imbalance = 0; 12166 } 12167 12168 out_all_pinned: 12169 /* 12170 * We reach balance because all tasks are pinned at this level so 12171 * we can't migrate them. Let the imbalance flag set so parent level 12172 * can try to migrate them. 12173 */ 12174 schedstat_inc(sd->lb_balanced[idle]); 12175 12176 sd->nr_balance_failed = 0; 12177 12178 out_one_pinned: 12179 ld_moved = 0; 12180 12181 /* 12182 * sched_balance_newidle() disregards balance intervals, so we could 12183 * repeatedly reach this code, which would lead to balance_interval 12184 * skyrocketing in a short amount of time. Skip the balance_interval 12185 * increase logic to avoid that. 12186 * 12187 * Similarly misfit migration which is not necessarily an indication of 12188 * the system being busy and requires lb to backoff to let it settle 12189 * down. 12190 */ 12191 if (env.idle == CPU_NEWLY_IDLE || 12192 env.migration_type == migrate_misfit) 12193 goto out; 12194 12195 /* tune up the balancing interval */ 12196 if ((env.flags & LBF_ALL_PINNED && 12197 sd->balance_interval < MAX_PINNED_INTERVAL) || 12198 sd->balance_interval < sd->max_interval) 12199 sd->balance_interval *= 2; 12200 out: 12201 if (need_unlock) 12202 atomic_set_release(&sched_balance_running, 0); 12203 12204 return ld_moved; 12205 } 12206 12207 static inline unsigned long 12208 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy) 12209 { 12210 unsigned long interval = sd->balance_interval; 12211 12212 if (cpu_busy) 12213 interval *= sd->busy_factor; 12214 12215 /* scale ms to jiffies */ 12216 interval = msecs_to_jiffies(interval); 12217 12218 /* 12219 * Reduce likelihood of busy balancing at higher domains racing with 12220 * balancing at lower domains by preventing their balancing periods 12221 * from being multiples of each other. 12222 */ 12223 if (cpu_busy) 12224 interval -= 1; 12225 12226 interval = clamp(interval, 1UL, max_load_balance_interval); 12227 12228 return interval; 12229 } 12230 12231 static inline void 12232 update_next_balance(struct sched_domain *sd, unsigned long *next_balance) 12233 { 12234 unsigned long interval, next; 12235 12236 /* used by idle balance, so cpu_busy = 0 */ 12237 interval = get_sd_balance_interval(sd, 0); 12238 next = sd->last_balance + interval; 12239 12240 if (time_after(*next_balance, next)) 12241 *next_balance = next; 12242 } 12243 12244 /* 12245 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes 12246 * running tasks off the busiest CPU onto idle CPUs. It requires at 12247 * least 1 task to be running on each physical CPU where possible, and 12248 * avoids physical / logical imbalances. 12249 */ 12250 static int active_load_balance_cpu_stop(void *data) 12251 { 12252 struct rq *busiest_rq = data; 12253 int busiest_cpu = cpu_of(busiest_rq); 12254 int target_cpu = busiest_rq->push_cpu; 12255 struct rq *target_rq = cpu_rq(target_cpu); 12256 struct sched_domain *sd; 12257 struct task_struct *p = NULL; 12258 struct rq_flags rf; 12259 12260 rq_lock_irq(busiest_rq, &rf); 12261 /* 12262 * Between queueing the stop-work and running it is a hole in which 12263 * CPUs can become inactive. We should not move tasks from or to 12264 * inactive CPUs. 12265 */ 12266 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu)) 12267 goto out_unlock; 12268 12269 /* Make sure the requested CPU hasn't gone down in the meantime: */ 12270 if (unlikely(busiest_cpu != smp_processor_id() || 12271 !busiest_rq->active_balance)) 12272 goto out_unlock; 12273 12274 /* Is there any task to move? */ 12275 if (busiest_rq->nr_running <= 1) 12276 goto out_unlock; 12277 12278 /* 12279 * This condition is "impossible", if it occurs 12280 * we need to fix it. Originally reported by 12281 * Bjorn Helgaas on a 128-CPU setup. 12282 */ 12283 WARN_ON_ONCE(busiest_rq == target_rq); 12284 12285 /* Search for an sd spanning us and the target CPU. */ 12286 rcu_read_lock(); 12287 for_each_domain(target_cpu, sd) { 12288 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) 12289 break; 12290 } 12291 12292 if (likely(sd)) { 12293 struct lb_env env = { 12294 .sd = sd, 12295 .dst_cpu = target_cpu, 12296 .dst_rq = target_rq, 12297 .src_cpu = busiest_rq->cpu, 12298 .src_rq = busiest_rq, 12299 .idle = CPU_IDLE, 12300 .flags = LBF_ACTIVE_LB, 12301 }; 12302 12303 schedstat_inc(sd->alb_count); 12304 update_rq_clock(busiest_rq); 12305 12306 p = detach_one_task(&env); 12307 if (p) { 12308 schedstat_inc(sd->alb_pushed); 12309 /* Active balancing done, reset the failure counter. */ 12310 sd->nr_balance_failed = 0; 12311 } else { 12312 schedstat_inc(sd->alb_failed); 12313 } 12314 } 12315 rcu_read_unlock(); 12316 out_unlock: 12317 busiest_rq->active_balance = 0; 12318 rq_unlock(busiest_rq, &rf); 12319 12320 if (p) 12321 attach_one_task(target_rq, p); 12322 12323 local_irq_enable(); 12324 12325 return 0; 12326 } 12327 12328 /* 12329 * Scale the max sched_balance_rq interval with the number of CPUs in the system. 12330 * This trades load-balance latency on larger machines for less cross talk. 12331 */ 12332 void update_max_interval(void) 12333 { 12334 max_load_balance_interval = HZ*num_online_cpus()/10; 12335 } 12336 12337 static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success) 12338 { 12339 sd->newidle_call++; 12340 sd->newidle_success += success; 12341 12342 if (sd->newidle_call >= 1024) { 12343 sd->newidle_ratio = sd->newidle_success; 12344 sd->newidle_call /= 2; 12345 sd->newidle_success /= 2; 12346 } 12347 } 12348 12349 static inline bool 12350 update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success) 12351 { 12352 unsigned long next_decay = sd->last_decay_max_lb_cost + HZ; 12353 unsigned long now = jiffies; 12354 12355 if (cost) 12356 update_newidle_stats(sd, success); 12357 12358 if (cost > sd->max_newidle_lb_cost) { 12359 /* 12360 * Track max cost of a domain to make sure to not delay the 12361 * next wakeup on the CPU. 12362 */ 12363 sd->max_newidle_lb_cost = cost; 12364 sd->last_decay_max_lb_cost = now; 12365 12366 } else if (time_after(now, next_decay)) { 12367 /* 12368 * Decay the newidle max times by ~1% per second to ensure that 12369 * it is not outdated and the current max cost is actually 12370 * shorter. 12371 */ 12372 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256; 12373 sd->last_decay_max_lb_cost = now; 12374 return true; 12375 } 12376 12377 return false; 12378 } 12379 12380 /* 12381 * It checks each scheduling domain to see if it is due to be balanced, 12382 * and initiates a balancing operation if so. 12383 * 12384 * Balancing parameters are set up in init_sched_domains. 12385 */ 12386 static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle) 12387 { 12388 int continue_balancing = 1; 12389 int cpu = rq->cpu; 12390 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu); 12391 unsigned long interval; 12392 struct sched_domain *sd; 12393 /* Earliest time when we have to do rebalance again */ 12394 unsigned long next_balance = jiffies + 60*HZ; 12395 int update_next_balance = 0; 12396 int need_decay = 0; 12397 u64 max_cost = 0; 12398 12399 rcu_read_lock(); 12400 for_each_domain(cpu, sd) { 12401 /* 12402 * Decay the newidle max times here because this is a regular 12403 * visit to all the domains. 12404 */ 12405 need_decay = update_newidle_cost(sd, 0, 0); 12406 max_cost += sd->max_newidle_lb_cost; 12407 12408 /* 12409 * Stop the load balance at this level. There is another 12410 * CPU in our sched group which is doing load balancing more 12411 * actively. 12412 */ 12413 if (!continue_balancing) { 12414 if (need_decay) 12415 continue; 12416 break; 12417 } 12418 12419 interval = get_sd_balance_interval(sd, busy); 12420 if (time_after_eq(jiffies, sd->last_balance + interval)) { 12421 if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) { 12422 /* 12423 * The LBF_DST_PINNED logic could have changed 12424 * env->dst_cpu, so we can't know our idle 12425 * state even if we migrated tasks. Update it. 12426 */ 12427 idle = idle_cpu(cpu); 12428 busy = !idle && !sched_idle_cpu(cpu); 12429 } 12430 sd->last_balance = jiffies; 12431 interval = get_sd_balance_interval(sd, busy); 12432 } 12433 if (time_after(next_balance, sd->last_balance + interval)) { 12434 next_balance = sd->last_balance + interval; 12435 update_next_balance = 1; 12436 } 12437 } 12438 if (need_decay) { 12439 /* 12440 * Ensure the rq-wide value also decays but keep it at a 12441 * reasonable floor to avoid funnies with rq->avg_idle. 12442 */ 12443 rq->max_idle_balance_cost = 12444 max((u64)sysctl_sched_migration_cost, max_cost); 12445 } 12446 rcu_read_unlock(); 12447 12448 /* 12449 * next_balance will be updated only when there is a need. 12450 * When the cpu is attached to null domain for ex, it will not be 12451 * updated. 12452 */ 12453 if (likely(update_next_balance)) 12454 rq->next_balance = next_balance; 12455 12456 } 12457 12458 static inline int on_null_domain(struct rq *rq) 12459 { 12460 return unlikely(!rcu_dereference_sched(rq->sd)); 12461 } 12462 12463 #ifdef CONFIG_NO_HZ_COMMON 12464 /* 12465 * NOHZ idle load balancing (ILB) details: 12466 * 12467 * - When one of the busy CPUs notices that there may be an idle rebalancing 12468 * needed, they will kick the idle load balancer, which then does idle 12469 * load balancing for all the idle CPUs. 12470 */ 12471 static inline int find_new_ilb(void) 12472 { 12473 const struct cpumask *hk_mask; 12474 int ilb_cpu; 12475 12476 hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); 12477 12478 for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) { 12479 12480 if (ilb_cpu == smp_processor_id()) 12481 continue; 12482 12483 if (idle_cpu(ilb_cpu)) 12484 return ilb_cpu; 12485 } 12486 12487 return -1; 12488 } 12489 12490 /* 12491 * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU 12492 * SMP function call (IPI). 12493 * 12494 * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set 12495 * (if there is one). 12496 */ 12497 static void kick_ilb(unsigned int flags) 12498 { 12499 int ilb_cpu; 12500 12501 /* 12502 * Increase nohz.next_balance only when if full ilb is triggered but 12503 * not if we only update stats. 12504 */ 12505 if (flags & NOHZ_BALANCE_KICK) 12506 nohz.next_balance = jiffies+1; 12507 12508 ilb_cpu = find_new_ilb(); 12509 if (ilb_cpu < 0) 12510 return; 12511 12512 /* 12513 * Don't bother if no new NOHZ balance work items for ilb_cpu, 12514 * i.e. all bits in flags are already set in ilb_cpu. 12515 */ 12516 if ((atomic_read(nohz_flags(ilb_cpu)) & flags) == flags) 12517 return; 12518 12519 /* 12520 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets 12521 * the first flag owns it; cleared by nohz_csd_func(). 12522 */ 12523 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); 12524 if (flags & NOHZ_KICK_MASK) 12525 return; 12526 12527 /* 12528 * This way we generate an IPI on the target CPU which 12529 * is idle, and the softirq performing NOHZ idle load balancing 12530 * will be run before returning from the IPI. 12531 */ 12532 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); 12533 } 12534 12535 /* 12536 * Current decision point for kicking the idle load balancer in the presence 12537 * of idle CPUs in the system. 12538 */ 12539 static void nohz_balancer_kick(struct rq *rq) 12540 { 12541 unsigned long now = jiffies; 12542 struct sched_domain_shared *sds; 12543 struct sched_domain *sd; 12544 int nr_busy, i, cpu = rq->cpu; 12545 unsigned int flags = 0; 12546 12547 if (unlikely(rq->idle_balance)) 12548 return; 12549 12550 /* 12551 * We may be recently in ticked or tickless idle mode. At the first 12552 * busy tick after returning from idle, we will update the busy stats. 12553 */ 12554 nohz_balance_exit_idle(rq); 12555 12556 /* 12557 * None are in tickless mode and hence no need for NOHZ idle load 12558 * balancing: 12559 */ 12560 if (likely(!atomic_read(&nohz.nr_cpus))) 12561 return; 12562 12563 if (READ_ONCE(nohz.has_blocked) && 12564 time_after(now, READ_ONCE(nohz.next_blocked))) 12565 flags = NOHZ_STATS_KICK; 12566 12567 if (time_before(now, nohz.next_balance)) 12568 goto out; 12569 12570 if (rq->nr_running >= 2) { 12571 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12572 goto out; 12573 } 12574 12575 rcu_read_lock(); 12576 12577 sd = rcu_dereference(rq->sd); 12578 if (sd) { 12579 /* 12580 * If there's a runnable CFS task and the current CPU has reduced 12581 * capacity, kick the ILB to see if there's a better CPU to run on: 12582 */ 12583 if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) { 12584 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12585 goto unlock; 12586 } 12587 } 12588 12589 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu)); 12590 if (sd) { 12591 /* 12592 * When ASYM_PACKING; see if there's a more preferred CPU 12593 * currently idle; in which case, kick the ILB to move tasks 12594 * around. 12595 * 12596 * When balancing between cores, all the SMT siblings of the 12597 * preferred CPU must be idle. 12598 */ 12599 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) { 12600 if (sched_asym(sd, i, cpu)) { 12601 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12602 goto unlock; 12603 } 12604 } 12605 } 12606 12607 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu)); 12608 if (sd) { 12609 /* 12610 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU 12611 * to run the misfit task on. 12612 */ 12613 if (check_misfit_status(rq)) { 12614 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12615 goto unlock; 12616 } 12617 12618 /* 12619 * For asymmetric systems, we do not want to nicely balance 12620 * cache use, instead we want to embrace asymmetry and only 12621 * ensure tasks have enough CPU capacity. 12622 * 12623 * Skip the LLC logic because it's not relevant in that case. 12624 */ 12625 goto unlock; 12626 } 12627 12628 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 12629 if (sds) { 12630 /* 12631 * If there is an imbalance between LLC domains (IOW we could 12632 * increase the overall cache utilization), we need a less-loaded LLC 12633 * domain to pull some load from. Likewise, we may need to spread 12634 * load within the current LLC domain (e.g. packed SMT cores but 12635 * other CPUs are idle). We can't really know from here how busy 12636 * the others are - so just get a NOHZ balance going if it looks 12637 * like this LLC domain has tasks we could move. 12638 */ 12639 nr_busy = atomic_read(&sds->nr_busy_cpus); 12640 if (nr_busy > 1) { 12641 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK; 12642 goto unlock; 12643 } 12644 } 12645 unlock: 12646 rcu_read_unlock(); 12647 out: 12648 if (READ_ONCE(nohz.needs_update)) 12649 flags |= NOHZ_NEXT_KICK; 12650 12651 if (flags) 12652 kick_ilb(flags); 12653 } 12654 12655 static void set_cpu_sd_state_busy(int cpu) 12656 { 12657 struct sched_domain *sd; 12658 12659 rcu_read_lock(); 12660 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 12661 12662 if (!sd || !sd->nohz_idle) 12663 goto unlock; 12664 sd->nohz_idle = 0; 12665 12666 atomic_inc(&sd->shared->nr_busy_cpus); 12667 unlock: 12668 rcu_read_unlock(); 12669 } 12670 12671 void nohz_balance_exit_idle(struct rq *rq) 12672 { 12673 WARN_ON_ONCE(rq != this_rq()); 12674 12675 if (likely(!rq->nohz_tick_stopped)) 12676 return; 12677 12678 rq->nohz_tick_stopped = 0; 12679 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask); 12680 atomic_dec(&nohz.nr_cpus); 12681 12682 set_cpu_sd_state_busy(rq->cpu); 12683 } 12684 12685 static void set_cpu_sd_state_idle(int cpu) 12686 { 12687 struct sched_domain *sd; 12688 12689 rcu_read_lock(); 12690 sd = rcu_dereference(per_cpu(sd_llc, cpu)); 12691 12692 if (!sd || sd->nohz_idle) 12693 goto unlock; 12694 sd->nohz_idle = 1; 12695 12696 atomic_dec(&sd->shared->nr_busy_cpus); 12697 unlock: 12698 rcu_read_unlock(); 12699 } 12700 12701 /* 12702 * This routine will record that the CPU is going idle with tick stopped. 12703 * This info will be used in performing idle load balancing in the future. 12704 */ 12705 void nohz_balance_enter_idle(int cpu) 12706 { 12707 struct rq *rq = cpu_rq(cpu); 12708 12709 WARN_ON_ONCE(cpu != smp_processor_id()); 12710 12711 /* If this CPU is going down, then nothing needs to be done: */ 12712 if (!cpu_active(cpu)) 12713 return; 12714 12715 /* 12716 * Can be set safely without rq->lock held 12717 * If a clear happens, it will have evaluated last additions because 12718 * rq->lock is held during the check and the clear 12719 */ 12720 rq->has_blocked_load = 1; 12721 12722 /* 12723 * The tick is still stopped but load could have been added in the 12724 * meantime. We set the nohz.has_blocked flag to trig a check of the 12725 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear 12726 * of nohz.has_blocked can only happen after checking the new load 12727 */ 12728 if (rq->nohz_tick_stopped) 12729 goto out; 12730 12731 /* If we're a completely isolated CPU, we don't play: */ 12732 if (on_null_domain(rq)) 12733 return; 12734 12735 rq->nohz_tick_stopped = 1; 12736 12737 cpumask_set_cpu(cpu, nohz.idle_cpus_mask); 12738 atomic_inc(&nohz.nr_cpus); 12739 12740 /* 12741 * Ensures that if nohz_idle_balance() fails to observe our 12742 * @idle_cpus_mask store, it must observe the @has_blocked 12743 * and @needs_update stores. 12744 */ 12745 smp_mb__after_atomic(); 12746 12747 set_cpu_sd_state_idle(cpu); 12748 12749 WRITE_ONCE(nohz.needs_update, 1); 12750 out: 12751 /* 12752 * Each time a cpu enter idle, we assume that it has blocked load and 12753 * enable the periodic update of the load of idle CPUs 12754 */ 12755 WRITE_ONCE(nohz.has_blocked, 1); 12756 } 12757 12758 static bool update_nohz_stats(struct rq *rq) 12759 { 12760 unsigned int cpu = rq->cpu; 12761 12762 if (!rq->has_blocked_load) 12763 return false; 12764 12765 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask)) 12766 return false; 12767 12768 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick))) 12769 return true; 12770 12771 sched_balance_update_blocked_averages(cpu); 12772 12773 return rq->has_blocked_load; 12774 } 12775 12776 /* 12777 * Internal function that runs load balance for all idle CPUs. The load balance 12778 * can be a simple update of blocked load or a complete load balance with 12779 * tasks movement depending of flags. 12780 */ 12781 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags) 12782 { 12783 /* Earliest time when we have to do rebalance again */ 12784 unsigned long now = jiffies; 12785 unsigned long next_balance = now + 60*HZ; 12786 bool has_blocked_load = false; 12787 int update_next_balance = 0; 12788 int this_cpu = this_rq->cpu; 12789 int balance_cpu; 12790 struct rq *rq; 12791 12792 WARN_ON_ONCE((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK); 12793 12794 /* 12795 * We assume there will be no idle load after this update and clear 12796 * the has_blocked flag. If a cpu enters idle in the mean time, it will 12797 * set the has_blocked flag and trigger another update of idle load. 12798 * Because a cpu that becomes idle, is added to idle_cpus_mask before 12799 * setting the flag, we are sure to not clear the state and not 12800 * check the load of an idle cpu. 12801 * 12802 * Same applies to idle_cpus_mask vs needs_update. 12803 */ 12804 if (flags & NOHZ_STATS_KICK) 12805 WRITE_ONCE(nohz.has_blocked, 0); 12806 if (flags & NOHZ_NEXT_KICK) 12807 WRITE_ONCE(nohz.needs_update, 0); 12808 12809 /* 12810 * Ensures that if we miss the CPU, we must see the has_blocked 12811 * store from nohz_balance_enter_idle(). 12812 */ 12813 smp_mb(); 12814 12815 /* 12816 * Start with the next CPU after this_cpu so we will end with this_cpu and let a 12817 * chance for other idle cpu to pull load. 12818 */ 12819 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) { 12820 if (!idle_cpu(balance_cpu)) 12821 continue; 12822 12823 /* 12824 * If this CPU gets work to do, stop the load balancing 12825 * work being done for other CPUs. Next load 12826 * balancing owner will pick it up. 12827 */ 12828 if (!idle_cpu(this_cpu) && need_resched()) { 12829 if (flags & NOHZ_STATS_KICK) 12830 has_blocked_load = true; 12831 if (flags & NOHZ_NEXT_KICK) 12832 WRITE_ONCE(nohz.needs_update, 1); 12833 goto abort; 12834 } 12835 12836 rq = cpu_rq(balance_cpu); 12837 12838 if (flags & NOHZ_STATS_KICK) 12839 has_blocked_load |= update_nohz_stats(rq); 12840 12841 /* 12842 * If time for next balance is due, 12843 * do the balance. 12844 */ 12845 if (time_after_eq(jiffies, rq->next_balance)) { 12846 struct rq_flags rf; 12847 12848 rq_lock_irqsave(rq, &rf); 12849 update_rq_clock(rq); 12850 rq_unlock_irqrestore(rq, &rf); 12851 12852 if (flags & NOHZ_BALANCE_KICK) 12853 sched_balance_domains(rq, CPU_IDLE); 12854 } 12855 12856 if (time_after(next_balance, rq->next_balance)) { 12857 next_balance = rq->next_balance; 12858 update_next_balance = 1; 12859 } 12860 } 12861 12862 /* 12863 * next_balance will be updated only when there is a need. 12864 * When the CPU is attached to null domain for ex, it will not be 12865 * updated. 12866 */ 12867 if (likely(update_next_balance)) 12868 nohz.next_balance = next_balance; 12869 12870 if (flags & NOHZ_STATS_KICK) 12871 WRITE_ONCE(nohz.next_blocked, 12872 now + msecs_to_jiffies(LOAD_AVG_PERIOD)); 12873 12874 abort: 12875 /* There is still blocked load, enable periodic update */ 12876 if (has_blocked_load) 12877 WRITE_ONCE(nohz.has_blocked, 1); 12878 } 12879 12880 /* 12881 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the 12882 * rebalancing for all the CPUs for whom scheduler ticks are stopped. 12883 */ 12884 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12885 { 12886 unsigned int flags = this_rq->nohz_idle_balance; 12887 12888 if (!flags) 12889 return false; 12890 12891 this_rq->nohz_idle_balance = 0; 12892 12893 if (idle != CPU_IDLE) 12894 return false; 12895 12896 _nohz_idle_balance(this_rq, flags); 12897 12898 return true; 12899 } 12900 12901 /* 12902 * Check if we need to directly run the ILB for updating blocked load before 12903 * entering idle state. Here we run ILB directly without issuing IPIs. 12904 * 12905 * Note that when this function is called, the tick may not yet be stopped on 12906 * this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and 12907 * cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates 12908 * don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle 12909 * entry/exit rate (usec). So it is possible that _nohz_idle_balance() is 12910 * called from this function on (this) CPU that's not yet in the mask. That's 12911 * OK because the goal of nohz_run_idle_balance() is to run ILB only for 12912 * updating the blocked load of already idle CPUs without waking up one of 12913 * those idle CPUs and outside the preempt disable / IRQ off phase of the local 12914 * cpu about to enter idle, because it can take a long time. 12915 */ 12916 void nohz_run_idle_balance(int cpu) 12917 { 12918 unsigned int flags; 12919 12920 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); 12921 12922 /* 12923 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen 12924 * (i.e. NOHZ_STATS_KICK set) and will do the same. 12925 */ 12926 if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) 12927 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK); 12928 } 12929 12930 static void nohz_newidle_balance(struct rq *this_rq) 12931 { 12932 int this_cpu = this_rq->cpu; 12933 12934 /* Will wake up very soon. No time for doing anything else*/ 12935 if (this_rq->avg_idle < sysctl_sched_migration_cost) 12936 return; 12937 12938 /* Don't need to update blocked load of idle CPUs*/ 12939 if (!READ_ONCE(nohz.has_blocked) || 12940 time_before(jiffies, READ_ONCE(nohz.next_blocked))) 12941 return; 12942 12943 /* 12944 * Set the need to trigger ILB in order to update blocked load 12945 * before entering idle state. 12946 */ 12947 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); 12948 } 12949 12950 #else /* !CONFIG_NO_HZ_COMMON: */ 12951 static inline void nohz_balancer_kick(struct rq *rq) { } 12952 12953 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) 12954 { 12955 return false; 12956 } 12957 12958 static inline void nohz_newidle_balance(struct rq *this_rq) { } 12959 #endif /* !CONFIG_NO_HZ_COMMON */ 12960 12961 /* 12962 * sched_balance_newidle is called by schedule() if this_cpu is about to become 12963 * idle. Attempts to pull tasks from other CPUs. 12964 * 12965 * Returns: 12966 * < 0 - we released the lock and there are !fair tasks present 12967 * 0 - failed, no new tasks 12968 * > 0 - success, new (fair) tasks present 12969 */ 12970 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) 12971 { 12972 unsigned long next_balance = jiffies + HZ; 12973 int this_cpu = this_rq->cpu; 12974 int continue_balancing = 1; 12975 u64 t0, t1, curr_cost = 0; 12976 struct sched_domain *sd; 12977 int pulled_task = 0; 12978 12979 update_misfit_status(NULL, this_rq); 12980 12981 /* 12982 * There is a task waiting to run. No need to search for one. 12983 * Return 0; the task will be enqueued when switching to idle. 12984 */ 12985 if (this_rq->ttwu_pending) 12986 return 0; 12987 12988 /* 12989 * We must set idle_stamp _before_ calling sched_balance_rq() 12990 * for CPU_NEWLY_IDLE, such that we measure the this duration 12991 * as idle time. 12992 */ 12993 this_rq->idle_stamp = rq_clock(this_rq); 12994 12995 /* 12996 * Do not pull tasks towards !active CPUs... 12997 */ 12998 if (!cpu_active(this_cpu)) 12999 return 0; 13000 13001 /* 13002 * This is OK, because current is on_cpu, which avoids it being picked 13003 * for load-balance and preemption/IRQs are still disabled avoiding 13004 * further scheduler activity on it and we're being very careful to 13005 * re-start the picking loop. 13006 */ 13007 rq_unpin_lock(this_rq, rf); 13008 13009 rcu_read_lock(); 13010 sd = rcu_dereference_check_sched_domain(this_rq->sd); 13011 if (!sd) { 13012 rcu_read_unlock(); 13013 goto out; 13014 } 13015 13016 if (!get_rd_overloaded(this_rq->rd) || 13017 this_rq->avg_idle < sd->max_newidle_lb_cost) { 13018 13019 update_next_balance(sd, &next_balance); 13020 rcu_read_unlock(); 13021 goto out; 13022 } 13023 rcu_read_unlock(); 13024 13025 raw_spin_rq_unlock(this_rq); 13026 13027 t0 = sched_clock_cpu(this_cpu); 13028 sched_balance_update_blocked_averages(this_cpu); 13029 13030 rcu_read_lock(); 13031 for_each_domain(this_cpu, sd) { 13032 u64 domain_cost; 13033 13034 update_next_balance(sd, &next_balance); 13035 13036 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) 13037 break; 13038 13039 if (sd->flags & SD_BALANCE_NEWIDLE) { 13040 unsigned int weight = 1; 13041 13042 if (sched_feat(NI_RANDOM)) { 13043 /* 13044 * Throw a 1k sided dice; and only run 13045 * newidle_balance according to the success 13046 * rate. 13047 */ 13048 u32 d1k = sched_rng() % 1024; 13049 weight = 1 + sd->newidle_ratio; 13050 if (d1k > weight) { 13051 update_newidle_stats(sd, 0); 13052 continue; 13053 } 13054 weight = (1024 + weight/2) / weight; 13055 } 13056 13057 pulled_task = sched_balance_rq(this_cpu, this_rq, 13058 sd, CPU_NEWLY_IDLE, 13059 &continue_balancing); 13060 13061 t1 = sched_clock_cpu(this_cpu); 13062 domain_cost = t1 - t0; 13063 curr_cost += domain_cost; 13064 t0 = t1; 13065 13066 /* 13067 * Track max cost of a domain to make sure to not delay the 13068 * next wakeup on the CPU. 13069 */ 13070 update_newidle_cost(sd, domain_cost, weight * !!pulled_task); 13071 } 13072 13073 /* 13074 * Stop searching for tasks to pull if there are 13075 * now runnable tasks on this rq. 13076 */ 13077 if (pulled_task || !continue_balancing) 13078 break; 13079 } 13080 rcu_read_unlock(); 13081 13082 raw_spin_rq_lock(this_rq); 13083 13084 if (curr_cost > this_rq->max_idle_balance_cost) 13085 this_rq->max_idle_balance_cost = curr_cost; 13086 13087 /* 13088 * While browsing the domains, we released the rq lock, a task could 13089 * have been enqueued in the meantime. Since we're not going idle, 13090 * pretend we pulled a task. 13091 */ 13092 if (this_rq->cfs.h_nr_queued && !pulled_task) 13093 pulled_task = 1; 13094 13095 /* Is there a task of a high priority class? */ 13096 if (this_rq->nr_running != this_rq->cfs.h_nr_queued) 13097 pulled_task = -1; 13098 13099 out: 13100 /* Move the next balance forward */ 13101 if (time_after(this_rq->next_balance, next_balance)) 13102 this_rq->next_balance = next_balance; 13103 13104 if (pulled_task) 13105 this_rq->idle_stamp = 0; 13106 else 13107 nohz_newidle_balance(this_rq); 13108 13109 rq_repin_lock(this_rq, rf); 13110 13111 return pulled_task; 13112 } 13113 13114 /* 13115 * This softirq handler is triggered via SCHED_SOFTIRQ from two places: 13116 * 13117 * - directly from the local sched_tick() for periodic load balancing 13118 * 13119 * - indirectly from a remote sched_tick() for NOHZ idle balancing 13120 * through the SMP cross-call nohz_csd_func() 13121 */ 13122 static __latent_entropy void sched_balance_softirq(void) 13123 { 13124 struct rq *this_rq = this_rq(); 13125 enum cpu_idle_type idle = this_rq->idle_balance; 13126 /* 13127 * If this CPU has a pending NOHZ_BALANCE_KICK, then do the 13128 * balancing on behalf of the other idle CPUs whose ticks are 13129 * stopped. Do nohz_idle_balance *before* sched_balance_domains to 13130 * give the idle CPUs a chance to load balance. Else we may 13131 * load balance only within the local sched_domain hierarchy 13132 * and abort nohz_idle_balance altogether if we pull some load. 13133 */ 13134 if (nohz_idle_balance(this_rq, idle)) 13135 return; 13136 13137 /* normal load balance */ 13138 sched_balance_update_blocked_averages(this_rq->cpu); 13139 sched_balance_domains(this_rq, idle); 13140 } 13141 13142 /* 13143 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing. 13144 */ 13145 void sched_balance_trigger(struct rq *rq) 13146 { 13147 /* 13148 * Don't need to rebalance while attached to NULL domain or 13149 * runqueue CPU is not active 13150 */ 13151 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq)))) 13152 return; 13153 13154 if (time_after_eq(jiffies, rq->next_balance)) 13155 raise_softirq(SCHED_SOFTIRQ); 13156 13157 nohz_balancer_kick(rq); 13158 } 13159 13160 static void rq_online_fair(struct rq *rq) 13161 { 13162 update_sysctl(); 13163 13164 update_runtime_enabled(rq); 13165 } 13166 13167 static void rq_offline_fair(struct rq *rq) 13168 { 13169 update_sysctl(); 13170 13171 /* Ensure any throttled groups are reachable by pick_next_task */ 13172 unthrottle_offline_cfs_rqs(rq); 13173 13174 /* Ensure that we remove rq contribution to group share: */ 13175 clear_tg_offline_cfs_rqs(rq); 13176 } 13177 13178 #ifdef CONFIG_SCHED_CORE 13179 static inline bool 13180 __entity_slice_used(struct sched_entity *se, int min_nr_tasks) 13181 { 13182 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime; 13183 u64 slice = se->slice; 13184 13185 return (rtime * min_nr_tasks > slice); 13186 } 13187 13188 #define MIN_NR_TASKS_DURING_FORCEIDLE 2 13189 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) 13190 { 13191 if (!sched_core_enabled(rq)) 13192 return; 13193 13194 /* 13195 * If runqueue has only one task which used up its slice and 13196 * if the sibling is forced idle, then trigger schedule to 13197 * give forced idle task a chance. 13198 * 13199 * sched_slice() considers only this active rq and it gets the 13200 * whole slice. But during force idle, we have siblings acting 13201 * like a single runqueue and hence we need to consider runnable 13202 * tasks on this CPU and the forced idle CPU. Ideally, we should 13203 * go through the forced idle rq, but that would be a perf hit. 13204 * We can assume that the forced idle CPU has at least 13205 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check 13206 * if we need to give up the CPU. 13207 */ 13208 if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && 13209 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) 13210 resched_curr(rq); 13211 } 13212 13213 /* 13214 * Consider any infeasible weight scenario. Take for instance two tasks, 13215 * each bound to their respective sibling, one with weight 1 and one with 13216 * weight 2. Then the lower weight task will run ahead of the higher weight 13217 * task without bound. 13218 * 13219 * This utterly destroys the concept of a shared time base. 13220 * 13221 * Remember; all this is about a proportionally fair scheduling, where each 13222 * tasks receives: 13223 * 13224 * w_i 13225 * dt_i = ---------- dt (1) 13226 * \Sum_j w_j 13227 * 13228 * which we do by tracking a virtual time, s_i: 13229 * 13230 * 1 13231 * s_i = --- d[t]_i (2) 13232 * w_i 13233 * 13234 * Where d[t] is a delta of discrete time, while dt is an infinitesimal. 13235 * The immediate corollary is that the ideal schedule S, where (2) to use 13236 * an infinitesimal delta, is: 13237 * 13238 * 1 13239 * S = ---------- dt (3) 13240 * \Sum_i w_i 13241 * 13242 * From which we can define the lag, or deviation from the ideal, as: 13243 * 13244 * lag(i) = S - s_i (4) 13245 * 13246 * And since the one and only purpose is to approximate S, we get that: 13247 * 13248 * \Sum_i w_i lag(i) := 0 (5) 13249 * 13250 * If this were not so, we no longer converge to S, and we can no longer 13251 * claim our scheduler has any of the properties we derive from S. This is 13252 * exactly what you did above, you broke it! 13253 * 13254 * 13255 * Let's continue for a while though; to see if there is anything useful to 13256 * be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i: 13257 * 13258 * \Sum_i w_i s_i 13259 * S = -------------- (6) 13260 * \Sum_i w_i 13261 * 13262 * Which gives us a way to compute S, given our s_i. Now, if you've read 13263 * our code, you know that we do not in fact do this, the reason for this 13264 * is two-fold. Firstly, computing S in that way requires a 64bit division 13265 * for every time we'd use it (see 12), and secondly, this only describes 13266 * the steady-state, it doesn't handle dynamics. 13267 * 13268 * Anyway, in (6): s_i -> x + (s_i - x), to get: 13269 * 13270 * \Sum_i w_i (s_i - x) 13271 * S - x = -------------------- (7) 13272 * \Sum_i w_i 13273 * 13274 * Which shows that S and s_i transform alike (which makes perfect sense 13275 * given that S is basically the (weighted) average of s_i). 13276 * 13277 * So the thing to remember is that the above is strictly UP. It is 13278 * possible to generalize to multiple runqueues -- however it gets really 13279 * yuck when you have to add affinity support, as illustrated by our very 13280 * first counter-example. 13281 * 13282 * Luckily I think we can avoid needing a full multi-queue variant for 13283 * core-scheduling (or load-balancing). The crucial observation is that we 13284 * only actually need this comparison in the presence of forced-idle; only 13285 * then do we need to tell if the stalled rq has higher priority over the 13286 * other. 13287 * 13288 * [XXX assumes SMT2; better consider the more general case, I suspect 13289 * it'll work out because our comparison is always between 2 rqs and the 13290 * answer is only interesting if one of them is forced-idle] 13291 * 13292 * And (under assumption of SMT2) when there is forced-idle, there is only 13293 * a single queue, so everything works like normal. 13294 * 13295 * Let, for our runqueue 'k': 13296 * 13297 * T_k = \Sum_i w_i s_i 13298 * W_k = \Sum_i w_i ; for all i of k (8) 13299 * 13300 * Then we can write (6) like: 13301 * 13302 * T_k 13303 * S_k = --- (9) 13304 * W_k 13305 * 13306 * From which immediately follows that: 13307 * 13308 * T_k + T_l 13309 * S_k+l = --------- (10) 13310 * W_k + W_l 13311 * 13312 * On which we can define a combined lag: 13313 * 13314 * lag_k+l(i) := S_k+l - s_i (11) 13315 * 13316 * And that gives us the tools to compare tasks across a combined runqueue. 13317 * 13318 * 13319 * Combined this gives the following: 13320 * 13321 * a) when a runqueue enters force-idle, sync it against it's sibling rq(s) 13322 * using (7); this only requires storing single 'time'-stamps. 13323 * 13324 * b) when comparing tasks between 2 runqueues of which one is forced-idle, 13325 * compare the combined lag, per (11). 13326 * 13327 * Now, of course cgroups (I so hate them) make this more interesting in 13328 * that a) seems to suggest we need to iterate all cgroup on a CPU at such 13329 * boundaries, but I think we can avoid that. The force-idle is for the 13330 * whole CPU, all it's rqs. So we can mark it in the root and lazily 13331 * propagate downward on demand. 13332 */ 13333 13334 /* 13335 * So this sync is basically a relative reset of S to 0. 13336 * 13337 * So with 2 queues, when one goes idle, we drop them both to 0 and one 13338 * then increases due to not being idle, and the idle one builds up lag to 13339 * get re-elected. So far so simple, right? 13340 * 13341 * When there's 3, we can have the situation where 2 run and one is idle, 13342 * we sync to 0 and let the idle one build up lag to get re-election. Now 13343 * suppose another one also drops idle. At this point dropping all to 0 13344 * again would destroy the built-up lag from the queue that was already 13345 * idle, not good. 13346 * 13347 * So instead of syncing everything, we can: 13348 * 13349 * less := !((s64)(s_a - s_b) <= 0) 13350 * 13351 * (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b 13352 * == v_a - (v_b - S_a + S_b) 13353 * 13354 * IOW, we can recast the (lag) comparison to a one-sided difference. 13355 * So if then, instead of syncing the whole queue, sync the idle queue 13356 * against the active queue with S_a + S_b at the point where we sync. 13357 * 13358 * (XXX consider the implication of living in a cyclic group: N / 2^n N) 13359 * 13360 * This gives us means of syncing single queues against the active queue, 13361 * and for already idle queues to preserve their build-up lag. 13362 * 13363 * Of course, then we get the situation where there's 2 active and one 13364 * going idle, who do we pick to sync against? Theory would have us sync 13365 * against the combined S, but as we've already demonstrated, there is no 13366 * such thing in infeasible weight scenarios. 13367 * 13368 * One thing I've considered; and this is where that core_active rudiment 13369 * came from, is having active queues sync up between themselves after 13370 * every tick. This limits the observed divergence due to the work 13371 * conservancy. 13372 * 13373 * On top of that, we can improve upon things by employing (10) here. 13374 */ 13375 13376 /* 13377 * se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed. 13378 */ 13379 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq, 13380 bool forceidle) 13381 { 13382 for_each_sched_entity(se) { 13383 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13384 13385 if (forceidle) { 13386 if (cfs_rq->forceidle_seq == fi_seq) 13387 break; 13388 cfs_rq->forceidle_seq = fi_seq; 13389 } 13390 13391 cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime; 13392 } 13393 } 13394 13395 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi) 13396 { 13397 struct sched_entity *se = &p->se; 13398 13399 if (p->sched_class != &fair_sched_class) 13400 return; 13401 13402 se_fi_update(se, rq->core->core_forceidle_seq, in_fi); 13403 } 13404 13405 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, 13406 bool in_fi) 13407 { 13408 struct rq *rq = task_rq(a); 13409 const struct sched_entity *sea = &a->se; 13410 const struct sched_entity *seb = &b->se; 13411 struct cfs_rq *cfs_rqa; 13412 struct cfs_rq *cfs_rqb; 13413 s64 delta; 13414 13415 WARN_ON_ONCE(task_rq(b)->core != rq->core); 13416 13417 #ifdef CONFIG_FAIR_GROUP_SCHED 13418 /* 13419 * Find an se in the hierarchy for tasks a and b, such that the se's 13420 * are immediate siblings. 13421 */ 13422 while (sea->cfs_rq->tg != seb->cfs_rq->tg) { 13423 int sea_depth = sea->depth; 13424 int seb_depth = seb->depth; 13425 13426 if (sea_depth >= seb_depth) 13427 sea = parent_entity(sea); 13428 if (sea_depth <= seb_depth) 13429 seb = parent_entity(seb); 13430 } 13431 13432 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); 13433 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); 13434 13435 cfs_rqa = sea->cfs_rq; 13436 cfs_rqb = seb->cfs_rq; 13437 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 13438 cfs_rqa = &task_rq(a)->cfs; 13439 cfs_rqb = &task_rq(b)->cfs; 13440 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 13441 13442 /* 13443 * Find delta after normalizing se's vruntime with its cfs_rq's 13444 * zero_vruntime_fi, which would have been updated in prior calls 13445 * to se_fi_update(). 13446 */ 13447 delta = vruntime_op(sea->vruntime, "-", seb->vruntime) + 13448 vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi); 13449 13450 return delta > 0; 13451 } 13452 13453 static int task_is_throttled_fair(struct task_struct *p, int cpu) 13454 { 13455 struct cfs_rq *cfs_rq; 13456 13457 #ifdef CONFIG_FAIR_GROUP_SCHED 13458 cfs_rq = task_group(p)->cfs_rq[cpu]; 13459 #else 13460 cfs_rq = &cpu_rq(cpu)->cfs; 13461 #endif 13462 return throttled_hierarchy(cfs_rq); 13463 } 13464 #else /* !CONFIG_SCHED_CORE: */ 13465 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} 13466 #endif /* !CONFIG_SCHED_CORE */ 13467 13468 /* 13469 * scheduler tick hitting a task of our scheduling class. 13470 * 13471 * NOTE: This function can be called remotely by the tick offload that 13472 * goes along full dynticks. Therefore no local assumption can be made 13473 * and everything must be accessed through the @rq and @curr passed in 13474 * parameters. 13475 */ 13476 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) 13477 { 13478 struct cfs_rq *cfs_rq; 13479 struct sched_entity *se = &curr->se; 13480 13481 for_each_sched_entity(se) { 13482 cfs_rq = cfs_rq_of(se); 13483 entity_tick(cfs_rq, se, queued); 13484 } 13485 13486 if (static_branch_unlikely(&sched_numa_balancing)) 13487 task_tick_numa(rq, curr); 13488 13489 update_misfit_status(curr, rq); 13490 check_update_overutilized_status(task_rq(curr)); 13491 13492 task_tick_core(rq, curr); 13493 } 13494 13495 /* 13496 * called on fork with the child task as argument from the parent's context 13497 * - child not yet on the tasklist 13498 * - preemption disabled 13499 */ 13500 static void task_fork_fair(struct task_struct *p) 13501 { 13502 set_task_max_allowed_capacity(p); 13503 } 13504 13505 /* 13506 * Priority of the task has changed. Check to see if we preempt 13507 * the current task. 13508 */ 13509 static void 13510 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio) 13511 { 13512 if (!task_on_rq_queued(p)) 13513 return; 13514 13515 if (rq->cfs.nr_queued == 1) 13516 return; 13517 13518 /* 13519 * Reschedule if we are currently running on this runqueue and 13520 * our priority decreased, or if we are not currently running on 13521 * this runqueue and our priority is higher than the current's 13522 */ 13523 if (task_current_donor(rq, p)) { 13524 if (p->prio > oldprio) 13525 resched_curr(rq); 13526 } else 13527 wakeup_preempt(rq, p, 0); 13528 } 13529 13530 #ifdef CONFIG_FAIR_GROUP_SCHED 13531 /* 13532 * Propagate the changes of the sched_entity across the tg tree to make it 13533 * visible to the root 13534 */ 13535 static void propagate_entity_cfs_rq(struct sched_entity *se) 13536 { 13537 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13538 13539 /* 13540 * If a task gets attached to this cfs_rq and before being queued, 13541 * it gets migrated to another CPU due to reasons like affinity 13542 * change, make sure this cfs_rq stays on leaf cfs_rq list to have 13543 * that removed load decayed or it can cause faireness problem. 13544 */ 13545 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 13546 list_add_leaf_cfs_rq(cfs_rq); 13547 13548 /* Start to propagate at parent */ 13549 se = se->parent; 13550 13551 for_each_sched_entity(se) { 13552 cfs_rq = cfs_rq_of(se); 13553 13554 update_load_avg(cfs_rq, se, UPDATE_TG); 13555 13556 if (!cfs_rq_pelt_clock_throttled(cfs_rq)) 13557 list_add_leaf_cfs_rq(cfs_rq); 13558 } 13559 13560 assert_list_leaf_cfs_rq(rq_of(cfs_rq)); 13561 } 13562 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 13563 static void propagate_entity_cfs_rq(struct sched_entity *se) { } 13564 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 13565 13566 static void detach_entity_cfs_rq(struct sched_entity *se) 13567 { 13568 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13569 13570 /* 13571 * In case the task sched_avg hasn't been attached: 13572 * - A forked task which hasn't been woken up by wake_up_new_task(). 13573 * - A task which has been woken up by try_to_wake_up() but is 13574 * waiting for actually being woken up by sched_ttwu_pending(). 13575 */ 13576 if (!se->avg.last_update_time) 13577 return; 13578 13579 /* Catch up with the cfs_rq and remove our load when we leave */ 13580 update_load_avg(cfs_rq, se, 0); 13581 detach_entity_load_avg(cfs_rq, se); 13582 update_tg_load_avg(cfs_rq); 13583 propagate_entity_cfs_rq(se); 13584 } 13585 13586 static void attach_entity_cfs_rq(struct sched_entity *se) 13587 { 13588 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13589 13590 /* Synchronize entity with its cfs_rq */ 13591 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD); 13592 attach_entity_load_avg(cfs_rq, se); 13593 update_tg_load_avg(cfs_rq); 13594 propagate_entity_cfs_rq(se); 13595 } 13596 13597 static void detach_task_cfs_rq(struct task_struct *p) 13598 { 13599 struct sched_entity *se = &p->se; 13600 13601 detach_entity_cfs_rq(se); 13602 } 13603 13604 static void attach_task_cfs_rq(struct task_struct *p) 13605 { 13606 struct sched_entity *se = &p->se; 13607 13608 attach_entity_cfs_rq(se); 13609 } 13610 13611 static void switched_from_fair(struct rq *rq, struct task_struct *p) 13612 { 13613 detach_task_cfs_rq(p); 13614 } 13615 13616 static void switched_to_fair(struct rq *rq, struct task_struct *p) 13617 { 13618 WARN_ON_ONCE(p->se.sched_delayed); 13619 13620 attach_task_cfs_rq(p); 13621 13622 set_task_max_allowed_capacity(p); 13623 13624 if (task_on_rq_queued(p)) { 13625 /* 13626 * We were most likely switched from sched_rt, so 13627 * kick off the schedule if running, otherwise just see 13628 * if we can still preempt the current task. 13629 */ 13630 if (task_current_donor(rq, p)) 13631 resched_curr(rq); 13632 else 13633 wakeup_preempt(rq, p, 0); 13634 } 13635 } 13636 13637 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13638 { 13639 struct sched_entity *se = &p->se; 13640 13641 if (task_on_rq_queued(p)) { 13642 /* 13643 * Move the next running task to the front of the list, so our 13644 * cfs_tasks list becomes MRU one. 13645 */ 13646 list_move(&se->group_node, &rq->cfs_tasks); 13647 } 13648 if (!first) 13649 return; 13650 13651 WARN_ON_ONCE(se->sched_delayed); 13652 13653 if (hrtick_enabled_fair(rq)) 13654 hrtick_start_fair(rq, p); 13655 13656 update_misfit_status(p, rq); 13657 sched_fair_update_stop_tick(rq, p); 13658 } 13659 13660 /* 13661 * Account for a task changing its policy or group. 13662 * 13663 * This routine is mostly called to set cfs_rq->curr field when a task 13664 * migrates between groups/classes. 13665 */ 13666 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) 13667 { 13668 struct sched_entity *se = &p->se; 13669 13670 for_each_sched_entity(se) { 13671 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13672 13673 set_next_entity(cfs_rq, se, first); 13674 /* ensure bandwidth has been allocated on our new cfs_rq */ 13675 account_cfs_rq_runtime(cfs_rq, 0); 13676 } 13677 13678 __set_next_task_fair(rq, p, first); 13679 } 13680 13681 void init_cfs_rq(struct cfs_rq *cfs_rq) 13682 { 13683 cfs_rq->tasks_timeline = RB_ROOT_CACHED; 13684 cfs_rq->zero_vruntime = (u64)(-(1LL << 20)); 13685 raw_spin_lock_init(&cfs_rq->removed.lock); 13686 } 13687 13688 #ifdef CONFIG_FAIR_GROUP_SCHED 13689 static void task_change_group_fair(struct task_struct *p) 13690 { 13691 /* 13692 * We couldn't detach or attach a forked task which 13693 * hasn't been woken up by wake_up_new_task(). 13694 */ 13695 if (READ_ONCE(p->__state) == TASK_NEW) 13696 return; 13697 13698 detach_task_cfs_rq(p); 13699 13700 /* Tell se's cfs_rq has been changed -- migrated */ 13701 p->se.avg.last_update_time = 0; 13702 set_task_rq(p, task_cpu(p)); 13703 attach_task_cfs_rq(p); 13704 } 13705 13706 void free_fair_sched_group(struct task_group *tg) 13707 { 13708 int i; 13709 13710 for_each_possible_cpu(i) { 13711 if (tg->cfs_rq) 13712 kfree(tg->cfs_rq[i]); 13713 if (tg->se) 13714 kfree(tg->se[i]); 13715 } 13716 13717 kfree(tg->cfs_rq); 13718 kfree(tg->se); 13719 } 13720 13721 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 13722 { 13723 struct sched_entity *se; 13724 struct cfs_rq *cfs_rq; 13725 int i; 13726 13727 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); 13728 if (!tg->cfs_rq) 13729 goto err; 13730 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL); 13731 if (!tg->se) 13732 goto err; 13733 13734 tg->shares = NICE_0_LOAD; 13735 13736 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent)); 13737 13738 for_each_possible_cpu(i) { 13739 cfs_rq = kzalloc_node(sizeof(struct cfs_rq), 13740 GFP_KERNEL, cpu_to_node(i)); 13741 if (!cfs_rq) 13742 goto err; 13743 13744 se = kzalloc_node(sizeof(struct sched_entity_stats), 13745 GFP_KERNEL, cpu_to_node(i)); 13746 if (!se) 13747 goto err_free_rq; 13748 13749 init_cfs_rq(cfs_rq); 13750 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]); 13751 init_entity_runnable_average(se); 13752 } 13753 13754 return 1; 13755 13756 err_free_rq: 13757 kfree(cfs_rq); 13758 err: 13759 return 0; 13760 } 13761 13762 void online_fair_sched_group(struct task_group *tg) 13763 { 13764 struct sched_entity *se; 13765 struct rq_flags rf; 13766 struct rq *rq; 13767 int i; 13768 13769 for_each_possible_cpu(i) { 13770 rq = cpu_rq(i); 13771 se = tg->se[i]; 13772 rq_lock_irq(rq, &rf); 13773 update_rq_clock(rq); 13774 attach_entity_cfs_rq(se); 13775 sync_throttle(tg, i); 13776 rq_unlock_irq(rq, &rf); 13777 } 13778 } 13779 13780 void unregister_fair_sched_group(struct task_group *tg) 13781 { 13782 int cpu; 13783 13784 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); 13785 13786 for_each_possible_cpu(cpu) { 13787 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu]; 13788 struct sched_entity *se = tg->se[cpu]; 13789 struct rq *rq = cpu_rq(cpu); 13790 13791 if (se) { 13792 if (se->sched_delayed) { 13793 guard(rq_lock_irqsave)(rq); 13794 if (se->sched_delayed) { 13795 update_rq_clock(rq); 13796 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 13797 } 13798 list_del_leaf_cfs_rq(cfs_rq); 13799 } 13800 remove_entity_load_avg(se); 13801 } 13802 13803 /* 13804 * Only empty task groups can be destroyed; so we can speculatively 13805 * check on_list without danger of it being re-added. 13806 */ 13807 if (cfs_rq->on_list) { 13808 guard(rq_lock_irqsave)(rq); 13809 list_del_leaf_cfs_rq(cfs_rq); 13810 } 13811 } 13812 } 13813 13814 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, 13815 struct sched_entity *se, int cpu, 13816 struct sched_entity *parent) 13817 { 13818 struct rq *rq = cpu_rq(cpu); 13819 13820 cfs_rq->tg = tg; 13821 cfs_rq->rq = rq; 13822 init_cfs_rq_runtime(cfs_rq); 13823 13824 tg->cfs_rq[cpu] = cfs_rq; 13825 tg->se[cpu] = se; 13826 13827 /* se could be NULL for root_task_group */ 13828 if (!se) 13829 return; 13830 13831 if (!parent) { 13832 se->cfs_rq = &rq->cfs; 13833 se->depth = 0; 13834 } else { 13835 se->cfs_rq = parent->my_q; 13836 se->depth = parent->depth + 1; 13837 } 13838 13839 se->my_q = cfs_rq; 13840 /* guarantee group entities always have weight */ 13841 update_load_set(&se->load, NICE_0_LOAD); 13842 se->parent = parent; 13843 } 13844 13845 static DEFINE_MUTEX(shares_mutex); 13846 13847 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares) 13848 { 13849 int i; 13850 13851 lockdep_assert_held(&shares_mutex); 13852 13853 /* 13854 * We can't change the weight of the root cgroup. 13855 */ 13856 if (!tg->se[0]) 13857 return -EINVAL; 13858 13859 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES)); 13860 13861 if (tg->shares == shares) 13862 return 0; 13863 13864 tg->shares = shares; 13865 for_each_possible_cpu(i) { 13866 struct rq *rq = cpu_rq(i); 13867 struct sched_entity *se = tg->se[i]; 13868 struct rq_flags rf; 13869 13870 /* Propagate contribution to hierarchy */ 13871 rq_lock_irqsave(rq, &rf); 13872 update_rq_clock(rq); 13873 for_each_sched_entity(se) { 13874 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 13875 update_cfs_group(se); 13876 } 13877 rq_unlock_irqrestore(rq, &rf); 13878 } 13879 13880 return 0; 13881 } 13882 13883 int sched_group_set_shares(struct task_group *tg, unsigned long shares) 13884 { 13885 int ret; 13886 13887 mutex_lock(&shares_mutex); 13888 if (tg_is_idle(tg)) 13889 ret = -EINVAL; 13890 else 13891 ret = __sched_group_set_shares(tg, shares); 13892 mutex_unlock(&shares_mutex); 13893 13894 return ret; 13895 } 13896 13897 int sched_group_set_idle(struct task_group *tg, long idle) 13898 { 13899 int i; 13900 13901 if (tg == &root_task_group) 13902 return -EINVAL; 13903 13904 if (idle < 0 || idle > 1) 13905 return -EINVAL; 13906 13907 mutex_lock(&shares_mutex); 13908 13909 if (tg->idle == idle) { 13910 mutex_unlock(&shares_mutex); 13911 return 0; 13912 } 13913 13914 tg->idle = idle; 13915 13916 for_each_possible_cpu(i) { 13917 struct rq *rq = cpu_rq(i); 13918 struct sched_entity *se = tg->se[i]; 13919 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i]; 13920 bool was_idle = cfs_rq_is_idle(grp_cfs_rq); 13921 long idle_task_delta; 13922 struct rq_flags rf; 13923 13924 rq_lock_irqsave(rq, &rf); 13925 13926 grp_cfs_rq->idle = idle; 13927 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq))) 13928 goto next_cpu; 13929 13930 idle_task_delta = grp_cfs_rq->h_nr_queued - 13931 grp_cfs_rq->h_nr_idle; 13932 if (!cfs_rq_is_idle(grp_cfs_rq)) 13933 idle_task_delta *= -1; 13934 13935 for_each_sched_entity(se) { 13936 struct cfs_rq *cfs_rq = cfs_rq_of(se); 13937 13938 if (!se->on_rq) 13939 break; 13940 13941 cfs_rq->h_nr_idle += idle_task_delta; 13942 13943 /* Already accounted at parent level and above. */ 13944 if (cfs_rq_is_idle(cfs_rq)) 13945 break; 13946 } 13947 13948 next_cpu: 13949 rq_unlock_irqrestore(rq, &rf); 13950 } 13951 13952 /* Idle groups have minimum weight. */ 13953 if (tg_is_idle(tg)) 13954 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO)); 13955 else 13956 __sched_group_set_shares(tg, NICE_0_LOAD); 13957 13958 mutex_unlock(&shares_mutex); 13959 return 0; 13960 } 13961 13962 #endif /* CONFIG_FAIR_GROUP_SCHED */ 13963 13964 13965 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task) 13966 { 13967 struct sched_entity *se = &task->se; 13968 unsigned int rr_interval = 0; 13969 13970 /* 13971 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise 13972 * idle runqueue: 13973 */ 13974 if (rq->cfs.load.weight) 13975 rr_interval = NS_TO_JIFFIES(se->slice); 13976 13977 return rr_interval; 13978 } 13979 13980 /* 13981 * All the scheduling class methods: 13982 */ 13983 DEFINE_SCHED_CLASS(fair) = { 13984 13985 .enqueue_task = enqueue_task_fair, 13986 .dequeue_task = dequeue_task_fair, 13987 .yield_task = yield_task_fair, 13988 .yield_to_task = yield_to_task_fair, 13989 13990 .wakeup_preempt = check_preempt_wakeup_fair, 13991 13992 .pick_task = pick_task_fair, 13993 .pick_next_task = __pick_next_task_fair, 13994 .put_prev_task = put_prev_task_fair, 13995 .set_next_task = set_next_task_fair, 13996 13997 .balance = balance_fair, 13998 .select_task_rq = select_task_rq_fair, 13999 .migrate_task_rq = migrate_task_rq_fair, 14000 14001 .rq_online = rq_online_fair, 14002 .rq_offline = rq_offline_fair, 14003 14004 .task_dead = task_dead_fair, 14005 .set_cpus_allowed = set_cpus_allowed_fair, 14006 14007 .task_tick = task_tick_fair, 14008 .task_fork = task_fork_fair, 14009 14010 .reweight_task = reweight_task_fair, 14011 .prio_changed = prio_changed_fair, 14012 .switched_from = switched_from_fair, 14013 .switched_to = switched_to_fair, 14014 14015 .get_rr_interval = get_rr_interval_fair, 14016 14017 .update_curr = update_curr_fair, 14018 14019 #ifdef CONFIG_FAIR_GROUP_SCHED 14020 .task_change_group = task_change_group_fair, 14021 #endif 14022 14023 #ifdef CONFIG_SCHED_CORE 14024 .task_is_throttled = task_is_throttled_fair, 14025 #endif 14026 14027 #ifdef CONFIG_UCLAMP_TASK 14028 .uclamp_enabled = 1, 14029 #endif 14030 }; 14031 14032 void print_cfs_stats(struct seq_file *m, int cpu) 14033 { 14034 struct cfs_rq *cfs_rq, *pos; 14035 14036 rcu_read_lock(); 14037 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos) 14038 print_cfs_rq(m, cpu, cfs_rq); 14039 rcu_read_unlock(); 14040 } 14041 14042 #ifdef CONFIG_NUMA_BALANCING 14043 void show_numa_stats(struct task_struct *p, struct seq_file *m) 14044 { 14045 int node; 14046 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0; 14047 struct numa_group *ng; 14048 14049 rcu_read_lock(); 14050 ng = rcu_dereference(p->numa_group); 14051 for_each_online_node(node) { 14052 if (p->numa_faults) { 14053 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)]; 14054 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)]; 14055 } 14056 if (ng) { 14057 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)], 14058 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 14059 } 14060 print_numa_stats(m, node, tsf, tpf, gsf, gpf); 14061 } 14062 rcu_read_unlock(); 14063 } 14064 #endif /* CONFIG_NUMA_BALANCING */ 14065 14066 __init void init_sched_fair_class(void) 14067 { 14068 int i; 14069 14070 for_each_possible_cpu(i) { 14071 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i)); 14072 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i)); 14073 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i), 14074 GFP_KERNEL, cpu_to_node(i)); 14075 14076 #ifdef CONFIG_CFS_BANDWIDTH 14077 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i)); 14078 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list); 14079 #endif 14080 } 14081 14082 open_softirq(SCHED_SOFTIRQ, sched_balance_softirq); 14083 14084 #ifdef CONFIG_NO_HZ_COMMON 14085 nohz.next_balance = jiffies; 14086 nohz.next_blocked = jiffies; 14087 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT); 14088 #endif 14089 }