1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH) 4 * 5 * Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <[email protected]> 6 * 7 * Interactivity improvements by Mike Galbraith 8 * (C) 2007 Mike Galbraith <[email protected]> 9 * 10 * Various enhancements by Dmitry Adamushko. 11 * (C) 2007 Dmitry Adamushko <[email protected]> 12 * 13 * Group scheduling enhancements by Srivatsa Vaddagiri 14 * Copyright IBM Corporation, 2007 15 * Author: Srivatsa Vaddagiri <[email protected]> 16 * 17 * Scaled math optimizations by Thomas Gleixner 18 * Copyright (C) 2007, Thomas Gleixner <[email protected]> 19 * 20 * Adaptive scheduling granularity, math enhancements by Peter Zijlstra 21 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 22 */ 23 #include <linux/energy_model.h> 24 #include <linux/mmap_lock.h> 25 #include <linux/hugetlb_inline.h> 26 #include <linux/jiffies.h> 27 #include <linux/mm_api.h> 28 #include <linux/highmem.h> 29 #include <linux/spinlock_api.h> 30 #include <linux/cpumask_api.h> 31 #include <linux/lockdep_api.h> 32 #include <linux/softirq.h> 33 #include <linux/refcount_api.h> 34 #include <linux/topology.h> 35 #include <linux/sched/clock.h> 36 #include <linux/sched/cond_resched.h> 37 #include <linux/sched/cputime.h> 38 #include <linux/sched/isolation.h> 39 #include <linux/sched/nohz.h> 40 #include <linux/sched/prio.h> 41 42 #include <linux/cpuidle.h> 43 #include <linux/interrupt.h> 44 #include <linux/memory-tiers.h> 45 #include <linux/mempolicy.h> 46 #include <linux/mutex_api.h> 47 #include <linux/profile.h> 48 #include <linux/psi.h> 49 #include <linux/ratelimit.h> 50 #include <linux/task_work.h> 51 #include <linux/rbtree_augmented.h> 52 53 #include <asm/switch_to.h> 54 55 #include <uapi/linux/sched/types.h> 56 57 #include "sched.h" 58 #include "stats.h" 59 #include "autogroup.h" 60 61 /* 62 * The initial- and re-scaling of tunables is configurable 63 * 64 * Options are: 65 * 66 * SCHED_TUNABLESCALING_NONE - unscaled, always *1 67 * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) 68 * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus 69 * 70 * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) 71 */ 72 unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; 73 74 /* 75 * Minimal preemption granularity for CPU-bound tasks: 76 * 77 * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) 78 */ 79 unsigned int sysctl_sched_base_slice = 700000ULL; 80 static unsigned int normalized_sysctl_sched_base_slice = 700000ULL; 81 82 __read_mostly unsigned int sysctl_sched_migration_cost = 500000UL; 83 84 static int __init setup_sched_thermal_decay_shift(char *str) 85 { 86 pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n"); 87 return 1; 88 } 89 __setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift); 90 91 /* 92 * For asym packing, by default the lower numbered CPU has higher priority. 93 */ 94 int __weak arch_asym_cpu_priority(int cpu) 95 { 96 return -cpu; 97 } 98 99 /* 100 * The margin used when comparing utilization with CPU capacity. 101 * 102 * (default: ~20%) 103 */ 104 #define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024) 105 106 /* 107 * The margin used when comparing CPU capacities. 108 * is 'cap1' noticeably greater than 'cap2' 109 * 110 * (default: ~5%) 111 */ 112 #define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078) 113 114 #ifdef CONFIG_CFS_BANDWIDTH 115 /* 116 * Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool 117 * each time a cfs_rq requests quota. 118 * 119 * Note: in the case that the slice exceeds the runtime remaining (either due 120 * to consumption or the quota being specified to be smaller than the slice) 121 * we will always only issue the remaining available time. 122 * 123 * (default: 5 msec, units: microseconds) 124 */ 125 static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; 126 #endif 127 128 #ifdef CONFIG_NUMA_BALANCING 129 /* Restrict the NUMA promotion throughput (MB/s) for each target node. */ 130 static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; 131 #endif 132 133 #ifdef CONFIG_SYSCTL 134 static const struct ctl_table sched_fair_sysctls[] = { 135 #ifdef CONFIG_CFS_BANDWIDTH 136 { 137 .procname = "sched_cfs_bandwidth_slice_us", 138 .data = &sysctl_sched_cfs_bandwidth_slice, 139 .maxlen = sizeof(unsigned int), 140 .mode = 0644, 141 .proc_handler = proc_dointvec_minmax, 142 .extra1 = SYSCTL_ONE, 143 }, 144 #endif 145 #ifdef CONFIG_NUMA_BALANCING 146 { 147 .procname = "numa_balancing_promote_rate_limit_MBps", 148 .data = &sysctl_numa_balancing_promote_rate_limit, 149 .maxlen = sizeof(unsigned int), 150 .mode = 0644, 151 .proc_handler = proc_dointvec_minmax, 152 .extra1 = SYSCTL_ZERO, 153 }, 154 #endif /* CONFIG_NUMA_BALANCING */ 155 }; 156 157 static int __init sched_fair_sysctl_init(void) 158 { 159 register_sysctl_init("kernel", sched_fair_sysctls); 160 return 0; 161 } 162 late_initcall(sched_fair_sysctl_init); 163 #endif /* CONFIG_SYSCTL */ 164 165 static inline void update_load_add(struct load_weight *lw, unsigned long inc) 166 { 167 lw->weight += inc; 168 lw->inv_weight = 0; 169 } 170 171 static inline void update_load_sub(struct load_weight *lw, unsigned long dec) 172 { 173 lw->weight -= dec; 174 lw->inv_weight = 0; 175 } 176 177 static inline void update_load_set(struct load_weight *lw, unsigned long w) 178 { 179 lw->weight = w; 180 lw->inv_weight = 0; 181 } 182 183 /* 184 * Increase the granularity value when there are more CPUs, 185 * because with more CPUs the 'effective latency' as visible 186 * to users decreases. But the relationship is not linear, 187 * so pick a second-best guess by going with the log2 of the 188 * number of CPUs. 189 * 190 * This idea comes from the SD scheduler of Con Kolivas: 191 */ 192 static unsigned int get_update_sysctl_factor(void) 193 { 194 unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); 195 unsigned int factor; 196 197 switch (sysctl_sched_tunable_scaling) { 198 case SCHED_TUNABLESCALING_NONE: 199 factor = 1; 200 break; 201 case SCHED_TUNABLESCALING_LINEAR: 202 factor = cpus; 203 break; 204 case SCHED_TUNABLESCALING_LOG: 205 default: 206 factor = 1 + ilog2(cpus); 207 break; 208 } 209 210 return factor; 211 } 212 213 static void update_sysctl(void) 214 { 215 unsigned int factor = get_update_sysctl_factor(); 216 217 #define SET_SYSCTL(name) \ 218 (sysctl_##name = (factor) * normalized_sysctl_##name) 219 SET_SYSCTL(sched_base_slice); 220 #undef SET_SYSCTL 221 } 222 223 void __init sched_init_granularity(void) 224 { 225 update_sysctl(); 226 } 227 228 #define WMULT_CONST (~0U) 229 #define WMULT_SHIFT 32 230 231 static void __update_inv_weight(struct load_weight *lw) 232 { 233 unsigned long w; 234 235 if (likely(lw->inv_weight)) 236 return; 237 238 w = scale_load_down(lw->weight); 239 240 if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) 241 lw->inv_weight = 1; 242 else if (unlikely(!w)) 243 lw->inv_weight = WMULT_CONST; 244 else 245 lw->inv_weight = WMULT_CONST / w; 246 } 247 248 /* 249 * delta_exec * weight / lw.weight 250 * OR 251 * (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT 252 * 253 * Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case 254 * we're guaranteed shift stays positive because inv_weight is guaranteed to 255 * fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22. 256 * 257 * Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus 258 * weight/lw.weight <= 1, and therefore our shift will also be positive. 259 */ 260 static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw) 261 { 262 u64 fact = scale_load_down(weight); 263 u32 fact_hi = (u32)(fact >> 32); 264 int shift = WMULT_SHIFT; 265 int fs; 266 267 __update_inv_weight(lw); 268 269 if (unlikely(fact_hi)) { 270 fs = fls(fact_hi); 271 shift -= fs; 272 fact >>= fs; 273 } 274 275 fact = mul_u32_u32(fact, lw->inv_weight); 276 277 fact_hi = (u32)(fact >> 32); 278 if (fact_hi) { 279 fs = fls(fact_hi); 280 shift -= fs; 281 fact >>= fs; 282 } 283 284 return mul_u64_u32_shr(delta_exec, fact, shift); 285 } 286 287 /* 288 * delta /= w 289 */ 290 static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) 291 { 292 if (unlikely(se->load.weight != NICE_0_LOAD)) 293 delta = __calc_delta(delta, NICE_0_LOAD, &se->load); 294 295 return delta; 296 } 297 298 const struct sched_class fair_sched_class; 299 300 /************************************************************** 301 * CFS operations on generic schedulable entities: 302 */ 303 304 #ifdef CONFIG_FAIR_GROUP_SCHED 305 306 /* Walk up scheduling entities hierarchy */ 307 #define for_each_sched_entity(se) \ 308 for (; se; se = se->parent) 309 310 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 311 { 312 struct rq *rq = rq_of(cfs_rq); 313 int cpu = cpu_of(rq); 314 315 if (cfs_rq->on_list) 316 return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; 317 318 cfs_rq->on_list = 1; 319 320 /* 321 * Ensure we either appear before our parent (if already 322 * enqueued) or force our parent to appear after us when it is 323 * enqueued. The fact that we always enqueue bottom-up 324 * reduces this to two cases and a special case for the root 325 * cfs_rq. Furthermore, it also means that we will always reset 326 * tmp_alone_branch either when the branch is connected 327 * to a tree or when we reach the top of the tree 328 */ 329 if (cfs_rq->tg->parent && 330 cfs_rq->tg->parent->cfs_rq[cpu]->on_list) { 331 /* 332 * If parent is already on the list, we add the child 333 * just before. Thanks to circular linked property of 334 * the list, this means to put the child at the tail 335 * of the list that starts by parent. 336 */ 337 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 338 &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list)); 339 /* 340 * The branch is now connected to its tree so we can 341 * reset tmp_alone_branch to the beginning of the 342 * list. 343 */ 344 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 345 return true; 346 } 347 348 if (!cfs_rq->tg->parent) { 349 /* 350 * cfs rq without parent should be put 351 * at the tail of the list. 352 */ 353 list_add_tail_rcu(&cfs_rq->leaf_cfs_rq_list, 354 &rq->leaf_cfs_rq_list); 355 /* 356 * We have reach the top of a tree so we can reset 357 * tmp_alone_branch to the beginning of the list. 358 */ 359 rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; 360 return true; 361 } 362 363 /* 364 * The parent has not already been added so we want to 365 * make sure that it will be put after us. 366 * tmp_alone_branch points to the begin of the branch 367 * where we will add parent. 368 */ 369 list_add_rcu(&cfs_rq->leaf_cfs_rq_list, rq->tmp_alone_branch); 370 /* 371 * update tmp_alone_branch to points to the new begin 372 * of the branch 373 */ 374 rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; 375 return false; 376 } 377 378 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 379 { 380 if (cfs_rq->on_list) { 381 struct rq *rq = rq_of(cfs_rq); 382 383 /* 384 * With cfs_rq being unthrottled/throttled during an enqueue, 385 * it can happen the tmp_alone_branch points to the leaf that 386 * we finally want to delete. In this case, tmp_alone_branch moves 387 * to the prev element but it will point to rq->leaf_cfs_rq_list 388 * at the end of the enqueue. 389 */ 390 if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list) 391 rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev; 392 393 list_del_rcu(&cfs_rq->leaf_cfs_rq_list); 394 cfs_rq->on_list = 0; 395 } 396 } 397 398 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 399 { 400 WARN_ON_ONCE(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); 401 } 402 403 /* Iterate through all leaf cfs_rq's on a runqueue */ 404 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 405 list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \ 406 leaf_cfs_rq_list) 407 408 /* Do the two (enqueued) entities belong to the same group ? */ 409 static inline struct cfs_rq * 410 is_same_group(struct sched_entity *se, struct sched_entity *pse) 411 { 412 if (se->cfs_rq == pse->cfs_rq) 413 return se->cfs_rq; 414 415 return NULL; 416 } 417 418 static inline struct sched_entity *parent_entity(const struct sched_entity *se) 419 { 420 return se->parent; 421 } 422 423 static void 424 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 425 { 426 int se_depth, pse_depth; 427 428 /* 429 * preemption test can be made between sibling entities who are in the 430 * same cfs_rq i.e who have a common parent. Walk up the hierarchy of 431 * both tasks until we find their ancestors who are siblings of common 432 * parent. 433 */ 434 435 /* First walk up until both entities are at same depth */ 436 se_depth = (*se)->depth; 437 pse_depth = (*pse)->depth; 438 439 while (se_depth > pse_depth) { 440 se_depth--; 441 *se = parent_entity(*se); 442 } 443 444 while (pse_depth > se_depth) { 445 pse_depth--; 446 *pse = parent_entity(*pse); 447 } 448 449 while (!is_same_group(*se, *pse)) { 450 *se = parent_entity(*se); 451 *pse = parent_entity(*pse); 452 } 453 } 454 455 static int tg_is_idle(struct task_group *tg) 456 { 457 return tg->idle > 0; 458 } 459 460 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 461 { 462 return cfs_rq->idle > 0; 463 } 464 465 static int se_is_idle(struct sched_entity *se) 466 { 467 if (entity_is_task(se)) 468 return task_has_idle_policy(task_of(se)); 469 return cfs_rq_is_idle(group_cfs_rq(se)); 470 } 471 472 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 473 474 #define for_each_sched_entity(se) \ 475 for (; se; se = NULL) 476 477 static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) 478 { 479 return true; 480 } 481 482 static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) 483 { 484 } 485 486 static inline void assert_list_leaf_cfs_rq(struct rq *rq) 487 { 488 } 489 490 #define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \ 491 for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos) 492 493 static inline struct sched_entity *parent_entity(struct sched_entity *se) 494 { 495 return NULL; 496 } 497 498 static inline void 499 find_matching_se(struct sched_entity **se, struct sched_entity **pse) 500 { 501 } 502 503 static inline int tg_is_idle(struct task_group *tg) 504 { 505 return 0; 506 } 507 508 static int cfs_rq_is_idle(struct cfs_rq *cfs_rq) 509 { 510 return 0; 511 } 512 513 static int se_is_idle(struct sched_entity *se) 514 { 515 return task_has_idle_policy(task_of(se)); 516 } 517 518 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 519 520 static __always_inline 521 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec); 522 523 /************************************************************** 524 * Scheduling class tree data structure manipulation methods: 525 */ 526 527 extern void __BUILD_BUG_vruntime_cmp(void); 528 529 /* Use __builtin_strcmp() because of __HAVE_ARCH_STRCMP: */ 530 531 #define vruntime_cmp(A, CMP_STR, B) ({ \ 532 int __res = 0; \ 533 \ 534 if (!__builtin_strcmp(CMP_STR, "<")) { \ 535 __res = ((s64)((A)-(B)) < 0); \ 536 } else if (!__builtin_strcmp(CMP_STR, "<=")) { \ 537 __res = ((s64)((A)-(B)) <= 0); \ 538 } else if (!__builtin_strcmp(CMP_STR, ">")) { \ 539 __res = ((s64)((A)-(B)) > 0); \ 540 } else if (!__builtin_strcmp(CMP_STR, ">=")) { \ 541 __res = ((s64)((A)-(B)) >= 0); \ 542 } else { \ 543 /* Unknown operator throws linker error: */ \ 544 __BUILD_BUG_vruntime_cmp(); \ 545 } \ 546 \ 547 __res; \ 548 }) 549 550 extern void __BUILD_BUG_vruntime_op(void); 551 552 #define vruntime_op(A, OP_STR, B) ({ \ 553 s64 __res = 0; \ 554 \ 555 if (!__builtin_strcmp(OP_STR, "-")) { \ 556 __res = (s64)((A)-(B)); \ 557 } else { \ 558 /* Unknown operator throws linker error: */ \ 559 __BUILD_BUG_vruntime_op(); \ 560 } \ 561 \ 562 __res; \ 563 }) 564 565 566 static inline __maybe_unused u64 max_vruntime(u64 max_vruntime, u64 vruntime) 567 { 568 if (vruntime_cmp(vruntime, ">", max_vruntime)) 569 max_vruntime = vruntime; 570 571 return max_vruntime; 572 } 573 574 static inline __maybe_unused u64 min_vruntime(u64 min_vruntime, u64 vruntime) 575 { 576 if (vruntime_cmp(vruntime, "<", min_vruntime)) 577 min_vruntime = vruntime; 578 579 return min_vruntime; 580 } 581 582 static inline bool entity_before(const struct sched_entity *a, 583 const struct sched_entity *b) 584 { 585 /* 586 * Tiebreak on vruntime seems unnecessary since it can 587 * hardly happen. 588 */ 589 return vruntime_cmp(a->deadline, "<", b->deadline); 590 } 591 592 /* 593 * Per avg_vruntime() below, cfs_rq::zero_vruntime is only slightly stale 594 * and this value should be no more than two lag bounds. Which puts it in the 595 * general order of: 596 * 597 * (slice + TICK_NSEC) << NICE_0_LOAD_SHIFT 598 * 599 * which is around 44 bits in size (on 64bit); that is 20 for 600 * NICE_0_LOAD_SHIFT, another 20 for NSEC_PER_MSEC and then a handful for 601 * however many msec the actual slice+tick ends up begin. 602 * 603 * (disregarding the actual divide-by-weight part makes for the worst case 604 * weight of 2, which nicely cancels vs the fuzz in zero_vruntime not actually 605 * being the zero-lag point). 606 */ 607 static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) 608 { 609 return vruntime_op(se->vruntime, "-", cfs_rq->zero_vruntime); 610 } 611 612 #define __node_2_se(node) \ 613 rb_entry((node), struct sched_entity, run_node) 614 615 /* 616 * Compute virtual time from the per-task service numbers: 617 * 618 * Fair schedulers conserve lag: 619 * 620 * \Sum lag_i = 0 621 * 622 * Where lag_i is given by: 623 * 624 * lag_i = S - s_i = w_i * (V - v_i) 625 * 626 * Where S is the ideal service time and V is it's virtual time counterpart. 627 * Therefore: 628 * 629 * \Sum lag_i = 0 630 * \Sum w_i * (V - v_i) = 0 631 * \Sum w_i * V - w_i * v_i = 0 632 * 633 * From which we can solve an expression for V in v_i (which we have in 634 * se->vruntime): 635 * 636 * \Sum v_i * w_i \Sum v_i * w_i 637 * V = -------------- = -------------- 638 * \Sum w_i W 639 * 640 * Specifically, this is the weighted average of all entity virtual runtimes. 641 * 642 * [[ NOTE: this is only equal to the ideal scheduler under the condition 643 * that join/leave operations happen at lag_i = 0, otherwise the 644 * virtual time has non-contiguous motion equivalent to: 645 * 646 * V +-= lag_i / W 647 * 648 * Also see the comment in place_entity() that deals with this. ]] 649 * 650 * However, since v_i is u64, and the multiplication could easily overflow 651 * transform it into a relative form that uses smaller quantities: 652 * 653 * Substitute: v_i == (v_i - v0) + v0 654 * 655 * \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i 656 * V = ---------------------------- = --------------------- + v0 657 * W W 658 * 659 * Which we track using: 660 * 661 * v0 := cfs_rq->zero_vruntime 662 * \Sum (v_i - v0) * w_i := cfs_rq->sum_w_vruntime 663 * \Sum w_i := cfs_rq->sum_weight 664 * 665 * Since zero_vruntime closely tracks the per-task service, these 666 * deltas: (v_i - v), will be in the order of the maximal (virtual) lag 667 * induced in the system due to quantisation. 668 * 669 * Also, we use scale_load_down() to reduce the size. 670 * 671 * As measured, the max (key * weight) value was ~44 bits for a kernel build. 672 */ 673 static void 674 sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 675 { 676 unsigned long weight = scale_load_down(se->load.weight); 677 s64 key = entity_key(cfs_rq, se); 678 679 cfs_rq->sum_w_vruntime += key * weight; 680 cfs_rq->sum_weight += weight; 681 } 682 683 static void 684 sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) 685 { 686 unsigned long weight = scale_load_down(se->load.weight); 687 s64 key = entity_key(cfs_rq, se); 688 689 cfs_rq->sum_w_vruntime -= key * weight; 690 cfs_rq->sum_weight -= weight; 691 } 692 693 static inline 694 void update_zero_vruntime(struct cfs_rq *cfs_rq, s64 delta) 695 { 696 /* 697 * v' = v + d ==> sum_w_vruntime' = sum_w_vruntime - d*sum_weight 698 */ 699 cfs_rq->sum_w_vruntime -= cfs_rq->sum_weight * delta; 700 cfs_rq->zero_vruntime += delta; 701 } 702 703 /* 704 * Specifically: avg_vruntime() + 0 must result in entity_eligible() := true 705 * For this to be so, the result of this function must have a left bias. 706 * 707 * Called in: 708 * - place_entity() -- before enqueue 709 * - update_entity_lag() -- before dequeue 710 * - update_deadline() -- slice expiration 711 * 712 * This means it is one entry 'behind' but that puts it close enough to where 713 * the bound on entity_key() is at most two lag bounds. 714 */ 715 u64 avg_vruntime(struct cfs_rq *cfs_rq) 716 { 717 struct sched_entity *curr = cfs_rq->curr; 718 long weight = cfs_rq->sum_weight; 719 s64 delta = 0; 720 721 if (curr && !curr->on_rq) 722 curr = NULL; 723 724 if (weight) { 725 s64 runtime = cfs_rq->sum_w_vruntime; 726 727 if (curr) { 728 unsigned long w = scale_load_down(curr->load.weight); 729 730 runtime += entity_key(cfs_rq, curr) * w; 731 weight += w; 732 } 733 734 /* sign flips effective floor / ceiling */ 735 if (runtime < 0) 736 runtime -= (weight - 1); 737 738 delta = div_s64(runtime, weight); 739 } else if (curr) { 740 /* 741 * When there is but one element, it is the average. 742 */ 743 delta = curr->vruntime - cfs_rq->zero_vruntime; 744 } 745 746 update_zero_vruntime(cfs_rq, delta); 747 748 return cfs_rq->zero_vruntime; 749 } 750 751 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); 752 753 /* 754 * lag_i = S - s_i = w_i * (V - v_i) 755 * 756 * However, since V is approximated by the weighted average of all entities it 757 * is possible -- by addition/removal/reweight to the tree -- to move V around 758 * and end up with a larger lag than we started with. 759 * 760 * Limit this to either double the slice length with a minimum of TICK_NSEC 761 * since that is the timing granularity. 762 * 763 * EEVDF gives the following limit for a steady state system: 764 * 765 * -r_max < lag < max(r_max, q) 766 */ 767 static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) 768 { 769 u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; 770 s64 vlag, limit; 771 772 WARN_ON_ONCE(!se->on_rq); 773 774 vlag = avg_vruntime(cfs_rq) - se->vruntime; 775 limit = calc_delta_fair(max_slice, se); 776 777 se->vlag = clamp(vlag, -limit, limit); 778 } 779 780 /* 781 * Entity is eligible once it received less service than it ought to have, 782 * eg. lag >= 0. 783 * 784 * lag_i = S - s_i = w_i*(V - v_i) 785 * 786 * lag_i >= 0 -> V >= v_i 787 * 788 * \Sum (v_i - v)*w_i 789 * V = ------------------ + v 790 * \Sum w_i 791 * 792 * lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i) 793 * 794 * Note: using 'avg_vruntime() > se->vruntime' is inaccurate due 795 * to the loss in precision caused by the division. 796 */ 797 static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) 798 { 799 struct sched_entity *curr = cfs_rq->curr; 800 s64 avg = cfs_rq->sum_w_vruntime; 801 long load = cfs_rq->sum_weight; 802 803 if (curr && curr->on_rq) { 804 unsigned long weight = scale_load_down(curr->load.weight); 805 806 avg += entity_key(cfs_rq, curr) * weight; 807 load += weight; 808 } 809 810 return avg >= vruntime_op(vruntime, "-", cfs_rq->zero_vruntime) * load; 811 } 812 813 int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se) 814 { 815 return vruntime_eligible(cfs_rq, se->vruntime); 816 } 817 818 static inline u64 cfs_rq_min_slice(struct cfs_rq *cfs_rq) 819 { 820 struct sched_entity *root = __pick_root_entity(cfs_rq); 821 struct sched_entity *curr = cfs_rq->curr; 822 u64 min_slice = ~0ULL; 823 824 if (curr && curr->on_rq) 825 min_slice = curr->slice; 826 827 if (root) 828 min_slice = min(min_slice, root->min_slice); 829 830 return min_slice; 831 } 832 833 static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq) 834 { 835 struct sched_entity *root = __pick_root_entity(cfs_rq); 836 struct sched_entity *curr = cfs_rq->curr; 837 u64 max_slice = 0ULL; 838 839 if (curr && curr->on_rq) 840 max_slice = curr->slice; 841 842 if (root) 843 max_slice = max(max_slice, root->max_slice); 844 845 return max_slice; 846 } 847 848 static inline bool __entity_less(struct rb_node *a, const struct rb_node *b) 849 { 850 return entity_before(__node_2_se(a), __node_2_se(b)); 851 } 852 853 static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node) 854 { 855 if (node) { 856 struct sched_entity *rse = __node_2_se(node); 857 858 if (vruntime_cmp(se->min_vruntime, ">", rse->min_vruntime)) 859 se->min_vruntime = rse->min_vruntime; 860 } 861 } 862 863 static inline void __min_slice_update(struct sched_entity *se, struct rb_node *node) 864 { 865 if (node) { 866 struct sched_entity *rse = __node_2_se(node); 867 if (rse->min_slice < se->min_slice) 868 se->min_slice = rse->min_slice; 869 } 870 } 871 872 static inline void __max_slice_update(struct sched_entity *se, struct rb_node *node) 873 { 874 if (node) { 875 struct sched_entity *rse = __node_2_se(node); 876 if (rse->max_slice > se->max_slice) 877 se->max_slice = rse->max_slice; 878 } 879 } 880 881 /* 882 * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) 883 */ 884 static inline bool min_vruntime_update(struct sched_entity *se, bool exit) 885 { 886 u64 old_min_vruntime = se->min_vruntime; 887 u64 old_min_slice = se->min_slice; 888 u64 old_max_slice = se->max_slice; 889 struct rb_node *node = &se->run_node; 890 891 se->min_vruntime = se->vruntime; 892 __min_vruntime_update(se, node->rb_right); 893 __min_vruntime_update(se, node->rb_left); 894 895 se->min_slice = se->slice; 896 __min_slice_update(se, node->rb_right); 897 __min_slice_update(se, node->rb_left); 898 899 se->max_slice = se->slice; 900 __max_slice_update(se, node->rb_right); 901 __max_slice_update(se, node->rb_left); 902 903 return se->min_vruntime == old_min_vruntime && 904 se->min_slice == old_min_slice && 905 se->max_slice == old_max_slice; 906 } 907 908 RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, 909 run_node, min_vruntime, min_vruntime_update); 910 911 /* 912 * Enqueue an entity into the rb-tree: 913 */ 914 static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 915 { 916 sum_w_vruntime_add(cfs_rq, se); 917 se->min_vruntime = se->vruntime; 918 se->min_slice = se->slice; 919 rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 920 __entity_less, &min_vruntime_cb); 921 } 922 923 static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) 924 { 925 rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, 926 &min_vruntime_cb); 927 sum_w_vruntime_sub(cfs_rq, se); 928 } 929 930 struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq) 931 { 932 struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node; 933 934 if (!root) 935 return NULL; 936 937 return __node_2_se(root); 938 } 939 940 struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) 941 { 942 struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); 943 944 if (!left) 945 return NULL; 946 947 return __node_2_se(left); 948 } 949 950 /* 951 * Set the vruntime up to which an entity can run before looking 952 * for another entity to pick. 953 * In case of run to parity, we use the shortest slice of the enqueued 954 * entities to set the protected period. 955 * When run to parity is disabled, we give a minimum quantum to the running 956 * entity to ensure progress. 957 */ 958 static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 959 { 960 u64 slice = normalized_sysctl_sched_base_slice; 961 u64 vprot = se->deadline; 962 963 if (sched_feat(RUN_TO_PARITY)) 964 slice = cfs_rq_min_slice(cfs_rq); 965 966 slice = min(slice, se->slice); 967 if (slice != se->slice) 968 vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); 969 970 se->vprot = vprot; 971 } 972 973 static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 974 { 975 u64 slice = cfs_rq_min_slice(cfs_rq); 976 977 se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); 978 } 979 980 static inline bool protect_slice(struct sched_entity *se) 981 { 982 return vruntime_cmp(se->vruntime, "<", se->vprot); 983 } 984 985 static inline void cancel_protect_slice(struct sched_entity *se) 986 { 987 if (protect_slice(se)) 988 se->vprot = se->vruntime; 989 } 990 991 /* 992 * Earliest Eligible Virtual Deadline First 993 * 994 * In order to provide latency guarantees for different request sizes 995 * EEVDF selects the best runnable task from two criteria: 996 * 997 * 1) the task must be eligible (must be owed service) 998 * 999 * 2) from those tasks that meet 1), we select the one 1000 * with the earliest virtual deadline. 1001 * 1002 * We can do this in O(log n) time due to an augmented RB-tree. The 1003 * tree keeps the entries sorted on deadline, but also functions as a 1004 * heap based on the vruntime by keeping: 1005 * 1006 * se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime) 1007 * 1008 * Which allows tree pruning through eligibility. 1009 */ 1010 static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) 1011 { 1012 struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; 1013 struct sched_entity *se = __pick_first_entity(cfs_rq); 1014 struct sched_entity *curr = cfs_rq->curr; 1015 struct sched_entity *best = NULL; 1016 1017 /* 1018 * We can safely skip eligibility check if there is only one entity 1019 * in this cfs_rq, saving some cycles. 1020 */ 1021 if (cfs_rq->nr_queued == 1) 1022 return curr && curr->on_rq ? curr : se; 1023 1024 /* 1025 * Picking the ->next buddy will affect latency but not fairness. 1026 */ 1027 if (sched_feat(PICK_BUDDY) && 1028 cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) { 1029 /* ->next will never be delayed */ 1030 WARN_ON_ONCE(cfs_rq->next->sched_delayed); 1031 return cfs_rq->next; 1032 } 1033 1034 if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) 1035 curr = NULL; 1036 1037 if (curr && protect && protect_slice(curr)) 1038 return curr; 1039 1040 /* Pick the leftmost entity if it's eligible */ 1041 if (se && entity_eligible(cfs_rq, se)) { 1042 best = se; 1043 goto found; 1044 } 1045 1046 /* Heap search for the EEVD entity */ 1047 while (node) { 1048 struct rb_node *left = node->rb_left; 1049 1050 /* 1051 * Eligible entities in left subtree are always better 1052 * choices, since they have earlier deadlines. 1053 */ 1054 if (left && vruntime_eligible(cfs_rq, 1055 __node_2_se(left)->min_vruntime)) { 1056 node = left; 1057 continue; 1058 } 1059 1060 se = __node_2_se(node); 1061 1062 /* 1063 * The left subtree either is empty or has no eligible 1064 * entity, so check the current node since it is the one 1065 * with earliest deadline that might be eligible. 1066 */ 1067 if (entity_eligible(cfs_rq, se)) { 1068 best = se; 1069 break; 1070 } 1071 1072 node = node->rb_right; 1073 } 1074 found: 1075 if (!best || (curr && entity_before(curr, best))) 1076 best = curr; 1077 1078 return best; 1079 } 1080 1081 struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) 1082 { 1083 struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); 1084 1085 if (!last) 1086 return NULL; 1087 1088 return __node_2_se(last); 1089 } 1090 1091 /************************************************************** 1092 * Scheduling class statistics methods: 1093 */ 1094 int sched_update_scaling(void) 1095 { 1096 unsigned int factor = get_update_sysctl_factor(); 1097 1098 #define WRT_SYSCTL(name) \ 1099 (normalized_sysctl_##name = sysctl_##name / (factor)) 1100 WRT_SYSCTL(sched_base_slice); 1101 #undef WRT_SYSCTL 1102 1103 return 0; 1104 } 1105 1106 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se); 1107 1108 /* 1109 * XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i 1110 * this is probably good enough. 1111 */ 1112 static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se) 1113 { 1114 if (vruntime_cmp(se->vruntime, "<", se->deadline)) 1115 return false; 1116 1117 /* 1118 * For EEVDF the virtual time slope is determined by w_i (iow. 1119 * nice) while the request time r_i is determined by 1120 * sysctl_sched_base_slice. 1121 */ 1122 if (!se->custom_slice) 1123 se->slice = sysctl_sched_base_slice; 1124 1125 /* 1126 * EEVDF: vd_i = ve_i + r_i / w_i 1127 */ 1128 se->deadline = se->vruntime + calc_delta_fair(se->slice, se); 1129 avg_vruntime(cfs_rq); 1130 1131 /* 1132 * The task has consumed its request, reschedule. 1133 */ 1134 return true; 1135 } 1136 1137 #include "pelt.h" 1138 1139 static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu); 1140 static unsigned long task_h_load(struct task_struct *p); 1141 static unsigned long capacity_of(int cpu); 1142 1143 /* Give new sched_entity start runnable values to heavy its load in infant time */ 1144 void init_entity_runnable_average(struct sched_entity *se) 1145 { 1146 struct sched_avg *sa = &se->avg; 1147 1148 memset(sa, 0, sizeof(*sa)); 1149 1150 /* 1151 * Tasks are initialized with full load to be seen as heavy tasks until 1152 * they get a chance to stabilize to their real load level. 1153 * Group entities are initialized with zero load to reflect the fact that 1154 * nothing has been attached to the task group yet. 1155 */ 1156 if (entity_is_task(se)) 1157 sa->load_avg = scale_load_down(se->load.weight); 1158 1159 /* when this task is enqueued, it will contribute to its cfs_rq's load_avg */ 1160 } 1161 1162 /* 1163 * With new tasks being created, their initial util_avgs are extrapolated 1164 * based on the cfs_rq's current util_avg: 1165 * 1166 * util_avg = cfs_rq->avg.util_avg / (cfs_rq->avg.load_avg + 1) 1167 * * se_weight(se) 1168 * 1169 * However, in many cases, the above util_avg does not give a desired 1170 * value. Moreover, the sum of the util_avgs may be divergent, such 1171 * as when the series is a harmonic series. 1172 * 1173 * To solve this problem, we also cap the util_avg of successive tasks to 1174 * only 1/2 of the left utilization budget: 1175 * 1176 * util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n 1177 * 1178 * where n denotes the nth task and cpu_scale the CPU capacity. 1179 * 1180 * For example, for a CPU with 1024 of capacity, a simplest series from 1181 * the beginning would be like: 1182 * 1183 * task util_avg: 512, 256, 128, 64, 32, 16, 8, ... 1184 * cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ... 1185 * 1186 * Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap) 1187 * if util_avg > util_avg_cap. 1188 */ 1189 void post_init_entity_util_avg(struct task_struct *p) 1190 { 1191 struct sched_entity *se = &p->se; 1192 struct cfs_rq *cfs_rq = cfs_rq_of(se); 1193 struct sched_avg *sa = &se->avg; 1194 long cpu_scale = arch_scale_cpu_capacity(cpu_of(rq_of(cfs_rq))); 1195 long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2; 1196 1197 if (p->sched_class != &fair_sched_class) { 1198 /* 1199 * For !fair tasks do: 1200 * 1201 update_cfs_rq_load_avg(now, cfs_rq); 1202 attach_entity_load_avg(cfs_rq, se); 1203 switched_from_fair(rq, p); 1204 * 1205 * such that the next switched_to_fair() has the 1206 * expected state. 1207 */ 1208 se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq); 1209 return; 1210 } 1211 1212 if (cap > 0) { 1213 if (cfs_rq->avg.util_avg != 0) { 1214 sa->util_avg = cfs_rq->avg.util_avg * se_weight(se); 1215 sa->util_avg /= (cfs_rq->avg.load_avg + 1); 1216 1217 if (sa->util_avg > cap) 1218 sa->util_avg = cap; 1219 } else { 1220 sa->util_avg = cap; 1221 } 1222 } 1223 1224 sa->runnable_avg = sa->util_avg; 1225 } 1226 1227 static s64 update_se(struct rq *rq, struct sched_entity *se) 1228 { 1229 u64 now = rq_clock_task(rq); 1230 s64 delta_exec; 1231 1232 delta_exec = now - se->exec_start; 1233 if (unlikely(delta_exec <= 0)) 1234 return delta_exec; 1235 1236 se->exec_start = now; 1237 if (entity_is_task(se)) { 1238 struct task_struct *donor = task_of(se); 1239 struct task_struct *running = rq->curr; 1240 /* 1241 * If se is a task, we account the time against the running 1242 * task, as w/ proxy-exec they may not be the same. 1243 */ 1244 running->se.exec_start = now; 1245 running->se.sum_exec_runtime += delta_exec; 1246 1247 trace_sched_stat_runtime(running, delta_exec); 1248 account_group_exec_runtime(running, delta_exec); 1249 1250 /* cgroup time is always accounted against the donor */ 1251 cgroup_account_cputime(donor, delta_exec); 1252 } else { 1253 /* If not task, account the time against donor se */ 1254 se->sum_exec_runtime += delta_exec; 1255 } 1256 1257 if (schedstat_enabled()) { 1258 struct sched_statistics *stats; 1259 1260 stats = __schedstats_from_se(se); 1261 __schedstat_set(stats->exec_max, 1262 max(delta_exec, stats->exec_max)); 1263 } 1264 1265 return delta_exec; 1266 } 1267 1268 static void set_next_buddy(struct sched_entity *se); 1269 1270 /* 1271 * Used by other classes to account runtime. 1272 */ 1273 s64 update_curr_common(struct rq *rq) 1274 { 1275 return update_se(rq, &rq->donor->se); 1276 } 1277 1278 /* 1279 * Update the current task's runtime statistics. 1280 */ 1281 static void update_curr(struct cfs_rq *cfs_rq) 1282 { 1283 /* 1284 * Note: cfs_rq->curr corresponds to the task picked to 1285 * run (ie: rq->donor.se) which due to proxy-exec may 1286 * not necessarily be the actual task running 1287 * (rq->curr.se). This is easy to confuse! 1288 */ 1289 struct sched_entity *curr = cfs_rq->curr; 1290 struct rq *rq = rq_of(cfs_rq); 1291 s64 delta_exec; 1292 bool resched; 1293 1294 if (unlikely(!curr)) 1295 return; 1296 1297 delta_exec = update_se(rq, curr); 1298 if (unlikely(delta_exec <= 0)) 1299 return; 1300 1301 curr->vruntime += calc_delta_fair(delta_exec, curr); 1302 resched = update_deadline(cfs_rq, curr); 1303 1304 if (entity_is_task(curr)) { 1305 /* 1306 * If the fair_server is active, we need to account for the 1307 * fair_server time whether or not the task is running on 1308 * behalf of fair_server or not: 1309 * - If the task is running on behalf of fair_server, we need 1310 * to limit its time based on the assigned runtime. 1311 * - Fair task that runs outside of fair_server should account 1312 * against fair_server such that it can account for this time 1313 * and possibly avoid running this period. 1314 */ 1315 if (dl_server_active(&rq->fair_server)) 1316 dl_server_update(&rq->fair_server, delta_exec); 1317 } 1318 1319 account_cfs_rq_runtime(cfs_rq, delta_exec); 1320 1321 if (cfs_rq->nr_queued == 1) 1322 return; 1323 1324 if (resched || !protect_slice(curr)) { 1325 resched_curr_lazy(rq); 1326 clear_buddies(cfs_rq, curr); 1327 } 1328 } 1329 1330 static void update_curr_fair(struct rq *rq) 1331 { 1332 update_curr(cfs_rq_of(&rq->donor->se)); 1333 } 1334 1335 static inline void 1336 update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1337 { 1338 struct sched_statistics *stats; 1339 struct task_struct *p = NULL; 1340 1341 if (!schedstat_enabled()) 1342 return; 1343 1344 stats = __schedstats_from_se(se); 1345 1346 if (entity_is_task(se)) 1347 p = task_of(se); 1348 1349 __update_stats_wait_start(rq_of(cfs_rq), p, stats); 1350 } 1351 1352 static inline void 1353 update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1354 { 1355 struct sched_statistics *stats; 1356 struct task_struct *p = NULL; 1357 1358 if (!schedstat_enabled()) 1359 return; 1360 1361 stats = __schedstats_from_se(se); 1362 1363 /* 1364 * When the sched_schedstat changes from 0 to 1, some sched se 1365 * maybe already in the runqueue, the se->statistics.wait_start 1366 * will be 0.So it will let the delta wrong. We need to avoid this 1367 * scenario. 1368 */ 1369 if (unlikely(!schedstat_val(stats->wait_start))) 1370 return; 1371 1372 if (entity_is_task(se)) 1373 p = task_of(se); 1374 1375 __update_stats_wait_end(rq_of(cfs_rq), p, stats); 1376 } 1377 1378 static inline void 1379 update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se) 1380 { 1381 struct sched_statistics *stats; 1382 struct task_struct *tsk = NULL; 1383 1384 if (!schedstat_enabled()) 1385 return; 1386 1387 stats = __schedstats_from_se(se); 1388 1389 if (entity_is_task(se)) 1390 tsk = task_of(se); 1391 1392 __update_stats_enqueue_sleeper(rq_of(cfs_rq), tsk, stats); 1393 } 1394 1395 /* 1396 * Task is being enqueued - update stats: 1397 */ 1398 static inline void 1399 update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1400 { 1401 if (!schedstat_enabled()) 1402 return; 1403 1404 /* 1405 * Are we enqueueing a waiting task? (for current tasks 1406 * a dequeue/enqueue event is a NOP) 1407 */ 1408 if (se != cfs_rq->curr) 1409 update_stats_wait_start_fair(cfs_rq, se); 1410 1411 if (flags & ENQUEUE_WAKEUP) 1412 update_stats_enqueue_sleeper_fair(cfs_rq, se); 1413 } 1414 1415 static inline void 1416 update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 1417 { 1418 1419 if (!schedstat_enabled()) 1420 return; 1421 1422 /* 1423 * Mark the end of the wait period if dequeueing a 1424 * waiting task: 1425 */ 1426 if (se != cfs_rq->curr) 1427 update_stats_wait_end_fair(cfs_rq, se); 1428 1429 if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { 1430 struct task_struct *tsk = task_of(se); 1431 unsigned int state; 1432 1433 /* XXX racy against TTWU */ 1434 state = READ_ONCE(tsk->__state); 1435 if (state & TASK_INTERRUPTIBLE) 1436 __schedstat_set(tsk->stats.sleep_start, 1437 rq_clock(rq_of(cfs_rq))); 1438 if (state & TASK_UNINTERRUPTIBLE) 1439 __schedstat_set(tsk->stats.block_start, 1440 rq_clock(rq_of(cfs_rq))); 1441 } 1442 } 1443 1444 /* 1445 * We are picking a new current task - update its stats: 1446 */ 1447 static inline void 1448 update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) 1449 { 1450 /* 1451 * We are starting a new run period: 1452 */ 1453 se->exec_start = rq_clock_task(rq_of(cfs_rq)); 1454 } 1455 1456 /************************************************** 1457 * Scheduling class queueing methods: 1458 */ 1459 1460 static inline bool is_core_idle(int cpu) 1461 { 1462 #ifdef CONFIG_SCHED_SMT 1463 int sibling; 1464 1465 for_each_cpu(sibling, cpu_smt_mask(cpu)) { 1466 if (cpu == sibling) 1467 continue; 1468 1469 if (!idle_cpu(sibling)) 1470 return false; 1471 } 1472 #endif 1473 1474 return true; 1475 } 1476 1477 #ifdef CONFIG_NUMA 1478 #define NUMA_IMBALANCE_MIN 2 1479 1480 static inline long 1481 adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr) 1482 { 1483 /* 1484 * Allow a NUMA imbalance if busy CPUs is less than the maximum 1485 * threshold. Above this threshold, individual tasks may be contending 1486 * for both memory bandwidth and any shared HT resources. This is an 1487 * approximation as the number of running tasks may not be related to 1488 * the number of busy CPUs due to sched_setaffinity. 1489 */ 1490 if (dst_running > imb_numa_nr) 1491 return imbalance; 1492 1493 /* 1494 * Allow a small imbalance based on a simple pair of communicating 1495 * tasks that remain local when the destination is lightly loaded. 1496 */ 1497 if (imbalance <= NUMA_IMBALANCE_MIN) 1498 return 0; 1499 1500 return imbalance; 1501 } 1502 #endif /* CONFIG_NUMA */ 1503 1504 #ifdef CONFIG_NUMA_BALANCING 1505 /* 1506 * Approximate time to scan a full NUMA task in ms. The task scan period is 1507 * calculated based on the tasks virtual memory size and 1508 * numa_balancing_scan_size. 1509 */ 1510 unsigned int sysctl_numa_balancing_scan_period_min = 1000; 1511 unsigned int sysctl_numa_balancing_scan_period_max = 60000; 1512 1513 /* Portion of address space to scan in MB */ 1514 unsigned int sysctl_numa_balancing_scan_size = 256; 1515 1516 /* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */ 1517 unsigned int sysctl_numa_balancing_scan_delay = 1000; 1518 1519 /* The page with hint page fault latency < threshold in ms is considered hot */ 1520 unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC; 1521 1522 struct numa_group { 1523 refcount_t refcount; 1524 1525 spinlock_t lock; /* nr_tasks, tasks */ 1526 int nr_tasks; 1527 pid_t gid; 1528 int active_nodes; 1529 1530 struct rcu_head rcu; 1531 unsigned long total_faults; 1532 unsigned long max_faults_cpu; 1533 /* 1534 * faults[] array is split into two regions: faults_mem and faults_cpu. 1535 * 1536 * Faults_cpu is used to decide whether memory should move 1537 * towards the CPU. As a consequence, these stats are weighted 1538 * more by CPU use than by memory faults. 1539 */ 1540 unsigned long faults[]; 1541 }; 1542 1543 /* 1544 * For functions that can be called in multiple contexts that permit reading 1545 * ->numa_group (see struct task_struct for locking rules). 1546 */ 1547 static struct numa_group *deref_task_numa_group(struct task_struct *p) 1548 { 1549 return rcu_dereference_check(p->numa_group, p == current || 1550 (lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu))); 1551 } 1552 1553 static struct numa_group *deref_curr_numa_group(struct task_struct *p) 1554 { 1555 return rcu_dereference_protected(p->numa_group, p == current); 1556 } 1557 1558 static inline unsigned long group_faults_priv(struct numa_group *ng); 1559 static inline unsigned long group_faults_shared(struct numa_group *ng); 1560 1561 static unsigned int task_nr_scan_windows(struct task_struct *p) 1562 { 1563 unsigned long rss = 0; 1564 unsigned long nr_scan_pages; 1565 1566 /* 1567 * Calculations based on RSS as non-present and empty pages are skipped 1568 * by the PTE scanner and NUMA hinting faults should be trapped based 1569 * on resident pages 1570 */ 1571 nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); 1572 rss = get_mm_rss(p->mm); 1573 if (!rss) 1574 rss = nr_scan_pages; 1575 1576 rss = round_up(rss, nr_scan_pages); 1577 return rss / nr_scan_pages; 1578 } 1579 1580 /* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */ 1581 #define MAX_SCAN_WINDOW 2560 1582 1583 static unsigned int task_scan_min(struct task_struct *p) 1584 { 1585 unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size); 1586 unsigned int scan, floor; 1587 unsigned int windows = 1; 1588 1589 if (scan_size < MAX_SCAN_WINDOW) 1590 windows = MAX_SCAN_WINDOW / scan_size; 1591 floor = 1000 / windows; 1592 1593 scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p); 1594 return max_t(unsigned int, floor, scan); 1595 } 1596 1597 static unsigned int task_scan_start(struct task_struct *p) 1598 { 1599 unsigned long smin = task_scan_min(p); 1600 unsigned long period = smin; 1601 struct numa_group *ng; 1602 1603 /* Scale the maximum scan period with the amount of shared memory. */ 1604 rcu_read_lock(); 1605 ng = rcu_dereference(p->numa_group); 1606 if (ng) { 1607 unsigned long shared = group_faults_shared(ng); 1608 unsigned long private = group_faults_priv(ng); 1609 1610 period *= refcount_read(&ng->refcount); 1611 period *= shared + 1; 1612 period /= private + shared + 1; 1613 } 1614 rcu_read_unlock(); 1615 1616 return max(smin, period); 1617 } 1618 1619 static unsigned int task_scan_max(struct task_struct *p) 1620 { 1621 unsigned long smin = task_scan_min(p); 1622 unsigned long smax; 1623 struct numa_group *ng; 1624 1625 /* Watch for min being lower than max due to floor calculations */ 1626 smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p); 1627 1628 /* Scale the maximum scan period with the amount of shared memory. */ 1629 ng = deref_curr_numa_group(p); 1630 if (ng) { 1631 unsigned long shared = group_faults_shared(ng); 1632 unsigned long private = group_faults_priv(ng); 1633 unsigned long period = smax; 1634 1635 period *= refcount_read(&ng->refcount); 1636 period *= shared + 1; 1637 period /= private + shared + 1; 1638 1639 smax = max(smax, period); 1640 } 1641 1642 return max(smin, smax); 1643 } 1644 1645 static void account_numa_enqueue(struct rq *rq, struct task_struct *p) 1646 { 1647 rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE); 1648 rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p)); 1649 } 1650 1651 static void account_numa_dequeue(struct rq *rq, struct task_struct *p) 1652 { 1653 rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE); 1654 rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p)); 1655 } 1656 1657 /* Shared or private faults. */ 1658 #define NR_NUMA_HINT_FAULT_TYPES 2 1659 1660 /* Memory and CPU locality */ 1661 #define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2) 1662 1663 /* Averaged statistics, and temporary buffers. */ 1664 #define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2) 1665 1666 pid_t task_numa_group_id(struct task_struct *p) 1667 { 1668 struct numa_group *ng; 1669 pid_t gid = 0; 1670 1671 rcu_read_lock(); 1672 ng = rcu_dereference(p->numa_group); 1673 if (ng) 1674 gid = ng->gid; 1675 rcu_read_unlock(); 1676 1677 return gid; 1678 } 1679 1680 /* 1681 * The averaged statistics, shared & private, memory & CPU, 1682 * occupy the first half of the array. The second half of the 1683 * array is for current counters, which are averaged into the 1684 * first set by task_numa_placement. 1685 */ 1686 static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv) 1687 { 1688 return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv; 1689 } 1690 1691 static inline unsigned long task_faults(struct task_struct *p, int nid) 1692 { 1693 if (!p->numa_faults) 1694 return 0; 1695 1696 return p->numa_faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1697 p->numa_faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1698 } 1699 1700 static inline unsigned long group_faults(struct task_struct *p, int nid) 1701 { 1702 struct numa_group *ng = deref_task_numa_group(p); 1703 1704 if (!ng) 1705 return 0; 1706 1707 return ng->faults[task_faults_idx(NUMA_MEM, nid, 0)] + 1708 ng->faults[task_faults_idx(NUMA_MEM, nid, 1)]; 1709 } 1710 1711 static inline unsigned long group_faults_cpu(struct numa_group *group, int nid) 1712 { 1713 return group->faults[task_faults_idx(NUMA_CPU, nid, 0)] + 1714 group->faults[task_faults_idx(NUMA_CPU, nid, 1)]; 1715 } 1716 1717 static inline unsigned long group_faults_priv(struct numa_group *ng) 1718 { 1719 unsigned long faults = 0; 1720 int node; 1721 1722 for_each_online_node(node) { 1723 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 1)]; 1724 } 1725 1726 return faults; 1727 } 1728 1729 static inline unsigned long group_faults_shared(struct numa_group *ng) 1730 { 1731 unsigned long faults = 0; 1732 int node; 1733 1734 for_each_online_node(node) { 1735 faults += ng->faults[task_faults_idx(NUMA_MEM, node, 0)]; 1736 } 1737 1738 return faults; 1739 } 1740 1741 /* 1742 * A node triggering more than 1/3 as many NUMA faults as the maximum is 1743 * considered part of a numa group's pseudo-interleaving set. Migrations 1744 * between these nodes are slowed down, to allow things to settle down. 1745 */ 1746 #define ACTIVE_NODE_FRACTION 3 1747 1748 static bool numa_is_active_node(int nid, struct numa_group *ng) 1749 { 1750 return group_faults_cpu(ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu; 1751 } 1752 1753 /* Handle placement on systems where not all nodes are directly connected. */ 1754 static unsigned long score_nearby_nodes(struct task_struct *p, int nid, 1755 int lim_dist, bool task) 1756 { 1757 unsigned long score = 0; 1758 int node, max_dist; 1759 1760 /* 1761 * All nodes are directly connected, and the same distance 1762 * from each other. No need for fancy placement algorithms. 1763 */ 1764 if (sched_numa_topology_type == NUMA_DIRECT) 1765 return 0; 1766 1767 /* sched_max_numa_distance may be changed in parallel. */ 1768 max_dist = READ_ONCE(sched_max_numa_distance); 1769 /* 1770 * This code is called for each node, introducing N^2 complexity, 1771 * which should be OK given the number of nodes rarely exceeds 8. 1772 */ 1773 for_each_online_node(node) { 1774 unsigned long faults; 1775 int dist = node_distance(nid, node); 1776 1777 /* 1778 * The furthest away nodes in the system are not interesting 1779 * for placement; nid was already counted. 1780 */ 1781 if (dist >= max_dist || node == nid) 1782 continue; 1783 1784 /* 1785 * On systems with a backplane NUMA topology, compare groups 1786 * of nodes, and move tasks towards the group with the most 1787 * memory accesses. When comparing two nodes at distance 1788 * "hoplimit", only nodes closer by than "hoplimit" are part 1789 * of each group. Skip other nodes. 1790 */ 1791 if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist) 1792 continue; 1793 1794 /* Add up the faults from nearby nodes. */ 1795 if (task) 1796 faults = task_faults(p, node); 1797 else 1798 faults = group_faults(p, node); 1799 1800 /* 1801 * On systems with a glueless mesh NUMA topology, there are 1802 * no fixed "groups of nodes". Instead, nodes that are not 1803 * directly connected bounce traffic through intermediate 1804 * nodes; a numa_group can occupy any set of nodes. 1805 * The further away a node is, the less the faults count. 1806 * This seems to result in good task placement. 1807 */ 1808 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 1809 faults *= (max_dist - dist); 1810 faults /= (max_dist - LOCAL_DISTANCE); 1811 } 1812 1813 score += faults; 1814 } 1815 1816 return score; 1817 } 1818 1819 /* 1820 * These return the fraction of accesses done by a particular task, or 1821 * task group, on a particular numa node. The group weight is given a 1822 * larger multiplier, in order to group tasks together that are almost 1823 * evenly spread out between numa nodes. 1824 */ 1825 static inline unsigned long task_weight(struct task_struct *p, int nid, 1826 int dist) 1827 { 1828 unsigned long faults, total_faults; 1829 1830 if (!p->numa_faults) 1831 return 0; 1832 1833 total_faults = p->total_numa_faults; 1834 1835 if (!total_faults) 1836 return 0; 1837 1838 faults = task_faults(p, nid); 1839 faults += score_nearby_nodes(p, nid, dist, true); 1840 1841 return 1000 * faults / total_faults; 1842 } 1843 1844 static inline unsigned long group_weight(struct task_struct *p, int nid, 1845 int dist) 1846 { 1847 struct numa_group *ng = deref_task_numa_group(p); 1848 unsigned long faults, total_faults; 1849 1850 if (!ng) 1851 return 0; 1852 1853 total_faults = ng->total_faults; 1854 1855 if (!total_faults) 1856 return 0; 1857 1858 faults = group_faults(p, nid); 1859 faults += score_nearby_nodes(p, nid, dist, false); 1860 1861 return 1000 * faults / total_faults; 1862 } 1863 1864 /* 1865 * If memory tiering mode is enabled, cpupid of slow memory page is 1866 * used to record scan time instead of CPU and PID. When tiering mode 1867 * is disabled at run time, the scan time (in cpupid) will be 1868 * interpreted as CPU and PID. So CPU needs to be checked to avoid to 1869 * access out of array bound. 1870 */ 1871 static inline bool cpupid_valid(int cpupid) 1872 { 1873 return cpupid_to_cpu(cpupid) < nr_cpu_ids; 1874 } 1875 1876 /* 1877 * For memory tiering mode, if there are enough free pages (more than 1878 * enough watermark defined here) in fast memory node, to take full 1879 * advantage of fast memory capacity, all recently accessed slow 1880 * memory pages will be migrated to fast memory node without 1881 * considering hot threshold. 1882 */ 1883 static bool pgdat_free_space_enough(struct pglist_data *pgdat) 1884 { 1885 int z; 1886 unsigned long enough_wmark; 1887 1888 enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT, 1889 pgdat->node_present_pages >> 4); 1890 for (z = pgdat->nr_zones - 1; z >= 0; z--) { 1891 struct zone *zone = pgdat->node_zones + z; 1892 1893 if (!populated_zone(zone)) 1894 continue; 1895 1896 if (zone_watermark_ok(zone, 0, 1897 promo_wmark_pages(zone) + enough_wmark, 1898 ZONE_MOVABLE, 0)) 1899 return true; 1900 } 1901 return false; 1902 } 1903 1904 /* 1905 * For memory tiering mode, when page tables are scanned, the scan 1906 * time will be recorded in struct page in addition to make page 1907 * PROT_NONE for slow memory page. So when the page is accessed, in 1908 * hint page fault handler, the hint page fault latency is calculated 1909 * via, 1910 * 1911 * hint page fault latency = hint page fault time - scan time 1912 * 1913 * The smaller the hint page fault latency, the higher the possibility 1914 * for the page to be hot. 1915 */ 1916 static int numa_hint_fault_latency(struct folio *folio) 1917 { 1918 int last_time, time; 1919 1920 time = jiffies_to_msecs(jiffies); 1921 last_time = folio_xchg_access_time(folio, time); 1922 1923 return (time - last_time) & PAGE_ACCESS_TIME_MASK; 1924 } 1925 1926 /* 1927 * For memory tiering mode, too high promotion/demotion throughput may 1928 * hurt application latency. So we provide a mechanism to rate limit 1929 * the number of pages that are tried to be promoted. 1930 */ 1931 static bool numa_promotion_rate_limit(struct pglist_data *pgdat, 1932 unsigned long rate_limit, int nr) 1933 { 1934 unsigned long nr_cand; 1935 unsigned int now, start; 1936 1937 now = jiffies_to_msecs(jiffies); 1938 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr); 1939 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1940 start = pgdat->nbp_rl_start; 1941 if (now - start > MSEC_PER_SEC && 1942 cmpxchg(&pgdat->nbp_rl_start, start, now) == start) 1943 pgdat->nbp_rl_nr_cand = nr_cand; 1944 if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit) 1945 return true; 1946 return false; 1947 } 1948 1949 #define NUMA_MIGRATION_ADJUST_STEPS 16 1950 1951 static void numa_promotion_adjust_threshold(struct pglist_data *pgdat, 1952 unsigned long rate_limit, 1953 unsigned int ref_th) 1954 { 1955 unsigned int now, start, th_period, unit_th, th; 1956 unsigned long nr_cand, ref_cand, diff_cand; 1957 1958 now = jiffies_to_msecs(jiffies); 1959 th_period = sysctl_numa_balancing_scan_period_max; 1960 start = pgdat->nbp_th_start; 1961 if (now - start > th_period && 1962 cmpxchg(&pgdat->nbp_th_start, start, now) == start) { 1963 ref_cand = rate_limit * 1964 sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC; 1965 nr_cand = node_page_state(pgdat, PGPROMOTE_CANDIDATE); 1966 diff_cand = nr_cand - pgdat->nbp_th_nr_cand; 1967 unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS; 1968 th = pgdat->nbp_threshold ? : ref_th; 1969 if (diff_cand > ref_cand * 11 / 10) 1970 th = max(th - unit_th, unit_th); 1971 else if (diff_cand < ref_cand * 9 / 10) 1972 th = min(th + unit_th, ref_th * 2); 1973 pgdat->nbp_th_nr_cand = nr_cand; 1974 pgdat->nbp_threshold = th; 1975 } 1976 } 1977 1978 bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio, 1979 int src_nid, int dst_cpu) 1980 { 1981 struct numa_group *ng = deref_curr_numa_group(p); 1982 int dst_nid = cpu_to_node(dst_cpu); 1983 int last_cpupid, this_cpupid; 1984 1985 /* 1986 * Cannot migrate to memoryless nodes. 1987 */ 1988 if (!node_state(dst_nid, N_MEMORY)) 1989 return false; 1990 1991 /* 1992 * The pages in slow memory node should be migrated according 1993 * to hot/cold instead of private/shared. 1994 */ 1995 if (folio_use_access_time(folio)) { 1996 struct pglist_data *pgdat; 1997 unsigned long rate_limit; 1998 unsigned int latency, th, def_th; 1999 long nr = folio_nr_pages(folio); 2000 2001 pgdat = NODE_DATA(dst_nid); 2002 if (pgdat_free_space_enough(pgdat)) { 2003 /* workload changed, reset hot threshold */ 2004 pgdat->nbp_threshold = 0; 2005 mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr); 2006 return true; 2007 } 2008 2009 def_th = sysctl_numa_balancing_hot_threshold; 2010 rate_limit = MB_TO_PAGES(sysctl_numa_balancing_promote_rate_limit); 2011 numa_promotion_adjust_threshold(pgdat, rate_limit, def_th); 2012 2013 th = pgdat->nbp_threshold ? : def_th; 2014 latency = numa_hint_fault_latency(folio); 2015 if (latency >= th) 2016 return false; 2017 2018 return !numa_promotion_rate_limit(pgdat, rate_limit, nr); 2019 } 2020 2021 this_cpupid = cpu_pid_to_cpupid(dst_cpu, current->pid); 2022 last_cpupid = folio_xchg_last_cpupid(folio, this_cpupid); 2023 2024 if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) && 2025 !node_is_toptier(src_nid) && !cpupid_valid(last_cpupid)) 2026 return false; 2027 2028 /* 2029 * Allow first faults or private faults to migrate immediately early in 2030 * the lifetime of a task. The magic number 4 is based on waiting for 2031 * two full passes of the "multi-stage node selection" test that is 2032 * executed below. 2033 */ 2034 if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) && 2035 (cpupid_pid_unset(last_cpupid) || cpupid_match_pid(p, last_cpupid))) 2036 return true; 2037 2038 /* 2039 * Multi-stage node selection is used in conjunction with a periodic 2040 * migration fault to build a temporal task<->page relation. By using 2041 * a two-stage filter we remove short/unlikely relations. 2042 * 2043 * Using P(p) ~ n_p / n_t as per frequentist probability, we can equate 2044 * a task's usage of a particular page (n_p) per total usage of this 2045 * page (n_t) (in a given time-span) to a probability. 2046 * 2047 * Our periodic faults will sample this probability and getting the 2048 * same result twice in a row, given these samples are fully 2049 * independent, is then given by P(n)^2, provided our sample period 2050 * is sufficiently short compared to the usage pattern. 2051 * 2052 * This quadric squishes small probabilities, making it less likely we 2053 * act on an unlikely task<->page relation. 2054 */ 2055 if (!cpupid_pid_unset(last_cpupid) && 2056 cpupid_to_nid(last_cpupid) != dst_nid) 2057 return false; 2058 2059 /* Always allow migrate on private faults */ 2060 if (cpupid_match_pid(p, last_cpupid)) 2061 return true; 2062 2063 /* A shared fault, but p->numa_group has not been set up yet. */ 2064 if (!ng) 2065 return true; 2066 2067 /* 2068 * Destination node is much more heavily used than the source 2069 * node? Allow migration. 2070 */ 2071 if (group_faults_cpu(ng, dst_nid) > group_faults_cpu(ng, src_nid) * 2072 ACTIVE_NODE_FRACTION) 2073 return true; 2074 2075 /* 2076 * Distribute memory according to CPU & memory use on each node, 2077 * with 3/4 hysteresis to avoid unnecessary memory migrations: 2078 * 2079 * faults_cpu(dst) 3 faults_cpu(src) 2080 * --------------- * - > --------------- 2081 * faults_mem(dst) 4 faults_mem(src) 2082 */ 2083 return group_faults_cpu(ng, dst_nid) * group_faults(p, src_nid) * 3 > 2084 group_faults_cpu(ng, src_nid) * group_faults(p, dst_nid) * 4; 2085 } 2086 2087 /* 2088 * 'numa_type' describes the node at the moment of load balancing. 2089 */ 2090 enum numa_type { 2091 /* The node has spare capacity that can be used to run more tasks. */ 2092 node_has_spare = 0, 2093 /* 2094 * The node is fully used and the tasks don't compete for more CPU 2095 * cycles. Nevertheless, some tasks might wait before running. 2096 */ 2097 node_fully_busy, 2098 /* 2099 * The node is overloaded and can't provide expected CPU cycles to all 2100 * tasks. 2101 */ 2102 node_overloaded 2103 }; 2104 2105 /* Cached statistics for all CPUs within a node */ 2106 struct numa_stats { 2107 unsigned long load; 2108 unsigned long runnable; 2109 unsigned long util; 2110 /* Total compute capacity of CPUs on a node */ 2111 unsigned long compute_capacity; 2112 unsigned int nr_running; 2113 unsigned int weight; 2114 enum numa_type node_type; 2115 int idle_cpu; 2116 }; 2117 2118 struct task_numa_env { 2119 struct task_struct *p; 2120 2121 int src_cpu, src_nid; 2122 int dst_cpu, dst_nid; 2123 int imb_numa_nr; 2124 2125 struct numa_stats src_stats, dst_stats; 2126 2127 int imbalance_pct; 2128 int dist; 2129 2130 struct task_struct *best_task; 2131 long best_imp; 2132 int best_cpu; 2133 }; 2134 2135 static unsigned long cpu_load(struct rq *rq); 2136 static unsigned long cpu_runnable(struct rq *rq); 2137 2138 static inline enum 2139 numa_type numa_classify(unsigned int imbalance_pct, 2140 struct numa_stats *ns) 2141 { 2142 if ((ns->nr_running > ns->weight) && 2143 (((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) || 2144 ((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100)))) 2145 return node_overloaded; 2146 2147 if ((ns->nr_running < ns->weight) || 2148 (((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) && 2149 ((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100)))) 2150 return node_has_spare; 2151 2152 return node_fully_busy; 2153 } 2154 2155 #ifdef CONFIG_SCHED_SMT 2156 /* Forward declarations of select_idle_sibling helpers */ 2157 static inline bool test_idle_cores(int cpu); 2158 static inline int numa_idle_core(int idle_core, int cpu) 2159 { 2160 if (!static_branch_likely(&sched_smt_present) || 2161 idle_core >= 0 || !test_idle_cores(cpu)) 2162 return idle_core; 2163 2164 /* 2165 * Prefer cores instead of packing HT siblings 2166 * and triggering future load balancing. 2167 */ 2168 if (is_core_idle(cpu)) 2169 idle_core = cpu; 2170 2171 return idle_core; 2172 } 2173 #else /* !CONFIG_SCHED_SMT: */ 2174 static inline int numa_idle_core(int idle_core, int cpu) 2175 { 2176 return idle_core; 2177 } 2178 #endif /* !CONFIG_SCHED_SMT */ 2179 2180 /* 2181 * Gather all necessary information to make NUMA balancing placement 2182 * decisions that are compatible with standard load balancer. This 2183 * borrows code and logic from update_sg_lb_stats but sharing a 2184 * common implementation is impractical. 2185 */ 2186 static void update_numa_stats(struct task_numa_env *env, 2187 struct numa_stats *ns, int nid, 2188 bool find_idle) 2189 { 2190 int cpu, idle_core = -1; 2191 2192 memset(ns, 0, sizeof(*ns)); 2193 ns->idle_cpu = -1; 2194 2195 rcu_read_lock(); 2196 for_each_cpu(cpu, cpumask_of_node(nid)) { 2197 struct rq *rq = cpu_rq(cpu); 2198 2199 ns->load += cpu_load(rq); 2200 ns->runnable += cpu_runnable(rq); 2201 ns->util += cpu_util_cfs(cpu); 2202 ns->nr_running += rq->cfs.h_nr_runnable; 2203 ns->compute_capacity += capacity_of(cpu); 2204 2205 if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) { 2206 if (READ_ONCE(rq->numa_migrate_on) || 2207 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2208 continue; 2209 2210 if (ns->idle_cpu == -1) 2211 ns->idle_cpu = cpu; 2212 2213 idle_core = numa_idle_core(idle_core, cpu); 2214 } 2215 } 2216 rcu_read_unlock(); 2217 2218 ns->weight = cpumask_weight(cpumask_of_node(nid)); 2219 2220 ns->node_type = numa_classify(env->imbalance_pct, ns); 2221 2222 if (idle_core >= 0) 2223 ns->idle_cpu = idle_core; 2224 } 2225 2226 static void task_numa_assign(struct task_numa_env *env, 2227 struct task_struct *p, long imp) 2228 { 2229 struct rq *rq = cpu_rq(env->dst_cpu); 2230 2231 /* Check if run-queue part of active NUMA balance. */ 2232 if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) { 2233 int cpu; 2234 int start = env->dst_cpu; 2235 2236 /* Find alternative idle CPU. */ 2237 for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) { 2238 if (cpu == env->best_cpu || !idle_cpu(cpu) || 2239 !cpumask_test_cpu(cpu, env->p->cpus_ptr)) { 2240 continue; 2241 } 2242 2243 env->dst_cpu = cpu; 2244 rq = cpu_rq(env->dst_cpu); 2245 if (!xchg(&rq->numa_migrate_on, 1)) 2246 goto assign; 2247 } 2248 2249 /* Failed to find an alternative idle CPU */ 2250 return; 2251 } 2252 2253 assign: 2254 /* 2255 * Clear previous best_cpu/rq numa-migrate flag, since task now 2256 * found a better CPU to move/swap. 2257 */ 2258 if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) { 2259 rq = cpu_rq(env->best_cpu); 2260 WRITE_ONCE(rq->numa_migrate_on, 0); 2261 } 2262 2263 if (env->best_task) 2264 put_task_struct(env->best_task); 2265 if (p) 2266 get_task_struct(p); 2267 2268 env->best_task = p; 2269 env->best_imp = imp; 2270 env->best_cpu = env->dst_cpu; 2271 } 2272 2273 static bool load_too_imbalanced(long src_load, long dst_load, 2274 struct task_numa_env *env) 2275 { 2276 long imb, old_imb; 2277 long orig_src_load, orig_dst_load; 2278 long src_capacity, dst_capacity; 2279 2280 /* 2281 * The load is corrected for the CPU capacity available on each node. 2282 * 2283 * src_load dst_load 2284 * ------------ vs --------- 2285 * src_capacity dst_capacity 2286 */ 2287 src_capacity = env->src_stats.compute_capacity; 2288 dst_capacity = env->dst_stats.compute_capacity; 2289 2290 imb = abs(dst_load * src_capacity - src_load * dst_capacity); 2291 2292 orig_src_load = env->src_stats.load; 2293 orig_dst_load = env->dst_stats.load; 2294 2295 old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity); 2296 2297 /* Would this change make things worse? */ 2298 return (imb > old_imb); 2299 } 2300 2301 /* 2302 * Maximum NUMA importance can be 1998 (2*999); 2303 * SMALLIMP @ 30 would be close to 1998/64. 2304 * Used to deter task migration. 2305 */ 2306 #define SMALLIMP 30 2307 2308 /* 2309 * This checks if the overall compute and NUMA accesses of the system would 2310 * be improved if the source tasks was migrated to the target dst_cpu taking 2311 * into account that it might be best if task running on the dst_cpu should 2312 * be exchanged with the source task 2313 */ 2314 static bool task_numa_compare(struct task_numa_env *env, 2315 long taskimp, long groupimp, bool maymove) 2316 { 2317 struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(env->p); 2318 struct rq *dst_rq = cpu_rq(env->dst_cpu); 2319 long imp = p_ng ? groupimp : taskimp; 2320 struct task_struct *cur; 2321 long src_load, dst_load; 2322 int dist = env->dist; 2323 long moveimp = imp; 2324 long load; 2325 bool stopsearch = false; 2326 2327 if (READ_ONCE(dst_rq->numa_migrate_on)) 2328 return false; 2329 2330 rcu_read_lock(); 2331 cur = rcu_dereference(dst_rq->curr); 2332 if (cur && ((cur->flags & (PF_EXITING | PF_KTHREAD)) || 2333 !cur->mm)) 2334 cur = NULL; 2335 2336 /* 2337 * Because we have preemption enabled we can get migrated around and 2338 * end try selecting ourselves (current == env->p) as a swap candidate. 2339 */ 2340 if (cur == env->p) { 2341 stopsearch = true; 2342 goto unlock; 2343 } 2344 2345 if (!cur) { 2346 if (maymove && moveimp >= env->best_imp) 2347 goto assign; 2348 else 2349 goto unlock; 2350 } 2351 2352 /* Skip this swap candidate if cannot move to the source cpu. */ 2353 if (!cpumask_test_cpu(env->src_cpu, cur->cpus_ptr)) 2354 goto unlock; 2355 2356 /* 2357 * Skip this swap candidate if it is not moving to its preferred 2358 * node and the best task is. 2359 */ 2360 if (env->best_task && 2361 env->best_task->numa_preferred_nid == env->src_nid && 2362 cur->numa_preferred_nid != env->src_nid) { 2363 goto unlock; 2364 } 2365 2366 /* 2367 * "imp" is the fault differential for the source task between the 2368 * source and destination node. Calculate the total differential for 2369 * the source task and potential destination task. The more negative 2370 * the value is, the more remote accesses that would be expected to 2371 * be incurred if the tasks were swapped. 2372 * 2373 * If dst and source tasks are in the same NUMA group, or not 2374 * in any group then look only at task weights. 2375 */ 2376 cur_ng = rcu_dereference(cur->numa_group); 2377 if (cur_ng == p_ng) { 2378 /* 2379 * Do not swap within a group or between tasks that have 2380 * no group if there is spare capacity. Swapping does 2381 * not address the load imbalance and helps one task at 2382 * the cost of punishing another. 2383 */ 2384 if (env->dst_stats.node_type == node_has_spare) 2385 goto unlock; 2386 2387 imp = taskimp + task_weight(cur, env->src_nid, dist) - 2388 task_weight(cur, env->dst_nid, dist); 2389 /* 2390 * Add some hysteresis to prevent swapping the 2391 * tasks within a group over tiny differences. 2392 */ 2393 if (cur_ng) 2394 imp -= imp / 16; 2395 } else { 2396 /* 2397 * Compare the group weights. If a task is all by itself 2398 * (not part of a group), use the task weight instead. 2399 */ 2400 if (cur_ng && p_ng) 2401 imp += group_weight(cur, env->src_nid, dist) - 2402 group_weight(cur, env->dst_nid, dist); 2403 else 2404 imp += task_weight(cur, env->src_nid, dist) - 2405 task_weight(cur, env->dst_nid, dist); 2406 } 2407 2408 /* Discourage picking a task already on its preferred node */ 2409 if (cur->numa_preferred_nid == env->dst_nid) 2410 imp -= imp / 16; 2411 2412 /* 2413 * Encourage picking a task that moves to its preferred node. 2414 * This potentially makes imp larger than it's maximum of 2415 * 1998 (see SMALLIMP and task_weight for why) but in this 2416 * case, it does not matter. 2417 */ 2418 if (cur->numa_preferred_nid == env->src_nid) 2419 imp += imp / 8; 2420 2421 if (maymove && moveimp > imp && moveimp > env->best_imp) { 2422 imp = moveimp; 2423 cur = NULL; 2424 goto assign; 2425 } 2426 2427 /* 2428 * Prefer swapping with a task moving to its preferred node over a 2429 * task that is not. 2430 */ 2431 if (env->best_task && cur->numa_preferred_nid == env->src_nid && 2432 env->best_task->numa_preferred_nid != env->src_nid) { 2433 goto assign; 2434 } 2435 2436 /* 2437 * If the NUMA importance is less than SMALLIMP, 2438 * task migration might only result in ping pong 2439 * of tasks and also hurt performance due to cache 2440 * misses. 2441 */ 2442 if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2) 2443 goto unlock; 2444 2445 /* 2446 * In the overloaded case, try and keep the load balanced. 2447 */ 2448 load = task_h_load(env->p) - task_h_load(cur); 2449 if (!load) 2450 goto assign; 2451 2452 dst_load = env->dst_stats.load + load; 2453 src_load = env->src_stats.load - load; 2454 2455 if (load_too_imbalanced(src_load, dst_load, env)) 2456 goto unlock; 2457 2458 assign: 2459 /* Evaluate an idle CPU for a task numa move. */ 2460 if (!cur) { 2461 int cpu = env->dst_stats.idle_cpu; 2462 2463 /* Nothing cached so current CPU went idle since the search. */ 2464 if (cpu < 0) 2465 cpu = env->dst_cpu; 2466 2467 /* 2468 * If the CPU is no longer truly idle and the previous best CPU 2469 * is, keep using it. 2470 */ 2471 if (!idle_cpu(cpu) && env->best_cpu >= 0 && 2472 idle_cpu(env->best_cpu)) { 2473 cpu = env->best_cpu; 2474 } 2475 2476 env->dst_cpu = cpu; 2477 } 2478 2479 task_numa_assign(env, cur, imp); 2480 2481 /* 2482 * If a move to idle is allowed because there is capacity or load 2483 * balance improves then stop the search. While a better swap 2484 * candidate may exist, a search is not free. 2485 */ 2486 if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(env->best_cpu)) 2487 stopsearch = true; 2488 2489 /* 2490 * If a swap candidate must be identified and the current best task 2491 * moves its preferred node then stop the search. 2492 */ 2493 if (!maymove && env->best_task && 2494 env->best_task->numa_preferred_nid == env->src_nid) { 2495 stopsearch = true; 2496 } 2497 unlock: 2498 rcu_read_unlock(); 2499 2500 return stopsearch; 2501 } 2502 2503 static void task_numa_find_cpu(struct task_numa_env *env, 2504 long taskimp, long groupimp) 2505 { 2506 bool maymove = false; 2507 int cpu; 2508 2509 /* 2510 * If dst node has spare capacity, then check if there is an 2511 * imbalance that would be overruled by the load balancer. 2512 */ 2513 if (env->dst_stats.node_type == node_has_spare) { 2514 unsigned int imbalance; 2515 int src_running, dst_running; 2516 2517 /* 2518 * Would movement cause an imbalance? Note that if src has 2519 * more running tasks that the imbalance is ignored as the 2520 * move improves the imbalance from the perspective of the 2521 * CPU load balancer. 2522 * */ 2523 src_running = env->src_stats.nr_running - 1; 2524 dst_running = env->dst_stats.nr_running + 1; 2525 imbalance = max(0, dst_running - src_running); 2526 imbalance = adjust_numa_imbalance(imbalance, dst_running, 2527 env->imb_numa_nr); 2528 2529 /* Use idle CPU if there is no imbalance */ 2530 if (!imbalance) { 2531 maymove = true; 2532 if (env->dst_stats.idle_cpu >= 0) { 2533 env->dst_cpu = env->dst_stats.idle_cpu; 2534 task_numa_assign(env, NULL, 0); 2535 return; 2536 } 2537 } 2538 } else { 2539 long src_load, dst_load, load; 2540 /* 2541 * If the improvement from just moving env->p direction is better 2542 * than swapping tasks around, check if a move is possible. 2543 */ 2544 load = task_h_load(env->p); 2545 dst_load = env->dst_stats.load + load; 2546 src_load = env->src_stats.load - load; 2547 maymove = !load_too_imbalanced(src_load, dst_load, env); 2548 } 2549 2550 for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) { 2551 /* Skip this CPU if the source task cannot migrate */ 2552 if (!cpumask_test_cpu(cpu, env->p->cpus_ptr)) 2553 continue; 2554 2555 env->dst_cpu = cpu; 2556 if (task_numa_compare(env, taskimp, groupimp, maymove)) 2557 break; 2558 } 2559 } 2560 2561 static int task_numa_migrate(struct task_struct *p) 2562 { 2563 struct task_numa_env env = { 2564 .p = p, 2565 2566 .src_cpu = task_cpu(p), 2567 .src_nid = task_node(p), 2568 2569 .imbalance_pct = 112, 2570 2571 .best_task = NULL, 2572 .best_imp = 0, 2573 .best_cpu = -1, 2574 }; 2575 unsigned long taskweight, groupweight; 2576 struct sched_domain *sd; 2577 long taskimp, groupimp; 2578 struct numa_group *ng; 2579 struct rq *best_rq; 2580 int nid, ret, dist; 2581 2582 /* 2583 * Pick the lowest SD_NUMA domain, as that would have the smallest 2584 * imbalance and would be the first to start moving tasks about. 2585 * 2586 * And we want to avoid any moving of tasks about, as that would create 2587 * random movement of tasks -- counter the numa conditions we're trying 2588 * to satisfy here. 2589 */ 2590 rcu_read_lock(); 2591 sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu)); 2592 if (sd) { 2593 env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2; 2594 env.imb_numa_nr = sd->imb_numa_nr; 2595 } 2596 rcu_read_unlock(); 2597 2598 /* 2599 * Cpusets can break the scheduler domain tree into smaller 2600 * balance domains, some of which do not cross NUMA boundaries. 2601 * Tasks that are "trapped" in such domains cannot be migrated 2602 * elsewhere, so there is no point in (re)trying. 2603 */ 2604 if (unlikely(!sd)) { 2605 sched_setnuma(p, task_node(p)); 2606 return -EINVAL; 2607 } 2608 2609 env.dst_nid = p->numa_preferred_nid; 2610 dist = env.dist = node_distance(env.src_nid, env.dst_nid); 2611 taskweight = task_weight(p, env.src_nid, dist); 2612 groupweight = group_weight(p, env.src_nid, dist); 2613 update_numa_stats(&env, &env.src_stats, env.src_nid, false); 2614 taskimp = task_weight(p, env.dst_nid, dist) - taskweight; 2615 groupimp = group_weight(p, env.dst_nid, dist) - groupweight; 2616 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2617 2618 /* Try to find a spot on the preferred nid. */ 2619 task_numa_find_cpu(&env, taskimp, groupimp); 2620 2621 /* 2622 * Look at other nodes in these cases: 2623 * - there is no space available on the preferred_nid 2624 * - the task is part of a numa_group that is interleaved across 2625 * multiple NUMA nodes; in order to better consolidate the group, 2626 * we need to check other locations. 2627 */ 2628 ng = deref_curr_numa_group(p); 2629 if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) { 2630 for_each_node_state(nid, N_CPU) { 2631 if (nid == env.src_nid || nid == p->numa_preferred_nid) 2632 continue; 2633 2634 dist = node_distance(env.src_nid, env.dst_nid); 2635 if (sched_numa_topology_type == NUMA_BACKPLANE && 2636 dist != env.dist) { 2637 taskweight = task_weight(p, env.src_nid, dist); 2638 groupweight = group_weight(p, env.src_nid, dist); 2639 } 2640 2641 /* Only consider nodes where both task and groups benefit */ 2642 taskimp = task_weight(p, nid, dist) - taskweight; 2643 groupimp = group_weight(p, nid, dist) - groupweight; 2644 if (taskimp < 0 && groupimp < 0) 2645 continue; 2646 2647 env.dist = dist; 2648 env.dst_nid = nid; 2649 update_numa_stats(&env, &env.dst_stats, env.dst_nid, true); 2650 task_numa_find_cpu(&env, taskimp, groupimp); 2651 } 2652 } 2653 2654 /* 2655 * If the task is part of a workload that spans multiple NUMA nodes, 2656 * and is migrating into one of the workload's active nodes, remember 2657 * this node as the task's preferred numa node, so the workload can 2658 * settle down. 2659 * A task that migrated to a second choice node will be better off 2660 * trying for a better one later. Do not set the preferred node here. 2661 */ 2662 if (ng) { 2663 if (env.best_cpu == -1) 2664 nid = env.src_nid; 2665 else 2666 nid = cpu_to_node(env.best_cpu); 2667 2668 if (nid != p->numa_preferred_nid) 2669 sched_setnuma(p, nid); 2670 } 2671 2672 /* No better CPU than the current one was found. */ 2673 if (env.best_cpu == -1) { 2674 trace_sched_stick_numa(p, env.src_cpu, NULL, -1); 2675 return -EAGAIN; 2676 } 2677 2678 best_rq = cpu_rq(env.best_cpu); 2679 if (env.best_task == NULL) { 2680 ret = migrate_task_to(p, env.best_cpu); 2681 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2682 if (ret != 0) 2683 trace_sched_stick_numa(p, env.src_cpu, NULL, env.best_cpu); 2684 return ret; 2685 } 2686 2687 ret = migrate_swap(p, env.best_task, env.best_cpu, env.src_cpu); 2688 WRITE_ONCE(best_rq->numa_migrate_on, 0); 2689 2690 if (ret != 0) 2691 trace_sched_stick_numa(p, env.src_cpu, env.best_task, env.best_cpu); 2692 put_task_struct(env.best_task); 2693 return ret; 2694 } 2695 2696 /* Attempt to migrate a task to a CPU on the preferred node. */ 2697 static void numa_migrate_preferred(struct task_struct *p) 2698 { 2699 unsigned long interval = HZ; 2700 2701 /* This task has no NUMA fault statistics yet */ 2702 if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults)) 2703 return; 2704 2705 /* Periodically retry migrating the task to the preferred node */ 2706 interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16); 2707 p->numa_migrate_retry = jiffies + interval; 2708 2709 /* Success if task is already running on preferred CPU */ 2710 if (task_node(p) == p->numa_preferred_nid) 2711 return; 2712 2713 /* Otherwise, try migrate to a CPU on the preferred node */ 2714 task_numa_migrate(p); 2715 } 2716 2717 /* 2718 * Find out how many nodes the workload is actively running on. Do this by 2719 * tracking the nodes from which NUMA hinting faults are triggered. This can 2720 * be different from the set of nodes where the workload's memory is currently 2721 * located. 2722 */ 2723 static void numa_group_count_active_nodes(struct numa_group *numa_group) 2724 { 2725 unsigned long faults, max_faults = 0; 2726 int nid, active_nodes = 0; 2727 2728 for_each_node_state(nid, N_CPU) { 2729 faults = group_faults_cpu(numa_group, nid); 2730 if (faults > max_faults) 2731 max_faults = faults; 2732 } 2733 2734 for_each_node_state(nid, N_CPU) { 2735 faults = group_faults_cpu(numa_group, nid); 2736 if (faults * ACTIVE_NODE_FRACTION > max_faults) 2737 active_nodes++; 2738 } 2739 2740 numa_group->max_faults_cpu = max_faults; 2741 numa_group->active_nodes = active_nodes; 2742 } 2743 2744 /* 2745 * When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS 2746 * increments. The more local the fault statistics are, the higher the scan 2747 * period will be for the next scan window. If local/(local+remote) ratio is 2748 * below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS) 2749 * the scan period will decrease. Aim for 70% local accesses. 2750 */ 2751 #define NUMA_PERIOD_SLOTS 10 2752 #define NUMA_PERIOD_THRESHOLD 7 2753 2754 /* 2755 * Increase the scan period (slow down scanning) if the majority of 2756 * our memory is already on our local node, or if the majority of 2757 * the page accesses are shared with other processes. 2758 * Otherwise, decrease the scan period. 2759 */ 2760 static void update_task_scan_period(struct task_struct *p, 2761 unsigned long shared, unsigned long private) 2762 { 2763 unsigned int period_slot; 2764 int lr_ratio, ps_ratio; 2765 int diff; 2766 2767 unsigned long remote = p->numa_faults_locality[0]; 2768 unsigned long local = p->numa_faults_locality[1]; 2769 2770 /* 2771 * If there were no record hinting faults then either the task is 2772 * completely idle or all activity is in areas that are not of interest 2773 * to automatic numa balancing. Related to that, if there were failed 2774 * migration then it implies we are migrating too quickly or the local 2775 * node is overloaded. In either case, scan slower 2776 */ 2777 if (local + shared == 0 || p->numa_faults_locality[2]) { 2778 p->numa_scan_period = min(p->numa_scan_period_max, 2779 p->numa_scan_period << 1); 2780 2781 p->mm->numa_next_scan = jiffies + 2782 msecs_to_jiffies(p->numa_scan_period); 2783 2784 return; 2785 } 2786 2787 /* 2788 * Prepare to scale scan period relative to the current period. 2789 * == NUMA_PERIOD_THRESHOLD scan period stays the same 2790 * < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster) 2791 * >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower) 2792 */ 2793 period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS); 2794 lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote); 2795 ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared); 2796 2797 if (ps_ratio >= NUMA_PERIOD_THRESHOLD) { 2798 /* 2799 * Most memory accesses are local. There is no need to 2800 * do fast NUMA scanning, since memory is already local. 2801 */ 2802 int slot = ps_ratio - NUMA_PERIOD_THRESHOLD; 2803 if (!slot) 2804 slot = 1; 2805 diff = slot * period_slot; 2806 } else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) { 2807 /* 2808 * Most memory accesses are shared with other tasks. 2809 * There is no point in continuing fast NUMA scanning, 2810 * since other tasks may just move the memory elsewhere. 2811 */ 2812 int slot = lr_ratio - NUMA_PERIOD_THRESHOLD; 2813 if (!slot) 2814 slot = 1; 2815 diff = slot * period_slot; 2816 } else { 2817 /* 2818 * Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS, 2819 * yet they are not on the local NUMA node. Speed up 2820 * NUMA scanning to get the memory moved over. 2821 */ 2822 int ratio = max(lr_ratio, ps_ratio); 2823 diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot; 2824 } 2825 2826 p->numa_scan_period = clamp(p->numa_scan_period + diff, 2827 task_scan_min(p), task_scan_max(p)); 2828 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 2829 } 2830 2831 /* 2832 * Get the fraction of time the task has been running since the last 2833 * NUMA placement cycle. The scheduler keeps similar statistics, but 2834 * decays those on a 32ms period, which is orders of magnitude off 2835 * from the dozens-of-seconds NUMA balancing period. Use the scheduler 2836 * stats only if the task is so new there are no NUMA statistics yet. 2837 */ 2838 static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) 2839 { 2840 u64 runtime, delta, now; 2841 /* Use the start of this time slice to avoid calculations. */ 2842 now = p->se.exec_start; 2843 runtime = p->se.sum_exec_runtime; 2844 2845 if (p->last_task_numa_placement) { 2846 delta = runtime - p->last_sum_exec_runtime; 2847 *period = now - p->last_task_numa_placement; 2848 2849 /* Avoid time going backwards, prevent potential divide error: */ 2850 if (unlikely((s64)*period < 0)) 2851 *period = 0; 2852 } else { 2853 delta = p->se.avg.load_sum; 2854 *period = LOAD_AVG_MAX; 2855 } 2856 2857 p->last_sum_exec_runtime = runtime; 2858 p->last_task_numa_placement = now; 2859 2860 return delta; 2861 } 2862 2863 /* 2864 * Determine the preferred nid for a task in a numa_group. This needs to 2865 * be done in a way that produces consistent results with group_weight, 2866 * otherwise workloads might not converge. 2867 */ 2868 static int preferred_group_nid(struct task_struct *p, int nid) 2869 { 2870 nodemask_t nodes; 2871 int dist; 2872 2873 /* Direct connections between all NUMA nodes. */ 2874 if (sched_numa_topology_type == NUMA_DIRECT) 2875 return nid; 2876 2877 /* 2878 * On a system with glueless mesh NUMA topology, group_weight 2879 * scores nodes according to the number of NUMA hinting faults on 2880 * both the node itself, and on nearby nodes. 2881 */ 2882 if (sched_numa_topology_type == NUMA_GLUELESS_MESH) { 2883 unsigned long score, max_score = 0; 2884 int node, max_node = nid; 2885 2886 dist = sched_max_numa_distance; 2887 2888 for_each_node_state(node, N_CPU) { 2889 score = group_weight(p, node, dist); 2890 if (score > max_score) { 2891 max_score = score; 2892 max_node = node; 2893 } 2894 } 2895 return max_node; 2896 } 2897 2898 /* 2899 * Finding the preferred nid in a system with NUMA backplane 2900 * interconnect topology is more involved. The goal is to locate 2901 * tasks from numa_groups near each other in the system, and 2902 * untangle workloads from different sides of the system. This requires 2903 * searching down the hierarchy of node groups, recursively searching 2904 * inside the highest scoring group of nodes. The nodemask tricks 2905 * keep the complexity of the search down. 2906 */ 2907 nodes = node_states[N_CPU]; 2908 for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) { 2909 unsigned long max_faults = 0; 2910 nodemask_t max_group = NODE_MASK_NONE; 2911 int a, b; 2912 2913 /* Are there nodes at this distance from each other? */ 2914 if (!find_numa_distance(dist)) 2915 continue; 2916 2917 for_each_node_mask(a, nodes) { 2918 unsigned long faults = 0; 2919 nodemask_t this_group; 2920 nodes_clear(this_group); 2921 2922 /* Sum group's NUMA faults; includes a==b case. */ 2923 for_each_node_mask(b, nodes) { 2924 if (node_distance(a, b) < dist) { 2925 faults += group_faults(p, b); 2926 node_set(b, this_group); 2927 node_clear(b, nodes); 2928 } 2929 } 2930 2931 /* Remember the top group. */ 2932 if (faults > max_faults) { 2933 max_faults = faults; 2934 max_group = this_group; 2935 /* 2936 * subtle: at the smallest distance there is 2937 * just one node left in each "group", the 2938 * winner is the preferred nid. 2939 */ 2940 nid = a; 2941 } 2942 } 2943 /* Next round, evaluate the nodes within max_group. */ 2944 if (!max_faults) 2945 break; 2946 nodes = max_group; 2947 } 2948 return nid; 2949 } 2950 2951 static void task_numa_placement(struct task_struct *p) 2952 { 2953 int seq, nid, max_nid = NUMA_NO_NODE; 2954 unsigned long max_faults = 0; 2955 unsigned long fault_types[2] = { 0, 0 }; 2956 unsigned long total_faults; 2957 u64 runtime, period; 2958 spinlock_t *group_lock = NULL; 2959 struct numa_group *ng; 2960 2961 /* 2962 * The p->mm->numa_scan_seq field gets updated without 2963 * exclusive access. Use READ_ONCE() here to ensure 2964 * that the field is read in a single access: 2965 */ 2966 seq = READ_ONCE(p->mm->numa_scan_seq); 2967 if (p->numa_scan_seq == seq) 2968 return; 2969 p->numa_scan_seq = seq; 2970 p->numa_scan_period_max = task_scan_max(p); 2971 2972 total_faults = p->numa_faults_locality[0] + 2973 p->numa_faults_locality[1]; 2974 runtime = numa_get_avg_runtime(p, &period); 2975 2976 /* If the task is part of a group prevent parallel updates to group stats */ 2977 ng = deref_curr_numa_group(p); 2978 if (ng) { 2979 group_lock = &ng->lock; 2980 spin_lock_irq(group_lock); 2981 } 2982 2983 /* Find the node with the highest number of faults */ 2984 for_each_online_node(nid) { 2985 /* Keep track of the offsets in numa_faults array */ 2986 int mem_idx, membuf_idx, cpu_idx, cpubuf_idx; 2987 unsigned long faults = 0, group_faults = 0; 2988 int priv; 2989 2990 for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) { 2991 long diff, f_diff, f_weight; 2992 2993 mem_idx = task_faults_idx(NUMA_MEM, nid, priv); 2994 membuf_idx = task_faults_idx(NUMA_MEMBUF, nid, priv); 2995 cpu_idx = task_faults_idx(NUMA_CPU, nid, priv); 2996 cpubuf_idx = task_faults_idx(NUMA_CPUBUF, nid, priv); 2997 2998 /* Decay existing window, copy faults since last scan */ 2999 diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2; 3000 fault_types[priv] += p->numa_faults[membuf_idx]; 3001 p->numa_faults[membuf_idx] = 0; 3002 3003 /* 3004 * Normalize the faults_from, so all tasks in a group 3005 * count according to CPU use, instead of by the raw 3006 * number of faults. Tasks with little runtime have 3007 * little over-all impact on throughput, and thus their 3008 * faults are less important. 3009 */ 3010 f_weight = div64_u64(runtime << 16, period + 1); 3011 f_weight = (f_weight * p->numa_faults[cpubuf_idx]) / 3012 (total_faults + 1); 3013 f_diff = f_weight - p->numa_faults[cpu_idx] / 2; 3014 p->numa_faults[cpubuf_idx] = 0; 3015 3016 p->numa_faults[mem_idx] += diff; 3017 p->numa_faults[cpu_idx] += f_diff; 3018 faults += p->numa_faults[mem_idx]; 3019 p->total_numa_faults += diff; 3020 if (ng) { 3021 /* 3022 * safe because we can only change our own group 3023 * 3024 * mem_idx represents the offset for a given 3025 * nid and priv in a specific region because it 3026 * is at the beginning of the numa_faults array. 3027 */ 3028 ng->faults[mem_idx] += diff; 3029 ng->faults[cpu_idx] += f_diff; 3030 ng->total_faults += diff; 3031 group_faults += ng->faults[mem_idx]; 3032 } 3033 } 3034 3035 if (!ng) { 3036 if (faults > max_faults) { 3037 max_faults = faults; 3038 max_nid = nid; 3039 } 3040 } else if (group_faults > max_faults) { 3041 max_faults = group_faults; 3042 max_nid = nid; 3043 } 3044 } 3045 3046 /* Cannot migrate task to CPU-less node */ 3047 max_nid = numa_nearest_node(max_nid, N_CPU); 3048 3049 if (ng) { 3050 numa_group_count_active_nodes(ng); 3051 spin_unlock_irq(group_lock); 3052 max_nid = preferred_group_nid(p, max_nid); 3053 } 3054 3055 if (max_faults) { 3056 /* Set the new preferred node */ 3057 if (max_nid != p->numa_preferred_nid) 3058 sched_setnuma(p, max_nid); 3059 } 3060 3061 update_task_scan_period(p, fault_types[0], fault_types[1]); 3062 } 3063 3064 static inline int get_numa_group(struct numa_group *grp) 3065 { 3066 return refcount_inc_not_zero(&grp->refcount); 3067 } 3068 3069 static inline void put_numa_group(struct numa_group *grp) 3070 { 3071 if (refcount_dec_and_test(&grp->refcount)) 3072 kfree_rcu(grp, rcu); 3073 } 3074 3075 static void task_numa_group(struct task_struct *p, int cpupid, int flags, 3076 int *priv) 3077 { 3078 struct numa_group *grp, *my_grp; 3079 struct task_struct *tsk; 3080 bool join = false; 3081 int cpu = cpupid_to_cpu(cpupid); 3082 int i; 3083 3084 if (unlikely(!deref_curr_numa_group(p))) { 3085 unsigned int size = sizeof(struct numa_group) + 3086 NR_NUMA_HINT_FAULT_STATS * 3087 nr_node_ids * sizeof(unsigned long); 3088 3089 grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 3090 if (!grp) 3091 return; 3092 3093 refcount_set(&grp->refcount, 1); 3094 grp->active_nodes = 1; 3095 grp->max_faults_cpu = 0; 3096 spin_lock_init(&grp->lock); 3097 grp->gid = p->pid; 3098 3099 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3100 grp->faults[i] = p->numa_faults[i]; 3101 3102 grp->total_faults = p->total_numa_faults; 3103 3104 grp->nr_tasks++; 3105 rcu_assign_pointer(p->numa_group, grp); 3106 } 3107 3108 rcu_read_lock(); 3109 tsk = READ_ONCE(cpu_rq(cpu)->curr); 3110 3111 if (!cpupid_match_pid(tsk, cpupid)) 3112 goto no_join; 3113 3114 grp = rcu_dereference(tsk->numa_group); 3115 if (!grp) 3116 goto no_join; 3117 3118 my_grp = deref_curr_numa_group(p); 3119 if (grp == my_grp) 3120 goto no_join; 3121 3122 /* 3123 * Only join the other group if its bigger; if we're the bigger group, 3124 * the other task will join us. 3125 */ 3126 if (my_grp->nr_tasks > grp->nr_tasks) 3127 goto no_join; 3128 3129 /* 3130 * Tie-break on the grp address. 3131 */ 3132 if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp) 3133 goto no_join; 3134 3135 /* Always join threads in the same process. */ 3136 if (tsk->mm == current->mm) 3137 join = true; 3138 3139 /* Simple filter to avoid false positives due to PID collisions */ 3140 if (flags & TNF_SHARED) 3141 join = true; 3142 3143 /* Update priv based on whether false sharing was detected */ 3144 *priv = !join; 3145 3146 if (join && !get_numa_group(grp)) 3147 goto no_join; 3148 3149 rcu_read_unlock(); 3150 3151 if (!join) 3152 return; 3153 3154 WARN_ON_ONCE(irqs_disabled()); 3155 double_lock_irq(&my_grp->lock, &grp->lock); 3156 3157 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) { 3158 my_grp->faults[i] -= p->numa_faults[i]; 3159 grp->faults[i] += p->numa_faults[i]; 3160 } 3161 my_grp->total_faults -= p->total_numa_faults; 3162 grp->total_faults += p->total_numa_faults; 3163 3164 my_grp->nr_tasks--; 3165 grp->nr_tasks++; 3166 3167 spin_unlock(&my_grp->lock); 3168 spin_unlock_irq(&grp->lock); 3169 3170 rcu_assign_pointer(p->numa_group, grp); 3171 3172 put_numa_group(my_grp); 3173 return; 3174 3175 no_join: 3176 rcu_read_unlock(); 3177 return; 3178 } 3179 3180 /* 3181 * Get rid of NUMA statistics associated with a task (either current or dead). 3182 * If @final is set, the task is dead and has reached refcount zero, so we can 3183 * safely free all relevant data structures. Otherwise, there might be 3184 * concurrent reads from places like load balancing and procfs, and we should 3185 * reset the data back to default state without freeing ->numa_faults. 3186 */ 3187 void task_numa_free(struct task_struct *p, bool final) 3188 { 3189 /* safe: p either is current or is being freed by current */ 3190 struct numa_group *grp = rcu_dereference_raw(p->numa_group); 3191 unsigned long *numa_faults = p->numa_faults; 3192 unsigned long flags; 3193 int i; 3194 3195 if (!numa_faults) 3196 return; 3197 3198 if (grp) { 3199 spin_lock_irqsave(&grp->lock, flags); 3200 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3201 grp->faults[i] -= p->numa_faults[i]; 3202 grp->total_faults -= p->total_numa_faults; 3203 3204 grp->nr_tasks--; 3205 spin_unlock_irqrestore(&grp->lock, flags); 3206 RCU_INIT_POINTER(p->numa_group, NULL); 3207 put_numa_group(grp); 3208 } 3209 3210 if (final) { 3211 p->numa_faults = NULL; 3212 kfree(numa_faults); 3213 } else { 3214 p->total_numa_faults = 0; 3215 for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) 3216 numa_faults[i] = 0; 3217 } 3218 } 3219 3220 /* 3221 * Got a PROT_NONE fault for a page on @node. 3222 */ 3223 void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) 3224 { 3225 struct task_struct *p = current; 3226 bool migrated = flags & TNF_MIGRATED; 3227 int cpu_node = task_node(current); 3228 int local = !!(flags & TNF_FAULT_LOCAL); 3229 struct numa_group *ng; 3230 int priv; 3231 3232 if (!static_branch_likely(&sched_numa_balancing)) 3233 return; 3234 3235 /* for example, ksmd faulting in a user's mm */ 3236 if (!p->mm) 3237 return; 3238 3239 /* 3240 * NUMA faults statistics are unnecessary for the slow memory 3241 * node for memory tiering mode. 3242 */ 3243 if (!node_is_toptier(mem_node) && 3244 (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING || 3245 !cpupid_valid(last_cpupid))) 3246 return; 3247 3248 /* Allocate buffer to track faults on a per-node basis */ 3249 if (unlikely(!p->numa_faults)) { 3250 int size = sizeof(*p->numa_faults) * 3251 NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids; 3252 3253 p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN); 3254 if (!p->numa_faults) 3255 return; 3256 3257 p->total_numa_faults = 0; 3258 memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality)); 3259 } 3260 3261 /* 3262 * First accesses are treated as private, otherwise consider accesses 3263 * to be private if the accessing pid has not changed 3264 */ 3265 if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) { 3266 priv = 1; 3267 } else { 3268 priv = cpupid_match_pid(p, last_cpupid); 3269 if (!priv && !(flags & TNF_NO_GROUP)) 3270 task_numa_group(p, last_cpupid, flags, &priv); 3271 } 3272 3273 /* 3274 * If a workload spans multiple NUMA nodes, a shared fault that 3275 * occurs wholly within the set of nodes that the workload is 3276 * actively using should be counted as local. This allows the 3277 * scan rate to slow down when a workload has settled down. 3278 */ 3279 ng = deref_curr_numa_group(p); 3280 if (!priv && !local && ng && ng->active_nodes > 1 && 3281 numa_is_active_node(cpu_node, ng) && 3282 numa_is_active_node(mem_node, ng)) 3283 local = 1; 3284 3285 /* 3286 * Retry to migrate task to preferred node periodically, in case it 3287 * previously failed, or the scheduler moved us. 3288 */ 3289 if (time_after(jiffies, p->numa_migrate_retry)) { 3290 task_numa_placement(p); 3291 numa_migrate_preferred(p); 3292 } 3293 3294 if (migrated) 3295 p->numa_pages_migrated += pages; 3296 if (flags & TNF_MIGRATE_FAIL) 3297 p->numa_faults_locality[2] += pages; 3298 3299 p->numa_faults[task_faults_idx(NUMA_MEMBUF, mem_node, priv)] += pages; 3300 p->numa_faults[task_faults_idx(NUMA_CPUBUF, cpu_node, priv)] += pages; 3301 p->numa_faults_locality[local] += pages; 3302 } 3303 3304 static void reset_ptenuma_scan(struct task_struct *p) 3305 { 3306 /* 3307 * We only did a read acquisition of the mmap sem, so 3308 * p->mm->numa_scan_seq is written to without exclusive access 3309 * and the update is not guaranteed to be atomic. That's not 3310 * much of an issue though, since this is just used for 3311 * statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not 3312 * expensive, to avoid any form of compiler optimizations: 3313 */ 3314 WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1); 3315 p->mm->numa_scan_offset = 0; 3316 } 3317 3318 static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma) 3319 { 3320 unsigned long pids; 3321 /* 3322 * Allow unconditional access first two times, so that all the (pages) 3323 * of VMAs get prot_none fault introduced irrespective of accesses. 3324 * This is also done to avoid any side effect of task scanning 3325 * amplifying the unfairness of disjoint set of VMAs' access. 3326 */ 3327 if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2) 3328 return true; 3329 3330 pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1]; 3331 if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids)) 3332 return true; 3333 3334 /* 3335 * Complete a scan that has already started regardless of PID access, or 3336 * some VMAs may never be scanned in multi-threaded applications: 3337 */ 3338 if (mm->numa_scan_offset > vma->vm_start) { 3339 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_IGNORE_PID); 3340 return true; 3341 } 3342 3343 /* 3344 * This vma has not been accessed for a while, and if the number 3345 * the threads in the same process is low, which means no other 3346 * threads can help scan this vma, force a vma scan. 3347 */ 3348 if (READ_ONCE(mm->numa_scan_seq) > 3349 (vma->numab_state->prev_scan_seq + get_nr_threads(current))) 3350 return true; 3351 3352 return false; 3353 } 3354 3355 #define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay) 3356 3357 /* 3358 * The expensive part of numa migration is done from task_work context. 3359 * Triggered from task_tick_numa(). 3360 */ 3361 static void task_numa_work(struct callback_head *work) 3362 { 3363 unsigned long migrate, next_scan, now = jiffies; 3364 struct task_struct *p = current; 3365 struct mm_struct *mm = p->mm; 3366 u64 runtime = p->se.sum_exec_runtime; 3367 struct vm_area_struct *vma; 3368 unsigned long start, end; 3369 unsigned long nr_pte_updates = 0; 3370 long pages, virtpages; 3371 struct vma_iterator vmi; 3372 bool vma_pids_skipped; 3373 bool vma_pids_forced = false; 3374 3375 WARN_ON_ONCE(p != container_of(work, struct task_struct, numa_work)); 3376 3377 work->next = work; 3378 /* 3379 * Who cares about NUMA placement when they're dying. 3380 * 3381 * NOTE: make sure not to dereference p->mm before this check, 3382 * exit_task_work() happens _after_ exit_mm() so we could be called 3383 * without p->mm even though we still had it when we enqueued this 3384 * work. 3385 */ 3386 if (p->flags & PF_EXITING) 3387 return; 3388 3389 /* 3390 * Memory is pinned to only one NUMA node via cpuset.mems, naturally 3391 * no page can be migrated. 3392 */ 3393 if (cpusets_enabled() && nodes_weight(cpuset_current_mems_allowed) == 1) { 3394 trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed); 3395 return; 3396 } 3397 3398 if (!mm->numa_next_scan) { 3399 mm->numa_next_scan = now + 3400 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3401 } 3402 3403 /* 3404 * Enforce maximal scan/migration frequency.. 3405 */ 3406 migrate = mm->numa_next_scan; 3407 if (time_before(now, migrate)) 3408 return; 3409 3410 if (p->numa_scan_period == 0) { 3411 p->numa_scan_period_max = task_scan_max(p); 3412 p->numa_scan_period = task_scan_start(p); 3413 } 3414 3415 next_scan = now + msecs_to_jiffies(p->numa_scan_period); 3416 if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan)) 3417 return; 3418 3419 /* 3420 * Delay this task enough that another task of this mm will likely win 3421 * the next time around. 3422 */ 3423 p->node_stamp += 2 * TICK_NSEC; 3424 3425 pages = sysctl_numa_balancing_scan_size; 3426 pages <<= 20 - PAGE_SHIFT; /* MB in pages */ 3427 virtpages = pages * 8; /* Scan up to this much virtual space */ 3428 if (!pages) 3429 return; 3430 3431 3432 if (!mmap_read_trylock(mm)) 3433 return; 3434 3435 /* 3436 * VMAs are skipped if the current PID has not trapped a fault within 3437 * the VMA recently. Allow scanning to be forced if there is no 3438 * suitable VMA remaining. 3439 */ 3440 vma_pids_skipped = false; 3441 3442 retry_pids: 3443 start = mm->numa_scan_offset; 3444 vma_iter_init(&vmi, mm, start); 3445 vma = vma_next(&vmi); 3446 if (!vma) { 3447 reset_ptenuma_scan(p); 3448 start = 0; 3449 vma_iter_set(&vmi, start); 3450 vma = vma_next(&vmi); 3451 } 3452 3453 for (; vma; vma = vma_next(&vmi)) { 3454 if (!vma_migratable(vma) || !vma_policy_mof(vma) || 3455 is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { 3456 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_UNSUITABLE); 3457 continue; 3458 } 3459 3460 /* 3461 * Shared library pages mapped by multiple processes are not 3462 * migrated as it is expected they are cache replicated. Avoid 3463 * hinting faults in read-only file-backed mappings or the vDSO 3464 * as migrating the pages will be of marginal benefit. 3465 */ 3466 if (!vma->vm_mm || 3467 (vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) { 3468 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SHARED_RO); 3469 continue; 3470 } 3471 3472 /* 3473 * Skip inaccessible VMAs to avoid any confusion between 3474 * PROT_NONE and NUMA hinting PTEs 3475 */ 3476 if (!vma_is_accessible(vma)) { 3477 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_INACCESSIBLE); 3478 continue; 3479 } 3480 3481 /* Initialise new per-VMA NUMAB state. */ 3482 if (!vma->numab_state) { 3483 struct vma_numab_state *ptr; 3484 3485 ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); 3486 if (!ptr) 3487 continue; 3488 3489 if (cmpxchg(&vma->numab_state, NULL, ptr)) { 3490 kfree(ptr); 3491 continue; 3492 } 3493 3494 vma->numab_state->start_scan_seq = mm->numa_scan_seq; 3495 3496 vma->numab_state->next_scan = now + 3497 msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3498 3499 /* Reset happens after 4 times scan delay of scan start */ 3500 vma->numab_state->pids_active_reset = vma->numab_state->next_scan + 3501 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3502 3503 /* 3504 * Ensure prev_scan_seq does not match numa_scan_seq, 3505 * to prevent VMAs being skipped prematurely on the 3506 * first scan: 3507 */ 3508 vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1; 3509 } 3510 3511 /* 3512 * Scanning the VMAs of short lived tasks add more overhead. So 3513 * delay the scan for new VMAs. 3514 */ 3515 if (mm->numa_scan_seq && time_before(jiffies, 3516 vma->numab_state->next_scan)) { 3517 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SCAN_DELAY); 3518 continue; 3519 } 3520 3521 /* RESET access PIDs regularly for old VMAs. */ 3522 if (mm->numa_scan_seq && 3523 time_after(jiffies, vma->numab_state->pids_active_reset)) { 3524 vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset + 3525 msecs_to_jiffies(VMA_PID_RESET_PERIOD); 3526 vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]); 3527 vma->numab_state->pids_active[1] = 0; 3528 } 3529 3530 /* Do not rescan VMAs twice within the same sequence. */ 3531 if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) { 3532 mm->numa_scan_offset = vma->vm_end; 3533 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_SEQ_COMPLETED); 3534 continue; 3535 } 3536 3537 /* 3538 * Do not scan the VMA if task has not accessed it, unless no other 3539 * VMA candidate exists. 3540 */ 3541 if (!vma_pids_forced && !vma_is_accessed(mm, vma)) { 3542 vma_pids_skipped = true; 3543 trace_sched_skip_vma_numa(mm, vma, NUMAB_SKIP_PID_INACTIVE); 3544 continue; 3545 } 3546 3547 do { 3548 start = max(start, vma->vm_start); 3549 end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE); 3550 end = min(end, vma->vm_end); 3551 nr_pte_updates = change_prot_numa(vma, start, end); 3552 3553 /* 3554 * Try to scan sysctl_numa_balancing_size worth of 3555 * hpages that have at least one present PTE that 3556 * is not already PTE-numa. If the VMA contains 3557 * areas that are unused or already full of prot_numa 3558 * PTEs, scan up to virtpages, to skip through those 3559 * areas faster. 3560 */ 3561 if (nr_pte_updates) 3562 pages -= (end - start) >> PAGE_SHIFT; 3563 virtpages -= (end - start) >> PAGE_SHIFT; 3564 3565 start = end; 3566 if (pages <= 0 || virtpages <= 0) 3567 goto out; 3568 3569 cond_resched(); 3570 } while (end != vma->vm_end); 3571 3572 /* VMA scan is complete, do not scan until next sequence. */ 3573 vma->numab_state->prev_scan_seq = mm->numa_scan_seq; 3574 3575 /* 3576 * Only force scan within one VMA at a time, to limit the 3577 * cost of scanning a potentially uninteresting VMA. 3578 */ 3579 if (vma_pids_forced) 3580 break; 3581 } 3582 3583 /* 3584 * If no VMAs are remaining and VMAs were skipped due to the PID 3585 * not accessing the VMA previously, then force a scan to ensure 3586 * forward progress: 3587 */ 3588 if (!vma && !vma_pids_forced && vma_pids_skipped) { 3589 vma_pids_forced = true; 3590 goto retry_pids; 3591 } 3592 3593 out: 3594 /* 3595 * It is possible to reach the end of the VMA list but the last few 3596 * VMAs are not guaranteed to the vma_migratable. If they are not, we 3597 * would find the !migratable VMA on the next scan but not reset the 3598 * scanner to the start so check it now. 3599 */ 3600 if (vma) 3601 mm->numa_scan_offset = start; 3602 else 3603 reset_ptenuma_scan(p); 3604 mmap_read_unlock(mm); 3605 3606 /* 3607 * Make sure tasks use at least 32x as much time to run other code 3608 * than they used here, to limit NUMA PTE scanning overhead to 3% max. 3609 * Usually update_task_scan_period slows down scanning enough; on an 3610 * overloaded system we need to limit overhead on a per task basis. 3611 */ 3612 if (unlikely(p->se.sum_exec_runtime != runtime)) { 3613 u64 diff = p->se.sum_exec_runtime - runtime; 3614 p->node_stamp += 32 * diff; 3615 } 3616 } 3617 3618 void init_numa_balancing(u64 clone_flags, struct task_struct *p) 3619 { 3620 int mm_users = 0; 3621 struct mm_struct *mm = p->mm; 3622 3623 if (mm) { 3624 mm_users = atomic_read(&mm->mm_users); 3625 if (mm_users == 1) { 3626 mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); 3627 mm->numa_scan_seq = 0; 3628 } 3629 } 3630 p->node_stamp = 0; 3631 p->numa_scan_seq = mm ? mm->numa_scan_seq : 0; 3632 p->numa_scan_period = sysctl_numa_balancing_scan_delay; 3633 p->numa_migrate_retry = 0; 3634 /* Protect against double add, see task_tick_numa and task_numa_work */ 3635 p->numa_work.next = &p->numa_work; 3636 p->numa_faults = NULL; 3637 p->numa_pages_migrated = 0; 3638 p->total_numa_faults = 0; 3639 RCU_INIT_POINTER(p->numa_group, NULL); 3640 p->last_task_numa_placement = 0; 3641 p->last_sum_exec_runtime = 0; 3642 3643 init_task_work(&p->numa_work, task_numa_work); 3644 3645 /* New address space, reset the preferred nid */ 3646 if (!(clone_flags & CLONE_VM)) { 3647 p->numa_preferred_nid = NUMA_NO_NODE; 3648 return; 3649 } 3650 3651 /* 3652 * New thread, keep existing numa_preferred_nid which should be copied 3653 * already by arch_dup_task_struct but stagger when scans start. 3654 */ 3655 if (mm) { 3656 unsigned int delay; 3657 3658 delay = min_t(unsigned int, task_scan_max(current), 3659 current->numa_scan_period * mm_users * NSEC_PER_MSEC); 3660 delay += 2 * TICK_NSEC; 3661 p->node_stamp = delay; 3662 } 3663 } 3664 3665 /* 3666 * Drive the periodic memory faults.. 3667 */ 3668 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3669 { 3670 struct callback_head *work = &curr->numa_work; 3671 u64 period, now; 3672 3673 /* 3674 * We don't care about NUMA placement if we don't have memory. 3675 */ 3676 if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work) 3677 return; 3678 3679 /* 3680 * Using runtime rather than walltime has the dual advantage that 3681 * we (mostly) drive the selection from busy threads and that the 3682 * task needs to have done some actual work before we bother with 3683 * NUMA placement. 3684 */ 3685 now = curr->se.sum_exec_runtime; 3686 period = (u64)curr->numa_scan_period * NSEC_PER_MSEC; 3687 3688 if (now > curr->node_stamp + period) { 3689 if (!curr->node_stamp) 3690 curr->numa_scan_period = task_scan_start(curr); 3691 curr->node_stamp += period; 3692 3693 if (!time_before(jiffies, curr->mm->numa_next_scan)) 3694 task_work_add(curr, work, TWA_RESUME); 3695 } 3696 } 3697 3698 static void update_scan_period(struct task_struct *p, int new_cpu) 3699 { 3700 int src_nid = cpu_to_node(task_cpu(p)); 3701 int dst_nid = cpu_to_node(new_cpu); 3702 3703 if (!static_branch_likely(&sched_numa_balancing)) 3704 return; 3705 3706 if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING)) 3707 return; 3708 3709 if (src_nid == dst_nid) 3710 return; 3711 3712 /* 3713 * Allow resets if faults have been trapped before one scan 3714 * has completed. This is most likely due to a new task that 3715 * is pulled cross-node due to wakeups or load balancing. 3716 */ 3717 if (p->numa_scan_seq) { 3718 /* 3719 * Avoid scan adjustments if moving to the preferred 3720 * node or if the task was not previously running on 3721 * the preferred node. 3722 */ 3723 if (dst_nid == p->numa_preferred_nid || 3724 (p->numa_preferred_nid != NUMA_NO_NODE && 3725 src_nid != p->numa_preferred_nid)) 3726 return; 3727 } 3728 3729 p->numa_scan_period = task_scan_start(p); 3730 } 3731 3732 #else /* !CONFIG_NUMA_BALANCING: */ 3733 3734 static void task_tick_numa(struct rq *rq, struct task_struct *curr) 3735 { 3736 } 3737 3738 static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p) 3739 { 3740 } 3741 3742 static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p) 3743 { 3744 } 3745 3746 static inline void update_scan_period(struct task_struct *p, int new_cpu) 3747 { 3748 } 3749 3750 #endif /* !CONFIG_NUMA_BALANCING */ 3751 3752 static void 3753 account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3754 { 3755 update_load_add(&cfs_rq->load, se->load.weight); 3756 if (entity_is_task(se)) { 3757 struct rq *rq = rq_of(cfs_rq); 3758 3759 account_numa_enqueue(rq, task_of(se)); 3760 list_add(&se->group_node, &rq->cfs_tasks); 3761 } 3762 cfs_rq->nr_queued++; 3763 } 3764 3765 static void 3766 account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 3767 { 3768 update_load_sub(&cfs_rq->load, se->load.weight); 3769 if (entity_is_task(se)) { 3770 account_numa_dequeue(rq_of(cfs_rq), task_of(se)); 3771 list_del_init(&se->group_node); 3772 } 3773 cfs_rq->nr_queued--; 3774 } 3775 3776 /* 3777 * Signed add and clamp on underflow. 3778 * 3779 * Explicitly do a load-store to ensure the intermediate value never hits 3780 * memory. This allows lockless observations without ever seeing the negative 3781 * values. 3782 */ 3783 #define add_positive(_ptr, _val) do { \ 3784 typeof(_ptr) ptr = (_ptr); \ 3785 typeof(_val) val = (_val); \ 3786 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3787 \ 3788 res = var + val; \ 3789 \ 3790 if (val < 0 && res > var) \ 3791 res = 0; \ 3792 \ 3793 WRITE_ONCE(*ptr, res); \ 3794 } while (0) 3795 3796 /* 3797 * Unsigned subtract and clamp on underflow. 3798 * 3799 * Explicitly do a load-store to ensure the intermediate value never hits 3800 * memory. This allows lockless observations without ever seeing the negative 3801 * values. 3802 */ 3803 #define sub_positive(_ptr, _val) do { \ 3804 typeof(_ptr) ptr = (_ptr); \ 3805 typeof(*ptr) val = (_val); \ 3806 typeof(*ptr) res, var = READ_ONCE(*ptr); \ 3807 res = var - val; \ 3808 if (res > var) \ 3809 res = 0; \ 3810 WRITE_ONCE(*ptr, res); \ 3811 } while (0) 3812 3813 /* 3814 * Remove and clamp on negative, from a local variable. 3815 * 3816 * A variant of sub_positive(), which does not use explicit load-store 3817 * and is thus optimized for local variable updates. 3818 */ 3819 #define lsub_positive(_ptr, _val) do { \ 3820 typeof(_ptr) ptr = (_ptr); \ 3821 *ptr -= min_t(typeof(*ptr), *ptr, _val); \ 3822 } while (0) 3823 3824 static inline void 3825 enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3826 { 3827 cfs_rq->avg.load_avg += se->avg.load_avg; 3828 cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum; 3829 } 3830 3831 static inline void 3832 dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 3833 { 3834 sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg); 3835 sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum); 3836 /* See update_cfs_rq_load_avg() */ 3837 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 3838 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 3839 } 3840 3841 static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags); 3842 3843 static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, 3844 unsigned long weight) 3845 { 3846 bool curr = cfs_rq->curr == se; 3847 bool rel_vprot = false; 3848 u64 vprot; 3849 3850 if (se->on_rq) { 3851 /* commit outstanding execution time */ 3852 update_curr(cfs_rq); 3853 update_entity_lag(cfs_rq, se); 3854 se->deadline -= se->vruntime; 3855 se->rel_deadline = 1; 3856 if (curr && protect_slice(se)) { 3857 vprot = se->vprot - se->vruntime; 3858 rel_vprot = true; 3859 } 3860 3861 cfs_rq->nr_queued--; 3862 if (!curr) 3863 __dequeue_entity(cfs_rq, se); 3864 update_load_sub(&cfs_rq->load, se->load.weight); 3865 } 3866 dequeue_load_avg(cfs_rq, se); 3867 3868 /* 3869 * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), 3870 * we need to scale se->vlag when w_i changes. 3871 */ 3872 se->vlag = div_s64(se->vlag * se->load.weight, weight); 3873 if (se->rel_deadline) 3874 se->deadline = div_s64(se->deadline * se->load.weight, weight); 3875 3876 if (rel_vprot) 3877 vprot = div_s64(vprot * se->load.weight, weight); 3878 3879 update_load_set(&se->load, weight); 3880 3881 do { 3882 u32 divider = get_pelt_divider(&se->avg); 3883 3884 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); 3885 } while (0); 3886 3887 enqueue_load_avg(cfs_rq, se); 3888 if (se->on_rq) { 3889 place_entity(cfs_rq, se, 0); 3890 if (rel_vprot) 3891 se->vprot = se->vruntime + vprot; 3892 update_load_add(&cfs_rq->load, se->load.weight); 3893 if (!curr) 3894 __enqueue_entity(cfs_rq, se); 3895 cfs_rq->nr_queued++; 3896 } 3897 } 3898 3899 static void reweight_task_fair(struct rq *rq, struct task_struct *p, 3900 const struct load_weight *lw) 3901 { 3902 struct sched_entity *se = &p->se; 3903 struct cfs_rq *cfs_rq = cfs_rq_of(se); 3904 struct load_weight *load = &se->load; 3905 3906 reweight_entity(cfs_rq, se, lw->weight); 3907 load->inv_weight = lw->inv_weight; 3908 } 3909 3910 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); 3911 3912 #ifdef CONFIG_FAIR_GROUP_SCHED 3913 /* 3914 * All this does is approximate the hierarchical proportion which includes that 3915 * global sum we all love to hate. 3916 * 3917 * That is, the weight of a group entity, is the proportional share of the 3918 * group weight based on the group runqueue weights. That is: 3919 * 3920 * tg->weight * grq->load.weight 3921 * ge->load.weight = ----------------------------- (1) 3922 * \Sum grq->load.weight 3923 * 3924 * Now, because computing that sum is prohibitively expensive to compute (been 3925 * there, done that) we approximate it with this average stuff. The average 3926 * moves slower and therefore the approximation is cheaper and more stable. 3927 * 3928 * So instead of the above, we substitute: 3929 * 3930 * grq->load.weight -> grq->avg.load_avg (2) 3931 * 3932 * which yields the following: 3933 * 3934 * tg->weight * grq->avg.load_avg 3935 * ge->load.weight = ------------------------------ (3) 3936 * tg->load_avg 3937 * 3938 * Where: tg->load_avg ~= \Sum grq->avg.load_avg 3939 * 3940 * That is shares_avg, and it is right (given the approximation (2)). 3941 * 3942 * The problem with it is that because the average is slow -- it was designed 3943 * to be exactly that of course -- this leads to transients in boundary 3944 * conditions. In specific, the case where the group was idle and we start the 3945 * one task. It takes time for our CPU's grq->avg.load_avg to build up, 3946 * yielding bad latency etc.. 3947 * 3948 * Now, in that special case (1) reduces to: 3949 * 3950 * tg->weight * grq->load.weight 3951 * ge->load.weight = ----------------------------- = tg->weight (4) 3952 * grp->load.weight 3953 * 3954 * That is, the sum collapses because all other CPUs are idle; the UP scenario. 3955 * 3956 * So what we do is modify our approximation (3) to approach (4) in the (near) 3957 * UP case, like: 3958 * 3959 * ge->load.weight = 3960 * 3961 * tg->weight * grq->load.weight 3962 * --------------------------------------------------- (5) 3963 * tg->load_avg - grq->avg.load_avg + grq->load.weight 3964 * 3965 * But because grq->load.weight can drop to 0, resulting in a divide by zero, 3966 * we need to use grq->avg.load_avg as its lower bound, which then gives: 3967 * 3968 * 3969 * tg->weight * grq->load.weight 3970 * ge->load.weight = ----------------------------- (6) 3971 * tg_load_avg' 3972 * 3973 * Where: 3974 * 3975 * tg_load_avg' = tg->load_avg - grq->avg.load_avg + 3976 * max(grq->load.weight, grq->avg.load_avg) 3977 * 3978 * And that is shares_weight and is icky. In the (near) UP case it approaches 3979 * (4) while in the normal case it approaches (3). It consistently 3980 * overestimates the ge->load.weight and therefore: 3981 * 3982 * \Sum ge->load.weight >= tg->weight 3983 * 3984 * hence icky! 3985 */ 3986 static long calc_group_shares(struct cfs_rq *cfs_rq) 3987 { 3988 long tg_weight, tg_shares, load, shares; 3989 struct task_group *tg = cfs_rq->tg; 3990 3991 tg_shares = READ_ONCE(tg->shares); 3992 3993 load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); 3994 3995 tg_weight = atomic_long_read(&tg->load_avg); 3996 3997 /* Ensure tg_weight >= load */ 3998 tg_weight -= cfs_rq->tg_load_avg_contrib; 3999 tg_weight += load; 4000 4001 shares = (tg_shares * load); 4002 if (tg_weight) 4003 shares /= tg_weight; 4004 4005 /* 4006 * MIN_SHARES has to be unscaled here to support per-CPU partitioning 4007 * of a group with small tg->shares value. It is a floor value which is 4008 * assigned as a minimum load.weight to the sched_entity representing 4009 * the group on a CPU. 4010 * 4011 * E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024 4012 * on an 8-core system with 8 tasks each runnable on one CPU shares has 4013 * to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In 4014 * case no task is runnable on a CPU MIN_SHARES=2 should be returned 4015 * instead of 0. 4016 */ 4017 return clamp_t(long, shares, MIN_SHARES, tg_shares); 4018 } 4019 4020 /* 4021 * Recomputes the group entity based on the current state of its group 4022 * runqueue. 4023 */ 4024 static void update_cfs_group(struct sched_entity *se) 4025 { 4026 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4027 long shares; 4028 4029 /* 4030 * When a group becomes empty, preserve its weight. This matters for 4031 * DELAY_DEQUEUE. 4032 */ 4033 if (!gcfs_rq || !gcfs_rq->load.weight) 4034 return; 4035 4036 shares = calc_group_shares(gcfs_rq); 4037 if (unlikely(se->load.weight != shares)) 4038 reweight_entity(cfs_rq_of(se), se, shares); 4039 } 4040 4041 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 4042 static inline void update_cfs_group(struct sched_entity *se) 4043 { 4044 } 4045 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 4046 4047 static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags) 4048 { 4049 struct rq *rq = rq_of(cfs_rq); 4050 4051 if (&rq->cfs == cfs_rq) { 4052 /* 4053 * There are a few boundary cases this might miss but it should 4054 * get called often enough that that should (hopefully) not be 4055 * a real problem. 4056 * 4057 * It will not get called when we go idle, because the idle 4058 * thread is a different class (!fair), nor will the utilization 4059 * number include things like RT tasks. 4060 * 4061 * As is, the util number is not freq-invariant (we'd have to 4062 * implement arch_scale_freq_capacity() for that). 4063 * 4064 * See cpu_util_cfs(). 4065 */ 4066 cpufreq_update_util(rq, flags); 4067 } 4068 } 4069 4070 static inline bool load_avg_is_decayed(struct sched_avg *sa) 4071 { 4072 if (sa->load_sum) 4073 return false; 4074 4075 if (sa->util_sum) 4076 return false; 4077 4078 if (sa->runnable_sum) 4079 return false; 4080 4081 /* 4082 * _avg must be null when _sum are null because _avg = _sum / divider 4083 * Make sure that rounding and/or propagation of PELT values never 4084 * break this. 4085 */ 4086 WARN_ON_ONCE(sa->load_avg || 4087 sa->util_avg || 4088 sa->runnable_avg); 4089 4090 return true; 4091 } 4092 4093 static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) 4094 { 4095 return u64_u32_load_copy(cfs_rq->avg.last_update_time, 4096 cfs_rq->last_update_time_copy); 4097 } 4098 #ifdef CONFIG_FAIR_GROUP_SCHED 4099 /* 4100 * Because list_add_leaf_cfs_rq always places a child cfs_rq on the list 4101 * immediately before a parent cfs_rq, and cfs_rqs are removed from the list 4102 * bottom-up, we only have to test whether the cfs_rq before us on the list 4103 * is our child. 4104 * If cfs_rq is not on the list, test whether a child needs its to be added to 4105 * connect a branch to the tree * (see list_add_leaf_cfs_rq() for details). 4106 */ 4107 static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq) 4108 { 4109 struct cfs_rq *prev_cfs_rq; 4110 struct list_head *prev; 4111 struct rq *rq = rq_of(cfs_rq); 4112 4113 if (cfs_rq->on_list) { 4114 prev = cfs_rq->leaf_cfs_rq_list.prev; 4115 } else { 4116 prev = rq->tmp_alone_branch; 4117 } 4118 4119 if (prev == &rq->leaf_cfs_rq_list) 4120 return false; 4121 4122 prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list); 4123 4124 return (prev_cfs_rq->tg->parent == cfs_rq->tg); 4125 } 4126 4127 static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) 4128 { 4129 if (cfs_rq->load.weight) 4130 return false; 4131 4132 if (!load_avg_is_decayed(&cfs_rq->avg)) 4133 return false; 4134 4135 if (child_cfs_rq_on_list(cfs_rq)) 4136 return false; 4137 4138 if (cfs_rq->tg_load_avg_contrib) 4139 return false; 4140 4141 return true; 4142 } 4143 4144 /** 4145 * update_tg_load_avg - update the tg's load avg 4146 * @cfs_rq: the cfs_rq whose avg changed 4147 * 4148 * This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load. 4149 * However, because tg->load_avg is a global value there are performance 4150 * considerations. 4151 * 4152 * In order to avoid having to look at the other cfs_rq's, we use a 4153 * differential update where we store the last value we propagated. This in 4154 * turn allows skipping updates if the differential is 'small'. 4155 * 4156 * Updating tg's load_avg is necessary before update_cfs_share(). 4157 */ 4158 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) 4159 { 4160 long delta; 4161 u64 now; 4162 4163 /* 4164 * No need to update load_avg for root_task_group as it is not used. 4165 */ 4166 if (cfs_rq->tg == &root_task_group) 4167 return; 4168 4169 /* rq has been offline and doesn't contribute to the share anymore: */ 4170 if (!cpu_active(cpu_of(rq_of(cfs_rq)))) 4171 return; 4172 4173 /* 4174 * For migration heavy workloads, access to tg->load_avg can be 4175 * unbound. Limit the update rate to at most once per ms. 4176 */ 4177 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4178 if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) 4179 return; 4180 4181 delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; 4182 if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { 4183 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4184 cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; 4185 cfs_rq->last_update_tg_load_avg = now; 4186 } 4187 } 4188 4189 static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) 4190 { 4191 long delta; 4192 u64 now; 4193 4194 /* 4195 * No need to update load_avg for root_task_group, as it is not used. 4196 */ 4197 if (cfs_rq->tg == &root_task_group) 4198 return; 4199 4200 now = sched_clock_cpu(cpu_of(rq_of(cfs_rq))); 4201 delta = 0 - cfs_rq->tg_load_avg_contrib; 4202 atomic_long_add(delta, &cfs_rq->tg->load_avg); 4203 cfs_rq->tg_load_avg_contrib = 0; 4204 cfs_rq->last_update_tg_load_avg = now; 4205 } 4206 4207 /* CPU offline callback: */ 4208 static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq) 4209 { 4210 struct task_group *tg; 4211 4212 lockdep_assert_rq_held(rq); 4213 4214 /* 4215 * The rq clock has already been updated in 4216 * set_rq_offline(), so we should skip updating 4217 * the rq clock again in unthrottle_cfs_rq(). 4218 */ 4219 rq_clock_start_loop_update(rq); 4220 4221 rcu_read_lock(); 4222 list_for_each_entry_rcu(tg, &task_groups, list) { 4223 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 4224 4225 clear_tg_load_avg(cfs_rq); 4226 } 4227 rcu_read_unlock(); 4228 4229 rq_clock_stop_loop_update(rq); 4230 } 4231 4232 /* 4233 * Called within set_task_rq() right before setting a task's CPU. The 4234 * caller only guarantees p->pi_lock is held; no other assumptions, 4235 * including the state of rq->lock, should be made. 4236 */ 4237 void set_task_rq_fair(struct sched_entity *se, 4238 struct cfs_rq *prev, struct cfs_rq *next) 4239 { 4240 u64 p_last_update_time; 4241 u64 n_last_update_time; 4242 4243 if (!sched_feat(ATTACH_AGE_LOAD)) 4244 return; 4245 4246 /* 4247 * We are supposed to update the task to "current" time, then its up to 4248 * date and ready to go to new CPU/cfs_rq. But we have difficulty in 4249 * getting what current time is, so simply throw away the out-of-date 4250 * time. This will result in the wakee task is less decayed, but giving 4251 * the wakee more load sounds not bad. 4252 */ 4253 if (!(se->avg.last_update_time && prev)) 4254 return; 4255 4256 p_last_update_time = cfs_rq_last_update_time(prev); 4257 n_last_update_time = cfs_rq_last_update_time(next); 4258 4259 __update_load_avg_blocked_se(p_last_update_time, se); 4260 se->avg.last_update_time = n_last_update_time; 4261 } 4262 4263 /* 4264 * When on migration a sched_entity joins/leaves the PELT hierarchy, we need to 4265 * propagate its contribution. The key to this propagation is the invariant 4266 * that for each group: 4267 * 4268 * ge->avg == grq->avg (1) 4269 * 4270 * _IFF_ we look at the pure running and runnable sums. Because they 4271 * represent the very same entity, just at different points in the hierarchy. 4272 * 4273 * Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial 4274 * and simply copies the running/runnable sum over (but still wrong, because 4275 * the group entity and group rq do not have their PELT windows aligned). 4276 * 4277 * However, update_tg_cfs_load() is more complex. So we have: 4278 * 4279 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2) 4280 * 4281 * And since, like util, the runnable part should be directly transferable, 4282 * the following would _appear_ to be the straight forward approach: 4283 * 4284 * grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3) 4285 * 4286 * And per (1) we have: 4287 * 4288 * ge->avg.runnable_avg == grq->avg.runnable_avg 4289 * 4290 * Which gives: 4291 * 4292 * ge->load.weight * grq->avg.load_avg 4293 * ge->avg.load_avg = ----------------------------------- (4) 4294 * grq->load.weight 4295 * 4296 * Except that is wrong! 4297 * 4298 * Because while for entities historical weight is not important and we 4299 * really only care about our future and therefore can consider a pure 4300 * runnable sum, runqueues can NOT do this. 4301 * 4302 * We specifically want runqueues to have a load_avg that includes 4303 * historical weights. Those represent the blocked load, the load we expect 4304 * to (shortly) return to us. This only works by keeping the weights as 4305 * integral part of the sum. We therefore cannot decompose as per (3). 4306 * 4307 * Another reason this doesn't work is that runnable isn't a 0-sum entity. 4308 * Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the 4309 * rq itself is runnable anywhere between 2/3 and 1 depending on how the 4310 * runnable section of these tasks overlap (or not). If they were to perfectly 4311 * align the rq as a whole would be runnable 2/3 of the time. If however we 4312 * always have at least 1 runnable task, the rq as a whole is always runnable. 4313 * 4314 * So we'll have to approximate.. :/ 4315 * 4316 * Given the constraint: 4317 * 4318 * ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX 4319 * 4320 * We can construct a rule that adds runnable to a rq by assuming minimal 4321 * overlap. 4322 * 4323 * On removal, we'll assume each task is equally runnable; which yields: 4324 * 4325 * grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight 4326 * 4327 * XXX: only do this for the part of runnable > running ? 4328 * 4329 */ 4330 static inline void 4331 update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4332 { 4333 long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg; 4334 u32 new_sum, divider; 4335 4336 /* Nothing to update */ 4337 if (!delta_avg) 4338 return; 4339 4340 /* 4341 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4342 * See ___update_load_avg() for details. 4343 */ 4344 divider = get_pelt_divider(&cfs_rq->avg); 4345 4346 4347 /* Set new sched_entity's utilization */ 4348 se->avg.util_avg = gcfs_rq->avg.util_avg; 4349 new_sum = se->avg.util_avg * divider; 4350 delta_sum = (long)new_sum - (long)se->avg.util_sum; 4351 se->avg.util_sum = new_sum; 4352 4353 /* Update parent cfs_rq utilization */ 4354 add_positive(&cfs_rq->avg.util_avg, delta_avg); 4355 add_positive(&cfs_rq->avg.util_sum, delta_sum); 4356 4357 /* See update_cfs_rq_load_avg() */ 4358 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4359 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4360 } 4361 4362 static inline void 4363 update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4364 { 4365 long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; 4366 u32 new_sum, divider; 4367 4368 /* Nothing to update */ 4369 if (!delta_avg) 4370 return; 4371 4372 /* 4373 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4374 * See ___update_load_avg() for details. 4375 */ 4376 divider = get_pelt_divider(&cfs_rq->avg); 4377 4378 /* Set new sched_entity's runnable */ 4379 se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; 4380 new_sum = se->avg.runnable_avg * divider; 4381 delta_sum = (long)new_sum - (long)se->avg.runnable_sum; 4382 se->avg.runnable_sum = new_sum; 4383 4384 /* Update parent cfs_rq runnable */ 4385 add_positive(&cfs_rq->avg.runnable_avg, delta_avg); 4386 add_positive(&cfs_rq->avg.runnable_sum, delta_sum); 4387 /* See update_cfs_rq_load_avg() */ 4388 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4389 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4390 } 4391 4392 static inline void 4393 update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) 4394 { 4395 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum; 4396 unsigned long load_avg; 4397 u64 load_sum = 0; 4398 s64 delta_sum; 4399 u32 divider; 4400 4401 if (!runnable_sum) 4402 return; 4403 4404 gcfs_rq->prop_runnable_sum = 0; 4405 4406 /* 4407 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4408 * See ___update_load_avg() for details. 4409 */ 4410 divider = get_pelt_divider(&cfs_rq->avg); 4411 4412 if (runnable_sum >= 0) { 4413 /* 4414 * Add runnable; clip at LOAD_AVG_MAX. Reflects that until 4415 * the CPU is saturated running == runnable. 4416 */ 4417 runnable_sum += se->avg.load_sum; 4418 runnable_sum = min_t(long, runnable_sum, divider); 4419 } else { 4420 /* 4421 * Estimate the new unweighted runnable_sum of the gcfs_rq by 4422 * assuming all tasks are equally runnable. 4423 */ 4424 if (scale_load_down(gcfs_rq->load.weight)) { 4425 load_sum = div_u64(gcfs_rq->avg.load_sum, 4426 scale_load_down(gcfs_rq->load.weight)); 4427 } 4428 4429 /* But make sure to not inflate se's runnable */ 4430 runnable_sum = min(se->avg.load_sum, load_sum); 4431 } 4432 4433 /* 4434 * runnable_sum can't be lower than running_sum 4435 * Rescale running sum to be in the same range as runnable sum 4436 * running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT] 4437 * runnable_sum is in [0 : LOAD_AVG_MAX] 4438 */ 4439 running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT; 4440 runnable_sum = max(runnable_sum, running_sum); 4441 4442 load_sum = se_weight(se) * runnable_sum; 4443 load_avg = div_u64(load_sum, divider); 4444 4445 delta_avg = load_avg - se->avg.load_avg; 4446 if (!delta_avg) 4447 return; 4448 4449 delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; 4450 4451 se->avg.load_sum = runnable_sum; 4452 se->avg.load_avg = load_avg; 4453 add_positive(&cfs_rq->avg.load_avg, delta_avg); 4454 add_positive(&cfs_rq->avg.load_sum, delta_sum); 4455 /* See update_cfs_rq_load_avg() */ 4456 cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum, 4457 cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); 4458 } 4459 4460 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) 4461 { 4462 cfs_rq->propagate = 1; 4463 cfs_rq->prop_runnable_sum += runnable_sum; 4464 } 4465 4466 /* Update task and its cfs_rq load average */ 4467 static inline int propagate_entity_load_avg(struct sched_entity *se) 4468 { 4469 struct cfs_rq *cfs_rq, *gcfs_rq; 4470 4471 if (entity_is_task(se)) 4472 return 0; 4473 4474 gcfs_rq = group_cfs_rq(se); 4475 if (!gcfs_rq->propagate) 4476 return 0; 4477 4478 gcfs_rq->propagate = 0; 4479 4480 cfs_rq = cfs_rq_of(se); 4481 4482 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum); 4483 4484 update_tg_cfs_util(cfs_rq, se, gcfs_rq); 4485 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq); 4486 update_tg_cfs_load(cfs_rq, se, gcfs_rq); 4487 4488 trace_pelt_cfs_tp(cfs_rq); 4489 trace_pelt_se_tp(se); 4490 4491 return 1; 4492 } 4493 4494 /* 4495 * Check if we need to update the load and the utilization of a blocked 4496 * group_entity: 4497 */ 4498 static inline bool skip_blocked_update(struct sched_entity *se) 4499 { 4500 struct cfs_rq *gcfs_rq = group_cfs_rq(se); 4501 4502 /* 4503 * If sched_entity still have not zero load or utilization, we have to 4504 * decay it: 4505 */ 4506 if (se->avg.load_avg || se->avg.util_avg) 4507 return false; 4508 4509 /* 4510 * If there is a pending propagation, we have to update the load and 4511 * the utilization of the sched_entity: 4512 */ 4513 if (gcfs_rq->propagate) 4514 return false; 4515 4516 /* 4517 * Otherwise, the load and the utilization of the sched_entity is 4518 * already zero and there is no pending propagation, so it will be a 4519 * waste of time to try to decay it: 4520 */ 4521 return true; 4522 } 4523 4524 #else /* !CONFIG_FAIR_GROUP_SCHED: */ 4525 4526 static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {} 4527 4528 static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {} 4529 4530 static inline int propagate_entity_load_avg(struct sched_entity *se) 4531 { 4532 return 0; 4533 } 4534 4535 static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {} 4536 4537 #endif /* !CONFIG_FAIR_GROUP_SCHED */ 4538 4539 #ifdef CONFIG_NO_HZ_COMMON 4540 static inline void migrate_se_pelt_lag(struct sched_entity *se) 4541 { 4542 u64 throttled = 0, now, lut; 4543 struct cfs_rq *cfs_rq; 4544 struct rq *rq; 4545 bool is_idle; 4546 4547 if (load_avg_is_decayed(&se->avg)) 4548 return; 4549 4550 cfs_rq = cfs_rq_of(se); 4551 rq = rq_of(cfs_rq); 4552 4553 rcu_read_lock(); 4554 is_idle = is_idle_task(rcu_dereference(rq->curr)); 4555 rcu_read_unlock(); 4556 4557 /* 4558 * The lag estimation comes with a cost we don't want to pay all the 4559 * time. Hence, limiting to the case where the source CPU is idle and 4560 * we know we are at the greatest risk to have an outdated clock. 4561 */ 4562 if (!is_idle) 4563 return; 4564 4565 /* 4566 * Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where: 4567 * 4568 * last_update_time (the cfs_rq's last_update_time) 4569 * = cfs_rq_clock_pelt()@cfs_rq_idle 4570 * = rq_clock_pelt()@cfs_rq_idle 4571 * - cfs->throttled_clock_pelt_time@cfs_rq_idle 4572 * 4573 * cfs_idle_lag (delta between rq's update and cfs_rq's update) 4574 * = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle 4575 * 4576 * rq_idle_lag (delta between now and rq's update) 4577 * = sched_clock_cpu() - rq_clock()@rq_idle 4578 * 4579 * We can then write: 4580 * 4581 * now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time + 4582 * sched_clock_cpu() - rq_clock()@rq_idle 4583 * Where: 4584 * rq_clock_pelt()@rq_idle is rq->clock_pelt_idle 4585 * rq_clock()@rq_idle is rq->clock_idle 4586 * cfs->throttled_clock_pelt_time@cfs_rq_idle 4587 * is cfs_rq->throttled_pelt_idle 4588 */ 4589 4590 #ifdef CONFIG_CFS_BANDWIDTH 4591 throttled = u64_u32_load(cfs_rq->throttled_pelt_idle); 4592 /* The clock has been stopped for throttling */ 4593 if (throttled == U64_MAX) 4594 return; 4595 #endif 4596 now = u64_u32_load(rq->clock_pelt_idle); 4597 /* 4598 * Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case 4599 * is observed the old clock_pelt_idle value and the new clock_idle, 4600 * which lead to an underestimation. The opposite would lead to an 4601 * overestimation. 4602 */ 4603 smp_rmb(); 4604 lut = cfs_rq_last_update_time(cfs_rq); 4605 4606 now -= throttled; 4607 if (now < lut) 4608 /* 4609 * cfs_rq->avg.last_update_time is more recent than our 4610 * estimation, let's use it. 4611 */ 4612 now = lut; 4613 else 4614 now += sched_clock_cpu(cpu_of(rq)) - u64_u32_load(rq->clock_idle); 4615 4616 __update_load_avg_blocked_se(now, se); 4617 } 4618 #else /* !CONFIG_NO_HZ_COMMON: */ 4619 static void migrate_se_pelt_lag(struct sched_entity *se) {} 4620 #endif /* !CONFIG_NO_HZ_COMMON */ 4621 4622 /** 4623 * update_cfs_rq_load_avg - update the cfs_rq's load/util averages 4624 * @now: current time, as per cfs_rq_clock_pelt() 4625 * @cfs_rq: cfs_rq to update 4626 * 4627 * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) 4628 * avg. The immediate corollary is that all (fair) tasks must be attached. 4629 * 4630 * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. 4631 * 4632 * Return: true if the load decayed or we removed load. 4633 * 4634 * Since both these conditions indicate a changed cfs_rq->avg.load we should 4635 * call update_tg_load_avg() when this function returns true. 4636 */ 4637 static inline int 4638 update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) 4639 { 4640 unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0; 4641 struct sched_avg *sa = &cfs_rq->avg; 4642 int decayed = 0; 4643 4644 if (cfs_rq->removed.nr) { 4645 unsigned long r; 4646 u32 divider = get_pelt_divider(&cfs_rq->avg); 4647 4648 raw_spin_lock(&cfs_rq->removed.lock); 4649 swap(cfs_rq->removed.util_avg, removed_util); 4650 swap(cfs_rq->removed.load_avg, removed_load); 4651 swap(cfs_rq->removed.runnable_avg, removed_runnable); 4652 cfs_rq->removed.nr = 0; 4653 raw_spin_unlock(&cfs_rq->removed.lock); 4654 4655 r = removed_load; 4656 sub_positive(&sa->load_avg, r); 4657 sub_positive(&sa->load_sum, r * divider); 4658 /* See sa->util_sum below */ 4659 sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER); 4660 4661 r = removed_util; 4662 sub_positive(&sa->util_avg, r); 4663 sub_positive(&sa->util_sum, r * divider); 4664 /* 4665 * Because of rounding, se->util_sum might ends up being +1 more than 4666 * cfs->util_sum. Although this is not a problem by itself, detaching 4667 * a lot of tasks with the rounding problem between 2 updates of 4668 * util_avg (~1ms) can make cfs->util_sum becoming null whereas 4669 * cfs_util_avg is not. 4670 * Check that util_sum is still above its lower bound for the new 4671 * util_avg. Given that period_contrib might have moved since the last 4672 * sync, we are only sure that util_sum must be above or equal to 4673 * util_avg * minimum possible divider 4674 */ 4675 sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER); 4676 4677 r = removed_runnable; 4678 sub_positive(&sa->runnable_avg, r); 4679 sub_positive(&sa->runnable_sum, r * divider); 4680 /* See sa->util_sum above */ 4681 sa->runnable_sum = max_t(u32, sa->runnable_sum, 4682 sa->runnable_avg * PELT_MIN_DIVIDER); 4683 4684 /* 4685 * removed_runnable is the unweighted version of removed_load so we 4686 * can use it to estimate removed_load_sum. 4687 */ 4688 add_tg_cfs_propagate(cfs_rq, 4689 -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT); 4690 4691 decayed = 1; 4692 } 4693 4694 decayed |= __update_load_avg_cfs_rq(now, cfs_rq); 4695 u64_u32_store_copy(sa->last_update_time, 4696 cfs_rq->last_update_time_copy, 4697 sa->last_update_time); 4698 return decayed; 4699 } 4700 4701 /** 4702 * attach_entity_load_avg - attach this entity to its cfs_rq load avg 4703 * @cfs_rq: cfs_rq to attach to 4704 * @se: sched_entity to attach 4705 * 4706 * Must call update_cfs_rq_load_avg() before this, since we rely on 4707 * cfs_rq->avg.last_update_time being current. 4708 */ 4709 static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4710 { 4711 /* 4712 * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. 4713 * See ___update_load_avg() for details. 4714 */ 4715 u32 divider = get_pelt_divider(&cfs_rq->avg); 4716 4717 /* 4718 * When we attach the @se to the @cfs_rq, we must align the decay 4719 * window because without that, really weird and wonderful things can 4720 * happen. 4721 * 4722 * XXX illustrate 4723 */ 4724 se->avg.last_update_time = cfs_rq->avg.last_update_time; 4725 se->avg.period_contrib = cfs_rq->avg.period_contrib; 4726 4727 /* 4728 * Hell(o) Nasty stuff.. we need to recompute _sum based on the new 4729 * period_contrib. This isn't strictly correct, but since we're 4730 * entirely outside of the PELT hierarchy, nobody cares if we truncate 4731 * _sum a little. 4732 */ 4733 se->avg.util_sum = se->avg.util_avg * divider; 4734 4735 se->avg.runnable_sum = se->avg.runnable_avg * divider; 4736 4737 se->avg.load_sum = se->avg.load_avg * divider; 4738 if (se_weight(se) < se->avg.load_sum) 4739 se->avg.load_sum = div_u64(se->avg.load_sum, se_weight(se)); 4740 else 4741 se->avg.load_sum = 1; 4742 4743 enqueue_load_avg(cfs_rq, se); 4744 cfs_rq->avg.util_avg += se->avg.util_avg; 4745 cfs_rq->avg.util_sum += se->avg.util_sum; 4746 cfs_rq->avg.runnable_avg += se->avg.runnable_avg; 4747 cfs_rq->avg.runnable_sum += se->avg.runnable_sum; 4748 4749 add_tg_cfs_propagate(cfs_rq, se->avg.load_sum); 4750 4751 cfs_rq_util_change(cfs_rq, 0); 4752 4753 trace_pelt_cfs_tp(cfs_rq); 4754 } 4755 4756 /** 4757 * detach_entity_load_avg - detach this entity from its cfs_rq load avg 4758 * @cfs_rq: cfs_rq to detach from 4759 * @se: sched_entity to detach 4760 * 4761 * Must call update_cfs_rq_load_avg() before this, since we rely on 4762 * cfs_rq->avg.last_update_time being current. 4763 */ 4764 static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) 4765 { 4766 dequeue_load_avg(cfs_rq, se); 4767 sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg); 4768 sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum); 4769 /* See update_cfs_rq_load_avg() */ 4770 cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum, 4771 cfs_rq->avg.util_avg * PELT_MIN_DIVIDER); 4772 4773 sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg); 4774 sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum); 4775 /* See update_cfs_rq_load_avg() */ 4776 cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum, 4777 cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER); 4778 4779 add_tg_cfs_propagate(cfs_rq, -se->avg.load_sum); 4780 4781 cfs_rq_util_change(cfs_rq, 0); 4782 4783 trace_pelt_cfs_tp(cfs_rq); 4784 } 4785 4786 /* 4787 * Optional action to be done while updating the load average 4788 */ 4789 #define UPDATE_TG 0x1 4790 #define SKIP_AGE_LOAD 0x2 4791 #define DO_ATTACH 0x4 4792 #define DO_DETACH 0x8 4793 4794 /* Update task and its cfs_rq load average */ 4795 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 4796 { 4797 u64 now = cfs_rq_clock_pelt(cfs_rq); 4798 int decayed; 4799 4800 /* 4801 * Track task load average for carrying it to new CPU after migrated, and 4802 * track group sched_entity load average for task_h_load calculation in migration 4803 */ 4804 if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD)) 4805 __update_load_avg_se(now, cfs_rq, se); 4806 4807 decayed = update_cfs_rq_load_avg(now, cfs_rq); 4808 decayed |= propagate_entity_load_avg(se); 4809 4810 if (!se->avg.last_update_time && (flags & DO_ATTACH)) { 4811 4812 /* 4813 * DO_ATTACH means we're here from enqueue_entity(). 4814 * !last_update_time means we've passed through 4815 * migrate_task_rq_fair() indicating we migrated. 4816 * 4817 * IOW we're enqueueing a task on a new CPU. 4818 */ 4819 attach_entity_load_avg(cfs_rq, se); 4820 update_tg_load_avg(cfs_rq); 4821 4822 } else if (flags & DO_DETACH) { 4823 /* 4824 * DO_DETACH means we're here from dequeue_entity() 4825 * and we are migrating task out of the CPU. 4826 */ 4827 detach_entity_load_avg(cfs_rq, se); 4828 update_tg_load_avg(cfs_rq); 4829 } else if (decayed) { 4830 cfs_rq_util_change(cfs_rq, 0); 4831 4832 if (flags & UPDATE_TG) 4833 update_tg_load_avg(cfs_rq); 4834 } 4835 } 4836 4837 /* 4838 * Synchronize entity load avg of dequeued entity without locking 4839 * the previous rq. 4840 */ 4841 static void sync_entity_load_avg(struct sched_entity *se) 4842 { 4843 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4844 u64 last_update_time; 4845 4846 last_update_time = cfs_rq_last_update_time(cfs_rq); 4847 __update_load_avg_blocked_se(last_update_time, se); 4848 } 4849 4850 /* 4851 * Task first catches up with cfs_rq, and then subtract 4852 * itself from the cfs_rq (task must be off the queue now). 4853 */ 4854 static void remove_entity_load_avg(struct sched_entity *se) 4855 { 4856 struct cfs_rq *cfs_rq = cfs_rq_of(se); 4857 unsigned long flags; 4858 4859 /* 4860 * tasks cannot exit without having gone through wake_up_new_task() -> 4861 * enqueue_task_fair() which will have added things to the cfs_rq, 4862 * so we can remove unconditionally. 4863 */ 4864 4865 sync_entity_load_avg(se); 4866 4867 raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags); 4868 ++cfs_rq->removed.nr; 4869 cfs_rq->removed.util_avg += se->avg.util_avg; 4870 cfs_rq->removed.load_avg += se->avg.load_avg; 4871 cfs_rq->removed.runnable_avg += se->avg.runnable_avg; 4872 raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags); 4873 } 4874 4875 static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq) 4876 { 4877 return cfs_rq->avg.runnable_avg; 4878 } 4879 4880 static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) 4881 { 4882 return cfs_rq->avg.load_avg; 4883 } 4884 4885 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf); 4886 4887 static inline unsigned long task_util(struct task_struct *p) 4888 { 4889 return READ_ONCE(p->se.avg.util_avg); 4890 } 4891 4892 static inline unsigned long task_runnable(struct task_struct *p) 4893 { 4894 return READ_ONCE(p->se.avg.runnable_avg); 4895 } 4896 4897 static inline unsigned long _task_util_est(struct task_struct *p) 4898 { 4899 return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED; 4900 } 4901 4902 static inline unsigned long task_util_est(struct task_struct *p) 4903 { 4904 return max(task_util(p), _task_util_est(p)); 4905 } 4906 4907 static inline void util_est_enqueue(struct cfs_rq *cfs_rq, 4908 struct task_struct *p) 4909 { 4910 unsigned int enqueued; 4911 4912 if (!sched_feat(UTIL_EST)) 4913 return; 4914 4915 /* Update root cfs_rq's estimated utilization */ 4916 enqueued = cfs_rq->avg.util_est; 4917 enqueued += _task_util_est(p); 4918 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4919 4920 trace_sched_util_est_cfs_tp(cfs_rq); 4921 } 4922 4923 static inline void util_est_dequeue(struct cfs_rq *cfs_rq, 4924 struct task_struct *p) 4925 { 4926 unsigned int enqueued; 4927 4928 if (!sched_feat(UTIL_EST)) 4929 return; 4930 4931 /* Update root cfs_rq's estimated utilization */ 4932 enqueued = cfs_rq->avg.util_est; 4933 enqueued -= min_t(unsigned int, enqueued, _task_util_est(p)); 4934 WRITE_ONCE(cfs_rq->avg.util_est, enqueued); 4935 4936 trace_sched_util_est_cfs_tp(cfs_rq); 4937 } 4938 4939 #define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100) 4940 4941 static inline void util_est_update(struct cfs_rq *cfs_rq, 4942 struct task_struct *p, 4943 bool task_sleep) 4944 { 4945 unsigned int ewma, dequeued, last_ewma_diff; 4946 4947 if (!sched_feat(UTIL_EST)) 4948 return; 4949 4950 /* 4951 * Skip update of task's estimated utilization when the task has not 4952 * yet completed an activation, e.g. being migrated. 4953 */ 4954 if (!task_sleep) 4955 return; 4956 4957 /* Get current estimate of utilization */ 4958 ewma = READ_ONCE(p->se.avg.util_est); 4959 4960 /* 4961 * If the PELT values haven't changed since enqueue time, 4962 * skip the util_est update. 4963 */ 4964 if (ewma & UTIL_AVG_UNCHANGED) 4965 return; 4966 4967 /* Get utilization at dequeue */ 4968 dequeued = task_util(p); 4969 4970 /* 4971 * Reset EWMA on utilization increases, the moving average is used only 4972 * to smooth utilization decreases. 4973 */ 4974 if (ewma <= dequeued) { 4975 ewma = dequeued; 4976 goto done; 4977 } 4978 4979 /* 4980 * Skip update of task's estimated utilization when its members are 4981 * already ~1% close to its last activation value. 4982 */ 4983 last_ewma_diff = ewma - dequeued; 4984 if (last_ewma_diff < UTIL_EST_MARGIN) 4985 goto done; 4986 4987 /* 4988 * To avoid underestimate of task utilization, skip updates of EWMA if 4989 * we cannot grant that thread got all CPU time it wanted. 4990 */ 4991 if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p)) 4992 goto done; 4993 4994 4995 /* 4996 * Update Task's estimated utilization 4997 * 4998 * When *p completes an activation we can consolidate another sample 4999 * of the task size. This is done by using this value to update the 5000 * Exponential Weighted Moving Average (EWMA): 5001 * 5002 * ewma(t) = w * task_util(p) + (1-w) * ewma(t-1) 5003 * = w * task_util(p) + ewma(t-1) - w * ewma(t-1) 5004 * = w * (task_util(p) - ewma(t-1)) + ewma(t-1) 5005 * = w * ( -last_ewma_diff ) + ewma(t-1) 5006 * = w * (-last_ewma_diff + ewma(t-1) / w) 5007 * 5008 * Where 'w' is the weight of new samples, which is configured to be 5009 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT) 5010 */ 5011 ewma <<= UTIL_EST_WEIGHT_SHIFT; 5012 ewma -= last_ewma_diff; 5013 ewma >>= UTIL_EST_WEIGHT_SHIFT; 5014 done: 5015 ewma |= UTIL_AVG_UNCHANGED; 5016 WRITE_ONCE(p->se.avg.util_est, ewma); 5017 5018 trace_sched_util_est_se_tp(&p->se); 5019 } 5020 5021 static inline unsigned long get_actual_cpu_capacity(int cpu) 5022 { 5023 unsigned long capacity = arch_scale_cpu_capacity(cpu); 5024 5025 capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu)); 5026 5027 return capacity; 5028 } 5029 5030 static inline int util_fits_cpu(unsigned long util, 5031 unsigned long uclamp_min, 5032 unsigned long uclamp_max, 5033 int cpu) 5034 { 5035 unsigned long capacity = capacity_of(cpu); 5036 unsigned long capacity_orig; 5037 bool fits, uclamp_max_fits; 5038 5039 /* 5040 * Check if the real util fits without any uclamp boost/cap applied. 5041 */ 5042 fits = fits_capacity(util, capacity); 5043 5044 if (!uclamp_is_used()) 5045 return fits; 5046 5047 /* 5048 * We must use arch_scale_cpu_capacity() for comparing against uclamp_min and 5049 * uclamp_max. We only care about capacity pressure (by using 5050 * capacity_of()) for comparing against the real util. 5051 * 5052 * If a task is boosted to 1024 for example, we don't want a tiny 5053 * pressure to skew the check whether it fits a CPU or not. 5054 * 5055 * Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it 5056 * should fit a little cpu even if there's some pressure. 5057 * 5058 * Only exception is for HW or cpufreq pressure since it has a direct impact 5059 * on available OPP of the system. 5060 * 5061 * We honour it for uclamp_min only as a drop in performance level 5062 * could result in not getting the requested minimum performance level. 5063 * 5064 * For uclamp_max, we can tolerate a drop in performance level as the 5065 * goal is to cap the task. So it's okay if it's getting less. 5066 */ 5067 capacity_orig = arch_scale_cpu_capacity(cpu); 5068 5069 /* 5070 * We want to force a task to fit a cpu as implied by uclamp_max. 5071 * But we do have some corner cases to cater for.. 5072 * 5073 * 5074 * C=z 5075 * | ___ 5076 * | C=y | | 5077 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5078 * | C=x | | | | 5079 * | ___ | | | | 5080 * | | | | | | | (util somewhere in this region) 5081 * | | | | | | | 5082 * | | | | | | | 5083 * +---------------------------------------- 5084 * CPU0 CPU1 CPU2 5085 * 5086 * In the above example if a task is capped to a specific performance 5087 * point, y, then when: 5088 * 5089 * * util = 80% of x then it does not fit on CPU0 and should migrate 5090 * to CPU1 5091 * * util = 80% of y then it is forced to fit on CPU1 to honour 5092 * uclamp_max request. 5093 * 5094 * which is what we're enforcing here. A task always fits if 5095 * uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig, 5096 * the normal upmigration rules should withhold still. 5097 * 5098 * Only exception is when we are on max capacity, then we need to be 5099 * careful not to block overutilized state. This is so because: 5100 * 5101 * 1. There's no concept of capping at max_capacity! We can't go 5102 * beyond this performance level anyway. 5103 * 2. The system is being saturated when we're operating near 5104 * max capacity, it doesn't make sense to block overutilized. 5105 */ 5106 uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE); 5107 uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig); 5108 fits = fits || uclamp_max_fits; 5109 5110 /* 5111 * 5112 * C=z 5113 * | ___ (region a, capped, util >= uclamp_max) 5114 * | C=y | | 5115 * |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max 5116 * | C=x | | | | 5117 * | ___ | | | | (region b, uclamp_min <= util <= uclamp_max) 5118 * |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min 5119 * | | | | | | | 5120 * | | | | | | | (region c, boosted, util < uclamp_min) 5121 * +---------------------------------------- 5122 * CPU0 CPU1 CPU2 5123 * 5124 * a) If util > uclamp_max, then we're capped, we don't care about 5125 * actual fitness value here. We only care if uclamp_max fits 5126 * capacity without taking margin/pressure into account. 5127 * See comment above. 5128 * 5129 * b) If uclamp_min <= util <= uclamp_max, then the normal 5130 * fits_capacity() rules apply. Except we need to ensure that we 5131 * enforce we remain within uclamp_max, see comment above. 5132 * 5133 * c) If util < uclamp_min, then we are boosted. Same as (b) but we 5134 * need to take into account the boosted value fits the CPU without 5135 * taking margin/pressure into account. 5136 * 5137 * Cases (a) and (b) are handled in the 'fits' variable already. We 5138 * just need to consider an extra check for case (c) after ensuring we 5139 * handle the case uclamp_min > uclamp_max. 5140 */ 5141 uclamp_min = min(uclamp_min, uclamp_max); 5142 if (fits && (util < uclamp_min) && 5143 (uclamp_min > get_actual_cpu_capacity(cpu))) 5144 return -1; 5145 5146 return fits; 5147 } 5148 5149 static inline int task_fits_cpu(struct task_struct *p, int cpu) 5150 { 5151 unsigned long uclamp_min = uclamp_eff_value(p, UCLAMP_MIN); 5152 unsigned long uclamp_max = uclamp_eff_value(p, UCLAMP_MAX); 5153 unsigned long util = task_util_est(p); 5154 /* 5155 * Return true only if the cpu fully fits the task requirements, which 5156 * include the utilization but also the performance hints. 5157 */ 5158 return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0); 5159 } 5160 5161 static inline void update_misfit_status(struct task_struct *p, struct rq *rq) 5162 { 5163 int cpu = cpu_of(rq); 5164 5165 if (!sched_asym_cpucap_active()) 5166 return; 5167 5168 /* 5169 * Affinity allows us to go somewhere higher? Or are we on biggest 5170 * available CPU already? Or do we fit into this CPU ? 5171 */ 5172 if (!p || (p->nr_cpus_allowed == 1) || 5173 (arch_scale_cpu_capacity(cpu) == p->max_allowed_capacity) || 5174 task_fits_cpu(p, cpu)) { 5175 5176 rq->misfit_task_load = 0; 5177 return; 5178 } 5179 5180 /* 5181 * Make sure that misfit_task_load will not be null even if 5182 * task_h_load() returns 0. 5183 */ 5184 rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1); 5185 } 5186 5187 void __setparam_fair(struct task_struct *p, const struct sched_attr *attr) 5188 { 5189 struct sched_entity *se = &p->se; 5190 5191 p->static_prio = NICE_TO_PRIO(attr->sched_nice); 5192 if (attr->sched_runtime) { 5193 se->custom_slice = 1; 5194 se->slice = clamp_t(u64, attr->sched_runtime, 5195 NSEC_PER_MSEC/10, /* HZ=1000 * 10 */ 5196 NSEC_PER_MSEC*100); /* HZ=100 / 10 */ 5197 } else { 5198 se->custom_slice = 0; 5199 se->slice = sysctl_sched_base_slice; 5200 } 5201 } 5202 5203 static void 5204 place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5205 { 5206 u64 vslice, vruntime = avg_vruntime(cfs_rq); 5207 s64 lag = 0; 5208 5209 if (!se->custom_slice) 5210 se->slice = sysctl_sched_base_slice; 5211 vslice = calc_delta_fair(se->slice, se); 5212 5213 /* 5214 * Due to how V is constructed as the weighted average of entities, 5215 * adding tasks with positive lag, or removing tasks with negative lag 5216 * will move 'time' backwards, this can screw around with the lag of 5217 * other tasks. 5218 * 5219 * EEVDF: placement strategy #1 / #2 5220 */ 5221 if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { 5222 struct sched_entity *curr = cfs_rq->curr; 5223 unsigned long load; 5224 5225 lag = se->vlag; 5226 5227 /* 5228 * If we want to place a task and preserve lag, we have to 5229 * consider the effect of the new entity on the weighted 5230 * average and compensate for this, otherwise lag can quickly 5231 * evaporate. 5232 * 5233 * Lag is defined as: 5234 * 5235 * lag_i = S - s_i = w_i * (V - v_i) 5236 * 5237 * To avoid the 'w_i' term all over the place, we only track 5238 * the virtual lag: 5239 * 5240 * vl_i = V - v_i <=> v_i = V - vl_i 5241 * 5242 * And we take V to be the weighted average of all v: 5243 * 5244 * V = (\Sum w_j*v_j) / W 5245 * 5246 * Where W is: \Sum w_j 5247 * 5248 * Then, the weighted average after adding an entity with lag 5249 * vl_i is given by: 5250 * 5251 * V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i) 5252 * = (W*V + w_i*(V - vl_i)) / (W + w_i) 5253 * = (W*V + w_i*V - w_i*vl_i) / (W + w_i) 5254 * = (V*(W + w_i) - w_i*vl_i) / (W + w_i) 5255 * = V - w_i*vl_i / (W + w_i) 5256 * 5257 * And the actual lag after adding an entity with vl_i is: 5258 * 5259 * vl'_i = V' - v_i 5260 * = V - w_i*vl_i / (W + w_i) - (V - vl_i) 5261 * = vl_i - w_i*vl_i / (W + w_i) 5262 * 5263 * Which is strictly less than vl_i. So in order to preserve lag 5264 * we should inflate the lag before placement such that the 5265 * effective lag after placement comes out right. 5266 * 5267 * As such, invert the above relation for vl'_i to get the vl_i 5268 * we need to use such that the lag after placement is the lag 5269 * we computed before dequeue. 5270 * 5271 * vl'_i = vl_i - w_i*vl_i / (W + w_i) 5272 * = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i) 5273 * 5274 * (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i 5275 * = W*vl_i 5276 * 5277 * vl_i = (W + w_i)*vl'_i / W 5278 */ 5279 load = cfs_rq->sum_weight; 5280 if (curr && curr->on_rq) 5281 load += scale_load_down(curr->load.weight); 5282 5283 lag *= load + scale_load_down(se->load.weight); 5284 if (WARN_ON_ONCE(!load)) 5285 load = 1; 5286 lag = div_s64(lag, load); 5287 } 5288 5289 se->vruntime = vruntime - lag; 5290 5291 if (se->rel_deadline) { 5292 se->deadline += se->vruntime; 5293 se->rel_deadline = 0; 5294 return; 5295 } 5296 5297 /* 5298 * When joining the competition; the existing tasks will be, 5299 * on average, halfway through their slice, as such start tasks 5300 * off with half a slice to ease into the competition. 5301 */ 5302 if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) 5303 vslice /= 2; 5304 5305 /* 5306 * EEVDF: vd_i = ve_i + r_i/w_i 5307 */ 5308 se->deadline = se->vruntime + vslice; 5309 } 5310 5311 static void check_enqueue_throttle(struct cfs_rq *cfs_rq); 5312 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); 5313 5314 static void 5315 requeue_delayed_entity(struct sched_entity *se); 5316 5317 static void 5318 enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5319 { 5320 bool curr = cfs_rq->curr == se; 5321 5322 /* 5323 * If we're the current task, we must renormalise before calling 5324 * update_curr(). 5325 */ 5326 if (curr) 5327 place_entity(cfs_rq, se, flags); 5328 5329 update_curr(cfs_rq); 5330 5331 /* 5332 * When enqueuing a sched_entity, we must: 5333 * - Update loads to have both entity and cfs_rq synced with now. 5334 * - For group_entity, update its runnable_weight to reflect the new 5335 * h_nr_runnable of its group cfs_rq. 5336 * - For group_entity, update its weight to reflect the new share of 5337 * its group cfs_rq 5338 * - Add its new weight to cfs_rq->load.weight 5339 */ 5340 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH); 5341 se_update_runnable(se); 5342 /* 5343 * XXX update_load_avg() above will have attached us to the pelt sum; 5344 * but update_cfs_group() here will re-adjust the weight and have to 5345 * undo/redo all that. Seems wasteful. 5346 */ 5347 update_cfs_group(se); 5348 5349 /* 5350 * XXX now that the entity has been re-weighted, and it's lag adjusted, 5351 * we can place the entity. 5352 */ 5353 if (!curr) 5354 place_entity(cfs_rq, se, flags); 5355 5356 account_entity_enqueue(cfs_rq, se); 5357 5358 /* Entity has migrated, no longer consider this task hot */ 5359 if (flags & ENQUEUE_MIGRATED) 5360 se->exec_start = 0; 5361 5362 check_schedstat_required(); 5363 update_stats_enqueue_fair(cfs_rq, se, flags); 5364 if (!curr) 5365 __enqueue_entity(cfs_rq, se); 5366 se->on_rq = 1; 5367 5368 if (cfs_rq->nr_queued == 1) { 5369 check_enqueue_throttle(cfs_rq); 5370 list_add_leaf_cfs_rq(cfs_rq); 5371 #ifdef CONFIG_CFS_BANDWIDTH 5372 if (cfs_rq->pelt_clock_throttled) { 5373 struct rq *rq = rq_of(cfs_rq); 5374 5375 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5376 cfs_rq->throttled_clock_pelt; 5377 cfs_rq->pelt_clock_throttled = 0; 5378 } 5379 #endif 5380 } 5381 } 5382 5383 static void __clear_buddies_next(struct sched_entity *se) 5384 { 5385 for_each_sched_entity(se) { 5386 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5387 if (cfs_rq->next != se) 5388 break; 5389 5390 cfs_rq->next = NULL; 5391 } 5392 } 5393 5394 static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) 5395 { 5396 if (cfs_rq->next == se) 5397 __clear_buddies_next(se); 5398 } 5399 5400 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5401 5402 static void set_delayed(struct sched_entity *se) 5403 { 5404 se->sched_delayed = 1; 5405 5406 /* 5407 * Delayed se of cfs_rq have no tasks queued on them. 5408 * Do not adjust h_nr_runnable since dequeue_entities() 5409 * will account it for blocked tasks. 5410 */ 5411 if (!entity_is_task(se)) 5412 return; 5413 5414 for_each_sched_entity(se) { 5415 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5416 5417 cfs_rq->h_nr_runnable--; 5418 } 5419 } 5420 5421 static void clear_delayed(struct sched_entity *se) 5422 { 5423 se->sched_delayed = 0; 5424 5425 /* 5426 * Delayed se of cfs_rq have no tasks queued on them. 5427 * Do not adjust h_nr_runnable since a dequeue has 5428 * already accounted for it or an enqueue of a task 5429 * below it will account for it in enqueue_task_fair(). 5430 */ 5431 if (!entity_is_task(se)) 5432 return; 5433 5434 for_each_sched_entity(se) { 5435 struct cfs_rq *cfs_rq = cfs_rq_of(se); 5436 5437 cfs_rq->h_nr_runnable++; 5438 } 5439 } 5440 5441 static inline void finish_delayed_dequeue_entity(struct sched_entity *se) 5442 { 5443 clear_delayed(se); 5444 if (sched_feat(DELAY_ZERO) && se->vlag > 0) 5445 se->vlag = 0; 5446 } 5447 5448 static bool 5449 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) 5450 { 5451 bool sleep = flags & DEQUEUE_SLEEP; 5452 int action = UPDATE_TG; 5453 5454 update_curr(cfs_rq); 5455 clear_buddies(cfs_rq, se); 5456 5457 if (flags & DEQUEUE_DELAYED) { 5458 WARN_ON_ONCE(!se->sched_delayed); 5459 } else { 5460 bool delay = sleep; 5461 /* 5462 * DELAY_DEQUEUE relies on spurious wakeups, special task 5463 * states must not suffer spurious wakeups, excempt them. 5464 */ 5465 if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) 5466 delay = false; 5467 5468 WARN_ON_ONCE(delay && se->sched_delayed); 5469 5470 if (sched_feat(DELAY_DEQUEUE) && delay && 5471 !entity_eligible(cfs_rq, se)) { 5472 update_load_avg(cfs_rq, se, 0); 5473 set_delayed(se); 5474 return false; 5475 } 5476 } 5477 5478 if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) 5479 action |= DO_DETACH; 5480 5481 /* 5482 * When dequeuing a sched_entity, we must: 5483 * - Update loads to have both entity and cfs_rq synced with now. 5484 * - For group_entity, update its runnable_weight to reflect the new 5485 * h_nr_runnable of its group cfs_rq. 5486 * - Subtract its previous weight from cfs_rq->load.weight. 5487 * - For group entity, update its weight to reflect the new share 5488 * of its group cfs_rq. 5489 */ 5490 update_load_avg(cfs_rq, se, action); 5491 se_update_runnable(se); 5492 5493 update_stats_dequeue_fair(cfs_rq, se, flags); 5494 5495 update_entity_lag(cfs_rq, se); 5496 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { 5497 se->deadline -= se->vruntime; 5498 se->rel_deadline = 1; 5499 } 5500 5501 if (se != cfs_rq->curr) 5502 __dequeue_entity(cfs_rq, se); 5503 se->on_rq = 0; 5504 account_entity_dequeue(cfs_rq, se); 5505 5506 /* return excess runtime on last dequeue */ 5507 return_cfs_rq_runtime(cfs_rq); 5508 5509 update_cfs_group(se); 5510 5511 if (flags & DEQUEUE_DELAYED) 5512 finish_delayed_dequeue_entity(se); 5513 5514 if (cfs_rq->nr_queued == 0) { 5515 update_idle_cfs_rq_clock_pelt(cfs_rq); 5516 #ifdef CONFIG_CFS_BANDWIDTH 5517 if (throttled_hierarchy(cfs_rq)) { 5518 struct rq *rq = rq_of(cfs_rq); 5519 5520 list_del_leaf_cfs_rq(cfs_rq); 5521 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 5522 cfs_rq->pelt_clock_throttled = 1; 5523 } 5524 #endif 5525 } 5526 5527 return true; 5528 } 5529 5530 static void 5531 set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) 5532 { 5533 clear_buddies(cfs_rq, se); 5534 5535 /* 'current' is not kept within the tree. */ 5536 if (se->on_rq) { 5537 /* 5538 * Any task has to be enqueued before it get to execute on 5539 * a CPU. So account for the time it spent waiting on the 5540 * runqueue. 5541 */ 5542 update_stats_wait_end_fair(cfs_rq, se); 5543 __dequeue_entity(cfs_rq, se); 5544 update_load_avg(cfs_rq, se, UPDATE_TG); 5545 5546 if (first) 5547 set_protect_slice(cfs_rq, se); 5548 } 5549 5550 update_stats_curr_start(cfs_rq, se); 5551 WARN_ON_ONCE(cfs_rq->curr); 5552 cfs_rq->curr = se; 5553 5554 /* 5555 * Track our maximum slice length, if the CPU's load is at 5556 * least twice that of our own weight (i.e. don't track it 5557 * when there are only lesser-weight tasks around): 5558 */ 5559 if (schedstat_enabled() && 5560 rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) { 5561 struct sched_statistics *stats; 5562 5563 stats = __schedstats_from_se(se); 5564 __schedstat_set(stats->slice_max, 5565 max((u64)stats->slice_max, 5566 se->sum_exec_runtime - se->prev_sum_exec_runtime)); 5567 } 5568 5569 se->prev_sum_exec_runtime = se->sum_exec_runtime; 5570 } 5571 5572 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); 5573 5574 /* 5575 * Pick the next process, keeping these things in mind, in this order: 5576 * 1) keep things fair between processes/task groups 5577 * 2) pick the "next" process, since someone really wants that to run 5578 * 3) pick the "last" process, for cache locality 5579 * 4) do not run the "skip" process, if something else is available 5580 */ 5581 static struct sched_entity * 5582 pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) 5583 { 5584 struct sched_entity *se; 5585 5586 se = pick_eevdf(cfs_rq, protect); 5587 if (se->sched_delayed) { 5588 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 5589 /* 5590 * Must not reference @se again, see __block_task(). 5591 */ 5592 return NULL; 5593 } 5594 return se; 5595 } 5596 5597 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); 5598 5599 static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) 5600 { 5601 /* 5602 * If still on the runqueue then deactivate_task() 5603 * was not called and update_curr() has to be done: 5604 */ 5605 if (prev->on_rq) 5606 update_curr(cfs_rq); 5607 5608 /* throttle cfs_rqs exceeding runtime */ 5609 check_cfs_rq_runtime(cfs_rq); 5610 5611 if (prev->on_rq) { 5612 update_stats_wait_start_fair(cfs_rq, prev); 5613 /* Put 'current' back into the tree. */ 5614 __enqueue_entity(cfs_rq, prev); 5615 /* in !on_rq case, update occurred at dequeue */ 5616 update_load_avg(cfs_rq, prev, 0); 5617 } 5618 WARN_ON_ONCE(cfs_rq->curr != prev); 5619 cfs_rq->curr = NULL; 5620 } 5621 5622 static void 5623 entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) 5624 { 5625 /* 5626 * Update run-time statistics of the 'current'. 5627 */ 5628 update_curr(cfs_rq); 5629 5630 /* 5631 * Ensure that runnable average is periodically updated. 5632 */ 5633 update_load_avg(cfs_rq, curr, UPDATE_TG); 5634 update_cfs_group(curr); 5635 5636 #ifdef CONFIG_SCHED_HRTICK 5637 /* 5638 * queued ticks are scheduled to match the slice, so don't bother 5639 * validating it and just reschedule. 5640 */ 5641 if (queued) { 5642 resched_curr_lazy(rq_of(cfs_rq)); 5643 return; 5644 } 5645 #endif 5646 } 5647 5648 5649 /************************************************** 5650 * CFS bandwidth control machinery 5651 */ 5652 5653 #ifdef CONFIG_CFS_BANDWIDTH 5654 5655 #ifdef CONFIG_JUMP_LABEL 5656 static struct static_key __cfs_bandwidth_used; 5657 5658 static inline bool cfs_bandwidth_used(void) 5659 { 5660 return static_key_false(&__cfs_bandwidth_used); 5661 } 5662 5663 void cfs_bandwidth_usage_inc(void) 5664 { 5665 static_key_slow_inc_cpuslocked(&__cfs_bandwidth_used); 5666 } 5667 5668 void cfs_bandwidth_usage_dec(void) 5669 { 5670 static_key_slow_dec_cpuslocked(&__cfs_bandwidth_used); 5671 } 5672 #else /* !CONFIG_JUMP_LABEL: */ 5673 static bool cfs_bandwidth_used(void) 5674 { 5675 return true; 5676 } 5677 5678 void cfs_bandwidth_usage_inc(void) {} 5679 void cfs_bandwidth_usage_dec(void) {} 5680 #endif /* !CONFIG_JUMP_LABEL */ 5681 5682 static inline u64 sched_cfs_bandwidth_slice(void) 5683 { 5684 return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC; 5685 } 5686 5687 /* 5688 * Replenish runtime according to assigned quota. We use sched_clock_cpu 5689 * directly instead of rq->clock to avoid adding additional synchronization 5690 * around rq->lock. 5691 * 5692 * requires cfs_b->lock 5693 */ 5694 void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b) 5695 { 5696 s64 runtime; 5697 5698 if (unlikely(cfs_b->quota == RUNTIME_INF)) 5699 return; 5700 5701 cfs_b->runtime += cfs_b->quota; 5702 runtime = cfs_b->runtime_snap - cfs_b->runtime; 5703 if (runtime > 0) { 5704 cfs_b->burst_time += runtime; 5705 cfs_b->nr_burst++; 5706 } 5707 5708 cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst); 5709 cfs_b->runtime_snap = cfs_b->runtime; 5710 } 5711 5712 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 5713 { 5714 return &tg->cfs_bandwidth; 5715 } 5716 5717 /* returns 0 on failure to allocate runtime */ 5718 static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, 5719 struct cfs_rq *cfs_rq, u64 target_runtime) 5720 { 5721 u64 min_amount, amount = 0; 5722 5723 lockdep_assert_held(&cfs_b->lock); 5724 5725 /* note: this is a positive sum as runtime_remaining <= 0 */ 5726 min_amount = target_runtime - cfs_rq->runtime_remaining; 5727 5728 if (cfs_b->quota == RUNTIME_INF) 5729 amount = min_amount; 5730 else { 5731 start_cfs_bandwidth(cfs_b); 5732 5733 if (cfs_b->runtime > 0) { 5734 amount = min(cfs_b->runtime, min_amount); 5735 cfs_b->runtime -= amount; 5736 cfs_b->idle = 0; 5737 } 5738 } 5739 5740 cfs_rq->runtime_remaining += amount; 5741 5742 return cfs_rq->runtime_remaining > 0; 5743 } 5744 5745 /* returns 0 on failure to allocate runtime */ 5746 static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) 5747 { 5748 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 5749 int ret; 5750 5751 raw_spin_lock(&cfs_b->lock); 5752 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); 5753 raw_spin_unlock(&cfs_b->lock); 5754 5755 return ret; 5756 } 5757 5758 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5759 { 5760 /* dock delta_exec before expiring quota (as it could span periods) */ 5761 cfs_rq->runtime_remaining -= delta_exec; 5762 5763 if (likely(cfs_rq->runtime_remaining > 0)) 5764 return; 5765 5766 if (cfs_rq->throttled) 5767 return; 5768 /* 5769 * if we're unable to extend our runtime we resched so that the active 5770 * hierarchy can be throttled 5771 */ 5772 if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) 5773 resched_curr(rq_of(cfs_rq)); 5774 } 5775 5776 static __always_inline 5777 void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) 5778 { 5779 if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled) 5780 return; 5781 5782 __account_cfs_rq_runtime(cfs_rq, delta_exec); 5783 } 5784 5785 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 5786 { 5787 return cfs_bandwidth_used() && cfs_rq->throttled; 5788 } 5789 5790 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 5791 { 5792 return cfs_bandwidth_used() && cfs_rq->pelt_clock_throttled; 5793 } 5794 5795 /* check whether cfs_rq, or any parent, is throttled */ 5796 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 5797 { 5798 return cfs_bandwidth_used() && cfs_rq->throttle_count; 5799 } 5800 5801 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 5802 { 5803 return throttled_hierarchy(task_group(p)->cfs_rq[dst_cpu]); 5804 } 5805 5806 static inline bool task_is_throttled(struct task_struct *p) 5807 { 5808 return cfs_bandwidth_used() && p->throttled; 5809 } 5810 5811 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5812 static void throttle_cfs_rq_work(struct callback_head *work) 5813 { 5814 struct task_struct *p = container_of(work, struct task_struct, sched_throttle_work); 5815 struct sched_entity *se; 5816 struct cfs_rq *cfs_rq; 5817 struct rq *rq; 5818 5819 WARN_ON_ONCE(p != current); 5820 p->sched_throttle_work.next = &p->sched_throttle_work; 5821 5822 /* 5823 * If task is exiting, then there won't be a return to userspace, so we 5824 * don't have to bother with any of this. 5825 */ 5826 if ((p->flags & PF_EXITING)) 5827 return; 5828 5829 scoped_guard(task_rq_lock, p) { 5830 se = &p->se; 5831 cfs_rq = cfs_rq_of(se); 5832 5833 /* Raced, forget */ 5834 if (p->sched_class != &fair_sched_class) 5835 return; 5836 5837 /* 5838 * If not in limbo, then either replenish has happened or this 5839 * task got migrated out of the throttled cfs_rq, move along. 5840 */ 5841 if (!cfs_rq->throttle_count) 5842 return; 5843 rq = scope.rq; 5844 update_rq_clock(rq); 5845 WARN_ON_ONCE(p->throttled || !list_empty(&p->throttle_node)); 5846 dequeue_task_fair(rq, p, DEQUEUE_SLEEP | DEQUEUE_THROTTLE); 5847 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5848 /* 5849 * Must not set throttled before dequeue or dequeue will 5850 * mistakenly regard this task as an already throttled one. 5851 */ 5852 p->throttled = true; 5853 resched_curr(rq); 5854 } 5855 } 5856 5857 void init_cfs_throttle_work(struct task_struct *p) 5858 { 5859 init_task_work(&p->sched_throttle_work, throttle_cfs_rq_work); 5860 /* Protect against double add, see throttle_cfs_rq() and throttle_cfs_rq_work() */ 5861 p->sched_throttle_work.next = &p->sched_throttle_work; 5862 INIT_LIST_HEAD(&p->throttle_node); 5863 } 5864 5865 /* 5866 * Task is throttled and someone wants to dequeue it again: 5867 * it could be sched/core when core needs to do things like 5868 * task affinity change, task group change, task sched class 5869 * change etc. and in these cases, DEQUEUE_SLEEP is not set; 5870 * or the task is blocked after throttled due to freezer etc. 5871 * and in these cases, DEQUEUE_SLEEP is set. 5872 */ 5873 static void detach_task_cfs_rq(struct task_struct *p); 5874 static void dequeue_throttled_task(struct task_struct *p, int flags) 5875 { 5876 WARN_ON_ONCE(p->se.on_rq); 5877 list_del_init(&p->throttle_node); 5878 5879 /* task blocked after throttled */ 5880 if (flags & DEQUEUE_SLEEP) { 5881 p->throttled = false; 5882 return; 5883 } 5884 5885 /* 5886 * task is migrating off its old cfs_rq, detach 5887 * the task's load from its old cfs_rq. 5888 */ 5889 if (task_on_rq_migrating(p)) 5890 detach_task_cfs_rq(p); 5891 } 5892 5893 static bool enqueue_throttled_task(struct task_struct *p) 5894 { 5895 struct cfs_rq *cfs_rq = cfs_rq_of(&p->se); 5896 5897 /* @p should have gone through dequeue_throttled_task() first */ 5898 WARN_ON_ONCE(!list_empty(&p->throttle_node)); 5899 5900 /* 5901 * If the throttled task @p is enqueued to a throttled cfs_rq, 5902 * take the fast path by directly putting the task on the 5903 * target cfs_rq's limbo list. 5904 * 5905 * Do not do that when @p is current because the following race can 5906 * cause @p's group_node to be incorectly re-insterted in its rq's 5907 * cfs_tasks list, despite being throttled: 5908 * 5909 * cpuX cpuY 5910 * p ret2user 5911 * throttle_cfs_rq_work() sched_move_task(p) 5912 * LOCK task_rq_lock 5913 * dequeue_task_fair(p) 5914 * UNLOCK task_rq_lock 5915 * LOCK task_rq_lock 5916 * task_current_donor(p) == true 5917 * task_on_rq_queued(p) == true 5918 * dequeue_task(p) 5919 * put_prev_task(p) 5920 * sched_change_group() 5921 * enqueue_task(p) -> p's new cfs_rq 5922 * is throttled, go 5923 * fast path and skip 5924 * actual enqueue 5925 * set_next_task(p) 5926 * list_move(&se->group_node, &rq->cfs_tasks); // bug 5927 * schedule() 5928 * 5929 * In the above race case, @p current cfs_rq is in the same rq as 5930 * its previous cfs_rq because sched_move_task() only moves a task 5931 * to a different group from the same rq, so we can use its current 5932 * cfs_rq to derive rq and test if the task is current. 5933 */ 5934 if (throttled_hierarchy(cfs_rq) && 5935 !task_current_donor(rq_of(cfs_rq), p)) { 5936 list_add(&p->throttle_node, &cfs_rq->throttled_limbo_list); 5937 return true; 5938 } 5939 5940 /* we can't take the fast path, do an actual enqueue*/ 5941 p->throttled = false; 5942 return false; 5943 } 5944 5945 static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags); 5946 static int tg_unthrottle_up(struct task_group *tg, void *data) 5947 { 5948 struct rq *rq = data; 5949 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 5950 struct task_struct *p, *tmp; 5951 5952 if (--cfs_rq->throttle_count) 5953 return 0; 5954 5955 if (cfs_rq->pelt_clock_throttled) { 5956 cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) - 5957 cfs_rq->throttled_clock_pelt; 5958 cfs_rq->pelt_clock_throttled = 0; 5959 } 5960 5961 if (cfs_rq->throttled_clock_self) { 5962 u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self; 5963 5964 cfs_rq->throttled_clock_self = 0; 5965 5966 if (WARN_ON_ONCE((s64)delta < 0)) 5967 delta = 0; 5968 5969 cfs_rq->throttled_clock_self_time += delta; 5970 } 5971 5972 /* Re-enqueue the tasks that have been throttled at this level. */ 5973 list_for_each_entry_safe(p, tmp, &cfs_rq->throttled_limbo_list, throttle_node) { 5974 list_del_init(&p->throttle_node); 5975 p->throttled = false; 5976 enqueue_task_fair(rq_of(cfs_rq), p, ENQUEUE_WAKEUP); 5977 } 5978 5979 /* Add cfs_rq with load or one or more already running entities to the list */ 5980 if (!cfs_rq_is_decayed(cfs_rq)) 5981 list_add_leaf_cfs_rq(cfs_rq); 5982 5983 return 0; 5984 } 5985 5986 static inline bool task_has_throttle_work(struct task_struct *p) 5987 { 5988 return p->sched_throttle_work.next != &p->sched_throttle_work; 5989 } 5990 5991 static inline void task_throttle_setup_work(struct task_struct *p) 5992 { 5993 if (task_has_throttle_work(p)) 5994 return; 5995 5996 /* 5997 * Kthreads and exiting tasks don't return to userspace, so adding the 5998 * work is pointless 5999 */ 6000 if ((p->flags & (PF_EXITING | PF_KTHREAD))) 6001 return; 6002 6003 task_work_add(p, &p->sched_throttle_work, TWA_RESUME); 6004 } 6005 6006 static void record_throttle_clock(struct cfs_rq *cfs_rq) 6007 { 6008 struct rq *rq = rq_of(cfs_rq); 6009 6010 if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock) 6011 cfs_rq->throttled_clock = rq_clock(rq); 6012 6013 if (!cfs_rq->throttled_clock_self) 6014 cfs_rq->throttled_clock_self = rq_clock(rq); 6015 } 6016 6017 static int tg_throttle_down(struct task_group *tg, void *data) 6018 { 6019 struct rq *rq = data; 6020 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6021 6022 if (cfs_rq->throttle_count++) 6023 return 0; 6024 6025 /* 6026 * For cfs_rqs that still have entities enqueued, PELT clock 6027 * stop happens at dequeue time when all entities are dequeued. 6028 */ 6029 if (!cfs_rq->nr_queued) { 6030 list_del_leaf_cfs_rq(cfs_rq); 6031 cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq); 6032 cfs_rq->pelt_clock_throttled = 1; 6033 } 6034 6035 WARN_ON_ONCE(cfs_rq->throttled_clock_self); 6036 WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_limbo_list)); 6037 return 0; 6038 } 6039 6040 static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) 6041 { 6042 struct rq *rq = rq_of(cfs_rq); 6043 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6044 int dequeue = 1; 6045 6046 raw_spin_lock(&cfs_b->lock); 6047 /* This will start the period timer if necessary */ 6048 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { 6049 /* 6050 * We have raced with bandwidth becoming available, and if we 6051 * actually throttled the timer might not unthrottle us for an 6052 * entire period. We additionally needed to make sure that any 6053 * subsequent check_cfs_rq_runtime calls agree not to throttle 6054 * us, as we may commit to do cfs put_prev+pick_next, so we ask 6055 * for 1ns of runtime rather than just check cfs_b. 6056 */ 6057 dequeue = 0; 6058 } else { 6059 list_add_tail_rcu(&cfs_rq->throttled_list, 6060 &cfs_b->throttled_cfs_rq); 6061 } 6062 raw_spin_unlock(&cfs_b->lock); 6063 6064 if (!dequeue) 6065 return false; /* Throttle no longer required. */ 6066 6067 /* freeze hierarchy runnable averages while throttled */ 6068 rcu_read_lock(); 6069 walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); 6070 rcu_read_unlock(); 6071 6072 /* 6073 * Note: distribution will already see us throttled via the 6074 * throttled-list. rq->lock protects completion. 6075 */ 6076 cfs_rq->throttled = 1; 6077 WARN_ON_ONCE(cfs_rq->throttled_clock); 6078 return true; 6079 } 6080 6081 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) 6082 { 6083 struct rq *rq = rq_of(cfs_rq); 6084 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6085 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)]; 6086 6087 /* 6088 * It's possible we are called with runtime_remaining < 0 due to things 6089 * like async unthrottled us with a positive runtime_remaining but other 6090 * still running entities consumed those runtime before we reached here. 6091 * 6092 * We can't unthrottle this cfs_rq without any runtime remaining because 6093 * any enqueue in tg_unthrottle_up() will immediately trigger a throttle, 6094 * which is not supposed to happen on unthrottle path. 6095 */ 6096 if (cfs_rq->runtime_enabled && cfs_rq->runtime_remaining <= 0) 6097 return; 6098 6099 cfs_rq->throttled = 0; 6100 6101 update_rq_clock(rq); 6102 6103 raw_spin_lock(&cfs_b->lock); 6104 if (cfs_rq->throttled_clock) { 6105 cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock; 6106 cfs_rq->throttled_clock = 0; 6107 } 6108 list_del_rcu(&cfs_rq->throttled_list); 6109 raw_spin_unlock(&cfs_b->lock); 6110 6111 /* update hierarchical throttle state */ 6112 walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); 6113 6114 if (!cfs_rq->load.weight) { 6115 if (!cfs_rq->on_list) 6116 return; 6117 /* 6118 * Nothing to run but something to decay (on_list)? 6119 * Complete the branch. 6120 */ 6121 for_each_sched_entity(se) { 6122 if (list_add_leaf_cfs_rq(cfs_rq_of(se))) 6123 break; 6124 } 6125 } 6126 6127 assert_list_leaf_cfs_rq(rq); 6128 6129 /* Determine whether we need to wake up potentially idle CPU: */ 6130 if (rq->curr == rq->idle && rq->cfs.nr_queued) 6131 resched_curr(rq); 6132 } 6133 6134 static void __cfsb_csd_unthrottle(void *arg) 6135 { 6136 struct cfs_rq *cursor, *tmp; 6137 struct rq *rq = arg; 6138 struct rq_flags rf; 6139 6140 rq_lock(rq, &rf); 6141 6142 /* 6143 * Iterating over the list can trigger several call to 6144 * update_rq_clock() in unthrottle_cfs_rq(). 6145 * Do it once and skip the potential next ones. 6146 */ 6147 update_rq_clock(rq); 6148 rq_clock_start_loop_update(rq); 6149 6150 /* 6151 * Since we hold rq lock we're safe from concurrent manipulation of 6152 * the CSD list. However, this RCU critical section annotates the 6153 * fact that we pair with sched_free_group_rcu(), so that we cannot 6154 * race with group being freed in the window between removing it 6155 * from the list and advancing to the next entry in the list. 6156 */ 6157 rcu_read_lock(); 6158 6159 list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list, 6160 throttled_csd_list) { 6161 list_del_init(&cursor->throttled_csd_list); 6162 6163 if (cfs_rq_throttled(cursor)) 6164 unthrottle_cfs_rq(cursor); 6165 } 6166 6167 rcu_read_unlock(); 6168 6169 rq_clock_stop_loop_update(rq); 6170 rq_unlock(rq, &rf); 6171 } 6172 6173 static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6174 { 6175 struct rq *rq = rq_of(cfs_rq); 6176 bool first; 6177 6178 if (rq == this_rq()) { 6179 unthrottle_cfs_rq(cfs_rq); 6180 return; 6181 } 6182 6183 /* Already enqueued */ 6184 if (WARN_ON_ONCE(!list_empty(&cfs_rq->throttled_csd_list))) 6185 return; 6186 6187 first = list_empty(&rq->cfsb_csd_list); 6188 list_add_tail(&cfs_rq->throttled_csd_list, &rq->cfsb_csd_list); 6189 if (first) 6190 smp_call_function_single_async(cpu_of(rq), &rq->cfsb_csd); 6191 } 6192 6193 static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq) 6194 { 6195 lockdep_assert_rq_held(rq_of(cfs_rq)); 6196 6197 if (WARN_ON_ONCE(!cfs_rq_throttled(cfs_rq) || 6198 cfs_rq->runtime_remaining <= 0)) 6199 return; 6200 6201 __unthrottle_cfs_rq_async(cfs_rq); 6202 } 6203 6204 static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) 6205 { 6206 int this_cpu = smp_processor_id(); 6207 u64 runtime, remaining = 1; 6208 bool throttled = false; 6209 struct cfs_rq *cfs_rq, *tmp; 6210 struct rq_flags rf; 6211 struct rq *rq; 6212 LIST_HEAD(local_unthrottle); 6213 6214 rcu_read_lock(); 6215 list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq, 6216 throttled_list) { 6217 rq = rq_of(cfs_rq); 6218 6219 if (!remaining) { 6220 throttled = true; 6221 break; 6222 } 6223 6224 rq_lock_irqsave(rq, &rf); 6225 if (!cfs_rq_throttled(cfs_rq)) 6226 goto next; 6227 6228 /* Already queued for async unthrottle */ 6229 if (!list_empty(&cfs_rq->throttled_csd_list)) 6230 goto next; 6231 6232 /* By the above checks, this should never be true */ 6233 WARN_ON_ONCE(cfs_rq->runtime_remaining > 0); 6234 6235 raw_spin_lock(&cfs_b->lock); 6236 runtime = -cfs_rq->runtime_remaining + 1; 6237 if (runtime > cfs_b->runtime) 6238 runtime = cfs_b->runtime; 6239 cfs_b->runtime -= runtime; 6240 remaining = cfs_b->runtime; 6241 raw_spin_unlock(&cfs_b->lock); 6242 6243 cfs_rq->runtime_remaining += runtime; 6244 6245 /* we check whether we're throttled above */ 6246 if (cfs_rq->runtime_remaining > 0) { 6247 if (cpu_of(rq) != this_cpu) { 6248 unthrottle_cfs_rq_async(cfs_rq); 6249 } else { 6250 /* 6251 * We currently only expect to be unthrottling 6252 * a single cfs_rq locally. 6253 */ 6254 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6255 list_add_tail(&cfs_rq->throttled_csd_list, 6256 &local_unthrottle); 6257 } 6258 } else { 6259 throttled = true; 6260 } 6261 6262 next: 6263 rq_unlock_irqrestore(rq, &rf); 6264 } 6265 6266 list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle, 6267 throttled_csd_list) { 6268 struct rq *rq = rq_of(cfs_rq); 6269 6270 rq_lock_irqsave(rq, &rf); 6271 6272 list_del_init(&cfs_rq->throttled_csd_list); 6273 6274 if (cfs_rq_throttled(cfs_rq)) 6275 unthrottle_cfs_rq(cfs_rq); 6276 6277 rq_unlock_irqrestore(rq, &rf); 6278 } 6279 WARN_ON_ONCE(!list_empty(&local_unthrottle)); 6280 6281 rcu_read_unlock(); 6282 6283 return throttled; 6284 } 6285 6286 /* 6287 * Responsible for refilling a task_group's bandwidth and unthrottling its 6288 * cfs_rqs as appropriate. If there has been no activity within the last 6289 * period the timer is deactivated until scheduling resumes; cfs_b->idle is 6290 * used to track this state. 6291 */ 6292 static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) 6293 { 6294 int throttled; 6295 6296 /* no need to continue the timer with no bandwidth constraint */ 6297 if (cfs_b->quota == RUNTIME_INF) 6298 goto out_deactivate; 6299 6300 throttled = !list_empty(&cfs_b->throttled_cfs_rq); 6301 cfs_b->nr_periods += overrun; 6302 6303 /* Refill extra burst quota even if cfs_b->idle */ 6304 __refill_cfs_bandwidth_runtime(cfs_b); 6305 6306 /* 6307 * idle depends on !throttled (for the case of a large deficit), and if 6308 * we're going inactive then everything else can be deferred 6309 */ 6310 if (cfs_b->idle && !throttled) 6311 goto out_deactivate; 6312 6313 if (!throttled) { 6314 /* mark as potentially idle for the upcoming period */ 6315 cfs_b->idle = 1; 6316 return 0; 6317 } 6318 6319 /* account preceding periods in which throttling occurred */ 6320 cfs_b->nr_throttled += overrun; 6321 6322 /* 6323 * This check is repeated as we release cfs_b->lock while we unthrottle. 6324 */ 6325 while (throttled && cfs_b->runtime > 0) { 6326 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6327 /* we can't nest cfs_b->lock while distributing bandwidth */ 6328 throttled = distribute_cfs_runtime(cfs_b); 6329 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6330 } 6331 6332 /* 6333 * While we are ensured activity in the period following an 6334 * unthrottle, this also covers the case in which the new bandwidth is 6335 * insufficient to cover the existing bandwidth deficit. (Forcing the 6336 * timer to remain active while there are any throttled entities.) 6337 */ 6338 cfs_b->idle = 0; 6339 6340 return 0; 6341 6342 out_deactivate: 6343 return 1; 6344 } 6345 6346 /* a cfs_rq won't donate quota below this amount */ 6347 static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC; 6348 /* minimum remaining period time to redistribute slack quota */ 6349 static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC; 6350 /* how long we wait to gather additional slack before distributing */ 6351 static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC; 6352 6353 /* 6354 * Are we near the end of the current quota period? 6355 * 6356 * Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the 6357 * hrtimer base being cleared by hrtimer_start. In the case of 6358 * migrate_hrtimers, base is never cleared, so we are fine. 6359 */ 6360 static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire) 6361 { 6362 struct hrtimer *refresh_timer = &cfs_b->period_timer; 6363 s64 remaining; 6364 6365 /* if the call-back is running a quota refresh is already occurring */ 6366 if (hrtimer_callback_running(refresh_timer)) 6367 return 1; 6368 6369 /* is a quota refresh about to occur? */ 6370 remaining = ktime_to_ns(hrtimer_expires_remaining(refresh_timer)); 6371 if (remaining < (s64)min_expire) 6372 return 1; 6373 6374 return 0; 6375 } 6376 6377 static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) 6378 { 6379 u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration; 6380 6381 /* if there's a quota refresh soon don't bother with slack */ 6382 if (runtime_refresh_within(cfs_b, min_left)) 6383 return; 6384 6385 /* don't push forwards an existing deferred unthrottle */ 6386 if (cfs_b->slack_started) 6387 return; 6388 cfs_b->slack_started = true; 6389 6390 hrtimer_start(&cfs_b->slack_timer, 6391 ns_to_ktime(cfs_bandwidth_slack_period), 6392 HRTIMER_MODE_REL); 6393 } 6394 6395 /* we know any runtime found here is valid as update_curr() precedes return */ 6396 static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6397 { 6398 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); 6399 s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime; 6400 6401 if (slack_runtime <= 0) 6402 return; 6403 6404 raw_spin_lock(&cfs_b->lock); 6405 if (cfs_b->quota != RUNTIME_INF) { 6406 cfs_b->runtime += slack_runtime; 6407 6408 /* we are under rq->lock, defer unthrottling using a timer */ 6409 if (cfs_b->runtime > sched_cfs_bandwidth_slice() && 6410 !list_empty(&cfs_b->throttled_cfs_rq)) 6411 start_cfs_slack_bandwidth(cfs_b); 6412 } 6413 raw_spin_unlock(&cfs_b->lock); 6414 6415 /* even if it's not valid for return we don't want to try again */ 6416 cfs_rq->runtime_remaining -= slack_runtime; 6417 } 6418 6419 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6420 { 6421 if (!cfs_bandwidth_used()) 6422 return; 6423 6424 if (!cfs_rq->runtime_enabled || cfs_rq->nr_queued) 6425 return; 6426 6427 __return_cfs_rq_runtime(cfs_rq); 6428 } 6429 6430 /* 6431 * This is done with a timer (instead of inline with bandwidth return) since 6432 * it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs. 6433 */ 6434 static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) 6435 { 6436 u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); 6437 unsigned long flags; 6438 6439 /* confirm we're still not at a refresh boundary */ 6440 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6441 cfs_b->slack_started = false; 6442 6443 if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { 6444 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6445 return; 6446 } 6447 6448 if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) 6449 runtime = cfs_b->runtime; 6450 6451 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6452 6453 if (!runtime) 6454 return; 6455 6456 distribute_cfs_runtime(cfs_b); 6457 } 6458 6459 /* 6460 * When a group wakes up we want to make sure that its quota is not already 6461 * expired/exceeded, otherwise it may be allowed to steal additional ticks of 6462 * runtime as update_curr() throttling can not trigger until it's on-rq. 6463 */ 6464 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) 6465 { 6466 if (!cfs_bandwidth_used()) 6467 return; 6468 6469 /* an active group must be handled by the update_curr()->put() path */ 6470 if (!cfs_rq->runtime_enabled || cfs_rq->curr) 6471 return; 6472 6473 /* ensure the group is not already throttled */ 6474 if (cfs_rq_throttled(cfs_rq)) 6475 return; 6476 6477 /* update runtime allocation */ 6478 account_cfs_rq_runtime(cfs_rq, 0); 6479 if (cfs_rq->runtime_remaining <= 0) 6480 throttle_cfs_rq(cfs_rq); 6481 } 6482 6483 static void sync_throttle(struct task_group *tg, int cpu) 6484 { 6485 struct cfs_rq *pcfs_rq, *cfs_rq; 6486 6487 if (!cfs_bandwidth_used()) 6488 return; 6489 6490 if (!tg->parent) 6491 return; 6492 6493 cfs_rq = tg->cfs_rq[cpu]; 6494 pcfs_rq = tg->parent->cfs_rq[cpu]; 6495 6496 cfs_rq->throttle_count = pcfs_rq->throttle_count; 6497 cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu)); 6498 6499 /* 6500 * It is not enough to sync the "pelt_clock_throttled" indicator 6501 * with the parent cfs_rq when the hierarchy is not queued. 6502 * Always join a throttled hierarchy with PELT clock throttled 6503 * and leaf it to the first enqueue, or distribution to 6504 * unthrottle the PELT clock. 6505 */ 6506 if (cfs_rq->throttle_count) 6507 cfs_rq->pelt_clock_throttled = 1; 6508 } 6509 6510 /* conditionally throttle active cfs_rq's from put_prev_entity() */ 6511 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6512 { 6513 if (!cfs_bandwidth_used()) 6514 return false; 6515 6516 if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0)) 6517 return false; 6518 6519 /* 6520 * it's possible for a throttled entity to be forced into a running 6521 * state (e.g. set_curr_task), in this case we're finished. 6522 */ 6523 if (cfs_rq_throttled(cfs_rq)) 6524 return true; 6525 6526 return throttle_cfs_rq(cfs_rq); 6527 } 6528 6529 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) 6530 { 6531 struct cfs_bandwidth *cfs_b = 6532 container_of(timer, struct cfs_bandwidth, slack_timer); 6533 6534 do_sched_cfs_slack_timer(cfs_b); 6535 6536 return HRTIMER_NORESTART; 6537 } 6538 6539 static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) 6540 { 6541 struct cfs_bandwidth *cfs_b = 6542 container_of(timer, struct cfs_bandwidth, period_timer); 6543 unsigned long flags; 6544 int overrun; 6545 int idle = 0; 6546 int count = 0; 6547 6548 raw_spin_lock_irqsave(&cfs_b->lock, flags); 6549 for (;;) { 6550 overrun = hrtimer_forward_now(timer, cfs_b->period); 6551 if (!overrun) 6552 break; 6553 6554 idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); 6555 6556 if (++count > 3) { 6557 u64 new, old = ktime_to_ns(cfs_b->period); 6558 6559 /* 6560 * Grow period by a factor of 2 to avoid losing precision. 6561 * Precision loss in the quota/period ratio can cause __cfs_schedulable 6562 * to fail. 6563 */ 6564 new = old * 2; 6565 if (new < max_bw_quota_period_us * NSEC_PER_USEC) { 6566 cfs_b->period = ns_to_ktime(new); 6567 cfs_b->quota *= 2; 6568 cfs_b->burst *= 2; 6569 6570 pr_warn_ratelimited( 6571 "cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6572 smp_processor_id(), 6573 div_u64(new, NSEC_PER_USEC), 6574 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6575 } else { 6576 pr_warn_ratelimited( 6577 "cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n", 6578 smp_processor_id(), 6579 div_u64(old, NSEC_PER_USEC), 6580 div_u64(cfs_b->quota, NSEC_PER_USEC)); 6581 } 6582 6583 /* reset count so we don't come right back in here */ 6584 count = 0; 6585 } 6586 } 6587 if (idle) 6588 cfs_b->period_active = 0; 6589 raw_spin_unlock_irqrestore(&cfs_b->lock, flags); 6590 6591 return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; 6592 } 6593 6594 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) 6595 { 6596 raw_spin_lock_init(&cfs_b->lock); 6597 cfs_b->runtime = 0; 6598 cfs_b->quota = RUNTIME_INF; 6599 cfs_b->period = us_to_ktime(default_bw_period_us()); 6600 cfs_b->burst = 0; 6601 cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF; 6602 6603 INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); 6604 hrtimer_setup(&cfs_b->period_timer, sched_cfs_period_timer, CLOCK_MONOTONIC, 6605 HRTIMER_MODE_ABS_PINNED); 6606 6607 /* Add a random offset so that timers interleave */ 6608 hrtimer_set_expires(&cfs_b->period_timer, 6609 get_random_u32_below(cfs_b->period)); 6610 hrtimer_setup(&cfs_b->slack_timer, sched_cfs_slack_timer, CLOCK_MONOTONIC, 6611 HRTIMER_MODE_REL); 6612 cfs_b->slack_started = false; 6613 } 6614 6615 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) 6616 { 6617 cfs_rq->runtime_enabled = 0; 6618 INIT_LIST_HEAD(&cfs_rq->throttled_list); 6619 INIT_LIST_HEAD(&cfs_rq->throttled_csd_list); 6620 INIT_LIST_HEAD(&cfs_rq->throttled_limbo_list); 6621 } 6622 6623 void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6624 { 6625 lockdep_assert_held(&cfs_b->lock); 6626 6627 if (cfs_b->period_active) 6628 return; 6629 6630 cfs_b->period_active = 1; 6631 hrtimer_forward_now(&cfs_b->period_timer, cfs_b->period); 6632 hrtimer_start_expires(&cfs_b->period_timer, HRTIMER_MODE_ABS_PINNED); 6633 } 6634 6635 static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) 6636 { 6637 int __maybe_unused i; 6638 6639 /* init_cfs_bandwidth() was not called */ 6640 if (!cfs_b->throttled_cfs_rq.next) 6641 return; 6642 6643 hrtimer_cancel(&cfs_b->period_timer); 6644 hrtimer_cancel(&cfs_b->slack_timer); 6645 6646 /* 6647 * It is possible that we still have some cfs_rq's pending on a CSD 6648 * list, though this race is very rare. In order for this to occur, we 6649 * must have raced with the last task leaving the group while there 6650 * exist throttled cfs_rq(s), and the period_timer must have queued the 6651 * CSD item but the remote cpu has not yet processed it. To handle this, 6652 * we can simply flush all pending CSD work inline here. We're 6653 * guaranteed at this point that no additional cfs_rq of this group can 6654 * join a CSD list. 6655 */ 6656 for_each_possible_cpu(i) { 6657 struct rq *rq = cpu_rq(i); 6658 unsigned long flags; 6659 6660 if (list_empty(&rq->cfsb_csd_list)) 6661 continue; 6662 6663 local_irq_save(flags); 6664 __cfsb_csd_unthrottle(rq); 6665 local_irq_restore(flags); 6666 } 6667 } 6668 6669 /* 6670 * Both these CPU hotplug callbacks race against unregister_fair_sched_group() 6671 * 6672 * The race is harmless, since modifying bandwidth settings of unhooked group 6673 * bits doesn't do much. 6674 */ 6675 6676 /* cpu online callback */ 6677 static void __maybe_unused update_runtime_enabled(struct rq *rq) 6678 { 6679 struct task_group *tg; 6680 6681 lockdep_assert_rq_held(rq); 6682 6683 rcu_read_lock(); 6684 list_for_each_entry_rcu(tg, &task_groups, list) { 6685 struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; 6686 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6687 6688 raw_spin_lock(&cfs_b->lock); 6689 cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF; 6690 raw_spin_unlock(&cfs_b->lock); 6691 } 6692 rcu_read_unlock(); 6693 } 6694 6695 /* cpu offline callback */ 6696 static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) 6697 { 6698 struct task_group *tg; 6699 6700 lockdep_assert_rq_held(rq); 6701 6702 // Do not unthrottle for an active CPU 6703 if (cpumask_test_cpu(cpu_of(rq), cpu_active_mask)) 6704 return; 6705 6706 /* 6707 * The rq clock has already been updated in the 6708 * set_rq_offline(), so we should skip updating 6709 * the rq clock again in unthrottle_cfs_rq(). 6710 */ 6711 rq_clock_start_loop_update(rq); 6712 6713 rcu_read_lock(); 6714 list_for_each_entry_rcu(tg, &task_groups, list) { 6715 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)]; 6716 6717 if (!cfs_rq->runtime_enabled) 6718 continue; 6719 6720 /* 6721 * Offline rq is schedulable till CPU is completely disabled 6722 * in take_cpu_down(), so we prevent new cfs throttling here. 6723 */ 6724 cfs_rq->runtime_enabled = 0; 6725 6726 if (!cfs_rq_throttled(cfs_rq)) 6727 continue; 6728 6729 /* 6730 * clock_task is not advancing so we just need to make sure 6731 * there's some valid quota amount 6732 */ 6733 cfs_rq->runtime_remaining = 1; 6734 unthrottle_cfs_rq(cfs_rq); 6735 } 6736 rcu_read_unlock(); 6737 6738 rq_clock_stop_loop_update(rq); 6739 } 6740 6741 bool cfs_task_bw_constrained(struct task_struct *p) 6742 { 6743 struct cfs_rq *cfs_rq = task_cfs_rq(p); 6744 6745 if (!cfs_bandwidth_used()) 6746 return false; 6747 6748 if (cfs_rq->runtime_enabled || 6749 tg_cfs_bandwidth(cfs_rq->tg)->hierarchical_quota != RUNTIME_INF) 6750 return true; 6751 6752 return false; 6753 } 6754 6755 #ifdef CONFIG_NO_HZ_FULL 6756 /* called from pick_next_task_fair() */ 6757 static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) 6758 { 6759 int cpu = cpu_of(rq); 6760 6761 if (!cfs_bandwidth_used()) 6762 return; 6763 6764 if (!tick_nohz_full_cpu(cpu)) 6765 return; 6766 6767 if (rq->nr_running != 1) 6768 return; 6769 6770 /* 6771 * We know there is only one task runnable and we've just picked it. The 6772 * normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will 6773 * be otherwise able to stop the tick. Just need to check if we are using 6774 * bandwidth control. 6775 */ 6776 if (cfs_task_bw_constrained(p)) 6777 tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED); 6778 } 6779 #endif /* CONFIG_NO_HZ_FULL */ 6780 6781 #else /* !CONFIG_CFS_BANDWIDTH: */ 6782 6783 static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {} 6784 static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; } 6785 static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {} 6786 static inline void sync_throttle(struct task_group *tg, int cpu) {} 6787 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6788 static void task_throttle_setup_work(struct task_struct *p) {} 6789 static bool task_is_throttled(struct task_struct *p) { return false; } 6790 static void dequeue_throttled_task(struct task_struct *p, int flags) {} 6791 static bool enqueue_throttled_task(struct task_struct *p) { return false; } 6792 static void record_throttle_clock(struct cfs_rq *cfs_rq) {} 6793 6794 static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq) 6795 { 6796 return 0; 6797 } 6798 6799 static inline bool cfs_rq_pelt_clock_throttled(struct cfs_rq *cfs_rq) 6800 { 6801 return false; 6802 } 6803 6804 static inline int throttled_hierarchy(struct cfs_rq *cfs_rq) 6805 { 6806 return 0; 6807 } 6808 6809 static inline int lb_throttled_hierarchy(struct task_struct *p, int dst_cpu) 6810 { 6811 return 0; 6812 } 6813 6814 #ifdef CONFIG_FAIR_GROUP_SCHED 6815 void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {} 6816 static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} 6817 #endif 6818 6819 static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) 6820 { 6821 return NULL; 6822 } 6823 static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} 6824 static inline void update_runtime_enabled(struct rq *rq) {} 6825 static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {} 6826 #ifdef CONFIG_CGROUP_SCHED 6827 bool cfs_task_bw_constrained(struct task_struct *p) 6828 { 6829 return false; 6830 } 6831 #endif 6832 #endif /* !CONFIG_CFS_BANDWIDTH */ 6833 6834 #if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL) 6835 static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {} 6836 #endif 6837 6838 /************************************************** 6839 * CFS operations on tasks: 6840 */ 6841 6842 #ifdef CONFIG_SCHED_HRTICK 6843 static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 6844 { 6845 struct sched_entity *se = &p->se; 6846 6847 WARN_ON_ONCE(task_rq(p) != rq); 6848 6849 if (rq->cfs.h_nr_queued > 1) { 6850 u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime; 6851 u64 slice = se->slice; 6852 s64 delta = slice - ran; 6853 6854 if (delta < 0) { 6855 if (task_current_donor(rq, p)) 6856 resched_curr(rq); 6857 return; 6858 } 6859 hrtick_start(rq, delta); 6860 } 6861 } 6862 6863 /* 6864 * called from enqueue/dequeue and updates the hrtick when the 6865 * current task is from our class and nr_running is low enough 6866 * to matter. 6867 */ 6868 static void hrtick_update(struct rq *rq) 6869 { 6870 struct task_struct *donor = rq->donor; 6871 6872 if (!hrtick_enabled_fair(rq) || donor->sched_class != &fair_sched_class) 6873 return; 6874 6875 hrtick_start_fair(rq, donor); 6876 } 6877 #else /* !CONFIG_SCHED_HRTICK: */ 6878 static inline void 6879 hrtick_start_fair(struct rq *rq, struct task_struct *p) 6880 { 6881 } 6882 6883 static inline void hrtick_update(struct rq *rq) 6884 { 6885 } 6886 #endif /* !CONFIG_SCHED_HRTICK */ 6887 6888 static inline bool cpu_overutilized(int cpu) 6889 { 6890 unsigned long rq_util_min, rq_util_max; 6891 6892 if (!sched_energy_enabled()) 6893 return false; 6894 6895 rq_util_min = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MIN); 6896 rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX); 6897 6898 /* Return true only if the utilization doesn't fit CPU's capacity */ 6899 return !util_fits_cpu(cpu_util_cfs(cpu), rq_util_min, rq_util_max, cpu); 6900 } 6901 6902 /* 6903 * overutilized value make sense only if EAS is enabled 6904 */ 6905 static inline bool is_rd_overutilized(struct root_domain *rd) 6906 { 6907 return !sched_energy_enabled() || READ_ONCE(rd->overutilized); 6908 } 6909 6910 static inline void set_rd_overutilized(struct root_domain *rd, bool flag) 6911 { 6912 if (!sched_energy_enabled()) 6913 return; 6914 6915 WRITE_ONCE(rd->overutilized, flag); 6916 trace_sched_overutilized_tp(rd, flag); 6917 } 6918 6919 static inline void check_update_overutilized_status(struct rq *rq) 6920 { 6921 /* 6922 * overutilized field is used for load balancing decisions only 6923 * if energy aware scheduler is being used 6924 */ 6925 6926 if (!is_rd_overutilized(rq->rd) && cpu_overutilized(rq->cpu)) 6927 set_rd_overutilized(rq->rd, 1); 6928 } 6929 6930 /* Runqueue only has SCHED_IDLE tasks enqueued */ 6931 static int sched_idle_rq(struct rq *rq) 6932 { 6933 return unlikely(rq->nr_running == rq->cfs.h_nr_idle && 6934 rq->nr_running); 6935 } 6936 6937 static int sched_idle_cpu(int cpu) 6938 { 6939 return sched_idle_rq(cpu_rq(cpu)); 6940 } 6941 6942 static void 6943 requeue_delayed_entity(struct sched_entity *se) 6944 { 6945 struct cfs_rq *cfs_rq = cfs_rq_of(se); 6946 6947 /* 6948 * se->sched_delayed should imply: se->on_rq == 1. 6949 * Because a delayed entity is one that is still on 6950 * the runqueue competing until elegibility. 6951 */ 6952 WARN_ON_ONCE(!se->sched_delayed); 6953 WARN_ON_ONCE(!se->on_rq); 6954 6955 if (sched_feat(DELAY_ZERO)) { 6956 update_entity_lag(cfs_rq, se); 6957 if (se->vlag > 0) { 6958 cfs_rq->nr_queued--; 6959 if (se != cfs_rq->curr) 6960 __dequeue_entity(cfs_rq, se); 6961 se->vlag = 0; 6962 place_entity(cfs_rq, se, 0); 6963 if (se != cfs_rq->curr) 6964 __enqueue_entity(cfs_rq, se); 6965 cfs_rq->nr_queued++; 6966 } 6967 } 6968 6969 update_load_avg(cfs_rq, se, 0); 6970 clear_delayed(se); 6971 } 6972 6973 /* 6974 * The enqueue_task method is called before nr_running is 6975 * increased. Here we update the fair scheduling stats and 6976 * then put the task into the rbtree: 6977 */ 6978 static void 6979 enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) 6980 { 6981 struct cfs_rq *cfs_rq; 6982 struct sched_entity *se = &p->se; 6983 int h_nr_idle = task_has_idle_policy(p); 6984 int h_nr_runnable = 1; 6985 int task_new = !(flags & ENQUEUE_WAKEUP); 6986 int rq_h_nr_queued = rq->cfs.h_nr_queued; 6987 u64 slice = 0; 6988 6989 if (task_is_throttled(p) && enqueue_throttled_task(p)) 6990 return; 6991 6992 /* 6993 * The code below (indirectly) updates schedutil which looks at 6994 * the cfs_rq utilization to select a frequency. 6995 * Let's add the task's estimated utilization to the cfs_rq's 6996 * estimated utilization, before we update schedutil. 6997 */ 6998 if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) 6999 util_est_enqueue(&rq->cfs, p); 7000 7001 if (flags & ENQUEUE_DELAYED) { 7002 requeue_delayed_entity(se); 7003 return; 7004 } 7005 7006 /* 7007 * If in_iowait is set, the code below may not trigger any cpufreq 7008 * utilization updates, so do it here explicitly with the IOWAIT flag 7009 * passed. 7010 */ 7011 if (p->in_iowait) 7012 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); 7013 7014 if (task_new && se->sched_delayed) 7015 h_nr_runnable = 0; 7016 7017 for_each_sched_entity(se) { 7018 if (se->on_rq) { 7019 if (se->sched_delayed) 7020 requeue_delayed_entity(se); 7021 break; 7022 } 7023 cfs_rq = cfs_rq_of(se); 7024 7025 /* 7026 * Basically set the slice of group entries to the min_slice of 7027 * their respective cfs_rq. This ensures the group can service 7028 * its entities in the desired time-frame. 7029 */ 7030 if (slice) { 7031 se->slice = slice; 7032 se->custom_slice = 1; 7033 } 7034 enqueue_entity(cfs_rq, se, flags); 7035 slice = cfs_rq_min_slice(cfs_rq); 7036 7037 cfs_rq->h_nr_runnable += h_nr_runnable; 7038 cfs_rq->h_nr_queued++; 7039 cfs_rq->h_nr_idle += h_nr_idle; 7040 7041 if (cfs_rq_is_idle(cfs_rq)) 7042 h_nr_idle = 1; 7043 7044 flags = ENQUEUE_WAKEUP; 7045 } 7046 7047 for_each_sched_entity(se) { 7048 cfs_rq = cfs_rq_of(se); 7049 7050 update_load_avg(cfs_rq, se, UPDATE_TG); 7051 se_update_runnable(se); 7052 update_cfs_group(se); 7053 7054 se->slice = slice; 7055 if (se != cfs_rq->curr) 7056 min_vruntime_cb_propagate(&se->run_node, NULL); 7057 slice = cfs_rq_min_slice(cfs_rq); 7058 7059 cfs_rq->h_nr_runnable += h_nr_runnable; 7060 cfs_rq->h_nr_queued++; 7061 cfs_rq->h_nr_idle += h_nr_idle; 7062 7063 if (cfs_rq_is_idle(cfs_rq)) 7064 h_nr_idle = 1; 7065 } 7066 7067 if (!rq_h_nr_queued && rq->cfs.h_nr_queued) { 7068 /* Account for idle runtime */ 7069 if (!rq->nr_running) 7070 dl_server_update_idle_time(rq, rq->curr); 7071 dl_server_start(&rq->fair_server); 7072 } 7073 7074 /* At this point se is NULL and we are at root level*/ 7075 add_nr_running(rq, 1); 7076 7077 /* 7078 * Since new tasks are assigned an initial util_avg equal to 7079 * half of the spare capacity of their CPU, tiny tasks have the 7080 * ability to cross the overutilized threshold, which will 7081 * result in the load balancer ruining all the task placement 7082 * done by EAS. As a way to mitigate that effect, do not account 7083 * for the first enqueue operation of new tasks during the 7084 * overutilized flag detection. 7085 * 7086 * A better way of solving this problem would be to wait for 7087 * the PELT signals of tasks to converge before taking them 7088 * into account, but that is not straightforward to implement, 7089 * and the following generally works well enough in practice. 7090 */ 7091 if (!task_new) 7092 check_update_overutilized_status(rq); 7093 7094 assert_list_leaf_cfs_rq(rq); 7095 7096 hrtick_update(rq); 7097 } 7098 7099 /* 7100 * Basically dequeue_task_fair(), except it can deal with dequeue_entity() 7101 * failing half-way through and resume the dequeue later. 7102 * 7103 * Returns: 7104 * -1 - dequeue delayed 7105 * 0 - dequeue throttled 7106 * 1 - dequeue complete 7107 */ 7108 static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) 7109 { 7110 bool was_sched_idle = sched_idle_rq(rq); 7111 bool task_sleep = flags & DEQUEUE_SLEEP; 7112 bool task_delayed = flags & DEQUEUE_DELAYED; 7113 bool task_throttled = flags & DEQUEUE_THROTTLE; 7114 struct task_struct *p = NULL; 7115 int h_nr_idle = 0; 7116 int h_nr_queued = 0; 7117 int h_nr_runnable = 0; 7118 struct cfs_rq *cfs_rq; 7119 u64 slice = 0; 7120 7121 if (entity_is_task(se)) { 7122 p = task_of(se); 7123 h_nr_queued = 1; 7124 h_nr_idle = task_has_idle_policy(p); 7125 if (task_sleep || task_delayed || !se->sched_delayed) 7126 h_nr_runnable = 1; 7127 } 7128 7129 for_each_sched_entity(se) { 7130 cfs_rq = cfs_rq_of(se); 7131 7132 if (!dequeue_entity(cfs_rq, se, flags)) { 7133 if (p && &p->se == se) 7134 return -1; 7135 7136 slice = cfs_rq_min_slice(cfs_rq); 7137 break; 7138 } 7139 7140 cfs_rq->h_nr_runnable -= h_nr_runnable; 7141 cfs_rq->h_nr_queued -= h_nr_queued; 7142 cfs_rq->h_nr_idle -= h_nr_idle; 7143 7144 if (cfs_rq_is_idle(cfs_rq)) 7145 h_nr_idle = h_nr_queued; 7146 7147 if (throttled_hierarchy(cfs_rq) && task_throttled) 7148 record_throttle_clock(cfs_rq); 7149 7150 /* Don't dequeue parent if it has other entities besides us */ 7151 if (cfs_rq->load.weight) { 7152 slice = cfs_rq_min_slice(cfs_rq); 7153 7154 /* Avoid re-evaluating load for this entity: */ 7155 se = parent_entity(se); 7156 /* 7157 * Bias pick_next to pick a task from this cfs_rq, as 7158 * p is sleeping when it is within its sched_slice. 7159 */ 7160 if (task_sleep && se) 7161 set_next_buddy(se); 7162 break; 7163 } 7164 flags |= DEQUEUE_SLEEP; 7165 flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); 7166 } 7167 7168 for_each_sched_entity(se) { 7169 cfs_rq = cfs_rq_of(se); 7170 7171 update_load_avg(cfs_rq, se, UPDATE_TG); 7172 se_update_runnable(se); 7173 update_cfs_group(se); 7174 7175 se->slice = slice; 7176 if (se != cfs_rq->curr) 7177 min_vruntime_cb_propagate(&se->run_node, NULL); 7178 slice = cfs_rq_min_slice(cfs_rq); 7179 7180 cfs_rq->h_nr_runnable -= h_nr_runnable; 7181 cfs_rq->h_nr_queued -= h_nr_queued; 7182 cfs_rq->h_nr_idle -= h_nr_idle; 7183 7184 if (cfs_rq_is_idle(cfs_rq)) 7185 h_nr_idle = h_nr_queued; 7186 7187 if (throttled_hierarchy(cfs_rq) && task_throttled) 7188 record_throttle_clock(cfs_rq); 7189 } 7190 7191 sub_nr_running(rq, h_nr_queued); 7192 7193 /* balance early to pull high priority tasks */ 7194 if (unlikely(!was_sched_idle && sched_idle_rq(rq))) 7195 rq->next_balance = jiffies; 7196 7197 if (p && task_delayed) { 7198 WARN_ON_ONCE(!task_sleep); 7199 WARN_ON_ONCE(p->on_rq != 1); 7200 7201 /* Fix-up what dequeue_task_fair() skipped */ 7202 hrtick_update(rq); 7203 7204 /* 7205 * Fix-up what block_task() skipped. 7206 * 7207 * Must be last, @p might not be valid after this. 7208 */ 7209 __block_task(rq, p); 7210 } 7211 7212 return 1; 7213 } 7214 7215 /* 7216 * The dequeue_task method is called before nr_running is 7217 * decreased. We remove the task from the rbtree and 7218 * update the fair scheduling stats: 7219 */ 7220 static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) 7221 { 7222 if (task_is_throttled(p)) { 7223 dequeue_throttled_task(p, flags); 7224 return true; 7225 } 7226 7227 if (!p->se.sched_delayed) 7228 util_est_dequeue(&rq->cfs, p); 7229 7230 util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP); 7231 if (dequeue_entities(rq, &p->se, flags) < 0) 7232 return false; 7233 7234 /* 7235 * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). 7236 */ 7237 7238 hrtick_update(rq); 7239 return true; 7240 } 7241 7242 static inline unsigned int cfs_h_nr_delayed(struct rq *rq) 7243 { 7244 return (rq->cfs.h_nr_queued - rq->cfs.h_nr_runnable); 7245 } 7246 7247 /* Working cpumask for: sched_balance_rq(), sched_balance_newidle(). */ 7248 static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask); 7249 static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask); 7250 static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask); 7251 7252 #ifdef CONFIG_NO_HZ_COMMON 7253 7254 static struct { 7255 cpumask_var_t idle_cpus_mask; 7256 atomic_t nr_cpus; 7257 int has_blocked; /* Idle CPUS has blocked load */ 7258 int needs_update; /* Newly idle CPUs need their next_balance collated */ 7259 unsigned long next_balance; /* in jiffy units */ 7260 unsigned long next_blocked; /* Next update of blocked load in jiffies */ 7261 } nohz ____cacheline_aligned; 7262 7263 #endif /* CONFIG_NO_HZ_COMMON */ 7264 7265 static unsigned long cpu_load(struct rq *rq) 7266 { 7267 return cfs_rq_load_avg(&rq->cfs); 7268 } 7269 7270 /* 7271 * cpu_load_without - compute CPU load without any contributions from *p 7272 * @cpu: the CPU which load is requested 7273 * @p: the task which load should be discounted 7274 * 7275 * The load of a CPU is defined by the load of tasks currently enqueued on that 7276 * CPU as well as tasks which are currently sleeping after an execution on that 7277 * CPU. 7278 * 7279 * This method returns the load of the specified CPU by discounting the load of 7280 * the specified task, whenever the task is currently contributing to the CPU 7281 * load. 7282 */ 7283 static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p) 7284 { 7285 struct cfs_rq *cfs_rq; 7286 unsigned int load; 7287 7288 /* Task has no contribution or is new */ 7289 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7290 return cpu_load(rq); 7291 7292 cfs_rq = &rq->cfs; 7293 load = READ_ONCE(cfs_rq->avg.load_avg); 7294 7295 /* Discount task's util from CPU's util */ 7296 lsub_positive(&load, task_h_load(p)); 7297 7298 return load; 7299 } 7300 7301 static unsigned long cpu_runnable(struct rq *rq) 7302 { 7303 return cfs_rq_runnable_avg(&rq->cfs); 7304 } 7305 7306 static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p) 7307 { 7308 struct cfs_rq *cfs_rq; 7309 unsigned int runnable; 7310 7311 /* Task has no contribution or is new */ 7312 if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 7313 return cpu_runnable(rq); 7314 7315 cfs_rq = &rq->cfs; 7316 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 7317 7318 /* Discount task's runnable from CPU's runnable */ 7319 lsub_positive(&runnable, p->se.avg.runnable_avg); 7320 7321 return runnable; 7322 } 7323 7324 static unsigned long capacity_of(int cpu) 7325 { 7326 return cpu_rq(cpu)->cpu_capacity; 7327 } 7328 7329 static void record_wakee(struct task_struct *p) 7330 { 7331 /* 7332 * Only decay a single time; tasks that have less then 1 wakeup per 7333 * jiffy will not have built up many flips. 7334 */ 7335 if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) { 7336 current->wakee_flips >>= 1; 7337 current->wakee_flip_decay_ts = jiffies; 7338 } 7339 7340 if (current->last_wakee != p) { 7341 current->last_wakee = p; 7342 current->wakee_flips++; 7343 } 7344 } 7345 7346 /* 7347 * Detect M:N waker/wakee relationships via a switching-frequency heuristic. 7348 * 7349 * A waker of many should wake a different task than the one last awakened 7350 * at a frequency roughly N times higher than one of its wakees. 7351 * 7352 * In order to determine whether we should let the load spread vs consolidating 7353 * to shared cache, we look for a minimum 'flip' frequency of llc_size in one 7354 * partner, and a factor of lls_size higher frequency in the other. 7355 * 7356 * With both conditions met, we can be relatively sure that the relationship is 7357 * non-monogamous, with partner count exceeding socket size. 7358 * 7359 * Waker/wakee being client/server, worker/dispatcher, interrupt source or 7360 * whatever is irrelevant, spread criteria is apparent partner count exceeds 7361 * socket size. 7362 */ 7363 static int wake_wide(struct task_struct *p) 7364 { 7365 unsigned int master = current->wakee_flips; 7366 unsigned int slave = p->wakee_flips; 7367 int factor = __this_cpu_read(sd_llc_size); 7368 7369 if (master < slave) 7370 swap(master, slave); 7371 if (slave < factor || master < slave * factor) 7372 return 0; 7373 return 1; 7374 } 7375 7376 /* 7377 * The purpose of wake_affine() is to quickly determine on which CPU we can run 7378 * soonest. For the purpose of speed we only consider the waking and previous 7379 * CPU. 7380 * 7381 * wake_affine_idle() - only considers 'now', it check if the waking CPU is 7382 * cache-affine and is (or will be) idle. 7383 * 7384 * wake_affine_weight() - considers the weight to reflect the average 7385 * scheduling latency of the CPUs. This seems to work 7386 * for the overloaded case. 7387 */ 7388 static int 7389 wake_affine_idle(int this_cpu, int prev_cpu, int sync) 7390 { 7391 /* 7392 * If this_cpu is idle, it implies the wakeup is from interrupt 7393 * context. Only allow the move if cache is shared. Otherwise an 7394 * interrupt intensive workload could force all tasks onto one 7395 * node depending on the IO topology or IRQ affinity settings. 7396 * 7397 * If the prev_cpu is idle and cache affine then avoid a migration. 7398 * There is no guarantee that the cache hot data from an interrupt 7399 * is more important than cache hot data on the prev_cpu and from 7400 * a cpufreq perspective, it's better to have higher utilisation 7401 * on one CPU. 7402 */ 7403 if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu)) 7404 return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu; 7405 7406 if (sync) { 7407 struct rq *rq = cpu_rq(this_cpu); 7408 7409 if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) 7410 return this_cpu; 7411 } 7412 7413 if (available_idle_cpu(prev_cpu)) 7414 return prev_cpu; 7415 7416 return nr_cpumask_bits; 7417 } 7418 7419 static int 7420 wake_affine_weight(struct sched_domain *sd, struct task_struct *p, 7421 int this_cpu, int prev_cpu, int sync) 7422 { 7423 s64 this_eff_load, prev_eff_load; 7424 unsigned long task_load; 7425 7426 this_eff_load = cpu_load(cpu_rq(this_cpu)); 7427 7428 if (sync) { 7429 unsigned long current_load = task_h_load(current); 7430 7431 if (current_load > this_eff_load) 7432 return this_cpu; 7433 7434 this_eff_load -= current_load; 7435 } 7436 7437 task_load = task_h_load(p); 7438 7439 this_eff_load += task_load; 7440 if (sched_feat(WA_BIAS)) 7441 this_eff_load *= 100; 7442 this_eff_load *= capacity_of(prev_cpu); 7443 7444 prev_eff_load = cpu_load(cpu_rq(prev_cpu)); 7445 prev_eff_load -= task_load; 7446 if (sched_feat(WA_BIAS)) 7447 prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2; 7448 prev_eff_load *= capacity_of(this_cpu); 7449 7450 /* 7451 * If sync, adjust the weight of prev_eff_load such that if 7452 * prev_eff == this_eff that select_idle_sibling() will consider 7453 * stacking the wakee on top of the waker if no other CPU is 7454 * idle. 7455 */ 7456 if (sync) 7457 prev_eff_load += 1; 7458 7459 return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits; 7460 } 7461 7462 static int wake_affine(struct sched_domain *sd, struct task_struct *p, 7463 int this_cpu, int prev_cpu, int sync) 7464 { 7465 int target = nr_cpumask_bits; 7466 7467 if (sched_feat(WA_IDLE)) 7468 target = wake_affine_idle(this_cpu, prev_cpu, sync); 7469 7470 if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits) 7471 target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync); 7472 7473 schedstat_inc(p->stats.nr_wakeups_affine_attempts); 7474 if (target != this_cpu) 7475 return prev_cpu; 7476 7477 schedstat_inc(sd->ttwu_move_affine); 7478 schedstat_inc(p->stats.nr_wakeups_affine); 7479 return target; 7480 } 7481 7482 static struct sched_group * 7483 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); 7484 7485 /* 7486 * sched_balance_find_dst_group_cpu - find the idlest CPU among the CPUs in the group. 7487 */ 7488 static int 7489 sched_balance_find_dst_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) 7490 { 7491 unsigned long load, min_load = ULONG_MAX; 7492 unsigned int min_exit_latency = UINT_MAX; 7493 u64 latest_idle_timestamp = 0; 7494 int least_loaded_cpu = this_cpu; 7495 int shallowest_idle_cpu = -1; 7496 int i; 7497 7498 /* Check if we have any choice: */ 7499 if (group->group_weight == 1) 7500 return cpumask_first(sched_group_span(group)); 7501 7502 /* Traverse only the allowed CPUs */ 7503 for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) { 7504 struct rq *rq = cpu_rq(i); 7505 7506 if (!sched_core_cookie_match(rq, p)) 7507 continue; 7508 7509 if (sched_idle_cpu(i)) 7510 return i; 7511 7512 if (available_idle_cpu(i)) { 7513 struct cpuidle_state *idle = idle_get_state(rq); 7514 if (idle && idle->exit_latency < min_exit_latency) { 7515 /* 7516 * We give priority to a CPU whose idle state 7517 * has the smallest exit latency irrespective 7518 * of any idle timestamp. 7519 */ 7520 min_exit_latency = idle->exit_latency; 7521 latest_idle_timestamp = rq->idle_stamp; 7522 shallowest_idle_cpu = i; 7523 } else if ((!idle || idle->exit_latency == min_exit_latency) && 7524 rq->idle_stamp > latest_idle_timestamp) { 7525 /* 7526 * If equal or no active idle state, then 7527 * the most recently idled CPU might have 7528 * a warmer cache. 7529 */ 7530 latest_idle_timestamp = rq->idle_stamp; 7531 shallowest_idle_cpu = i; 7532 } 7533 } else if (shallowest_idle_cpu == -1) { 7534 load = cpu_load(cpu_rq(i)); 7535 if (load < min_load) { 7536 min_load = load; 7537 least_loaded_cpu = i; 7538 } 7539 } 7540 } 7541 7542 return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu; 7543 } 7544 7545 static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p, 7546 int cpu, int prev_cpu, int sd_flag) 7547 { 7548 int new_cpu = cpu; 7549 7550 if (!cpumask_intersects(sched_domain_span(sd), p->cpus_ptr)) 7551 return prev_cpu; 7552 7553 /* 7554 * We need task's util for cpu_util_without, sync it up to 7555 * prev_cpu's last_update_time. 7556 */ 7557 if (!(sd_flag & SD_BALANCE_FORK)) 7558 sync_entity_load_avg(&p->se); 7559 7560 while (sd) { 7561 struct sched_group *group; 7562 struct sched_domain *tmp; 7563 int weight; 7564 7565 if (!(sd->flags & sd_flag)) { 7566 sd = sd->child; 7567 continue; 7568 } 7569 7570 group = sched_balance_find_dst_group(sd, p, cpu); 7571 if (!group) { 7572 sd = sd->child; 7573 continue; 7574 } 7575 7576 new_cpu = sched_balance_find_dst_group_cpu(group, p, cpu); 7577 if (new_cpu == cpu) { 7578 /* Now try balancing at a lower domain level of 'cpu': */ 7579 sd = sd->child; 7580 continue; 7581 } 7582 7583 /* Now try balancing at a lower domain level of 'new_cpu': */ 7584 cpu = new_cpu; 7585 weight = sd->span_weight; 7586 sd = NULL; 7587 for_each_domain(cpu, tmp) { 7588 if (weight <= tmp->span_weight) 7589 break; 7590 if (tmp->flags & sd_flag) 7591 sd = tmp; 7592 } 7593 } 7594 7595 return new_cpu; 7596 } 7597 7598 static inline int __select_idle_cpu(int cpu, struct task_struct *p) 7599 { 7600 if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) && 7601 sched_cpu_cookie_match(cpu_rq(cpu), p)) 7602 return cpu; 7603 7604 return -1; 7605 } 7606 7607 #ifdef CONFIG_SCHED_SMT 7608 DEFINE_STATIC_KEY_FALSE(sched_smt_present); 7609 EXPORT_SYMBOL_GPL(sched_smt_present); 7610 7611 static inline void set_idle_cores(int cpu, int val) 7612 { 7613 struct sched_domain_shared *sds; 7614 7615 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7616 if (sds) 7617 WRITE_ONCE(sds->has_idle_cores, val); 7618 } 7619 7620 static inline bool test_idle_cores(int cpu) 7621 { 7622 struct sched_domain_shared *sds; 7623 7624 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu)); 7625 if (sds) 7626 return READ_ONCE(sds->has_idle_cores); 7627 7628 return false; 7629 } 7630 7631 /* 7632 * Scans the local SMT mask to see if the entire core is idle, and records this 7633 * information in sd_llc_shared->has_idle_cores. 7634 * 7635 * Since SMT siblings share all cache levels, inspecting this limited remote 7636 * state should be fairly cheap. 7637 */ 7638 void __update_idle_core(struct rq *rq) 7639 { 7640 int core = cpu_of(rq); 7641 int cpu; 7642 7643 rcu_read_lock(); 7644 if (test_idle_cores(core)) 7645 goto unlock; 7646 7647 for_each_cpu(cpu, cpu_smt_mask(core)) { 7648 if (cpu == core) 7649 continue; 7650 7651 if (!available_idle_cpu(cpu)) 7652 goto unlock; 7653 } 7654 7655 set_idle_cores(core, 1); 7656 unlock: 7657 rcu_read_unlock(); 7658 } 7659 7660 /* 7661 * Scan the entire LLC domain for idle cores; this dynamically switches off if 7662 * there are no idle cores left in the system; tracked through 7663 * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above. 7664 */ 7665 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7666 { 7667 bool idle = true; 7668 int cpu; 7669 7670 for_each_cpu(cpu, cpu_smt_mask(core)) { 7671 if (!available_idle_cpu(cpu)) { 7672 idle = false; 7673 if (*idle_cpu == -1) { 7674 if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpus)) { 7675 *idle_cpu = cpu; 7676 break; 7677 } 7678 continue; 7679 } 7680 break; 7681 } 7682 if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpus)) 7683 *idle_cpu = cpu; 7684 } 7685 7686 if (idle) 7687 return core; 7688 7689 cpumask_andnot(cpus, cpus, cpu_smt_mask(core)); 7690 return -1; 7691 } 7692 7693 /* 7694 * Scan the local SMT mask for idle CPUs. 7695 */ 7696 static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7697 { 7698 int cpu; 7699 7700 for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) { 7701 if (cpu == target) 7702 continue; 7703 /* 7704 * Check if the CPU is in the LLC scheduling domain of @target. 7705 * Due to isolcpus, there is no guarantee that all the siblings are in the domain. 7706 */ 7707 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 7708 continue; 7709 if (available_idle_cpu(cpu) || sched_idle_cpu(cpu)) 7710 return cpu; 7711 } 7712 7713 return -1; 7714 } 7715 7716 #else /* !CONFIG_SCHED_SMT: */ 7717 7718 static inline void set_idle_cores(int cpu, int val) 7719 { 7720 } 7721 7722 static inline bool test_idle_cores(int cpu) 7723 { 7724 return false; 7725 } 7726 7727 static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) 7728 { 7729 return __select_idle_cpu(core, p); 7730 } 7731 7732 static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) 7733 { 7734 return -1; 7735 } 7736 7737 #endif /* !CONFIG_SCHED_SMT */ 7738 7739 /* 7740 * Scan the LLC domain for idle CPUs; this is dynamically regulated by 7741 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the 7742 * average idle time for this rq (as found in rq->avg_idle). 7743 */ 7744 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target) 7745 { 7746 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7747 int i, cpu, idle_cpu = -1, nr = INT_MAX; 7748 struct sched_domain_shared *sd_share; 7749 7750 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7751 7752 if (sched_feat(SIS_UTIL)) { 7753 sd_share = rcu_dereference(per_cpu(sd_llc_shared, target)); 7754 if (sd_share) { 7755 /* because !--nr is the condition to stop scan */ 7756 nr = READ_ONCE(sd_share->nr_idle_scan) + 1; 7757 /* overloaded LLC is unlikely to have idle cpu/core */ 7758 if (nr == 1) 7759 return -1; 7760 } 7761 } 7762 7763 if (static_branch_unlikely(&sched_cluster_active)) { 7764 struct sched_group *sg = sd->groups; 7765 7766 if (sg->flags & SD_CLUSTER) { 7767 for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) { 7768 if (!cpumask_test_cpu(cpu, cpus)) 7769 continue; 7770 7771 if (has_idle_core) { 7772 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7773 if ((unsigned int)i < nr_cpumask_bits) 7774 return i; 7775 } else { 7776 if (--nr <= 0) 7777 return -1; 7778 idle_cpu = __select_idle_cpu(cpu, p); 7779 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7780 return idle_cpu; 7781 } 7782 } 7783 cpumask_andnot(cpus, cpus, sched_group_span(sg)); 7784 } 7785 } 7786 7787 for_each_cpu_wrap(cpu, cpus, target + 1) { 7788 if (has_idle_core) { 7789 i = select_idle_core(p, cpu, cpus, &idle_cpu); 7790 if ((unsigned int)i < nr_cpumask_bits) 7791 return i; 7792 7793 } else { 7794 if (--nr <= 0) 7795 return -1; 7796 idle_cpu = __select_idle_cpu(cpu, p); 7797 if ((unsigned int)idle_cpu < nr_cpumask_bits) 7798 break; 7799 } 7800 } 7801 7802 if (has_idle_core) 7803 set_idle_cores(target, false); 7804 7805 return idle_cpu; 7806 } 7807 7808 /* 7809 * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which 7810 * the task fits. If no CPU is big enough, but there are idle ones, try to 7811 * maximize capacity. 7812 */ 7813 static int 7814 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target) 7815 { 7816 unsigned long task_util, util_min, util_max, best_cap = 0; 7817 int fits, best_fits = 0; 7818 int cpu, best_cpu = -1; 7819 struct cpumask *cpus; 7820 7821 cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 7822 cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr); 7823 7824 task_util = task_util_est(p); 7825 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7826 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7827 7828 for_each_cpu_wrap(cpu, cpus, target) { 7829 unsigned long cpu_cap = capacity_of(cpu); 7830 7831 if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu)) 7832 continue; 7833 7834 fits = util_fits_cpu(task_util, util_min, util_max, cpu); 7835 7836 /* This CPU fits with all requirements */ 7837 if (fits > 0) 7838 return cpu; 7839 /* 7840 * Only the min performance hint (i.e. uclamp_min) doesn't fit. 7841 * Look for the CPU with best capacity. 7842 */ 7843 else if (fits < 0) 7844 cpu_cap = get_actual_cpu_capacity(cpu); 7845 7846 /* 7847 * First, select CPU which fits better (-1 being better than 0). 7848 * Then, select the one with best capacity at same level. 7849 */ 7850 if ((fits < best_fits) || 7851 ((fits == best_fits) && (cpu_cap > best_cap))) { 7852 best_cap = cpu_cap; 7853 best_cpu = cpu; 7854 best_fits = fits; 7855 } 7856 } 7857 7858 return best_cpu; 7859 } 7860 7861 static inline bool asym_fits_cpu(unsigned long util, 7862 unsigned long util_min, 7863 unsigned long util_max, 7864 int cpu) 7865 { 7866 if (sched_asym_cpucap_active()) 7867 /* 7868 * Return true only if the cpu fully fits the task requirements 7869 * which include the utilization and the performance hints. 7870 */ 7871 return (util_fits_cpu(util, util_min, util_max, cpu) > 0); 7872 7873 return true; 7874 } 7875 7876 /* 7877 * Try and locate an idle core/thread in the LLC cache domain. 7878 */ 7879 static int select_idle_sibling(struct task_struct *p, int prev, int target) 7880 { 7881 bool has_idle_core = false; 7882 struct sched_domain *sd; 7883 unsigned long task_util, util_min, util_max; 7884 int i, recent_used_cpu, prev_aff = -1; 7885 7886 /* 7887 * On asymmetric system, update task utilization because we will check 7888 * that the task fits with CPU's capacity. 7889 */ 7890 if (sched_asym_cpucap_active()) { 7891 sync_entity_load_avg(&p->se); 7892 task_util = task_util_est(p); 7893 util_min = uclamp_eff_value(p, UCLAMP_MIN); 7894 util_max = uclamp_eff_value(p, UCLAMP_MAX); 7895 } 7896 7897 /* 7898 * per-cpu select_rq_mask usage 7899 */ 7900 lockdep_assert_irqs_disabled(); 7901 7902 if ((available_idle_cpu(target) || sched_idle_cpu(target)) && 7903 asym_fits_cpu(task_util, util_min, util_max, target)) 7904 return target; 7905 7906 /* 7907 * If the previous CPU is cache affine and idle, don't be stupid: 7908 */ 7909 if (prev != target && cpus_share_cache(prev, target) && 7910 (available_idle_cpu(prev) || sched_idle_cpu(prev)) && 7911 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7912 7913 if (!static_branch_unlikely(&sched_cluster_active) || 7914 cpus_share_resources(prev, target)) 7915 return prev; 7916 7917 prev_aff = prev; 7918 } 7919 7920 /* 7921 * Allow a per-cpu kthread to stack with the wakee if the 7922 * kworker thread and the tasks previous CPUs are the same. 7923 * The assumption is that the wakee queued work for the 7924 * per-cpu kthread that is now complete and the wakeup is 7925 * essentially a sync wakeup. An obvious example of this 7926 * pattern is IO completions. 7927 */ 7928 if (is_per_cpu_kthread(current) && 7929 in_task() && 7930 prev == smp_processor_id() && 7931 this_rq()->nr_running <= 1 && 7932 asym_fits_cpu(task_util, util_min, util_max, prev)) { 7933 return prev; 7934 } 7935 7936 /* Check a recently used CPU as a potential idle candidate: */ 7937 recent_used_cpu = p->recent_used_cpu; 7938 p->recent_used_cpu = prev; 7939 if (recent_used_cpu != prev && 7940 recent_used_cpu != target && 7941 cpus_share_cache(recent_used_cpu, target) && 7942 (available_idle_cpu(recent_used_cpu) || sched_idle_cpu(recent_used_cpu)) && 7943 cpumask_test_cpu(recent_used_cpu, p->cpus_ptr) && 7944 asym_fits_cpu(task_util, util_min, util_max, recent_used_cpu)) { 7945 7946 if (!static_branch_unlikely(&sched_cluster_active) || 7947 cpus_share_resources(recent_used_cpu, target)) 7948 return recent_used_cpu; 7949 7950 } else { 7951 recent_used_cpu = -1; 7952 } 7953 7954 /* 7955 * For asymmetric CPU capacity systems, our domain of interest is 7956 * sd_asym_cpucapacity rather than sd_llc. 7957 */ 7958 if (sched_asym_cpucap_active()) { 7959 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target)); 7960 /* 7961 * On an asymmetric CPU capacity system where an exclusive 7962 * cpuset defines a symmetric island (i.e. one unique 7963 * capacity_orig value through the cpuset), the key will be set 7964 * but the CPUs within that cpuset will not have a domain with 7965 * SD_ASYM_CPUCAPACITY. These should follow the usual symmetric 7966 * capacity path. 7967 */ 7968 if (sd) { 7969 i = select_idle_capacity(p, sd, target); 7970 return ((unsigned)i < nr_cpumask_bits) ? i : target; 7971 } 7972 } 7973 7974 sd = rcu_dereference(per_cpu(sd_llc, target)); 7975 if (!sd) 7976 return target; 7977 7978 if (sched_smt_active()) { 7979 has_idle_core = test_idle_cores(target); 7980 7981 if (!has_idle_core && cpus_share_cache(prev, target)) { 7982 i = select_idle_smt(p, sd, prev); 7983 if ((unsigned int)i < nr_cpumask_bits) 7984 return i; 7985 } 7986 } 7987 7988 i = select_idle_cpu(p, sd, has_idle_core, target); 7989 if ((unsigned)i < nr_cpumask_bits) 7990 return i; 7991 7992 /* 7993 * For cluster machines which have lower sharing cache like L2 or 7994 * LLC Tag, we tend to find an idle CPU in the target's cluster 7995 * first. But prev_cpu or recent_used_cpu may also be a good candidate, 7996 * use them if possible when no idle CPU found in select_idle_cpu(). 7997 */ 7998 if ((unsigned int)prev_aff < nr_cpumask_bits) 7999 return prev_aff; 8000 if ((unsigned int)recent_used_cpu < nr_cpumask_bits) 8001 return recent_used_cpu; 8002 8003 return target; 8004 } 8005 8006 /** 8007 * cpu_util() - Estimates the amount of CPU capacity used by CFS tasks. 8008 * @cpu: the CPU to get the utilization for 8009 * @p: task for which the CPU utilization should be predicted or NULL 8010 * @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL 8011 * @boost: 1 to enable boosting, otherwise 0 8012 * 8013 * The unit of the return value must be the same as the one of CPU capacity 8014 * so that CPU utilization can be compared with CPU capacity. 8015 * 8016 * CPU utilization is the sum of running time of runnable tasks plus the 8017 * recent utilization of currently non-runnable tasks on that CPU. 8018 * It represents the amount of CPU capacity currently used by CFS tasks in 8019 * the range [0..max CPU capacity] with max CPU capacity being the CPU 8020 * capacity at f_max. 8021 * 8022 * The estimated CPU utilization is defined as the maximum between CPU 8023 * utilization and sum of the estimated utilization of the currently 8024 * runnable tasks on that CPU. It preserves a utilization "snapshot" of 8025 * previously-executed tasks, which helps better deduce how busy a CPU will 8026 * be when a long-sleeping task wakes up. The contribution to CPU utilization 8027 * of such a task would be significantly decayed at this point of time. 8028 * 8029 * Boosted CPU utilization is defined as max(CPU runnable, CPU utilization). 8030 * CPU contention for CFS tasks can be detected by CPU runnable > CPU 8031 * utilization. Boosting is implemented in cpu_util() so that internal 8032 * users (e.g. EAS) can use it next to external users (e.g. schedutil), 8033 * latter via cpu_util_cfs_boost(). 8034 * 8035 * CPU utilization can be higher than the current CPU capacity 8036 * (f_curr/f_max * max CPU capacity) or even the max CPU capacity because 8037 * of rounding errors as well as task migrations or wakeups of new tasks. 8038 * CPU utilization has to be capped to fit into the [0..max CPU capacity] 8039 * range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%) 8040 * could be seen as over-utilized even though CPU1 has 20% of spare CPU 8041 * capacity. CPU utilization is allowed to overshoot current CPU capacity 8042 * though since this is useful for predicting the CPU capacity required 8043 * after task migrations (scheduler-driven DVFS). 8044 * 8045 * Return: (Boosted) (estimated) utilization for the specified CPU. 8046 */ 8047 static unsigned long 8048 cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost) 8049 { 8050 struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs; 8051 unsigned long util = READ_ONCE(cfs_rq->avg.util_avg); 8052 unsigned long runnable; 8053 8054 if (boost) { 8055 runnable = READ_ONCE(cfs_rq->avg.runnable_avg); 8056 util = max(util, runnable); 8057 } 8058 8059 /* 8060 * If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its 8061 * contribution. If @p migrates from another CPU to @cpu add its 8062 * contribution. In all the other cases @cpu is not impacted by the 8063 * migration so its util_avg is already correct. 8064 */ 8065 if (p && task_cpu(p) == cpu && dst_cpu != cpu) 8066 lsub_positive(&util, task_util(p)); 8067 else if (p && task_cpu(p) != cpu && dst_cpu == cpu) 8068 util += task_util(p); 8069 8070 if (sched_feat(UTIL_EST)) { 8071 unsigned long util_est; 8072 8073 util_est = READ_ONCE(cfs_rq->avg.util_est); 8074 8075 /* 8076 * During wake-up @p isn't enqueued yet and doesn't contribute 8077 * to any cpu_rq(cpu)->cfs.avg.util_est. 8078 * If @dst_cpu == @cpu add it to "simulate" cpu_util after @p 8079 * has been enqueued. 8080 * 8081 * During exec (@dst_cpu = -1) @p is enqueued and does 8082 * contribute to cpu_rq(cpu)->cfs.util_est. 8083 * Remove it to "simulate" cpu_util without @p's contribution. 8084 * 8085 * Despite the task_on_rq_queued(@p) check there is still a 8086 * small window for a possible race when an exec 8087 * select_task_rq_fair() races with LB's detach_task(). 8088 * 8089 * detach_task() 8090 * deactivate_task() 8091 * p->on_rq = TASK_ON_RQ_MIGRATING; 8092 * -------------------------------- A 8093 * dequeue_task() \ 8094 * dequeue_task_fair() + Race Time 8095 * util_est_dequeue() / 8096 * -------------------------------- B 8097 * 8098 * The additional check "current == p" is required to further 8099 * reduce the race window. 8100 */ 8101 if (dst_cpu == cpu) 8102 util_est += _task_util_est(p); 8103 else if (p && unlikely(task_on_rq_queued(p) || current == p)) 8104 lsub_positive(&util_est, _task_util_est(p)); 8105 8106 util = max(util, util_est); 8107 } 8108 8109 return min(util, arch_scale_cpu_capacity(cpu)); 8110 } 8111 8112 unsigned long cpu_util_cfs(int cpu) 8113 { 8114 return cpu_util(cpu, NULL, -1, 0); 8115 } 8116 8117 unsigned long cpu_util_cfs_boost(int cpu) 8118 { 8119 return cpu_util(cpu, NULL, -1, 1); 8120 } 8121 8122 /* 8123 * cpu_util_without: compute cpu utilization without any contributions from *p 8124 * @cpu: the CPU which utilization is requested 8125 * @p: the task which utilization should be discounted 8126 * 8127 * The utilization of a CPU is defined by the utilization of tasks currently 8128 * enqueued on that CPU as well as tasks which are currently sleeping after an 8129 * execution on that CPU. 8130 * 8131 * This method returns the utilization of the specified CPU by discounting the 8132 * utilization of the specified task, whenever the task is currently 8133 * contributing to the CPU utilization. 8134 */ 8135 static unsigned long cpu_util_without(int cpu, struct task_struct *p) 8136 { 8137 /* Task has no contribution or is new */ 8138 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time)) 8139 p = NULL; 8140 8141 return cpu_util(cpu, p, -1, 0); 8142 } 8143 8144 /* 8145 * This function computes an effective utilization for the given CPU, to be 8146 * used for frequency selection given the linear relation: f = u * f_max. 8147 * 8148 * The scheduler tracks the following metrics: 8149 * 8150 * cpu_util_{cfs,rt,dl,irq}() 8151 * cpu_bw_dl() 8152 * 8153 * Where the cfs,rt and dl util numbers are tracked with the same metric and 8154 * synchronized windows and are thus directly comparable. 8155 * 8156 * The cfs,rt,dl utilization are the running times measured with rq->clock_task 8157 * which excludes things like IRQ and steal-time. These latter are then accrued 8158 * in the IRQ utilization. 8159 * 8160 * The DL bandwidth number OTOH is not a measured metric but a value computed 8161 * based on the task model parameters and gives the minimal utilization 8162 * required to meet deadlines. 8163 */ 8164 unsigned long effective_cpu_util(int cpu, unsigned long util_cfs, 8165 unsigned long *min, 8166 unsigned long *max) 8167 { 8168 unsigned long util, irq, scale; 8169 struct rq *rq = cpu_rq(cpu); 8170 8171 scale = arch_scale_cpu_capacity(cpu); 8172 8173 /* 8174 * Early check to see if IRQ/steal time saturates the CPU, can be 8175 * because of inaccuracies in how we track these -- see 8176 * update_irq_load_avg(). 8177 */ 8178 irq = cpu_util_irq(rq); 8179 if (unlikely(irq >= scale)) { 8180 if (min) 8181 *min = scale; 8182 if (max) 8183 *max = scale; 8184 return scale; 8185 } 8186 8187 if (min) { 8188 /* 8189 * The minimum utilization returns the highest level between: 8190 * - the computed DL bandwidth needed with the IRQ pressure which 8191 * steals time to the deadline task. 8192 * - The minimum performance requirement for CFS and/or RT. 8193 */ 8194 *min = max(irq + cpu_bw_dl(rq), uclamp_rq_get(rq, UCLAMP_MIN)); 8195 8196 /* 8197 * When an RT task is runnable and uclamp is not used, we must 8198 * ensure that the task will run at maximum compute capacity. 8199 */ 8200 if (!uclamp_is_used() && rt_rq_is_runnable(&rq->rt)) 8201 *min = max(*min, scale); 8202 } 8203 8204 /* 8205 * Because the time spend on RT/DL tasks is visible as 'lost' time to 8206 * CFS tasks and we use the same metric to track the effective 8207 * utilization (PELT windows are synchronized) we can directly add them 8208 * to obtain the CPU's actual utilization. 8209 */ 8210 util = util_cfs + cpu_util_rt(rq); 8211 util += cpu_util_dl(rq); 8212 8213 /* 8214 * The maximum hint is a soft bandwidth requirement, which can be lower 8215 * than the actual utilization because of uclamp_max requirements. 8216 */ 8217 if (max) 8218 *max = min(scale, uclamp_rq_get(rq, UCLAMP_MAX)); 8219 8220 if (util >= scale) 8221 return scale; 8222 8223 /* 8224 * There is still idle time; further improve the number by using the 8225 * IRQ metric. Because IRQ/steal time is hidden from the task clock we 8226 * need to scale the task numbers: 8227 * 8228 * max - irq 8229 * U' = irq + --------- * U 8230 * max 8231 */ 8232 util = scale_irq_capacity(util, irq, scale); 8233 util += irq; 8234 8235 return min(scale, util); 8236 } 8237 8238 unsigned long sched_cpu_util(int cpu) 8239 { 8240 return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL); 8241 } 8242 8243 /* 8244 * energy_env - Utilization landscape for energy estimation. 8245 * @task_busy_time: Utilization contribution by the task for which we test the 8246 * placement. Given by eenv_task_busy_time(). 8247 * @pd_busy_time: Utilization of the whole perf domain without the task 8248 * contribution. Given by eenv_pd_busy_time(). 8249 * @cpu_cap: Maximum CPU capacity for the perf domain. 8250 * @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap). 8251 */ 8252 struct energy_env { 8253 unsigned long task_busy_time; 8254 unsigned long pd_busy_time; 8255 unsigned long cpu_cap; 8256 unsigned long pd_cap; 8257 }; 8258 8259 /* 8260 * Compute the task busy time for compute_energy(). This time cannot be 8261 * injected directly into effective_cpu_util() because of the IRQ scaling. 8262 * The latter only makes sense with the most recent CPUs where the task has 8263 * run. 8264 */ 8265 static inline void eenv_task_busy_time(struct energy_env *eenv, 8266 struct task_struct *p, int prev_cpu) 8267 { 8268 unsigned long busy_time, max_cap = arch_scale_cpu_capacity(prev_cpu); 8269 unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu)); 8270 8271 if (unlikely(irq >= max_cap)) 8272 busy_time = max_cap; 8273 else 8274 busy_time = scale_irq_capacity(task_util_est(p), irq, max_cap); 8275 8276 eenv->task_busy_time = busy_time; 8277 } 8278 8279 /* 8280 * Compute the perf_domain (PD) busy time for compute_energy(). Based on the 8281 * utilization for each @pd_cpus, it however doesn't take into account 8282 * clamping since the ratio (utilization / cpu_capacity) is already enough to 8283 * scale the EM reported power consumption at the (eventually clamped) 8284 * cpu_capacity. 8285 * 8286 * The contribution of the task @p for which we want to estimate the 8287 * energy cost is removed (by cpu_util()) and must be calculated 8288 * separately (see eenv_task_busy_time). This ensures: 8289 * 8290 * - A stable PD utilization, no matter which CPU of that PD we want to place 8291 * the task on. 8292 * 8293 * - A fair comparison between CPUs as the task contribution (task_util()) 8294 * will always be the same no matter which CPU utilization we rely on 8295 * (util_avg or util_est). 8296 * 8297 * Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't 8298 * exceed @eenv->pd_cap. 8299 */ 8300 static inline void eenv_pd_busy_time(struct energy_env *eenv, 8301 struct cpumask *pd_cpus, 8302 struct task_struct *p) 8303 { 8304 unsigned long busy_time = 0; 8305 int cpu; 8306 8307 for_each_cpu(cpu, pd_cpus) { 8308 unsigned long util = cpu_util(cpu, p, -1, 0); 8309 8310 busy_time += effective_cpu_util(cpu, util, NULL, NULL); 8311 } 8312 8313 eenv->pd_busy_time = min(eenv->pd_cap, busy_time); 8314 } 8315 8316 /* 8317 * Compute the maximum utilization for compute_energy() when the task @p 8318 * is placed on the cpu @dst_cpu. 8319 * 8320 * Returns the maximum utilization among @eenv->cpus. This utilization can't 8321 * exceed @eenv->cpu_cap. 8322 */ 8323 static inline unsigned long 8324 eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus, 8325 struct task_struct *p, int dst_cpu) 8326 { 8327 unsigned long max_util = 0; 8328 int cpu; 8329 8330 for_each_cpu(cpu, pd_cpus) { 8331 struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL; 8332 unsigned long util = cpu_util(cpu, p, dst_cpu, 1); 8333 unsigned long eff_util, min, max; 8334 8335 /* 8336 * Performance domain frequency: utilization clamping 8337 * must be considered since it affects the selection 8338 * of the performance domain frequency. 8339 * NOTE: in case RT tasks are running, by default the min 8340 * utilization can be max OPP. 8341 */ 8342 eff_util = effective_cpu_util(cpu, util, &min, &max); 8343 8344 /* Task's uclamp can modify min and max value */ 8345 if (tsk && uclamp_is_used()) { 8346 min = max(min, uclamp_eff_value(p, UCLAMP_MIN)); 8347 8348 /* 8349 * If there is no active max uclamp constraint, 8350 * directly use task's one, otherwise keep max. 8351 */ 8352 if (uclamp_rq_is_idle(cpu_rq(cpu))) 8353 max = uclamp_eff_value(p, UCLAMP_MAX); 8354 else 8355 max = max(max, uclamp_eff_value(p, UCLAMP_MAX)); 8356 } 8357 8358 eff_util = sugov_effective_cpu_perf(cpu, eff_util, min, max); 8359 max_util = max(max_util, eff_util); 8360 } 8361 8362 return min(max_util, eenv->cpu_cap); 8363 } 8364 8365 /* 8366 * compute_energy(): Use the Energy Model to estimate the energy that @pd would 8367 * consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task 8368 * contribution is ignored. 8369 */ 8370 static inline unsigned long 8371 compute_energy(struct energy_env *eenv, struct perf_domain *pd, 8372 struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu) 8373 { 8374 unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu); 8375 unsigned long busy_time = eenv->pd_busy_time; 8376 unsigned long energy; 8377 8378 if (dst_cpu >= 0) 8379 busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time); 8380 8381 energy = em_cpu_energy(pd->em_pd, max_util, busy_time, eenv->cpu_cap); 8382 8383 trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time); 8384 8385 return energy; 8386 } 8387 8388 /* 8389 * find_energy_efficient_cpu(): Find most energy-efficient target CPU for the 8390 * waking task. find_energy_efficient_cpu() looks for the CPU with maximum 8391 * spare capacity in each performance domain and uses it as a potential 8392 * candidate to execute the task. Then, it uses the Energy Model to figure 8393 * out which of the CPU candidates is the most energy-efficient. 8394 * 8395 * The rationale for this heuristic is as follows. In a performance domain, 8396 * all the most energy efficient CPU candidates (according to the Energy 8397 * Model) are those for which we'll request a low frequency. When there are 8398 * several CPUs for which the frequency request will be the same, we don't 8399 * have enough data to break the tie between them, because the Energy Model 8400 * only includes active power costs. With this model, if we assume that 8401 * frequency requests follow utilization (e.g. using schedutil), the CPU with 8402 * the maximum spare capacity in a performance domain is guaranteed to be among 8403 * the best candidates of the performance domain. 8404 * 8405 * In practice, it could be preferable from an energy standpoint to pack 8406 * small tasks on a CPU in order to let other CPUs go in deeper idle states, 8407 * but that could also hurt our chances to go cluster idle, and we have no 8408 * ways to tell with the current Energy Model if this is actually a good 8409 * idea or not. So, find_energy_efficient_cpu() basically favors 8410 * cluster-packing, and spreading inside a cluster. That should at least be 8411 * a good thing for latency, and this is consistent with the idea that most 8412 * of the energy savings of EAS come from the asymmetry of the system, and 8413 * not so much from breaking the tie between identical CPUs. That's also the 8414 * reason why EAS is enabled in the topology code only for systems where 8415 * SD_ASYM_CPUCAPACITY is set. 8416 * 8417 * NOTE: Forkees are not accepted in the energy-aware wake-up path because 8418 * they don't have any useful utilization data yet and it's not possible to 8419 * forecast their impact on energy consumption. Consequently, they will be 8420 * placed by sched_balance_find_dst_cpu() on the least loaded CPU, which might turn out 8421 * to be energy-inefficient in some use-cases. The alternative would be to 8422 * bias new tasks towards specific types of CPUs first, or to try to infer 8423 * their util_avg from the parent task, but those heuristics could hurt 8424 * other use-cases too. So, until someone finds a better way to solve this, 8425 * let's keep things simple by re-using the existing slow path. 8426 */ 8427 static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu) 8428 { 8429 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask); 8430 unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX; 8431 unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MIN) : 0; 8432 unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, UCLAMP_MAX) : 1024; 8433 struct root_domain *rd = this_rq()->rd; 8434 int cpu, best_energy_cpu, target = -1; 8435 int prev_fits = -1, best_fits = -1; 8436 unsigned long best_actual_cap = 0; 8437 unsigned long prev_actual_cap = 0; 8438 struct sched_domain *sd; 8439 struct perf_domain *pd; 8440 struct energy_env eenv; 8441 8442 rcu_read_lock(); 8443 pd = rcu_dereference(rd->pd); 8444 if (!pd) 8445 goto unlock; 8446 8447 /* 8448 * Energy-aware wake-up happens on the lowest sched_domain starting 8449 * from sd_asym_cpucapacity spanning over this_cpu and prev_cpu. 8450 */ 8451 sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity)); 8452 while (sd && !cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) 8453 sd = sd->parent; 8454 if (!sd) 8455 goto unlock; 8456 8457 target = prev_cpu; 8458 8459 sync_entity_load_avg(&p->se); 8460 if (!task_util_est(p) && p_util_min == 0) 8461 goto unlock; 8462 8463 eenv_task_busy_time(&eenv, p, prev_cpu); 8464 8465 for (; pd; pd = pd->next) { 8466 unsigned long util_min = p_util_min, util_max = p_util_max; 8467 unsigned long cpu_cap, cpu_actual_cap, util; 8468 long prev_spare_cap = -1, max_spare_cap = -1; 8469 unsigned long rq_util_min, rq_util_max; 8470 unsigned long cur_delta, base_energy; 8471 int max_spare_cap_cpu = -1; 8472 int fits, max_fits = -1; 8473 8474 cpumask_and(cpus, perf_domain_span(pd), cpu_online_mask); 8475 8476 if (cpumask_empty(cpus)) 8477 continue; 8478 8479 /* Account external pressure for the energy estimation */ 8480 cpu = cpumask_first(cpus); 8481 cpu_actual_cap = get_actual_cpu_capacity(cpu); 8482 8483 eenv.cpu_cap = cpu_actual_cap; 8484 eenv.pd_cap = 0; 8485 8486 for_each_cpu(cpu, cpus) { 8487 struct rq *rq = cpu_rq(cpu); 8488 8489 eenv.pd_cap += cpu_actual_cap; 8490 8491 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) 8492 continue; 8493 8494 if (!cpumask_test_cpu(cpu, p->cpus_ptr)) 8495 continue; 8496 8497 util = cpu_util(cpu, p, cpu, 0); 8498 cpu_cap = capacity_of(cpu); 8499 8500 /* 8501 * Skip CPUs that cannot satisfy the capacity request. 8502 * IOW, placing the task there would make the CPU 8503 * overutilized. Take uclamp into account to see how 8504 * much capacity we can get out of the CPU; this is 8505 * aligned with sched_cpu_util(). 8506 */ 8507 if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) { 8508 /* 8509 * Open code uclamp_rq_util_with() except for 8510 * the clamp() part. I.e.: apply max aggregation 8511 * only. util_fits_cpu() logic requires to 8512 * operate on non clamped util but must use the 8513 * max-aggregated uclamp_{min, max}. 8514 */ 8515 rq_util_min = uclamp_rq_get(rq, UCLAMP_MIN); 8516 rq_util_max = uclamp_rq_get(rq, UCLAMP_MAX); 8517 8518 util_min = max(rq_util_min, p_util_min); 8519 util_max = max(rq_util_max, p_util_max); 8520 } 8521 8522 fits = util_fits_cpu(util, util_min, util_max, cpu); 8523 if (!fits) 8524 continue; 8525 8526 lsub_positive(&cpu_cap, util); 8527 8528 if (cpu == prev_cpu) { 8529 /* Always use prev_cpu as a candidate. */ 8530 prev_spare_cap = cpu_cap; 8531 prev_fits = fits; 8532 } else if ((fits > max_fits) || 8533 ((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) { 8534 /* 8535 * Find the CPU with the maximum spare capacity 8536 * among the remaining CPUs in the performance 8537 * domain. 8538 */ 8539 max_spare_cap = cpu_cap; 8540 max_spare_cap_cpu = cpu; 8541 max_fits = fits; 8542 } 8543 } 8544 8545 if (max_spare_cap_cpu < 0 && prev_spare_cap < 0) 8546 continue; 8547 8548 eenv_pd_busy_time(&eenv, cpus, p); 8549 /* Compute the 'base' energy of the pd, without @p */ 8550 base_energy = compute_energy(&eenv, pd, cpus, p, -1); 8551 8552 /* Evaluate the energy impact of using prev_cpu. */ 8553 if (prev_spare_cap > -1) { 8554 prev_delta = compute_energy(&eenv, pd, cpus, p, 8555 prev_cpu); 8556 /* CPU utilization has changed */ 8557 if (prev_delta < base_energy) 8558 goto unlock; 8559 prev_delta -= base_energy; 8560 prev_actual_cap = cpu_actual_cap; 8561 best_delta = min(best_delta, prev_delta); 8562 } 8563 8564 /* Evaluate the energy impact of using max_spare_cap_cpu. */ 8565 if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) { 8566 /* Current best energy cpu fits better */ 8567 if (max_fits < best_fits) 8568 continue; 8569 8570 /* 8571 * Both don't fit performance hint (i.e. uclamp_min) 8572 * but best energy cpu has better capacity. 8573 */ 8574 if ((max_fits < 0) && 8575 (cpu_actual_cap <= best_actual_cap)) 8576 continue; 8577 8578 cur_delta = compute_energy(&eenv, pd, cpus, p, 8579 max_spare_cap_cpu); 8580 /* CPU utilization has changed */ 8581 if (cur_delta < base_energy) 8582 goto unlock; 8583 cur_delta -= base_energy; 8584 8585 /* 8586 * Both fit for the task but best energy cpu has lower 8587 * energy impact. 8588 */ 8589 if ((max_fits > 0) && (best_fits > 0) && 8590 (cur_delta >= best_delta)) 8591 continue; 8592 8593 best_delta = cur_delta; 8594 best_energy_cpu = max_spare_cap_cpu; 8595 best_fits = max_fits; 8596 best_actual_cap = cpu_actual_cap; 8597 } 8598 } 8599 rcu_read_unlock(); 8600 8601 if ((best_fits > prev_fits) || 8602 ((best_fits > 0) && (best_delta < prev_delta)) || 8603 ((best_fits < 0) && (best_actual_cap > prev_actual_cap))) 8604 target = best_energy_cpu; 8605 8606 return target; 8607 8608 unlock: 8609 rcu_read_unlock(); 8610 8611 return target; 8612 } 8613 8614 /* 8615 * select_task_rq_fair: Select target runqueue for the waking task in domains 8616 * that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE, 8617 * SD_BALANCE_FORK, or SD_BALANCE_EXEC. 8618 * 8619 * Balances load by selecting the idlest CPU in the idlest group, or under 8620 * certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set. 8621 * 8622 * Returns the target CPU number. 8623 */ 8624 static int 8625 select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags) 8626 { 8627 int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); 8628 struct sched_domain *tmp, *sd = NULL; 8629 int cpu = smp_processor_id(); 8630 int new_cpu = prev_cpu; 8631 int want_affine = 0; 8632 /* SD_flags and WF_flags share the first nibble */ 8633 int sd_flag = wake_flags & 0xF; 8634 8635 /* 8636 * required for stable ->cpus_allowed 8637 */ 8638 lockdep_assert_held(&p->pi_lock); 8639 if (wake_flags & WF_TTWU) { 8640 record_wakee(p); 8641 8642 if ((wake_flags & WF_CURRENT_CPU) && 8643 cpumask_test_cpu(cpu, p->cpus_ptr)) 8644 return cpu; 8645 8646 if (!is_rd_overutilized(this_rq()->rd)) { 8647 new_cpu = find_energy_efficient_cpu(p, prev_cpu); 8648 if (new_cpu >= 0) 8649 return new_cpu; 8650 new_cpu = prev_cpu; 8651 } 8652 8653 want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr); 8654 } 8655 8656 rcu_read_lock(); 8657 for_each_domain(cpu, tmp) { 8658 /* 8659 * If both 'cpu' and 'prev_cpu' are part of this domain, 8660 * cpu is a valid SD_WAKE_AFFINE target. 8661 */ 8662 if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && 8663 cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { 8664 if (cpu != prev_cpu) 8665 new_cpu = wake_affine(tmp, p, cpu, prev_cpu, sync); 8666 8667 sd = NULL; /* Prefer wake_affine over balance flags */ 8668 break; 8669 } 8670 8671 /* 8672 * Usually only true for WF_EXEC and WF_FORK, as sched_domains 8673 * usually do not have SD_BALANCE_WAKE set. That means wakeup 8674 * will usually go to the fast path. 8675 */ 8676 if (tmp->flags & sd_flag) 8677 sd = tmp; 8678 else if (!want_affine) 8679 break; 8680 } 8681 8682 if (unlikely(sd)) { 8683 /* Slow path */ 8684 new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag); 8685 } else if (wake_flags & WF_TTWU) { /* XXX always ? */ 8686 /* Fast path */ 8687 new_cpu = select_idle_sibling(p, prev_cpu, new_cpu); 8688 } 8689 rcu_read_unlock(); 8690 8691 return new_cpu; 8692 } 8693 8694 /* 8695 * Called immediately before a task is migrated to a new CPU; task_cpu(p) and 8696 * cfs_rq_of(p) references at time of call are still valid and identify the 8697 * previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held. 8698 */ 8699 static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) 8700 { 8701 struct sched_entity *se = &p->se; 8702 8703 if (!task_on_rq_migrating(p)) { 8704 remove_entity_load_avg(se); 8705 8706 /* 8707 * Here, the task's PELT values have been updated according to 8708 * the current rq's clock. But if that clock hasn't been 8709 * updated in a while, a substantial idle time will be missed, 8710 * leading to an inflation after wake-up on the new rq. 8711 * 8712 * Estimate the missing time from the cfs_rq last_update_time 8713 * and update sched_avg to improve the PELT continuity after 8714 * migration. 8715 */ 8716 migrate_se_pelt_lag(se); 8717 } 8718 8719 /* Tell new CPU we are migrated */ 8720 se->avg.last_update_time = 0; 8721 8722 update_scan_period(p, new_cpu); 8723 } 8724 8725 static void task_dead_fair(struct task_struct *p) 8726 { 8727 struct sched_entity *se = &p->se; 8728 8729 if (se->sched_delayed) { 8730 struct rq_flags rf; 8731 struct rq *rq; 8732 8733 rq = task_rq_lock(p, &rf); 8734 if (se->sched_delayed) { 8735 update_rq_clock(rq); 8736 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); 8737 } 8738 task_rq_unlock(rq, p, &rf); 8739 } 8740 8741 remove_entity_load_avg(se); 8742 } 8743 8744 /* 8745 * Set the max capacity the task is allowed to run at for misfit detection. 8746 */ 8747 static void set_task_max_allowed_capacity(struct task_struct *p) 8748 { 8749 struct asym_cap_data *entry; 8750 8751 if (!sched_asym_cpucap_active()) 8752 return; 8753 8754 rcu_read_lock(); 8755 list_for_each_entry_rcu(entry, &asym_cap_list, link) { 8756 cpumask_t *cpumask; 8757 8758 cpumask = cpu_capacity_span(entry); 8759 if (!cpumask_intersects(p->cpus_ptr, cpumask)) 8760 continue; 8761 8762 p->max_allowed_capacity = entry->capacity; 8763 break; 8764 } 8765 rcu_read_unlock(); 8766 } 8767 8768 static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context *ctx) 8769 { 8770 set_cpus_allowed_common(p, ctx); 8771 set_task_max_allowed_capacity(p); 8772 } 8773 8774 static int 8775 balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 8776 { 8777 if (sched_fair_runnable(rq)) 8778 return 1; 8779 8780 return sched_balance_newidle(rq, rf) != 0; 8781 } 8782 8783 static void set_next_buddy(struct sched_entity *se) 8784 { 8785 for_each_sched_entity(se) { 8786 if (WARN_ON_ONCE(!se->on_rq)) 8787 return; 8788 if (se_is_idle(se)) 8789 return; 8790 cfs_rq_of(se)->next = se; 8791 } 8792 } 8793 8794 enum preempt_wakeup_action { 8795 PREEMPT_WAKEUP_NONE, /* No preemption. */ 8796 PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ 8797 PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ 8798 PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ 8799 }; 8800 8801 static inline bool 8802 set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, 8803 struct sched_entity *pse, struct sched_entity *se) 8804 { 8805 /* 8806 * Keep existing buddy if the deadline is sooner than pse. 8807 * The older buddy may be cache cold and completely unrelated 8808 * to the current wakeup but that is unpredictable where as 8809 * obeying the deadline is more in line with EEVDF objectives. 8810 */ 8811 if (cfs_rq->next && entity_before(cfs_rq->next, pse)) 8812 return false; 8813 8814 set_next_buddy(pse); 8815 return true; 8816 } 8817 8818 /* 8819 * WF_SYNC|WF_TTWU indicates the waker expects to sleep but it is not 8820 * strictly enforced because the hint is either misunderstood or 8821 * multiple tasks must be woken up. 8822 */ 8823 static inline enum preempt_wakeup_action 8824 preempt_sync(struct rq *rq, int wake_flags, 8825 struct sched_entity *pse, struct sched_entity *se) 8826 { 8827 u64 threshold, delta; 8828 8829 /* 8830 * WF_SYNC without WF_TTWU is not expected so warn if it happens even 8831 * though it is likely harmless. 8832 */ 8833 WARN_ON_ONCE(!(wake_flags & WF_TTWU)); 8834 8835 threshold = sysctl_sched_migration_cost; 8836 delta = rq_clock_task(rq) - se->exec_start; 8837 if ((s64)delta < 0) 8838 delta = 0; 8839 8840 /* 8841 * WF_RQ_SELECTED implies the tasks are stacking on a CPU when they 8842 * could run on other CPUs. Reduce the threshold before preemption is 8843 * allowed to an arbitrary lower value as it is more likely (but not 8844 * guaranteed) the waker requires the wakee to finish. 8845 */ 8846 if (wake_flags & WF_RQ_SELECTED) 8847 threshold >>= 2; 8848 8849 /* 8850 * As WF_SYNC is not strictly obeyed, allow some runtime for batch 8851 * wakeups to be issued. 8852 */ 8853 if (entity_before(pse, se) && delta >= threshold) 8854 return PREEMPT_WAKEUP_RESCHED; 8855 8856 return PREEMPT_WAKEUP_NONE; 8857 } 8858 8859 /* 8860 * Preempt the current task with a newly woken task if needed: 8861 */ 8862 static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int wake_flags) 8863 { 8864 enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; 8865 struct task_struct *donor = rq->donor; 8866 struct sched_entity *nse, *se = &donor->se, *pse = &p->se; 8867 struct cfs_rq *cfs_rq = task_cfs_rq(donor); 8868 int cse_is_idle, pse_is_idle; 8869 8870 if (unlikely(se == pse)) 8871 return; 8872 8873 /* 8874 * This is possible from callers such as attach_tasks(), in which we 8875 * unconditionally wakeup_preempt() after an enqueue (which may have 8876 * lead to a throttle). This both saves work and prevents false 8877 * next-buddy nomination below. 8878 */ 8879 if (task_is_throttled(p)) 8880 return; 8881 8882 /* 8883 * We can come here with TIF_NEED_RESCHED already set from new task 8884 * wake up path. 8885 * 8886 * Note: this also catches the edge-case of curr being in a throttled 8887 * group (e.g. via set_curr_task), since update_curr() (in the 8888 * enqueue of curr) will have resulted in resched being set. This 8889 * prevents us from potentially nominating it as a false LAST_BUDDY 8890 * below. 8891 */ 8892 if (test_tsk_need_resched(rq->curr)) 8893 return; 8894 8895 if (!sched_feat(WAKEUP_PREEMPTION)) 8896 return; 8897 8898 find_matching_se(&se, &pse); 8899 WARN_ON_ONCE(!pse); 8900 8901 cse_is_idle = se_is_idle(se); 8902 pse_is_idle = se_is_idle(pse); 8903 8904 /* 8905 * Preempt an idle entity in favor of a non-idle entity (and don't preempt 8906 * in the inverse case). 8907 */ 8908 if (cse_is_idle && !pse_is_idle) { 8909 /* 8910 * When non-idle entity preempt an idle entity, 8911 * don't give idle entity slice protection. 8912 */ 8913 preempt_action = PREEMPT_WAKEUP_SHORT; 8914 goto preempt; 8915 } 8916 8917 if (cse_is_idle != pse_is_idle) 8918 return; 8919 8920 /* 8921 * BATCH and IDLE tasks do not preempt others. 8922 */ 8923 if (unlikely(!normal_policy(p->policy))) 8924 return; 8925 8926 cfs_rq = cfs_rq_of(se); 8927 update_curr(cfs_rq); 8928 /* 8929 * If @p has a shorter slice than current and @p is eligible, override 8930 * current's slice protection in order to allow preemption. 8931 */ 8932 if (sched_feat(PREEMPT_SHORT) && (pse->slice < se->slice)) { 8933 preempt_action = PREEMPT_WAKEUP_SHORT; 8934 goto pick; 8935 } 8936 8937 /* 8938 * Ignore wakee preemption on WF_FORK as it is less likely that 8939 * there is shared data as exec often follow fork. Do not 8940 * preempt for tasks that are sched_delayed as it would violate 8941 * EEVDF to forcibly queue an ineligible task. 8942 */ 8943 if ((wake_flags & WF_FORK) || pse->sched_delayed) 8944 return; 8945 8946 /* Prefer picking wakee soon if appropriate. */ 8947 if (sched_feat(NEXT_BUDDY) && 8948 set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { 8949 8950 /* 8951 * Decide whether to obey WF_SYNC hint for a new buddy. Old 8952 * buddies are ignored as they may not be relevant to the 8953 * waker and less likely to be cache hot. 8954 */ 8955 if (wake_flags & WF_SYNC) 8956 preempt_action = preempt_sync(rq, wake_flags, pse, se); 8957 } 8958 8959 switch (preempt_action) { 8960 case PREEMPT_WAKEUP_NONE: 8961 return; 8962 case PREEMPT_WAKEUP_RESCHED: 8963 goto preempt; 8964 case PREEMPT_WAKEUP_SHORT: 8965 fallthrough; 8966 case PREEMPT_WAKEUP_PICK: 8967 break; 8968 } 8969 8970 pick: 8971 nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); 8972 /* If @p has become the most eligible task, force preemption */ 8973 if (nse == pse) 8974 goto preempt; 8975 8976 /* 8977 * Because p is enqueued, nse being null can only mean that we 8978 * dequeued a delayed task. If there are still entities queued in 8979 * cfs, check if the next one will be p. 8980 */ 8981 if (!nse && cfs_rq->nr_queued) 8982 goto pick; 8983 8984 if (sched_feat(RUN_TO_PARITY)) 8985 update_protect_slice(cfs_rq, se); 8986 8987 return; 8988 8989 preempt: 8990 if (preempt_action == PREEMPT_WAKEUP_SHORT) 8991 cancel_protect_slice(se); 8992 8993 resched_curr_lazy(rq); 8994 } 8995 8996 static struct task_struct *pick_task_fair(struct rq *rq) 8997 { 8998 struct sched_entity *se; 8999 struct cfs_rq *cfs_rq; 9000 struct task_struct *p; 9001 bool throttled; 9002 9003 again: 9004 cfs_rq = &rq->cfs; 9005 if (!cfs_rq->nr_queued) 9006 return NULL; 9007 9008 throttled = false; 9009 9010 do { 9011 /* Might not have done put_prev_entity() */ 9012 if (cfs_rq->curr && cfs_rq->curr->on_rq) 9013 update_curr(cfs_rq); 9014 9015 throttled |= check_cfs_rq_runtime(cfs_rq); 9016 9017 se = pick_next_entity(rq, cfs_rq, true); 9018 if (!se) 9019 goto again; 9020 cfs_rq = group_cfs_rq(se); 9021 } while (cfs_rq); 9022 9023 p = task_of(se); 9024 if (unlikely(throttled)) 9025 task_throttle_setup_work(p); 9026 return p; 9027 } 9028 9029 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 9030 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first); 9031 9032 struct task_struct * 9033 pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) 9034 { 9035 struct sched_entity *se; 9036 struct task_struct *p; 9037 int new_tasks; 9038 9039 again: 9040 p = pick_task_fair(rq); 9041 if (!p) 9042 goto idle; 9043 se = &p->se; 9044 9045 #ifdef CONFIG_FAIR_GROUP_SCHED 9046 if (prev->sched_class != &fair_sched_class) 9047 goto simple; 9048 9049 __put_prev_set_next_dl_server(rq, prev, p); 9050 9051 /* 9052 * Because of the set_next_buddy() in dequeue_task_fair() it is rather 9053 * likely that a next task is from the same cgroup as the current. 9054 * 9055 * Therefore attempt to avoid putting and setting the entire cgroup 9056 * hierarchy, only change the part that actually changes. 9057 * 9058 * Since we haven't yet done put_prev_entity and if the selected task 9059 * is a different task than we started out with, try and touch the 9060 * least amount of cfs_rqs. 9061 */ 9062 if (prev != p) { 9063 struct sched_entity *pse = &prev->se; 9064 struct cfs_rq *cfs_rq; 9065 9066 while (!(cfs_rq = is_same_group(se, pse))) { 9067 int se_depth = se->depth; 9068 int pse_depth = pse->depth; 9069 9070 if (se_depth <= pse_depth) { 9071 put_prev_entity(cfs_rq_of(pse), pse); 9072 pse = parent_entity(pse); 9073 } 9074 if (se_depth >= pse_depth) { 9075 set_next_entity(cfs_rq_of(se), se, true); 9076 se = parent_entity(se); 9077 } 9078 } 9079 9080 put_prev_entity(cfs_rq, pse); 9081 set_next_entity(cfs_rq, se, true); 9082 9083 __set_next_task_fair(rq, p, true); 9084 } 9085 9086 return p; 9087 9088 simple: 9089 #endif /* CONFIG_FAIR_GROUP_SCHED */ 9090 put_prev_set_next_task(rq, prev, p); 9091 return p; 9092 9093 idle: 9094 if (rf) { 9095 new_tasks = sched_balance_newidle(rq, rf); 9096 9097 /* 9098 * Because sched_balance_newidle() releases (and re-acquires) 9099 * rq->lock, it is possible for any higher priority task to 9100 * appear. In that case we must re-start the pick_next_entity() 9101 * loop. 9102 */ 9103 if (new_tasks < 0) 9104 return RETRY_TASK; 9105 9106 if (new_tasks > 0) 9107 goto again; 9108 } 9109 9110 return NULL; 9111 } 9112 9113 static struct task_struct *__pick_next_task_fair(struct rq *rq, struct task_struct *prev) 9114 { 9115 return pick_next_task_fair(rq, prev, NULL); 9116 } 9117 9118 static struct task_struct *fair_server_pick_task(struct sched_dl_entity *dl_se) 9119 { 9120 return pick_task_fair(dl_se->rq); 9121 } 9122 9123 void fair_server_init(struct rq *rq) 9124 { 9125 struct sched_dl_entity *dl_se = &rq->fair_server; 9126 9127 init_dl_entity(dl_se); 9128 9129 dl_server_init(dl_se, rq, fair_server_pick_task); 9130 } 9131 9132 /* 9133 * Account for a descheduled task: 9134 */ 9135 static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) 9136 { 9137 struct sched_entity *se = &prev->se; 9138 struct cfs_rq *cfs_rq; 9139 9140 for_each_sched_entity(se) { 9141 cfs_rq = cfs_rq_of(se); 9142 put_prev_entity(cfs_rq, se); 9143 } 9144 } 9145 9146 /* 9147 * sched_yield() is very simple 9148 */ 9149 static void yield_task_fair(struct rq *rq) 9150 { 9151 struct task_struct *curr = rq->donor; 9152 struct cfs_rq *cfs_rq = task_cfs_rq(curr); 9153 struct sched_entity *se = &curr->se; 9154 9155 /* 9156 * Are we the only task in the tree? 9157 */ 9158 if (unlikely(rq->nr_running == 1)) 9159 return; 9160 9161 clear_buddies(cfs_rq, se); 9162 9163 update_rq_clock(rq); 9164 /* 9165 * Update run-time statistics of the 'current'. 9166 */ 9167 update_curr(cfs_rq); 9168 /* 9169 * Tell update_rq_clock() that we've just updated, 9170 * so we don't do microscopic update in schedule() 9171 * and double the fastpath cost. 9172 */ 9173 rq_clock_skip_update(rq); 9174 9175 /* 9176 * Forfeit the remaining vruntime, only if the entity is eligible. This 9177 * condition is necessary because in core scheduling we prefer to run 9178 * ineligible tasks rather than force idling. If this happens we may 9179 * end up in a loop where the core scheduler picks the yielding task, 9180 * which yields immediately again; without the condition the vruntime 9181 * ends up quickly running away. 9182 */ 9183 if (entity_eligible(cfs_rq, se)) { 9184 se->vruntime = se->deadline; 9185 update_deadline(cfs_rq, se); 9186 } 9187 } 9188 9189 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) 9190 { 9191 struct sched_entity *se = &p->se; 9192 9193 /* !se->on_rq also covers throttled task */ 9194 if (!se->on_rq) 9195 return false; 9196 9197 /* Tell the scheduler that we'd really like se to run next. */ 9198 set_next_buddy(se); 9199 9200 yield_task_fair(rq); 9201 9202 return true; 9203 } 9204 9205 /************************************************** 9206 * Fair scheduling class load-balancing methods. 9207 * 9208 * BASICS 9209 * 9210 * The purpose of load-balancing is to achieve the same basic fairness the 9211 * per-CPU scheduler provides, namely provide a proportional amount of compute 9212 * time to each task. This is expressed in the following equation: 9213 * 9214 * W_i,n/P_i == W_j,n/P_j for all i,j (1) 9215 * 9216 * Where W_i,n is the n-th weight average for CPU i. The instantaneous weight 9217 * W_i,0 is defined as: 9218 * 9219 * W_i,0 = \Sum_j w_i,j (2) 9220 * 9221 * Where w_i,j is the weight of the j-th runnable task on CPU i. This weight 9222 * is derived from the nice value as per sched_prio_to_weight[]. 9223 * 9224 * The weight average is an exponential decay average of the instantaneous 9225 * weight: 9226 * 9227 * W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3) 9228 * 9229 * C_i is the compute capacity of CPU i, typically it is the 9230 * fraction of 'recent' time available for SCHED_OTHER task execution. But it 9231 * can also include other factors [XXX]. 9232 * 9233 * To achieve this balance we define a measure of imbalance which follows 9234 * directly from (1): 9235 * 9236 * imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4) 9237 * 9238 * We them move tasks around to minimize the imbalance. In the continuous 9239 * function space it is obvious this converges, in the discrete case we get 9240 * a few fun cases generally called infeasible weight scenarios. 9241 * 9242 * [XXX expand on: 9243 * - infeasible weights; 9244 * - local vs global optima in the discrete case. ] 9245 * 9246 * 9247 * SCHED DOMAINS 9248 * 9249 * In order to solve the imbalance equation (4), and avoid the obvious O(n^2) 9250 * for all i,j solution, we create a tree of CPUs that follows the hardware 9251 * topology where each level pairs two lower groups (or better). This results 9252 * in O(log n) layers. Furthermore we reduce the number of CPUs going up the 9253 * tree to only the first of the previous level and we decrease the frequency 9254 * of load-balance at each level inversely proportional to the number of CPUs in 9255 * the groups. 9256 * 9257 * This yields: 9258 * 9259 * log_2 n 1 n 9260 * \Sum { --- * --- * 2^i } = O(n) (5) 9261 * i = 0 2^i 2^i 9262 * `- size of each group 9263 * | | `- number of CPUs doing load-balance 9264 * | `- freq 9265 * `- sum over all levels 9266 * 9267 * Coupled with a limit on how many tasks we can migrate every balance pass, 9268 * this makes (5) the runtime complexity of the balancer. 9269 * 9270 * An important property here is that each CPU is still (indirectly) connected 9271 * to every other CPU in at most O(log n) steps: 9272 * 9273 * The adjacency matrix of the resulting graph is given by: 9274 * 9275 * log_2 n 9276 * A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6) 9277 * k = 0 9278 * 9279 * And you'll find that: 9280 * 9281 * A^(log_2 n)_i,j != 0 for all i,j (7) 9282 * 9283 * Showing there's indeed a path between every CPU in at most O(log n) steps. 9284 * The task movement gives a factor of O(m), giving a convergence complexity 9285 * of: 9286 * 9287 * O(nm log n), n := nr_cpus, m := nr_tasks (8) 9288 * 9289 * 9290 * WORK CONSERVING 9291 * 9292 * In order to avoid CPUs going idle while there's still work to do, new idle 9293 * balancing is more aggressive and has the newly idle CPU iterate up the domain 9294 * tree itself instead of relying on other CPUs to bring it work. 9295 * 9296 * This adds some complexity to both (5) and (8) but it reduces the total idle 9297 * time. 9298 * 9299 * [XXX more?] 9300 * 9301 * 9302 * CGROUPS 9303 * 9304 * Cgroups make a horror show out of (2), instead of a simple sum we get: 9305 * 9306 * s_k,i 9307 * W_i,0 = \Sum_j \Prod_k w_k * ----- (9) 9308 * S_k 9309 * 9310 * Where 9311 * 9312 * s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10) 9313 * 9314 * w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i. 9315 * 9316 * The big problem is S_k, its a global sum needed to compute a local (W_i) 9317 * property. 9318 * 9319 * [XXX write more on how we solve this.. _after_ merging pjt's patches that 9320 * rewrite all of this once again.] 9321 */ 9322 9323 static unsigned long __read_mostly max_load_balance_interval = HZ/10; 9324 9325 enum fbq_type { regular, remote, all }; 9326 9327 /* 9328 * 'group_type' describes the group of CPUs at the moment of load balancing. 9329 * 9330 * The enum is ordered by pulling priority, with the group with lowest priority 9331 * first so the group_type can simply be compared when selecting the busiest 9332 * group. See update_sd_pick_busiest(). 9333 */ 9334 enum group_type { 9335 /* The group has spare capacity that can be used to run more tasks. */ 9336 group_has_spare = 0, 9337 /* 9338 * The group is fully used and the tasks don't compete for more CPU 9339 * cycles. Nevertheless, some tasks might wait before running. 9340 */ 9341 group_fully_busy, 9342 /* 9343 * One task doesn't fit with CPU's capacity and must be migrated to a 9344 * more powerful CPU. 9345 */ 9346 group_misfit_task, 9347 /* 9348 * Balance SMT group that's fully busy. Can benefit from migration 9349 * a task on SMT with busy sibling to another CPU on idle core. 9350 */ 9351 group_smt_balance, 9352 /* 9353 * SD_ASYM_PACKING only: One local CPU with higher capacity is available, 9354 * and the task should be migrated to it instead of running on the 9355 * current CPU. 9356 */ 9357 group_asym_packing, 9358 /* 9359 * The tasks' affinity constraints previously prevented the scheduler 9360 * from balancing the load across the system. 9361 */ 9362 group_imbalanced, 9363 /* 9364 * The CPU is overloaded and can't provide expected CPU cycles to all 9365 * tasks. 9366 */ 9367 group_overloaded 9368 }; 9369 9370 enum migration_type { 9371 migrate_load = 0, 9372 migrate_util, 9373 migrate_task, 9374 migrate_misfit 9375 }; 9376 9377 #define LBF_ALL_PINNED 0x01 9378 #define LBF_NEED_BREAK 0x02 9379 #define LBF_DST_PINNED 0x04 9380 #define LBF_SOME_PINNED 0x08 9381 #define LBF_ACTIVE_LB 0x10 9382 9383 struct lb_env { 9384 struct sched_domain *sd; 9385 9386 struct rq *src_rq; 9387 int src_cpu; 9388 9389 int dst_cpu; 9390 struct rq *dst_rq; 9391 9392 struct cpumask *dst_grpmask; 9393 int new_dst_cpu; 9394 enum cpu_idle_type idle; 9395 long imbalance; 9396 /* The set of CPUs under consideration for load-balancing */ 9397 struct cpumask *cpus; 9398 9399 unsigned int flags; 9400 9401 unsigned int loop; 9402 unsigned int loop_break; 9403 unsigned int loop_max; 9404 9405 enum fbq_type fbq_type; 9406 enum migration_type migration_type; 9407 struct list_head tasks; 9408 }; 9409 9410 /* 9411 * Is this task likely cache-hot: 9412 */ 9413 static int task_hot(struct task_struct *p, struct lb_env *env) 9414 { 9415 s64 delta; 9416 9417 lockdep_assert_rq_held(env->src_rq); 9418 9419 if (p->sched_class != &fair_sched_class) 9420 return 0; 9421 9422 if (unlikely(task_has_idle_policy(p))) 9423 return 0; 9424 9425 /* SMT siblings share cache */ 9426 if (env->sd->flags & SD_SHARE_CPUCAPACITY) 9427 return 0; 9428 9429 /* 9430 * Buddy candidates are cache hot: 9431 */ 9432 if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running && 9433 (&p->se == cfs_rq_of(&p->se)->next)) 9434 return 1; 9435 9436 if (sysctl_sched_migration_cost == -1) 9437 return 1; 9438 9439 /* 9440 * Don't migrate task if the task's cookie does not match 9441 * with the destination CPU's core cookie. 9442 */ 9443 if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p)) 9444 return 1; 9445 9446 if (sysctl_sched_migration_cost == 0) 9447 return 0; 9448 9449 delta = rq_clock_task(env->src_rq) - p->se.exec_start; 9450 9451 return delta < (s64)sysctl_sched_migration_cost; 9452 } 9453 9454 #ifdef CONFIG_NUMA_BALANCING 9455 /* 9456 * Returns a positive value, if task migration degrades locality. 9457 * Returns 0, if task migration is not affected by locality. 9458 * Returns a negative value, if task migration improves locality i.e migration preferred. 9459 */ 9460 static long migrate_degrades_locality(struct task_struct *p, struct lb_env *env) 9461 { 9462 struct numa_group *numa_group = rcu_dereference(p->numa_group); 9463 unsigned long src_weight, dst_weight; 9464 int src_nid, dst_nid, dist; 9465 9466 if (!static_branch_likely(&sched_numa_balancing)) 9467 return 0; 9468 9469 if (!p->numa_faults || !(env->sd->flags & SD_NUMA)) 9470 return 0; 9471 9472 src_nid = cpu_to_node(env->src_cpu); 9473 dst_nid = cpu_to_node(env->dst_cpu); 9474 9475 if (src_nid == dst_nid) 9476 return 0; 9477 9478 /* Migrating away from the preferred node is always bad. */ 9479 if (src_nid == p->numa_preferred_nid) { 9480 if (env->src_rq->nr_running > env->src_rq->nr_preferred_running) 9481 return 1; 9482 else 9483 return 0; 9484 } 9485 9486 /* Encourage migration to the preferred node. */ 9487 if (dst_nid == p->numa_preferred_nid) 9488 return -1; 9489 9490 /* Leaving a core idle is often worse than degrading locality. */ 9491 if (env->idle == CPU_IDLE) 9492 return 0; 9493 9494 dist = node_distance(src_nid, dst_nid); 9495 if (numa_group) { 9496 src_weight = group_weight(p, src_nid, dist); 9497 dst_weight = group_weight(p, dst_nid, dist); 9498 } else { 9499 src_weight = task_weight(p, src_nid, dist); 9500 dst_weight = task_weight(p, dst_nid, dist); 9501 } 9502 9503 return src_weight - dst_weight; 9504 } 9505 9506 #else /* !CONFIG_NUMA_BALANCING: */ 9507 static inline long migrate_degrades_locality(struct task_struct *p, 9508 struct lb_env *env) 9509 { 9510 return 0; 9511 } 9512 #endif /* !CONFIG_NUMA_BALANCING */ 9513 9514 /* 9515 * Check whether the task is ineligible on the destination cpu 9516 * 9517 * When the PLACE_LAG scheduling feature is enabled and 9518 * dst_cfs_rq->nr_queued is greater than 1, if the task 9519 * is ineligible, it will also be ineligible when 9520 * it is migrated to the destination cpu. 9521 */ 9522 static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) 9523 { 9524 struct cfs_rq *dst_cfs_rq; 9525 9526 #ifdef CONFIG_FAIR_GROUP_SCHED 9527 dst_cfs_rq = task_group(p)->cfs_rq[dest_cpu]; 9528 #else 9529 dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; 9530 #endif 9531 if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && 9532 !entity_eligible(task_cfs_rq(p), &p->se)) 9533 return 1; 9534 9535 return 0; 9536 } 9537 9538 /* 9539 * can_migrate_task - may task p from runqueue rq be migrated to this_cpu? 9540 */ 9541 static 9542 int can_migrate_task(struct task_struct *p, struct lb_env *env) 9543 { 9544 long degrades, hot; 9545 9546 lockdep_assert_rq_held(env->src_rq); 9547 if (p->sched_task_hot) 9548 p->sched_task_hot = 0; 9549 9550 /* 9551 * We do not migrate tasks that are: 9552 * 1) delayed dequeued unless we migrate load, or 9553 * 2) target cfs_rq is in throttled hierarchy, or 9554 * 3) cannot be migrated to this CPU due to cpus_ptr, or 9555 * 4) running (obviously), or 9556 * 5) are cache-hot on their current CPU, or 9557 * 6) are blocked on mutexes (if SCHED_PROXY_EXEC is enabled) 9558 */ 9559 if ((p->se.sched_delayed) && (env->migration_type != migrate_load)) 9560 return 0; 9561 9562 if (lb_throttled_hierarchy(p, env->dst_cpu)) 9563 return 0; 9564 9565 /* 9566 * We want to prioritize the migration of eligible tasks. 9567 * For ineligible tasks we soft-limit them and only allow 9568 * them to migrate when nr_balance_failed is non-zero to 9569 * avoid load-balancing trying very hard to balance the load. 9570 */ 9571 if (!env->sd->nr_balance_failed && 9572 task_is_ineligible_on_dst_cpu(p, env->dst_cpu)) 9573 return 0; 9574 9575 /* Disregard percpu kthreads; they are where they need to be. */ 9576 if (kthread_is_per_cpu(p)) 9577 return 0; 9578 9579 if (task_is_blocked(p)) 9580 return 0; 9581 9582 if (!cpumask_test_cpu(env->dst_cpu, p->cpus_ptr)) { 9583 int cpu; 9584 9585 schedstat_inc(p->stats.nr_failed_migrations_affine); 9586 9587 env->flags |= LBF_SOME_PINNED; 9588 9589 /* 9590 * Remember if this task can be migrated to any other CPU in 9591 * our sched_group. We may want to revisit it if we couldn't 9592 * meet load balance goals by pulling other tasks on src_cpu. 9593 * 9594 * Avoid computing new_dst_cpu 9595 * - for NEWLY_IDLE 9596 * - if we have already computed one in current iteration 9597 * - if it's an active balance 9598 */ 9599 if (env->idle == CPU_NEWLY_IDLE || 9600 env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB)) 9601 return 0; 9602 9603 /* Prevent to re-select dst_cpu via env's CPUs: */ 9604 cpu = cpumask_first_and_and(env->dst_grpmask, env->cpus, p->cpus_ptr); 9605 9606 if (cpu < nr_cpu_ids) { 9607 env->flags |= LBF_DST_PINNED; 9608 env->new_dst_cpu = cpu; 9609 } 9610 9611 return 0; 9612 } 9613 9614 /* Record that we found at least one task that could run on dst_cpu */ 9615 env->flags &= ~LBF_ALL_PINNED; 9616 9617 if (task_on_cpu(env->src_rq, p) || 9618 task_current_donor(env->src_rq, p)) { 9619 schedstat_inc(p->stats.nr_failed_migrations_running); 9620 return 0; 9621 } 9622 9623 /* 9624 * Aggressive migration if: 9625 * 1) active balance 9626 * 2) destination numa is preferred 9627 * 3) task is cache cold, or 9628 * 4) too many balance attempts have failed. 9629 */ 9630 if (env->flags & LBF_ACTIVE_LB) 9631 return 1; 9632 9633 degrades = migrate_degrades_locality(p, env); 9634 if (!degrades) 9635 hot = task_hot(p, env); 9636 else 9637 hot = degrades > 0; 9638 9639 if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { 9640 if (hot) 9641 p->sched_task_hot = 1; 9642 return 1; 9643 } 9644 9645 schedstat_inc(p->stats.nr_failed_migrations_hot); 9646 return 0; 9647 } 9648 9649 /* 9650 * detach_task() -- detach the task for the migration specified in env 9651 */ 9652 static void detach_task(struct task_struct *p, struct lb_env *env) 9653 { 9654 lockdep_assert_rq_held(env->src_rq); 9655 9656 if (p->sched_task_hot) { 9657 p->sched_task_hot = 0; 9658 schedstat_inc(env->sd->lb_hot_gained[env->idle]); 9659 schedstat_inc(p->stats.nr_forced_migrations); 9660 } 9661 9662 WARN_ON(task_current(env->src_rq, p)); 9663 WARN_ON(task_current_donor(env->src_rq, p)); 9664 9665 deactivate_task(env->src_rq, p, DEQUEUE_NOCLOCK); 9666 set_task_cpu(p, env->dst_cpu); 9667 } 9668 9669 /* 9670 * detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as 9671 * part of active balancing operations within "domain". 9672 * 9673 * Returns a task if successful and NULL otherwise. 9674 */ 9675 static struct task_struct *detach_one_task(struct lb_env *env) 9676 { 9677 struct task_struct *p; 9678 9679 lockdep_assert_rq_held(env->src_rq); 9680 9681 list_for_each_entry_reverse(p, 9682 &env->src_rq->cfs_tasks, se.group_node) { 9683 if (!can_migrate_task(p, env)) 9684 continue; 9685 9686 detach_task(p, env); 9687 9688 /* 9689 * Right now, this is only the second place where 9690 * lb_gained[env->idle] is updated (other is detach_tasks) 9691 * so we can safely collect stats here rather than 9692 * inside detach_tasks(). 9693 */ 9694 schedstat_inc(env->sd->lb_gained[env->idle]); 9695 return p; 9696 } 9697 return NULL; 9698 } 9699 9700 /* 9701 * detach_tasks() -- tries to detach up to imbalance load/util/tasks from 9702 * busiest_rq, as part of a balancing operation within domain "sd". 9703 * 9704 * Returns number of detached tasks if successful and 0 otherwise. 9705 */ 9706 static int detach_tasks(struct lb_env *env) 9707 { 9708 struct list_head *tasks = &env->src_rq->cfs_tasks; 9709 unsigned long util, load; 9710 struct task_struct *p; 9711 int detached = 0; 9712 9713 lockdep_assert_rq_held(env->src_rq); 9714 9715 /* 9716 * Source run queue has been emptied by another CPU, clear 9717 * LBF_ALL_PINNED flag as we will not test any task. 9718 */ 9719 if (env->src_rq->nr_running <= 1) { 9720 env->flags &= ~LBF_ALL_PINNED; 9721 return 0; 9722 } 9723 9724 if (env->imbalance <= 0) 9725 return 0; 9726 9727 while (!list_empty(tasks)) { 9728 /* 9729 * We don't want to steal all, otherwise we may be treated likewise, 9730 * which could at worst lead to a livelock crash. 9731 */ 9732 if (env->idle && env->src_rq->nr_running <= 1) 9733 break; 9734 9735 env->loop++; 9736 /* We've more or less seen every task there is, call it quits */ 9737 if (env->loop > env->loop_max) 9738 break; 9739 9740 /* take a breather every nr_migrate tasks */ 9741 if (env->loop > env->loop_break) { 9742 env->loop_break += SCHED_NR_MIGRATE_BREAK; 9743 env->flags |= LBF_NEED_BREAK; 9744 break; 9745 } 9746 9747 p = list_last_entry(tasks, struct task_struct, se.group_node); 9748 9749 if (!can_migrate_task(p, env)) 9750 goto next; 9751 9752 switch (env->migration_type) { 9753 case migrate_load: 9754 /* 9755 * Depending of the number of CPUs and tasks and the 9756 * cgroup hierarchy, task_h_load() can return a null 9757 * value. Make sure that env->imbalance decreases 9758 * otherwise detach_tasks() will stop only after 9759 * detaching up to loop_max tasks. 9760 */ 9761 load = max_t(unsigned long, task_h_load(p), 1); 9762 9763 if (sched_feat(LB_MIN) && 9764 load < 16 && !env->sd->nr_balance_failed) 9765 goto next; 9766 9767 /* 9768 * Make sure that we don't migrate too much load. 9769 * Nevertheless, let relax the constraint if 9770 * scheduler fails to find a good waiting task to 9771 * migrate. 9772 */ 9773 if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance) 9774 goto next; 9775 9776 env->imbalance -= load; 9777 break; 9778 9779 case migrate_util: 9780 util = task_util_est(p); 9781 9782 if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance) 9783 goto next; 9784 9785 env->imbalance -= util; 9786 break; 9787 9788 case migrate_task: 9789 env->imbalance--; 9790 break; 9791 9792 case migrate_misfit: 9793 /* This is not a misfit task */ 9794 if (task_fits_cpu(p, env->src_cpu)) 9795 goto next; 9796 9797 env->imbalance = 0; 9798 break; 9799 } 9800 9801 detach_task(p, env); 9802 list_add(&p->se.group_node, &env->tasks); 9803 9804 detached++; 9805 9806 #ifdef CONFIG_PREEMPTION 9807 /* 9808 * NEWIDLE balancing is a source of latency, so preemptible 9809 * kernels will stop after the first task is detached to minimize 9810 * the critical section. 9811 */ 9812 if (env->idle == CPU_NEWLY_IDLE) 9813 break; 9814 #endif 9815 9816 /* 9817 * We only want to steal up to the prescribed amount of 9818 * load/util/tasks. 9819 */ 9820 if (env->imbalance <= 0) 9821 break; 9822 9823 continue; 9824 next: 9825 if (p->sched_task_hot) 9826 schedstat_inc(p->stats.nr_failed_migrations_hot); 9827 9828 list_move(&p->se.group_node, tasks); 9829 } 9830 9831 /* 9832 * Right now, this is one of only two places we collect this stat 9833 * so we can safely collect detach_one_task() stats here rather 9834 * than inside detach_one_task(). 9835 */ 9836 schedstat_add(env->sd->lb_gained[env->idle], detached); 9837 9838 return detached; 9839 } 9840 9841 /* 9842 * attach_task() -- attach the task detached by detach_task() to its new rq. 9843 */ 9844 static void attach_task(struct rq *rq, struct task_struct *p) 9845 { 9846 lockdep_assert_rq_held(rq); 9847 9848 WARN_ON_ONCE(task_rq(p) != rq); 9849 activate_task(rq, p, ENQUEUE_NOCLOCK); 9850 wakeup_preempt(rq, p, 0); 9851 } 9852 9853 /* 9854 * attach_one_task() -- attaches the task returned from detach_one_task() to 9855 * its new rq. 9856 */ 9857 static void attach_one_task(struct rq *rq, struct task_struct *p) 9858 { 9859 struct rq_flags rf; 9860 9861 rq_lock(rq, &rf); 9862 update_rq_clock(rq); 9863 attach_task(rq, p); 9864 rq_unlock(rq, &rf); 9865 } 9866 9867 /* 9868 * attach_tasks() -- attaches all tasks detached by detach_tasks() to their 9869 * new rq. 9870 */ 9871 static void attach_tasks(struct lb_env *env) 9872 { 9873 struct list_head *tasks = &env->tasks; 9874 struct task_struct *p; 9875 struct rq_flags rf; 9876 9877 rq_lock(env->dst_rq, &rf); 9878 update_rq_clock(env->dst_rq); 9879 9880 while (!list_empty(tasks)) { 9881 p = list_first_entry(tasks, struct task_struct, se.group_node); 9882 list_del_init(&p->se.group_node); 9883 9884 attach_task(env->dst_rq, p); 9885 } 9886 9887 rq_unlock(env->dst_rq, &rf); 9888 } 9889 9890 #ifdef CONFIG_NO_HZ_COMMON 9891 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) 9892 { 9893 if (cfs_rq->avg.load_avg) 9894 return true; 9895 9896 if (cfs_rq->avg.util_avg) 9897 return true; 9898 9899 return false; 9900 } 9901 9902 static inline bool others_have_blocked(struct rq *rq) 9903 { 9904 if (cpu_util_rt(rq)) 9905 return true; 9906 9907 if (cpu_util_dl(rq)) 9908 return true; 9909 9910 if (hw_load_avg(rq)) 9911 return true; 9912 9913 if (cpu_util_irq(rq)) 9914 return true; 9915 9916 return false; 9917 } 9918 9919 static inline void update_blocked_load_tick(struct rq *rq) 9920 { 9921 WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies); 9922 } 9923 9924 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) 9925 { 9926 if (!has_blocked) 9927 rq->has_blocked_load = 0; 9928 } 9929 #else /* !CONFIG_NO_HZ_COMMON: */ 9930 static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; } 9931 static inline bool others_have_blocked(struct rq *rq) { return false; } 9932 static inline void update_blocked_load_tick(struct rq *rq) {} 9933 static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {} 9934 #endif /* !CONFIG_NO_HZ_COMMON */ 9935 9936 static bool __update_blocked_others(struct rq *rq, bool *done) 9937 { 9938 bool updated; 9939 9940 /* 9941 * update_load_avg() can call cpufreq_update_util(). Make sure that RT, 9942 * DL and IRQ signals have been updated before updating CFS. 9943 */ 9944 updated = update_other_load_avgs(rq); 9945 9946 if (others_have_blocked(rq)) 9947 *done = false; 9948 9949 return updated; 9950 } 9951 9952 #ifdef CONFIG_FAIR_GROUP_SCHED 9953 9954 static bool __update_blocked_fair(struct rq *rq, bool *done) 9955 { 9956 struct cfs_rq *cfs_rq, *pos; 9957 bool decayed = false; 9958 int cpu = cpu_of(rq); 9959 9960 /* 9961 * Iterates the task_group tree in a bottom up fashion, see 9962 * list_add_leaf_cfs_rq() for details. 9963 */ 9964 for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) { 9965 struct sched_entity *se; 9966 9967 if (update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq)) { 9968 update_tg_load_avg(cfs_rq); 9969 9970 if (cfs_rq->nr_queued == 0) 9971 update_idle_cfs_rq_clock_pelt(cfs_rq); 9972 9973 if (cfs_rq == &rq->cfs) 9974 decayed = true; 9975 } 9976 9977 /* Propagate pending load changes to the parent, if any: */ 9978 se = cfs_rq->tg->se[cpu]; 9979 if (se && !skip_blocked_update(se)) 9980 update_load_avg(cfs_rq_of(se), se, UPDATE_TG); 9981 9982 /* 9983 * There can be a lot of idle CPU cgroups. Don't let fully 9984 * decayed cfs_rqs linger on the list. 9985 */ 9986 if (cfs_rq_is_decayed(cfs_rq)) 9987 list_del_leaf_cfs_rq(cfs_rq); 9988 9989 /* Don't need periodic decay once load/util_avg are null */ 9990 if (cfs_rq_has_blocked(cfs_rq)) 9991 *done = false; 9992 } 9993 9994 return decayed; 9995 } 9996 9997 /* 9998 * Compute the hierarchical load factor for cfs_rq and all its ascendants. 9999 * This needs to be done in a top-down fashion because the load of a child10000 * group is a fraction of its parents load.10001 */10002 static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)10003 {10004 struct rq *rq = rq_of(cfs_rq);10005 struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];10006 unsigned long now = jiffies;10007 unsigned long load;1000810009 if (cfs_rq->last_h_load_update == now)10010 return;1001110012 WRITE_ONCE(cfs_rq->h_load_next, NULL);10013 for_each_sched_entity(se) {10014 cfs_rq = cfs_rq_of(se);10015 WRITE_ONCE(cfs_rq->h_load_next, se);10016 if (cfs_rq->last_h_load_update == now)10017 break;10018 }1001910020 if (!se) {10021 cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);10022 cfs_rq->last_h_load_update = now;10023 }1002410025 while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {10026 load = cfs_rq->h_load;10027 load = div64_ul(load * se->avg.load_avg,10028 cfs_rq_load_avg(cfs_rq) + 1);10029 cfs_rq = group_cfs_rq(se);10030 cfs_rq->h_load = load;10031 cfs_rq->last_h_load_update = now;10032 }10033 }1003410035 static unsigned long task_h_load(struct task_struct *p)10036 {10037 struct cfs_rq *cfs_rq = task_cfs_rq(p);1003810039 update_cfs_rq_h_load(cfs_rq);10040 return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,10041 cfs_rq_load_avg(cfs_rq) + 1);10042 }10043 #else /* !CONFIG_FAIR_GROUP_SCHED: */10044 static bool __update_blocked_fair(struct rq *rq, bool *done)10045 {10046 struct cfs_rq *cfs_rq = &rq->cfs;10047 bool decayed;1004810049 decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);10050 if (cfs_rq_has_blocked(cfs_rq))10051 *done = false;1005210053 return decayed;10054 }1005510056 static unsigned long task_h_load(struct task_struct *p)10057 {10058 return p->se.avg.load_avg;10059 }10060 #endif /* !CONFIG_FAIR_GROUP_SCHED */1006110062 static void sched_balance_update_blocked_averages(int cpu)10063 {10064 bool decayed = false, done = true;10065 struct rq *rq = cpu_rq(cpu);10066 struct rq_flags rf;1006710068 rq_lock_irqsave(rq, &rf);10069 update_blocked_load_tick(rq);10070 update_rq_clock(rq);1007110072 decayed |= __update_blocked_others(rq, &done);10073 decayed |= __update_blocked_fair(rq, &done);1007410075 update_blocked_load_status(rq, !done);10076 if (decayed)10077 cpufreq_update_util(rq, 0);10078 rq_unlock_irqrestore(rq, &rf);10079 }1008010081 /********** Helpers for sched_balance_find_src_group ************************/1008210083 /*10084 * sg_lb_stats - stats of a sched_group required for load-balancing:10085 */10086 struct sg_lb_stats {10087 unsigned long avg_load; /* Avg load over the CPUs of the group */10088 unsigned long group_load; /* Total load over the CPUs of the group */10089 unsigned long group_capacity; /* Capacity over the CPUs of the group */10090 unsigned long group_util; /* Total utilization over the CPUs of the group */10091 unsigned long group_runnable; /* Total runnable time over the CPUs of the group */10092 unsigned int sum_nr_running; /* Nr of all tasks running in the group */10093 unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */10094 unsigned int idle_cpus; /* Nr of idle CPUs in the group */10095 unsigned int group_weight;10096 enum group_type group_type;10097 unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */10098 unsigned int group_smt_balance; /* Task on busy SMT be moved */10099 unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */10100 #ifdef CONFIG_NUMA_BALANCING10101 unsigned int nr_numa_running;10102 unsigned int nr_preferred_running;10103 #endif10104 };1010510106 /*10107 * sd_lb_stats - stats of a sched_domain required for load-balancing:10108 */10109 struct sd_lb_stats {10110 struct sched_group *busiest; /* Busiest group in this sd */10111 struct sched_group *local; /* Local group in this sd */10112 unsigned long total_load; /* Total load of all groups in sd */10113 unsigned long total_capacity; /* Total capacity of all groups in sd */10114 unsigned long avg_load; /* Average load across all groups in sd */10115 unsigned int prefer_sibling; /* Tasks should go to sibling first */1011610117 struct sg_lb_stats busiest_stat; /* Statistics of the busiest group */10118 struct sg_lb_stats local_stat; /* Statistics of the local group */10119 };1012010121 static inline void init_sd_lb_stats(struct sd_lb_stats *sds)10122 {10123 /*10124 * Skimp on the clearing to avoid duplicate work. We can avoid clearing10125 * local_stat because update_sg_lb_stats() does a full clear/assignment.10126 * We must however set busiest_stat::group_type and10127 * busiest_stat::idle_cpus to the worst busiest group because10128 * update_sd_pick_busiest() reads these before assignment.10129 */10130 *sds = (struct sd_lb_stats){10131 .busiest = NULL,10132 .local = NULL,10133 .total_load = 0UL,10134 .total_capacity = 0UL,10135 .busiest_stat = {10136 .idle_cpus = UINT_MAX,10137 .group_type = group_has_spare,10138 },10139 };10140 }1014110142 static unsigned long scale_rt_capacity(int cpu)10143 {10144 unsigned long max = get_actual_cpu_capacity(cpu);10145 struct rq *rq = cpu_rq(cpu);10146 unsigned long used, free;10147 unsigned long irq;1014810149 irq = cpu_util_irq(rq);1015010151 if (unlikely(irq >= max))10152 return 1;1015310154 /*10155 * avg_rt.util_avg and avg_dl.util_avg track binary signals10156 * (running and not running) with weights 0 and 1024 respectively.10157 */10158 used = cpu_util_rt(rq);10159 used += cpu_util_dl(rq);1016010161 if (unlikely(used >= max))10162 return 1;1016310164 free = max - used;1016510166 return scale_irq_capacity(free, irq, max);10167 }1016810169 static void update_cpu_capacity(struct sched_domain *sd, int cpu)10170 {10171 unsigned long capacity = scale_rt_capacity(cpu);10172 struct sched_group *sdg = sd->groups;1017310174 if (!capacity)10175 capacity = 1;1017610177 cpu_rq(cpu)->cpu_capacity = capacity;10178 trace_sched_cpu_capacity_tp(cpu_rq(cpu));1017910180 sdg->sgc->capacity = capacity;10181 sdg->sgc->min_capacity = capacity;10182 sdg->sgc->max_capacity = capacity;10183 }1018410185 void update_group_capacity(struct sched_domain *sd, int cpu)10186 {10187 struct sched_domain *child = sd->child;10188 struct sched_group *group, *sdg = sd->groups;10189 unsigned long capacity, min_capacity, max_capacity;10190 unsigned long interval;1019110192 interval = msecs_to_jiffies(sd->balance_interval);10193 interval = clamp(interval, 1UL, max_load_balance_interval);10194 sdg->sgc->next_update = jiffies + interval;1019510196 if (!child) {10197 update_cpu_capacity(sd, cpu);10198 return;10199 }1020010201 capacity = 0;10202 min_capacity = ULONG_MAX;10203 max_capacity = 0;1020410205 if (child->flags & SD_NUMA) {10206 /*10207 * SD_NUMA domains cannot assume that child groups10208 * span the current group.10209 */1021010211 for_each_cpu(cpu, sched_group_span(sdg)) {10212 unsigned long cpu_cap = capacity_of(cpu);1021310214 capacity += cpu_cap;10215 min_capacity = min(cpu_cap, min_capacity);10216 max_capacity = max(cpu_cap, max_capacity);10217 }10218 } else {10219 /*10220 * !SD_NUMA domains can assume that child groups10221 * span the current group.10222 */1022310224 group = child->groups;10225 do {10226 struct sched_group_capacity *sgc = group->sgc;1022710228 capacity += sgc->capacity;10229 min_capacity = min(sgc->min_capacity, min_capacity);10230 max_capacity = max(sgc->max_capacity, max_capacity);10231 group = group->next;10232 } while (group != child->groups);10233 }1023410235 sdg->sgc->capacity = capacity;10236 sdg->sgc->min_capacity = min_capacity;10237 sdg->sgc->max_capacity = max_capacity;10238 }1023910240 /*10241 * Check whether the capacity of the rq has been noticeably reduced by side10242 * activity. The imbalance_pct is used for the threshold.10243 * Return true is the capacity is reduced10244 */10245 static inline int10246 check_cpu_capacity(struct rq *rq, struct sched_domain *sd)10247 {10248 return ((rq->cpu_capacity * sd->imbalance_pct) <10249 (arch_scale_cpu_capacity(cpu_of(rq)) * 100));10250 }1025110252 /* Check if the rq has a misfit task */10253 static inline bool check_misfit_status(struct rq *rq)10254 {10255 return rq->misfit_task_load;10256 }1025710258 /*10259 * Group imbalance indicates (and tries to solve) the problem where balancing10260 * groups is inadequate due to ->cpus_ptr constraints.10261 *10262 * Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a10263 * cpumask covering 1 CPU of the first group and 3 CPUs of the second group.10264 * Something like:10265 *10266 * { 0 1 2 3 } { 4 5 6 7 }10267 * * * * *10268 *10269 * If we were to balance group-wise we'd place two tasks in the first group and10270 * two tasks in the second group. Clearly this is undesired as it will overload10271 * cpu 3 and leave one of the CPUs in the second group unused.10272 *10273 * The current solution to this issue is detecting the skew in the first group10274 * by noticing the lower domain failed to reach balance and had difficulty10275 * moving tasks due to affinity constraints.10276 *10277 * When this is so detected; this group becomes a candidate for busiest; see10278 * update_sd_pick_busiest(). And calculate_imbalance() and10279 * sched_balance_find_src_group() avoid some of the usual balance conditions to allow it10280 * to create an effective group imbalance.10281 *10282 * This is a somewhat tricky proposition since the next run might not find the10283 * group imbalance and decide the groups need to be balanced again. A most10284 * subtle and fragile situation.10285 */1028610287 static inline int sg_imbalanced(struct sched_group *group)10288 {10289 return group->sgc->imbalance;10290 }1029110292 /*10293 * group_has_capacity returns true if the group has spare capacity that could10294 * be used by some tasks.10295 * We consider that a group has spare capacity if the number of task is10296 * smaller than the number of CPUs or if the utilization is lower than the10297 * available capacity for CFS tasks.10298 * For the latter, we use a threshold to stabilize the state, to take into10299 * account the variance of the tasks' load and to return true if the available10300 * capacity in meaningful for the load balancer.10301 * As an example, an available capacity of 1% can appear but it doesn't make10302 * any benefit for the load balance.10303 */10304 static inline bool10305 group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)10306 {10307 if (sgs->sum_nr_running < sgs->group_weight)10308 return true;1030910310 if ((sgs->group_capacity * imbalance_pct) <10311 (sgs->group_runnable * 100))10312 return false;1031310314 if ((sgs->group_capacity * 100) >10315 (sgs->group_util * imbalance_pct))10316 return true;1031710318 return false;10319 }1032010321 /*10322 * group_is_overloaded returns true if the group has more tasks than it can10323 * handle.10324 * group_is_overloaded is not equals to !group_has_capacity because a group10325 * with the exact right number of tasks, has no more spare capacity but is not10326 * overloaded so both group_has_capacity and group_is_overloaded return10327 * false.10328 */10329 static inline bool10330 group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)10331 {10332 if (sgs->sum_nr_running <= sgs->group_weight)10333 return false;1033410335 if ((sgs->group_capacity * 100) <10336 (sgs->group_util * imbalance_pct))10337 return true;1033810339 if ((sgs->group_capacity * imbalance_pct) <10340 (sgs->group_runnable * 100))10341 return true;1034210343 return false;10344 }1034510346 static inline enum10347 group_type group_classify(unsigned int imbalance_pct,10348 struct sched_group *group,10349 struct sg_lb_stats *sgs)10350 {10351 if (group_is_overloaded(imbalance_pct, sgs))10352 return group_overloaded;1035310354 if (sg_imbalanced(group))10355 return group_imbalanced;1035610357 if (sgs->group_asym_packing)10358 return group_asym_packing;1035910360 if (sgs->group_smt_balance)10361 return group_smt_balance;1036210363 if (sgs->group_misfit_task_load)10364 return group_misfit_task;1036510366 if (!group_has_capacity(imbalance_pct, sgs))10367 return group_fully_busy;1036810369 return group_has_spare;10370 }1037110372 /**10373 * sched_use_asym_prio - Check whether asym_packing priority must be used10374 * @sd: The scheduling domain of the load balancing10375 * @cpu: A CPU10376 *10377 * Always use CPU priority when balancing load between SMT siblings. When10378 * balancing load between cores, it is not sufficient that @cpu is idle. Only10379 * use CPU priority if the whole core is idle.10380 *10381 * Returns: True if the priority of @cpu must be followed. False otherwise.10382 */10383 static bool sched_use_asym_prio(struct sched_domain *sd, int cpu)10384 {10385 if (!(sd->flags & SD_ASYM_PACKING))10386 return false;1038710388 if (!sched_smt_active())10389 return true;1039010391 return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu);10392 }1039310394 static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu)10395 {10396 /*10397 * First check if @dst_cpu can do asym_packing load balance. Only do it10398 * if it has higher priority than @src_cpu.10399 */10400 return sched_use_asym_prio(sd, dst_cpu) &&10401 sched_asym_prefer(dst_cpu, src_cpu);10402 }1040310404 /**10405 * sched_group_asym - Check if the destination CPU can do asym_packing balance10406 * @env: The load balancing environment10407 * @sgs: Load-balancing statistics of the candidate busiest group10408 * @group: The candidate busiest group10409 *10410 * @env::dst_cpu can do asym_packing if it has higher priority than the10411 * preferred CPU of @group.10412 *10413 * Return: true if @env::dst_cpu can do with asym_packing load balance. False10414 * otherwise.10415 */10416 static inline bool10417 sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group)10418 {10419 /*10420 * CPU priorities do not make sense for SMT cores with more than one10421 * busy sibling.10422 */10423 if ((group->flags & SD_SHARE_CPUCAPACITY) &&10424 (sgs->group_weight - sgs->idle_cpus != 1))10425 return false;1042610427 return sched_asym(env->sd, env->dst_cpu, READ_ONCE(group->asym_prefer_cpu));10428 }1042910430 /* One group has more than one SMT CPU while the other group does not */10431 static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1,10432 struct sched_group *sg2)10433 {10434 if (!sg1 || !sg2)10435 return false;1043610437 return (sg1->flags & SD_SHARE_CPUCAPACITY) !=10438 (sg2->flags & SD_SHARE_CPUCAPACITY);10439 }1044010441 static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs,10442 struct sched_group *group)10443 {10444 if (!env->idle)10445 return false;1044610447 /*10448 * For SMT source group, it is better to move a task10449 * to a CPU that doesn't have multiple tasks sharing its CPU capacity.10450 * Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY10451 * will not be on.10452 */10453 if (group->flags & SD_SHARE_CPUCAPACITY &&10454 sgs->sum_h_nr_running > 1)10455 return true;1045610457 return false;10458 }1045910460 static inline long sibling_imbalance(struct lb_env *env,10461 struct sd_lb_stats *sds,10462 struct sg_lb_stats *busiest,10463 struct sg_lb_stats *local)10464 {10465 int ncores_busiest, ncores_local;10466 long imbalance;1046710468 if (!env->idle || !busiest->sum_nr_running)10469 return 0;1047010471 ncores_busiest = sds->busiest->cores;10472 ncores_local = sds->local->cores;1047310474 if (ncores_busiest == ncores_local) {10475 imbalance = busiest->sum_nr_running;10476 lsub_positive(&imbalance, local->sum_nr_running);10477 return imbalance;10478 }1047910480 /* Balance such that nr_running/ncores ratio are same on both groups */10481 imbalance = ncores_local * busiest->sum_nr_running;10482 lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running);10483 /* Normalize imbalance and do rounding on normalization */10484 imbalance = 2 * imbalance + ncores_local + ncores_busiest;10485 imbalance /= ncores_local + ncores_busiest;1048610487 /* Take advantage of resource in an empty sched group */10488 if (imbalance <= 1 && local->sum_nr_running == 0 &&10489 busiest->sum_nr_running > 1)10490 imbalance = 2;1049110492 return imbalance;10493 }1049410495 static inline bool10496 sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)10497 {10498 /*10499 * When there is more than 1 task, the group_overloaded case already10500 * takes care of cpu with reduced capacity10501 */10502 if (rq->cfs.h_nr_runnable != 1)10503 return false;1050410505 return check_cpu_capacity(rq, sd);10506 }1050710508 /**10509 * update_sg_lb_stats - Update sched_group's statistics for load balancing.10510 * @env: The load balancing environment.10511 * @sds: Load-balancing data with statistics of the local group.10512 * @group: sched_group whose statistics are to be updated.10513 * @sgs: variable to hold the statistics for this group.10514 * @sg_overloaded: sched_group is overloaded10515 * @sg_overutilized: sched_group is overutilized10516 */10517 static inline void update_sg_lb_stats(struct lb_env *env,10518 struct sd_lb_stats *sds,10519 struct sched_group *group,10520 struct sg_lb_stats *sgs,10521 bool *sg_overloaded,10522 bool *sg_overutilized)10523 {10524 int i, nr_running, local_group, sd_flags = env->sd->flags;10525 bool balancing_at_rd = !env->sd->parent;1052610527 memset(sgs, 0, sizeof(*sgs));1052810529 local_group = group == sds->local;1053010531 for_each_cpu_and(i, sched_group_span(group), env->cpus) {10532 struct rq *rq = cpu_rq(i);10533 unsigned long load = cpu_load(rq);1053410535 sgs->group_load += load;10536 sgs->group_util += cpu_util_cfs(i);10537 sgs->group_runnable += cpu_runnable(rq);10538 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable;1053910540 nr_running = rq->nr_running;10541 sgs->sum_nr_running += nr_running;1054210543 if (cpu_overutilized(i))10544 *sg_overutilized = 1;1054510546 /*10547 * No need to call idle_cpu() if nr_running is not 010548 */10549 if (!nr_running && idle_cpu(i)) {10550 sgs->idle_cpus++;10551 /* Idle cpu can't have misfit task */10552 continue;10553 }1055410555 /* Overload indicator is only updated at root domain */10556 if (balancing_at_rd && nr_running > 1)10557 *sg_overloaded = 1;1055810559 #ifdef CONFIG_NUMA_BALANCING10560 /* Only fbq_classify_group() uses this to classify NUMA groups */10561 if (sd_flags & SD_NUMA) {10562 sgs->nr_numa_running += rq->nr_numa_running;10563 sgs->nr_preferred_running += rq->nr_preferred_running;10564 }10565 #endif10566 if (local_group)10567 continue;1056810569 if (sd_flags & SD_ASYM_CPUCAPACITY) {10570 /* Check for a misfit task on the cpu */10571 if (sgs->group_misfit_task_load < rq->misfit_task_load) {10572 sgs->group_misfit_task_load = rq->misfit_task_load;10573 *sg_overloaded = 1;10574 }10575 } else if (env->idle && sched_reduced_capacity(rq, env->sd)) {10576 /* Check for a task running on a CPU with reduced capacity */10577 if (sgs->group_misfit_task_load < load)10578 sgs->group_misfit_task_load = load;10579 }10580 }1058110582 sgs->group_capacity = group->sgc->capacity;1058310584 sgs->group_weight = group->group_weight;1058510586 /* Check if dst CPU is idle and preferred to this group */10587 if (!local_group && env->idle && sgs->sum_h_nr_running &&10588 sched_group_asym(env, sgs, group))10589 sgs->group_asym_packing = 1;1059010591 /* Check for loaded SMT group to be balanced to dst CPU */10592 if (!local_group && smt_balance(env, sgs, group))10593 sgs->group_smt_balance = 1;1059410595 sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);1059610597 /* Computing avg_load makes sense only when group is overloaded */10598 if (sgs->group_type == group_overloaded)10599 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /10600 sgs->group_capacity;10601 }1060210603 /**10604 * update_sd_pick_busiest - return 1 on busiest group10605 * @env: The load balancing environment.10606 * @sds: sched_domain statistics10607 * @sg: sched_group candidate to be checked for being the busiest10608 * @sgs: sched_group statistics10609 *10610 * Determine if @sg is a busier group than the previously selected10611 * busiest group.10612 *10613 * Return: %true if @sg is a busier group than the previously selected10614 * busiest group. %false otherwise.10615 */10616 static bool update_sd_pick_busiest(struct lb_env *env,10617 struct sd_lb_stats *sds,10618 struct sched_group *sg,10619 struct sg_lb_stats *sgs)10620 {10621 struct sg_lb_stats *busiest = &sds->busiest_stat;1062210623 /* Make sure that there is at least one task to pull */10624 if (!sgs->sum_h_nr_running)10625 return false;1062610627 /*10628 * Don't try to pull misfit tasks we can't help.10629 * We can use max_capacity here as reduction in capacity on some10630 * CPUs in the group should either be possible to resolve10631 * internally or be covered by avg_load imbalance (eventually).10632 */10633 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&10634 (sgs->group_type == group_misfit_task) &&10635 (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||10636 sds->local_stat.group_type != group_has_spare))10637 return false;1063810639 if (sgs->group_type > busiest->group_type)10640 return true;1064110642 if (sgs->group_type < busiest->group_type)10643 return false;1064410645 /*10646 * The candidate and the current busiest group are the same type of10647 * group. Let check which one is the busiest according to the type.10648 */1064910650 switch (sgs->group_type) {10651 case group_overloaded:10652 /* Select the overloaded group with highest avg_load. */10653 return sgs->avg_load > busiest->avg_load;1065410655 case group_imbalanced:10656 /*10657 * Select the 1st imbalanced group as we don't have any way to10658 * choose one more than another.10659 */10660 return false;1066110662 case group_asym_packing:10663 /* Prefer to move from lowest priority CPU's work */10664 return sched_asym_prefer(READ_ONCE(sds->busiest->asym_prefer_cpu),10665 READ_ONCE(sg->asym_prefer_cpu));1066610667 case group_misfit_task:10668 /*10669 * If we have more than one misfit sg go with the biggest10670 * misfit.10671 */10672 return sgs->group_misfit_task_load > busiest->group_misfit_task_load;1067310674 case group_smt_balance:10675 /*10676 * Check if we have spare CPUs on either SMT group to10677 * choose has spare or fully busy handling.10678 */10679 if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0)10680 goto has_spare;1068110682 fallthrough;1068310684 case group_fully_busy:10685 /*10686 * Select the fully busy group with highest avg_load. In10687 * theory, there is no need to pull task from such kind of10688 * group because tasks have all compute capacity that they need10689 * but we can still improve the overall throughput by reducing10690 * contention when accessing shared HW resources.10691 *10692 * XXX for now avg_load is not computed and always 0 so we10693 * select the 1st one, except if @sg is composed of SMT10694 * siblings.10695 */1069610697 if (sgs->avg_load < busiest->avg_load)10698 return false;1069910700 if (sgs->avg_load == busiest->avg_load) {10701 /*10702 * SMT sched groups need more help than non-SMT groups.10703 * If @sg happens to also be SMT, either choice is good.10704 */10705 if (sds->busiest->flags & SD_SHARE_CPUCAPACITY)10706 return false;10707 }1070810709 break;1071010711 case group_has_spare:10712 /*10713 * Do not pick sg with SMT CPUs over sg with pure CPUs,10714 * as we do not want to pull task off SMT core with one task10715 * and make the core idle.10716 */10717 if (smt_vs_nonsmt_groups(sds->busiest, sg)) {10718 if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1)10719 return false;10720 else10721 return true;10722 }10723 has_spare:1072410725 /*10726 * Select not overloaded group with lowest number of idle CPUs10727 * and highest number of running tasks. We could also compare10728 * the spare capacity which is more stable but it can end up10729 * that the group has less spare capacity but finally more idle10730 * CPUs which means less opportunity to pull tasks.10731 */10732 if (sgs->idle_cpus > busiest->idle_cpus)10733 return false;10734 else if ((sgs->idle_cpus == busiest->idle_cpus) &&10735 (sgs->sum_nr_running <= busiest->sum_nr_running))10736 return false;1073710738 break;10739 }1074010741 /*10742 * Candidate sg has no more than one task per CPU and has higher10743 * per-CPU capacity. Migrating tasks to less capable CPUs may harm10744 * throughput. Maximize throughput, power/energy consequences are not10745 * considered.10746 */10747 if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&10748 (sgs->group_type <= group_fully_busy) &&10749 (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))10750 return false;1075110752 return true;10753 }1075410755 #ifdef CONFIG_NUMA_BALANCING10756 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)10757 {10758 if (sgs->sum_h_nr_running > sgs->nr_numa_running)10759 return regular;10760 if (sgs->sum_h_nr_running > sgs->nr_preferred_running)10761 return remote;10762 return all;10763 }1076410765 static inline enum fbq_type fbq_classify_rq(struct rq *rq)10766 {10767 if (rq->nr_running > rq->nr_numa_running)10768 return regular;10769 if (rq->nr_running > rq->nr_preferred_running)10770 return remote;10771 return all;10772 }10773 #else /* !CONFIG_NUMA_BALANCING: */10774 static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)10775 {10776 return all;10777 }1077810779 static inline enum fbq_type fbq_classify_rq(struct rq *rq)10780 {10781 return regular;10782 }10783 #endif /* !CONFIG_NUMA_BALANCING */107841078510786 struct sg_lb_stats;1078710788 /*10789 * task_running_on_cpu - return 1 if @p is running on @cpu.10790 */1079110792 static unsigned int task_running_on_cpu(int cpu, struct task_struct *p)10793 {10794 /* Task has no contribution or is new */10795 if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))10796 return 0;1079710798 if (task_on_rq_queued(p))10799 return 1;1080010801 return 0;10802 }1080310804 /**10805 * idle_cpu_without - would a given CPU be idle without p ?10806 * @cpu: the processor on which idleness is tested.10807 * @p: task which should be ignored.10808 *10809 * Return: 1 if the CPU would be idle. 0 otherwise.10810 */10811 static int idle_cpu_without(int cpu, struct task_struct *p)10812 {10813 struct rq *rq = cpu_rq(cpu);1081410815 if (rq->curr != rq->idle && rq->curr != p)10816 return 0;1081710818 /*10819 * rq->nr_running can't be used but an updated version without the10820 * impact of p on cpu must be used instead. The updated nr_running10821 * be computed and tested before calling idle_cpu_without().10822 */1082310824 if (rq->ttwu_pending)10825 return 0;1082610827 return 1;10828 }1082910830 /*10831 * update_sg_wakeup_stats - Update sched_group's statistics for wakeup.10832 * @sd: The sched_domain level to look for idlest group.10833 * @group: sched_group whose statistics are to be updated.10834 * @sgs: variable to hold the statistics for this group.10835 * @p: The task for which we look for the idlest group/CPU.10836 */10837 static inline void update_sg_wakeup_stats(struct sched_domain *sd,10838 struct sched_group *group,10839 struct sg_lb_stats *sgs,10840 struct task_struct *p)10841 {10842 int i, nr_running;1084310844 memset(sgs, 0, sizeof(*sgs));1084510846 /* Assume that task can't fit any CPU of the group */10847 if (sd->flags & SD_ASYM_CPUCAPACITY)10848 sgs->group_misfit_task_load = 1;1084910850 for_each_cpu(i, sched_group_span(group)) {10851 struct rq *rq = cpu_rq(i);10852 unsigned int local;1085310854 sgs->group_load += cpu_load_without(rq, p);10855 sgs->group_util += cpu_util_without(i, p);10856 sgs->group_runnable += cpu_runnable_without(rq, p);10857 local = task_running_on_cpu(i, p);10858 sgs->sum_h_nr_running += rq->cfs.h_nr_runnable - local;1085910860 nr_running = rq->nr_running - local;10861 sgs->sum_nr_running += nr_running;1086210863 /*10864 * No need to call idle_cpu_without() if nr_running is not 010865 */10866 if (!nr_running && idle_cpu_without(i, p))10867 sgs->idle_cpus++;1086810869 /* Check if task fits in the CPU */10870 if (sd->flags & SD_ASYM_CPUCAPACITY &&10871 sgs->group_misfit_task_load &&10872 task_fits_cpu(p, i))10873 sgs->group_misfit_task_load = 0;1087410875 }1087610877 sgs->group_capacity = group->sgc->capacity;1087810879 sgs->group_weight = group->group_weight;1088010881 sgs->group_type = group_classify(sd->imbalance_pct, group, sgs);1088210883 /*10884 * Computing avg_load makes sense only when group is fully busy or10885 * overloaded10886 */10887 if (sgs->group_type == group_fully_busy ||10888 sgs->group_type == group_overloaded)10889 sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /10890 sgs->group_capacity;10891 }1089210893 static bool update_pick_idlest(struct sched_group *idlest,10894 struct sg_lb_stats *idlest_sgs,10895 struct sched_group *group,10896 struct sg_lb_stats *sgs)10897 {10898 if (sgs->group_type < idlest_sgs->group_type)10899 return true;1090010901 if (sgs->group_type > idlest_sgs->group_type)10902 return false;1090310904 /*10905 * The candidate and the current idlest group are the same type of10906 * group. Let check which one is the idlest according to the type.10907 */1090810909 switch (sgs->group_type) {10910 case group_overloaded:10911 case group_fully_busy:10912 /* Select the group with lowest avg_load. */10913 if (idlest_sgs->avg_load <= sgs->avg_load)10914 return false;10915 break;1091610917 case group_imbalanced:10918 case group_asym_packing:10919 case group_smt_balance:10920 /* Those types are not used in the slow wakeup path */10921 return false;1092210923 case group_misfit_task:10924 /* Select group with the highest max capacity */10925 if (idlest->sgc->max_capacity >= group->sgc->max_capacity)10926 return false;10927 break;1092810929 case group_has_spare:10930 /* Select group with most idle CPUs */10931 if (idlest_sgs->idle_cpus > sgs->idle_cpus)10932 return false;1093310934 /* Select group with lowest group_util */10935 if (idlest_sgs->idle_cpus == sgs->idle_cpus &&10936 idlest_sgs->group_util <= sgs->group_util)10937 return false;1093810939 break;10940 }1094110942 return true;10943 }1094410945 /*10946 * sched_balance_find_dst_group() finds and returns the least busy CPU group within the10947 * domain.10948 *10949 * Assumes p is allowed on at least one CPU in sd.10950 */10951 static struct sched_group *10952 sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)10953 {10954 struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups;10955 struct sg_lb_stats local_sgs, tmp_sgs;10956 struct sg_lb_stats *sgs;10957 unsigned long imbalance;10958 struct sg_lb_stats idlest_sgs = {10959 .avg_load = UINT_MAX,10960 .group_type = group_overloaded,10961 };1096210963 do {10964 int local_group;1096510966 /* Skip over this group if it has no CPUs allowed */10967 if (!cpumask_intersects(sched_group_span(group),10968 p->cpus_ptr))10969 continue;1097010971 /* Skip over this group if no cookie matched */10972 if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))10973 continue;1097410975 local_group = cpumask_test_cpu(this_cpu,10976 sched_group_span(group));1097710978 if (local_group) {10979 sgs = &local_sgs;10980 local = group;10981 } else {10982 sgs = &tmp_sgs;10983 }1098410985 update_sg_wakeup_stats(sd, group, sgs, p);1098610987 if (!local_group && update_pick_idlest(idlest, &idlest_sgs, group, sgs)) {10988 idlest = group;10989 idlest_sgs = *sgs;10990 }1099110992 } while (group = group->next, group != sd->groups);109931099410995 /* There is no idlest group to push tasks to */10996 if (!idlest)10997 return NULL;1099810999 /* The local group has been skipped because of CPU affinity */11000 if (!local)11001 return idlest;1100211003 /*11004 * If the local group is idler than the selected idlest group11005 * don't try and push the task.11006 */11007 if (local_sgs.group_type < idlest_sgs.group_type)11008 return NULL;1100911010 /*11011 * If the local group is busier than the selected idlest group11012 * try and push the task.11013 */11014 if (local_sgs.group_type > idlest_sgs.group_type)11015 return idlest;1101611017 switch (local_sgs.group_type) {11018 case group_overloaded:11019 case group_fully_busy:1102011021 /* Calculate allowed imbalance based on load */11022 imbalance = scale_load_down(NICE_0_LOAD) *11023 (sd->imbalance_pct-100) / 100;1102411025 /*11026 * When comparing groups across NUMA domains, it's possible for11027 * the local domain to be very lightly loaded relative to the11028 * remote domains but "imbalance" skews the comparison making11029 * remote CPUs look much more favourable. When considering11030 * cross-domain, add imbalance to the load on the remote node11031 * and consider staying local.11032 */1103311034 if ((sd->flags & SD_NUMA) &&11035 ((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load))11036 return NULL;1103711038 /*11039 * If the local group is less loaded than the selected11040 * idlest group don't try and push any tasks.11041 */11042 if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance))11043 return NULL;1104411045 if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load)11046 return NULL;11047 break;1104811049 case group_imbalanced:11050 case group_asym_packing:11051 case group_smt_balance:11052 /* Those type are not used in the slow wakeup path */11053 return NULL;1105411055 case group_misfit_task:11056 /* Select group with the highest max capacity */11057 if (local->sgc->max_capacity >= idlest->sgc->max_capacity)11058 return NULL;11059 break;1106011061 case group_has_spare:11062 #ifdef CONFIG_NUMA11063 if (sd->flags & SD_NUMA) {11064 int imb_numa_nr = sd->imb_numa_nr;11065 #ifdef CONFIG_NUMA_BALANCING11066 int idlest_cpu;11067 /*11068 * If there is spare capacity at NUMA, try to select11069 * the preferred node11070 */11071 if (cpu_to_node(this_cpu) == p->numa_preferred_nid)11072 return NULL;1107311074 idlest_cpu = cpumask_first(sched_group_span(idlest));11075 if (cpu_to_node(idlest_cpu) == p->numa_preferred_nid)11076 return idlest;11077 #endif /* CONFIG_NUMA_BALANCING */11078 /*11079 * Otherwise, keep the task close to the wakeup source11080 * and improve locality if the number of running tasks11081 * would remain below threshold where an imbalance is11082 * allowed while accounting for the possibility the11083 * task is pinned to a subset of CPUs. If there is a11084 * real need of migration, periodic load balance will11085 * take care of it.11086 */11087 if (p->nr_cpus_allowed != NR_CPUS) {11088 struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);1108911090 cpumask_and(cpus, sched_group_span(local), p->cpus_ptr);11091 imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr);11092 }1109311094 imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus);11095 if (!adjust_numa_imbalance(imbalance,11096 local_sgs.sum_nr_running + 1,11097 imb_numa_nr)) {11098 return NULL;11099 }11100 }11101 #endif /* CONFIG_NUMA */1110211103 /*11104 * Select group with highest number of idle CPUs. We could also11105 * compare the utilization which is more stable but it can end11106 * up that the group has less spare capacity but finally more11107 * idle CPUs which means more opportunity to run task.11108 */11109 if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus)11110 return NULL;11111 break;11112 }1111311114 return idlest;11115 }1111611117 static void update_idle_cpu_scan(struct lb_env *env,11118 unsigned long sum_util)11119 {11120 struct sched_domain_shared *sd_share;11121 int llc_weight, pct;11122 u64 x, y, tmp;11123 /*11124 * Update the number of CPUs to scan in LLC domain, which could11125 * be used as a hint in select_idle_cpu(). The update of sd_share11126 * could be expensive because it is within a shared cache line.11127 * So the write of this hint only occurs during periodic load11128 * balancing, rather than CPU_NEWLY_IDLE, because the latter11129 * can fire way more frequently than the former.11130 */11131 if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)11132 return;1113311134 llc_weight = per_cpu(sd_llc_size, env->dst_cpu);11135 if (env->sd->span_weight != llc_weight)11136 return;1113711138 sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu));11139 if (!sd_share)11140 return;1114111142 /*11143 * The number of CPUs to search drops as sum_util increases, when11144 * sum_util hits 85% or above, the scan stops.11145 * The reason to choose 85% as the threshold is because this is the11146 * imbalance_pct(117) when a LLC sched group is overloaded.11147 *11148 * let y = SCHED_CAPACITY_SCALE - p * x^2 [1]11149 * and y'= y / SCHED_CAPACITY_SCALE11150 *11151 * x is the ratio of sum_util compared to the CPU capacity:11152 * x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE)11153 * y' is the ratio of CPUs to be scanned in the LLC domain,11154 * and the number of CPUs to scan is calculated by:11155 *11156 * nr_scan = llc_weight * y' [2]11157 *11158 * When x hits the threshold of overloaded, AKA, when11159 * x = 100 / pct, y drops to 0. According to [1],11160 * p should be SCHED_CAPACITY_SCALE * pct^2 / 1000011161 *11162 * Scale x by SCHED_CAPACITY_SCALE:11163 * x' = sum_util / llc_weight; [3]11164 *11165 * and finally [1] becomes:11166 * y = SCHED_CAPACITY_SCALE -11167 * x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4]11168 *11169 */11170 /* equation [3] */11171 x = sum_util;11172 do_div(x, llc_weight);1117311174 /* equation [4] */11175 pct = env->sd->imbalance_pct;11176 tmp = x * x * pct * pct;11177 do_div(tmp, 10000 * SCHED_CAPACITY_SCALE);11178 tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE);11179 y = SCHED_CAPACITY_SCALE - tmp;1118011181 /* equation [2] */11182 y *= llc_weight;11183 do_div(y, SCHED_CAPACITY_SCALE);11184 if ((int)y != sd_share->nr_idle_scan)11185 WRITE_ONCE(sd_share->nr_idle_scan, (int)y);11186 }1118711188 /**11189 * update_sd_lb_stats - Update sched_domain's statistics for load balancing.11190 * @env: The load balancing environment.11191 * @sds: variable to hold the statistics for this sched_domain.11192 */1119311194 static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)11195 {11196 struct sched_group *sg = env->sd->groups;11197 struct sg_lb_stats *local = &sds->local_stat;11198 struct sg_lb_stats tmp_sgs;11199 unsigned long sum_util = 0;11200 bool sg_overloaded = 0, sg_overutilized = 0;1120111202 do {11203 struct sg_lb_stats *sgs = &tmp_sgs;11204 int local_group;1120511206 local_group = cpumask_test_cpu(env->dst_cpu, sched_group_span(sg));11207 if (local_group) {11208 sds->local = sg;11209 sgs = local;1121011211 if (env->idle != CPU_NEWLY_IDLE ||11212 time_after_eq(jiffies, sg->sgc->next_update))11213 update_group_capacity(env->sd, env->dst_cpu);11214 }1121511216 update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized);1121711218 if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {11219 sds->busiest = sg;11220 sds->busiest_stat = *sgs;11221 }1122211223 /* Now, start updating sd_lb_stats */11224 sds->total_load += sgs->group_load;11225 sds->total_capacity += sgs->group_capacity;1122611227 sum_util += sgs->group_util;11228 sg = sg->next;11229 } while (sg != env->sd->groups);1123011231 /*11232 * Indicate that the child domain of the busiest group prefers tasks11233 * go to a child's sibling domains first. NB the flags of a sched group11234 * are those of the child domain.11235 */11236 if (sds->busiest)11237 sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING);112381123911240 if (env->sd->flags & SD_NUMA)11241 env->fbq_type = fbq_classify_group(&sds->busiest_stat);1124211243 if (!env->sd->parent) {11244 /* update overload indicator if we are at root domain */11245 set_rd_overloaded(env->dst_rq->rd, sg_overloaded);1124611247 /* Update over-utilization (tipping point, U >= 0) indicator */11248 set_rd_overutilized(env->dst_rq->rd, sg_overutilized);11249 } else if (sg_overutilized) {11250 set_rd_overutilized(env->dst_rq->rd, sg_overutilized);11251 }1125211253 update_idle_cpu_scan(env, sum_util);11254 }1125511256 /**11257 * calculate_imbalance - Calculate the amount of imbalance present within the11258 * groups of a given sched_domain during load balance.11259 * @env: load balance environment11260 * @sds: statistics of the sched_domain whose imbalance is to be calculated.11261 */11262 static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)11263 {11264 struct sg_lb_stats *local, *busiest;1126511266 local = &sds->local_stat;11267 busiest = &sds->busiest_stat;1126811269 if (busiest->group_type == group_misfit_task) {11270 if (env->sd->flags & SD_ASYM_CPUCAPACITY) {11271 /* Set imbalance to allow misfit tasks to be balanced. */11272 env->migration_type = migrate_misfit;11273 env->imbalance = 1;11274 } else {11275 /*11276 * Set load imbalance to allow moving task from cpu11277 * with reduced capacity.11278 */11279 env->migration_type = migrate_load;11280 env->imbalance = busiest->group_misfit_task_load;11281 }11282 return;11283 }1128411285 if (busiest->group_type == group_asym_packing) {11286 /*11287 * In case of asym capacity, we will try to migrate all load to11288 * the preferred CPU.11289 */11290 env->migration_type = migrate_task;11291 env->imbalance = busiest->sum_h_nr_running;11292 return;11293 }1129411295 if (busiest->group_type == group_smt_balance) {11296 /* Reduce number of tasks sharing CPU capacity */11297 env->migration_type = migrate_task;11298 env->imbalance = 1;11299 return;11300 }1130111302 if (busiest->group_type == group_imbalanced) {11303 /*11304 * In the group_imb case we cannot rely on group-wide averages11305 * to ensure CPU-load equilibrium, try to move any task to fix11306 * the imbalance. The next load balance will take care of11307 * balancing back the system.11308 */11309 env->migration_type = migrate_task;11310 env->imbalance = 1;11311 return;11312 }1131311314 /*11315 * Try to use spare capacity of local group without overloading it or11316 * emptying busiest.11317 */11318 if (local->group_type == group_has_spare) {11319 if ((busiest->group_type > group_fully_busy) &&11320 !(env->sd->flags & SD_SHARE_LLC)) {11321 /*11322 * If busiest is overloaded, try to fill spare11323 * capacity. This might end up creating spare capacity11324 * in busiest or busiest still being overloaded but11325 * there is no simple way to directly compute the11326 * amount of load to migrate in order to balance the11327 * system.11328 */11329 env->migration_type = migrate_util;11330 env->imbalance = max(local->group_capacity, local->group_util) -11331 local->group_util;1133211333 /*11334 * In some cases, the group's utilization is max or even11335 * higher than capacity because of migrations but the11336 * local CPU is (newly) idle. There is at least one11337 * waiting task in this overloaded busiest group. Let's11338 * try to pull it.11339 */11340 if (env->idle && env->imbalance == 0) {11341 env->migration_type = migrate_task;11342 env->imbalance = 1;11343 }1134411345 return;11346 }1134711348 if (busiest->group_weight == 1 || sds->prefer_sibling) {11349 /*11350 * When prefer sibling, evenly spread running tasks on11351 * groups.11352 */11353 env->migration_type = migrate_task;11354 env->imbalance = sibling_imbalance(env, sds, busiest, local);11355 } else {1135611357 /*11358 * If there is no overload, we just want to even the number of11359 * idle CPUs.11360 */11361 env->migration_type = migrate_task;11362 env->imbalance = max_t(long, 0,11363 (local->idle_cpus - busiest->idle_cpus));11364 }1136511366 #ifdef CONFIG_NUMA11367 /* Consider allowing a small imbalance between NUMA groups */11368 if (env->sd->flags & SD_NUMA) {11369 env->imbalance = adjust_numa_imbalance(env->imbalance,11370 local->sum_nr_running + 1,11371 env->sd->imb_numa_nr);11372 }11373 #endif1137411375 /* Number of tasks to move to restore balance */11376 env->imbalance >>= 1;1137711378 return;11379 }1138011381 /*11382 * Local is fully busy but has to take more load to relieve the11383 * busiest group11384 */11385 if (local->group_type < group_overloaded) {11386 /*11387 * Local will become overloaded so the avg_load metrics are11388 * finally needed.11389 */1139011391 local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /11392 local->group_capacity;1139311394 /*11395 * If the local group is more loaded than the selected11396 * busiest group don't try to pull any tasks.11397 */11398 if (local->avg_load >= busiest->avg_load) {11399 env->imbalance = 0;11400 return;11401 }1140211403 sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /11404 sds->total_capacity;1140511406 /*11407 * If the local group is more loaded than the average system11408 * load, don't try to pull any tasks.11409 */11410 if (local->avg_load >= sds->avg_load) {11411 env->imbalance = 0;11412 return;11413 }1141411415 }1141611417 /*11418 * Both group are or will become overloaded and we're trying to get all11419 * the CPUs to the average_load, so we don't want to push ourselves11420 * above the average load, nor do we wish to reduce the max loaded CPU11421 * below the average load. At the same time, we also don't want to11422 * reduce the group load below the group capacity. Thus we look for11423 * the minimum possible imbalance.11424 */11425 env->migration_type = migrate_load;11426 env->imbalance = min(11427 (busiest->avg_load - sds->avg_load) * busiest->group_capacity,11428 (sds->avg_load - local->avg_load) * local->group_capacity11429 ) / SCHED_CAPACITY_SCALE;11430 }1143111432 /******* sched_balance_find_src_group() helpers end here *********************/1143311434 /*11435 * Decision matrix according to the local and busiest group type:11436 *11437 * busiest \ local has_spare fully_busy misfit asym imbalanced overloaded11438 * has_spare nr_idle balanced N/A N/A balanced balanced11439 * fully_busy nr_idle nr_idle N/A N/A balanced balanced11440 * misfit_task force N/A N/A N/A N/A N/A11441 * asym_packing force force N/A N/A force force11442 * imbalanced force force N/A N/A force force11443 * overloaded force force N/A N/A force avg_load11444 *11445 * N/A : Not Applicable because already filtered while updating11446 * statistics.11447 * balanced : The system is balanced for these 2 groups.11448 * force : Calculate the imbalance as load migration is probably needed.11449 * avg_load : Only if imbalance is significant enough.11450 * nr_idle : dst_cpu is not busy and the number of idle CPUs is quite11451 * different in groups.11452 */1145311454 /**11455 * sched_balance_find_src_group - Returns the busiest group within the sched_domain11456 * if there is an imbalance.11457 * @env: The load balancing environment.11458 *11459 * Also calculates the amount of runnable load which should be moved11460 * to restore balance.11461 *11462 * Return: - The busiest group if imbalance exists.11463 */11464 static struct sched_group *sched_balance_find_src_group(struct lb_env *env)11465 {11466 struct sg_lb_stats *local, *busiest;11467 struct sd_lb_stats sds;1146811469 init_sd_lb_stats(&sds);1147011471 /*11472 * Compute the various statistics relevant for load balancing at11473 * this level.11474 */11475 update_sd_lb_stats(env, &sds);1147611477 /* There is no busy sibling group to pull tasks from */11478 if (!sds.busiest)11479 goto out_balanced;1148011481 busiest = &sds.busiest_stat;1148211483 /* Misfit tasks should be dealt with regardless of the avg load */11484 if (busiest->group_type == group_misfit_task)11485 goto force_balance;1148611487 if (!is_rd_overutilized(env->dst_rq->rd) &&11488 rcu_dereference(env->dst_rq->rd->pd))11489 goto out_balanced;1149011491 /* ASYM feature bypasses nice load balance check */11492 if (busiest->group_type == group_asym_packing)11493 goto force_balance;1149411495 /*11496 * If the busiest group is imbalanced the below checks don't11497 * work because they assume all things are equal, which typically11498 * isn't true due to cpus_ptr constraints and the like.11499 */11500 if (busiest->group_type == group_imbalanced)11501 goto force_balance;1150211503 local = &sds.local_stat;11504 /*11505 * If the local group is busier than the selected busiest group11506 * don't try and pull any tasks.11507 */11508 if (local->group_type > busiest->group_type)11509 goto out_balanced;1151011511 /*11512 * When groups are overloaded, use the avg_load to ensure fairness11513 * between tasks.11514 */11515 if (local->group_type == group_overloaded) {11516 /*11517 * If the local group is more loaded than the selected11518 * busiest group don't try to pull any tasks.11519 */11520 if (local->avg_load >= busiest->avg_load)11521 goto out_balanced;1152211523 /* XXX broken for overlapping NUMA groups */11524 sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) /11525 sds.total_capacity;1152611527 /*11528 * Don't pull any tasks if this group is already above the11529 * domain average load.11530 */11531 if (local->avg_load >= sds.avg_load)11532 goto out_balanced;1153311534 /*11535 * If the busiest group is more loaded, use imbalance_pct to be11536 * conservative.11537 */11538 if (100 * busiest->avg_load <=11539 env->sd->imbalance_pct * local->avg_load)11540 goto out_balanced;11541 }1154211543 /*11544 * Try to move all excess tasks to a sibling domain of the busiest11545 * group's child domain.11546 */11547 if (sds.prefer_sibling && local->group_type == group_has_spare &&11548 sibling_imbalance(env, &sds, busiest, local) > 1)11549 goto force_balance;1155011551 if (busiest->group_type != group_overloaded) {11552 if (!env->idle) {11553 /*11554 * If the busiest group is not overloaded (and as a11555 * result the local one too) but this CPU is already11556 * busy, let another idle CPU try to pull task.11557 */11558 goto out_balanced;11559 }1156011561 if (busiest->group_type == group_smt_balance &&11562 smt_vs_nonsmt_groups(sds.local, sds.busiest)) {11563 /* Let non SMT CPU pull from SMT CPU sharing with sibling */11564 goto force_balance;11565 }1156611567 if (busiest->group_weight > 1 &&11568 local->idle_cpus <= (busiest->idle_cpus + 1)) {11569 /*11570 * If the busiest group is not overloaded11571 * and there is no imbalance between this and busiest11572 * group wrt idle CPUs, it is balanced. The imbalance11573 * becomes significant if the diff is greater than 111574 * otherwise we might end up to just move the imbalance11575 * on another group. Of course this applies only if11576 * there is more than 1 CPU per group.11577 */11578 goto out_balanced;11579 }1158011581 if (busiest->sum_h_nr_running == 1) {11582 /*11583 * busiest doesn't have any tasks waiting to run11584 */11585 goto out_balanced;11586 }11587 }1158811589 force_balance:11590 /* Looks like there is an imbalance. Compute it */11591 calculate_imbalance(env, &sds);11592 return env->imbalance ? sds.busiest : NULL;1159311594 out_balanced:11595 env->imbalance = 0;11596 return NULL;11597 }1159811599 /*11600 * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group.11601 */11602 static struct rq *sched_balance_find_src_rq(struct lb_env *env,11603 struct sched_group *group)11604 {11605 struct rq *busiest = NULL, *rq;11606 unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;11607 unsigned int busiest_nr = 0;11608 int i;1160911610 for_each_cpu_and(i, sched_group_span(group), env->cpus) {11611 unsigned long capacity, load, util;11612 unsigned int nr_running;11613 enum fbq_type rt;1161411615 rq = cpu_rq(i);11616 rt = fbq_classify_rq(rq);1161711618 /*11619 * We classify groups/runqueues into three groups:11620 * - regular: there are !numa tasks11621 * - remote: there are numa tasks that run on the 'wrong' node11622 * - all: there is no distinction11623 *11624 * In order to avoid migrating ideally placed numa tasks,11625 * ignore those when there's better options.11626 *11627 * If we ignore the actual busiest queue to migrate another11628 * task, the next balance pass can still reduce the busiest11629 * queue by moving tasks around inside the node.11630 *11631 * If we cannot move enough load due to this classification11632 * the next pass will adjust the group classification and11633 * allow migration of more tasks.11634 *11635 * Both cases only affect the total convergence complexity.11636 */11637 if (rt > env->fbq_type)11638 continue;1163911640 nr_running = rq->cfs.h_nr_runnable;11641 if (!nr_running)11642 continue;1164311644 capacity = capacity_of(i);1164511646 /*11647 * For ASYM_CPUCAPACITY domains, don't pick a CPU that could11648 * eventually lead to active_balancing high->low capacity.11649 * Higher per-CPU capacity is considered better than balancing11650 * average load.11651 */11652 if (env->sd->flags & SD_ASYM_CPUCAPACITY &&11653 !capacity_greater(capacity_of(env->dst_cpu), capacity) &&11654 nr_running == 1)11655 continue;1165611657 /*11658 * Make sure we only pull tasks from a CPU of lower priority11659 * when balancing between SMT siblings.11660 *11661 * If balancing between cores, let lower priority CPUs help11662 * SMT cores with more than one busy sibling.11663 */11664 if (sched_asym(env->sd, i, env->dst_cpu) && nr_running == 1)11665 continue;1166611667 switch (env->migration_type) {11668 case migrate_load:11669 /*11670 * When comparing with load imbalance, use cpu_load()11671 * which is not scaled with the CPU capacity.11672 */11673 load = cpu_load(rq);1167411675 if (nr_running == 1 && load > env->imbalance &&11676 !check_cpu_capacity(rq, env->sd))11677 break;1167811679 /*11680 * For the load comparisons with the other CPUs,11681 * consider the cpu_load() scaled with the CPU11682 * capacity, so that the load can be moved away11683 * from the CPU that is potentially running at a11684 * lower capacity.11685 *11686 * Thus we're looking for max(load_i / capacity_i),11687 * crosswise multiplication to rid ourselves of the11688 * division works out to:11689 * load_i * capacity_j > load_j * capacity_i;11690 * where j is our previous maximum.11691 */11692 if (load * busiest_capacity > busiest_load * capacity) {11693 busiest_load = load;11694 busiest_capacity = capacity;11695 busiest = rq;11696 }11697 break;1169811699 case migrate_util:11700 util = cpu_util_cfs_boost(i);1170111702 /*11703 * Don't try to pull utilization from a CPU with one11704 * running task. Whatever its utilization, we will fail11705 * detach the task.11706 */11707 if (nr_running <= 1)11708 continue;1170911710 if (busiest_util < util) {11711 busiest_util = util;11712 busiest = rq;11713 }11714 break;1171511716 case migrate_task:11717 if (busiest_nr < nr_running) {11718 busiest_nr = nr_running;11719 busiest = rq;11720 }11721 break;1172211723 case migrate_misfit:11724 /*11725 * For ASYM_CPUCAPACITY domains with misfit tasks we11726 * simply seek the "biggest" misfit task.11727 */11728 if (rq->misfit_task_load > busiest_load) {11729 busiest_load = rq->misfit_task_load;11730 busiest = rq;11731 }1173211733 break;1173411735 }11736 }1173711738 return busiest;11739 }1174011741 /*11742 * Max backoff if we encounter pinned tasks. Pretty arbitrary value, but11743 * so long as it is large enough.11744 */11745 #define MAX_PINNED_INTERVAL 5121174611747 static inline bool11748 asym_active_balance(struct lb_env *env)11749 {11750 /*11751 * ASYM_PACKING needs to force migrate tasks from busy but lower11752 * priority CPUs in order to pack all tasks in the highest priority11753 * CPUs. When done between cores, do it only if the whole core if the11754 * whole core is idle.11755 *11756 * If @env::src_cpu is an SMT core with busy siblings, let11757 * the lower priority @env::dst_cpu help it. Do not follow11758 * CPU priority.11759 */11760 return env->idle && sched_use_asym_prio(env->sd, env->dst_cpu) &&11761 (sched_asym_prefer(env->dst_cpu, env->src_cpu) ||11762 !sched_use_asym_prio(env->sd, env->src_cpu));11763 }1176411765 static inline bool11766 imbalanced_active_balance(struct lb_env *env)11767 {11768 struct sched_domain *sd = env->sd;1176911770 /*11771 * The imbalanced case includes the case of pinned tasks preventing a fair11772 * distribution of the load on the system but also the even distribution of the11773 * threads on a system with spare capacity11774 */11775 if ((env->migration_type == migrate_task) &&11776 (sd->nr_balance_failed > sd->cache_nice_tries+2))11777 return 1;1177811779 return 0;11780 }1178111782 static int need_active_balance(struct lb_env *env)11783 {11784 struct sched_domain *sd = env->sd;1178511786 if (asym_active_balance(env))11787 return 1;1178811789 if (imbalanced_active_balance(env))11790 return 1;1179111792 /*11793 * The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.11794 * It's worth migrating the task if the src_cpu's capacity is reduced11795 * because of other sched_class or IRQs if more capacity stays11796 * available on dst_cpu.11797 */11798 if (env->idle &&11799 (env->src_rq->cfs.h_nr_runnable == 1)) {11800 if ((check_cpu_capacity(env->src_rq, sd)) &&11801 (capacity_of(env->src_cpu)*sd->imbalance_pct < capacity_of(env->dst_cpu)*100))11802 return 1;11803 }1180411805 if (env->migration_type == migrate_misfit)11806 return 1;1180711808 return 0;11809 }1181011811 static int active_load_balance_cpu_stop(void *data);1181211813 static int should_we_balance(struct lb_env *env)11814 {11815 struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask);11816 struct sched_group *sg = env->sd->groups;11817 int cpu, idle_smt = -1;1181811819 /*11820 * Ensure the balancing environment is consistent; can happen11821 * when the softirq triggers 'during' hotplug.11822 */11823 if (!cpumask_test_cpu(env->dst_cpu, env->cpus))11824 return 0;1182511826 /*11827 * In the newly idle case, we will allow all the CPUs11828 * to do the newly idle load balance.11829 *11830 * However, we bail out if we already have tasks or a wakeup pending,11831 * to optimize wakeup latency.11832 */11833 if (env->idle == CPU_NEWLY_IDLE) {11834 if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending)11835 return 0;11836 return 1;11837 }1183811839 cpumask_copy(swb_cpus, group_balance_mask(sg));11840 /* Try to find first idle CPU */11841 for_each_cpu_and(cpu, swb_cpus, env->cpus) {11842 if (!idle_cpu(cpu))11843 continue;1184411845 /*11846 * Don't balance to idle SMT in busy core right away when11847 * balancing cores, but remember the first idle SMT CPU for11848 * later consideration. Find CPU on an idle core first.11849 */11850 if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) {11851 if (idle_smt == -1)11852 idle_smt = cpu;11853 /*11854 * If the core is not idle, and first SMT sibling which is11855 * idle has been found, then its not needed to check other11856 * SMT siblings for idleness:11857 */11858 #ifdef CONFIG_SCHED_SMT11859 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu));11860 #endif11861 continue;11862 }1186311864 /*11865 * Are we the first idle core in a non-SMT domain or higher,11866 * or the first idle CPU in a SMT domain?11867 */11868 return cpu == env->dst_cpu;11869 }1187011871 /* Are we the first idle CPU with busy siblings? */11872 if (idle_smt != -1)11873 return idle_smt == env->dst_cpu;1187411875 /* Are we the first CPU of this group ? */11876 return group_balance_cpu(sg) == env->dst_cpu;11877 }1187811879 static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd,11880 enum cpu_idle_type idle)11881 {11882 if (!schedstat_enabled())11883 return;1188411885 switch (env->migration_type) {11886 case migrate_load:11887 __schedstat_add(sd->lb_imbalance_load[idle], env->imbalance);11888 break;11889 case migrate_util:11890 __schedstat_add(sd->lb_imbalance_util[idle], env->imbalance);11891 break;11892 case migrate_task:11893 __schedstat_add(sd->lb_imbalance_task[idle], env->imbalance);11894 break;11895 case migrate_misfit:11896 __schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance);11897 break;11898 }11899 }1190011901 /*11902 * This flag serializes load-balancing passes over large domains11903 * (above the NODE topology level) - only one load-balancing instance11904 * may run at a time, to reduce overhead on very large systems with11905 * lots of CPUs and large NUMA distances.11906 *11907 * - Note that load-balancing passes triggered while another one11908 * is executing are skipped and not re-tried.11909 *11910 * - Also note that this does not serialize rebalance_domains()11911 * execution, as non-SD_SERIALIZE domains will still be11912 * load-balanced in parallel.11913 */11914 static atomic_t sched_balance_running = ATOMIC_INIT(0);1191511916 /*11917 * Check this_cpu to ensure it is balanced within domain. Attempt to move11918 * tasks if there is an imbalance.11919 */11920 static int sched_balance_rq(int this_cpu, struct rq *this_rq,11921 struct sched_domain *sd, enum cpu_idle_type idle,11922 int *continue_balancing)11923 {11924 int ld_moved, cur_ld_moved, active_balance = 0;11925 struct sched_domain *sd_parent = sd->parent;11926 struct sched_group *group;11927 struct rq *busiest;11928 struct rq_flags rf;11929 struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);11930 struct lb_env env = {11931 .sd = sd,11932 .dst_cpu = this_cpu,11933 .dst_rq = this_rq,11934 .dst_grpmask = group_balance_mask(sd->groups),11935 .idle = idle,11936 .loop_break = SCHED_NR_MIGRATE_BREAK,11937 .cpus = cpus,11938 .fbq_type = all,11939 .tasks = LIST_HEAD_INIT(env.tasks),11940 };11941 bool need_unlock = false;1194211943 cpumask_and(cpus, sched_domain_span(sd), cpu_active_mask);1194411945 schedstat_inc(sd->lb_count[idle]);1194611947 redo:11948 if (!should_we_balance(&env)) {11949 *continue_balancing = 0;11950 goto out_balanced;11951 }1195211953 if (!need_unlock && (sd->flags & SD_SERIALIZE)) {11954 int zero = 0;11955 if (!atomic_try_cmpxchg_acquire(&sched_balance_running, &zero, 1))11956 goto out_balanced;1195711958 need_unlock = true;11959 }1196011961 group = sched_balance_find_src_group(&env);11962 if (!group) {11963 schedstat_inc(sd->lb_nobusyg[idle]);11964 goto out_balanced;11965 }1196611967 busiest = sched_balance_find_src_rq(&env, group);11968 if (!busiest) {11969 schedstat_inc(sd->lb_nobusyq[idle]);11970 goto out_balanced;11971 }1197211973 WARN_ON_ONCE(busiest == env.dst_rq);1197411975 update_lb_imbalance_stat(&env, sd, idle);1197611977 env.src_cpu = busiest->cpu;11978 env.src_rq = busiest;1197911980 ld_moved = 0;11981 /* Clear this flag as soon as we find a pullable task */11982 env.flags |= LBF_ALL_PINNED;11983 if (busiest->nr_running > 1) {11984 /*11985 * Attempt to move tasks. If sched_balance_find_src_group has found11986 * an imbalance but busiest->nr_running <= 1, the group is11987 * still unbalanced. ld_moved simply stays zero, so it is11988 * correctly treated as an imbalance.11989 */11990 env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);1199111992 more_balance:11993 rq_lock_irqsave(busiest, &rf);11994 update_rq_clock(busiest);1199511996 /*11997 * cur_ld_moved - load moved in current iteration11998 * ld_moved - cumulative load moved across iterations11999 */12000 cur_ld_moved = detach_tasks(&env);1200112002 /*12003 * We've detached some tasks from busiest_rq. Every12004 * task is masked "TASK_ON_RQ_MIGRATING", so we can safely12005 * unlock busiest->lock, and we are able to be sure12006 * that nobody can manipulate the tasks in parallel.12007 * See task_rq_lock() family for the details.12008 */1200912010 rq_unlock(busiest, &rf);1201112012 if (cur_ld_moved) {12013 attach_tasks(&env);12014 ld_moved += cur_ld_moved;12015 }1201612017 local_irq_restore(rf.flags);1201812019 if (env.flags & LBF_NEED_BREAK) {12020 env.flags &= ~LBF_NEED_BREAK;12021 goto more_balance;12022 }1202312024 /*12025 * Revisit (affine) tasks on src_cpu that couldn't be moved to12026 * us and move them to an alternate dst_cpu in our sched_group12027 * where they can run. The upper limit on how many times we12028 * iterate on same src_cpu is dependent on number of CPUs in our12029 * sched_group.12030 *12031 * This changes load balance semantics a bit on who can move12032 * load to a given_cpu. In addition to the given_cpu itself12033 * (or a ilb_cpu acting on its behalf where given_cpu is12034 * nohz-idle), we now have balance_cpu in a position to move12035 * load to given_cpu. In rare situations, this may cause12036 * conflicts (balance_cpu and given_cpu/ilb_cpu deciding12037 * _independently_ and at _same_ time to move some load to12038 * given_cpu) causing excess load to be moved to given_cpu.12039 * This however should not happen so much in practice and12040 * moreover subsequent load balance cycles should correct the12041 * excess load moved.12042 */12043 if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {1204412045 /* Prevent to re-select dst_cpu via env's CPUs */12046 __cpumask_clear_cpu(env.dst_cpu, env.cpus);1204712048 env.dst_rq = cpu_rq(env.new_dst_cpu);12049 env.dst_cpu = env.new_dst_cpu;12050 env.flags &= ~LBF_DST_PINNED;12051 env.loop = 0;12052 env.loop_break = SCHED_NR_MIGRATE_BREAK;1205312054 /*12055 * Go back to "more_balance" rather than "redo" since we12056 * need to continue with same src_cpu.12057 */12058 goto more_balance;12059 }1206012061 /*12062 * We failed to reach balance because of affinity.12063 */12064 if (sd_parent) {12065 int *group_imbalance = &sd_parent->groups->sgc->imbalance;1206612067 if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)12068 *group_imbalance = 1;12069 }1207012071 /* All tasks on this runqueue were pinned by CPU affinity */12072 if (unlikely(env.flags & LBF_ALL_PINNED)) {12073 __cpumask_clear_cpu(cpu_of(busiest), cpus);12074 /*12075 * Attempting to continue load balancing at the current12076 * sched_domain level only makes sense if there are12077 * active CPUs remaining as possible busiest CPUs to12078 * pull load from which are not contained within the12079 * destination group that is receiving any migrated12080 * load.12081 */12082 if (!cpumask_subset(cpus, env.dst_grpmask)) {12083 env.loop = 0;12084 env.loop_break = SCHED_NR_MIGRATE_BREAK;12085 goto redo;12086 }12087 goto out_all_pinned;12088 }12089 }1209012091 if (!ld_moved) {12092 schedstat_inc(sd->lb_failed[idle]);12093 /*12094 * Increment the failure counter only on periodic balance.12095 * We do not want newidle balance, which can be very12096 * frequent, pollute the failure counter causing12097 * excessive cache_hot migrations and active balances.12098 *12099 * Similarly for migration_misfit which is not related to12100 * load/util migration, don't pollute nr_balance_failed.12101 */12102 if (idle != CPU_NEWLY_IDLE &&12103 env.migration_type != migrate_misfit)12104 sd->nr_balance_failed++;1210512106 if (need_active_balance(&env)) {12107 unsigned long flags;1210812109 raw_spin_rq_lock_irqsave(busiest, flags);1211012111 /*12112 * Don't kick the active_load_balance_cpu_stop,12113 * if the curr task on busiest CPU can't be12114 * moved to this_cpu:12115 */12116 if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) {12117 raw_spin_rq_unlock_irqrestore(busiest, flags);12118 goto out_one_pinned;12119 }1212012121 /* Record that we found at least one task that could run on this_cpu */12122 env.flags &= ~LBF_ALL_PINNED;1212312124 /*12125 * ->active_balance synchronizes accesses to12126 * ->active_balance_work. Once set, it's cleared12127 * only after active load balance is finished.12128 */12129 if (!busiest->active_balance) {12130 busiest->active_balance = 1;12131 busiest->push_cpu = this_cpu;12132 active_balance = 1;12133 }1213412135 preempt_disable();12136 raw_spin_rq_unlock_irqrestore(busiest, flags);12137 if (active_balance) {12138 stop_one_cpu_nowait(cpu_of(busiest),12139 active_load_balance_cpu_stop, busiest,12140 &busiest->active_balance_work);12141 }12142 preempt_enable();12143 }12144 } else {12145 sd->nr_balance_failed = 0;12146 }1214712148 if (likely(!active_balance) || need_active_balance(&env)) {12149 /* We were unbalanced, so reset the balancing interval */12150 sd->balance_interval = sd->min_interval;12151 }1215212153 goto out;1215412155 out_balanced:12156 /*12157 * We reach balance although we may have faced some affinity12158 * constraints. Clear the imbalance flag only if other tasks got12159 * a chance to move and fix the imbalance.12160 */12161 if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {12162 int *group_imbalance = &sd_parent->groups->sgc->imbalance;1216312164 if (*group_imbalance)12165 *group_imbalance = 0;12166 }1216712168 out_all_pinned:12169 /*12170 * We reach balance because all tasks are pinned at this level so12171 * we can't migrate them. Let the imbalance flag set so parent level12172 * can try to migrate them.12173 */12174 schedstat_inc(sd->lb_balanced[idle]);1217512176 sd->nr_balance_failed = 0;1217712178 out_one_pinned:12179 ld_moved = 0;1218012181 /*12182 * sched_balance_newidle() disregards balance intervals, so we could12183 * repeatedly reach this code, which would lead to balance_interval12184 * skyrocketing in a short amount of time. Skip the balance_interval12185 * increase logic to avoid that.12186 *12187 * Similarly misfit migration which is not necessarily an indication of12188 * the system being busy and requires lb to backoff to let it settle12189 * down.12190 */12191 if (env.idle == CPU_NEWLY_IDLE ||12192 env.migration_type == migrate_misfit)12193 goto out;1219412195 /* tune up the balancing interval */12196 if ((env.flags & LBF_ALL_PINNED &&12197 sd->balance_interval < MAX_PINNED_INTERVAL) ||12198 sd->balance_interval < sd->max_interval)12199 sd->balance_interval *= 2;12200 out:12201 if (need_unlock)12202 atomic_set_release(&sched_balance_running, 0);1220312204 return ld_moved;12205 }1220612207 static inline unsigned long12208 get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)12209 {12210 unsigned long interval = sd->balance_interval;1221112212 if (cpu_busy)12213 interval *= sd->busy_factor;1221412215 /* scale ms to jiffies */12216 interval = msecs_to_jiffies(interval);1221712218 /*12219 * Reduce likelihood of busy balancing at higher domains racing with12220 * balancing at lower domains by preventing their balancing periods12221 * from being multiples of each other.12222 */12223 if (cpu_busy)12224 interval -= 1;1222512226 interval = clamp(interval, 1UL, max_load_balance_interval);1222712228 return interval;12229 }1223012231 static inline void12232 update_next_balance(struct sched_domain *sd, unsigned long *next_balance)12233 {12234 unsigned long interval, next;1223512236 /* used by idle balance, so cpu_busy = 0 */12237 interval = get_sd_balance_interval(sd, 0);12238 next = sd->last_balance + interval;1223912240 if (time_after(*next_balance, next))12241 *next_balance = next;12242 }1224312244 /*12245 * active_load_balance_cpu_stop is run by the CPU stopper. It pushes12246 * running tasks off the busiest CPU onto idle CPUs. It requires at12247 * least 1 task to be running on each physical CPU where possible, and12248 * avoids physical / logical imbalances.12249 */12250 static int active_load_balance_cpu_stop(void *data)12251 {12252 struct rq *busiest_rq = data;12253 int busiest_cpu = cpu_of(busiest_rq);12254 int target_cpu = busiest_rq->push_cpu;12255 struct rq *target_rq = cpu_rq(target_cpu);12256 struct sched_domain *sd;12257 struct task_struct *p = NULL;12258 struct rq_flags rf;1225912260 rq_lock_irq(busiest_rq, &rf);12261 /*12262 * Between queueing the stop-work and running it is a hole in which12263 * CPUs can become inactive. We should not move tasks from or to12264 * inactive CPUs.12265 */12266 if (!cpu_active(busiest_cpu) || !cpu_active(target_cpu))12267 goto out_unlock;1226812269 /* Make sure the requested CPU hasn't gone down in the meantime: */12270 if (unlikely(busiest_cpu != smp_processor_id() ||12271 !busiest_rq->active_balance))12272 goto out_unlock;1227312274 /* Is there any task to move? */12275 if (busiest_rq->nr_running <= 1)12276 goto out_unlock;1227712278 /*12279 * This condition is "impossible", if it occurs12280 * we need to fix it. Originally reported by12281 * Bjorn Helgaas on a 128-CPU setup.12282 */12283 WARN_ON_ONCE(busiest_rq == target_rq);1228412285 /* Search for an sd spanning us and the target CPU. */12286 rcu_read_lock();12287 for_each_domain(target_cpu, sd) {12288 if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))12289 break;12290 }1229112292 if (likely(sd)) {12293 struct lb_env env = {12294 .sd = sd,12295 .dst_cpu = target_cpu,12296 .dst_rq = target_rq,12297 .src_cpu = busiest_rq->cpu,12298 .src_rq = busiest_rq,12299 .idle = CPU_IDLE,12300 .flags = LBF_ACTIVE_LB,12301 };1230212303 schedstat_inc(sd->alb_count);12304 update_rq_clock(busiest_rq);1230512306 p = detach_one_task(&env);12307 if (p) {12308 schedstat_inc(sd->alb_pushed);12309 /* Active balancing done, reset the failure counter. */12310 sd->nr_balance_failed = 0;12311 } else {12312 schedstat_inc(sd->alb_failed);12313 }12314 }12315 rcu_read_unlock();12316 out_unlock:12317 busiest_rq->active_balance = 0;12318 rq_unlock(busiest_rq, &rf);1231912320 if (p)12321 attach_one_task(target_rq, p);1232212323 local_irq_enable();1232412325 return 0;12326 }1232712328 /*12329 * Scale the max sched_balance_rq interval with the number of CPUs in the system.12330 * This trades load-balance latency on larger machines for less cross talk.12331 */12332 void update_max_interval(void)12333 {12334 max_load_balance_interval = HZ*num_online_cpus()/10;12335 }1233612337 static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success)12338 {12339 sd->newidle_call++;12340 sd->newidle_success += success;1234112342 if (sd->newidle_call >= 1024) {12343 sd->newidle_ratio = sd->newidle_success;12344 sd->newidle_call /= 2;12345 sd->newidle_success /= 2;12346 }12347 }1234812349 static inline bool12350 update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success)12351 {12352 unsigned long next_decay = sd->last_decay_max_lb_cost + HZ;12353 unsigned long now = jiffies;1235412355 if (cost)12356 update_newidle_stats(sd, success);1235712358 if (cost > sd->max_newidle_lb_cost) {12359 /*12360 * Track max cost of a domain to make sure to not delay the12361 * next wakeup on the CPU.12362 */12363 sd->max_newidle_lb_cost = cost;12364 sd->last_decay_max_lb_cost = now;1236512366 } else if (time_after(now, next_decay)) {12367 /*12368 * Decay the newidle max times by ~1% per second to ensure that12369 * it is not outdated and the current max cost is actually12370 * shorter.12371 */12372 sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256;12373 sd->last_decay_max_lb_cost = now;12374 return true;12375 }1237612377 return false;12378 }1237912380 /*12381 * It checks each scheduling domain to see if it is due to be balanced,12382 * and initiates a balancing operation if so.12383 *12384 * Balancing parameters are set up in init_sched_domains.12385 */12386 static void sched_balance_domains(struct rq *rq, enum cpu_idle_type idle)12387 {12388 int continue_balancing = 1;12389 int cpu = rq->cpu;12390 int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);12391 unsigned long interval;12392 struct sched_domain *sd;12393 /* Earliest time when we have to do rebalance again */12394 unsigned long next_balance = jiffies + 60*HZ;12395 int update_next_balance = 0;12396 int need_decay = 0;12397 u64 max_cost = 0;1239812399 rcu_read_lock();12400 for_each_domain(cpu, sd) {12401 /*12402 * Decay the newidle max times here because this is a regular12403 * visit to all the domains.12404 */12405 need_decay = update_newidle_cost(sd, 0, 0);12406 max_cost += sd->max_newidle_lb_cost;1240712408 /*12409 * Stop the load balance at this level. There is another12410 * CPU in our sched group which is doing load balancing more12411 * actively.12412 */12413 if (!continue_balancing) {12414 if (need_decay)12415 continue;12416 break;12417 }1241812419 interval = get_sd_balance_interval(sd, busy);12420 if (time_after_eq(jiffies, sd->last_balance + interval)) {12421 if (sched_balance_rq(cpu, rq, sd, idle, &continue_balancing)) {12422 /*12423 * The LBF_DST_PINNED logic could have changed12424 * env->dst_cpu, so we can't know our idle12425 * state even if we migrated tasks. Update it.12426 */12427 idle = idle_cpu(cpu);12428 busy = !idle && !sched_idle_cpu(cpu);12429 }12430 sd->last_balance = jiffies;12431 interval = get_sd_balance_interval(sd, busy);12432 }12433 if (time_after(next_balance, sd->last_balance + interval)) {12434 next_balance = sd->last_balance + interval;12435 update_next_balance = 1;12436 }12437 }12438 if (need_decay) {12439 /*12440 * Ensure the rq-wide value also decays but keep it at a12441 * reasonable floor to avoid funnies with rq->avg_idle.12442 */12443 rq->max_idle_balance_cost =12444 max((u64)sysctl_sched_migration_cost, max_cost);12445 }12446 rcu_read_unlock();1244712448 /*12449 * next_balance will be updated only when there is a need.12450 * When the cpu is attached to null domain for ex, it will not be12451 * updated.12452 */12453 if (likely(update_next_balance))12454 rq->next_balance = next_balance;1245512456 }1245712458 static inline int on_null_domain(struct rq *rq)12459 {12460 return unlikely(!rcu_dereference_sched(rq->sd));12461 }1246212463 #ifdef CONFIG_NO_HZ_COMMON12464 /*12465 * NOHZ idle load balancing (ILB) details:12466 *12467 * - When one of the busy CPUs notices that there may be an idle rebalancing12468 * needed, they will kick the idle load balancer, which then does idle12469 * load balancing for all the idle CPUs.12470 */12471 static inline int find_new_ilb(void)12472 {12473 const struct cpumask *hk_mask;12474 int ilb_cpu;1247512476 hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE);1247712478 for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) {1247912480 if (ilb_cpu == smp_processor_id())12481 continue;1248212483 if (idle_cpu(ilb_cpu))12484 return ilb_cpu;12485 }1248612487 return -1;12488 }1248912490 /*12491 * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU12492 * SMP function call (IPI).12493 *12494 * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set12495 * (if there is one).12496 */12497 static void kick_ilb(unsigned int flags)12498 {12499 int ilb_cpu;1250012501 /*12502 * Increase nohz.next_balance only when if full ilb is triggered but12503 * not if we only update stats.12504 */12505 if (flags & NOHZ_BALANCE_KICK)12506 nohz.next_balance = jiffies+1;1250712508 ilb_cpu = find_new_ilb();12509 if (ilb_cpu < 0)12510 return;1251112512 /*12513 * Don't bother if no new NOHZ balance work items for ilb_cpu,12514 * i.e. all bits in flags are already set in ilb_cpu.12515 */12516 if ((atomic_read(nohz_flags(ilb_cpu)) & flags) == flags)12517 return;1251812519 /*12520 * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets12521 * the first flag owns it; cleared by nohz_csd_func().12522 */12523 flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu));12524 if (flags & NOHZ_KICK_MASK)12525 return;1252612527 /*12528 * This way we generate an IPI on the target CPU which12529 * is idle, and the softirq performing NOHZ idle load balancing12530 * will be run before returning from the IPI.12531 */12532 smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);12533 }1253412535 /*12536 * Current decision point for kicking the idle load balancer in the presence12537 * of idle CPUs in the system.12538 */12539 static void nohz_balancer_kick(struct rq *rq)12540 {12541 unsigned long now = jiffies;12542 struct sched_domain_shared *sds;12543 struct sched_domain *sd;12544 int nr_busy, i, cpu = rq->cpu;12545 unsigned int flags = 0;1254612547 if (unlikely(rq->idle_balance))12548 return;1254912550 /*12551 * We may be recently in ticked or tickless idle mode. At the first12552 * busy tick after returning from idle, we will update the busy stats.12553 */12554 nohz_balance_exit_idle(rq);1255512556 /*12557 * None are in tickless mode and hence no need for NOHZ idle load12558 * balancing:12559 */12560 if (likely(!atomic_read(&nohz.nr_cpus)))12561 return;1256212563 if (READ_ONCE(nohz.has_blocked) &&12564 time_after(now, READ_ONCE(nohz.next_blocked)))12565 flags = NOHZ_STATS_KICK;1256612567 if (time_before(now, nohz.next_balance))12568 goto out;1256912570 if (rq->nr_running >= 2) {12571 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;12572 goto out;12573 }1257412575 rcu_read_lock();1257612577 sd = rcu_dereference(rq->sd);12578 if (sd) {12579 /*12580 * If there's a runnable CFS task and the current CPU has reduced12581 * capacity, kick the ILB to see if there's a better CPU to run on:12582 */12583 if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) {12584 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;12585 goto unlock;12586 }12587 }1258812589 sd = rcu_dereference(per_cpu(sd_asym_packing, cpu));12590 if (sd) {12591 /*12592 * When ASYM_PACKING; see if there's a more preferred CPU12593 * currently idle; in which case, kick the ILB to move tasks12594 * around.12595 *12596 * When balancing between cores, all the SMT siblings of the12597 * preferred CPU must be idle.12598 */12599 for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {12600 if (sched_asym(sd, i, cpu)) {12601 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;12602 goto unlock;12603 }12604 }12605 }1260612607 sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu));12608 if (sd) {12609 /*12610 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU12611 * to run the misfit task on.12612 */12613 if (check_misfit_status(rq)) {12614 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;12615 goto unlock;12616 }1261712618 /*12619 * For asymmetric systems, we do not want to nicely balance12620 * cache use, instead we want to embrace asymmetry and only12621 * ensure tasks have enough CPU capacity.12622 *12623 * Skip the LLC logic because it's not relevant in that case.12624 */12625 goto unlock;12626 }1262712628 sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));12629 if (sds) {12630 /*12631 * If there is an imbalance between LLC domains (IOW we could12632 * increase the overall cache utilization), we need a less-loaded LLC12633 * domain to pull some load from. Likewise, we may need to spread12634 * load within the current LLC domain (e.g. packed SMT cores but12635 * other CPUs are idle). We can't really know from here how busy12636 * the others are - so just get a NOHZ balance going if it looks12637 * like this LLC domain has tasks we could move.12638 */12639 nr_busy = atomic_read(&sds->nr_busy_cpus);12640 if (nr_busy > 1) {12641 flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;12642 goto unlock;12643 }12644 }12645 unlock:12646 rcu_read_unlock();12647 out:12648 if (READ_ONCE(nohz.needs_update))12649 flags |= NOHZ_NEXT_KICK;1265012651 if (flags)12652 kick_ilb(flags);12653 }1265412655 static void set_cpu_sd_state_busy(int cpu)12656 {12657 struct sched_domain *sd;1265812659 rcu_read_lock();12660 sd = rcu_dereference(per_cpu(sd_llc, cpu));1266112662 if (!sd || !sd->nohz_idle)12663 goto unlock;12664 sd->nohz_idle = 0;1266512666 atomic_inc(&sd->shared->nr_busy_cpus);12667 unlock:12668 rcu_read_unlock();12669 }1267012671 void nohz_balance_exit_idle(struct rq *rq)12672 {12673 WARN_ON_ONCE(rq != this_rq());1267412675 if (likely(!rq->nohz_tick_stopped))12676 return;1267712678 rq->nohz_tick_stopped = 0;12679 cpumask_clear_cpu(rq->cpu, nohz.idle_cpus_mask);12680 atomic_dec(&nohz.nr_cpus);1268112682 set_cpu_sd_state_busy(rq->cpu);12683 }1268412685 static void set_cpu_sd_state_idle(int cpu)12686 {12687 struct sched_domain *sd;1268812689 rcu_read_lock();12690 sd = rcu_dereference(per_cpu(sd_llc, cpu));1269112692 if (!sd || sd->nohz_idle)12693 goto unlock;12694 sd->nohz_idle = 1;1269512696 atomic_dec(&sd->shared->nr_busy_cpus);12697 unlock:12698 rcu_read_unlock();12699 }1270012701 /*12702 * This routine will record that the CPU is going idle with tick stopped.12703 * This info will be used in performing idle load balancing in the future.12704 */12705 void nohz_balance_enter_idle(int cpu)12706 {12707 struct rq *rq = cpu_rq(cpu);1270812709 WARN_ON_ONCE(cpu != smp_processor_id());1271012711 /* If this CPU is going down, then nothing needs to be done: */12712 if (!cpu_active(cpu))12713 return;1271412715 /*12716 * Can be set safely without rq->lock held12717 * If a clear happens, it will have evaluated last additions because12718 * rq->lock is held during the check and the clear12719 */12720 rq->has_blocked_load = 1;1272112722 /*12723 * The tick is still stopped but load could have been added in the12724 * meantime. We set the nohz.has_blocked flag to trig a check of the12725 * *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear12726 * of nohz.has_blocked can only happen after checking the new load12727 */12728 if (rq->nohz_tick_stopped)12729 goto out;1273012731 /* If we're a completely isolated CPU, we don't play: */12732 if (on_null_domain(rq))12733 return;1273412735 rq->nohz_tick_stopped = 1;1273612737 cpumask_set_cpu(cpu, nohz.idle_cpus_mask);12738 atomic_inc(&nohz.nr_cpus);1273912740 /*12741 * Ensures that if nohz_idle_balance() fails to observe our12742 * @idle_cpus_mask store, it must observe the @has_blocked12743 * and @needs_update stores.12744 */12745 smp_mb__after_atomic();1274612747 set_cpu_sd_state_idle(cpu);1274812749 WRITE_ONCE(nohz.needs_update, 1);12750 out:12751 /*12752 * Each time a cpu enter idle, we assume that it has blocked load and12753 * enable the periodic update of the load of idle CPUs12754 */12755 WRITE_ONCE(nohz.has_blocked, 1);12756 }1275712758 static bool update_nohz_stats(struct rq *rq)12759 {12760 unsigned int cpu = rq->cpu;1276112762 if (!rq->has_blocked_load)12763 return false;1276412765 if (!cpumask_test_cpu(cpu, nohz.idle_cpus_mask))12766 return false;1276712768 if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick)))12769 return true;1277012771 sched_balance_update_blocked_averages(cpu);1277212773 return rq->has_blocked_load;12774 }1277512776 /*12777 * Internal function that runs load balance for all idle CPUs. The load balance12778 * can be a simple update of blocked load or a complete load balance with12779 * tasks movement depending of flags.12780 */12781 static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)12782 {12783 /* Earliest time when we have to do rebalance again */12784 unsigned long now = jiffies;12785 unsigned long next_balance = now + 60*HZ;12786 bool has_blocked_load = false;12787 int update_next_balance = 0;12788 int this_cpu = this_rq->cpu;12789 int balance_cpu;12790 struct rq *rq;1279112792 WARN_ON_ONCE((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);1279312794 /*12795 * We assume there will be no idle load after this update and clear12796 * the has_blocked flag. If a cpu enters idle in the mean time, it will12797 * set the has_blocked flag and trigger another update of idle load.12798 * Because a cpu that becomes idle, is added to idle_cpus_mask before12799 * setting the flag, we are sure to not clear the state and not12800 * check the load of an idle cpu.12801 *12802 * Same applies to idle_cpus_mask vs needs_update.12803 */12804 if (flags & NOHZ_STATS_KICK)12805 WRITE_ONCE(nohz.has_blocked, 0);12806 if (flags & NOHZ_NEXT_KICK)12807 WRITE_ONCE(nohz.needs_update, 0);1280812809 /*12810 * Ensures that if we miss the CPU, we must see the has_blocked12811 * store from nohz_balance_enter_idle().12812 */12813 smp_mb();1281412815 /*12816 * Start with the next CPU after this_cpu so we will end with this_cpu and let a12817 * chance for other idle cpu to pull load.12818 */12819 for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {12820 if (!idle_cpu(balance_cpu))12821 continue;1282212823 /*12824 * If this CPU gets work to do, stop the load balancing12825 * work being done for other CPUs. Next load12826 * balancing owner will pick it up.12827 */12828 if (!idle_cpu(this_cpu) && need_resched()) {12829 if (flags & NOHZ_STATS_KICK)12830 has_blocked_load = true;12831 if (flags & NOHZ_NEXT_KICK)12832 WRITE_ONCE(nohz.needs_update, 1);12833 goto abort;12834 }1283512836 rq = cpu_rq(balance_cpu);1283712838 if (flags & NOHZ_STATS_KICK)12839 has_blocked_load |= update_nohz_stats(rq);1284012841 /*12842 * If time for next balance is due,12843 * do the balance.12844 */12845 if (time_after_eq(jiffies, rq->next_balance)) {12846 struct rq_flags rf;1284712848 rq_lock_irqsave(rq, &rf);12849 update_rq_clock(rq);12850 rq_unlock_irqrestore(rq, &rf);1285112852 if (flags & NOHZ_BALANCE_KICK)12853 sched_balance_domains(rq, CPU_IDLE);12854 }1285512856 if (time_after(next_balance, rq->next_balance)) {12857 next_balance = rq->next_balance;12858 update_next_balance = 1;12859 }12860 }1286112862 /*12863 * next_balance will be updated only when there is a need.12864 * When the CPU is attached to null domain for ex, it will not be12865 * updated.12866 */12867 if (likely(update_next_balance))12868 nohz.next_balance = next_balance;1286912870 if (flags & NOHZ_STATS_KICK)12871 WRITE_ONCE(nohz.next_blocked,12872 now + msecs_to_jiffies(LOAD_AVG_PERIOD));1287312874 abort:12875 /* There is still blocked load, enable periodic update */12876 if (has_blocked_load)12877 WRITE_ONCE(nohz.has_blocked, 1);12878 }1287912880 /*12881 * In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the12882 * rebalancing for all the CPUs for whom scheduler ticks are stopped.12883 */12884 static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)12885 {12886 unsigned int flags = this_rq->nohz_idle_balance;1288712888 if (!flags)12889 return false;1289012891 this_rq->nohz_idle_balance = 0;1289212893 if (idle != CPU_IDLE)12894 return false;1289512896 _nohz_idle_balance(this_rq, flags);1289712898 return true;12899 }1290012901 /*12902 * Check if we need to directly run the ILB for updating blocked load before12903 * entering idle state. Here we run ILB directly without issuing IPIs.12904 *12905 * Note that when this function is called, the tick may not yet be stopped on12906 * this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and12907 * cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates12908 * don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle12909 * entry/exit rate (usec). So it is possible that _nohz_idle_balance() is12910 * called from this function on (this) CPU that's not yet in the mask. That's12911 * OK because the goal of nohz_run_idle_balance() is to run ILB only for12912 * updating the blocked load of already idle CPUs without waking up one of12913 * those idle CPUs and outside the preempt disable / IRQ off phase of the local12914 * cpu about to enter idle, because it can take a long time.12915 */12916 void nohz_run_idle_balance(int cpu)12917 {12918 unsigned int flags;1291912920 flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));1292112922 /*12923 * Update the blocked load only if no SCHED_SOFTIRQ is about to happen12924 * (i.e. NOHZ_STATS_KICK set) and will do the same.12925 */12926 if ((flags == NOHZ_NEWILB_KICK) && !need_resched())12927 _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK);12928 }1292912930 static void nohz_newidle_balance(struct rq *this_rq)12931 {12932 int this_cpu = this_rq->cpu;1293312934 /* Will wake up very soon. No time for doing anything else*/12935 if (this_rq->avg_idle < sysctl_sched_migration_cost)12936 return;1293712938 /* Don't need to update blocked load of idle CPUs*/12939 if (!READ_ONCE(nohz.has_blocked) ||12940 time_before(jiffies, READ_ONCE(nohz.next_blocked)))12941 return;1294212943 /*12944 * Set the need to trigger ILB in order to update blocked load12945 * before entering idle state.12946 */12947 atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));12948 }1294912950 #else /* !CONFIG_NO_HZ_COMMON: */12951 static inline void nohz_balancer_kick(struct rq *rq) { }1295212953 static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)12954 {12955 return false;12956 }1295712958 static inline void nohz_newidle_balance(struct rq *this_rq) { }12959 #endif /* !CONFIG_NO_HZ_COMMON */1296012961 /*12962 * sched_balance_newidle is called by schedule() if this_cpu is about to become12963 * idle. Attempts to pull tasks from other CPUs.12964 *12965 * Returns:12966 * < 0 - we released the lock and there are !fair tasks present12967 * 0 - failed, no new tasks12968 * > 0 - success, new (fair) tasks present12969 */12970 static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)12971 {12972 unsigned long next_balance = jiffies + HZ;12973 int this_cpu = this_rq->cpu;12974 int continue_balancing = 1;12975 u64 t0, t1, curr_cost = 0;12976 struct sched_domain *sd;12977 int pulled_task = 0;1297812979 update_misfit_status(NULL, this_rq);1298012981 /*12982 * There is a task waiting to run. No need to search for one.12983 * Return 0; the task will be enqueued when switching to idle.12984 */12985 if (this_rq->ttwu_pending)12986 return 0;1298712988 /*12989 * We must set idle_stamp _before_ calling sched_balance_rq()12990 * for CPU_NEWLY_IDLE, such that we measure the this duration12991 * as idle time.12992 */12993 this_rq->idle_stamp = rq_clock(this_rq);1299412995 /*12996 * Do not pull tasks towards !active CPUs...12997 */12998 if (!cpu_active(this_cpu))12999 return 0;1300013001 /*13002 * This is OK, because current is on_cpu, which avoids it being picked13003 * for load-balance and preemption/IRQs are still disabled avoiding13004 * further scheduler activity on it and we're being very careful to13005 * re-start the picking loop.13006 */13007 rq_unpin_lock(this_rq, rf);1300813009 rcu_read_lock();13010 sd = rcu_dereference_check_sched_domain(this_rq->sd);13011 if (!sd) {13012 rcu_read_unlock();13013 goto out;13014 }1301513016 if (!get_rd_overloaded(this_rq->rd) ||13017 this_rq->avg_idle < sd->max_newidle_lb_cost) {1301813019 update_next_balance(sd, &next_balance);13020 rcu_read_unlock();13021 goto out;13022 }13023 rcu_read_unlock();1302413025 raw_spin_rq_unlock(this_rq);1302613027 t0 = sched_clock_cpu(this_cpu);13028 sched_balance_update_blocked_averages(this_cpu);1302913030 rcu_read_lock();13031 for_each_domain(this_cpu, sd) {13032 u64 domain_cost;1303313034 update_next_balance(sd, &next_balance);1303513036 if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)13037 break;1303813039 if (sd->flags & SD_BALANCE_NEWIDLE) {13040 unsigned int weight = 1;1304113042 if (sched_feat(NI_RANDOM)) {13043 /*13044 * Throw a 1k sided dice; and only run13045 * newidle_balance according to the success13046 * rate.13047 */13048 u32 d1k = sched_rng() % 1024;13049 weight = 1 + sd->newidle_ratio;13050 if (d1k > weight) {13051 update_newidle_stats(sd, 0);13052 continue;13053 }13054 weight = (1024 + weight/2) / weight;13055 }1305613057 pulled_task = sched_balance_rq(this_cpu, this_rq,13058 sd, CPU_NEWLY_IDLE,13059 &continue_balancing);1306013061 t1 = sched_clock_cpu(this_cpu);13062 domain_cost = t1 - t0;13063 curr_cost += domain_cost;13064 t0 = t1;1306513066 /*13067 * Track max cost of a domain to make sure to not delay the13068 * next wakeup on the CPU.13069 */13070 update_newidle_cost(sd, domain_cost, weight * !!pulled_task);13071 }1307213073 /*13074 * Stop searching for tasks to pull if there are13075 * now runnable tasks on this rq.13076 */13077 if (pulled_task || !continue_balancing)13078 break;13079 }13080 rcu_read_unlock();1308113082 raw_spin_rq_lock(this_rq);1308313084 if (curr_cost > this_rq->max_idle_balance_cost)13085 this_rq->max_idle_balance_cost = curr_cost;1308613087 /*13088 * While browsing the domains, we released the rq lock, a task could13089 * have been enqueued in the meantime. Since we're not going idle,13090 * pretend we pulled a task.13091 */13092 if (this_rq->cfs.h_nr_queued && !pulled_task)13093 pulled_task = 1;1309413095 /* Is there a task of a high priority class? */13096 if (this_rq->nr_running != this_rq->cfs.h_nr_queued)13097 pulled_task = -1;1309813099 out:13100 /* Move the next balance forward */13101 if (time_after(this_rq->next_balance, next_balance))13102 this_rq->next_balance = next_balance;1310313104 if (pulled_task)13105 this_rq->idle_stamp = 0;13106 else13107 nohz_newidle_balance(this_rq);1310813109 rq_repin_lock(this_rq, rf);1311013111 return pulled_task;13112 }1311313114 /*13115 * This softirq handler is triggered via SCHED_SOFTIRQ from two places:13116 *13117 * - directly from the local sched_tick() for periodic load balancing13118 *13119 * - indirectly from a remote sched_tick() for NOHZ idle balancing13120 * through the SMP cross-call nohz_csd_func()13121 */13122 static __latent_entropy void sched_balance_softirq(void)13123 {13124 struct rq *this_rq = this_rq();13125 enum cpu_idle_type idle = this_rq->idle_balance;13126 /*13127 * If this CPU has a pending NOHZ_BALANCE_KICK, then do the13128 * balancing on behalf of the other idle CPUs whose ticks are13129 * stopped. Do nohz_idle_balance *before* sched_balance_domains to13130 * give the idle CPUs a chance to load balance. Else we may13131 * load balance only within the local sched_domain hierarchy13132 * and abort nohz_idle_balance altogether if we pull some load.13133 */13134 if (nohz_idle_balance(this_rq, idle))13135 return;1313613137 /* normal load balance */13138 sched_balance_update_blocked_averages(this_rq->cpu);13139 sched_balance_domains(this_rq, idle);13140 }1314113142 /*13143 * Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.13144 */13145 void sched_balance_trigger(struct rq *rq)13146 {13147 /*13148 * Don't need to rebalance while attached to NULL domain or13149 * runqueue CPU is not active13150 */13151 if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq))))13152 return;1315313154 if (time_after_eq(jiffies, rq->next_balance))13155 raise_softirq(SCHED_SOFTIRQ);1315613157 nohz_balancer_kick(rq);13158 }1315913160 static void rq_online_fair(struct rq *rq)13161 {13162 update_sysctl();1316313164 update_runtime_enabled(rq);13165 }1316613167 static void rq_offline_fair(struct rq *rq)13168 {13169 update_sysctl();1317013171 /* Ensure any throttled groups are reachable by pick_next_task */13172 unthrottle_offline_cfs_rqs(rq);1317313174 /* Ensure that we remove rq contribution to group share: */13175 clear_tg_offline_cfs_rqs(rq);13176 }1317713178 #ifdef CONFIG_SCHED_CORE13179 static inline bool13180 __entity_slice_used(struct sched_entity *se, int min_nr_tasks)13181 {13182 u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime;13183 u64 slice = se->slice;1318413185 return (rtime * min_nr_tasks > slice);13186 }1318713188 #define MIN_NR_TASKS_DURING_FORCEIDLE 213189 static inline void task_tick_core(struct rq *rq, struct task_struct *curr)13190 {13191 if (!sched_core_enabled(rq))13192 return;1319313194 /*13195 * If runqueue has only one task which used up its slice and13196 * if the sibling is forced idle, then trigger schedule to13197 * give forced idle task a chance.13198 *13199 * sched_slice() considers only this active rq and it gets the13200 * whole slice. But during force idle, we have siblings acting13201 * like a single runqueue and hence we need to consider runnable13202 * tasks on this CPU and the forced idle CPU. Ideally, we should13203 * go through the forced idle rq, but that would be a perf hit.13204 * We can assume that the forced idle CPU has at least13205 * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check13206 * if we need to give up the CPU.13207 */13208 if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 &&13209 __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))13210 resched_curr(rq);13211 }1321213213 /*13214 * Consider any infeasible weight scenario. Take for instance two tasks,13215 * each bound to their respective sibling, one with weight 1 and one with13216 * weight 2. Then the lower weight task will run ahead of the higher weight13217 * task without bound.13218 *13219 * This utterly destroys the concept of a shared time base.13220 *13221 * Remember; all this is about a proportionally fair scheduling, where each13222 * tasks receives:13223 *13224 * w_i13225 * dt_i = ---------- dt (1)13226 * \Sum_j w_j13227 *13228 * which we do by tracking a virtual time, s_i:13229 *13230 * 113231 * s_i = --- d[t]_i (2)13232 * w_i13233 *13234 * Where d[t] is a delta of discrete time, while dt is an infinitesimal.13235 * The immediate corollary is that the ideal schedule S, where (2) to use13236 * an infinitesimal delta, is:13237 *13238 * 113239 * S = ---------- dt (3)13240 * \Sum_i w_i13241 *13242 * From which we can define the lag, or deviation from the ideal, as:13243 *13244 * lag(i) = S - s_i (4)13245 *13246 * And since the one and only purpose is to approximate S, we get that:13247 *13248 * \Sum_i w_i lag(i) := 0 (5)13249 *13250 * If this were not so, we no longer converge to S, and we can no longer13251 * claim our scheduler has any of the properties we derive from S. This is13252 * exactly what you did above, you broke it!13253 *13254 *13255 * Let's continue for a while though; to see if there is anything useful to13256 * be learned. We can combine (1)-(3) or (4)-(5) and express S in s_i:13257 *13258 * \Sum_i w_i s_i13259 * S = -------------- (6)13260 * \Sum_i w_i13261 *13262 * Which gives us a way to compute S, given our s_i. Now, if you've read13263 * our code, you know that we do not in fact do this, the reason for this13264 * is two-fold. Firstly, computing S in that way requires a 64bit division13265 * for every time we'd use it (see 12), and secondly, this only describes13266 * the steady-state, it doesn't handle dynamics.13267 *13268 * Anyway, in (6): s_i -> x + (s_i - x), to get:13269 *13270 * \Sum_i w_i (s_i - x)13271 * S - x = -------------------- (7)13272 * \Sum_i w_i13273 *13274 * Which shows that S and s_i transform alike (which makes perfect sense13275 * given that S is basically the (weighted) average of s_i).13276 *13277 * So the thing to remember is that the above is strictly UP. It is13278 * possible to generalize to multiple runqueues -- however it gets really13279 * yuck when you have to add affinity support, as illustrated by our very13280 * first counter-example.13281 *13282 * Luckily I think we can avoid needing a full multi-queue variant for13283 * core-scheduling (or load-balancing). The crucial observation is that we13284 * only actually need this comparison in the presence of forced-idle; only13285 * then do we need to tell if the stalled rq has higher priority over the13286 * other.13287 *13288 * [XXX assumes SMT2; better consider the more general case, I suspect13289 * it'll work out because our comparison is always between 2 rqs and the13290 * answer is only interesting if one of them is forced-idle]13291 *13292 * And (under assumption of SMT2) when there is forced-idle, there is only13293 * a single queue, so everything works like normal.13294 *13295 * Let, for our runqueue 'k':13296 *13297 * T_k = \Sum_i w_i s_i13298 * W_k = \Sum_i w_i ; for all i of k (8)13299 *13300 * Then we can write (6) like:13301 *13302 * T_k13303 * S_k = --- (9)13304 * W_k13305 *13306 * From which immediately follows that:13307 *13308 * T_k + T_l13309 * S_k+l = --------- (10)13310 * W_k + W_l13311 *13312 * On which we can define a combined lag:13313 *13314 * lag_k+l(i) := S_k+l - s_i (11)13315 *13316 * And that gives us the tools to compare tasks across a combined runqueue.13317 *13318 *13319 * Combined this gives the following:13320 *13321 * a) when a runqueue enters force-idle, sync it against it's sibling rq(s)13322 * using (7); this only requires storing single 'time'-stamps.13323 *13324 * b) when comparing tasks between 2 runqueues of which one is forced-idle,13325 * compare the combined lag, per (11).13326 *13327 * Now, of course cgroups (I so hate them) make this more interesting in13328 * that a) seems to suggest we need to iterate all cgroup on a CPU at such13329 * boundaries, but I think we can avoid that. The force-idle is for the13330 * whole CPU, all it's rqs. So we can mark it in the root and lazily13331 * propagate downward on demand.13332 */1333313334 /*13335 * So this sync is basically a relative reset of S to 0.13336 *13337 * So with 2 queues, when one goes idle, we drop them both to 0 and one13338 * then increases due to not being idle, and the idle one builds up lag to13339 * get re-elected. So far so simple, right?13340 *13341 * When there's 3, we can have the situation where 2 run and one is idle,13342 * we sync to 0 and let the idle one build up lag to get re-election. Now13343 * suppose another one also drops idle. At this point dropping all to 013344 * again would destroy the built-up lag from the queue that was already13345 * idle, not good.13346 *13347 * So instead of syncing everything, we can:13348 *13349 * less := !((s64)(s_a - s_b) <= 0)13350 *13351 * (v_a - S_a) - (v_b - S_b) == v_a - v_b - S_a + S_b13352 * == v_a - (v_b - S_a + S_b)13353 *13354 * IOW, we can recast the (lag) comparison to a one-sided difference.13355 * So if then, instead of syncing the whole queue, sync the idle queue13356 * against the active queue with S_a + S_b at the point where we sync.13357 *13358 * (XXX consider the implication of living in a cyclic group: N / 2^n N)13359 *13360 * This gives us means of syncing single queues against the active queue,13361 * and for already idle queues to preserve their build-up lag.13362 *13363 * Of course, then we get the situation where there's 2 active and one13364 * going idle, who do we pick to sync against? Theory would have us sync13365 * against the combined S, but as we've already demonstrated, there is no13366 * such thing in infeasible weight scenarios.13367 *13368 * One thing I've considered; and this is where that core_active rudiment13369 * came from, is having active queues sync up between themselves after13370 * every tick. This limits the observed divergence due to the work13371 * conservancy.13372 *13373 * On top of that, we can improve upon things by employing (10) here.13374 */1337513376 /*13377 * se_fi_update - Update the cfs_rq->zero_vruntime_fi in a CFS hierarchy if needed.13378 */13379 static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq,13380 bool forceidle)13381 {13382 for_each_sched_entity(se) {13383 struct cfs_rq *cfs_rq = cfs_rq_of(se);1338413385 if (forceidle) {13386 if (cfs_rq->forceidle_seq == fi_seq)13387 break;13388 cfs_rq->forceidle_seq = fi_seq;13389 }1339013391 cfs_rq->zero_vruntime_fi = cfs_rq->zero_vruntime;13392 }13393 }1339413395 void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi)13396 {13397 struct sched_entity *se = &p->se;1339813399 if (p->sched_class != &fair_sched_class)13400 return;1340113402 se_fi_update(se, rq->core->core_forceidle_seq, in_fi);13403 }1340413405 bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b,13406 bool in_fi)13407 {13408 struct rq *rq = task_rq(a);13409 const struct sched_entity *sea = &a->se;13410 const struct sched_entity *seb = &b->se;13411 struct cfs_rq *cfs_rqa;13412 struct cfs_rq *cfs_rqb;13413 s64 delta;1341413415 WARN_ON_ONCE(task_rq(b)->core != rq->core);1341613417 #ifdef CONFIG_FAIR_GROUP_SCHED13418 /*13419 * Find an se in the hierarchy for tasks a and b, such that the se's13420 * are immediate siblings.13421 */13422 while (sea->cfs_rq->tg != seb->cfs_rq->tg) {13423 int sea_depth = sea->depth;13424 int seb_depth = seb->depth;1342513426 if (sea_depth >= seb_depth)13427 sea = parent_entity(sea);13428 if (sea_depth <= seb_depth)13429 seb = parent_entity(seb);13430 }1343113432 se_fi_update(sea, rq->core->core_forceidle_seq, in_fi);13433 se_fi_update(seb, rq->core->core_forceidle_seq, in_fi);1343413435 cfs_rqa = sea->cfs_rq;13436 cfs_rqb = seb->cfs_rq;13437 #else /* !CONFIG_FAIR_GROUP_SCHED: */13438 cfs_rqa = &task_rq(a)->cfs;13439 cfs_rqb = &task_rq(b)->cfs;13440 #endif /* !CONFIG_FAIR_GROUP_SCHED */1344113442 /*13443 * Find delta after normalizing se's vruntime with its cfs_rq's13444 * zero_vruntime_fi, which would have been updated in prior calls13445 * to se_fi_update().13446 */13447 delta = vruntime_op(sea->vruntime, "-", seb->vruntime) +13448 vruntime_op(cfs_rqb->zero_vruntime_fi, "-", cfs_rqa->zero_vruntime_fi);1344913450 return delta > 0;13451 }1345213453 static int task_is_throttled_fair(struct task_struct *p, int cpu)13454 {13455 struct cfs_rq *cfs_rq;1345613457 #ifdef CONFIG_FAIR_GROUP_SCHED13458 cfs_rq = task_group(p)->cfs_rq[cpu];13459 #else13460 cfs_rq = &cpu_rq(cpu)->cfs;13461 #endif13462 return throttled_hierarchy(cfs_rq);13463 }13464 #else /* !CONFIG_SCHED_CORE: */13465 static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}13466 #endif /* !CONFIG_SCHED_CORE */1346713468 /*13469 * scheduler tick hitting a task of our scheduling class.13470 *13471 * NOTE: This function can be called remotely by the tick offload that13472 * goes along full dynticks. Therefore no local assumption can be made13473 * and everything must be accessed through the @rq and @curr passed in13474 * parameters.13475 */13476 static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)13477 {13478 struct cfs_rq *cfs_rq;13479 struct sched_entity *se = &curr->se;1348013481 for_each_sched_entity(se) {13482 cfs_rq = cfs_rq_of(se);13483 entity_tick(cfs_rq, se, queued);13484 }1348513486 if (static_branch_unlikely(&sched_numa_balancing))13487 task_tick_numa(rq, curr);1348813489 update_misfit_status(curr, rq);13490 check_update_overutilized_status(task_rq(curr));1349113492 task_tick_core(rq, curr);13493 }1349413495 /*13496 * called on fork with the child task as argument from the parent's context13497 * - child not yet on the tasklist13498 * - preemption disabled13499 */13500 static void task_fork_fair(struct task_struct *p)13501 {13502 set_task_max_allowed_capacity(p);13503 }1350413505 /*13506 * Priority of the task has changed. Check to see if we preempt13507 * the current task.13508 */13509 static void13510 prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)13511 {13512 if (!task_on_rq_queued(p))13513 return;1351413515 if (rq->cfs.nr_queued == 1)13516 return;1351713518 /*13519 * Reschedule if we are currently running on this runqueue and13520 * our priority decreased, or if we are not currently running on13521 * this runqueue and our priority is higher than the current's13522 */13523 if (task_current_donor(rq, p)) {13524 if (p->prio > oldprio)13525 resched_curr(rq);13526 } else13527 wakeup_preempt(rq, p, 0);13528 }1352913530 #ifdef CONFIG_FAIR_GROUP_SCHED13531 /*13532 * Propagate the changes of the sched_entity across the tg tree to make it13533 * visible to the root13534 */13535 static void propagate_entity_cfs_rq(struct sched_entity *se)13536 {13537 struct cfs_rq *cfs_rq = cfs_rq_of(se);1353813539 /*13540 * If a task gets attached to this cfs_rq and before being queued,13541 * it gets migrated to another CPU due to reasons like affinity13542 * change, make sure this cfs_rq stays on leaf cfs_rq list to have13543 * that removed load decayed or it can cause faireness problem.13544 */13545 if (!cfs_rq_pelt_clock_throttled(cfs_rq))13546 list_add_leaf_cfs_rq(cfs_rq);1354713548 /* Start to propagate at parent */13549 se = se->parent;1355013551 for_each_sched_entity(se) {13552 cfs_rq = cfs_rq_of(se);1355313554 update_load_avg(cfs_rq, se, UPDATE_TG);1355513556 if (!cfs_rq_pelt_clock_throttled(cfs_rq))13557 list_add_leaf_cfs_rq(cfs_rq);13558 }1355913560 assert_list_leaf_cfs_rq(rq_of(cfs_rq));13561 }13562 #else /* !CONFIG_FAIR_GROUP_SCHED: */13563 static void propagate_entity_cfs_rq(struct sched_entity *se) { }13564 #endif /* !CONFIG_FAIR_GROUP_SCHED */1356513566 static void detach_entity_cfs_rq(struct sched_entity *se)13567 {13568 struct cfs_rq *cfs_rq = cfs_rq_of(se);1356913570 /*13571 * In case the task sched_avg hasn't been attached:13572 * - A forked task which hasn't been woken up by wake_up_new_task().13573 * - A task which has been woken up by try_to_wake_up() but is13574 * waiting for actually being woken up by sched_ttwu_pending().13575 */13576 if (!se->avg.last_update_time)13577 return;1357813579 /* Catch up with the cfs_rq and remove our load when we leave */13580 update_load_avg(cfs_rq, se, 0);13581 detach_entity_load_avg(cfs_rq, se);13582 update_tg_load_avg(cfs_rq);13583 propagate_entity_cfs_rq(se);13584 }1358513586 static void attach_entity_cfs_rq(struct sched_entity *se)13587 {13588 struct cfs_rq *cfs_rq = cfs_rq_of(se);1358913590 /* Synchronize entity with its cfs_rq */13591 update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);13592 attach_entity_load_avg(cfs_rq, se);13593 update_tg_load_avg(cfs_rq);13594 propagate_entity_cfs_rq(se);13595 }1359613597 static void detach_task_cfs_rq(struct task_struct *p)13598 {13599 struct sched_entity *se = &p->se;1360013601 detach_entity_cfs_rq(se);13602 }1360313604 static void attach_task_cfs_rq(struct task_struct *p)13605 {13606 struct sched_entity *se = &p->se;1360713608 attach_entity_cfs_rq(se);13609 }1361013611 static void switched_from_fair(struct rq *rq, struct task_struct *p)13612 {13613 detach_task_cfs_rq(p);13614 }1361513616 static void switched_to_fair(struct rq *rq, struct task_struct *p)13617 {13618 WARN_ON_ONCE(p->se.sched_delayed);1361913620 attach_task_cfs_rq(p);1362113622 set_task_max_allowed_capacity(p);1362313624 if (task_on_rq_queued(p)) {13625 /*13626 * We were most likely switched from sched_rt, so13627 * kick off the schedule if running, otherwise just see13628 * if we can still preempt the current task.13629 */13630 if (task_current_donor(rq, p))13631 resched_curr(rq);13632 else13633 wakeup_preempt(rq, p, 0);13634 }13635 }1363613637 static void __set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)13638 {13639 struct sched_entity *se = &p->se;1364013641 if (task_on_rq_queued(p)) {13642 /*13643 * Move the next running task to the front of the list, so our13644 * cfs_tasks list becomes MRU one.13645 */13646 list_move(&se->group_node, &rq->cfs_tasks);13647 }13648 if (!first)13649 return;1365013651 WARN_ON_ONCE(se->sched_delayed);1365213653 if (hrtick_enabled_fair(rq))13654 hrtick_start_fair(rq, p);1365513656 update_misfit_status(p, rq);13657 sched_fair_update_stop_tick(rq, p);13658 }1365913660 /*13661 * Account for a task changing its policy or group.13662 *13663 * This routine is mostly called to set cfs_rq->curr field when a task13664 * migrates between groups/classes.13665 */13666 static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)13667 {13668 struct sched_entity *se = &p->se;1366913670 for_each_sched_entity(se) {13671 struct cfs_rq *cfs_rq = cfs_rq_of(se);1367213673 set_next_entity(cfs_rq, se, first);13674 /* ensure bandwidth has been allocated on our new cfs_rq */13675 account_cfs_rq_runtime(cfs_rq, 0);13676 }1367713678 __set_next_task_fair(rq, p, first);13679 }1368013681 void init_cfs_rq(struct cfs_rq *cfs_rq)13682 {13683 cfs_rq->tasks_timeline = RB_ROOT_CACHED;13684 cfs_rq->zero_vruntime = (u64)(-(1LL << 20));13685 raw_spin_lock_init(&cfs_rq->removed.lock);13686 }1368713688 #ifdef CONFIG_FAIR_GROUP_SCHED13689 static void task_change_group_fair(struct task_struct *p)13690 {13691 /*13692 * We couldn't detach or attach a forked task which13693 * hasn't been woken up by wake_up_new_task().13694 */13695 if (READ_ONCE(p->__state) == TASK_NEW)13696 return;1369713698 detach_task_cfs_rq(p);1369913700 /* Tell se's cfs_rq has been changed -- migrated */13701 p->se.avg.last_update_time = 0;13702 set_task_rq(p, task_cpu(p));13703 attach_task_cfs_rq(p);13704 }1370513706 void free_fair_sched_group(struct task_group *tg)13707 {13708 int i;1370913710 for_each_possible_cpu(i) {13711 if (tg->cfs_rq)13712 kfree(tg->cfs_rq[i]);13713 if (tg->se)13714 kfree(tg->se[i]);13715 }1371613717 kfree(tg->cfs_rq);13718 kfree(tg->se);13719 }1372013721 int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)13722 {13723 struct sched_entity *se;13724 struct cfs_rq *cfs_rq;13725 int i;1372613727 tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL);13728 if (!tg->cfs_rq)13729 goto err;13730 tg->se = kcalloc(nr_cpu_ids, sizeof(se), GFP_KERNEL);13731 if (!tg->se)13732 goto err;1373313734 tg->shares = NICE_0_LOAD;1373513736 init_cfs_bandwidth(tg_cfs_bandwidth(tg), tg_cfs_bandwidth(parent));1373713738 for_each_possible_cpu(i) {13739 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),13740 GFP_KERNEL, cpu_to_node(i));13741 if (!cfs_rq)13742 goto err;1374313744 se = kzalloc_node(sizeof(struct sched_entity_stats),13745 GFP_KERNEL, cpu_to_node(i));13746 if (!se)13747 goto err_free_rq;1374813749 init_cfs_rq(cfs_rq);13750 init_tg_cfs_entry(tg, cfs_rq, se, i, parent->se[i]);13751 init_entity_runnable_average(se);13752 }1375313754 return 1;1375513756 err_free_rq:13757 kfree(cfs_rq);13758 err:13759 return 0;13760 }1376113762 void online_fair_sched_group(struct task_group *tg)13763 {13764 struct sched_entity *se;13765 struct rq_flags rf;13766 struct rq *rq;13767 int i;1376813769 for_each_possible_cpu(i) {13770 rq = cpu_rq(i);13771 se = tg->se[i];13772 rq_lock_irq(rq, &rf);13773 update_rq_clock(rq);13774 attach_entity_cfs_rq(se);13775 sync_throttle(tg, i);13776 rq_unlock_irq(rq, &rf);13777 }13778 }1377913780 void unregister_fair_sched_group(struct task_group *tg)13781 {13782 int cpu;1378313784 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));1378513786 for_each_possible_cpu(cpu) {13787 struct cfs_rq *cfs_rq = tg->cfs_rq[cpu];13788 struct sched_entity *se = tg->se[cpu];13789 struct rq *rq = cpu_rq(cpu);1379013791 if (se) {13792 if (se->sched_delayed) {13793 guard(rq_lock_irqsave)(rq);13794 if (se->sched_delayed) {13795 update_rq_clock(rq);13796 dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED);13797 }13798 list_del_leaf_cfs_rq(cfs_rq);13799 }13800 remove_entity_load_avg(se);13801 }1380213803 /*13804 * Only empty task groups can be destroyed; so we can speculatively13805 * check on_list without danger of it being re-added.13806 */13807 if (cfs_rq->on_list) {13808 guard(rq_lock_irqsave)(rq);13809 list_del_leaf_cfs_rq(cfs_rq);13810 }13811 }13812 }1381313814 void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,13815 struct sched_entity *se, int cpu,13816 struct sched_entity *parent)13817 {13818 struct rq *rq = cpu_rq(cpu);1381913820 cfs_rq->tg = tg;13821 cfs_rq->rq = rq;13822 init_cfs_rq_runtime(cfs_rq);1382313824 tg->cfs_rq[cpu] = cfs_rq;13825 tg->se[cpu] = se;1382613827 /* se could be NULL for root_task_group */13828 if (!se)13829 return;1383013831 if (!parent) {13832 se->cfs_rq = &rq->cfs;13833 se->depth = 0;13834 } else {13835 se->cfs_rq = parent->my_q;13836 se->depth = parent->depth + 1;13837 }1383813839 se->my_q = cfs_rq;13840 /* guarantee group entities always have weight */13841 update_load_set(&se->load, NICE_0_LOAD);13842 se->parent = parent;13843 }1384413845 static DEFINE_MUTEX(shares_mutex);1384613847 static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)13848 {13849 int i;1385013851 lockdep_assert_held(&shares_mutex);1385213853 /*13854 * We can't change the weight of the root cgroup.13855 */13856 if (!tg->se[0])13857 return -EINVAL;1385813859 shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));1386013861 if (tg->shares == shares)13862 return 0;1386313864 tg->shares = shares;13865 for_each_possible_cpu(i) {13866 struct rq *rq = cpu_rq(i);13867 struct sched_entity *se = tg->se[i];13868 struct rq_flags rf;1386913870 /* Propagate contribution to hierarchy */13871 rq_lock_irqsave(rq, &rf);13872 update_rq_clock(rq);13873 for_each_sched_entity(se) {13874 update_load_avg(cfs_rq_of(se), se, UPDATE_TG);13875 update_cfs_group(se);13876 }13877 rq_unlock_irqrestore(rq, &rf);13878 }1387913880 return 0;13881 }1388213883 int sched_group_set_shares(struct task_group *tg, unsigned long shares)13884 {13885 int ret;1388613887 mutex_lock(&shares_mutex);13888 if (tg_is_idle(tg))13889 ret = -EINVAL;13890 else13891 ret = __sched_group_set_shares(tg, shares);13892 mutex_unlock(&shares_mutex);1389313894 return ret;13895 }1389613897 int sched_group_set_idle(struct task_group *tg, long idle)13898 {13899 int i;1390013901 if (tg == &root_task_group)13902 return -EINVAL;1390313904 if (idle < 0 || idle > 1)13905 return -EINVAL;1390613907 mutex_lock(&shares_mutex);1390813909 if (tg->idle == idle) {13910 mutex_unlock(&shares_mutex);13911 return 0;13912 }1391313914 tg->idle = idle;1391513916 for_each_possible_cpu(i) {13917 struct rq *rq = cpu_rq(i);13918 struct sched_entity *se = tg->se[i];13919 struct cfs_rq *grp_cfs_rq = tg->cfs_rq[i];13920 bool was_idle = cfs_rq_is_idle(grp_cfs_rq);13921 long idle_task_delta;13922 struct rq_flags rf;1392313924 rq_lock_irqsave(rq, &rf);1392513926 grp_cfs_rq->idle = idle;13927 if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq)))13928 goto next_cpu;1392913930 idle_task_delta = grp_cfs_rq->h_nr_queued -13931 grp_cfs_rq->h_nr_idle;13932 if (!cfs_rq_is_idle(grp_cfs_rq))13933 idle_task_delta *= -1;1393413935 for_each_sched_entity(se) {13936 struct cfs_rq *cfs_rq = cfs_rq_of(se);1393713938 if (!se->on_rq)13939 break;1394013941 cfs_rq->h_nr_idle += idle_task_delta;1394213943 /* Already accounted at parent level and above. */13944 if (cfs_rq_is_idle(cfs_rq))13945 break;13946 }1394713948 next_cpu:13949 rq_unlock_irqrestore(rq, &rf);13950 }1395113952 /* Idle groups have minimum weight. */13953 if (tg_is_idle(tg))13954 __sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO));13955 else13956 __sched_group_set_shares(tg, NICE_0_LOAD);1395713958 mutex_unlock(&shares_mutex);13959 return 0;13960 }1396113962 #endif /* CONFIG_FAIR_GROUP_SCHED */139631396413965 static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)13966 {13967 struct sched_entity *se = &task->se;13968 unsigned int rr_interval = 0;1396913970 /*13971 * Time slice is 0 for SCHED_OTHER tasks that are on an otherwise13972 * idle runqueue:13973 */13974 if (rq->cfs.load.weight)13975 rr_interval = NS_TO_JIFFIES(se->slice);1397613977 return rr_interval;13978 }1397913980 /*13981 * All the scheduling class methods:13982 */13983 DEFINE_SCHED_CLASS(fair) = {1398413985 .enqueue_task = enqueue_task_fair,13986 .dequeue_task = dequeue_task_fair,13987 .yield_task = yield_task_fair,13988 .yield_to_task = yield_to_task_fair,1398913990 .wakeup_preempt = check_preempt_wakeup_fair,1399113992 .pick_task = pick_task_fair,13993 .pick_next_task = __pick_next_task_fair,13994 .put_prev_task = put_prev_task_fair,13995 .set_next_task = set_next_task_fair,1399613997 .balance = balance_fair,13998 .select_task_rq = select_task_rq_fair,13999 .migrate_task_rq = migrate_task_rq_fair,1400014001 .rq_online = rq_online_fair,14002 .rq_offline = rq_offline_fair,1400314004 .task_dead = task_dead_fair,14005 .set_cpus_allowed = set_cpus_allowed_fair,1400614007 .task_tick = task_tick_fair,14008 .task_fork = task_fork_fair,1400914010 .reweight_task = reweight_task_fair,14011 .prio_changed = prio_changed_fair,14012 .switched_from = switched_from_fair,14013 .switched_to = switched_to_fair,1401414015 .get_rr_interval = get_rr_interval_fair,1401614017 .update_curr = update_curr_fair,1401814019 #ifdef CONFIG_FAIR_GROUP_SCHED14020 .task_change_group = task_change_group_fair,14021 #endif1402214023 #ifdef CONFIG_SCHED_CORE14024 .task_is_throttled = task_is_throttled_fair,14025 #endif1402614027 #ifdef CONFIG_UCLAMP_TASK14028 .uclamp_enabled = 1,14029 #endif14030 };1403114032 void print_cfs_stats(struct seq_file *m, int cpu)14033 {14034 struct cfs_rq *cfs_rq, *pos;1403514036 rcu_read_lock();14037 for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)14038 print_cfs_rq(m, cpu, cfs_rq);14039 rcu_read_unlock();14040 }1404114042 #ifdef CONFIG_NUMA_BALANCING14043 void show_numa_stats(struct task_struct *p, struct seq_file *m)14044 {14045 int node;14046 unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0;14047 struct numa_group *ng;1404814049 rcu_read_lock();14050 ng = rcu_dereference(p->numa_group);14051 for_each_online_node(node) {14052 if (p->numa_faults) {14053 tsf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 0)];14054 tpf = p->numa_faults[task_faults_idx(NUMA_MEM, node, 1)];14055 }14056 if (ng) {14057 gsf = ng->faults[task_faults_idx(NUMA_MEM, node, 0)],14058 gpf = ng->faults[task_faults_idx(NUMA_MEM, node, 1)];14059 }14060 print_numa_stats(m, node, tsf, tpf, gsf, gpf);14061 }14062 rcu_read_unlock();14063 }14064 #endif /* CONFIG_NUMA_BALANCING */1406514066 __init void init_sched_fair_class(void)14067 {14068 int i;1406914070 for_each_possible_cpu(i) {14071 zalloc_cpumask_var_node(&per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(i));14072 zalloc_cpumask_var_node(&per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(i));14073 zalloc_cpumask_var_node(&per_cpu(should_we_balance_tmpmask, i),14074 GFP_KERNEL, cpu_to_node(i));1407514076 #ifdef CONFIG_CFS_BANDWIDTH14077 INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i));14078 INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list);14079 #endif14080 }1408114082 open_softirq(SCHED_SOFTIRQ, sched_balance_softirq);1408314084 #ifdef CONFIG_NO_HZ_COMMON14085 nohz.next_balance = jiffies;14086 nohz.next_blocked = jiffies;14087 zalloc_cpumask_var(&nohz.idle_cpus_mask, GFP_NOWAIT);14088 #endif14089 }