개념 설명 전체 · v6.18.37 / mm/vmscan.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 4 * 5 * Swap reorganised 29.12.95, Stephen Tweedie. 6 * kswapd added: 7.1.96 sct 7 * Removed kswapd_ctl limits, and swap out as many pages as needed 8 * to bring the system back to freepages.high: 2.4.97, Rik van Riel. 9 * Zone aware kswapd started 02/00, Kanoj Sarcar ([email protected]). 10 * Multiqueue VM started 5.8.00, Rik van Riel. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/mm.h> 16 #include <linux/sched/mm.h> 17 #include <linux/module.h> 18 #include <linux/gfp.h> 19 #include <linux/kernel_stat.h> 20 #include <linux/swap.h> 21 #include <linux/pagemap.h> 22 #include <linux/init.h> 23 #include <linux/highmem.h> 24 #include <linux/vmpressure.h> 25 #include <linux/vmstat.h> 26 #include <linux/file.h> 27 #include <linux/writeback.h> 28 #include <linux/blkdev.h> 29 #include <linux/buffer_head.h> /* for buffer_heads_over_limit */ 30 #include <linux/mm_inline.h> 31 #include <linux/backing-dev.h> 32 #include <linux/rmap.h> 33 #include <linux/topology.h> 34 #include <linux/cpu.h> 35 #include <linux/cpuset.h> 36 #include <linux/compaction.h> 37 #include <linux/notifier.h> 38 #include <linux/delay.h> 39 #include <linux/kthread.h> 40 #include <linux/freezer.h> 41 #include <linux/memcontrol.h> 42 #include <linux/migrate.h> 43 #include <linux/delayacct.h> 44 #include <linux/sysctl.h> 45 #include <linux/memory-tiers.h> 46 #include <linux/oom.h> 47 #include <linux/pagevec.h> 48 #include <linux/prefetch.h> 49 #include <linux/printk.h> 50 #include <linux/dax.h> 51 #include <linux/psi.h> 52 #include <linux/pagewalk.h> 53 #include <linux/shmem_fs.h> 54 #include <linux/ctype.h> 55 #include <linux/debugfs.h> 56 #include <linux/khugepaged.h> 57 #include <linux/rculist_nulls.h> 58 #include <linux/random.h> 59 #include <linux/mmu_notifier.h> 60 #include <linux/parser.h> 61 62 #include <asm/tlbflush.h> 63 #include <asm/div64.h> 64 65 #include <linux/swapops.h> 66 #include <linux/balloon_compaction.h> 67 #include <linux/sched/sysctl.h> 68 69 #include "internal.h" 70 #include "swap.h" 71 72 #define CREATE_TRACE_POINTS 73 #include <trace/events/vmscan.h> 74 75 struct scan_control { 76 /* How many pages shrink_list() should reclaim */ 77 unsigned long nr_to_reclaim; 78 79 /* 80 * Nodemask of nodes allowed by the caller. If NULL, all nodes 81 * are scanned. 82 */ 83 nodemask_t *nodemask; 84 85 /* 86 * The memory cgroup that hit its limit and as a result is the 87 * primary target of this reclaim invocation. 88 */ 89 struct mem_cgroup *target_mem_cgroup; 90 91 /* 92 * Scan pressure balancing between anon and file LRUs 93 */ 94 unsigned long anon_cost; 95 unsigned long file_cost; 96 97 /* Swappiness value for proactive reclaim. Always use sc_swappiness()! */ 98 int *proactive_swappiness; 99 100 /* Can active folios be deactivated as part of reclaim? */ 101 #define DEACTIVATE_ANON 1 102 #define DEACTIVATE_FILE 2 103 unsigned int may_deactivate:2; 104 unsigned int force_deactivate:1; 105 unsigned int skipped_deactivate:1; 106 107 /* Writepage batching in laptop mode; RECLAIM_WRITE */ 108 unsigned int may_writepage:1; 109 110 /* Can mapped folios be reclaimed? */ 111 unsigned int may_unmap:1; 112 113 /* Can folios be swapped as part of reclaim? */ 114 unsigned int may_swap:1; 115 116 /* Not allow cache_trim_mode to be turned on as part of reclaim? */ 117 unsigned int no_cache_trim_mode:1; 118 119 /* Has cache_trim_mode failed at least once? */ 120 unsigned int cache_trim_mode_failed:1; 121 122 /* Proactive reclaim invoked by userspace */ 123 unsigned int proactive:1; 124 125 /* 126 * Cgroup memory below memory.low is protected as long as we 127 * don't threaten to OOM. If any cgroup is reclaimed at 128 * reduced force or passed over entirely due to its memory.low 129 * setting (memcg_low_skipped), and nothing is reclaimed as a 130 * result, then go back for one more cycle that reclaims the protected 131 * memory (memcg_low_reclaim) to avert OOM. 132 */ 133 unsigned int memcg_low_reclaim:1; 134 unsigned int memcg_low_skipped:1; 135 136 /* Shared cgroup tree walk failed, rescan the whole tree */ 137 unsigned int memcg_full_walk:1; 138 139 unsigned int hibernation_mode:1; 140 141 /* One of the zones is ready for compaction */ 142 unsigned int compaction_ready:1; 143 144 /* There is easily reclaimable cold cache in the current node */ 145 unsigned int cache_trim_mode:1; 146 147 /* The file folios on the current node are dangerously low */ 148 unsigned int file_is_tiny:1; 149 150 /* Always discard instead of demoting to lower tier memory */ 151 unsigned int no_demotion:1; 152 153 /* Allocation order */ 154 s8 order; 155 156 /* Scan (total_size >> priority) pages at once */ 157 s8 priority; 158 159 /* The highest zone to isolate folios for reclaim from */ 160 s8 reclaim_idx; 161 162 /* This context's GFP mask */ 163 gfp_t gfp_mask; 164 165 /* Incremented by the number of inactive pages that were scanned */ 166 unsigned long nr_scanned; 167 168 /* Number of pages freed so far during a call to shrink_zones() */ 169 unsigned long nr_reclaimed; 170 171 struct { 172 unsigned int dirty; 173 unsigned int unqueued_dirty; 174 unsigned int congested; 175 unsigned int writeback; 176 unsigned int immediate; 177 unsigned int file_taken; 178 unsigned int taken; 179 } nr; 180 181 /* for recording the reclaimed slab by now */ 182 struct reclaim_state reclaim_state; 183 }; 184 185 #ifdef ARCH_HAS_PREFETCHW 186 #define prefetchw_prev_lru_folio(_folio, _base, _field) \ 187 do { \ 188 if ((_folio)->lru.prev != _base) { \ 189 struct folio *prev; \ 190 \ 191 prev = lru_to_folio(&(_folio->lru)); \ 192 prefetchw(&prev->_field); \ 193 } \ 194 } while (0) 195 #else 196 #define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) 197 #endif 198 199 /* 200 * From 0 .. MAX_SWAPPINESS. Higher means more swappy. 201 */ 202 int vm_swappiness = 60; 203 204 #ifdef CONFIG_MEMCG 205 206 /* Returns true for reclaim through cgroup limits or cgroup interfaces. */ 207 static bool cgroup_reclaim(struct scan_control *sc) 208 { 209 return sc->target_mem_cgroup; 210 } 211 212 /* 213 * Returns true for reclaim on the root cgroup. This is true for direct 214 * allocator reclaim and reclaim through cgroup interfaces on the root cgroup. 215 */ 216 static bool root_reclaim(struct scan_control *sc) 217 { 218 return !sc->target_mem_cgroup || mem_cgroup_is_root(sc->target_mem_cgroup); 219 } 220 221 /** 222 * writeback_throttling_sane - is the usual dirty throttling mechanism available? 223 * @sc: scan_control in question 224 * 225 * The normal page dirty throttling mechanism in balance_dirty_pages() is 226 * completely broken with the legacy memcg and direct stalling in 227 * shrink_folio_list() is used for throttling instead, which lacks all the 228 * niceties such as fairness, adaptive pausing, bandwidth proportional 229 * allocation and configurability. 230 * 231 * This function tests whether the vmscan currently in progress can assume 232 * that the normal dirty throttling mechanism is operational. 233 */ 234 static bool writeback_throttling_sane(struct scan_control *sc) 235 { 236 if (!cgroup_reclaim(sc)) 237 return true; 238 #ifdef CONFIG_CGROUP_WRITEBACK 239 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) 240 return true; 241 #endif 242 return false; 243 } 244 245 static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) 246 { 247 if (sc->proactive && sc->proactive_swappiness) 248 return *sc->proactive_swappiness; 249 return mem_cgroup_swappiness(memcg); 250 } 251 #else 252 static bool cgroup_reclaim(struct scan_control *sc) 253 { 254 return false; 255 } 256 257 static bool root_reclaim(struct scan_control *sc) 258 { 259 return true; 260 } 261 262 static bool writeback_throttling_sane(struct scan_control *sc) 263 { 264 return true; 265 } 266 267 static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) 268 { 269 return READ_ONCE(vm_swappiness); 270 } 271 #endif 272 273 /* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to 274 * and including the specified highidx 275 * @zone: The current zone in the iterator 276 * @pgdat: The pgdat which node_zones are being iterated 277 * @idx: The index variable 278 * @highidx: The index of the highest zone to return 279 * 280 * This macro iterates through all managed zones up to and including the specified highidx. 281 * The zone iterator enters an invalid state after macro call and must be reinitialized 282 * before it can be used again. 283 */ 284 #define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx) \ 285 for ((idx) = 0, (zone) = (pgdat)->node_zones; \ 286 (idx) <= (highidx); \ 287 (idx)++, (zone)++) \ 288 if (!managed_zone(zone)) \ 289 continue; \ 290 else 291 292 static void set_task_reclaim_state(struct task_struct *task, 293 struct reclaim_state *rs) 294 { 295 /* Check for an overwrite */ 296 WARN_ON_ONCE(rs && task->reclaim_state); 297 298 /* Check for the nulling of an already-nulled member */ 299 WARN_ON_ONCE(!rs && !task->reclaim_state); 300 301 task->reclaim_state = rs; 302 } 303 304 /* 305 * flush_reclaim_state(): add pages reclaimed outside of LRU-based reclaim to 306 * scan_control->nr_reclaimed. 307 */ 308 static void flush_reclaim_state(struct scan_control *sc) 309 { 310 /* 311 * Currently, reclaim_state->reclaimed includes three types of pages 312 * freed outside of vmscan: 313 * (1) Slab pages. 314 * (2) Clean file pages from pruned inodes (on highmem systems). 315 * (3) XFS freed buffer pages. 316 * 317 * For all of these cases, we cannot universally link the pages to a 318 * single memcg. For example, a memcg-aware shrinker can free one object 319 * charged to the target memcg, causing an entire page to be freed. 320 * If we count the entire page as reclaimed from the memcg, we end up 321 * overestimating the reclaimed amount (potentially under-reclaiming). 322 * 323 * Only count such pages for global reclaim to prevent under-reclaiming 324 * from the target memcg; preventing unnecessary retries during memcg 325 * charging and false positives from proactive reclaim. 326 * 327 * For uncommon cases where the freed pages were actually mostly 328 * charged to the target memcg, we end up underestimating the reclaimed 329 * amount. This should be fine. The freed pages will be uncharged 330 * anyway, even if they are not counted here properly, and we will be 331 * able to make forward progress in charging (which is usually in a 332 * retry loop). 333 * 334 * We can go one step further, and report the uncharged objcg pages in 335 * memcg reclaim, to make reporting more accurate and reduce 336 * underestimation, but it's probably not worth the complexity for now. 337 */ 338 if (current->reclaim_state && root_reclaim(sc)) { 339 sc->nr_reclaimed += current->reclaim_state->reclaimed; 340 current->reclaim_state->reclaimed = 0; 341 } 342 } 343 344 static bool can_demote(int nid, struct scan_control *sc, 345 struct mem_cgroup *memcg) 346 { 347 struct pglist_data *pgdat = NODE_DATA(nid); 348 nodemask_t allowed_mask; 349 350 if (!pgdat || !numa_demotion_enabled) 351 return false; 352 if (sc && sc->no_demotion) 353 return false; 354 355 node_get_allowed_targets(pgdat, &allowed_mask); 356 if (nodes_empty(allowed_mask)) 357 return false; 358 359 /* Filter out nodes that are not in cgroup's mems_allowed. */ 360 mem_cgroup_node_filter_allowed(memcg, &allowed_mask); 361 return !nodes_empty(allowed_mask); 362 } 363 364 static inline bool can_reclaim_anon_pages(struct mem_cgroup *memcg, 365 int nid, 366 struct scan_control *sc) 367 { 368 if (memcg == NULL) { 369 /* 370 * For non-memcg reclaim, is there 371 * space in any swap device? 372 */ 373 if (get_nr_swap_pages() > 0) 374 return true; 375 } else { 376 /* Is the memcg below its swap limit? */ 377 if (mem_cgroup_get_nr_swap_pages(memcg) > 0) 378 return true; 379 } 380 381 /* 382 * The page can not be swapped. 383 * 384 * Can it be reclaimed from this node via demotion? 385 */ 386 return can_demote(nid, sc, memcg); 387 } 388 389 /* 390 * This misses isolated folios which are not accounted for to save counters. 391 * As the data only determines if reclaim or compaction continues, it is 392 * not expected that isolated folios will be a dominating factor. 393 */ 394 unsigned long zone_reclaimable_pages(struct zone *zone) 395 { 396 unsigned long nr; 397 398 nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) + 399 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE); 400 if (can_reclaim_anon_pages(NULL, zone_to_nid(zone), NULL)) 401 nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) + 402 zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON); 403 404 return nr; 405 } 406 407 /** 408 * lruvec_lru_size - Returns the number of pages on the given LRU list. 409 * @lruvec: lru vector 410 * @lru: lru to use 411 * @zone_idx: zones to consider (use MAX_NR_ZONES - 1 for the whole LRU list) 412 */ 413 static unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, 414 int zone_idx) 415 { 416 unsigned long size = 0; 417 int zid; 418 struct zone *zone; 419 420 for_each_managed_zone_pgdat(zone, lruvec_pgdat(lruvec), zid, zone_idx) { 421 if (!mem_cgroup_disabled()) 422 size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid); 423 else 424 size += zone_page_state(zone, NR_ZONE_LRU_BASE + lru); 425 } 426 return size; 427 } 428 429 static unsigned long drop_slab_node(int nid) 430 { 431 unsigned long freed = 0; 432 struct mem_cgroup *memcg = NULL; 433 434 memcg = mem_cgroup_iter(NULL, NULL, NULL); 435 do { 436 freed += shrink_slab(GFP_KERNEL, nid, memcg, 0); 437 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL); 438 439 return freed; 440 } 441 442 void drop_slab(void) 443 { 444 int nid; 445 int shift = 0; 446 unsigned long freed; 447 448 do { 449 freed = 0; 450 for_each_online_node(nid) { 451 if (fatal_signal_pending(current)) 452 return; 453 454 freed += drop_slab_node(nid); 455 } 456 } while ((freed >> shift++) > 1); 457 } 458 459 #define CHECK_RECLAIMER_OFFSET(type) \ 460 do { \ 461 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \ 462 PGDEMOTE_##type - PGDEMOTE_KSWAPD); \ 463 BUILD_BUG_ON(PGSTEAL_##type - PGSTEAL_KSWAPD != \ 464 PGSCAN_##type - PGSCAN_KSWAPD); \ 465 } while (0) 466 467 static int reclaimer_offset(struct scan_control *sc) 468 { 469 CHECK_RECLAIMER_OFFSET(DIRECT); 470 CHECK_RECLAIMER_OFFSET(KHUGEPAGED); 471 CHECK_RECLAIMER_OFFSET(PROACTIVE); 472 473 if (current_is_kswapd()) 474 return 0; 475 if (current_is_khugepaged()) 476 return PGSTEAL_KHUGEPAGED - PGSTEAL_KSWAPD; 477 if (sc->proactive) 478 return PGSTEAL_PROACTIVE - PGSTEAL_KSWAPD; 479 return PGSTEAL_DIRECT - PGSTEAL_KSWAPD; 480 } 481 482 static inline int is_page_cache_freeable(struct folio *folio) 483 { 484 /* 485 * A freeable page cache folio is referenced only by the caller 486 * that isolated the folio, the page cache and optional filesystem 487 * private data at folio->private. 488 */ 489 return folio_ref_count(folio) - folio_test_private(folio) == 490 1 + folio_nr_pages(folio); 491 } 492 493 /* 494 * We detected a synchronous write error writing a folio out. Probably 495 * -ENOSPC. We need to propagate that into the address_space for a subsequent 496 * fsync(), msync() or close(). 497 * 498 * The tricky part is that after writepage we cannot touch the mapping: nothing 499 * prevents it from being freed up. But we have a ref on the folio and once 500 * that folio is locked, the mapping is pinned. 501 * 502 * We're allowed to run sleeping folio_lock() here because we know the caller has 503 * __GFP_FS. 504 */ 505 static void handle_write_error(struct address_space *mapping, 506 struct folio *folio, int error) 507 { 508 folio_lock(folio); 509 if (folio_mapping(folio) == mapping) 510 mapping_set_error(mapping, error); 511 folio_unlock(folio); 512 } 513 514 static bool skip_throttle_noprogress(pg_data_t *pgdat) 515 { 516 int reclaimable = 0, write_pending = 0; 517 int i; 518 struct zone *zone; 519 /* 520 * If kswapd is disabled, reschedule if necessary but do not 521 * throttle as the system is likely near OOM. 522 */ 523 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES) 524 return true; 525 526 /* 527 * If there are a lot of dirty/writeback folios then do not 528 * throttle as throttling will occur when the folios cycle 529 * towards the end of the LRU if still under writeback. 530 */ 531 for_each_managed_zone_pgdat(zone, pgdat, i, MAX_NR_ZONES - 1) { 532 reclaimable += zone_reclaimable_pages(zone); 533 write_pending += zone_page_state_snapshot(zone, 534 NR_ZONE_WRITE_PENDING); 535 } 536 if (2 * write_pending <= reclaimable) 537 return true; 538 539 return false; 540 } 541 542 void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason) 543 { 544 wait_queue_head_t *wqh = &pgdat->reclaim_wait[reason]; 545 long timeout, ret; 546 DEFINE_WAIT(wait); 547 548 /* 549 * Do not throttle user workers, kthreads other than kswapd or 550 * workqueues. They may be required for reclaim to make 551 * forward progress (e.g. journalling workqueues or kthreads). 552 */ 553 if (!current_is_kswapd() && 554 current->flags & (PF_USER_WORKER|PF_KTHREAD)) { 555 cond_resched(); 556 return; 557 } 558 559 /* 560 * These figures are pulled out of thin air. 561 * VMSCAN_THROTTLE_ISOLATED is a transient condition based on too many 562 * parallel reclaimers which is a short-lived event so the timeout is 563 * short. Failing to make progress or waiting on writeback are 564 * potentially long-lived events so use a longer timeout. This is shaky 565 * logic as a failure to make progress could be due to anything from 566 * writeback to a slow device to excessive referenced folios at the tail 567 * of the inactive LRU. 568 */ 569 switch(reason) { 570 case VMSCAN_THROTTLE_WRITEBACK: 571 timeout = HZ/10; 572 573 if (atomic_inc_return(&pgdat->nr_writeback_throttled) == 1) { 574 WRITE_ONCE(pgdat->nr_reclaim_start, 575 node_page_state(pgdat, NR_THROTTLED_WRITTEN)); 576 } 577 578 break; 579 case VMSCAN_THROTTLE_CONGESTED: 580 fallthrough; 581 case VMSCAN_THROTTLE_NOPROGRESS: 582 if (skip_throttle_noprogress(pgdat)) { 583 cond_resched(); 584 return; 585 } 586 587 timeout = 1; 588 589 break; 590 case VMSCAN_THROTTLE_ISOLATED: 591 timeout = HZ/50; 592 break; 593 default: 594 WARN_ON_ONCE(1); 595 timeout = HZ; 596 break; 597 } 598 599 prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE); 600 ret = schedule_timeout(timeout); 601 finish_wait(wqh, &wait); 602 603 if (reason == VMSCAN_THROTTLE_WRITEBACK) 604 atomic_dec(&pgdat->nr_writeback_throttled); 605 606 trace_mm_vmscan_throttled(pgdat->node_id, jiffies_to_usecs(timeout), 607 jiffies_to_usecs(timeout - ret), 608 reason); 609 } 610 611 /* 612 * Account for folios written if tasks are throttled waiting on dirty 613 * folios to clean. If enough folios have been cleaned since throttling 614 * started then wakeup the throttled tasks. 615 */ 616 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio, 617 int nr_throttled) 618 { 619 unsigned long nr_written; 620 621 node_stat_add_folio(folio, NR_THROTTLED_WRITTEN); 622 623 /* 624 * This is an inaccurate read as the per-cpu deltas may not 625 * be synchronised. However, given that the system is 626 * writeback throttled, it is not worth taking the penalty 627 * of getting an accurate count. At worst, the throttle 628 * timeout guarantees forward progress. 629 */ 630 nr_written = node_page_state(pgdat, NR_THROTTLED_WRITTEN) - 631 READ_ONCE(pgdat->nr_reclaim_start); 632 633 if (nr_written > SWAP_CLUSTER_MAX * nr_throttled) 634 wake_up(&pgdat->reclaim_wait[VMSCAN_THROTTLE_WRITEBACK]); 635 } 636 637 /* possible outcome of pageout() */ 638 typedef enum { 639 /* failed to write folio out, folio is locked */ 640 PAGE_KEEP, 641 /* move folio to the active list, folio is locked */ 642 PAGE_ACTIVATE, 643 /* folio has been sent to the disk successfully, folio is unlocked */ 644 PAGE_SUCCESS, 645 /* folio is clean and locked */ 646 PAGE_CLEAN, 647 } pageout_t; 648 649 static pageout_t writeout(struct folio *folio, struct address_space *mapping, 650 struct swap_iocb **plug, struct list_head *folio_list) 651 { 652 int res; 653 654 folio_set_reclaim(folio); 655 656 /* 657 * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled 658 * or we failed to allocate contiguous swap entries, in which case 659 * the split out folios get added back to folio_list. 660 */ 661 if (shmem_mapping(mapping)) 662 res = shmem_writeout(folio, plug, folio_list); 663 else 664 res = swap_writeout(folio, plug); 665 666 if (res < 0) 667 handle_write_error(mapping, folio, res); 668 if (res == AOP_WRITEPAGE_ACTIVATE) { 669 folio_clear_reclaim(folio); 670 return PAGE_ACTIVATE; 671 } 672 673 /* synchronous write? */ 674 if (!folio_test_writeback(folio)) 675 folio_clear_reclaim(folio); 676 677 trace_mm_vmscan_write_folio(folio); 678 node_stat_add_folio(folio, NR_VMSCAN_WRITE); 679 return PAGE_SUCCESS; 680 } 681 682 /* 683 * pageout is called by shrink_folio_list() for each dirty folio. 684 */ 685 static pageout_t pageout(struct folio *folio, struct address_space *mapping, 686 struct swap_iocb **plug, struct list_head *folio_list) 687 { 688 /* 689 * We no longer attempt to writeback filesystem folios here, other 690 * than tmpfs/shmem. That's taken care of in page-writeback. 691 * If we find a dirty filesystem folio at the end of the LRU list, 692 * typically that means the filesystem is saturating the storage 693 * with contiguous writes and telling it to write a folio here 694 * would only make the situation worse by injecting an element 695 * of random access. 696 * 697 * If the folio is swapcache, write it back even if that would 698 * block, for some throttling. This happens by accident, because 699 * swap_backing_dev_info is bust: it doesn't reflect the 700 * congestion state of the swapdevs. Easy to fix, if needed. 701 */ 702 if (!is_page_cache_freeable(folio)) 703 return PAGE_KEEP; 704 if (!mapping) { 705 /* 706 * Some data journaling orphaned folios can have 707 * folio->mapping == NULL while being dirty with clean buffers. 708 */ 709 if (folio_test_private(folio)) { 710 if (try_to_free_buffers(folio)) { 711 folio_clear_dirty(folio); 712 pr_info("%s: orphaned folio\n", __func__); 713 return PAGE_CLEAN; 714 } 715 } 716 return PAGE_KEEP; 717 } 718 719 if (!shmem_mapping(mapping) && !folio_test_anon(folio)) 720 return PAGE_ACTIVATE; 721 if (!folio_clear_dirty_for_io(folio)) 722 return PAGE_CLEAN; 723 return writeout(folio, mapping, plug, folio_list); 724 } 725 726 /* 727 * Same as remove_mapping, but if the folio is removed from the mapping, it 728 * gets returned with a refcount of 0. 729 */ 730 static int __remove_mapping(struct address_space *mapping, struct folio *folio, 731 bool reclaimed, struct mem_cgroup *target_memcg) 732 { 733 int refcount; 734 void *shadow = NULL; 735 struct swap_cluster_info *ci; 736 737 BUG_ON(!folio_test_locked(folio)); 738 BUG_ON(mapping != folio_mapping(folio)); 739 740 if (folio_test_swapcache(folio)) { 741 ci = swap_cluster_get_and_lock_irq(folio); 742 } else { 743 spin_lock(&mapping->host->i_lock); 744 xa_lock_irq(&mapping->i_pages); 745 } 746 747 /* 748 * The non racy check for a busy folio. 749 * 750 * Must be careful with the order of the tests. When someone has 751 * a ref to the folio, it may be possible that they dirty it then 752 * drop the reference. So if the dirty flag is tested before the 753 * refcount here, then the following race may occur: 754 * 755 * get_user_pages(&page); 756 * [user mapping goes away] 757 * write_to(page); 758 * !folio_test_dirty(folio) [good] 759 * folio_set_dirty(folio); 760 * folio_put(folio); 761 * !refcount(folio) [good, discard it] 762 * 763 * [oops, our write_to data is lost] 764 * 765 * Reversing the order of the tests ensures such a situation cannot 766 * escape unnoticed. The smp_rmb is needed to ensure the folio->flags 767 * load is not satisfied before that of folio->_refcount. 768 * 769 * Note that if the dirty flag is always set via folio_mark_dirty, 770 * and thus under the i_pages lock, then this ordering is not required. 771 */ 772 refcount = 1 + folio_nr_pages(folio); 773 if (!folio_ref_freeze(folio, refcount)) 774 goto cannot_free; 775 /* note: atomic_cmpxchg in folio_ref_freeze provides the smp_rmb */ 776 if (unlikely(folio_test_dirty(folio))) { 777 folio_ref_unfreeze(folio, refcount); 778 goto cannot_free; 779 } 780 781 if (folio_test_swapcache(folio)) { 782 swp_entry_t swap = folio->swap; 783 784 if (reclaimed && !mapping_exiting(mapping)) 785 shadow = workingset_eviction(folio, target_memcg); 786 __swap_cache_del_folio(ci, folio, swap, shadow); 787 memcg1_swapout(folio, swap); 788 swap_cluster_unlock_irq(ci); 789 put_swap_folio(folio, swap); 790 } else { 791 void (*free_folio)(struct folio *); 792 793 free_folio = mapping->a_ops->free_folio; 794 /* 795 * Remember a shadow entry for reclaimed file cache in 796 * order to detect refaults, thus thrashing, later on. 797 * 798 * But don't store shadows in an address space that is 799 * already exiting. This is not just an optimization, 800 * inode reclaim needs to empty out the radix tree or 801 * the nodes are lost. Don't plant shadows behind its 802 * back. 803 * 804 * We also don't store shadows for DAX mappings because the 805 * only page cache folios found in these are zero pages 806 * covering holes, and because we don't want to mix DAX 807 * exceptional entries and shadow exceptional entries in the 808 * same address_space. 809 */ 810 if (reclaimed && folio_is_file_lru(folio) && 811 !mapping_exiting(mapping) && !dax_mapping(mapping)) 812 shadow = workingset_eviction(folio, target_memcg); 813 __filemap_remove_folio(folio, shadow); 814 xa_unlock_irq(&mapping->i_pages); 815 if (mapping_shrinkable(mapping)) 816 inode_add_lru(mapping->host); 817 spin_unlock(&mapping->host->i_lock); 818 819 if (free_folio) 820 free_folio(folio); 821 } 822 823 return 1; 824 825 cannot_free: 826 if (folio_test_swapcache(folio)) { 827 swap_cluster_unlock_irq(ci); 828 } else { 829 xa_unlock_irq(&mapping->i_pages); 830 spin_unlock(&mapping->host->i_lock); 831 } 832 return 0; 833 } 834 835 /** 836 * remove_mapping() - Attempt to remove a folio from its mapping. 837 * @mapping: The address space. 838 * @folio: The folio to remove. 839 * 840 * If the folio is dirty, under writeback or if someone else has a ref 841 * on it, removal will fail. 842 * Return: The number of pages removed from the mapping. 0 if the folio 843 * could not be removed. 844 * Context: The caller should have a single refcount on the folio and 845 * hold its lock. 846 */ 847 long remove_mapping(struct address_space *mapping, struct folio *folio) 848 { 849 if (__remove_mapping(mapping, folio, false, NULL)) { 850 /* 851 * Unfreezing the refcount with 1 effectively 852 * drops the pagecache ref for us without requiring another 853 * atomic operation. 854 */ 855 folio_ref_unfreeze(folio, 1); 856 return folio_nr_pages(folio); 857 } 858 return 0; 859 } 860 861 /** 862 * folio_putback_lru - Put previously isolated folio onto appropriate LRU list. 863 * @folio: Folio to be returned to an LRU list. 864 * 865 * Add previously isolated @folio to appropriate LRU list. 866 * The folio may still be unevictable for other reasons. 867 * 868 * Context: lru_lock must not be held, interrupts must be enabled. 869 */ 870 void folio_putback_lru(struct folio *folio) 871 { 872 folio_add_lru(folio); 873 folio_put(folio); /* drop ref from isolate */ 874 } 875 876 enum folio_references { 877 FOLIOREF_RECLAIM, 878 FOLIOREF_RECLAIM_CLEAN, 879 FOLIOREF_KEEP, 880 FOLIOREF_ACTIVATE, 881 }; 882 883 #ifdef CONFIG_LRU_GEN 884 /* 885 * Only used on a mapped folio in the eviction (rmap walk) path, where promotion 886 * needs to be done by taking the folio off the LRU list and then adding it back 887 * with PG_active set. In contrast, the aging (page table walk) path uses 888 * folio_update_gen(). 889 */ 890 static bool lru_gen_set_refs(struct folio *folio) 891 { 892 /* see the comment on LRU_REFS_FLAGS */ 893 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { 894 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); 895 return false; 896 } 897 898 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_workingset)); 899 return true; 900 } 901 #else 902 static bool lru_gen_set_refs(struct folio *folio) 903 { 904 return false; 905 } 906 #endif /* CONFIG_LRU_GEN */ 907 908 static enum folio_references folio_check_references(struct folio *folio, 909 struct scan_control *sc) 910 { 911 int referenced_ptes, referenced_folio; 912 vm_flags_t vm_flags; 913 914 referenced_ptes = folio_referenced(folio, 1, sc->target_mem_cgroup, 915 &vm_flags); 916 917 /* 918 * The supposedly reclaimable folio was found to be in a VM_LOCKED vma. 919 * Let the folio, now marked Mlocked, be moved to the unevictable list. 920 */ 921 if (vm_flags & VM_LOCKED) 922 return FOLIOREF_ACTIVATE; 923 924 /* 925 * There are two cases to consider. 926 * 1) Rmap lock contention: rotate. 927 * 2) Skip the non-shared swapbacked folio mapped solely by 928 * the exiting or OOM-reaped process. 929 */ 930 if (referenced_ptes == -1) 931 return FOLIOREF_KEEP; 932 933 if (lru_gen_enabled()) { 934 if (!referenced_ptes) 935 return FOLIOREF_RECLAIM; 936 937 return lru_gen_set_refs(folio) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP; 938 } 939 940 referenced_folio = folio_test_clear_referenced(folio); 941 942 if (referenced_ptes) { 943 /* 944 * All mapped folios start out with page table 945 * references from the instantiating fault, so we need 946 * to look twice if a mapped file/anon folio is used more 947 * than once. 948 * 949 * Mark it and spare it for another trip around the 950 * inactive list. Another page table reference will 951 * lead to its activation. 952 * 953 * Note: the mark is set for activated folios as well 954 * so that recently deactivated but used folios are 955 * quickly recovered. 956 */ 957 folio_set_referenced(folio); 958 959 if (referenced_folio || referenced_ptes > 1) 960 return FOLIOREF_ACTIVATE; 961 962 /* 963 * Activate file-backed executable folios after first usage. 964 */ 965 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) 966 return FOLIOREF_ACTIVATE; 967 968 return FOLIOREF_KEEP; 969 } 970 971 /* Reclaim if clean, defer dirty folios to writeback */ 972 if (referenced_folio && folio_is_file_lru(folio)) 973 return FOLIOREF_RECLAIM_CLEAN; 974 975 return FOLIOREF_RECLAIM; 976 } 977 978 /* Check if a folio is dirty or under writeback */ 979 static void folio_check_dirty_writeback(struct folio *folio, 980 bool *dirty, bool *writeback) 981 { 982 struct address_space *mapping; 983 984 /* 985 * Anonymous folios are not handled by flushers and must be written 986 * from reclaim context. Do not stall reclaim based on them. 987 * MADV_FREE anonymous folios are put into inactive file list too. 988 * They could be mistakenly treated as file lru. So further anon 989 * test is needed. 990 */ 991 if (!folio_is_file_lru(folio) || 992 (folio_test_anon(folio) && !folio_test_swapbacked(folio))) { 993 *dirty = false; 994 *writeback = false; 995 return; 996 } 997 998 /* By default assume that the folio flags are accurate */ 999 *dirty = folio_test_dirty(folio); 1000 *writeback = folio_test_writeback(folio); 1001 1002 /* Verify dirty/writeback state if the filesystem supports it */ 1003 if (!folio_test_private(folio)) 1004 return; 1005 1006 mapping = folio_mapping(folio); 1007 if (mapping && mapping->a_ops->is_dirty_writeback) 1008 mapping->a_ops->is_dirty_writeback(folio, dirty, writeback); 1009 } 1010 1011 static struct folio *alloc_demote_folio(struct folio *src, 1012 unsigned long private) 1013 { 1014 struct folio *dst; 1015 nodemask_t *allowed_mask; 1016 struct migration_target_control *mtc; 1017 1018 mtc = (struct migration_target_control *)private; 1019 1020 allowed_mask = mtc->nmask; 1021 /* 1022 * make sure we allocate from the target node first also trying to 1023 * demote or reclaim pages from the target node via kswapd if we are 1024 * low on free memory on target node. If we don't do this and if 1025 * we have free memory on the slower(lower) memtier, we would start 1026 * allocating pages from slower(lower) memory tiers without even forcing 1027 * a demotion of cold pages from the target memtier. This can result 1028 * in the kernel placing hot pages in slower(lower) memory tiers. 1029 */ 1030 mtc->nmask = NULL; 1031 mtc->gfp_mask |= __GFP_THISNODE; 1032 dst = alloc_migration_target(src, (unsigned long)mtc); 1033 if (dst) 1034 return dst; 1035 1036 mtc->gfp_mask &= ~__GFP_THISNODE; 1037 mtc->nmask = allowed_mask; 1038 1039 return alloc_migration_target(src, (unsigned long)mtc); 1040 } 1041 1042 /* 1043 * Take folios on @demote_folios and attempt to demote them to another node. 1044 * Folios which are not demoted are left on @demote_folios. 1045 */ 1046 static unsigned int demote_folio_list(struct list_head *demote_folios, 1047 struct pglist_data *pgdat, 1048 struct mem_cgroup *memcg) 1049 { 1050 int target_nid; 1051 unsigned int nr_succeeded; 1052 nodemask_t allowed_mask; 1053 1054 struct migration_target_control mtc = { 1055 /* 1056 * Allocate from 'node', or fail quickly and quietly. 1057 * When this happens, 'page' will likely just be discarded 1058 * instead of migrated. 1059 */ 1060 .gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) | __GFP_NOWARN | 1061 __GFP_NOMEMALLOC | GFP_NOWAIT, 1062 .nmask = &allowed_mask, 1063 .reason = MR_DEMOTION, 1064 }; 1065 1066 if (list_empty(demote_folios)) 1067 return 0; 1068 1069 node_get_allowed_targets(pgdat, &allowed_mask); 1070 mem_cgroup_node_filter_allowed(memcg, &allowed_mask); 1071 if (nodes_empty(allowed_mask)) 1072 return 0; 1073 1074 target_nid = next_demotion_node(pgdat->node_id); 1075 if (target_nid == NUMA_NO_NODE) 1076 /* No lower-tier nodes or nodes were hot-unplugged. */ 1077 return 0; 1078 if (!node_isset(target_nid, allowed_mask)) 1079 target_nid = node_random(&allowed_mask); 1080 mtc.nid = target_nid; 1081 1082 /* Demotion ignores all cpuset and mempolicy settings */ 1083 migrate_pages(demote_folios, alloc_demote_folio, NULL, 1084 (unsigned long)&mtc, MIGRATE_ASYNC, MR_DEMOTION, 1085 &nr_succeeded); 1086 1087 return nr_succeeded; 1088 } 1089 1090 static bool may_enter_fs(struct folio *folio, gfp_t gfp_mask) 1091 { 1092 if (gfp_mask & __GFP_FS) 1093 return true; 1094 if (!folio_test_swapcache(folio) || !(gfp_mask & __GFP_IO)) 1095 return false; 1096 /* 1097 * We can "enter_fs" for swap-cache with only __GFP_IO 1098 * providing this isn't SWP_FS_OPS. 1099 * ->flags can be updated non-atomicially (scan_swap_map_slots), 1100 * but that will never affect SWP_FS_OPS, so the data_race 1101 * is safe. 1102 */ 1103 return !data_race(folio_swap_flags(folio) & SWP_FS_OPS); 1104 } 1105 1106 /* 1107 * shrink_folio_list() returns the number of reclaimed pages 1108 */ 1109 static unsigned int shrink_folio_list(struct list_head *folio_list, 1110 struct pglist_data *pgdat, struct scan_control *sc, 1111 struct reclaim_stat *stat, bool ignore_references, 1112 struct mem_cgroup *memcg) 1113 { 1114 struct folio_batch free_folios; 1115 LIST_HEAD(ret_folios); 1116 LIST_HEAD(demote_folios); 1117 unsigned int nr_reclaimed = 0, nr_demoted = 0; 1118 unsigned int pgactivate = 0; 1119 bool do_demote_pass; 1120 struct swap_iocb *plug = NULL; 1121 1122 folio_batch_init(&free_folios); 1123 memset(stat, 0, sizeof(*stat)); 1124 cond_resched(); 1125 do_demote_pass = can_demote(pgdat->node_id, sc, memcg); 1126 1127 retry: 1128 while (!list_empty(folio_list)) { 1129 struct address_space *mapping; 1130 struct folio *folio; 1131 enum folio_references references = FOLIOREF_RECLAIM; 1132 bool dirty, writeback; 1133 unsigned int nr_pages; 1134 1135 cond_resched(); 1136 1137 folio = lru_to_folio(folio_list); 1138 list_del(&folio->lru); 1139 1140 if (!folio_trylock(folio)) 1141 goto keep; 1142 1143 if (folio_contain_hwpoisoned_page(folio)) { 1144 /* 1145 * unmap_poisoned_folio() can't handle large 1146 * folio, just skip it. memory_failure() will 1147 * handle it if the UCE is triggered again. 1148 */ 1149 if (folio_test_large(folio)) 1150 goto keep_locked; 1151 1152 unmap_poisoned_folio(folio, folio_pfn(folio), false); 1153 folio_unlock(folio); 1154 folio_put(folio); 1155 continue; 1156 } 1157 1158 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1159 1160 nr_pages = folio_nr_pages(folio); 1161 1162 /* Account the number of base pages */ 1163 sc->nr_scanned += nr_pages; 1164 1165 if (unlikely(!folio_evictable(folio))) 1166 goto activate_locked; 1167 1168 if (!sc->may_unmap && folio_mapped(folio)) 1169 goto keep_locked; 1170 1171 /* 1172 * The number of dirty pages determines if a node is marked 1173 * reclaim_congested. kswapd will stall and start writing 1174 * folios if the tail of the LRU is all dirty unqueued folios. 1175 */ 1176 folio_check_dirty_writeback(folio, &dirty, &writeback); 1177 if (dirty || writeback) 1178 stat->nr_dirty += nr_pages; 1179 1180 if (dirty && !writeback) 1181 stat->nr_unqueued_dirty += nr_pages; 1182 1183 /* 1184 * Treat this folio as congested if folios are cycling 1185 * through the LRU so quickly that the folios marked 1186 * for immediate reclaim are making it to the end of 1187 * the LRU a second time. 1188 */ 1189 if (writeback && folio_test_reclaim(folio)) 1190 stat->nr_congested += nr_pages; 1191 1192 /* 1193 * If a folio at the tail of the LRU is under writeback, there 1194 * are three cases to consider. 1195 * 1196 * 1) If reclaim is encountering an excessive number 1197 * of folios under writeback and this folio has both 1198 * the writeback and reclaim flags set, then it 1199 * indicates that folios are being queued for I/O but 1200 * are being recycled through the LRU before the I/O 1201 * can complete. Waiting on the folio itself risks an 1202 * indefinite stall if it is impossible to writeback 1203 * the folio due to I/O error or disconnected storage 1204 * so instead note that the LRU is being scanned too 1205 * quickly and the caller can stall after the folio 1206 * list has been processed. 1207 * 1208 * 2) Global or new memcg reclaim encounters a folio that is 1209 * not marked for immediate reclaim, or the caller does not 1210 * have __GFP_FS (or __GFP_IO if it's simply going to swap, 1211 * not to fs), or the folio belongs to a mapping where 1212 * waiting on writeback during reclaim may lead to a deadlock. 1213 * In this case mark the folio for immediate reclaim and 1214 * continue scanning. 1215 * 1216 * Require may_enter_fs() because we would wait on fs, which 1217 * may not have submitted I/O yet. And the loop driver might 1218 * enter reclaim, and deadlock if it waits on a folio for 1219 * which it is needed to do the write (loop masks off 1220 * __GFP_IO|__GFP_FS for this reason); but more thought 1221 * would probably show more reasons. 1222 * 1223 * 3) Legacy memcg encounters a folio that already has the 1224 * reclaim flag set. memcg does not have any dirty folio 1225 * throttling so we could easily OOM just because too many 1226 * folios are in writeback and there is nothing else to 1227 * reclaim. Wait for the writeback to complete. 1228 * 1229 * In cases 1) and 2) we activate the folios to get them out of 1230 * the way while we continue scanning for clean folios on the 1231 * inactive list and refilling from the active list. The 1232 * observation here is that waiting for disk writes is more 1233 * expensive than potentially causing reloads down the line. 1234 * Since they're marked for immediate reclaim, they won't put 1235 * memory pressure on the cache working set any longer than it 1236 * takes to write them to disk. 1237 */ 1238 if (folio_test_writeback(folio)) { 1239 mapping = folio_mapping(folio); 1240 1241 /* Case 1 above */ 1242 if (current_is_kswapd() && 1243 folio_test_reclaim(folio) && 1244 test_bit(PGDAT_WRITEBACK, &pgdat->flags)) { 1245 stat->nr_immediate += nr_pages; 1246 goto activate_locked; 1247 1248 /* Case 2 above */ 1249 } else if (writeback_throttling_sane(sc) || 1250 !folio_test_reclaim(folio) || 1251 !may_enter_fs(folio, sc->gfp_mask) || 1252 (mapping && 1253 mapping_writeback_may_deadlock_on_reclaim(mapping))) { 1254 /* 1255 * This is slightly racy - 1256 * folio_end_writeback() might have 1257 * just cleared the reclaim flag, then 1258 * setting the reclaim flag here ends up 1259 * interpreted as the readahead flag - but 1260 * that does not matter enough to care. 1261 * What we do want is for this folio to 1262 * have the reclaim flag set next time 1263 * memcg reclaim reaches the tests above, 1264 * so it will then wait for writeback to 1265 * avoid OOM; and it's also appropriate 1266 * in global reclaim. 1267 */ 1268 folio_set_reclaim(folio); 1269 stat->nr_writeback += nr_pages; 1270 goto activate_locked; 1271 1272 /* Case 3 above */ 1273 } else { 1274 folio_unlock(folio); 1275 folio_wait_writeback(folio); 1276 /* then go back and try same folio again */ 1277 list_add_tail(&folio->lru, folio_list); 1278 continue; 1279 } 1280 } 1281 1282 if (!ignore_references) 1283 references = folio_check_references(folio, sc); 1284 1285 switch (references) { 1286 case FOLIOREF_ACTIVATE: 1287 goto activate_locked; 1288 case FOLIOREF_KEEP: 1289 stat->nr_ref_keep += nr_pages; 1290 goto keep_locked; 1291 case FOLIOREF_RECLAIM: 1292 case FOLIOREF_RECLAIM_CLEAN: 1293 ; /* try to reclaim the folio below */ 1294 } 1295 1296 /* 1297 * Before reclaiming the folio, try to relocate 1298 * its contents to another node. 1299 */ 1300 if (do_demote_pass && 1301 (thp_migration_supported() || !folio_test_large(folio))) { 1302 list_add(&folio->lru, &demote_folios); 1303 folio_unlock(folio); 1304 continue; 1305 } 1306 1307 /* 1308 * Anonymous process memory has backing store? 1309 * Try to allocate it some swap space here. 1310 * Lazyfree folio could be freed directly 1311 */ 1312 if (folio_test_anon(folio) && folio_test_swapbacked(folio)) { 1313 if (!folio_test_swapcache(folio)) { 1314 if (!(sc->gfp_mask & __GFP_IO)) 1315 goto keep_locked; 1316 if (folio_maybe_dma_pinned(folio)) 1317 goto keep_locked; 1318 if (folio_test_large(folio)) { 1319 /* cannot split folio, skip it */ 1320 if (!can_split_folio(folio, 1, NULL)) 1321 goto activate_locked; 1322 /* 1323 * Split partially mapped folios right away. 1324 * We can free the unmapped pages without IO. 1325 */ 1326 if (data_race(!list_empty(&folio->_deferred_list) && 1327 folio_test_partially_mapped(folio)) && 1328 split_folio_to_list(folio, folio_list)) 1329 goto activate_locked; 1330 } 1331 if (folio_alloc_swap(folio, __GFP_HIGH | __GFP_NOWARN)) { 1332 int __maybe_unused order = folio_order(folio); 1333 1334 if (!folio_test_large(folio)) 1335 goto activate_locked_split; 1336 /* Fallback to swap normal pages */ 1337 if (split_folio_to_list(folio, folio_list)) 1338 goto activate_locked; 1339 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1340 if (nr_pages >= HPAGE_PMD_NR) { 1341 count_memcg_folio_events(folio, 1342 THP_SWPOUT_FALLBACK, 1); 1343 count_vm_event(THP_SWPOUT_FALLBACK); 1344 } 1345 #endif 1346 count_mthp_stat(order, MTHP_STAT_SWPOUT_FALLBACK); 1347 if (folio_alloc_swap(folio, __GFP_HIGH | __GFP_NOWARN)) 1348 goto activate_locked_split; 1349 } 1350 /* 1351 * Normally the folio will be dirtied in unmap because its 1352 * pte should be dirty. A special case is MADV_FREE page. The 1353 * page's pte could have dirty bit cleared but the folio's 1354 * SwapBacked flag is still set because clearing the dirty bit 1355 * and SwapBacked flag has no lock protected. For such folio, 1356 * unmap will not set dirty bit for it, so folio reclaim will 1357 * not write the folio out. This can cause data corruption when 1358 * the folio is swapped in later. Always setting the dirty flag 1359 * for the folio solves the problem. 1360 */ 1361 folio_mark_dirty(folio); 1362 } 1363 } 1364 1365 /* 1366 * If the folio was split above, the tail pages will make 1367 * their own pass through this function and be accounted 1368 * then. 1369 */ 1370 if ((nr_pages > 1) && !folio_test_large(folio)) { 1371 sc->nr_scanned -= (nr_pages - 1); 1372 nr_pages = 1; 1373 } 1374 1375 /* 1376 * The folio is mapped into the page tables of one or more 1377 * processes. Try to unmap it here. 1378 */ 1379 if (folio_mapped(folio)) { 1380 enum ttu_flags flags = TTU_BATCH_FLUSH; 1381 bool was_swapbacked = folio_test_swapbacked(folio); 1382 1383 if (folio_test_pmd_mappable(folio)) 1384 flags |= TTU_SPLIT_HUGE_PMD; 1385 /* 1386 * Without TTU_SYNC, try_to_unmap will only begin to 1387 * hold PTL from the first present PTE within a large 1388 * folio. Some initial PTEs might be skipped due to 1389 * races with parallel PTE writes in which PTEs can be 1390 * cleared temporarily before being written new present 1391 * values. This will lead to a large folio is still 1392 * mapped while some subpages have been partially 1393 * unmapped after try_to_unmap; TTU_SYNC helps 1394 * try_to_unmap acquire PTL from the first PTE, 1395 * eliminating the influence of temporary PTE values. 1396 */ 1397 if (folio_test_large(folio)) 1398 flags |= TTU_SYNC; 1399 1400 try_to_unmap(folio, flags); 1401 if (folio_mapped(folio)) { 1402 stat->nr_unmap_fail += nr_pages; 1403 if (!was_swapbacked && 1404 folio_test_swapbacked(folio)) 1405 stat->nr_lazyfree_fail += nr_pages; 1406 goto activate_locked; 1407 } 1408 } 1409 1410 /* 1411 * Folio is unmapped now so it cannot be newly pinned anymore. 1412 * No point in trying to reclaim folio if it is pinned. 1413 * Furthermore we don't want to reclaim underlying fs metadata 1414 * if the folio is pinned and thus potentially modified by the 1415 * pinning process as that may upset the filesystem. 1416 */ 1417 if (folio_maybe_dma_pinned(folio)) 1418 goto activate_locked; 1419 1420 mapping = folio_mapping(folio); 1421 if (folio_test_dirty(folio)) { 1422 /* 1423 * Only kswapd can writeback filesystem folios 1424 * to avoid risk of stack overflow. But avoid 1425 * injecting inefficient single-folio I/O into 1426 * flusher writeback as much as possible: only 1427 * write folios when we've encountered many 1428 * dirty folios, and when we've already scanned 1429 * the rest of the LRU for clean folios and see 1430 * the same dirty folios again (with the reclaim 1431 * flag set). 1432 */ 1433 if (folio_is_file_lru(folio) && 1434 (!current_is_kswapd() || 1435 !folio_test_reclaim(folio) || 1436 !test_bit(PGDAT_DIRTY, &pgdat->flags))) { 1437 /* 1438 * Immediately reclaim when written back. 1439 * Similar in principle to folio_deactivate() 1440 * except we already have the folio isolated 1441 * and know it's dirty 1442 */ 1443 node_stat_mod_folio(folio, NR_VMSCAN_IMMEDIATE, 1444 nr_pages); 1445 folio_set_reclaim(folio); 1446 1447 goto activate_locked; 1448 } 1449 1450 if (references == FOLIOREF_RECLAIM_CLEAN) 1451 goto keep_locked; 1452 if (!may_enter_fs(folio, sc->gfp_mask)) 1453 goto keep_locked; 1454 if (!sc->may_writepage) 1455 goto keep_locked; 1456 1457 /* 1458 * Folio is dirty. Flush the TLB if a writable entry 1459 * potentially exists to avoid CPU writes after I/O 1460 * starts and then write it out here. 1461 */ 1462 try_to_unmap_flush_dirty(); 1463 switch (pageout(folio, mapping, &plug, folio_list)) { 1464 case PAGE_KEEP: 1465 goto keep_locked; 1466 case PAGE_ACTIVATE: 1467 /* 1468 * If shmem folio is split when writeback to swap, 1469 * the tail pages will make their own pass through 1470 * this function and be accounted then. 1471 */ 1472 if (nr_pages > 1 && !folio_test_large(folio)) { 1473 sc->nr_scanned -= (nr_pages - 1); 1474 nr_pages = 1; 1475 } 1476 goto activate_locked; 1477 case PAGE_SUCCESS: 1478 if (nr_pages > 1 && !folio_test_large(folio)) { 1479 sc->nr_scanned -= (nr_pages - 1); 1480 nr_pages = 1; 1481 } 1482 stat->nr_pageout += nr_pages; 1483 1484 if (folio_test_writeback(folio)) 1485 goto keep; 1486 if (folio_test_dirty(folio)) 1487 goto keep; 1488 1489 /* 1490 * A synchronous write - probably a ramdisk. Go 1491 * ahead and try to reclaim the folio. 1492 */ 1493 if (!folio_trylock(folio)) 1494 goto keep; 1495 if (folio_test_dirty(folio) || 1496 folio_test_writeback(folio)) 1497 goto keep_locked; 1498 mapping = folio_mapping(folio); 1499 fallthrough; 1500 case PAGE_CLEAN: 1501 ; /* try to free the folio below */ 1502 } 1503 } 1504 1505 /* 1506 * If the folio has buffers, try to free the buffer 1507 * mappings associated with this folio. If we succeed 1508 * we try to free the folio as well. 1509 * 1510 * We do this even if the folio is dirty. 1511 * filemap_release_folio() does not perform I/O, but it 1512 * is possible for a folio to have the dirty flag set, 1513 * but it is actually clean (all its buffers are clean). 1514 * This happens if the buffers were written out directly, 1515 * with submit_bh(). ext3 will do this, as well as 1516 * the blockdev mapping. filemap_release_folio() will 1517 * discover that cleanness and will drop the buffers 1518 * and mark the folio clean - it can be freed. 1519 * 1520 * Rarely, folios can have buffers and no ->mapping. 1521 * These are the folios which were not successfully 1522 * invalidated in truncate_cleanup_folio(). We try to 1523 * drop those buffers here and if that worked, and the 1524 * folio is no longer mapped into process address space 1525 * (refcount == 1) it can be freed. Otherwise, leave 1526 * the folio on the LRU so it is swappable. 1527 */ 1528 if (folio_needs_release(folio)) { 1529 if (!filemap_release_folio(folio, sc->gfp_mask)) 1530 goto activate_locked; 1531 if (!mapping && folio_ref_count(folio) == 1) { 1532 folio_unlock(folio); 1533 if (folio_put_testzero(folio)) 1534 goto free_it; 1535 else { 1536 /* 1537 * rare race with speculative reference. 1538 * the speculative reference will free 1539 * this folio shortly, so we may 1540 * increment nr_reclaimed here (and 1541 * leave it off the LRU). 1542 */ 1543 nr_reclaimed += nr_pages; 1544 continue; 1545 } 1546 } 1547 } 1548 1549 if (folio_test_anon(folio) && !folio_test_swapbacked(folio)) { 1550 /* follow __remove_mapping for reference */ 1551 if (!folio_ref_freeze(folio, 1)) 1552 goto keep_locked; 1553 /* 1554 * The folio has only one reference left, which is 1555 * from the isolation. After the caller puts the 1556 * folio back on the lru and drops the reference, the 1557 * folio will be freed anyway. It doesn't matter 1558 * which lru it goes on. So we don't bother checking 1559 * the dirty flag here. 1560 */ 1561 count_vm_events(PGLAZYFREED, nr_pages); 1562 count_memcg_folio_events(folio, PGLAZYFREED, nr_pages); 1563 } else if (!mapping || !__remove_mapping(mapping, folio, true, 1564 sc->target_mem_cgroup)) 1565 goto keep_locked; 1566 1567 folio_unlock(folio); 1568 free_it: 1569 /* 1570 * Folio may get swapped out as a whole, need to account 1571 * all pages in it. 1572 */ 1573 nr_reclaimed += nr_pages; 1574 1575 folio_unqueue_deferred_split(folio); 1576 if (folio_batch_add(&free_folios, folio) == 0) { 1577 mem_cgroup_uncharge_folios(&free_folios); 1578 try_to_unmap_flush(); 1579 free_unref_folios(&free_folios); 1580 } 1581 continue; 1582 1583 activate_locked_split: 1584 /* 1585 * The tail pages that are failed to add into swap cache 1586 * reach here. Fixup nr_scanned and nr_pages. 1587 */ 1588 if (nr_pages > 1) { 1589 sc->nr_scanned -= (nr_pages - 1); 1590 nr_pages = 1; 1591 } 1592 activate_locked: 1593 /* Not a candidate for swapping, so reclaim swap space. */ 1594 if (folio_test_swapcache(folio) && 1595 (mem_cgroup_swap_full(folio) || folio_test_mlocked(folio))) 1596 folio_free_swap(folio); 1597 VM_BUG_ON_FOLIO(folio_test_active(folio), folio); 1598 if (!folio_test_mlocked(folio)) { 1599 int type = folio_is_file_lru(folio); 1600 folio_set_active(folio); 1601 stat->nr_activate[type] += nr_pages; 1602 count_memcg_folio_events(folio, PGACTIVATE, nr_pages); 1603 } 1604 keep_locked: 1605 folio_unlock(folio); 1606 keep: 1607 list_add(&folio->lru, &ret_folios); 1608 VM_BUG_ON_FOLIO(folio_test_lru(folio) || 1609 folio_test_unevictable(folio), folio); 1610 } 1611 /* 'folio_list' is always empty here */ 1612 1613 /* Migrate folios selected for demotion */ 1614 nr_demoted = demote_folio_list(&demote_folios, pgdat, memcg); 1615 nr_reclaimed += nr_demoted; 1616 stat->nr_demoted += nr_demoted; 1617 /* Folios that could not be demoted are still in @demote_folios */ 1618 if (!list_empty(&demote_folios)) { 1619 /* Folios which weren't demoted go back on @folio_list */ 1620 list_splice_init(&demote_folios, folio_list); 1621 1622 /* 1623 * goto retry to reclaim the undemoted folios in folio_list if 1624 * desired. 1625 * 1626 * Reclaiming directly from top tier nodes is not often desired 1627 * due to it breaking the LRU ordering: in general memory 1628 * should be reclaimed from lower tier nodes and demoted from 1629 * top tier nodes. 1630 * 1631 * However, disabling reclaim from top tier nodes entirely 1632 * would cause ooms in edge scenarios where lower tier memory 1633 * is unreclaimable for whatever reason, eg memory being 1634 * mlocked or too hot to reclaim. We can disable reclaim 1635 * from top tier nodes in proactive reclaim though as that is 1636 * not real memory pressure. 1637 */ 1638 if (!sc->proactive) { 1639 do_demote_pass = false; 1640 goto retry; 1641 } 1642 } 1643 1644 pgactivate = stat->nr_activate[0] + stat->nr_activate[1]; 1645 1646 mem_cgroup_uncharge_folios(&free_folios); 1647 try_to_unmap_flush(); 1648 free_unref_folios(&free_folios); 1649 1650 list_splice(&ret_folios, folio_list); 1651 count_vm_events(PGACTIVATE, pgactivate); 1652 1653 if (plug) 1654 swap_write_unplug(plug); 1655 return nr_reclaimed; 1656 } 1657 1658 unsigned int reclaim_clean_pages_from_list(struct zone *zone, 1659 struct list_head *folio_list) 1660 { 1661 struct scan_control sc = { 1662 .gfp_mask = GFP_KERNEL, 1663 .may_unmap = 1, 1664 }; 1665 struct reclaim_stat stat; 1666 unsigned int nr_reclaimed; 1667 struct folio *folio, *next; 1668 LIST_HEAD(clean_folios); 1669 unsigned int noreclaim_flag; 1670 1671 list_for_each_entry_safe(folio, next, folio_list, lru) { 1672 /* TODO: these pages should not even appear in this list. */ 1673 if (page_has_movable_ops(&folio->page)) 1674 continue; 1675 if (!folio_test_hugetlb(folio) && folio_is_file_lru(folio) && 1676 !folio_test_dirty(folio) && !folio_test_unevictable(folio)) { 1677 folio_clear_active(folio); 1678 list_move(&folio->lru, &clean_folios); 1679 } 1680 } 1681 1682 /* 1683 * We should be safe here since we are only dealing with file pages and 1684 * we are not kswapd and therefore cannot write dirty file pages. But 1685 * call memalloc_noreclaim_save() anyway, just in case these conditions 1686 * change in the future. 1687 */ 1688 noreclaim_flag = memalloc_noreclaim_save(); 1689 nr_reclaimed = shrink_folio_list(&clean_folios, zone->zone_pgdat, &sc, 1690 &stat, true, NULL); 1691 memalloc_noreclaim_restore(noreclaim_flag); 1692 1693 list_splice(&clean_folios, folio_list); 1694 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 1695 -(long)nr_reclaimed); 1696 /* 1697 * Since lazyfree pages are isolated from file LRU from the beginning, 1698 * they will rotate back to anonymous LRU in the end if it failed to 1699 * discard so isolated count will be mismatched. 1700 * Compensate the isolated count for both LRU lists. 1701 */ 1702 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, 1703 stat.nr_lazyfree_fail); 1704 mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, 1705 -(long)stat.nr_lazyfree_fail); 1706 return nr_reclaimed; 1707 } 1708 1709 /* 1710 * Update LRU sizes after isolating pages. The LRU size updates must 1711 * be complete before mem_cgroup_update_lru_size due to a sanity check. 1712 */ 1713 static __always_inline void update_lru_sizes(struct lruvec *lruvec, 1714 enum lru_list lru, unsigned long *nr_zone_taken) 1715 { 1716 int zid; 1717 1718 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1719 if (!nr_zone_taken[zid]) 1720 continue; 1721 1722 update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]); 1723 } 1724 1725 } 1726 1727 /* 1728 * Isolating page from the lruvec to fill in @dst list by nr_to_scan times. 1729 * 1730 * lruvec->lru_lock is heavily contended. Some of the functions that 1731 * shrink the lists perform better by taking out a batch of pages 1732 * and working on them outside the LRU lock. 1733 * 1734 * For pagecache intensive workloads, this function is the hottest 1735 * spot in the kernel (apart from copy_*_user functions). 1736 * 1737 * Lru_lock must be held before calling this function. 1738 * 1739 * @nr_to_scan: The number of eligible pages to look through on the list. 1740 * @lruvec: The LRU vector to pull pages from. 1741 * @dst: The temp list to put pages on to. 1742 * @nr_scanned: The number of pages that were scanned. 1743 * @sc: The scan_control struct for this reclaim session 1744 * @lru: LRU list id for isolating 1745 * 1746 * returns how many pages were moved onto *@dst. 1747 */ 1748 static unsigned long isolate_lru_folios(unsigned long nr_to_scan, 1749 struct lruvec *lruvec, struct list_head *dst, 1750 unsigned long *nr_scanned, struct scan_control *sc, 1751 enum lru_list lru) 1752 { 1753 struct list_head *src = &lruvec->lists[lru]; 1754 unsigned long nr_taken = 0; 1755 unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 }; 1756 unsigned long nr_skipped[MAX_NR_ZONES] = { 0, }; 1757 unsigned long skipped = 0, total_scan = 0, scan = 0; 1758 unsigned long nr_pages; 1759 unsigned long max_nr_skipped = 0; 1760 LIST_HEAD(folios_skipped); 1761 1762 while (scan < nr_to_scan && !list_empty(src)) { 1763 struct list_head *move_to = src; 1764 struct folio *folio; 1765 1766 folio = lru_to_folio(src); 1767 prefetchw_prev_lru_folio(folio, src, flags); 1768 1769 nr_pages = folio_nr_pages(folio); 1770 total_scan += nr_pages; 1771 1772 /* Using max_nr_skipped to prevent hard LOCKUP*/ 1773 if (max_nr_skipped < SWAP_CLUSTER_MAX_SKIPPED && 1774 (folio_zonenum(folio) > sc->reclaim_idx)) { 1775 nr_skipped[folio_zonenum(folio)] += nr_pages; 1776 move_to = &folios_skipped; 1777 max_nr_skipped++; 1778 goto move; 1779 } 1780 1781 /* 1782 * Do not count skipped folios because that makes the function 1783 * return with no isolated folios if the LRU mostly contains 1784 * ineligible folios. This causes the VM to not reclaim any 1785 * folios, triggering a premature OOM. 1786 * Account all pages in a folio. 1787 */ 1788 scan += nr_pages; 1789 1790 if (!folio_test_lru(folio)) 1791 goto move; 1792 if (!sc->may_unmap && folio_mapped(folio)) 1793 goto move; 1794 1795 /* 1796 * Be careful not to clear the lru flag until after we're 1797 * sure the folio is not being freed elsewhere -- the 1798 * folio release code relies on it. 1799 */ 1800 if (unlikely(!folio_try_get(folio))) 1801 goto move; 1802 1803 if (!folio_test_clear_lru(folio)) { 1804 /* Another thread is already isolating this folio */ 1805 folio_put(folio); 1806 goto move; 1807 } 1808 1809 nr_taken += nr_pages; 1810 nr_zone_taken[folio_zonenum(folio)] += nr_pages; 1811 move_to = dst; 1812 move: 1813 list_move(&folio->lru, move_to); 1814 } 1815 1816 /* 1817 * Splice any skipped folios to the start of the LRU list. Note that 1818 * this disrupts the LRU order when reclaiming for lower zones but 1819 * we cannot splice to the tail. If we did then the SWAP_CLUSTER_MAX 1820 * scanning would soon rescan the same folios to skip and waste lots 1821 * of cpu cycles. 1822 */ 1823 if (!list_empty(&folios_skipped)) { 1824 int zid; 1825 1826 list_splice(&folios_skipped, src); 1827 for (zid = 0; zid < MAX_NR_ZONES; zid++) { 1828 if (!nr_skipped[zid]) 1829 continue; 1830 1831 __count_zid_vm_events(PGSCAN_SKIP, zid, nr_skipped[zid]); 1832 skipped += nr_skipped[zid]; 1833 } 1834 } 1835 *nr_scanned = total_scan; 1836 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan, 1837 total_scan, skipped, nr_taken, lru); 1838 update_lru_sizes(lruvec, lru, nr_zone_taken); 1839 return nr_taken; 1840 } 1841 1842 /** 1843 * folio_isolate_lru() - Try to isolate a folio from its LRU list. 1844 * @folio: Folio to isolate from its LRU list. 1845 * 1846 * Isolate a @folio from an LRU list and adjust the vmstat statistic 1847 * corresponding to whatever LRU list the folio was on. 1848 * 1849 * The folio will have its LRU flag cleared. If it was found on the 1850 * active list, it will have the Active flag set. If it was found on the 1851 * unevictable list, it will have the Unevictable flag set. These flags 1852 * may need to be cleared by the caller before letting the page go. 1853 * 1854 * Context: 1855 * 1856 * (1) Must be called with an elevated refcount on the folio. This is a 1857 * fundamental difference from isolate_lru_folios() (which is called 1858 * without a stable reference). 1859 * (2) The lru_lock must not be held. 1860 * (3) Interrupts must be enabled. 1861 * 1862 * Return: true if the folio was removed from an LRU list. 1863 * false if the folio was not on an LRU list. 1864 */ 1865 bool folio_isolate_lru(struct folio *folio) 1866 { 1867 bool ret = false; 1868 1869 VM_BUG_ON_FOLIO(!folio_ref_count(folio), folio); 1870 1871 if (folio_test_clear_lru(folio)) { 1872 struct lruvec *lruvec; 1873 1874 folio_get(folio); 1875 lruvec = folio_lruvec_lock_irq(folio); 1876 lruvec_del_folio(lruvec, folio); 1877 unlock_page_lruvec_irq(lruvec); 1878 ret = true; 1879 } 1880 1881 return ret; 1882 } 1883 1884 /* 1885 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and 1886 * then get rescheduled. When there are massive number of tasks doing page 1887 * allocation, such sleeping direct reclaimers may keep piling up on each CPU, 1888 * the LRU list will go small and be scanned faster than necessary, leading to 1889 * unnecessary swapping, thrashing and OOM. 1890 */ 1891 static bool too_many_isolated(struct pglist_data *pgdat, int file, 1892 struct scan_control *sc) 1893 { 1894 unsigned long inactive, isolated; 1895 bool too_many; 1896 1897 if (current_is_kswapd()) 1898 return false; 1899 1900 if (!writeback_throttling_sane(sc)) 1901 return false; 1902 1903 if (file) { 1904 inactive = node_page_state(pgdat, NR_INACTIVE_FILE); 1905 isolated = node_page_state(pgdat, NR_ISOLATED_FILE); 1906 } else { 1907 inactive = node_page_state(pgdat, NR_INACTIVE_ANON); 1908 isolated = node_page_state(pgdat, NR_ISOLATED_ANON); 1909 } 1910 1911 /* 1912 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they 1913 * won't get blocked by normal direct-reclaimers, forming a circular 1914 * deadlock. 1915 */ 1916 if (gfp_has_io_fs(sc->gfp_mask)) 1917 inactive >>= 3; 1918 1919 too_many = isolated > inactive; 1920 1921 /* Wake up tasks throttled due to too_many_isolated. */ 1922 if (!too_many) 1923 wake_throttle_isolated(pgdat); 1924 1925 return too_many; 1926 } 1927 1928 /* 1929 * move_folios_to_lru() moves folios from private @list to appropriate LRU list. 1930 * 1931 * Returns the number of pages moved to the given lruvec. 1932 */ 1933 static unsigned int move_folios_to_lru(struct lruvec *lruvec, 1934 struct list_head *list) 1935 { 1936 int nr_pages, nr_moved = 0; 1937 struct folio_batch free_folios; 1938 1939 folio_batch_init(&free_folios); 1940 while (!list_empty(list)) { 1941 struct folio *folio = lru_to_folio(list); 1942 1943 VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); 1944 list_del(&folio->lru); 1945 if (unlikely(!folio_evictable(folio))) { 1946 spin_unlock_irq(&lruvec->lru_lock); 1947 folio_putback_lru(folio); 1948 spin_lock_irq(&lruvec->lru_lock); 1949 continue; 1950 } 1951 1952 /* 1953 * The folio_set_lru needs to be kept here for list integrity. 1954 * Otherwise: 1955 * #0 move_folios_to_lru #1 release_pages 1956 * if (!folio_put_testzero()) 1957 * if (folio_put_testzero()) 1958 * !lru //skip lru_lock 1959 * folio_set_lru() 1960 * list_add(&folio->lru,) 1961 * list_add(&folio->lru,) 1962 */ 1963 folio_set_lru(folio); 1964 1965 if (unlikely(folio_put_testzero(folio))) { 1966 __folio_clear_lru_flags(folio); 1967 1968 folio_unqueue_deferred_split(folio); 1969 if (folio_batch_add(&free_folios, folio) == 0) { 1970 spin_unlock_irq(&lruvec->lru_lock); 1971 mem_cgroup_uncharge_folios(&free_folios); 1972 free_unref_folios(&free_folios); 1973 spin_lock_irq(&lruvec->lru_lock); 1974 } 1975 1976 continue; 1977 } 1978 1979 /* 1980 * All pages were isolated from the same lruvec (and isolation 1981 * inhibits memcg migration). 1982 */ 1983 VM_BUG_ON_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); 1984 lruvec_add_folio(lruvec, folio); 1985 nr_pages = folio_nr_pages(folio); 1986 nr_moved += nr_pages; 1987 if (folio_test_active(folio)) 1988 workingset_age_nonresident(lruvec, nr_pages); 1989 } 1990 1991 if (free_folios.nr) { 1992 spin_unlock_irq(&lruvec->lru_lock); 1993 mem_cgroup_uncharge_folios(&free_folios); 1994 free_unref_folios(&free_folios); 1995 spin_lock_irq(&lruvec->lru_lock); 1996 } 1997 1998 return nr_moved; 1999 } 2000 2001 /* 2002 * If a kernel thread (such as nfsd for loop-back mounts) services a backing 2003 * device by writing to the page cache it sets PF_LOCAL_THROTTLE. In this case 2004 * we should not throttle. Otherwise it is safe to do so. 2005 */ 2006 static int current_may_throttle(void) 2007 { 2008 return !(current->flags & PF_LOCAL_THROTTLE); 2009 } 2010 2011 /* 2012 * shrink_inactive_list() is a helper for shrink_node(). It returns the number 2013 * of reclaimed pages 2014 */ 2015 static unsigned long shrink_inactive_list(unsigned long nr_to_scan, 2016 struct lruvec *lruvec, struct scan_control *sc, 2017 enum lru_list lru) 2018 { 2019 LIST_HEAD(folio_list); 2020 unsigned long nr_scanned; 2021 unsigned int nr_reclaimed = 0; 2022 unsigned long nr_taken; 2023 struct reclaim_stat stat; 2024 bool file = is_file_lru(lru); 2025 enum vm_event_item item; 2026 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2027 bool stalled = false; 2028 2029 while (unlikely(too_many_isolated(pgdat, file, sc))) { 2030 if (stalled) 2031 return 0; 2032 2033 /* wait a bit for the reclaimer. */ 2034 stalled = true; 2035 reclaim_throttle(pgdat, VMSCAN_THROTTLE_ISOLATED); 2036 2037 /* We are about to die and free our memory. Return now. */ 2038 if (fatal_signal_pending(current)) 2039 return SWAP_CLUSTER_MAX; 2040 } 2041 2042 lru_add_drain(); 2043 2044 spin_lock_irq(&lruvec->lru_lock); 2045 2046 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list, 2047 &nr_scanned, sc, lru); 2048 2049 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2050 item = PGSCAN_KSWAPD + reclaimer_offset(sc); 2051 if (!cgroup_reclaim(sc)) 2052 __count_vm_events(item, nr_scanned); 2053 count_memcg_events(lruvec_memcg(lruvec), item, nr_scanned); 2054 __count_vm_events(PGSCAN_ANON + file, nr_scanned); 2055 2056 spin_unlock_irq(&lruvec->lru_lock); 2057 2058 if (nr_taken == 0) 2059 return 0; 2060 2061 nr_reclaimed = shrink_folio_list(&folio_list, pgdat, sc, &stat, false, 2062 lruvec_memcg(lruvec)); 2063 2064 spin_lock_irq(&lruvec->lru_lock); 2065 move_folios_to_lru(lruvec, &folio_list); 2066 2067 __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), 2068 stat.nr_demoted); 2069 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2070 item = PGSTEAL_KSWAPD + reclaimer_offset(sc); 2071 if (!cgroup_reclaim(sc)) 2072 __count_vm_events(item, nr_reclaimed); 2073 count_memcg_events(lruvec_memcg(lruvec), item, nr_reclaimed); 2074 __count_vm_events(PGSTEAL_ANON + file, nr_reclaimed); 2075 2076 lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout, 2077 nr_scanned - nr_reclaimed); 2078 2079 /* 2080 * If dirty folios are scanned that are not queued for IO, it 2081 * implies that flushers are not doing their job. This can 2082 * happen when memory pressure pushes dirty folios to the end of 2083 * the LRU before the dirty limits are breached and the dirty 2084 * data has expired. It can also happen when the proportion of 2085 * dirty folios grows not through writes but through memory 2086 * pressure reclaiming all the clean cache. And in some cases, 2087 * the flushers simply cannot keep up with the allocation 2088 * rate. Nudge the flusher threads in case they are asleep. 2089 */ 2090 if (stat.nr_unqueued_dirty == nr_taken) { 2091 wakeup_flusher_threads(WB_REASON_VMSCAN); 2092 /* 2093 * For cgroupv1 dirty throttling is achieved by waking up 2094 * the kernel flusher here and later waiting on folios 2095 * which are in writeback to finish (see shrink_folio_list()). 2096 * 2097 * Flusher may not be able to issue writeback quickly 2098 * enough for cgroupv1 writeback throttling to work 2099 * on a large system. 2100 */ 2101 if (!writeback_throttling_sane(sc)) 2102 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 2103 } 2104 2105 sc->nr.dirty += stat.nr_dirty; 2106 sc->nr.congested += stat.nr_congested; 2107 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 2108 sc->nr.writeback += stat.nr_writeback; 2109 sc->nr.immediate += stat.nr_immediate; 2110 sc->nr.taken += nr_taken; 2111 if (file) 2112 sc->nr.file_taken += nr_taken; 2113 2114 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 2115 nr_scanned, nr_reclaimed, &stat, sc->priority, file); 2116 return nr_reclaimed; 2117 } 2118 2119 /* 2120 * shrink_active_list() moves folios from the active LRU to the inactive LRU. 2121 * 2122 * We move them the other way if the folio is referenced by one or more 2123 * processes. 2124 * 2125 * If the folios are mostly unmapped, the processing is fast and it is 2126 * appropriate to hold lru_lock across the whole operation. But if 2127 * the folios are mapped, the processing is slow (folio_referenced()), so 2128 * we should drop lru_lock around each folio. It's impossible to balance 2129 * this, so instead we remove the folios from the LRU while processing them. 2130 * It is safe to rely on the active flag against the non-LRU folios in here 2131 * because nobody will play with that bit on a non-LRU folio. 2132 * 2133 * The downside is that we have to touch folio->_refcount against each folio. 2134 * But we had to alter folio->flags anyway. 2135 */ 2136 static void shrink_active_list(unsigned long nr_to_scan, 2137 struct lruvec *lruvec, 2138 struct scan_control *sc, 2139 enum lru_list lru) 2140 { 2141 unsigned long nr_taken; 2142 unsigned long nr_scanned; 2143 vm_flags_t vm_flags; 2144 LIST_HEAD(l_hold); /* The folios which were snipped off */ 2145 LIST_HEAD(l_active); 2146 LIST_HEAD(l_inactive); 2147 unsigned nr_deactivate, nr_activate; 2148 unsigned nr_rotated = 0; 2149 bool file = is_file_lru(lru); 2150 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2151 2152 lru_add_drain(); 2153 2154 spin_lock_irq(&lruvec->lru_lock); 2155 2156 nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold, 2157 &nr_scanned, sc, lru); 2158 2159 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, nr_taken); 2160 2161 if (!cgroup_reclaim(sc)) 2162 __count_vm_events(PGREFILL, nr_scanned); 2163 count_memcg_events(lruvec_memcg(lruvec), PGREFILL, nr_scanned); 2164 2165 spin_unlock_irq(&lruvec->lru_lock); 2166 2167 while (!list_empty(&l_hold)) { 2168 struct folio *folio; 2169 2170 cond_resched(); 2171 folio = lru_to_folio(&l_hold); 2172 list_del(&folio->lru); 2173 2174 if (unlikely(!folio_evictable(folio))) { 2175 folio_putback_lru(folio); 2176 continue; 2177 } 2178 2179 if (unlikely(buffer_heads_over_limit)) { 2180 if (folio_needs_release(folio) && 2181 folio_trylock(folio)) { 2182 filemap_release_folio(folio, 0); 2183 folio_unlock(folio); 2184 } 2185 } 2186 2187 /* Referenced or rmap lock contention: rotate */ 2188 if (folio_referenced(folio, 0, sc->target_mem_cgroup, 2189 &vm_flags) != 0) { 2190 /* 2191 * Identify referenced, file-backed active folios and 2192 * give them one more trip around the active list. So 2193 * that executable code get better chances to stay in 2194 * memory under moderate memory pressure. Anon folios 2195 * are not likely to be evicted by use-once streaming 2196 * IO, plus JVM can create lots of anon VM_EXEC folios, 2197 * so we ignore them here. 2198 */ 2199 if ((vm_flags & VM_EXEC) && folio_is_file_lru(folio)) { 2200 nr_rotated += folio_nr_pages(folio); 2201 list_add(&folio->lru, &l_active); 2202 continue; 2203 } 2204 } 2205 2206 folio_clear_active(folio); /* we are de-activating */ 2207 folio_set_workingset(folio); 2208 list_add(&folio->lru, &l_inactive); 2209 } 2210 2211 /* 2212 * Move folios back to the lru list. 2213 */ 2214 spin_lock_irq(&lruvec->lru_lock); 2215 2216 nr_activate = move_folios_to_lru(lruvec, &l_active); 2217 nr_deactivate = move_folios_to_lru(lruvec, &l_inactive); 2218 2219 __count_vm_events(PGDEACTIVATE, nr_deactivate); 2220 count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); 2221 2222 __mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); 2223 2224 lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated); 2225 trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, 2226 nr_deactivate, nr_rotated, sc->priority, file); 2227 } 2228 2229 static unsigned int reclaim_folio_list(struct list_head *folio_list, 2230 struct pglist_data *pgdat) 2231 { 2232 struct reclaim_stat stat; 2233 unsigned int nr_reclaimed; 2234 struct folio *folio; 2235 struct scan_control sc = { 2236 .gfp_mask = GFP_KERNEL, 2237 .may_writepage = 1, 2238 .may_unmap = 1, 2239 .may_swap = 1, 2240 .no_demotion = 1, 2241 }; 2242 2243 nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &stat, true, NULL); 2244 while (!list_empty(folio_list)) { 2245 folio = lru_to_folio(folio_list); 2246 list_del(&folio->lru); 2247 folio_putback_lru(folio); 2248 } 2249 trace_mm_vmscan_reclaim_pages(pgdat->node_id, sc.nr_scanned, nr_reclaimed, &stat); 2250 2251 return nr_reclaimed; 2252 } 2253 2254 unsigned long reclaim_pages(struct list_head *folio_list) 2255 { 2256 int nid; 2257 unsigned int nr_reclaimed = 0; 2258 LIST_HEAD(node_folio_list); 2259 unsigned int noreclaim_flag; 2260 2261 if (list_empty(folio_list)) 2262 return nr_reclaimed; 2263 2264 noreclaim_flag = memalloc_noreclaim_save(); 2265 2266 nid = folio_nid(lru_to_folio(folio_list)); 2267 do { 2268 struct folio *folio = lru_to_folio(folio_list); 2269 2270 if (nid == folio_nid(folio)) { 2271 folio_clear_active(folio); 2272 list_move(&folio->lru, &node_folio_list); 2273 continue; 2274 } 2275 2276 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2277 nid = folio_nid(lru_to_folio(folio_list)); 2278 } while (!list_empty(folio_list)); 2279 2280 nr_reclaimed += reclaim_folio_list(&node_folio_list, NODE_DATA(nid)); 2281 2282 memalloc_noreclaim_restore(noreclaim_flag); 2283 2284 return nr_reclaimed; 2285 } 2286 2287 static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, 2288 struct lruvec *lruvec, struct scan_control *sc) 2289 { 2290 if (is_active_lru(lru)) { 2291 if (sc->may_deactivate & (1 << is_file_lru(lru))) 2292 shrink_active_list(nr_to_scan, lruvec, sc, lru); 2293 else 2294 sc->skipped_deactivate = 1; 2295 return 0; 2296 } 2297 2298 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru); 2299 } 2300 2301 /* 2302 * The inactive anon list should be small enough that the VM never has 2303 * to do too much work. 2304 * 2305 * The inactive file list should be small enough to leave most memory 2306 * to the established workingset on the scan-resistant active list, 2307 * but large enough to avoid thrashing the aggregate readahead window. 2308 * 2309 * Both inactive lists should also be large enough that each inactive 2310 * folio has a chance to be referenced again before it is reclaimed. 2311 * 2312 * If that fails and refaulting is observed, the inactive list grows. 2313 * 2314 * The inactive_ratio is the target ratio of ACTIVE to INACTIVE folios 2315 * on this LRU, maintained by the pageout code. An inactive_ratio 2316 * of 3 means 3:1 or 25% of the folios are kept on the inactive list. 2317 * 2318 * total target max 2319 * memory ratio inactive 2320 * ------------------------------------- 2321 * 10MB 1 5MB 2322 * 100MB 1 50MB 2323 * 1GB 3 250MB 2324 * 10GB 10 0.9GB 2325 * 100GB 31 3GB 2326 * 1TB 101 10GB 2327 * 10TB 320 32GB 2328 */ 2329 static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru) 2330 { 2331 enum lru_list active_lru = inactive_lru + LRU_ACTIVE; 2332 unsigned long inactive, active; 2333 unsigned long inactive_ratio; 2334 unsigned long gb; 2335 2336 inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru); 2337 active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru); 2338 2339 gb = (inactive + active) >> (30 - PAGE_SHIFT); 2340 if (gb) 2341 inactive_ratio = int_sqrt(10 * gb); 2342 else 2343 inactive_ratio = 1; 2344 2345 return inactive * inactive_ratio < active; 2346 } 2347 2348 enum scan_balance { 2349 SCAN_EQUAL, 2350 SCAN_FRACT, 2351 SCAN_ANON, 2352 SCAN_FILE, 2353 }; 2354 2355 static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) 2356 { 2357 unsigned long file; 2358 struct lruvec *target_lruvec; 2359 2360 if (lru_gen_enabled()) 2361 return; 2362 2363 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 2364 2365 /* 2366 * Flush the memory cgroup stats in rate-limited way as we don't need 2367 * most accurate stats here. We may switch to regular stats flushing 2368 * in the future once it is cheap enough. 2369 */ 2370 mem_cgroup_flush_stats_ratelimited(sc->target_mem_cgroup); 2371 2372 /* 2373 * Determine the scan balance between anon and file LRUs. 2374 */ 2375 spin_lock_irq(&target_lruvec->lru_lock); 2376 sc->anon_cost = target_lruvec->anon_cost; 2377 sc->file_cost = target_lruvec->file_cost; 2378 spin_unlock_irq(&target_lruvec->lru_lock); 2379 2380 /* 2381 * Target desirable inactive:active list ratios for the anon 2382 * and file LRU lists. 2383 */ 2384 if (!sc->force_deactivate) { 2385 unsigned long refaults; 2386 2387 /* 2388 * When refaults are being observed, it means a new 2389 * workingset is being established. Deactivate to get 2390 * rid of any stale active pages quickly. 2391 */ 2392 refaults = lruvec_page_state(target_lruvec, 2393 WORKINGSET_ACTIVATE_ANON); 2394 if (refaults != target_lruvec->refaults[WORKINGSET_ANON] || 2395 inactive_is_low(target_lruvec, LRU_INACTIVE_ANON)) 2396 sc->may_deactivate |= DEACTIVATE_ANON; 2397 else 2398 sc->may_deactivate &= ~DEACTIVATE_ANON; 2399 2400 refaults = lruvec_page_state(target_lruvec, 2401 WORKINGSET_ACTIVATE_FILE); 2402 if (refaults != target_lruvec->refaults[WORKINGSET_FILE] || 2403 inactive_is_low(target_lruvec, LRU_INACTIVE_FILE)) 2404 sc->may_deactivate |= DEACTIVATE_FILE; 2405 else 2406 sc->may_deactivate &= ~DEACTIVATE_FILE; 2407 } else 2408 sc->may_deactivate = DEACTIVATE_ANON | DEACTIVATE_FILE; 2409 2410 /* 2411 * If we have plenty of inactive file pages that aren't 2412 * thrashing, try to reclaim those first before touching 2413 * anonymous pages. 2414 */ 2415 file = lruvec_page_state(target_lruvec, NR_INACTIVE_FILE); 2416 if (file >> sc->priority && !(sc->may_deactivate & DEACTIVATE_FILE) && 2417 !sc->no_cache_trim_mode) 2418 sc->cache_trim_mode = 1; 2419 else 2420 sc->cache_trim_mode = 0; 2421 2422 /* 2423 * Prevent the reclaimer from falling into the cache trap: as 2424 * cache pages start out inactive, every cache fault will tip 2425 * the scan balance towards the file LRU. And as the file LRU 2426 * shrinks, so does the window for rotation from references. 2427 * This means we have a runaway feedback loop where a tiny 2428 * thrashing file LRU becomes infinitely more attractive than 2429 * anon pages. Try to detect this based on file LRU size. 2430 */ 2431 if (!cgroup_reclaim(sc)) { 2432 unsigned long total_high_wmark = 0; 2433 unsigned long free, anon; 2434 int z; 2435 struct zone *zone; 2436 2437 free = sum_zone_node_page_state(pgdat->node_id, NR_FREE_PAGES); 2438 file = node_page_state(pgdat, NR_ACTIVE_FILE) + 2439 node_page_state(pgdat, NR_INACTIVE_FILE); 2440 2441 for_each_managed_zone_pgdat(zone, pgdat, z, MAX_NR_ZONES - 1) { 2442 total_high_wmark += high_wmark_pages(zone); 2443 } 2444 2445 /* 2446 * Consider anon: if that's low too, this isn't a 2447 * runaway file reclaim problem, but rather just 2448 * extreme pressure. Reclaim as per usual then. 2449 */ 2450 anon = node_page_state(pgdat, NR_INACTIVE_ANON); 2451 2452 sc->file_is_tiny = 2453 file + free <= total_high_wmark && 2454 !(sc->may_deactivate & DEACTIVATE_ANON) && 2455 anon >> sc->priority; 2456 } 2457 } 2458 2459 static inline void calculate_pressure_balance(struct scan_control *sc, 2460 int swappiness, u64 *fraction, u64 *denominator) 2461 { 2462 unsigned long anon_cost, file_cost, total_cost; 2463 unsigned long ap, fp; 2464 2465 /* 2466 * Calculate the pressure balance between anon and file pages. 2467 * 2468 * The amount of pressure we put on each LRU is inversely 2469 * proportional to the cost of reclaiming each list, as 2470 * determined by the share of pages that are refaulting, times 2471 * the relative IO cost of bringing back a swapped out 2472 * anonymous page vs reloading a filesystem page (swappiness). 2473 * 2474 * Although we limit that influence to ensure no list gets 2475 * left behind completely: at least a third of the pressure is 2476 * applied, before swappiness. 2477 * 2478 * With swappiness at 100, anon and file have equal IO cost. 2479 */ 2480 total_cost = sc->anon_cost + sc->file_cost; 2481 anon_cost = total_cost + sc->anon_cost; 2482 file_cost = total_cost + sc->file_cost; 2483 total_cost = anon_cost + file_cost; 2484 2485 ap = swappiness * (total_cost + 1); 2486 ap /= anon_cost + 1; 2487 2488 fp = (MAX_SWAPPINESS - swappiness) * (total_cost + 1); 2489 fp /= file_cost + 1; 2490 2491 fraction[WORKINGSET_ANON] = ap; 2492 fraction[WORKINGSET_FILE] = fp; 2493 *denominator = ap + fp; 2494 } 2495 2496 static unsigned long apply_proportional_protection(struct mem_cgroup *memcg, 2497 struct scan_control *sc, unsigned long scan) 2498 { 2499 unsigned long min, low; 2500 2501 mem_cgroup_protection(sc->target_mem_cgroup, memcg, &min, &low); 2502 2503 if (min || low) { 2504 /* 2505 * Scale a cgroup's reclaim pressure by proportioning 2506 * its current usage to its memory.low or memory.min 2507 * setting. 2508 * 2509 * This is important, as otherwise scanning aggression 2510 * becomes extremely binary -- from nothing as we 2511 * approach the memory protection threshold, to totally 2512 * nominal as we exceed it. This results in requiring 2513 * setting extremely liberal protection thresholds. It 2514 * also means we simply get no protection at all if we 2515 * set it too low, which is not ideal. 2516 * 2517 * If there is any protection in place, we reduce scan 2518 * pressure by how much of the total memory used is 2519 * within protection thresholds. 2520 * 2521 * There is one special case: in the first reclaim pass, 2522 * we skip over all groups that are within their low 2523 * protection. If that fails to reclaim enough pages to 2524 * satisfy the reclaim goal, we come back and override 2525 * the best-effort low protection. However, we still 2526 * ideally want to honor how well-behaved groups are in 2527 * that case instead of simply punishing them all 2528 * equally. As such, we reclaim them based on how much 2529 * memory they are using, reducing the scan pressure 2530 * again by how much of the total memory used is under 2531 * hard protection. 2532 */ 2533 unsigned long cgroup_size = mem_cgroup_size(memcg); 2534 unsigned long protection; 2535 2536 /* memory.low scaling, make sure we retry before OOM */ 2537 if (!sc->memcg_low_reclaim && low > min) { 2538 protection = low; 2539 sc->memcg_low_skipped = 1; 2540 } else { 2541 protection = min; 2542 } 2543 2544 /* Avoid TOCTOU with earlier protection check */ 2545 cgroup_size = max(cgroup_size, protection); 2546 2547 scan -= scan * protection / (cgroup_size + 1); 2548 2549 /* 2550 * Minimally target SWAP_CLUSTER_MAX pages to keep 2551 * reclaim moving forwards, avoiding decrementing 2552 * sc->priority further than desirable. 2553 */ 2554 scan = max(scan, SWAP_CLUSTER_MAX); 2555 } 2556 return scan; 2557 } 2558 2559 /* 2560 * Determine how aggressively the anon and file LRU lists should be 2561 * scanned. 2562 * 2563 * nr[0] = anon inactive folios to scan; nr[1] = anon active folios to scan 2564 * nr[2] = file inactive folios to scan; nr[3] = file active folios to scan 2565 */ 2566 static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, 2567 unsigned long *nr) 2568 { 2569 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2570 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2571 int swappiness = sc_swappiness(sc, memcg); 2572 u64 fraction[ANON_AND_FILE]; 2573 u64 denominator = 0; /* gcc */ 2574 enum scan_balance scan_balance; 2575 enum lru_list lru; 2576 2577 /* If we have no swap space, do not bother scanning anon folios. */ 2578 if (!sc->may_swap || !can_reclaim_anon_pages(memcg, pgdat->node_id, sc)) { 2579 scan_balance = SCAN_FILE; 2580 goto out; 2581 } 2582 2583 /* 2584 * Global reclaim will swap to prevent OOM even with no 2585 * swappiness, but memcg users want to use this knob to 2586 * disable swapping for individual groups completely when 2587 * using the memory controller's swap limit feature would be 2588 * too expensive. 2589 */ 2590 if (cgroup_reclaim(sc) && !swappiness) { 2591 scan_balance = SCAN_FILE; 2592 goto out; 2593 } 2594 2595 /* Proactive reclaim initiated by userspace for anonymous memory only */ 2596 if (swappiness == SWAPPINESS_ANON_ONLY) { 2597 WARN_ON_ONCE(!sc->proactive); 2598 scan_balance = SCAN_ANON; 2599 goto out; 2600 } 2601 2602 /* 2603 * Do not apply any pressure balancing cleverness when the 2604 * system is close to OOM, scan both anon and file equally 2605 * (unless the swappiness setting disagrees with swapping). 2606 */ 2607 if (!sc->priority && swappiness) { 2608 scan_balance = SCAN_EQUAL; 2609 goto out; 2610 } 2611 2612 /* 2613 * If the system is almost out of file pages, force-scan anon. 2614 */ 2615 if (sc->file_is_tiny) { 2616 scan_balance = SCAN_ANON; 2617 goto out; 2618 } 2619 2620 /* 2621 * If there is enough inactive page cache, we do not reclaim 2622 * anything from the anonymous working right now to make sure 2623 * a streaming file access pattern doesn't cause swapping. 2624 */ 2625 if (sc->cache_trim_mode) { 2626 scan_balance = SCAN_FILE; 2627 goto out; 2628 } 2629 2630 scan_balance = SCAN_FRACT; 2631 calculate_pressure_balance(sc, swappiness, fraction, &denominator); 2632 2633 out: 2634 for_each_evictable_lru(lru) { 2635 bool file = is_file_lru(lru); 2636 unsigned long lruvec_size; 2637 unsigned long scan; 2638 2639 lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx); 2640 scan = apply_proportional_protection(memcg, sc, lruvec_size); 2641 scan >>= sc->priority; 2642 2643 /* 2644 * If the cgroup's already been deleted, make sure to 2645 * scrape out the remaining cache. 2646 */ 2647 if (!scan && !mem_cgroup_online(memcg)) 2648 scan = min(lruvec_size, SWAP_CLUSTER_MAX); 2649 2650 switch (scan_balance) { 2651 case SCAN_EQUAL: 2652 /* Scan lists relative to size */ 2653 break; 2654 case SCAN_FRACT: 2655 /* 2656 * Scan types proportional to swappiness and 2657 * their relative recent reclaim efficiency. 2658 * Make sure we don't miss the last page on 2659 * the offlined memory cgroups because of a 2660 * round-off error. 2661 */ 2662 scan = mem_cgroup_online(memcg) ? 2663 div64_u64(scan * fraction[file], denominator) : 2664 DIV64_U64_ROUND_UP(scan * fraction[file], 2665 denominator); 2666 break; 2667 case SCAN_FILE: 2668 case SCAN_ANON: 2669 /* Scan one type exclusively */ 2670 if ((scan_balance == SCAN_FILE) != file) 2671 scan = 0; 2672 break; 2673 default: 2674 /* Look ma, no brain */ 2675 BUG(); 2676 } 2677 2678 nr[lru] = scan; 2679 } 2680 } 2681 2682 /* 2683 * Anonymous LRU management is a waste if there is 2684 * ultimately no way to reclaim the memory. 2685 */ 2686 static bool can_age_anon_pages(struct lruvec *lruvec, 2687 struct scan_control *sc) 2688 { 2689 /* Aging the anon LRU is valuable if swap is present: */ 2690 if (total_swap_pages > 0) 2691 return true; 2692 2693 /* Also valuable if anon pages can be demoted: */ 2694 return can_demote(lruvec_pgdat(lruvec)->node_id, sc, 2695 lruvec_memcg(lruvec)); 2696 } 2697 2698 #ifdef CONFIG_LRU_GEN 2699 2700 #ifdef CONFIG_LRU_GEN_ENABLED 2701 DEFINE_STATIC_KEY_ARRAY_TRUE(lru_gen_caps, NR_LRU_GEN_CAPS); 2702 #define get_cap(cap) static_branch_likely(&lru_gen_caps[cap]) 2703 #else 2704 DEFINE_STATIC_KEY_ARRAY_FALSE(lru_gen_caps, NR_LRU_GEN_CAPS); 2705 #define get_cap(cap) static_branch_unlikely(&lru_gen_caps[cap]) 2706 #endif 2707 2708 static bool should_walk_mmu(void) 2709 { 2710 return arch_has_hw_pte_young() && get_cap(LRU_GEN_MM_WALK); 2711 } 2712 2713 static bool should_clear_pmd_young(void) 2714 { 2715 return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG); 2716 } 2717 2718 /****************************************************************************** 2719 * shorthand helpers 2720 ******************************************************************************/ 2721 2722 #define DEFINE_MAX_SEQ(lruvec) \ 2723 unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq) 2724 2725 #define DEFINE_MIN_SEQ(lruvec) \ 2726 unsigned long min_seq[ANON_AND_FILE] = { \ 2727 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \ 2728 READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \ 2729 } 2730 2731 /* Get the min/max evictable type based on swappiness */ 2732 #define min_type(swappiness) (!(swappiness)) 2733 #define max_type(swappiness) ((swappiness) < SWAPPINESS_ANON_ONLY) 2734 2735 #define evictable_min_seq(min_seq, swappiness) \ 2736 min((min_seq)[min_type(swappiness)], (min_seq)[max_type(swappiness)]) 2737 2738 #define for_each_gen_type_zone(gen, type, zone) \ 2739 for ((gen) = 0; (gen) < MAX_NR_GENS; (gen)++) \ 2740 for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ 2741 for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) 2742 2743 #define for_each_evictable_type(type, swappiness) \ 2744 for ((type) = min_type(swappiness); (type) <= max_type(swappiness); (type)++) 2745 2746 #define get_memcg_gen(seq) ((seq) % MEMCG_NR_GENS) 2747 #define get_memcg_bin(bin) ((bin) % MEMCG_NR_BINS) 2748 2749 static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid) 2750 { 2751 struct pglist_data *pgdat = NODE_DATA(nid); 2752 2753 #ifdef CONFIG_MEMCG 2754 if (memcg) { 2755 struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; 2756 2757 /* see the comment in mem_cgroup_lruvec() */ 2758 if (!lruvec->pgdat) 2759 lruvec->pgdat = pgdat; 2760 2761 return lruvec; 2762 } 2763 #endif 2764 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 2765 2766 return &pgdat->__lruvec; 2767 } 2768 2769 static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc) 2770 { 2771 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 2772 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 2773 2774 if (!sc->may_swap) 2775 return 0; 2776 2777 if (!can_demote(pgdat->node_id, sc, memcg) && 2778 mem_cgroup_get_nr_swap_pages(memcg) < MIN_LRU_BATCH) 2779 return 0; 2780 2781 return sc_swappiness(sc, memcg); 2782 } 2783 2784 static int get_nr_gens(struct lruvec *lruvec, int type) 2785 { 2786 return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1; 2787 } 2788 2789 static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) 2790 { 2791 int type; 2792 2793 for (type = 0; type < ANON_AND_FILE; type++) { 2794 int n = get_nr_gens(lruvec, type); 2795 2796 if (n < MIN_NR_GENS || n > MAX_NR_GENS) 2797 return false; 2798 } 2799 2800 return true; 2801 } 2802 2803 /****************************************************************************** 2804 * Bloom filters 2805 ******************************************************************************/ 2806 2807 /* 2808 * Bloom filters with m=1<<15, k=2 and the false positive rates of ~1/5 when 2809 * n=10,000 and ~1/2 when n=20,000, where, conventionally, m is the number of 2810 * bits in a bitmap, k is the number of hash functions and n is the number of 2811 * inserted items. 2812 * 2813 * Page table walkers use one of the two filters to reduce their search space. 2814 * To get rid of non-leaf entries that no longer have enough leaf entries, the 2815 * aging uses the double-buffering technique to flip to the other filter each 2816 * time it produces a new generation. For non-leaf entries that have enough 2817 * leaf entries, the aging carries them over to the next generation in 2818 * walk_pmd_range(); the eviction also report them when walking the rmap 2819 * in lru_gen_look_around(). 2820 * 2821 * For future optimizations: 2822 * 1. It's not necessary to keep both filters all the time. The spare one can be 2823 * freed after the RCU grace period and reallocated if needed again. 2824 * 2. And when reallocating, it's worth scaling its size according to the number 2825 * of inserted entries in the other filter, to reduce the memory overhead on 2826 * small systems and false positives on large systems. 2827 * 3. Jenkins' hash function is an alternative to Knuth's. 2828 */ 2829 #define BLOOM_FILTER_SHIFT 15 2830 2831 static inline int filter_gen_from_seq(unsigned long seq) 2832 { 2833 return seq % NR_BLOOM_FILTERS; 2834 } 2835 2836 static void get_item_key(void *item, int *key) 2837 { 2838 u32 hash = hash_ptr(item, BLOOM_FILTER_SHIFT * 2); 2839 2840 BUILD_BUG_ON(BLOOM_FILTER_SHIFT * 2 > BITS_PER_TYPE(u32)); 2841 2842 key[0] = hash & (BIT(BLOOM_FILTER_SHIFT) - 1); 2843 key[1] = hash >> BLOOM_FILTER_SHIFT; 2844 } 2845 2846 static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, 2847 void *item) 2848 { 2849 int key[2]; 2850 unsigned long *filter; 2851 int gen = filter_gen_from_seq(seq); 2852 2853 filter = READ_ONCE(mm_state->filters[gen]); 2854 if (!filter) 2855 return true; 2856 2857 get_item_key(item, key); 2858 2859 return test_bit(key[0], filter) && test_bit(key[1], filter); 2860 } 2861 2862 static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq, 2863 void *item) 2864 { 2865 int key[2]; 2866 unsigned long *filter; 2867 int gen = filter_gen_from_seq(seq); 2868 2869 filter = READ_ONCE(mm_state->filters[gen]); 2870 if (!filter) 2871 return; 2872 2873 get_item_key(item, key); 2874 2875 if (!test_bit(key[0], filter)) 2876 set_bit(key[0], filter); 2877 if (!test_bit(key[1], filter)) 2878 set_bit(key[1], filter); 2879 } 2880 2881 static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long seq) 2882 { 2883 unsigned long *filter; 2884 int gen = filter_gen_from_seq(seq); 2885 2886 filter = mm_state->filters[gen]; 2887 if (filter) { 2888 bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT)); 2889 return; 2890 } 2891 2892 filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT), 2893 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 2894 WRITE_ONCE(mm_state->filters[gen], filter); 2895 } 2896 2897 /****************************************************************************** 2898 * mm_struct list 2899 ******************************************************************************/ 2900 2901 #ifdef CONFIG_LRU_GEN_WALKS_MMU 2902 2903 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 2904 { 2905 static struct lru_gen_mm_list mm_list = { 2906 .fifo = LIST_HEAD_INIT(mm_list.fifo), 2907 .lock = __SPIN_LOCK_UNLOCKED(mm_list.lock), 2908 }; 2909 2910 #ifdef CONFIG_MEMCG 2911 if (memcg) 2912 return &memcg->mm_list; 2913 #endif 2914 VM_WARN_ON_ONCE(!mem_cgroup_disabled()); 2915 2916 return &mm_list; 2917 } 2918 2919 static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec) 2920 { 2921 return &lruvec->mm_state; 2922 } 2923 2924 static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk) 2925 { 2926 int key; 2927 struct mm_struct *mm; 2928 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 2929 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); 2930 2931 mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list); 2932 key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap); 2933 2934 if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap)) 2935 return NULL; 2936 2937 clear_bit(key, &mm->lru_gen.bitmap); 2938 2939 return mmget_not_zero(mm) ? mm : NULL; 2940 } 2941 2942 void lru_gen_add_mm(struct mm_struct *mm) 2943 { 2944 int nid; 2945 struct mem_cgroup *memcg = get_mem_cgroup_from_mm(mm); 2946 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 2947 2948 VM_WARN_ON_ONCE(!list_empty(&mm->lru_gen.list)); 2949 #ifdef CONFIG_MEMCG 2950 VM_WARN_ON_ONCE(mm->lru_gen.memcg); 2951 mm->lru_gen.memcg = memcg; 2952 #endif 2953 spin_lock(&mm_list->lock); 2954 2955 for_each_node_state(nid, N_MEMORY) { 2956 struct lruvec *lruvec = get_lruvec(memcg, nid); 2957 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 2958 2959 /* the first addition since the last iteration */ 2960 if (mm_state->tail == &mm_list->fifo) 2961 mm_state->tail = &mm->lru_gen.list; 2962 } 2963 2964 list_add_tail(&mm->lru_gen.list, &mm_list->fifo); 2965 2966 spin_unlock(&mm_list->lock); 2967 } 2968 2969 void lru_gen_del_mm(struct mm_struct *mm) 2970 { 2971 int nid; 2972 struct lru_gen_mm_list *mm_list; 2973 struct mem_cgroup *memcg = NULL; 2974 2975 if (list_empty(&mm->lru_gen.list)) 2976 return; 2977 2978 #ifdef CONFIG_MEMCG 2979 memcg = mm->lru_gen.memcg; 2980 #endif 2981 mm_list = get_mm_list(memcg); 2982 2983 spin_lock(&mm_list->lock); 2984 2985 for_each_node(nid) { 2986 struct lruvec *lruvec = get_lruvec(memcg, nid); 2987 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 2988 2989 /* where the current iteration continues after */ 2990 if (mm_state->head == &mm->lru_gen.list) 2991 mm_state->head = mm_state->head->prev; 2992 2993 /* where the last iteration ended before */ 2994 if (mm_state->tail == &mm->lru_gen.list) 2995 mm_state->tail = mm_state->tail->next; 2996 } 2997 2998 list_del_init(&mm->lru_gen.list); 2999 3000 spin_unlock(&mm_list->lock); 3001 3002 #ifdef CONFIG_MEMCG 3003 mem_cgroup_put(mm->lru_gen.memcg); 3004 mm->lru_gen.memcg = NULL; 3005 #endif 3006 } 3007 3008 #ifdef CONFIG_MEMCG 3009 void lru_gen_migrate_mm(struct mm_struct *mm) 3010 { 3011 struct mem_cgroup *memcg; 3012 struct task_struct *task = rcu_dereference_protected(mm->owner, true); 3013 3014 VM_WARN_ON_ONCE(task->mm != mm); 3015 lockdep_assert_held(&task->alloc_lock); 3016 3017 /* for mm_update_next_owner() */ 3018 if (mem_cgroup_disabled()) 3019 return; 3020 3021 /* migration can happen before addition */ 3022 if (!mm->lru_gen.memcg) 3023 return; 3024 3025 rcu_read_lock(); 3026 memcg = mem_cgroup_from_task(task); 3027 rcu_read_unlock(); 3028 if (memcg == mm->lru_gen.memcg) 3029 return; 3030 3031 VM_WARN_ON_ONCE(list_empty(&mm->lru_gen.list)); 3032 3033 lru_gen_del_mm(mm); 3034 lru_gen_add_mm(mm); 3035 } 3036 #endif 3037 3038 #else /* !CONFIG_LRU_GEN_WALKS_MMU */ 3039 3040 static struct lru_gen_mm_list *get_mm_list(struct mem_cgroup *memcg) 3041 { 3042 return NULL; 3043 } 3044 3045 static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec) 3046 { 3047 return NULL; 3048 } 3049 3050 static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk) 3051 { 3052 return NULL; 3053 } 3054 3055 #endif 3056 3057 static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last) 3058 { 3059 int i; 3060 int hist; 3061 struct lruvec *lruvec = walk->lruvec; 3062 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3063 3064 lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock); 3065 3066 hist = lru_hist_from_seq(walk->seq); 3067 3068 for (i = 0; i < NR_MM_STATS; i++) { 3069 WRITE_ONCE(mm_state->stats[hist][i], 3070 mm_state->stats[hist][i] + walk->mm_stats[i]); 3071 walk->mm_stats[i] = 0; 3072 } 3073 3074 if (NR_HIST_GENS > 1 && last) { 3075 hist = lru_hist_from_seq(walk->seq + 1); 3076 3077 for (i = 0; i < NR_MM_STATS; i++) 3078 WRITE_ONCE(mm_state->stats[hist][i], 0); 3079 } 3080 } 3081 3082 static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **iter) 3083 { 3084 bool first = false; 3085 bool last = false; 3086 struct mm_struct *mm = NULL; 3087 struct lruvec *lruvec = walk->lruvec; 3088 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3089 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3090 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3091 3092 /* 3093 * mm_state->seq is incremented after each iteration of mm_list. There 3094 * are three interesting cases for this page table walker: 3095 * 1. It tries to start a new iteration with a stale max_seq: there is 3096 * nothing left to do. 3097 * 2. It started the next iteration: it needs to reset the Bloom filter 3098 * so that a fresh set of PTE tables can be recorded. 3099 * 3. It ended the current iteration: it needs to reset the mm stats 3100 * counters and tell its caller to increment max_seq. 3101 */ 3102 spin_lock(&mm_list->lock); 3103 3104 VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq); 3105 3106 if (walk->seq <= mm_state->seq) 3107 goto done; 3108 3109 if (!mm_state->head) 3110 mm_state->head = &mm_list->fifo; 3111 3112 if (mm_state->head == &mm_list->fifo) 3113 first = true; 3114 3115 do { 3116 mm_state->head = mm_state->head->next; 3117 if (mm_state->head == &mm_list->fifo) { 3118 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3119 last = true; 3120 break; 3121 } 3122 3123 /* force scan for those added after the last iteration */ 3124 if (!mm_state->tail || mm_state->tail == mm_state->head) { 3125 mm_state->tail = mm_state->head->next; 3126 walk->force_scan = true; 3127 } 3128 } while (!(mm = get_next_mm(walk))); 3129 done: 3130 if (*iter || last) 3131 reset_mm_stats(walk, last); 3132 3133 spin_unlock(&mm_list->lock); 3134 3135 if (mm && first) 3136 reset_bloom_filter(mm_state, walk->seq + 1); 3137 3138 if (*iter) 3139 mmput_async(*iter); 3140 3141 *iter = mm; 3142 3143 return last; 3144 } 3145 3146 static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq) 3147 { 3148 bool success = false; 3149 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 3150 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 3151 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 3152 3153 spin_lock(&mm_list->lock); 3154 3155 VM_WARN_ON_ONCE(mm_state->seq + 1 < seq); 3156 3157 if (seq > mm_state->seq) { 3158 mm_state->head = NULL; 3159 mm_state->tail = NULL; 3160 WRITE_ONCE(mm_state->seq, mm_state->seq + 1); 3161 success = true; 3162 } 3163 3164 spin_unlock(&mm_list->lock); 3165 3166 return success; 3167 } 3168 3169 /****************************************************************************** 3170 * PID controller 3171 ******************************************************************************/ 3172 3173 /* 3174 * A feedback loop based on Proportional-Integral-Derivative (PID) controller. 3175 * 3176 * The P term is refaulted/(evicted+protected) from a tier in the generation 3177 * currently being evicted; the I term is the exponential moving average of the 3178 * P term over the generations previously evicted, using the smoothing factor 3179 * 1/2; the D term isn't supported. 3180 * 3181 * The setpoint (SP) is always the first tier of one type; the process variable 3182 * (PV) is either any tier of the other type or any other tier of the same 3183 * type. 3184 * 3185 * The error is the difference between the SP and the PV; the correction is to 3186 * turn off protection when SP>PV or turn on protection when SP<PV. 3187 * 3188 * For future optimizations: 3189 * 1. The D term may discount the other two terms over time so that long-lived 3190 * generations can resist stale information. 3191 */ 3192 struct ctrl_pos { 3193 unsigned long refaulted; 3194 unsigned long total; 3195 int gain; 3196 }; 3197 3198 static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain, 3199 struct ctrl_pos *pos) 3200 { 3201 int i; 3202 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3203 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3204 3205 pos->gain = gain; 3206 pos->refaulted = pos->total = 0; 3207 3208 for (i = tier % MAX_NR_TIERS; i <= min(tier, MAX_NR_TIERS - 1); i++) { 3209 pos->refaulted += lrugen->avg_refaulted[type][i] + 3210 atomic_long_read(&lrugen->refaulted[hist][type][i]); 3211 pos->total += lrugen->avg_total[type][i] + 3212 lrugen->protected[hist][type][i] + 3213 atomic_long_read(&lrugen->evicted[hist][type][i]); 3214 } 3215 } 3216 3217 static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover) 3218 { 3219 int hist, tier; 3220 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3221 bool clear = carryover ? NR_HIST_GENS == 1 : NR_HIST_GENS > 1; 3222 unsigned long seq = carryover ? lrugen->min_seq[type] : lrugen->max_seq + 1; 3223 3224 lockdep_assert_held(&lruvec->lru_lock); 3225 3226 if (!carryover && !clear) 3227 return; 3228 3229 hist = lru_hist_from_seq(seq); 3230 3231 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 3232 if (carryover) { 3233 unsigned long sum; 3234 3235 sum = lrugen->avg_refaulted[type][tier] + 3236 atomic_long_read(&lrugen->refaulted[hist][type][tier]); 3237 WRITE_ONCE(lrugen->avg_refaulted[type][tier], sum / 2); 3238 3239 sum = lrugen->avg_total[type][tier] + 3240 lrugen->protected[hist][type][tier] + 3241 atomic_long_read(&lrugen->evicted[hist][type][tier]); 3242 WRITE_ONCE(lrugen->avg_total[type][tier], sum / 2); 3243 } 3244 3245 if (clear) { 3246 atomic_long_set(&lrugen->refaulted[hist][type][tier], 0); 3247 atomic_long_set(&lrugen->evicted[hist][type][tier], 0); 3248 WRITE_ONCE(lrugen->protected[hist][type][tier], 0); 3249 } 3250 } 3251 } 3252 3253 static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv) 3254 { 3255 /* 3256 * Return true if the PV has a limited number of refaults or a lower 3257 * refaulted/total than the SP. 3258 */ 3259 return pv->refaulted < MIN_LRU_BATCH || 3260 pv->refaulted * (sp->total + MIN_LRU_BATCH) * sp->gain <= 3261 (sp->refaulted + 1) * pv->total * pv->gain; 3262 } 3263 3264 /****************************************************************************** 3265 * the aging 3266 ******************************************************************************/ 3267 3268 /* promote pages accessed through page tables */ 3269 static int folio_update_gen(struct folio *folio, int gen) 3270 { 3271 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); 3272 3273 VM_WARN_ON_ONCE(gen >= MAX_NR_GENS); 3274 3275 /* see the comment on LRU_REFS_FLAGS */ 3276 if (!folio_test_referenced(folio) && !folio_test_workingset(folio)) { 3277 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, BIT(PG_referenced)); 3278 return -1; 3279 } 3280 3281 do { 3282 /* lru_gen_del_folio() has isolated this page? */ 3283 if (!(old_flags & LRU_GEN_MASK)) 3284 return -1; 3285 3286 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); 3287 new_flags |= ((gen + 1UL) << LRU_GEN_PGOFF) | BIT(PG_workingset); 3288 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); 3289 3290 return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3291 } 3292 3293 /* protect pages accessed multiple times through file descriptors */ 3294 static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming) 3295 { 3296 int type = folio_is_file_lru(folio); 3297 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3298 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3299 unsigned long new_flags, old_flags = READ_ONCE(folio->flags.f); 3300 3301 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); 3302 3303 do { 3304 new_gen = ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; 3305 /* folio_update_gen() has promoted this page? */ 3306 if (new_gen >= 0 && new_gen != old_gen) 3307 return new_gen; 3308 3309 new_gen = (old_gen + 1) % MAX_NR_GENS; 3310 3311 new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS); 3312 new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF; 3313 /* for folio_end_writeback() */ 3314 if (reclaiming) 3315 new_flags |= BIT(PG_reclaim); 3316 } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); 3317 3318 lru_gen_update_size(lruvec, folio, old_gen, new_gen); 3319 3320 return new_gen; 3321 } 3322 3323 static void update_batch_size(struct lru_gen_mm_walk *walk, struct folio *folio, 3324 int old_gen, int new_gen) 3325 { 3326 int type = folio_is_file_lru(folio); 3327 int zone = folio_zonenum(folio); 3328 int delta = folio_nr_pages(folio); 3329 3330 VM_WARN_ON_ONCE(old_gen >= MAX_NR_GENS); 3331 VM_WARN_ON_ONCE(new_gen >= MAX_NR_GENS); 3332 3333 walk->batched++; 3334 3335 walk->nr_pages[old_gen][type][zone] -= delta; 3336 walk->nr_pages[new_gen][type][zone] += delta; 3337 } 3338 3339 static void reset_batch_size(struct lru_gen_mm_walk *walk) 3340 { 3341 int gen, type, zone; 3342 struct lruvec *lruvec = walk->lruvec; 3343 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3344 3345 walk->batched = 0; 3346 3347 for_each_gen_type_zone(gen, type, zone) { 3348 enum lru_list lru = type * LRU_INACTIVE_FILE; 3349 int delta = walk->nr_pages[gen][type][zone]; 3350 3351 if (!delta) 3352 continue; 3353 3354 walk->nr_pages[gen][type][zone] = 0; 3355 WRITE_ONCE(lrugen->nr_pages[gen][type][zone], 3356 lrugen->nr_pages[gen][type][zone] + delta); 3357 3358 if (lru_gen_is_active(lruvec, gen)) 3359 lru += LRU_ACTIVE; 3360 __update_lru_size(lruvec, lru, zone, delta); 3361 } 3362 } 3363 3364 static int should_skip_vma(unsigned long start, unsigned long end, struct mm_walk *args) 3365 { 3366 struct address_space *mapping; 3367 struct vm_area_struct *vma = args->vma; 3368 struct lru_gen_mm_walk *walk = args->private; 3369 3370 if (!vma_is_accessible(vma)) 3371 return true; 3372 3373 if (is_vm_hugetlb_page(vma)) 3374 return true; 3375 3376 if (!vma_has_recency(vma)) 3377 return true; 3378 3379 if (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) 3380 return true; 3381 3382 if (vma == get_gate_vma(vma->vm_mm)) 3383 return true; 3384 3385 if (vma_is_anonymous(vma)) 3386 return !walk->swappiness; 3387 3388 if (WARN_ON_ONCE(!vma->vm_file || !vma->vm_file->f_mapping)) 3389 return true; 3390 3391 mapping = vma->vm_file->f_mapping; 3392 if (mapping_unevictable(mapping)) 3393 return true; 3394 3395 if (shmem_mapping(mapping)) 3396 return !walk->swappiness; 3397 3398 if (walk->swappiness > MAX_SWAPPINESS) 3399 return true; 3400 3401 /* to exclude special mappings like dax, etc. */ 3402 return !mapping->a_ops->read_folio; 3403 } 3404 3405 /* 3406 * Some userspace memory allocators map many single-page VMAs. Instead of 3407 * returning back to the PGD table for each of such VMAs, finish an entire PMD 3408 * table to reduce zigzags and improve cache performance. 3409 */ 3410 static bool get_next_vma(unsigned long mask, unsigned long size, struct mm_walk *args, 3411 unsigned long *vm_start, unsigned long *vm_end) 3412 { 3413 unsigned long start = round_up(*vm_end, size); 3414 unsigned long end = (start | ~mask) + 1; 3415 VMA_ITERATOR(vmi, args->mm, start); 3416 3417 VM_WARN_ON_ONCE(mask & size); 3418 VM_WARN_ON_ONCE((start & mask) != (*vm_start & mask)); 3419 3420 for_each_vma(vmi, args->vma) { 3421 if (end && end <= args->vma->vm_start) 3422 return false; 3423 3424 if (should_skip_vma(args->vma->vm_start, args->vma->vm_end, args)) 3425 continue; 3426 3427 *vm_start = max(start, args->vma->vm_start); 3428 *vm_end = min(end - 1, args->vma->vm_end - 1) + 1; 3429 3430 return true; 3431 } 3432 3433 return false; 3434 } 3435 3436 static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr, 3437 struct pglist_data *pgdat) 3438 { 3439 unsigned long pfn = pte_pfn(pte); 3440 3441 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3442 3443 if (!pte_present(pte) || is_zero_pfn(pfn)) 3444 return -1; 3445 3446 if (WARN_ON_ONCE(pte_special(pte))) 3447 return -1; 3448 3449 if (!pte_young(pte) && !mm_has_notifiers(vma->vm_mm)) 3450 return -1; 3451 3452 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3453 return -1; 3454 3455 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3456 return -1; 3457 3458 return pfn; 3459 } 3460 3461 static unsigned long get_pmd_pfn(pmd_t pmd, struct vm_area_struct *vma, unsigned long addr, 3462 struct pglist_data *pgdat) 3463 { 3464 unsigned long pfn = pmd_pfn(pmd); 3465 3466 VM_WARN_ON_ONCE(addr < vma->vm_start || addr >= vma->vm_end); 3467 3468 if (!pmd_present(pmd) || is_huge_zero_pmd(pmd)) 3469 return -1; 3470 3471 if (!pmd_young(pmd) && !mm_has_notifiers(vma->vm_mm)) 3472 return -1; 3473 3474 if (WARN_ON_ONCE(!pfn_valid(pfn))) 3475 return -1; 3476 3477 if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat)) 3478 return -1; 3479 3480 return pfn; 3481 } 3482 3483 static struct folio *get_pfn_folio(unsigned long pfn, struct mem_cgroup *memcg, 3484 struct pglist_data *pgdat) 3485 { 3486 struct folio *folio = pfn_folio(pfn); 3487 3488 if (folio_lru_gen(folio) < 0) 3489 return NULL; 3490 3491 if (folio_nid(folio) != pgdat->node_id) 3492 return NULL; 3493 3494 if (folio_memcg(folio) != memcg) 3495 return NULL; 3496 3497 return folio; 3498 } 3499 3500 static bool suitable_to_scan(int total, int young) 3501 { 3502 int n = clamp_t(int, cache_line_size() / sizeof(pte_t), 2, 8); 3503 3504 /* suitable if the average number of young PTEs per cacheline is >=1 */ 3505 return young * n >= total; 3506 } 3507 3508 static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, 3509 int new_gen, bool dirty) 3510 { 3511 int old_gen; 3512 3513 if (!folio) 3514 return; 3515 3516 if (dirty && !folio_test_dirty(folio) && 3517 !(folio_test_anon(folio) && folio_test_swapbacked(folio) && 3518 !folio_test_swapcache(folio))) 3519 folio_mark_dirty(folio); 3520 3521 if (walk) { 3522 old_gen = folio_update_gen(folio, new_gen); 3523 if (old_gen >= 0 && old_gen != new_gen) 3524 update_batch_size(walk, folio, old_gen, new_gen); 3525 } else if (lru_gen_set_refs(folio)) { 3526 old_gen = folio_lru_gen(folio); 3527 if (old_gen >= 0 && old_gen != new_gen) 3528 folio_activate(folio); 3529 } 3530 } 3531 3532 static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 3533 struct mm_walk *args) 3534 { 3535 int i; 3536 bool dirty; 3537 pte_t *pte; 3538 spinlock_t *ptl; 3539 unsigned long addr; 3540 int total = 0; 3541 int young = 0; 3542 struct folio *last = NULL; 3543 struct lru_gen_mm_walk *walk = args->private; 3544 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3545 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3546 DEFINE_MAX_SEQ(walk->lruvec); 3547 int gen = lru_gen_from_seq(max_seq); 3548 pmd_t pmdval; 3549 3550 pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); 3551 if (!pte) 3552 return false; 3553 3554 if (!spin_trylock(ptl)) { 3555 pte_unmap(pte); 3556 return true; 3557 } 3558 3559 if (unlikely(!pmd_same(pmdval, pmdp_get_lockless(pmd)))) { 3560 pte_unmap_unlock(pte, ptl); 3561 return false; 3562 } 3563 3564 arch_enter_lazy_mmu_mode(); 3565 restart: 3566 for (i = pte_index(start), addr = start; addr != end; i++, addr += PAGE_SIZE) { 3567 unsigned long pfn; 3568 struct folio *folio; 3569 pte_t ptent = ptep_get(pte + i); 3570 3571 total++; 3572 walk->mm_stats[MM_LEAF_TOTAL]++; 3573 3574 pfn = get_pte_pfn(ptent, args->vma, addr, pgdat); 3575 if (pfn == -1) 3576 continue; 3577 3578 folio = get_pfn_folio(pfn, memcg, pgdat); 3579 if (!folio) 3580 continue; 3581 3582 if (!ptep_clear_young_notify(args->vma, addr, pte + i)) 3583 continue; 3584 3585 if (last != folio) { 3586 walk_update_folio(walk, last, gen, dirty); 3587 3588 last = folio; 3589 dirty = false; 3590 } 3591 3592 if (pte_dirty(ptent)) 3593 dirty = true; 3594 3595 young++; 3596 walk->mm_stats[MM_LEAF_YOUNG]++; 3597 } 3598 3599 walk_update_folio(walk, last, gen, dirty); 3600 last = NULL; 3601 3602 if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) 3603 goto restart; 3604 3605 arch_leave_lazy_mmu_mode(); 3606 pte_unmap_unlock(pte, ptl); 3607 3608 return suitable_to_scan(total, young); 3609 } 3610 3611 static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma, 3612 struct mm_walk *args, unsigned long *bitmap, unsigned long *first) 3613 { 3614 int i; 3615 bool dirty; 3616 pmd_t *pmd; 3617 spinlock_t *ptl; 3618 struct folio *last = NULL; 3619 struct lru_gen_mm_walk *walk = args->private; 3620 struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec); 3621 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3622 DEFINE_MAX_SEQ(walk->lruvec); 3623 int gen = lru_gen_from_seq(max_seq); 3624 3625 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3626 3627 /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ 3628 if (*first == -1) { 3629 *first = addr; 3630 bitmap_zero(bitmap, MIN_LRU_BATCH); 3631 return; 3632 } 3633 3634 i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); 3635 if (i && i <= MIN_LRU_BATCH) { 3636 __set_bit(i - 1, bitmap); 3637 return; 3638 } 3639 3640 pmd = pmd_offset(pud, *first); 3641 3642 ptl = pmd_lockptr(args->mm, pmd); 3643 if (!spin_trylock(ptl)) 3644 goto done; 3645 3646 arch_enter_lazy_mmu_mode(); 3647 3648 do { 3649 unsigned long pfn; 3650 struct folio *folio; 3651 3652 /* don't round down the first address */ 3653 addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; 3654 3655 if (!pmd_present(pmd[i])) 3656 goto next; 3657 3658 if (!pmd_trans_huge(pmd[i])) { 3659 if (!walk->force_scan && should_clear_pmd_young() && 3660 !mm_has_notifiers(args->mm)) 3661 pmdp_test_and_clear_young(vma, addr, pmd + i); 3662 goto next; 3663 } 3664 3665 pfn = get_pmd_pfn(pmd[i], vma, addr, pgdat); 3666 if (pfn == -1) 3667 goto next; 3668 3669 folio = get_pfn_folio(pfn, memcg, pgdat); 3670 if (!folio) 3671 goto next; 3672 3673 if (!pmdp_clear_young_notify(vma, addr, pmd + i)) 3674 goto next; 3675 3676 if (last != folio) { 3677 walk_update_folio(walk, last, gen, dirty); 3678 3679 last = folio; 3680 dirty = false; 3681 } 3682 3683 if (pmd_dirty(pmd[i])) 3684 dirty = true; 3685 3686 walk->mm_stats[MM_LEAF_YOUNG]++; 3687 next: 3688 i = i > MIN_LRU_BATCH ? 0 : find_next_bit(bitmap, MIN_LRU_BATCH, i) + 1; 3689 } while (i <= MIN_LRU_BATCH); 3690 3691 walk_update_folio(walk, last, gen, dirty); 3692 3693 arch_leave_lazy_mmu_mode(); 3694 spin_unlock(ptl); 3695 done: 3696 *first = -1; 3697 } 3698 3699 static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 3700 struct mm_walk *args) 3701 { 3702 int i; 3703 pmd_t *pmd; 3704 unsigned long next; 3705 unsigned long addr; 3706 struct vm_area_struct *vma; 3707 DECLARE_BITMAP(bitmap, MIN_LRU_BATCH); 3708 unsigned long first = -1; 3709 struct lru_gen_mm_walk *walk = args->private; 3710 struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); 3711 3712 VM_WARN_ON_ONCE(pud_leaf(*pud)); 3713 3714 /* 3715 * Finish an entire PMD in two passes: the first only reaches to PTE 3716 * tables to avoid taking the PMD lock; the second, if necessary, takes 3717 * the PMD lock to clear the accessed bit in PMD entries. 3718 */ 3719 pmd = pmd_offset(pud, start & PUD_MASK); 3720 restart: 3721 /* walk_pte_range() may call get_next_vma() */ 3722 vma = args->vma; 3723 for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { 3724 pmd_t val = pmdp_get_lockless(pmd + i); 3725 3726 next = pmd_addr_end(addr, end); 3727 3728 if (!pmd_present(val) || is_huge_zero_pmd(val)) { 3729 walk->mm_stats[MM_LEAF_TOTAL]++; 3730 continue; 3731 } 3732 3733 if (pmd_trans_huge(val)) { 3734 struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec); 3735 unsigned long pfn = get_pmd_pfn(val, vma, addr, pgdat); 3736 3737 walk->mm_stats[MM_LEAF_TOTAL]++; 3738 3739 if (pfn != -1) 3740 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); 3741 continue; 3742 } 3743 3744 if (!walk->force_scan && should_clear_pmd_young() && 3745 !mm_has_notifiers(args->mm)) { 3746 if (!pmd_young(val)) 3747 continue; 3748 3749 walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first); 3750 } 3751 3752 if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i)) 3753 continue; 3754 3755 walk->mm_stats[MM_NONLEAF_FOUND]++; 3756 3757 if (!walk_pte_range(&val, addr, next, args)) 3758 continue; 3759 3760 walk->mm_stats[MM_NONLEAF_ADDED]++; 3761 3762 /* carry over to the next generation */ 3763 update_bloom_filter(mm_state, walk->seq + 1, pmd + i); 3764 } 3765 3766 walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); 3767 3768 if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) 3769 goto restart; 3770 } 3771 3772 static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, 3773 struct mm_walk *args) 3774 { 3775 int i; 3776 pud_t *pud; 3777 unsigned long addr; 3778 unsigned long next; 3779 struct lru_gen_mm_walk *walk = args->private; 3780 3781 VM_WARN_ON_ONCE(p4d_leaf(*p4d)); 3782 3783 pud = pud_offset(p4d, start & P4D_MASK); 3784 restart: 3785 for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { 3786 pud_t val = pudp_get(pud + i); 3787 3788 next = pud_addr_end(addr, end); 3789 3790 if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) 3791 continue; 3792 3793 walk_pmd_range(&val, addr, next, args); 3794 3795 if (need_resched() || walk->batched >= MAX_LRU_BATCH) { 3796 end = (addr | ~PUD_MASK) + 1; 3797 goto done; 3798 } 3799 } 3800 3801 if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) 3802 goto restart; 3803 3804 end = round_up(end, P4D_SIZE); 3805 done: 3806 if (!end || !args->vma) 3807 return 1; 3808 3809 walk->next_addr = max(end, args->vma->vm_start); 3810 3811 return -EAGAIN; 3812 } 3813 3814 static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk) 3815 { 3816 static const struct mm_walk_ops mm_walk_ops = { 3817 .test_walk = should_skip_vma, 3818 .p4d_entry = walk_pud_range, 3819 .walk_lock = PGWALK_RDLOCK, 3820 }; 3821 int err; 3822 struct lruvec *lruvec = walk->lruvec; 3823 3824 walk->next_addr = FIRST_USER_ADDRESS; 3825 3826 do { 3827 DEFINE_MAX_SEQ(lruvec); 3828 3829 err = -EBUSY; 3830 3831 /* another thread might have called inc_max_seq() */ 3832 if (walk->seq != max_seq) 3833 break; 3834 3835 /* the caller might be holding the lock for write */ 3836 if (mmap_read_trylock(mm)) { 3837 err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk); 3838 3839 mmap_read_unlock(mm); 3840 } 3841 3842 if (walk->batched) { 3843 spin_lock_irq(&lruvec->lru_lock); 3844 reset_batch_size(walk); 3845 spin_unlock_irq(&lruvec->lru_lock); 3846 } 3847 3848 cond_resched(); 3849 } while (err == -EAGAIN); 3850 } 3851 3852 static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc) 3853 { 3854 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 3855 3856 if (pgdat && current_is_kswapd()) { 3857 VM_WARN_ON_ONCE(walk); 3858 3859 walk = &pgdat->mm_walk; 3860 } else if (!walk && force_alloc) { 3861 VM_WARN_ON_ONCE(current_is_kswapd()); 3862 3863 walk = kzalloc(sizeof(*walk), __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN); 3864 } 3865 3866 current->reclaim_state->mm_walk = walk; 3867 3868 return walk; 3869 } 3870 3871 static void clear_mm_walk(void) 3872 { 3873 struct lru_gen_mm_walk *walk = current->reclaim_state->mm_walk; 3874 3875 VM_WARN_ON_ONCE(walk && memchr_inv(walk->nr_pages, 0, sizeof(walk->nr_pages))); 3876 VM_WARN_ON_ONCE(walk && memchr_inv(walk->mm_stats, 0, sizeof(walk->mm_stats))); 3877 3878 current->reclaim_state->mm_walk = NULL; 3879 3880 if (!current_is_kswapd()) 3881 kfree(walk); 3882 } 3883 3884 static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) 3885 { 3886 int zone; 3887 int remaining = MAX_LRU_BATCH; 3888 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3889 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 3890 int new_gen, old_gen = lru_gen_from_seq(lrugen->min_seq[type]); 3891 3892 /* For file type, skip the check if swappiness is anon only */ 3893 if (type && (swappiness == SWAPPINESS_ANON_ONLY)) 3894 goto done; 3895 3896 /* For anon type, skip the check if swappiness is zero (file only) */ 3897 if (!type && !swappiness) 3898 goto done; 3899 3900 /* prevent cold/hot inversion if the type is evictable */ 3901 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 3902 struct list_head *head = &lrugen->folios[old_gen][type][zone]; 3903 3904 while (!list_empty(head)) { 3905 struct folio *folio = lru_to_folio(head); 3906 int refs = folio_lru_refs(folio); 3907 bool workingset = folio_test_workingset(folio); 3908 3909 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 3910 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 3911 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 3912 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 3913 3914 new_gen = folio_inc_gen(lruvec, folio, false); 3915 list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); 3916 3917 /* don't count the workingset being lazily promoted */ 3918 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { 3919 int tier = lru_tier_from_refs(refs, workingset); 3920 int delta = folio_nr_pages(folio); 3921 3922 WRITE_ONCE(lrugen->protected[hist][type][tier], 3923 lrugen->protected[hist][type][tier] + delta); 3924 } 3925 3926 if (!--remaining) 3927 return false; 3928 } 3929 } 3930 done: 3931 reset_ctrl_pos(lruvec, type, true); 3932 WRITE_ONCE(lrugen->min_seq[type], lrugen->min_seq[type] + 1); 3933 3934 return true; 3935 } 3936 3937 static bool try_to_inc_min_seq(struct lruvec *lruvec, int swappiness) 3938 { 3939 int gen, type, zone; 3940 bool success = false; 3941 bool seq_inc_flag = false; 3942 struct lru_gen_folio *lrugen = &lruvec->lrugen; 3943 DEFINE_MIN_SEQ(lruvec); 3944 3945 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 3946 3947 /* find the oldest populated generation */ 3948 for_each_evictable_type(type, swappiness) { 3949 while (min_seq[type] + MIN_NR_GENS <= lrugen->max_seq) { 3950 gen = lru_gen_from_seq(min_seq[type]); 3951 3952 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 3953 if (!list_empty(&lrugen->folios[gen][type][zone])) 3954 goto next; 3955 } 3956 3957 min_seq[type]++; 3958 seq_inc_flag = true; 3959 } 3960 next: 3961 ; 3962 } 3963 3964 /* 3965 * If min_seq[type] of both anonymous and file is not increased, 3966 * we can directly return false to avoid unnecessary checking 3967 * overhead later. 3968 */ 3969 if (!seq_inc_flag) 3970 return success; 3971 3972 /* see the comment on lru_gen_folio */ 3973 if (swappiness && swappiness <= MAX_SWAPPINESS) { 3974 unsigned long seq = lrugen->max_seq - MIN_NR_GENS; 3975 3976 if (min_seq[LRU_GEN_ANON] > seq && min_seq[LRU_GEN_FILE] < seq) 3977 min_seq[LRU_GEN_ANON] = seq; 3978 else if (min_seq[LRU_GEN_FILE] > seq && min_seq[LRU_GEN_ANON] < seq) 3979 min_seq[LRU_GEN_FILE] = seq; 3980 } 3981 3982 for_each_evictable_type(type, swappiness) { 3983 if (min_seq[type] <= lrugen->min_seq[type]) 3984 continue; 3985 3986 reset_ctrl_pos(lruvec, type, true); 3987 WRITE_ONCE(lrugen->min_seq[type], min_seq[type]); 3988 success = true; 3989 } 3990 3991 return success; 3992 } 3993 3994 static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness) 3995 { 3996 bool success; 3997 int prev, next; 3998 int type, zone; 3999 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4000 restart: 4001 if (seq < READ_ONCE(lrugen->max_seq)) 4002 return false; 4003 4004 spin_lock_irq(&lruvec->lru_lock); 4005 4006 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 4007 4008 success = seq == lrugen->max_seq; 4009 if (!success) 4010 goto unlock; 4011 4012 for (type = 0; type < ANON_AND_FILE; type++) { 4013 if (get_nr_gens(lruvec, type) != MAX_NR_GENS) 4014 continue; 4015 4016 if (inc_min_seq(lruvec, type, swappiness)) 4017 continue; 4018 4019 spin_unlock_irq(&lruvec->lru_lock); 4020 cond_resched(); 4021 goto restart; 4022 } 4023 4024 /* 4025 * Update the active/inactive LRU sizes for compatibility. Both sides of 4026 * the current max_seq need to be covered, since max_seq+1 can overlap 4027 * with min_seq[LRU_GEN_ANON] if swapping is constrained. And if they do 4028 * overlap, cold/hot inversion happens. 4029 */ 4030 prev = lru_gen_from_seq(lrugen->max_seq - 1); 4031 next = lru_gen_from_seq(lrugen->max_seq + 1); 4032 4033 for (type = 0; type < ANON_AND_FILE; type++) { 4034 for (zone = 0; zone < MAX_NR_ZONES; zone++) { 4035 enum lru_list lru = type * LRU_INACTIVE_FILE; 4036 long delta = lrugen->nr_pages[prev][type][zone] - 4037 lrugen->nr_pages[next][type][zone]; 4038 4039 if (!delta) 4040 continue; 4041 4042 __update_lru_size(lruvec, lru, zone, delta); 4043 __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta); 4044 } 4045 } 4046 4047 for (type = 0; type < ANON_AND_FILE; type++) 4048 reset_ctrl_pos(lruvec, type, false); 4049 4050 WRITE_ONCE(lrugen->timestamps[next], jiffies); 4051 /* make sure preceding modifications appear */ 4052 smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); 4053 unlock: 4054 spin_unlock_irq(&lruvec->lru_lock); 4055 4056 return success; 4057 } 4058 4059 static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq, 4060 int swappiness, bool force_scan) 4061 { 4062 bool success; 4063 struct lru_gen_mm_walk *walk; 4064 struct mm_struct *mm = NULL; 4065 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4066 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 4067 4068 VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq)); 4069 4070 if (!mm_state) 4071 return inc_max_seq(lruvec, seq, swappiness); 4072 4073 /* see the comment in iterate_mm_list() */ 4074 if (seq <= READ_ONCE(mm_state->seq)) 4075 return false; 4076 4077 /* 4078 * If the hardware doesn't automatically set the accessed bit, fallback 4079 * to lru_gen_look_around(), which only clears the accessed bit in a 4080 * handful of PTEs. Spreading the work out over a period of time usually 4081 * is less efficient, but it avoids bursty page faults. 4082 */ 4083 if (!should_walk_mmu()) { 4084 success = iterate_mm_list_nowalk(lruvec, seq); 4085 goto done; 4086 } 4087 4088 walk = set_mm_walk(NULL, true); 4089 if (!walk) { 4090 success = iterate_mm_list_nowalk(lruvec, seq); 4091 goto done; 4092 } 4093 4094 walk->lruvec = lruvec; 4095 walk->seq = seq; 4096 walk->swappiness = swappiness; 4097 walk->force_scan = force_scan; 4098 4099 do { 4100 success = iterate_mm_list(walk, &mm); 4101 if (mm) 4102 walk_mm(mm, walk); 4103 } while (mm); 4104 done: 4105 if (success) { 4106 success = inc_max_seq(lruvec, seq, swappiness); 4107 WARN_ON_ONCE(!success); 4108 } 4109 4110 return success; 4111 } 4112 4113 /****************************************************************************** 4114 * working set protection 4115 ******************************************************************************/ 4116 4117 static void set_initial_priority(struct pglist_data *pgdat, struct scan_control *sc) 4118 { 4119 int priority; 4120 unsigned long reclaimable; 4121 4122 if (sc->priority != DEF_PRIORITY || sc->nr_to_reclaim < MIN_LRU_BATCH) 4123 return; 4124 /* 4125 * Determine the initial priority based on 4126 * (total >> priority) * reclaimed_to_scanned_ratio = nr_to_reclaim, 4127 * where reclaimed_to_scanned_ratio = inactive / total. 4128 */ 4129 reclaimable = node_page_state(pgdat, NR_INACTIVE_FILE); 4130 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 4131 reclaimable += node_page_state(pgdat, NR_INACTIVE_ANON); 4132 4133 /* round down reclaimable and round up sc->nr_to_reclaim */ 4134 priority = fls_long(reclaimable) - 1 - fls_long(sc->nr_to_reclaim - 1); 4135 4136 /* 4137 * The estimation is based on LRU pages only, so cap it to prevent 4138 * overshoots of shrinker objects by large margins. 4139 */ 4140 sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY); 4141 } 4142 4143 static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc) 4144 { 4145 int gen, type, zone; 4146 unsigned long total = 0; 4147 int swappiness = get_swappiness(lruvec, sc); 4148 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4149 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4150 DEFINE_MAX_SEQ(lruvec); 4151 DEFINE_MIN_SEQ(lruvec); 4152 4153 for_each_evictable_type(type, swappiness) { 4154 unsigned long seq; 4155 4156 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4157 gen = lru_gen_from_seq(seq); 4158 4159 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4160 total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4161 } 4162 } 4163 4164 /* whether the size is big enough to be helpful */ 4165 return mem_cgroup_online(memcg) ? (total >> sc->priority) : total; 4166 } 4167 4168 static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc, 4169 unsigned long min_ttl) 4170 { 4171 int gen; 4172 unsigned long birth; 4173 int swappiness = get_swappiness(lruvec, sc); 4174 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4175 DEFINE_MIN_SEQ(lruvec); 4176 4177 if (mem_cgroup_below_min(NULL, memcg)) 4178 return false; 4179 4180 if (!lruvec_is_sizable(lruvec, sc)) 4181 return false; 4182 4183 gen = lru_gen_from_seq(evictable_min_seq(min_seq, swappiness)); 4184 birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 4185 4186 return time_is_before_jiffies(birth + min_ttl); 4187 } 4188 4189 /* to protect the working set of the last N jiffies */ 4190 static unsigned long lru_gen_min_ttl __read_mostly; 4191 4192 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 4193 { 4194 struct mem_cgroup *memcg; 4195 unsigned long min_ttl = READ_ONCE(lru_gen_min_ttl); 4196 bool reclaimable = !min_ttl; 4197 4198 VM_WARN_ON_ONCE(!current_is_kswapd()); 4199 4200 set_initial_priority(pgdat, sc); 4201 4202 memcg = mem_cgroup_iter(NULL, NULL, NULL); 4203 do { 4204 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4205 4206 mem_cgroup_calculate_protection(NULL, memcg); 4207 4208 if (!reclaimable) 4209 reclaimable = lruvec_is_reclaimable(lruvec, sc, min_ttl); 4210 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 4211 4212 /* 4213 * The main goal is to OOM kill if every generation from all memcgs is 4214 * younger than min_ttl. However, another possibility is all memcgs are 4215 * either too small or below min. 4216 */ 4217 if (!reclaimable && mutex_trylock(&oom_lock)) { 4218 struct oom_control oc = { 4219 .gfp_mask = sc->gfp_mask, 4220 }; 4221 4222 out_of_memory(&oc); 4223 4224 mutex_unlock(&oom_lock); 4225 } 4226 } 4227 4228 /****************************************************************************** 4229 * rmap/PT walk feedback 4230 ******************************************************************************/ 4231 4232 /* 4233 * This function exploits spatial locality when shrink_folio_list() walks the 4234 * rmap. It scans the adjacent PTEs of a young PTE and promotes hot pages. If 4235 * the scan was done cacheline efficiently, it adds the PMD entry pointing to 4236 * the PTE table to the Bloom filter. This forms a feedback loop between the 4237 * eviction and the aging. 4238 */ 4239 bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw) 4240 { 4241 int i; 4242 bool dirty; 4243 unsigned long start; 4244 unsigned long end; 4245 struct lru_gen_mm_walk *walk; 4246 struct folio *last = NULL; 4247 int young = 1; 4248 pte_t *pte = pvmw->pte; 4249 unsigned long addr = pvmw->address; 4250 struct vm_area_struct *vma = pvmw->vma; 4251 struct folio *folio = pfn_folio(pvmw->pfn); 4252 struct mem_cgroup *memcg = folio_memcg(folio); 4253 struct pglist_data *pgdat = folio_pgdat(folio); 4254 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 4255 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 4256 DEFINE_MAX_SEQ(lruvec); 4257 int gen = lru_gen_from_seq(max_seq); 4258 4259 lockdep_assert_held(pvmw->ptl); 4260 VM_WARN_ON_ONCE_FOLIO(folio_test_lru(folio), folio); 4261 4262 if (!ptep_clear_young_notify(vma, addr, pte)) 4263 return false; 4264 4265 if (spin_is_contended(pvmw->ptl)) 4266 return true; 4267 4268 /* exclude special VMAs containing anon pages from COW */ 4269 if (vma->vm_flags & VM_SPECIAL) 4270 return true; 4271 4272 /* avoid taking the LRU lock under the PTL when possible */ 4273 walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; 4274 4275 start = max(addr & PMD_MASK, vma->vm_start); 4276 end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; 4277 4278 if (end - start == PAGE_SIZE) 4279 return true; 4280 4281 if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { 4282 if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) 4283 end = start + MIN_LRU_BATCH * PAGE_SIZE; 4284 else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) 4285 start = end - MIN_LRU_BATCH * PAGE_SIZE; 4286 else { 4287 start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; 4288 end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; 4289 } 4290 } 4291 4292 arch_enter_lazy_mmu_mode(); 4293 4294 pte -= (addr - start) / PAGE_SIZE; 4295 4296 for (i = 0, addr = start; addr != end; i++, addr += PAGE_SIZE) { 4297 unsigned long pfn; 4298 pte_t ptent = ptep_get(pte + i); 4299 4300 pfn = get_pte_pfn(ptent, vma, addr, pgdat); 4301 if (pfn == -1) 4302 continue; 4303 4304 folio = get_pfn_folio(pfn, memcg, pgdat); 4305 if (!folio) 4306 continue; 4307 4308 if (!ptep_clear_young_notify(vma, addr, pte + i)) 4309 continue; 4310 4311 if (last != folio) { 4312 walk_update_folio(walk, last, gen, dirty); 4313 4314 last = folio; 4315 dirty = false; 4316 } 4317 4318 if (pte_dirty(ptent)) 4319 dirty = true; 4320 4321 young++; 4322 } 4323 4324 walk_update_folio(walk, last, gen, dirty); 4325 4326 arch_leave_lazy_mmu_mode(); 4327 4328 /* feedback from rmap walkers to page table walkers */ 4329 if (mm_state && suitable_to_scan(i, young)) 4330 update_bloom_filter(mm_state, max_seq, pvmw->pmd); 4331 4332 return true; 4333 } 4334 4335 /****************************************************************************** 4336 * memcg LRU 4337 ******************************************************************************/ 4338 4339 /* see the comment on MEMCG_NR_GENS */ 4340 enum { 4341 MEMCG_LRU_NOP, 4342 MEMCG_LRU_HEAD, 4343 MEMCG_LRU_TAIL, 4344 MEMCG_LRU_OLD, 4345 MEMCG_LRU_YOUNG, 4346 }; 4347 4348 static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) 4349 { 4350 int seg; 4351 int old, new; 4352 unsigned long flags; 4353 int bin = get_random_u32_below(MEMCG_NR_BINS); 4354 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4355 4356 spin_lock_irqsave(&pgdat->memcg_lru.lock, flags); 4357 4358 VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list)); 4359 4360 seg = 0; 4361 new = old = lruvec->lrugen.gen; 4362 4363 /* see the comment on MEMCG_NR_GENS */ 4364 if (op == MEMCG_LRU_HEAD) 4365 seg = MEMCG_LRU_HEAD; 4366 else if (op == MEMCG_LRU_TAIL) 4367 seg = MEMCG_LRU_TAIL; 4368 else if (op == MEMCG_LRU_OLD) 4369 new = get_memcg_gen(pgdat->memcg_lru.seq); 4370 else if (op == MEMCG_LRU_YOUNG) 4371 new = get_memcg_gen(pgdat->memcg_lru.seq + 1); 4372 else 4373 VM_WARN_ON_ONCE(true); 4374 4375 WRITE_ONCE(lruvec->lrugen.seg, seg); 4376 WRITE_ONCE(lruvec->lrugen.gen, new); 4377 4378 hlist_nulls_del_rcu(&lruvec->lrugen.list); 4379 4380 if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) 4381 hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); 4382 else 4383 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); 4384 4385 pgdat->memcg_lru.nr_memcgs[old]--; 4386 pgdat->memcg_lru.nr_memcgs[new]++; 4387 4388 if (!pgdat->memcg_lru.nr_memcgs[old] && old == get_memcg_gen(pgdat->memcg_lru.seq)) 4389 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); 4390 4391 spin_unlock_irqrestore(&pgdat->memcg_lru.lock, flags); 4392 } 4393 4394 #ifdef CONFIG_MEMCG 4395 4396 void lru_gen_online_memcg(struct mem_cgroup *memcg) 4397 { 4398 int gen; 4399 int nid; 4400 int bin = get_random_u32_below(MEMCG_NR_BINS); 4401 4402 for_each_node(nid) { 4403 struct pglist_data *pgdat = NODE_DATA(nid); 4404 struct lruvec *lruvec = get_lruvec(memcg, nid); 4405 4406 spin_lock_irq(&pgdat->memcg_lru.lock); 4407 4408 VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list)); 4409 4410 gen = get_memcg_gen(pgdat->memcg_lru.seq); 4411 4412 lruvec->lrugen.gen = gen; 4413 4414 hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); 4415 pgdat->memcg_lru.nr_memcgs[gen]++; 4416 4417 spin_unlock_irq(&pgdat->memcg_lru.lock); 4418 } 4419 } 4420 4421 void lru_gen_offline_memcg(struct mem_cgroup *memcg) 4422 { 4423 int nid; 4424 4425 for_each_node(nid) { 4426 struct lruvec *lruvec = get_lruvec(memcg, nid); 4427 4428 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD); 4429 } 4430 } 4431 4432 void lru_gen_release_memcg(struct mem_cgroup *memcg) 4433 { 4434 int gen; 4435 int nid; 4436 4437 for_each_node(nid) { 4438 struct pglist_data *pgdat = NODE_DATA(nid); 4439 struct lruvec *lruvec = get_lruvec(memcg, nid); 4440 4441 spin_lock_irq(&pgdat->memcg_lru.lock); 4442 4443 if (hlist_nulls_unhashed(&lruvec->lrugen.list)) 4444 goto unlock; 4445 4446 gen = lruvec->lrugen.gen; 4447 4448 hlist_nulls_del_init_rcu(&lruvec->lrugen.list); 4449 pgdat->memcg_lru.nr_memcgs[gen]--; 4450 4451 if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq)) 4452 WRITE_ONCE(pgdat->memcg_lru.seq, pgdat->memcg_lru.seq + 1); 4453 unlock: 4454 spin_unlock_irq(&pgdat->memcg_lru.lock); 4455 } 4456 } 4457 4458 void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) 4459 { 4460 struct lruvec *lruvec = get_lruvec(memcg, nid); 4461 4462 /* see the comment on MEMCG_NR_GENS */ 4463 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD) 4464 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD); 4465 } 4466 4467 #endif /* CONFIG_MEMCG */ 4468 4469 /****************************************************************************** 4470 * the eviction 4471 ******************************************************************************/ 4472 4473 static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, 4474 int tier_idx) 4475 { 4476 bool success; 4477 bool dirty, writeback; 4478 int gen = folio_lru_gen(folio); 4479 int type = folio_is_file_lru(folio); 4480 int zone = folio_zonenum(folio); 4481 int delta = folio_nr_pages(folio); 4482 int refs = folio_lru_refs(folio); 4483 bool workingset = folio_test_workingset(folio); 4484 int tier = lru_tier_from_refs(refs, workingset); 4485 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4486 4487 VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); 4488 4489 /* unevictable */ 4490 if (!folio_evictable(folio)) { 4491 success = lru_gen_del_folio(lruvec, folio, true); 4492 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4493 folio_set_unevictable(folio); 4494 lruvec_add_folio(lruvec, folio); 4495 __count_vm_events(UNEVICTABLE_PGCULLED, delta); 4496 return true; 4497 } 4498 4499 /* promoted */ 4500 if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { 4501 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4502 return true; 4503 } 4504 4505 /* protected */ 4506 if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) { 4507 gen = folio_inc_gen(lruvec, folio, false); 4508 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4509 4510 /* don't count the workingset being lazily promoted */ 4511 if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) { 4512 int hist = lru_hist_from_seq(lrugen->min_seq[type]); 4513 4514 WRITE_ONCE(lrugen->protected[hist][type][tier], 4515 lrugen->protected[hist][type][tier] + delta); 4516 } 4517 return true; 4518 } 4519 4520 /* ineligible */ 4521 if (zone > sc->reclaim_idx) { 4522 gen = folio_inc_gen(lruvec, folio, false); 4523 list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]); 4524 return true; 4525 } 4526 4527 dirty = folio_test_dirty(folio); 4528 writeback = folio_test_writeback(folio); 4529 if (type == LRU_GEN_FILE && dirty) { 4530 sc->nr.file_taken += delta; 4531 if (!writeback) 4532 sc->nr.unqueued_dirty += delta; 4533 } 4534 4535 /* waiting for writeback */ 4536 if (writeback || (type == LRU_GEN_FILE && dirty)) { 4537 gen = folio_inc_gen(lruvec, folio, true); 4538 list_move(&folio->lru, &lrugen->folios[gen][type][zone]); 4539 return true; 4540 } 4541 4542 return false; 4543 } 4544 4545 static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc) 4546 { 4547 bool success; 4548 4549 /* swap constrained */ 4550 if (!(sc->gfp_mask & __GFP_IO) && 4551 (folio_test_dirty(folio) || 4552 (folio_test_anon(folio) && !folio_test_swapcache(folio)))) 4553 return false; 4554 4555 /* raced with release_pages() */ 4556 if (!folio_try_get(folio)) 4557 return false; 4558 4559 /* raced with another isolation */ 4560 if (!folio_test_clear_lru(folio)) { 4561 folio_put(folio); 4562 return false; 4563 } 4564 4565 /* see the comment on LRU_REFS_FLAGS */ 4566 if (!folio_test_referenced(folio)) 4567 set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); 4568 4569 /* for shrink_folio_list() */ 4570 folio_clear_reclaim(folio); 4571 4572 success = lru_gen_del_folio(lruvec, folio, true); 4573 VM_WARN_ON_ONCE_FOLIO(!success, folio); 4574 4575 return true; 4576 } 4577 4578 static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec, 4579 struct scan_control *sc, int type, int tier, 4580 struct list_head *list) 4581 { 4582 int i; 4583 int gen; 4584 enum vm_event_item item; 4585 int sorted = 0; 4586 int scanned = 0; 4587 int isolated = 0; 4588 int skipped = 0; 4589 int remaining = min(nr_to_scan, MAX_LRU_BATCH); 4590 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4591 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4592 4593 VM_WARN_ON_ONCE(!list_empty(list)); 4594 4595 if (get_nr_gens(lruvec, type) == MIN_NR_GENS) 4596 return 0; 4597 4598 gen = lru_gen_from_seq(lrugen->min_seq[type]); 4599 4600 for (i = MAX_NR_ZONES; i > 0; i--) { 4601 LIST_HEAD(moved); 4602 int skipped_zone = 0; 4603 int zone = (sc->reclaim_idx + i) % MAX_NR_ZONES; 4604 struct list_head *head = &lrugen->folios[gen][type][zone]; 4605 4606 while (!list_empty(head)) { 4607 struct folio *folio = lru_to_folio(head); 4608 int delta = folio_nr_pages(folio); 4609 4610 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 4611 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 4612 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 4613 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 4614 4615 scanned += delta; 4616 4617 if (sort_folio(lruvec, folio, sc, tier)) 4618 sorted += delta; 4619 else if (isolate_folio(lruvec, folio, sc)) { 4620 list_add(&folio->lru, list); 4621 isolated += delta; 4622 } else { 4623 list_move(&folio->lru, &moved); 4624 skipped_zone += delta; 4625 } 4626 4627 if (!--remaining || max(isolated, skipped_zone) >= MIN_LRU_BATCH) 4628 break; 4629 } 4630 4631 if (skipped_zone) { 4632 list_splice(&moved, head); 4633 __count_zid_vm_events(PGSCAN_SKIP, zone, skipped_zone); 4634 skipped += skipped_zone; 4635 } 4636 4637 if (!remaining || isolated >= MIN_LRU_BATCH) 4638 break; 4639 } 4640 4641 item = PGSCAN_KSWAPD + reclaimer_offset(sc); 4642 if (!cgroup_reclaim(sc)) { 4643 __count_vm_events(item, isolated); 4644 __count_vm_events(PGREFILL, sorted); 4645 } 4646 count_memcg_events(memcg, item, isolated); 4647 count_memcg_events(memcg, PGREFILL, sorted); 4648 __count_vm_events(PGSCAN_ANON + type, isolated); 4649 trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, MAX_LRU_BATCH, 4650 scanned, skipped, isolated, 4651 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); 4652 if (type == LRU_GEN_FILE) 4653 sc->nr.file_taken += isolated; 4654 /* 4655 * There might not be eligible folios due to reclaim_idx. Check the 4656 * remaining to prevent livelock if it's not making progress. 4657 */ 4658 return isolated || !remaining ? scanned : 0; 4659 } 4660 4661 static int get_tier_idx(struct lruvec *lruvec, int type) 4662 { 4663 int tier; 4664 struct ctrl_pos sp, pv; 4665 4666 /* 4667 * To leave a margin for fluctuations, use a larger gain factor (2:3). 4668 * This value is chosen because any other tier would have at least twice 4669 * as many refaults as the first tier. 4670 */ 4671 read_ctrl_pos(lruvec, type, 0, 2, &sp); 4672 for (tier = 1; tier < MAX_NR_TIERS; tier++) { 4673 read_ctrl_pos(lruvec, type, tier, 3, &pv); 4674 if (!positive_ctrl_err(&sp, &pv)) 4675 break; 4676 } 4677 4678 return tier - 1; 4679 } 4680 4681 static int get_type_to_scan(struct lruvec *lruvec, int swappiness) 4682 { 4683 struct ctrl_pos sp, pv; 4684 4685 if (swappiness <= MIN_SWAPPINESS + 1) 4686 return LRU_GEN_FILE; 4687 4688 if (swappiness >= MAX_SWAPPINESS) 4689 return LRU_GEN_ANON; 4690 /* 4691 * Compare the sum of all tiers of anon with that of file to determine 4692 * which type to scan. 4693 */ 4694 read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp); 4695 read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv); 4696 4697 return positive_ctrl_err(&sp, &pv); 4698 } 4699 4700 static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec, 4701 struct scan_control *sc, int swappiness, 4702 int *type_scanned, struct list_head *list) 4703 { 4704 int i; 4705 int type = get_type_to_scan(lruvec, swappiness); 4706 4707 for_each_evictable_type(i, swappiness) { 4708 int scanned; 4709 int tier = get_tier_idx(lruvec, type); 4710 4711 *type_scanned = type; 4712 4713 scanned = scan_folios(nr_to_scan, lruvec, sc, type, tier, list); 4714 if (scanned) 4715 return scanned; 4716 4717 type = !type; 4718 } 4719 4720 return 0; 4721 } 4722 4723 static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, 4724 struct scan_control *sc, int swappiness) 4725 { 4726 int type; 4727 int scanned; 4728 int reclaimed; 4729 LIST_HEAD(list); 4730 LIST_HEAD(clean); 4731 struct folio *folio; 4732 struct folio *next; 4733 enum vm_event_item item; 4734 struct reclaim_stat stat; 4735 struct lru_gen_mm_walk *walk; 4736 bool skip_retry = false; 4737 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4738 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4739 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4740 4741 spin_lock_irq(&lruvec->lru_lock); 4742 4743 scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &type, &list); 4744 4745 scanned += try_to_inc_min_seq(lruvec, swappiness); 4746 4747 if (evictable_min_seq(lrugen->min_seq, swappiness) + MIN_NR_GENS > lrugen->max_seq) 4748 scanned = 0; 4749 4750 spin_unlock_irq(&lruvec->lru_lock); 4751 4752 if (list_empty(&list)) 4753 return scanned; 4754 retry: 4755 reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg); 4756 sc->nr.unqueued_dirty += stat.nr_unqueued_dirty; 4757 sc->nr_reclaimed += reclaimed; 4758 trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, 4759 scanned, reclaimed, &stat, sc->priority, 4760 type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON); 4761 4762 list_for_each_entry_safe_reverse(folio, next, &list, lru) { 4763 DEFINE_MIN_SEQ(lruvec); 4764 4765 if (!folio_evictable(folio)) { 4766 list_del(&folio->lru); 4767 folio_putback_lru(folio); 4768 continue; 4769 } 4770 4771 /* retry folios that may have missed folio_rotate_reclaimable() */ 4772 if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) && 4773 !folio_test_dirty(folio) && !folio_test_writeback(folio)) { 4774 list_move(&folio->lru, &clean); 4775 continue; 4776 } 4777 4778 /* don't add rejected folios to the oldest generation */ 4779 if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) 4780 set_mask_bits(&folio->flags.f, LRU_REFS_FLAGS, BIT(PG_active)); 4781 } 4782 4783 spin_lock_irq(&lruvec->lru_lock); 4784 4785 move_folios_to_lru(lruvec, &list); 4786 4787 walk = current->reclaim_state->mm_walk; 4788 if (walk && walk->batched) { 4789 walk->lruvec = lruvec; 4790 reset_batch_size(walk); 4791 } 4792 4793 __mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc), 4794 stat.nr_demoted); 4795 4796 item = PGSTEAL_KSWAPD + reclaimer_offset(sc); 4797 if (!cgroup_reclaim(sc)) 4798 __count_vm_events(item, reclaimed); 4799 count_memcg_events(memcg, item, reclaimed); 4800 __count_vm_events(PGSTEAL_ANON + type, reclaimed); 4801 4802 spin_unlock_irq(&lruvec->lru_lock); 4803 4804 list_splice_init(&clean, &list); 4805 4806 if (!list_empty(&list)) { 4807 skip_retry = true; 4808 goto retry; 4809 } 4810 4811 return scanned; 4812 } 4813 4814 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, 4815 int swappiness, unsigned long *nr_to_scan) 4816 { 4817 int gen, type, zone; 4818 unsigned long size = 0; 4819 struct lru_gen_folio *lrugen = &lruvec->lrugen; 4820 DEFINE_MIN_SEQ(lruvec); 4821 4822 *nr_to_scan = 0; 4823 /* have to run aging, since eviction is not possible anymore */ 4824 if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq) 4825 return true; 4826 4827 for_each_evictable_type(type, swappiness) { 4828 unsigned long seq; 4829 4830 for (seq = min_seq[type]; seq <= max_seq; seq++) { 4831 gen = lru_gen_from_seq(seq); 4832 4833 for (zone = 0; zone < MAX_NR_ZONES; zone++) 4834 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 4835 } 4836 } 4837 4838 *nr_to_scan = size; 4839 /* better to run aging even though eviction is still possible */ 4840 return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq; 4841 } 4842 4843 /* 4844 * For future optimizations: 4845 * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg 4846 * reclaim. 4847 */ 4848 static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness) 4849 { 4850 bool success; 4851 unsigned long nr_to_scan; 4852 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4853 DEFINE_MAX_SEQ(lruvec); 4854 4855 if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) 4856 return -1; 4857 4858 success = should_run_aging(lruvec, max_seq, swappiness, &nr_to_scan); 4859 4860 /* try to scrape all its memory if this memcg was deleted */ 4861 if (nr_to_scan && !mem_cgroup_online(memcg)) 4862 return nr_to_scan; 4863 4864 nr_to_scan = apply_proportional_protection(memcg, sc, nr_to_scan); 4865 4866 /* try to get away with not aging at the default priority */ 4867 if (!success || sc->priority == DEF_PRIORITY) 4868 return nr_to_scan >> sc->priority; 4869 4870 /* stop scanning this lruvec as it's low on cold folios */ 4871 return try_to_inc_max_seq(lruvec, max_seq, swappiness, false) ? -1 : 0; 4872 } 4873 4874 static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc) 4875 { 4876 int i; 4877 enum zone_watermarks mark; 4878 4879 /* don't abort memcg reclaim to ensure fairness */ 4880 if (!root_reclaim(sc)) 4881 return false; 4882 4883 if (sc->nr_reclaimed >= max(sc->nr_to_reclaim, compact_gap(sc->order))) 4884 return true; 4885 4886 /* check the order to exclude compaction-induced reclaim */ 4887 if (!current_is_kswapd() || sc->order) 4888 return false; 4889 4890 mark = sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ? 4891 WMARK_PROMO : WMARK_HIGH; 4892 4893 for (i = 0; i <= sc->reclaim_idx; i++) { 4894 struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i; 4895 unsigned long size = wmark_pages(zone, mark) + MIN_LRU_BATCH; 4896 4897 if (managed_zone(zone) && !zone_watermark_ok(zone, 0, size, sc->reclaim_idx, 0)) 4898 return false; 4899 } 4900 4901 /* kswapd should abort if all eligible zones are safe */ 4902 return true; 4903 } 4904 4905 static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 4906 { 4907 long nr_to_scan; 4908 unsigned long scanned = 0; 4909 int swappiness = get_swappiness(lruvec, sc); 4910 4911 while (true) { 4912 int delta; 4913 4914 nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness); 4915 if (nr_to_scan <= 0) 4916 break; 4917 4918 delta = evict_folios(nr_to_scan, lruvec, sc, swappiness); 4919 if (!delta) 4920 break; 4921 4922 scanned += delta; 4923 if (scanned >= nr_to_scan) 4924 break; 4925 4926 if (should_abort_scan(lruvec, sc)) 4927 break; 4928 4929 cond_resched(); 4930 } 4931 4932 /* 4933 * If too many file cache in the coldest generation can't be evicted 4934 * due to being dirty, wake up the flusher. 4935 */ 4936 if (sc->nr.unqueued_dirty && sc->nr.unqueued_dirty == sc->nr.file_taken) 4937 wakeup_flusher_threads(WB_REASON_VMSCAN); 4938 4939 /* whether this lruvec should be rotated */ 4940 return nr_to_scan < 0; 4941 } 4942 4943 static int shrink_one(struct lruvec *lruvec, struct scan_control *sc) 4944 { 4945 bool success; 4946 unsigned long scanned = sc->nr_scanned; 4947 unsigned long reclaimed = sc->nr_reclaimed; 4948 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 4949 struct pglist_data *pgdat = lruvec_pgdat(lruvec); 4950 4951 /* lru_gen_age_node() called mem_cgroup_calculate_protection() */ 4952 if (mem_cgroup_below_min(NULL, memcg)) 4953 return MEMCG_LRU_YOUNG; 4954 4955 if (mem_cgroup_below_low(NULL, memcg)) { 4956 /* see the comment on MEMCG_NR_GENS */ 4957 if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL) 4958 return MEMCG_LRU_TAIL; 4959 4960 memcg_memory_event(memcg, MEMCG_LOW); 4961 } 4962 4963 success = try_to_shrink_lruvec(lruvec, sc); 4964 4965 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority); 4966 4967 if (!sc->proactive) 4968 vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned, 4969 sc->nr_reclaimed - reclaimed); 4970 4971 flush_reclaim_state(sc); 4972 4973 if (success && mem_cgroup_online(memcg)) 4974 return MEMCG_LRU_YOUNG; 4975 4976 if (!success && lruvec_is_sizable(lruvec, sc)) 4977 return 0; 4978 4979 /* one retry if offlined or too small */ 4980 return READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL ? 4981 MEMCG_LRU_TAIL : MEMCG_LRU_YOUNG; 4982 } 4983 4984 static void shrink_many(struct pglist_data *pgdat, struct scan_control *sc) 4985 { 4986 int op; 4987 int gen; 4988 int bin; 4989 int first_bin; 4990 struct lruvec *lruvec; 4991 struct lru_gen_folio *lrugen; 4992 struct mem_cgroup *memcg; 4993 struct hlist_nulls_node *pos; 4994 4995 gen = get_memcg_gen(READ_ONCE(pgdat->memcg_lru.seq)); 4996 bin = first_bin = get_random_u32_below(MEMCG_NR_BINS); 4997 restart: 4998 op = 0; 4999 memcg = NULL; 5000 5001 rcu_read_lock(); 5002 5003 hlist_nulls_for_each_entry_rcu(lrugen, pos, &pgdat->memcg_lru.fifo[gen][bin], list) { 5004 if (op) { 5005 lru_gen_rotate_memcg(lruvec, op); 5006 op = 0; 5007 } 5008 5009 mem_cgroup_put(memcg); 5010 memcg = NULL; 5011 5012 if (gen != READ_ONCE(lrugen->gen)) 5013 continue; 5014 5015 lruvec = container_of(lrugen, struct lruvec, lrugen); 5016 memcg = lruvec_memcg(lruvec); 5017 5018 if (!mem_cgroup_tryget(memcg)) { 5019 lru_gen_release_memcg(memcg); 5020 memcg = NULL; 5021 continue; 5022 } 5023 5024 rcu_read_unlock(); 5025 5026 op = shrink_one(lruvec, sc); 5027 5028 rcu_read_lock(); 5029 5030 if (should_abort_scan(lruvec, sc)) 5031 break; 5032 } 5033 5034 rcu_read_unlock(); 5035 5036 if (op) 5037 lru_gen_rotate_memcg(lruvec, op); 5038 5039 mem_cgroup_put(memcg); 5040 5041 if (!is_a_nulls(pos)) 5042 return; 5043 5044 /* restart if raced with lru_gen_rotate_memcg() */ 5045 if (gen != get_nulls_value(pos)) 5046 goto restart; 5047 5048 /* try the rest of the bins of the current generation */ 5049 bin = get_memcg_bin(bin + 1); 5050 if (bin != first_bin) 5051 goto restart; 5052 } 5053 5054 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5055 { 5056 struct blk_plug plug; 5057 5058 VM_WARN_ON_ONCE(root_reclaim(sc)); 5059 VM_WARN_ON_ONCE(!sc->may_writepage || !sc->may_unmap); 5060 5061 lru_add_drain(); 5062 5063 blk_start_plug(&plug); 5064 5065 set_mm_walk(NULL, sc->proactive); 5066 5067 if (try_to_shrink_lruvec(lruvec, sc)) 5068 lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG); 5069 5070 clear_mm_walk(); 5071 5072 blk_finish_plug(&plug); 5073 } 5074 5075 static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) 5076 { 5077 struct blk_plug plug; 5078 unsigned long reclaimed = sc->nr_reclaimed; 5079 5080 VM_WARN_ON_ONCE(!root_reclaim(sc)); 5081 5082 /* 5083 * Unmapped clean folios are already prioritized. Scanning for more of 5084 * them is likely futile and can cause high reclaim latency when there 5085 * is a large number of memcgs. 5086 */ 5087 if (!sc->may_writepage || !sc->may_unmap) 5088 goto done; 5089 5090 lru_add_drain(); 5091 5092 blk_start_plug(&plug); 5093 5094 set_mm_walk(pgdat, sc->proactive); 5095 5096 set_initial_priority(pgdat, sc); 5097 5098 if (current_is_kswapd()) 5099 sc->nr_reclaimed = 0; 5100 5101 if (mem_cgroup_disabled()) 5102 shrink_one(&pgdat->__lruvec, sc); 5103 else 5104 shrink_many(pgdat, sc); 5105 5106 if (current_is_kswapd()) 5107 sc->nr_reclaimed += reclaimed; 5108 5109 clear_mm_walk(); 5110 5111 blk_finish_plug(&plug); 5112 done: 5113 if (sc->nr_reclaimed > reclaimed) 5114 atomic_set(&pgdat->kswapd_failures, 0); 5115 } 5116 5117 /****************************************************************************** 5118 * state change 5119 ******************************************************************************/ 5120 5121 static bool __maybe_unused state_is_valid(struct lruvec *lruvec) 5122 { 5123 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5124 5125 if (lrugen->enabled) { 5126 enum lru_list lru; 5127 5128 for_each_evictable_lru(lru) { 5129 if (!list_empty(&lruvec->lists[lru])) 5130 return false; 5131 } 5132 } else { 5133 int gen, type, zone; 5134 5135 for_each_gen_type_zone(gen, type, zone) { 5136 if (!list_empty(&lrugen->folios[gen][type][zone])) 5137 return false; 5138 } 5139 } 5140 5141 return true; 5142 } 5143 5144 static bool fill_evictable(struct lruvec *lruvec) 5145 { 5146 enum lru_list lru; 5147 int remaining = MAX_LRU_BATCH; 5148 5149 for_each_evictable_lru(lru) { 5150 int type = is_file_lru(lru); 5151 bool active = is_active_lru(lru); 5152 struct list_head *head = &lruvec->lists[lru]; 5153 5154 while (!list_empty(head)) { 5155 bool success; 5156 struct folio *folio = lru_to_folio(head); 5157 5158 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5159 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio) != active, folio); 5160 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5161 VM_WARN_ON_ONCE_FOLIO(folio_lru_gen(folio) != -1, folio); 5162 5163 lruvec_del_folio(lruvec, folio); 5164 success = lru_gen_add_folio(lruvec, folio, false); 5165 VM_WARN_ON_ONCE(!success); 5166 5167 if (!--remaining) 5168 return false; 5169 } 5170 } 5171 5172 return true; 5173 } 5174 5175 static bool drain_evictable(struct lruvec *lruvec) 5176 { 5177 int gen, type, zone; 5178 int remaining = MAX_LRU_BATCH; 5179 5180 for_each_gen_type_zone(gen, type, zone) { 5181 struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; 5182 5183 while (!list_empty(head)) { 5184 bool success; 5185 struct folio *folio = lru_to_folio(head); 5186 5187 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); 5188 VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); 5189 VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio); 5190 VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio); 5191 5192 success = lru_gen_del_folio(lruvec, folio, false); 5193 VM_WARN_ON_ONCE(!success); 5194 lruvec_add_folio(lruvec, folio); 5195 5196 if (!--remaining) 5197 return false; 5198 } 5199 } 5200 5201 return true; 5202 } 5203 5204 static void lru_gen_change_state(bool enabled) 5205 { 5206 static DEFINE_MUTEX(state_mutex); 5207 5208 struct mem_cgroup *memcg; 5209 5210 cgroup_lock(); 5211 cpus_read_lock(); 5212 get_online_mems(); 5213 mutex_lock(&state_mutex); 5214 5215 if (enabled == lru_gen_enabled()) 5216 goto unlock; 5217 5218 if (enabled) 5219 static_branch_enable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5220 else 5221 static_branch_disable_cpuslocked(&lru_gen_caps[LRU_GEN_CORE]); 5222 5223 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5224 do { 5225 int nid; 5226 5227 for_each_node(nid) { 5228 struct lruvec *lruvec = get_lruvec(memcg, nid); 5229 5230 spin_lock_irq(&lruvec->lru_lock); 5231 5232 VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); 5233 VM_WARN_ON_ONCE(!state_is_valid(lruvec)); 5234 5235 lruvec->lrugen.enabled = enabled; 5236 5237 while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) { 5238 spin_unlock_irq(&lruvec->lru_lock); 5239 cond_resched(); 5240 spin_lock_irq(&lruvec->lru_lock); 5241 } 5242 5243 spin_unlock_irq(&lruvec->lru_lock); 5244 } 5245 5246 cond_resched(); 5247 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5248 unlock: 5249 mutex_unlock(&state_mutex); 5250 put_online_mems(); 5251 cpus_read_unlock(); 5252 cgroup_unlock(); 5253 } 5254 5255 /****************************************************************************** 5256 * sysfs interface 5257 ******************************************************************************/ 5258 5259 static ssize_t min_ttl_ms_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5260 { 5261 return sysfs_emit(buf, "%u\n", jiffies_to_msecs(READ_ONCE(lru_gen_min_ttl))); 5262 } 5263 5264 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5265 static ssize_t min_ttl_ms_store(struct kobject *kobj, struct kobj_attribute *attr, 5266 const char *buf, size_t len) 5267 { 5268 unsigned int msecs; 5269 5270 if (kstrtouint(buf, 0, &msecs)) 5271 return -EINVAL; 5272 5273 WRITE_ONCE(lru_gen_min_ttl, msecs_to_jiffies(msecs)); 5274 5275 return len; 5276 } 5277 5278 static struct kobj_attribute lru_gen_min_ttl_attr = __ATTR_RW(min_ttl_ms); 5279 5280 static ssize_t enabled_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) 5281 { 5282 unsigned int caps = 0; 5283 5284 if (get_cap(LRU_GEN_CORE)) 5285 caps |= BIT(LRU_GEN_CORE); 5286 5287 if (should_walk_mmu()) 5288 caps |= BIT(LRU_GEN_MM_WALK); 5289 5290 if (should_clear_pmd_young()) 5291 caps |= BIT(LRU_GEN_NONLEAF_YOUNG); 5292 5293 return sysfs_emit(buf, "0x%04x\n", caps); 5294 } 5295 5296 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5297 static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, 5298 const char *buf, size_t len) 5299 { 5300 int i; 5301 unsigned int caps; 5302 5303 if (tolower(*buf) == 'n') 5304 caps = 0; 5305 else if (tolower(*buf) == 'y') 5306 caps = -1; 5307 else if (kstrtouint(buf, 0, &caps)) 5308 return -EINVAL; 5309 5310 for (i = 0; i < NR_LRU_GEN_CAPS; i++) { 5311 bool enabled = caps & BIT(i); 5312 5313 if (i == LRU_GEN_CORE) 5314 lru_gen_change_state(enabled); 5315 else if (enabled) 5316 static_branch_enable(&lru_gen_caps[i]); 5317 else 5318 static_branch_disable(&lru_gen_caps[i]); 5319 } 5320 5321 return len; 5322 } 5323 5324 static struct kobj_attribute lru_gen_enabled_attr = __ATTR_RW(enabled); 5325 5326 static struct attribute *lru_gen_attrs[] = { 5327 &lru_gen_min_ttl_attr.attr, 5328 &lru_gen_enabled_attr.attr, 5329 NULL 5330 }; 5331 5332 static const struct attribute_group lru_gen_attr_group = { 5333 .name = "lru_gen", 5334 .attrs = lru_gen_attrs, 5335 }; 5336 5337 /****************************************************************************** 5338 * debugfs interface 5339 ******************************************************************************/ 5340 5341 static void *lru_gen_seq_start(struct seq_file *m, loff_t *pos) 5342 { 5343 struct mem_cgroup *memcg; 5344 loff_t nr_to_skip = *pos; 5345 5346 m->private = kvmalloc(PATH_MAX, GFP_KERNEL); 5347 if (!m->private) 5348 return ERR_PTR(-ENOMEM); 5349 5350 memcg = mem_cgroup_iter(NULL, NULL, NULL); 5351 do { 5352 int nid; 5353 5354 for_each_node_state(nid, N_MEMORY) { 5355 if (!nr_to_skip--) 5356 return get_lruvec(memcg, nid); 5357 } 5358 } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); 5359 5360 return NULL; 5361 } 5362 5363 static void lru_gen_seq_stop(struct seq_file *m, void *v) 5364 { 5365 if (!IS_ERR_OR_NULL(v)) 5366 mem_cgroup_iter_break(NULL, lruvec_memcg(v)); 5367 5368 kvfree(m->private); 5369 m->private = NULL; 5370 } 5371 5372 static void *lru_gen_seq_next(struct seq_file *m, void *v, loff_t *pos) 5373 { 5374 int nid = lruvec_pgdat(v)->node_id; 5375 struct mem_cgroup *memcg = lruvec_memcg(v); 5376 5377 ++*pos; 5378 5379 nid = next_memory_node(nid); 5380 if (nid == MAX_NUMNODES) { 5381 memcg = mem_cgroup_iter(NULL, memcg, NULL); 5382 if (!memcg) 5383 return NULL; 5384 5385 nid = first_memory_node; 5386 } 5387 5388 return get_lruvec(memcg, nid); 5389 } 5390 5391 static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec, 5392 unsigned long max_seq, unsigned long *min_seq, 5393 unsigned long seq) 5394 { 5395 int i; 5396 int type, tier; 5397 int hist = lru_hist_from_seq(seq); 5398 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5399 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5400 5401 for (tier = 0; tier < MAX_NR_TIERS; tier++) { 5402 seq_printf(m, " %10d", tier); 5403 for (type = 0; type < ANON_AND_FILE; type++) { 5404 const char *s = "xxx"; 5405 unsigned long n[3] = {}; 5406 5407 if (seq == max_seq) { 5408 s = "RTx"; 5409 n[0] = READ_ONCE(lrugen->avg_refaulted[type][tier]); 5410 n[1] = READ_ONCE(lrugen->avg_total[type][tier]); 5411 } else if (seq == min_seq[type] || NR_HIST_GENS > 1) { 5412 s = "rep"; 5413 n[0] = atomic_long_read(&lrugen->refaulted[hist][type][tier]); 5414 n[1] = atomic_long_read(&lrugen->evicted[hist][type][tier]); 5415 n[2] = READ_ONCE(lrugen->protected[hist][type][tier]); 5416 } 5417 5418 for (i = 0; i < 3; i++) 5419 seq_printf(m, " %10lu%c", n[i], s[i]); 5420 } 5421 seq_putc(m, '\n'); 5422 } 5423 5424 if (!mm_state) 5425 return; 5426 5427 seq_puts(m, " "); 5428 for (i = 0; i < NR_MM_STATS; i++) { 5429 const char *s = "xxxx"; 5430 unsigned long n = 0; 5431 5432 if (seq == max_seq && NR_HIST_GENS == 1) { 5433 s = "TYFA"; 5434 n = READ_ONCE(mm_state->stats[hist][i]); 5435 } else if (seq != max_seq && NR_HIST_GENS > 1) { 5436 s = "tyfa"; 5437 n = READ_ONCE(mm_state->stats[hist][i]); 5438 } 5439 5440 seq_printf(m, " %10lu%c", n, s[i]); 5441 } 5442 seq_putc(m, '\n'); 5443 } 5444 5445 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5446 static int lru_gen_seq_show(struct seq_file *m, void *v) 5447 { 5448 unsigned long seq; 5449 bool full = debugfs_get_aux_num(m->file); 5450 struct lruvec *lruvec = v; 5451 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5452 int nid = lruvec_pgdat(lruvec)->node_id; 5453 struct mem_cgroup *memcg = lruvec_memcg(lruvec); 5454 DEFINE_MAX_SEQ(lruvec); 5455 DEFINE_MIN_SEQ(lruvec); 5456 5457 if (nid == first_memory_node) { 5458 const char *path = memcg ? m->private : ""; 5459 5460 #ifdef CONFIG_MEMCG 5461 if (memcg) 5462 cgroup_path(memcg->css.cgroup, m->private, PATH_MAX); 5463 #endif 5464 seq_printf(m, "memcg %5hu %s\n", mem_cgroup_id(memcg), path); 5465 } 5466 5467 seq_printf(m, " node %5d\n", nid); 5468 5469 if (!full) 5470 seq = evictable_min_seq(min_seq, MAX_SWAPPINESS / 2); 5471 else if (max_seq >= MAX_NR_GENS) 5472 seq = max_seq - MAX_NR_GENS + 1; 5473 else 5474 seq = 0; 5475 5476 for (; seq <= max_seq; seq++) { 5477 int type, zone; 5478 int gen = lru_gen_from_seq(seq); 5479 unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]); 5480 5481 seq_printf(m, " %10lu %10u", seq, jiffies_to_msecs(jiffies - birth)); 5482 5483 for (type = 0; type < ANON_AND_FILE; type++) { 5484 unsigned long size = 0; 5485 char mark = full && seq < min_seq[type] ? 'x' : ' '; 5486 5487 for (zone = 0; zone < MAX_NR_ZONES; zone++) 5488 size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L); 5489 5490 seq_printf(m, " %10lu%c", size, mark); 5491 } 5492 5493 seq_putc(m, '\n'); 5494 5495 if (full) 5496 lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq); 5497 } 5498 5499 return 0; 5500 } 5501 5502 static const struct seq_operations lru_gen_seq_ops = { 5503 .start = lru_gen_seq_start, 5504 .stop = lru_gen_seq_stop, 5505 .next = lru_gen_seq_next, 5506 .show = lru_gen_seq_show, 5507 }; 5508 5509 static int run_aging(struct lruvec *lruvec, unsigned long seq, 5510 int swappiness, bool force_scan) 5511 { 5512 DEFINE_MAX_SEQ(lruvec); 5513 5514 if (seq > max_seq) 5515 return -EINVAL; 5516 5517 return try_to_inc_max_seq(lruvec, max_seq, swappiness, force_scan) ? 0 : -EEXIST; 5518 } 5519 5520 static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc, 5521 int swappiness, unsigned long nr_to_reclaim) 5522 { 5523 DEFINE_MAX_SEQ(lruvec); 5524 5525 if (seq + MIN_NR_GENS > max_seq) 5526 return -EINVAL; 5527 5528 sc->nr_reclaimed = 0; 5529 5530 while (!signal_pending(current)) { 5531 DEFINE_MIN_SEQ(lruvec); 5532 5533 if (seq < evictable_min_seq(min_seq, swappiness)) 5534 return 0; 5535 5536 if (sc->nr_reclaimed >= nr_to_reclaim) 5537 return 0; 5538 5539 if (!evict_folios(nr_to_reclaim - sc->nr_reclaimed, lruvec, sc, 5540 swappiness)) 5541 return 0; 5542 5543 cond_resched(); 5544 } 5545 5546 return -EINTR; 5547 } 5548 5549 static int run_cmd(char cmd, int memcg_id, int nid, unsigned long seq, 5550 struct scan_control *sc, int swappiness, unsigned long opt) 5551 { 5552 struct lruvec *lruvec; 5553 int err = -EINVAL; 5554 struct mem_cgroup *memcg = NULL; 5555 5556 if (nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY)) 5557 return -EINVAL; 5558 5559 if (!mem_cgroup_disabled()) { 5560 rcu_read_lock(); 5561 5562 memcg = mem_cgroup_from_id(memcg_id); 5563 if (!mem_cgroup_tryget(memcg)) 5564 memcg = NULL; 5565 5566 rcu_read_unlock(); 5567 5568 if (!memcg) 5569 return -EINVAL; 5570 } 5571 5572 if (memcg_id != mem_cgroup_id(memcg)) 5573 goto done; 5574 5575 sc->target_mem_cgroup = memcg; 5576 lruvec = get_lruvec(memcg, nid); 5577 5578 if (swappiness < MIN_SWAPPINESS) 5579 swappiness = get_swappiness(lruvec, sc); 5580 else if (swappiness > SWAPPINESS_ANON_ONLY) 5581 goto done; 5582 5583 switch (cmd) { 5584 case '+': 5585 err = run_aging(lruvec, seq, swappiness, opt); 5586 break; 5587 case '-': 5588 err = run_eviction(lruvec, seq, sc, swappiness, opt); 5589 break; 5590 } 5591 done: 5592 mem_cgroup_put(memcg); 5593 5594 return err; 5595 } 5596 5597 /* see Documentation/admin-guide/mm/multigen_lru.rst for details */ 5598 static ssize_t lru_gen_seq_write(struct file *file, const char __user *src, 5599 size_t len, loff_t *pos) 5600 { 5601 void *buf; 5602 char *cur, *next; 5603 unsigned int flags; 5604 struct blk_plug plug; 5605 int err = -EINVAL; 5606 struct scan_control sc = { 5607 .may_writepage = true, 5608 .may_unmap = true, 5609 .may_swap = true, 5610 .reclaim_idx = MAX_NR_ZONES - 1, 5611 .gfp_mask = GFP_KERNEL, 5612 .proactive = true, 5613 }; 5614 5615 buf = kvmalloc(len + 1, GFP_KERNEL); 5616 if (!buf) 5617 return -ENOMEM; 5618 5619 if (copy_from_user(buf, src, len)) { 5620 kvfree(buf); 5621 return -EFAULT; 5622 } 5623 5624 set_task_reclaim_state(current, &sc.reclaim_state); 5625 flags = memalloc_noreclaim_save(); 5626 blk_start_plug(&plug); 5627 if (!set_mm_walk(NULL, true)) { 5628 err = -ENOMEM; 5629 goto done; 5630 } 5631 5632 next = buf; 5633 next[len] = '\0'; 5634 5635 while ((cur = strsep(&next, ",;\n"))) { 5636 int n; 5637 int end; 5638 char cmd, swap_string[5]; 5639 unsigned int memcg_id; 5640 unsigned int nid; 5641 unsigned long seq; 5642 unsigned int swappiness; 5643 unsigned long opt = -1; 5644 5645 cur = skip_spaces(cur); 5646 if (!*cur) 5647 continue; 5648 5649 n = sscanf(cur, "%c %u %u %lu %n %4s %n %lu %n", &cmd, &memcg_id, &nid, 5650 &seq, &end, swap_string, &end, &opt, &end); 5651 if (n < 4 || cur[end]) { 5652 err = -EINVAL; 5653 break; 5654 } 5655 5656 if (n == 4) { 5657 swappiness = -1; 5658 } else if (!strcmp("max", swap_string)) { 5659 /* set by userspace for anonymous memory only */ 5660 swappiness = SWAPPINESS_ANON_ONLY; 5661 } else { 5662 err = kstrtouint(swap_string, 0, &swappiness); 5663 if (err) 5664 break; 5665 } 5666 5667 err = run_cmd(cmd, memcg_id, nid, seq, &sc, swappiness, opt); 5668 if (err) 5669 break; 5670 } 5671 done: 5672 clear_mm_walk(); 5673 blk_finish_plug(&plug); 5674 memalloc_noreclaim_restore(flags); 5675 set_task_reclaim_state(current, NULL); 5676 5677 kvfree(buf); 5678 5679 return err ? : len; 5680 } 5681 5682 static int lru_gen_seq_open(struct inode *inode, struct file *file) 5683 { 5684 return seq_open(file, &lru_gen_seq_ops); 5685 } 5686 5687 static const struct file_operations lru_gen_rw_fops = { 5688 .open = lru_gen_seq_open, 5689 .read = seq_read, 5690 .write = lru_gen_seq_write, 5691 .llseek = seq_lseek, 5692 .release = seq_release, 5693 }; 5694 5695 static const struct file_operations lru_gen_ro_fops = { 5696 .open = lru_gen_seq_open, 5697 .read = seq_read, 5698 .llseek = seq_lseek, 5699 .release = seq_release, 5700 }; 5701 5702 /****************************************************************************** 5703 * initialization 5704 ******************************************************************************/ 5705 5706 void lru_gen_init_pgdat(struct pglist_data *pgdat) 5707 { 5708 int i, j; 5709 5710 spin_lock_init(&pgdat->memcg_lru.lock); 5711 5712 for (i = 0; i < MEMCG_NR_GENS; i++) { 5713 for (j = 0; j < MEMCG_NR_BINS; j++) 5714 INIT_HLIST_NULLS_HEAD(&pgdat->memcg_lru.fifo[i][j], i); 5715 } 5716 } 5717 5718 void lru_gen_init_lruvec(struct lruvec *lruvec) 5719 { 5720 int i; 5721 int gen, type, zone; 5722 struct lru_gen_folio *lrugen = &lruvec->lrugen; 5723 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5724 5725 lrugen->max_seq = MIN_NR_GENS + 1; 5726 lrugen->enabled = lru_gen_enabled(); 5727 5728 for (i = 0; i <= MIN_NR_GENS + 1; i++) 5729 lrugen->timestamps[i] = jiffies; 5730 5731 for_each_gen_type_zone(gen, type, zone) 5732 INIT_LIST_HEAD(&lrugen->folios[gen][type][zone]); 5733 5734 if (mm_state) 5735 mm_state->seq = MIN_NR_GENS; 5736 } 5737 5738 #ifdef CONFIG_MEMCG 5739 5740 void lru_gen_init_memcg(struct mem_cgroup *memcg) 5741 { 5742 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 5743 5744 if (!mm_list) 5745 return; 5746 5747 INIT_LIST_HEAD(&mm_list->fifo); 5748 spin_lock_init(&mm_list->lock); 5749 } 5750 5751 void lru_gen_exit_memcg(struct mem_cgroup *memcg) 5752 { 5753 int i; 5754 int nid; 5755 struct lru_gen_mm_list *mm_list = get_mm_list(memcg); 5756 5757 VM_WARN_ON_ONCE(mm_list && !list_empty(&mm_list->fifo)); 5758 5759 for_each_node(nid) { 5760 struct lruvec *lruvec = get_lruvec(memcg, nid); 5761 struct lru_gen_mm_state *mm_state = get_mm_state(lruvec); 5762 5763 VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, 5764 sizeof(lruvec->lrugen.nr_pages))); 5765 5766 lruvec->lrugen.list.next = LIST_POISON1; 5767 5768 if (!mm_state) 5769 continue; 5770 5771 for (i = 0; i < NR_BLOOM_FILTERS; i++) { 5772 bitmap_free(mm_state->filters[i]); 5773 mm_state->filters[i] = NULL; 5774 } 5775 } 5776 } 5777 5778 #endif /* CONFIG_MEMCG */ 5779 5780 static int __init init_lru_gen(void) 5781 { 5782 BUILD_BUG_ON(MIN_NR_GENS + 1 >= MAX_NR_GENS); 5783 BUILD_BUG_ON(BIT(LRU_GEN_WIDTH) <= MAX_NR_GENS); 5784 5785 if (sysfs_create_group(mm_kobj, &lru_gen_attr_group)) 5786 pr_err("lru_gen: failed to create sysfs group\n"); 5787 5788 debugfs_create_file_aux_num("lru_gen", 0644, NULL, NULL, false, 5789 &lru_gen_rw_fops); 5790 debugfs_create_file_aux_num("lru_gen_full", 0444, NULL, NULL, true, 5791 &lru_gen_ro_fops); 5792 5793 return 0; 5794 }; 5795 late_initcall(init_lru_gen); 5796 5797 #else /* !CONFIG_LRU_GEN */ 5798 5799 static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) 5800 { 5801 BUILD_BUG(); 5802 } 5803 5804 static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5805 { 5806 BUILD_BUG(); 5807 } 5808 5809 static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *sc) 5810 { 5811 BUILD_BUG(); 5812 } 5813 5814 #endif /* CONFIG_LRU_GEN */ 5815 5816 static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) 5817 { 5818 unsigned long nr[NR_LRU_LISTS]; 5819 unsigned long targets[NR_LRU_LISTS]; 5820 unsigned long nr_to_scan; 5821 enum lru_list lru; 5822 unsigned long nr_reclaimed = 0; 5823 unsigned long nr_to_reclaim = sc->nr_to_reclaim; 5824 bool proportional_reclaim; 5825 struct blk_plug plug; 5826 5827 if (lru_gen_enabled() && !root_reclaim(sc)) { 5828 lru_gen_shrink_lruvec(lruvec, sc); 5829 return; 5830 } 5831 5832 get_scan_count(lruvec, sc, nr); 5833 5834 /* Record the original scan target for proportional adjustments later */ 5835 memcpy(targets, nr, sizeof(nr)); 5836 5837 /* 5838 * Global reclaiming within direct reclaim at DEF_PRIORITY is a normal 5839 * event that can occur when there is little memory pressure e.g. 5840 * multiple streaming readers/writers. Hence, we do not abort scanning 5841 * when the requested number of pages are reclaimed when scanning at 5842 * DEF_PRIORITY on the assumption that the fact we are direct 5843 * reclaiming implies that kswapd is not keeping up and it is best to 5844 * do a batch of work at once. For memcg reclaim one check is made to 5845 * abort proportional reclaim if either the file or anon lru has already 5846 * dropped to zero at the first pass. 5847 */ 5848 proportional_reclaim = (!cgroup_reclaim(sc) && !current_is_kswapd() && 5849 sc->priority == DEF_PRIORITY); 5850 5851 blk_start_plug(&plug); 5852 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || 5853 nr[LRU_INACTIVE_FILE]) { 5854 unsigned long nr_anon, nr_file, percentage; 5855 unsigned long nr_scanned; 5856 5857 for_each_evictable_lru(lru) { 5858 if (nr[lru]) { 5859 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); 5860 nr[lru] -= nr_to_scan; 5861 5862 nr_reclaimed += shrink_list(lru, nr_to_scan, 5863 lruvec, sc); 5864 } 5865 } 5866 5867 cond_resched(); 5868 5869 if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) 5870 continue; 5871 5872 /* 5873 * For kswapd and memcg, reclaim at least the number of pages 5874 * requested. Ensure that the anon and file LRUs are scanned 5875 * proportionally what was requested by get_scan_count(). We 5876 * stop reclaiming one LRU and reduce the amount scanning 5877 * proportional to the original scan target. 5878 */ 5879 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE]; 5880 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON]; 5881 5882 /* 5883 * It's just vindictive to attack the larger once the smaller 5884 * has gone to zero. And given the way we stop scanning the 5885 * smaller below, this makes sure that we only make one nudge 5886 * towards proportionality once we've got nr_to_reclaim. 5887 */ 5888 if (!nr_file || !nr_anon) 5889 break; 5890 5891 if (nr_file > nr_anon) { 5892 unsigned long scan_target = targets[LRU_INACTIVE_ANON] + 5893 targets[LRU_ACTIVE_ANON] + 1; 5894 lru = LRU_BASE; 5895 percentage = nr_anon * 100 / scan_target; 5896 } else { 5897 unsigned long scan_target = targets[LRU_INACTIVE_FILE] + 5898 targets[LRU_ACTIVE_FILE] + 1; 5899 lru = LRU_FILE; 5900 percentage = nr_file * 100 / scan_target; 5901 } 5902 5903 /* Stop scanning the smaller of the LRU */ 5904 nr[lru] = 0; 5905 nr[lru + LRU_ACTIVE] = 0; 5906 5907 /* 5908 * Recalculate the other LRU scan count based on its original 5909 * scan target and the percentage scanning already complete 5910 */ 5911 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE; 5912 nr_scanned = targets[lru] - nr[lru]; 5913 nr[lru] = targets[lru] * (100 - percentage) / 100; 5914 nr[lru] -= min(nr[lru], nr_scanned); 5915 5916 lru += LRU_ACTIVE; 5917 nr_scanned = targets[lru] - nr[lru]; 5918 nr[lru] = targets[lru] * (100 - percentage) / 100; 5919 nr[lru] -= min(nr[lru], nr_scanned); 5920 } 5921 blk_finish_plug(&plug); 5922 sc->nr_reclaimed += nr_reclaimed; 5923 5924 /* 5925 * Even if we did not try to evict anon pages at all, we want to 5926 * rebalance the anon lru active/inactive ratio. 5927 */ 5928 if (can_age_anon_pages(lruvec, sc) && 5929 inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 5930 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 5931 sc, LRU_ACTIVE_ANON); 5932 } 5933 5934 /* Use reclaim/compaction for costly allocs or under memory pressure */ 5935 static bool in_reclaim_compaction(struct scan_control *sc) 5936 { 5937 if (gfp_compaction_allowed(sc->gfp_mask) && sc->order && 5938 (sc->order > PAGE_ALLOC_COSTLY_ORDER || 5939 sc->priority < DEF_PRIORITY - 2)) 5940 return true; 5941 5942 return false; 5943 } 5944 5945 /* 5946 * Reclaim/compaction is used for high-order allocation requests. It reclaims 5947 * order-0 pages before compacting the zone. should_continue_reclaim() returns 5948 * true if more pages should be reclaimed such that when the page allocator 5949 * calls try_to_compact_pages() that it will have enough free pages to succeed. 5950 * It will give up earlier than that if there is difficulty reclaiming pages. 5951 */ 5952 static inline bool should_continue_reclaim(struct pglist_data *pgdat, 5953 unsigned long nr_reclaimed, 5954 struct scan_control *sc) 5955 { 5956 unsigned long pages_for_compaction; 5957 unsigned long inactive_lru_pages; 5958 int z; 5959 struct zone *zone; 5960 5961 /* If not in reclaim/compaction mode, stop */ 5962 if (!in_reclaim_compaction(sc)) 5963 return false; 5964 5965 /* 5966 * Stop if we failed to reclaim any pages from the last SWAP_CLUSTER_MAX 5967 * number of pages that were scanned. This will return to the caller 5968 * with the risk reclaim/compaction and the resulting allocation attempt 5969 * fails. In the past we have tried harder for __GFP_RETRY_MAYFAIL 5970 * allocations through requiring that the full LRU list has been scanned 5971 * first, by assuming that zero delta of sc->nr_scanned means full LRU 5972 * scan, but that approximation was wrong, and there were corner cases 5973 * where always a non-zero amount of pages were scanned. 5974 */ 5975 if (!nr_reclaimed) 5976 return false; 5977 5978 /* If compaction would go ahead or the allocation would succeed, stop */ 5979 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) { 5980 unsigned long watermark = min_wmark_pages(zone); 5981 5982 /* Allocation can already succeed, nothing to do */ 5983 if (zone_watermark_ok(zone, sc->order, watermark, 5984 sc->reclaim_idx, 0)) 5985 return false; 5986 5987 if (compaction_suitable(zone, sc->order, watermark, 5988 sc->reclaim_idx)) 5989 return false; 5990 } 5991 5992 /* 5993 * If we have not reclaimed enough pages for compaction and the 5994 * inactive lists are large enough, continue reclaiming 5995 */ 5996 pages_for_compaction = compact_gap(sc->order); 5997 inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE); 5998 if (can_reclaim_anon_pages(NULL, pgdat->node_id, sc)) 5999 inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON); 6000 6001 return inactive_lru_pages > pages_for_compaction; 6002 } 6003 6004 static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc) 6005 { 6006 struct mem_cgroup *target_memcg = sc->target_mem_cgroup; 6007 struct mem_cgroup_reclaim_cookie reclaim = { 6008 .pgdat = pgdat, 6009 }; 6010 struct mem_cgroup_reclaim_cookie *partial = &reclaim; 6011 struct mem_cgroup *memcg; 6012 6013 /* 6014 * In most cases, direct reclaimers can do partial walks 6015 * through the cgroup tree, using an iterator state that 6016 * persists across invocations. This strikes a balance between 6017 * fairness and allocation latency. 6018 * 6019 * For kswapd, reliable forward progress is more important 6020 * than a quick return to idle. Always do full walks. 6021 */ 6022 if (current_is_kswapd() || sc->memcg_full_walk) 6023 partial = NULL; 6024 6025 memcg = mem_cgroup_iter(target_memcg, NULL, partial); 6026 do { 6027 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6028 unsigned long reclaimed; 6029 unsigned long scanned; 6030 6031 /* 6032 * This loop can become CPU-bound when target memcgs 6033 * aren't eligible for reclaim - either because they 6034 * don't have any reclaimable pages, or because their 6035 * memory is explicitly protected. Avoid soft lockups. 6036 */ 6037 cond_resched(); 6038 6039 mem_cgroup_calculate_protection(target_memcg, memcg); 6040 6041 if (mem_cgroup_below_min(target_memcg, memcg)) { 6042 /* 6043 * Hard protection. 6044 * If there is no reclaimable memory, OOM. 6045 */ 6046 continue; 6047 } else if (mem_cgroup_below_low(target_memcg, memcg)) { 6048 /* 6049 * Soft protection. 6050 * Respect the protection only as long as 6051 * there is an unprotected supply 6052 * of reclaimable memory from other cgroups. 6053 */ 6054 if (!sc->memcg_low_reclaim) { 6055 sc->memcg_low_skipped = 1; 6056 continue; 6057 } 6058 memcg_memory_event(memcg, MEMCG_LOW); 6059 } 6060 6061 reclaimed = sc->nr_reclaimed; 6062 scanned = sc->nr_scanned; 6063 6064 shrink_lruvec(lruvec, sc); 6065 6066 shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, 6067 sc->priority); 6068 6069 /* Record the group's reclaim efficiency */ 6070 if (!sc->proactive) 6071 vmpressure(sc->gfp_mask, memcg, false, 6072 sc->nr_scanned - scanned, 6073 sc->nr_reclaimed - reclaimed); 6074 6075 /* If partial walks are allowed, bail once goal is reached */ 6076 if (partial && sc->nr_reclaimed >= sc->nr_to_reclaim) { 6077 mem_cgroup_iter_break(target_memcg, memcg); 6078 break; 6079 } 6080 } while ((memcg = mem_cgroup_iter(target_memcg, memcg, partial))); 6081 } 6082 6083 static void shrink_node(pg_data_t *pgdat, struct scan_control *sc) 6084 { 6085 unsigned long nr_reclaimed, nr_scanned, nr_node_reclaimed; 6086 struct lruvec *target_lruvec; 6087 bool reclaimable = false; 6088 6089 if (lru_gen_enabled() && root_reclaim(sc)) { 6090 memset(&sc->nr, 0, sizeof(sc->nr)); 6091 lru_gen_shrink_node(pgdat, sc); 6092 return; 6093 } 6094 6095 target_lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, pgdat); 6096 6097 again: 6098 memset(&sc->nr, 0, sizeof(sc->nr)); 6099 6100 nr_reclaimed = sc->nr_reclaimed; 6101 nr_scanned = sc->nr_scanned; 6102 6103 prepare_scan_control(pgdat, sc); 6104 6105 shrink_node_memcgs(pgdat, sc); 6106 6107 flush_reclaim_state(sc); 6108 6109 nr_node_reclaimed = sc->nr_reclaimed - nr_reclaimed; 6110 6111 /* Record the subtree's reclaim efficiency */ 6112 if (!sc->proactive) 6113 vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true, 6114 sc->nr_scanned - nr_scanned, nr_node_reclaimed); 6115 6116 if (nr_node_reclaimed) 6117 reclaimable = true; 6118 6119 if (current_is_kswapd()) { 6120 /* 6121 * If reclaim is isolating dirty pages under writeback, 6122 * it implies that the long-lived page allocation rate 6123 * is exceeding the page laundering rate. Either the 6124 * global limits are not being effective at throttling 6125 * processes due to the page distribution throughout 6126 * zones or there is heavy usage of a slow backing 6127 * device. The only option is to throttle from reclaim 6128 * context which is not ideal as there is no guarantee 6129 * the dirtying process is throttled in the same way 6130 * balance_dirty_pages() manages. 6131 * 6132 * Once a node is flagged PGDAT_WRITEBACK, kswapd will 6133 * count the number of pages under pages flagged for 6134 * immediate reclaim and stall if any are encountered 6135 * in the nr_immediate check below. 6136 */ 6137 if (sc->nr.writeback && sc->nr.writeback == sc->nr.taken) 6138 set_bit(PGDAT_WRITEBACK, &pgdat->flags); 6139 6140 /* Allow kswapd to start writing pages during reclaim.*/ 6141 if (sc->nr.unqueued_dirty && 6142 sc->nr.unqueued_dirty == sc->nr.file_taken) 6143 set_bit(PGDAT_DIRTY, &pgdat->flags); 6144 6145 /* 6146 * If kswapd scans pages marked for immediate 6147 * reclaim and under writeback (nr_immediate), it 6148 * implies that pages are cycling through the LRU 6149 * faster than they are written so forcibly stall 6150 * until some pages complete writeback. 6151 */ 6152 if (sc->nr.immediate) 6153 reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK); 6154 } 6155 6156 /* 6157 * Tag a node/memcg as congested if all the dirty pages were marked 6158 * for writeback and immediate reclaim (counted in nr.congested). 6159 * 6160 * Legacy memcg will stall in page writeback so avoid forcibly 6161 * stalling in reclaim_throttle(). 6162 */ 6163 if (sc->nr.dirty && sc->nr.dirty == sc->nr.congested) { 6164 if (cgroup_reclaim(sc) && writeback_throttling_sane(sc)) 6165 set_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags); 6166 6167 if (current_is_kswapd()) 6168 set_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags); 6169 } 6170 6171 /* 6172 * Stall direct reclaim for IO completions if the lruvec is 6173 * node is congested. Allow kswapd to continue until it 6174 * starts encountering unqueued dirty pages or cycling through 6175 * the LRU too quickly. 6176 */ 6177 if (!current_is_kswapd() && current_may_throttle() && 6178 !sc->hibernation_mode && 6179 (test_bit(LRUVEC_CGROUP_CONGESTED, &target_lruvec->flags) || 6180 test_bit(LRUVEC_NODE_CONGESTED, &target_lruvec->flags))) 6181 reclaim_throttle(pgdat, VMSCAN_THROTTLE_CONGESTED); 6182 6183 if (should_continue_reclaim(pgdat, nr_node_reclaimed, sc)) 6184 goto again; 6185 6186 /* 6187 * Kswapd gives up on balancing particular nodes after too 6188 * many failures to reclaim anything from them and goes to 6189 * sleep. On reclaim progress, reset the failure counter. A 6190 * successful direct reclaim run will revive a dormant kswapd. 6191 */ 6192 if (reclaimable) 6193 atomic_set(&pgdat->kswapd_failures, 0); 6194 else if (sc->cache_trim_mode) 6195 sc->cache_trim_mode_failed = 1; 6196 } 6197 6198 /* 6199 * Returns true if compaction should go ahead for a costly-order request, or 6200 * the allocation would already succeed without compaction. Return false if we 6201 * should reclaim first. 6202 */ 6203 static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) 6204 { 6205 unsigned long watermark; 6206 6207 if (!gfp_compaction_allowed(sc->gfp_mask)) 6208 return false; 6209 6210 /* Allocation can already succeed, nothing to do */ 6211 if (zone_watermark_ok(zone, sc->order, min_wmark_pages(zone), 6212 sc->reclaim_idx, 0)) 6213 return true; 6214 6215 /* 6216 * Direct reclaim usually targets the min watermark, but compaction 6217 * takes time to run and there are potentially other callers using the 6218 * pages just freed. So target a higher buffer to give compaction a 6219 * reasonable chance of completing and allocating the pages. 6220 * 6221 * Note that we won't actually reclaim the whole buffer in one attempt 6222 * as the target watermark in should_continue_reclaim() is lower. But if 6223 * we are already above the high+gap watermark, don't reclaim at all. 6224 */ 6225 watermark = high_wmark_pages(zone); 6226 if (compaction_suitable(zone, sc->order, watermark, sc->reclaim_idx)) 6227 return true; 6228 6229 return false; 6230 } 6231 6232 static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc) 6233 { 6234 /* 6235 * If reclaim is making progress greater than 12% efficiency then 6236 * wake all the NOPROGRESS throttled tasks. 6237 */ 6238 if (sc->nr_reclaimed > (sc->nr_scanned >> 3)) { 6239 wait_queue_head_t *wqh; 6240 6241 wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_NOPROGRESS]; 6242 if (waitqueue_active(wqh)) 6243 wake_up(wqh); 6244 6245 return; 6246 } 6247 6248 /* 6249 * Do not throttle kswapd or cgroup reclaim on NOPROGRESS as it will 6250 * throttle on VMSCAN_THROTTLE_WRITEBACK if there are too many pages 6251 * under writeback and marked for immediate reclaim at the tail of the 6252 * LRU. 6253 */ 6254 if (current_is_kswapd() || cgroup_reclaim(sc)) 6255 return; 6256 6257 /* Throttle if making no progress at high prioities. */ 6258 if (sc->priority == 1 && !sc->nr_reclaimed) 6259 reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS); 6260 } 6261 6262 /* 6263 * This is the direct reclaim path, for page-allocating processes. We only 6264 * try to reclaim pages from zones which will satisfy the caller's allocation 6265 * request. 6266 * 6267 * If a zone is deemed to be full of pinned pages then just give it a light 6268 * scan then give up on it. 6269 */ 6270 static void shrink_zones(struct zonelist *zonelist, struct scan_control *sc) 6271 { 6272 struct zoneref *z; 6273 struct zone *zone; 6274 unsigned long nr_soft_reclaimed; 6275 unsigned long nr_soft_scanned; 6276 gfp_t orig_mask; 6277 pg_data_t *last_pgdat = NULL; 6278 pg_data_t *first_pgdat = NULL; 6279 6280 /* 6281 * If the number of buffer_heads in the machine exceeds the maximum 6282 * allowed level, force direct reclaim to scan the highmem zone as 6283 * highmem pages could be pinning lowmem pages storing buffer_heads 6284 */ 6285 orig_mask = sc->gfp_mask; 6286 if (buffer_heads_over_limit) { 6287 sc->gfp_mask |= __GFP_HIGHMEM; 6288 sc->reclaim_idx = gfp_zone(sc->gfp_mask); 6289 } 6290 6291 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6292 sc->reclaim_idx, sc->nodemask) { 6293 /* 6294 * Take care memory controller reclaiming has small influence 6295 * to global LRU. 6296 */ 6297 if (!cgroup_reclaim(sc)) { 6298 if (!cpuset_zone_allowed(zone, 6299 GFP_KERNEL | __GFP_HARDWALL)) 6300 continue; 6301 6302 /* 6303 * If we already have plenty of memory free for 6304 * compaction in this zone, don't free any more. 6305 * Even though compaction is invoked for any 6306 * non-zero order, only frequent costly order 6307 * reclamation is disruptive enough to become a 6308 * noticeable problem, like transparent huge 6309 * page allocations. 6310 */ 6311 if (IS_ENABLED(CONFIG_COMPACTION) && 6312 sc->order > PAGE_ALLOC_COSTLY_ORDER && 6313 compaction_ready(zone, sc)) { 6314 sc->compaction_ready = true; 6315 continue; 6316 } 6317 6318 /* 6319 * Shrink each node in the zonelist once. If the 6320 * zonelist is ordered by zone (not the default) then a 6321 * node may be shrunk multiple times but in that case 6322 * the user prefers lower zones being preserved. 6323 */ 6324 if (zone->zone_pgdat == last_pgdat) 6325 continue; 6326 6327 /* 6328 * This steals pages from memory cgroups over softlimit 6329 * and returns the number of reclaimed pages and 6330 * scanned pages. This works for global memory pressure 6331 * and balancing, not for a memcg's limit. 6332 */ 6333 nr_soft_scanned = 0; 6334 nr_soft_reclaimed = memcg1_soft_limit_reclaim(zone->zone_pgdat, 6335 sc->order, sc->gfp_mask, 6336 &nr_soft_scanned); 6337 sc->nr_reclaimed += nr_soft_reclaimed; 6338 sc->nr_scanned += nr_soft_scanned; 6339 /* need some check for avoid more shrink_zone() */ 6340 } 6341 6342 if (!first_pgdat) 6343 first_pgdat = zone->zone_pgdat; 6344 6345 /* See comment about same check for global reclaim above */ 6346 if (zone->zone_pgdat == last_pgdat) 6347 continue; 6348 last_pgdat = zone->zone_pgdat; 6349 shrink_node(zone->zone_pgdat, sc); 6350 } 6351 6352 if (first_pgdat) 6353 consider_reclaim_throttle(first_pgdat, sc); 6354 6355 /* 6356 * Restore to original mask to avoid the impact on the caller if we 6357 * promoted it to __GFP_HIGHMEM. 6358 */ 6359 sc->gfp_mask = orig_mask; 6360 } 6361 6362 static void snapshot_refaults(struct mem_cgroup *target_memcg, pg_data_t *pgdat) 6363 { 6364 struct lruvec *target_lruvec; 6365 unsigned long refaults; 6366 6367 if (lru_gen_enabled()) 6368 return; 6369 6370 target_lruvec = mem_cgroup_lruvec(target_memcg, pgdat); 6371 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_ANON); 6372 target_lruvec->refaults[WORKINGSET_ANON] = refaults; 6373 refaults = lruvec_page_state(target_lruvec, WORKINGSET_ACTIVATE_FILE); 6374 target_lruvec->refaults[WORKINGSET_FILE] = refaults; 6375 } 6376 6377 /* 6378 * This is the main entry point to direct page reclaim. 6379 * 6380 * If a full scan of the inactive list fails to free enough memory then we 6381 * are "out of memory" and something needs to be killed. 6382 * 6383 * If the caller is !__GFP_FS then the probability of a failure is reasonably 6384 * high - the zone may be full of dirty or under-writeback pages, which this 6385 * caller can't do much about. We kick the writeback threads and take explicit 6386 * naps in the hope that some of these pages can be written. But if the 6387 * allocating task holds filesystem locks which prevent writeout this might not 6388 * work, and the allocation attempt will fail. 6389 * 6390 * returns: 0, if no pages reclaimed 6391 * else, the number of pages reclaimed 6392 */ 6393 static unsigned long do_try_to_free_pages(struct zonelist *zonelist, 6394 struct scan_control *sc) 6395 { 6396 int initial_priority = sc->priority; 6397 pg_data_t *last_pgdat; 6398 struct zoneref *z; 6399 struct zone *zone; 6400 retry: 6401 delayacct_freepages_start(); 6402 6403 if (!cgroup_reclaim(sc)) 6404 __count_zid_vm_events(ALLOCSTALL, sc->reclaim_idx, 1); 6405 6406 do { 6407 if (!sc->proactive) 6408 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, 6409 sc->priority); 6410 sc->nr_scanned = 0; 6411 shrink_zones(zonelist, sc); 6412 6413 if (sc->nr_reclaimed >= sc->nr_to_reclaim) 6414 break; 6415 6416 if (sc->compaction_ready) 6417 break; 6418 6419 /* 6420 * If we're getting trouble reclaiming, start doing 6421 * writepage even in laptop mode. 6422 */ 6423 if (sc->priority < DEF_PRIORITY - 2) 6424 sc->may_writepage = 1; 6425 } while (--sc->priority >= 0); 6426 6427 last_pgdat = NULL; 6428 for_each_zone_zonelist_nodemask(zone, z, zonelist, sc->reclaim_idx, 6429 sc->nodemask) { 6430 if (zone->zone_pgdat == last_pgdat) 6431 continue; 6432 last_pgdat = zone->zone_pgdat; 6433 6434 snapshot_refaults(sc->target_mem_cgroup, zone->zone_pgdat); 6435 6436 if (cgroup_reclaim(sc)) { 6437 struct lruvec *lruvec; 6438 6439 lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup, 6440 zone->zone_pgdat); 6441 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); 6442 } 6443 } 6444 6445 delayacct_freepages_end(); 6446 6447 if (sc->nr_reclaimed) 6448 return sc->nr_reclaimed; 6449 6450 /* Aborted reclaim to try compaction? don't OOM, then */ 6451 if (sc->compaction_ready) 6452 return 1; 6453 6454 /* 6455 * In most cases, direct reclaimers can do partial walks 6456 * through the cgroup tree to meet the reclaim goal while 6457 * keeping latency low. Since the iterator state is shared 6458 * among all direct reclaim invocations (to retain fairness 6459 * among cgroups), though, high concurrency can result in 6460 * individual threads not seeing enough cgroups to make 6461 * meaningful forward progress. Avoid false OOMs in this case. 6462 */ 6463 if (!sc->memcg_full_walk) { 6464 sc->priority = initial_priority; 6465 sc->memcg_full_walk = 1; 6466 goto retry; 6467 } 6468 6469 /* 6470 * We make inactive:active ratio decisions based on the node's 6471 * composition of memory, but a restrictive reclaim_idx or a 6472 * memory.low cgroup setting can exempt large amounts of 6473 * memory from reclaim. Neither of which are very common, so 6474 * instead of doing costly eligibility calculations of the 6475 * entire cgroup subtree up front, we assume the estimates are 6476 * good, and retry with forcible deactivation if that fails. 6477 */ 6478 if (sc->skipped_deactivate) { 6479 sc->priority = initial_priority; 6480 sc->force_deactivate = 1; 6481 sc->skipped_deactivate = 0; 6482 goto retry; 6483 } 6484 6485 /* Untapped cgroup reserves? Don't OOM, retry. */ 6486 if (sc->memcg_low_skipped) { 6487 sc->priority = initial_priority; 6488 sc->force_deactivate = 0; 6489 sc->memcg_low_reclaim = 1; 6490 sc->memcg_low_skipped = 0; 6491 goto retry; 6492 } 6493 6494 return 0; 6495 } 6496 6497 static bool allow_direct_reclaim(pg_data_t *pgdat) 6498 { 6499 struct zone *zone; 6500 unsigned long pfmemalloc_reserve = 0; 6501 unsigned long free_pages = 0; 6502 int i; 6503 bool wmark_ok; 6504 6505 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES) 6506 return true; 6507 6508 for_each_managed_zone_pgdat(zone, pgdat, i, ZONE_NORMAL) { 6509 if (!zone_reclaimable_pages(zone) && zone_page_state_snapshot(zone, NR_FREE_PAGES)) 6510 continue; 6511 6512 pfmemalloc_reserve += min_wmark_pages(zone); 6513 free_pages += zone_page_state_snapshot(zone, NR_FREE_PAGES); 6514 } 6515 6516 /* If there are no reserves (unexpected config) then do not throttle */ 6517 if (!pfmemalloc_reserve) 6518 return true; 6519 6520 wmark_ok = free_pages > pfmemalloc_reserve / 2; 6521 6522 /* kswapd must be awake if processes are being throttled */ 6523 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) { 6524 if (READ_ONCE(pgdat->kswapd_highest_zoneidx) > ZONE_NORMAL) 6525 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, ZONE_NORMAL); 6526 6527 wake_up_interruptible(&pgdat->kswapd_wait); 6528 } 6529 6530 return wmark_ok; 6531 } 6532 6533 /* 6534 * Throttle direct reclaimers if backing storage is backed by the network 6535 * and the PFMEMALLOC reserve for the preferred node is getting dangerously 6536 * depleted. kswapd will continue to make progress and wake the processes 6537 * when the low watermark is reached. 6538 * 6539 * Returns true if a fatal signal was delivered during throttling. If this 6540 * happens, the page allocator should not consider triggering the OOM killer. 6541 */ 6542 static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist, 6543 nodemask_t *nodemask) 6544 { 6545 struct zoneref *z; 6546 struct zone *zone; 6547 pg_data_t *pgdat = NULL; 6548 6549 /* 6550 * Kernel threads should not be throttled as they may be indirectly 6551 * responsible for cleaning pages necessary for reclaim to make forward 6552 * progress. kjournald for example may enter direct reclaim while 6553 * committing a transaction where throttling it could forcing other 6554 * processes to block on log_wait_commit(). 6555 */ 6556 if (current->flags & PF_KTHREAD) 6557 goto out; 6558 6559 /* 6560 * If a fatal signal is pending, this process should not throttle. 6561 * It should return quickly so it can exit and free its memory 6562 */ 6563 if (fatal_signal_pending(current)) 6564 goto out; 6565 6566 /* 6567 * Check if the pfmemalloc reserves are ok by finding the first node 6568 * with a usable ZONE_NORMAL or lower zone. The expectation is that 6569 * GFP_KERNEL will be required for allocating network buffers when 6570 * swapping over the network so ZONE_HIGHMEM is unusable. 6571 * 6572 * Throttling is based on the first usable node and throttled processes 6573 * wait on a queue until kswapd makes progress and wakes them. There 6574 * is an affinity then between processes waking up and where reclaim 6575 * progress has been made assuming the process wakes on the same node. 6576 * More importantly, processes running on remote nodes will not compete 6577 * for remote pfmemalloc reserves and processes on different nodes 6578 * should make reasonable progress. 6579 */ 6580 for_each_zone_zonelist_nodemask(zone, z, zonelist, 6581 gfp_zone(gfp_mask), nodemask) { 6582 if (zone_idx(zone) > ZONE_NORMAL) 6583 continue; 6584 6585 /* Throttle based on the first usable node */ 6586 pgdat = zone->zone_pgdat; 6587 if (allow_direct_reclaim(pgdat)) 6588 goto out; 6589 break; 6590 } 6591 6592 /* If no zone was usable by the allocation flags then do not throttle */ 6593 if (!pgdat) 6594 goto out; 6595 6596 /* Account for the throttling */ 6597 count_vm_event(PGSCAN_DIRECT_THROTTLE); 6598 6599 /* 6600 * If the caller cannot enter the filesystem, it's possible that it 6601 * is due to the caller holding an FS lock or performing a journal 6602 * transaction in the case of a filesystem like ext[3|4]. In this case, 6603 * it is not safe to block on pfmemalloc_wait as kswapd could be 6604 * blocked waiting on the same lock. Instead, throttle for up to a 6605 * second before continuing. 6606 */ 6607 if (!(gfp_mask & __GFP_FS)) 6608 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait, 6609 allow_direct_reclaim(pgdat), HZ); 6610 else 6611 /* Throttle until kswapd wakes the process */ 6612 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait, 6613 allow_direct_reclaim(pgdat)); 6614 6615 if (fatal_signal_pending(current)) 6616 return true; 6617 6618 out: 6619 return false; 6620 } 6621 6622 unsigned long try_to_free_pages(struct zonelist *zonelist, int order, 6623 gfp_t gfp_mask, nodemask_t *nodemask) 6624 { 6625 unsigned long nr_reclaimed; 6626 struct scan_control sc = { 6627 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6628 .gfp_mask = current_gfp_context(gfp_mask), 6629 .reclaim_idx = gfp_zone(gfp_mask), 6630 .order = order, 6631 .nodemask = nodemask, 6632 .priority = DEF_PRIORITY, 6633 .may_writepage = !laptop_mode, 6634 .may_unmap = 1, 6635 .may_swap = 1, 6636 }; 6637 6638 /* 6639 * scan_control uses s8 fields for order, priority, and reclaim_idx. 6640 * Confirm they are large enough for max values. 6641 */ 6642 BUILD_BUG_ON(MAX_PAGE_ORDER >= S8_MAX); 6643 BUILD_BUG_ON(DEF_PRIORITY > S8_MAX); 6644 BUILD_BUG_ON(MAX_NR_ZONES > S8_MAX); 6645 6646 /* 6647 * Do not enter reclaim if fatal signal was delivered while throttled. 6648 * 1 is returned so that the page allocator does not OOM kill at this 6649 * point. 6650 */ 6651 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) 6652 return 1; 6653 6654 set_task_reclaim_state(current, &sc.reclaim_state); 6655 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); 6656 6657 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6658 6659 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); 6660 set_task_reclaim_state(current, NULL); 6661 6662 return nr_reclaimed; 6663 } 6664 6665 #ifdef CONFIG_MEMCG 6666 6667 /* Only used by soft limit reclaim. Do not reuse for anything else. */ 6668 unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg, 6669 gfp_t gfp_mask, bool noswap, 6670 pg_data_t *pgdat, 6671 unsigned long *nr_scanned) 6672 { 6673 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); 6674 struct scan_control sc = { 6675 .nr_to_reclaim = SWAP_CLUSTER_MAX, 6676 .target_mem_cgroup = memcg, 6677 .may_writepage = !laptop_mode, 6678 .may_unmap = 1, 6679 .reclaim_idx = MAX_NR_ZONES - 1, 6680 .may_swap = !noswap, 6681 }; 6682 6683 WARN_ON_ONCE(!current->reclaim_state); 6684 6685 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 6686 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 6687 6688 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order, 6689 sc.gfp_mask); 6690 6691 /* 6692 * NOTE: Although we can get the priority field, using it 6693 * here is not a good idea, since it limits the pages we can scan. 6694 * if we don't reclaim here, the shrink_node from balance_pgdat 6695 * will pick up pages from other mem cgroup's as well. We hack 6696 * the priority and make it zero. 6697 */ 6698 shrink_lruvec(lruvec, &sc); 6699 6700 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); 6701 6702 *nr_scanned = sc.nr_scanned; 6703 6704 return sc.nr_reclaimed; 6705 } 6706 6707 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6708 unsigned long nr_pages, 6709 gfp_t gfp_mask, 6710 unsigned int reclaim_options, 6711 int *swappiness) 6712 { 6713 unsigned long nr_reclaimed; 6714 unsigned int noreclaim_flag; 6715 struct scan_control sc = { 6716 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 6717 .proactive_swappiness = swappiness, 6718 .gfp_mask = (current_gfp_context(gfp_mask) & GFP_RECLAIM_MASK) | 6719 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), 6720 .reclaim_idx = MAX_NR_ZONES - 1, 6721 .target_mem_cgroup = memcg, 6722 .priority = DEF_PRIORITY, 6723 .may_writepage = !laptop_mode, 6724 .may_unmap = 1, 6725 .may_swap = !!(reclaim_options & MEMCG_RECLAIM_MAY_SWAP), 6726 .proactive = !!(reclaim_options & MEMCG_RECLAIM_PROACTIVE), 6727 }; 6728 /* 6729 * Traverse the ZONELIST_FALLBACK zonelist of the current node to put 6730 * equal pressure on all the nodes. This is based on the assumption that 6731 * the reclaim does not bail out early. 6732 */ 6733 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 6734 6735 set_task_reclaim_state(current, &sc.reclaim_state); 6736 trace_mm_vmscan_memcg_reclaim_begin(0, sc.gfp_mask); 6737 noreclaim_flag = memalloc_noreclaim_save(); 6738 6739 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 6740 6741 memalloc_noreclaim_restore(noreclaim_flag); 6742 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 6743 set_task_reclaim_state(current, NULL); 6744 6745 return nr_reclaimed; 6746 } 6747 #else 6748 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, 6749 unsigned long nr_pages, 6750 gfp_t gfp_mask, 6751 unsigned int reclaim_options, 6752 int *swappiness) 6753 { 6754 return 0; 6755 } 6756 #endif 6757 6758 static void kswapd_age_node(struct pglist_data *pgdat, struct scan_control *sc) 6759 { 6760 struct mem_cgroup *memcg; 6761 struct lruvec *lruvec; 6762 6763 if (lru_gen_enabled()) { 6764 lru_gen_age_node(pgdat, sc); 6765 return; 6766 } 6767 6768 lruvec = mem_cgroup_lruvec(NULL, pgdat); 6769 if (!can_age_anon_pages(lruvec, sc)) 6770 return; 6771 6772 if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON)) 6773 return; 6774 6775 memcg = mem_cgroup_iter(NULL, NULL, NULL); 6776 do { 6777 lruvec = mem_cgroup_lruvec(memcg, pgdat); 6778 shrink_active_list(SWAP_CLUSTER_MAX, lruvec, 6779 sc, LRU_ACTIVE_ANON); 6780 memcg = mem_cgroup_iter(NULL, memcg, NULL); 6781 } while (memcg); 6782 } 6783 6784 static bool pgdat_watermark_boosted(pg_data_t *pgdat, int highest_zoneidx) 6785 { 6786 int i; 6787 struct zone *zone; 6788 6789 /* 6790 * Check for watermark boosts top-down as the higher zones 6791 * are more likely to be boosted. Both watermarks and boosts 6792 * should not be checked at the same time as reclaim would 6793 * start prematurely when there is no boosting and a lower 6794 * zone is balanced. 6795 */ 6796 for (i = highest_zoneidx; i >= 0; i--) { 6797 zone = pgdat->node_zones + i; 6798 if (!managed_zone(zone)) 6799 continue; 6800 6801 if (zone->watermark_boost) 6802 return true; 6803 } 6804 6805 return false; 6806 } 6807 6808 /* 6809 * Returns true if there is an eligible zone balanced for the request order 6810 * and highest_zoneidx 6811 */ 6812 static bool pgdat_balanced(pg_data_t *pgdat, int order, int highest_zoneidx) 6813 { 6814 int i; 6815 unsigned long mark = -1; 6816 struct zone *zone; 6817 6818 /* 6819 * Check watermarks bottom-up as lower zones are more likely to 6820 * meet watermarks. 6821 */ 6822 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6823 enum zone_stat_item item; 6824 unsigned long free_pages; 6825 6826 if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) 6827 mark = promo_wmark_pages(zone); 6828 else 6829 mark = high_wmark_pages(zone); 6830 6831 /* 6832 * In defrag_mode, watermarks must be met in whole 6833 * blocks to avoid polluting allocator fallbacks. 6834 * 6835 * However, kswapd usually cannot accomplish this on 6836 * its own and needs kcompactd support. Once it's 6837 * reclaimed a compaction gap, and kswapd_shrink_node 6838 * has dropped order, simply ensure there are enough 6839 * base pages for compaction, wake kcompactd & sleep. 6840 */ 6841 if (defrag_mode && order) 6842 item = NR_FREE_PAGES_BLOCKS; 6843 else 6844 item = NR_FREE_PAGES; 6845 6846 /* 6847 * When there is a high number of CPUs in the system, 6848 * the cumulative error from the vmstat per-cpu cache 6849 * can blur the line between the watermarks. In that 6850 * case, be safe and get an accurate snapshot. 6851 * 6852 * TODO: NR_FREE_PAGES_BLOCKS moves in steps of 6853 * pageblock_nr_pages, while the vmstat pcp threshold 6854 * is limited to 125. On many configurations that 6855 * counter won't actually be per-cpu cached. But keep 6856 * things simple for now; revisit when somebody cares. 6857 */ 6858 free_pages = zone_page_state(zone, item); 6859 if (zone->percpu_drift_mark && free_pages < zone->percpu_drift_mark) 6860 free_pages = zone_page_state_snapshot(zone, item); 6861 6862 if (__zone_watermark_ok(zone, order, mark, highest_zoneidx, 6863 0, free_pages)) 6864 return true; 6865 } 6866 6867 /* 6868 * If a node has no managed zone within highest_zoneidx, it does not 6869 * need balancing by definition. This can happen if a zone-restricted 6870 * allocation tries to wake a remote kswapd. 6871 */ 6872 if (mark == -1) 6873 return true; 6874 6875 return false; 6876 } 6877 6878 /* Clear pgdat state for congested, dirty or under writeback. */ 6879 static void clear_pgdat_congested(pg_data_t *pgdat) 6880 { 6881 struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat); 6882 6883 clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags); 6884 clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags); 6885 clear_bit(PGDAT_DIRTY, &pgdat->flags); 6886 clear_bit(PGDAT_WRITEBACK, &pgdat->flags); 6887 } 6888 6889 /* 6890 * Prepare kswapd for sleeping. This verifies that there are no processes 6891 * waiting in throttle_direct_reclaim() and that watermarks have been met. 6892 * 6893 * Returns true if kswapd is ready to sleep 6894 */ 6895 static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, 6896 int highest_zoneidx) 6897 { 6898 /* 6899 * The throttled processes are normally woken up in balance_pgdat() as 6900 * soon as allow_direct_reclaim() is true. But there is a potential 6901 * race between when kswapd checks the watermarks and a process gets 6902 * throttled. There is also a potential race if processes get 6903 * throttled, kswapd wakes, a large process exits thereby balancing the 6904 * zones, which causes kswapd to exit balance_pgdat() before reaching 6905 * the wake up checks. If kswapd is going to sleep, no process should 6906 * be sleeping on pfmemalloc_wait, so wake them now if necessary. If 6907 * the wake up is premature, processes will wake kswapd and get 6908 * throttled again. The difference from wake ups in balance_pgdat() is 6909 * that here we are under prepare_to_wait(). 6910 */ 6911 if (waitqueue_active(&pgdat->pfmemalloc_wait)) 6912 wake_up_all(&pgdat->pfmemalloc_wait); 6913 6914 /* Hopeless node, leave it to direct reclaim */ 6915 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES) 6916 return true; 6917 6918 if (pgdat_balanced(pgdat, order, highest_zoneidx)) { 6919 clear_pgdat_congested(pgdat); 6920 return true; 6921 } 6922 6923 return false; 6924 } 6925 6926 /* 6927 * kswapd shrinks a node of pages that are at or below the highest usable 6928 * zone that is currently unbalanced. 6929 * 6930 * Returns true if kswapd scanned at least the requested number of pages to 6931 * reclaim or if the lack of progress was due to pages under writeback. 6932 * This is used to determine if the scanning priority needs to be raised. 6933 */ 6934 static bool kswapd_shrink_node(pg_data_t *pgdat, 6935 struct scan_control *sc) 6936 { 6937 struct zone *zone; 6938 int z; 6939 unsigned long nr_reclaimed = sc->nr_reclaimed; 6940 6941 /* Reclaim a number of pages proportional to the number of zones */ 6942 sc->nr_to_reclaim = 0; 6943 for_each_managed_zone_pgdat(zone, pgdat, z, sc->reclaim_idx) { 6944 sc->nr_to_reclaim += max(high_wmark_pages(zone), SWAP_CLUSTER_MAX); 6945 } 6946 6947 /* 6948 * Historically care was taken to put equal pressure on all zones but 6949 * now pressure is applied based on node LRU order. 6950 */ 6951 shrink_node(pgdat, sc); 6952 6953 /* 6954 * Fragmentation may mean that the system cannot be rebalanced for 6955 * high-order allocations. If twice the allocation size has been 6956 * reclaimed then recheck watermarks only at order-0 to prevent 6957 * excessive reclaim. Assume that a process requested a high-order 6958 * can direct reclaim/compact. 6959 */ 6960 if (sc->order && sc->nr_reclaimed >= compact_gap(sc->order)) 6961 sc->order = 0; 6962 6963 /* account for progress from mm_account_reclaimed_pages() */ 6964 return max(sc->nr_scanned, sc->nr_reclaimed - nr_reclaimed) >= sc->nr_to_reclaim; 6965 } 6966 6967 /* Page allocator PCP high watermark is lowered if reclaim is active. */ 6968 static inline void 6969 update_reclaim_active(pg_data_t *pgdat, int highest_zoneidx, bool active) 6970 { 6971 int i; 6972 struct zone *zone; 6973 6974 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 6975 if (active) 6976 set_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6977 else 6978 clear_bit(ZONE_RECLAIM_ACTIVE, &zone->flags); 6979 } 6980 } 6981 6982 static inline void 6983 set_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6984 { 6985 update_reclaim_active(pgdat, highest_zoneidx, true); 6986 } 6987 6988 static inline void 6989 clear_reclaim_active(pg_data_t *pgdat, int highest_zoneidx) 6990 { 6991 update_reclaim_active(pgdat, highest_zoneidx, false); 6992 } 6993 6994 /* 6995 * For kswapd, balance_pgdat() will reclaim pages across a node from zones 6996 * that are eligible for use by the caller until at least one zone is 6997 * balanced. 6998 * 6999 * Returns the order kswapd finished reclaiming at. 7000 * 7001 * kswapd scans the zones in the highmem->normal->dma direction. It skips 7002 * zones which have free_pages > high_wmark_pages(zone), but once a zone is 7003 * found to have free_pages <= high_wmark_pages(zone), any page in that zone 7004 * or lower is eligible for reclaim until at least one usable zone is 7005 * balanced. 7006 */ 7007 static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx) 7008 { 7009 int i; 7010 unsigned long nr_soft_reclaimed; 7011 unsigned long nr_soft_scanned; 7012 unsigned long pflags; 7013 unsigned long nr_boost_reclaim; 7014 unsigned long zone_boosts[MAX_NR_ZONES] = { 0, }; 7015 bool boosted; 7016 struct zone *zone; 7017 struct scan_control sc = { 7018 .gfp_mask = GFP_KERNEL, 7019 .order = order, 7020 .may_unmap = 1, 7021 }; 7022 7023 set_task_reclaim_state(current, &sc.reclaim_state); 7024 psi_memstall_enter(&pflags); 7025 __fs_reclaim_acquire(_THIS_IP_); 7026 7027 count_vm_event(PAGEOUTRUN); 7028 7029 /* 7030 * Account for the reclaim boost. Note that the zone boost is left in 7031 * place so that parallel allocations that are near the watermark will 7032 * stall or direct reclaim until kswapd is finished. 7033 */ 7034 nr_boost_reclaim = 0; 7035 for_each_managed_zone_pgdat(zone, pgdat, i, highest_zoneidx) { 7036 nr_boost_reclaim += zone->watermark_boost; 7037 zone_boosts[i] = zone->watermark_boost; 7038 } 7039 boosted = nr_boost_reclaim; 7040 7041 restart: 7042 set_reclaim_active(pgdat, highest_zoneidx); 7043 sc.priority = DEF_PRIORITY; 7044 do { 7045 unsigned long nr_reclaimed = sc.nr_reclaimed; 7046 bool raise_priority = true; 7047 bool balanced; 7048 bool ret; 7049 bool was_frozen; 7050 7051 sc.reclaim_idx = highest_zoneidx; 7052 7053 /* 7054 * If the number of buffer_heads exceeds the maximum allowed 7055 * then consider reclaiming from all zones. This has a dual 7056 * purpose -- on 64-bit systems it is expected that 7057 * buffer_heads are stripped during active rotation. On 32-bit 7058 * systems, highmem pages can pin lowmem memory and shrinking 7059 * buffers can relieve lowmem pressure. Reclaim may still not 7060 * go ahead if all eligible zones for the original allocation 7061 * request are balanced to avoid excessive reclaim from kswapd. 7062 */ 7063 if (buffer_heads_over_limit) { 7064 for (i = MAX_NR_ZONES - 1; i >= 0; i--) { 7065 zone = pgdat->node_zones + i; 7066 if (!managed_zone(zone)) 7067 continue; 7068 7069 sc.reclaim_idx = i; 7070 break; 7071 } 7072 } 7073 7074 /* 7075 * If the pgdat is imbalanced then ignore boosting and preserve 7076 * the watermarks for a later time and restart. Note that the 7077 * zone watermarks will be still reset at the end of balancing 7078 * on the grounds that the normal reclaim should be enough to 7079 * re-evaluate if boosting is required when kswapd next wakes. 7080 */ 7081 balanced = pgdat_balanced(pgdat, sc.order, highest_zoneidx); 7082 if (!balanced && nr_boost_reclaim) { 7083 nr_boost_reclaim = 0; 7084 goto restart; 7085 } 7086 7087 /* 7088 * If boosting is not active then only reclaim if there are no 7089 * eligible zones. Note that sc.reclaim_idx is not used as 7090 * buffer_heads_over_limit may have adjusted it. 7091 */ 7092 if (!nr_boost_reclaim && balanced) 7093 goto out; 7094 7095 /* Limit the priority of boosting to avoid reclaim writeback */ 7096 if (nr_boost_reclaim && sc.priority == DEF_PRIORITY - 2) 7097 raise_priority = false; 7098 7099 /* 7100 * Do not writeback or swap pages for boosted reclaim. The 7101 * intent is to relieve pressure not issue sub-optimal IO 7102 * from reclaim context. If no pages are reclaimed, the 7103 * reclaim will be aborted. 7104 */ 7105 sc.may_writepage = !laptop_mode && !nr_boost_reclaim; 7106 sc.may_swap = !nr_boost_reclaim; 7107 7108 /* 7109 * Do some background aging, to give pages a chance to be 7110 * referenced before reclaiming. All pages are rotated 7111 * regardless of classzone as this is about consistent aging. 7112 */ 7113 kswapd_age_node(pgdat, &sc); 7114 7115 /* 7116 * If we're getting trouble reclaiming, start doing writepage 7117 * even in laptop mode. 7118 */ 7119 if (sc.priority < DEF_PRIORITY - 2) 7120 sc.may_writepage = 1; 7121 7122 /* Call soft limit reclaim before calling shrink_node. */ 7123 sc.nr_scanned = 0; 7124 nr_soft_scanned = 0; 7125 nr_soft_reclaimed = memcg1_soft_limit_reclaim(pgdat, sc.order, 7126 sc.gfp_mask, &nr_soft_scanned); 7127 sc.nr_reclaimed += nr_soft_reclaimed; 7128 7129 /* 7130 * There should be no need to raise the scanning priority if 7131 * enough pages are already being scanned that that high 7132 * watermark would be met at 100% efficiency. 7133 */ 7134 if (kswapd_shrink_node(pgdat, &sc)) 7135 raise_priority = false; 7136 7137 /* 7138 * If the low watermark is met there is no need for processes 7139 * to be throttled on pfmemalloc_wait as they should not be 7140 * able to safely make forward progress. Wake them 7141 */ 7142 if (waitqueue_active(&pgdat->pfmemalloc_wait) && 7143 allow_direct_reclaim(pgdat)) 7144 wake_up_all(&pgdat->pfmemalloc_wait); 7145 7146 /* Check if kswapd should be suspending */ 7147 __fs_reclaim_release(_THIS_IP_); 7148 ret = kthread_freezable_should_stop(&was_frozen); 7149 __fs_reclaim_acquire(_THIS_IP_); 7150 if (was_frozen || ret) 7151 break; 7152 7153 /* 7154 * Raise priority if scanning rate is too low or there was no 7155 * progress in reclaiming pages 7156 */ 7157 nr_reclaimed = sc.nr_reclaimed - nr_reclaimed; 7158 nr_boost_reclaim -= min(nr_boost_reclaim, nr_reclaimed); 7159 7160 /* 7161 * If reclaim made no progress for a boost, stop reclaim as 7162 * IO cannot be queued and it could be an infinite loop in 7163 * extreme circumstances. 7164 */ 7165 if (nr_boost_reclaim && !nr_reclaimed) 7166 break; 7167 7168 if (raise_priority || !nr_reclaimed) 7169 sc.priority--; 7170 } while (sc.priority >= 1); 7171 7172 /* 7173 * Restart only if it went through the priority loop all the way, 7174 * but cache_trim_mode didn't work. 7175 */ 7176 if (!sc.nr_reclaimed && sc.priority < 1 && 7177 !sc.no_cache_trim_mode && sc.cache_trim_mode_failed) { 7178 sc.no_cache_trim_mode = 1; 7179 goto restart; 7180 } 7181 7182 if (!sc.nr_reclaimed) 7183 atomic_inc(&pgdat->kswapd_failures); 7184 7185 out: 7186 clear_reclaim_active(pgdat, highest_zoneidx); 7187 7188 /* If reclaim was boosted, account for the reclaim done in this pass */ 7189 if (boosted) { 7190 unsigned long flags; 7191 7192 for (i = 0; i <= highest_zoneidx; i++) { 7193 if (!zone_boosts[i]) 7194 continue; 7195 7196 /* Increments are under the zone lock */ 7197 zone = pgdat->node_zones + i; 7198 spin_lock_irqsave(&zone->lock, flags); 7199 zone->watermark_boost -= min(zone->watermark_boost, zone_boosts[i]); 7200 spin_unlock_irqrestore(&zone->lock, flags); 7201 } 7202 7203 /* 7204 * As there is now likely space, wakeup kcompact to defragment 7205 * pageblocks. 7206 */ 7207 wakeup_kcompactd(pgdat, pageblock_order, highest_zoneidx); 7208 } 7209 7210 snapshot_refaults(NULL, pgdat); 7211 __fs_reclaim_release(_THIS_IP_); 7212 psi_memstall_leave(&pflags); 7213 set_task_reclaim_state(current, NULL); 7214 7215 /* 7216 * Return the order kswapd stopped reclaiming at as 7217 * prepare_kswapd_sleep() takes it into account. If another caller 7218 * entered the allocator slow path while kswapd was awake, order will 7219 * remain at the higher level. 7220 */ 7221 return sc.order; 7222 } 7223 7224 /* 7225 * The pgdat->kswapd_highest_zoneidx is used to pass the highest zone index to 7226 * be reclaimed by kswapd from the waker. If the value is MAX_NR_ZONES which is 7227 * not a valid index then either kswapd runs for first time or kswapd couldn't 7228 * sleep after previous reclaim attempt (node is still unbalanced). In that 7229 * case return the zone index of the previous kswapd reclaim cycle. 7230 */ 7231 static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, 7232 enum zone_type prev_highest_zoneidx) 7233 { 7234 enum zone_type curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7235 7236 return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; 7237 } 7238 7239 static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, 7240 unsigned int highest_zoneidx) 7241 { 7242 long remaining = 0; 7243 DEFINE_WAIT(wait); 7244 7245 if (freezing(current) || kthread_should_stop()) 7246 return; 7247 7248 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7249 7250 /* 7251 * Try to sleep for a short interval. Note that kcompactd will only be 7252 * woken if it is possible to sleep for a short interval. This is 7253 * deliberate on the assumption that if reclaim cannot keep an 7254 * eligible zone balanced that it's also unlikely that compaction will 7255 * succeed. 7256 */ 7257 if (prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7258 /* 7259 * Compaction records what page blocks it recently failed to 7260 * isolate pages from and skips them in the future scanning. 7261 * When kswapd is going to sleep, it is reasonable to assume 7262 * that pages and compaction may succeed so reset the cache. 7263 */ 7264 reset_isolation_suitable(pgdat); 7265 7266 /* 7267 * We have freed the memory, now we should compact it to make 7268 * allocation of the requested order possible. 7269 */ 7270 wakeup_kcompactd(pgdat, alloc_order, highest_zoneidx); 7271 7272 remaining = schedule_timeout(HZ/10); 7273 7274 /* 7275 * If woken prematurely then reset kswapd_highest_zoneidx and 7276 * order. The values will either be from a wakeup request or 7277 * the previous request that slept prematurely. 7278 */ 7279 if (remaining) { 7280 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, 7281 kswapd_highest_zoneidx(pgdat, 7282 highest_zoneidx)); 7283 7284 if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) 7285 WRITE_ONCE(pgdat->kswapd_order, reclaim_order); 7286 } 7287 7288 finish_wait(&pgdat->kswapd_wait, &wait); 7289 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE); 7290 } 7291 7292 /* 7293 * After a short sleep, check if it was a premature sleep. If not, then 7294 * go fully to sleep until explicitly woken up. 7295 */ 7296 if (!remaining && 7297 prepare_kswapd_sleep(pgdat, reclaim_order, highest_zoneidx)) { 7298 trace_mm_vmscan_kswapd_sleep(pgdat->node_id); 7299 7300 /* 7301 * vmstat counters are not perfectly accurate and the estimated 7302 * value for counters such as NR_FREE_PAGES can deviate from the 7303 * true value by nr_online_cpus * threshold. To avoid the zone 7304 * watermarks being breached while under pressure, we reduce the 7305 * per-cpu vmstat threshold while kswapd is awake and restore 7306 * them before going back to sleep. 7307 */ 7308 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold); 7309 7310 if (!kthread_should_stop()) 7311 schedule(); 7312 7313 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold); 7314 } else { 7315 if (remaining) 7316 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); 7317 else 7318 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY); 7319 } 7320 finish_wait(&pgdat->kswapd_wait, &wait); 7321 } 7322 7323 /* 7324 * The background pageout daemon, started as a kernel thread 7325 * from the init process. 7326 * 7327 * This basically trickles out pages so that we have _some_ 7328 * free memory available even if there is no other activity 7329 * that frees anything up. This is needed for things like routing 7330 * etc, where we otherwise might have all activity going on in 7331 * asynchronous contexts that cannot page things out. 7332 * 7333 * If there are applications that are active memory-allocators 7334 * (most normal use), this basically shouldn't matter. 7335 */ 7336 static int kswapd(void *p) 7337 { 7338 unsigned int alloc_order, reclaim_order; 7339 unsigned int highest_zoneidx = MAX_NR_ZONES - 1; 7340 pg_data_t *pgdat = (pg_data_t *)p; 7341 struct task_struct *tsk = current; 7342 7343 /* 7344 * Tell the memory management that we're a "memory allocator", 7345 * and that if we need more memory we should get access to it 7346 * regardless (see "__alloc_pages()"). "kswapd" should 7347 * never get caught in the normal page freeing logic. 7348 * 7349 * (Kswapd normally doesn't need memory anyway, but sometimes 7350 * you need a small amount of memory in order to be able to 7351 * page out something else, and this flag essentially protects 7352 * us from recursively trying to free more memory as we're 7353 * trying to free the first piece of memory in the first place). 7354 */ 7355 tsk->flags |= PF_MEMALLOC | PF_KSWAPD; 7356 set_freezable(); 7357 7358 WRITE_ONCE(pgdat->kswapd_order, 0); 7359 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7360 atomic_set(&pgdat->nr_writeback_throttled, 0); 7361 for ( ; ; ) { 7362 bool was_frozen; 7363 7364 alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); 7365 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7366 highest_zoneidx); 7367 7368 kswapd_try_sleep: 7369 kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, 7370 highest_zoneidx); 7371 7372 /* Read the new order and highest_zoneidx */ 7373 alloc_order = READ_ONCE(pgdat->kswapd_order); 7374 highest_zoneidx = kswapd_highest_zoneidx(pgdat, 7375 highest_zoneidx); 7376 WRITE_ONCE(pgdat->kswapd_order, 0); 7377 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); 7378 7379 if (kthread_freezable_should_stop(&was_frozen)) 7380 break; 7381 7382 /* 7383 * We can speed up thawing tasks if we don't call balance_pgdat 7384 * after returning from the refrigerator 7385 */ 7386 if (was_frozen) 7387 continue; 7388 7389 /* 7390 * Reclaim begins at the requested order but if a high-order 7391 * reclaim fails then kswapd falls back to reclaiming for 7392 * order-0. If that happens, kswapd will consider sleeping 7393 * for the order it finished reclaiming at (reclaim_order) 7394 * but kcompactd is woken to compact for the original 7395 * request (alloc_order). 7396 */ 7397 trace_mm_vmscan_kswapd_wake(pgdat->node_id, highest_zoneidx, 7398 alloc_order); 7399 reclaim_order = balance_pgdat(pgdat, alloc_order, 7400 highest_zoneidx); 7401 if (reclaim_order < alloc_order) 7402 goto kswapd_try_sleep; 7403 } 7404 7405 tsk->flags &= ~(PF_MEMALLOC | PF_KSWAPD); 7406 7407 return 0; 7408 } 7409 7410 /* 7411 * A zone is low on free memory or too fragmented for high-order memory. If 7412 * kswapd should reclaim (direct reclaim is deferred), wake it up for the zone's 7413 * pgdat. It will wake up kcompactd after reclaiming memory. If kswapd reclaim 7414 * has failed or is not needed, still wake up kcompactd if only compaction is 7415 * needed. 7416 */ 7417 void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, 7418 enum zone_type highest_zoneidx) 7419 { 7420 pg_data_t *pgdat; 7421 enum zone_type curr_idx; 7422 7423 if (!managed_zone(zone)) 7424 return; 7425 7426 if (!cpuset_zone_allowed(zone, gfp_flags)) 7427 return; 7428 7429 pgdat = zone->zone_pgdat; 7430 curr_idx = READ_ONCE(pgdat->kswapd_highest_zoneidx); 7431 7432 if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) 7433 WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); 7434 7435 if (READ_ONCE(pgdat->kswapd_order) < order) 7436 WRITE_ONCE(pgdat->kswapd_order, order); 7437 7438 if (!waitqueue_active(&pgdat->kswapd_wait)) 7439 return; 7440 7441 /* Hopeless node, leave it to direct reclaim if possible */ 7442 if (atomic_read(&pgdat->kswapd_failures) >= MAX_RECLAIM_RETRIES || 7443 (pgdat_balanced(pgdat, order, highest_zoneidx) && 7444 !pgdat_watermark_boosted(pgdat, highest_zoneidx))) { 7445 /* 7446 * There may be plenty of free memory available, but it's too 7447 * fragmented for high-order allocations. Wake up kcompactd 7448 * and rely on compaction_suitable() to determine if it's 7449 * needed. If it fails, it will defer subsequent attempts to 7450 * ratelimit its work. 7451 */ 7452 if (!(gfp_flags & __GFP_DIRECT_RECLAIM)) 7453 wakeup_kcompactd(pgdat, order, highest_zoneidx); 7454 return; 7455 } 7456 7457 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, highest_zoneidx, order, 7458 gfp_flags); 7459 wake_up_interruptible(&pgdat->kswapd_wait); 7460 } 7461 7462 #ifdef CONFIG_HIBERNATION 7463 /* 7464 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of 7465 * freed pages. 7466 * 7467 * Rather than trying to age LRUs the aim is to preserve the overall 7468 * LRU order by reclaiming preferentially 7469 * inactive > active > active referenced > active mapped 7470 */ 7471 unsigned long shrink_all_memory(unsigned long nr_to_reclaim) 7472 { 7473 struct scan_control sc = { 7474 .nr_to_reclaim = nr_to_reclaim, 7475 .gfp_mask = GFP_HIGHUSER_MOVABLE, 7476 .reclaim_idx = MAX_NR_ZONES - 1, 7477 .priority = DEF_PRIORITY, 7478 .may_writepage = 1, 7479 .may_unmap = 1, 7480 .may_swap = 1, 7481 .hibernation_mode = 1, 7482 }; 7483 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 7484 unsigned long nr_reclaimed; 7485 unsigned int noreclaim_flag; 7486 7487 fs_reclaim_acquire(sc.gfp_mask); 7488 noreclaim_flag = memalloc_noreclaim_save(); 7489 set_task_reclaim_state(current, &sc.reclaim_state); 7490 7491 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 7492 7493 set_task_reclaim_state(current, NULL); 7494 memalloc_noreclaim_restore(noreclaim_flag); 7495 fs_reclaim_release(sc.gfp_mask); 7496 7497 return nr_reclaimed; 7498 } 7499 #endif /* CONFIG_HIBERNATION */ 7500 7501 /* 7502 * This kswapd start function will be called by init and node-hot-add. 7503 */ 7504 void __meminit kswapd_run(int nid) 7505 { 7506 pg_data_t *pgdat = NODE_DATA(nid); 7507 7508 pgdat_kswapd_lock(pgdat); 7509 if (!pgdat->kswapd) { 7510 pgdat->kswapd = kthread_create_on_node(kswapd, pgdat, nid, "kswapd%d", nid); 7511 if (IS_ERR(pgdat->kswapd)) { 7512 /* failure at boot is fatal */ 7513 pr_err("Failed to start kswapd on node %d，ret=%ld\n", 7514 nid, PTR_ERR(pgdat->kswapd)); 7515 BUG_ON(system_state < SYSTEM_RUNNING); 7516 pgdat->kswapd = NULL; 7517 } else { 7518 wake_up_process(pgdat->kswapd); 7519 } 7520 } 7521 pgdat_kswapd_unlock(pgdat); 7522 } 7523 7524 /* 7525 * Called by memory hotplug when all memory in a node is offlined. Caller must 7526 * be holding mem_hotplug_begin/done(). 7527 */ 7528 void __meminit kswapd_stop(int nid) 7529 { 7530 pg_data_t *pgdat = NODE_DATA(nid); 7531 struct task_struct *kswapd; 7532 7533 pgdat_kswapd_lock(pgdat); 7534 kswapd = pgdat->kswapd; 7535 if (kswapd) { 7536 kthread_stop(kswapd); 7537 pgdat->kswapd = NULL; 7538 } 7539 pgdat_kswapd_unlock(pgdat); 7540 } 7541 7542 static const struct ctl_table vmscan_sysctl_table[] = { 7543 { 7544 .procname = "swappiness", 7545 .data = &vm_swappiness, 7546 .maxlen = sizeof(vm_swappiness), 7547 .mode = 0644, 7548 .proc_handler = proc_dointvec_minmax, 7549 .extra1 = SYSCTL_ZERO, 7550 .extra2 = SYSCTL_TWO_HUNDRED, 7551 }, 7552 #ifdef CONFIG_NUMA 7553 { 7554 .procname = "zone_reclaim_mode", 7555 .data = &node_reclaim_mode, 7556 .maxlen = sizeof(node_reclaim_mode), 7557 .mode = 0644, 7558 .proc_handler = proc_dointvec_minmax, 7559 .extra1 = SYSCTL_ZERO, 7560 } 7561 #endif 7562 }; 7563 7564 static int __init kswapd_init(void) 7565 { 7566 int nid; 7567 7568 swap_setup(); 7569 for_each_node_state(nid, N_MEMORY) 7570 kswapd_run(nid); 7571 register_sysctl_init("vm", vmscan_sysctl_table); 7572 return 0; 7573 } 7574 7575 module_init(kswapd_init) 7576 7577 #ifdef CONFIG_NUMA 7578 /* 7579 * Node reclaim mode 7580 * 7581 * If non-zero call node_reclaim when the number of free pages falls below 7582 * the watermarks. 7583 */ 7584 int node_reclaim_mode __read_mostly; 7585 7586 /* 7587 * Priority for NODE_RECLAIM. This determines the fraction of pages 7588 * of a node considered for each zone_reclaim. 4 scans 1/16th of 7589 * a zone. 7590 */ 7591 #define NODE_RECLAIM_PRIORITY 4 7592 7593 /* 7594 * Percentage of pages in a zone that must be unmapped for node_reclaim to 7595 * occur. 7596 */ 7597 int sysctl_min_unmapped_ratio = 1; 7598 7599 /* 7600 * If the number of slab pages in a zone grows beyond this percentage then 7601 * slab reclaim needs to occur. 7602 */ 7603 int sysctl_min_slab_ratio = 5; 7604 7605 static inline unsigned long node_unmapped_file_pages(struct pglist_data *pgdat) 7606 { 7607 unsigned long file_mapped = node_page_state(pgdat, NR_FILE_MAPPED); 7608 unsigned long file_lru = node_page_state(pgdat, NR_INACTIVE_FILE) + 7609 node_page_state(pgdat, NR_ACTIVE_FILE); 7610 7611 /* 7612 * It's possible for there to be more file mapped pages than 7613 * accounted for by the pages on the file LRU lists because 7614 * tmpfs pages accounted for as ANON can also be FILE_MAPPED 7615 */ 7616 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0; 7617 } 7618 7619 /* Work out how many page cache pages we can reclaim in this reclaim_mode */ 7620 static unsigned long node_pagecache_reclaimable(struct pglist_data *pgdat) 7621 { 7622 unsigned long nr_pagecache_reclaimable; 7623 unsigned long delta = 0; 7624 7625 /* 7626 * If RECLAIM_UNMAP is set, then all file pages are considered 7627 * potentially reclaimable. Otherwise, we have to worry about 7628 * pages like swapcache and node_unmapped_file_pages() provides 7629 * a better estimate 7630 */ 7631 if (node_reclaim_mode & RECLAIM_UNMAP) 7632 nr_pagecache_reclaimable = node_page_state(pgdat, NR_FILE_PAGES); 7633 else 7634 nr_pagecache_reclaimable = node_unmapped_file_pages(pgdat); 7635 7636 /* If we can't clean pages, remove dirty pages from consideration */ 7637 if (!(node_reclaim_mode & RECLAIM_WRITE)) 7638 delta += node_page_state(pgdat, NR_FILE_DIRTY); 7639 7640 /* Watch for any possible underflows due to delta */ 7641 if (unlikely(delta > nr_pagecache_reclaimable)) 7642 delta = nr_pagecache_reclaimable; 7643 7644 return nr_pagecache_reclaimable - delta; 7645 } 7646 7647 /* 7648 * Try to free up some pages from this node through reclaim. 7649 */ 7650 static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, 7651 unsigned long nr_pages, 7652 struct scan_control *sc) 7653 { 7654 struct task_struct *p = current; 7655 unsigned int noreclaim_flag; 7656 unsigned long pflags; 7657 7658 trace_mm_vmscan_node_reclaim_begin(pgdat->node_id, sc->order, 7659 sc->gfp_mask); 7660 7661 cond_resched(); 7662 psi_memstall_enter(&pflags); 7663 delayacct_freepages_start(); 7664 fs_reclaim_acquire(sc->gfp_mask); 7665 /* 7666 * We need to be able to allocate from the reserves for RECLAIM_UNMAP 7667 */ 7668 noreclaim_flag = memalloc_noreclaim_save(); 7669 set_task_reclaim_state(p, &sc->reclaim_state); 7670 7671 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages || 7672 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) > pgdat->min_slab_pages) { 7673 /* 7674 * Free memory by calling shrink node with increasing 7675 * priorities until we have enough memory freed. 7676 */ 7677 do { 7678 shrink_node(pgdat, sc); 7679 } while (sc->nr_reclaimed < nr_pages && --sc->priority >= 0); 7680 } 7681 7682 set_task_reclaim_state(p, NULL); 7683 memalloc_noreclaim_restore(noreclaim_flag); 7684 fs_reclaim_release(sc->gfp_mask); 7685 delayacct_freepages_end(); 7686 psi_memstall_leave(&pflags); 7687 7688 trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed); 7689 7690 return sc->nr_reclaimed; 7691 } 7692 7693 int node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned int order) 7694 { 7695 int ret; 7696 /* Minimum pages needed in order to stay on node */ 7697 const unsigned long nr_pages = 1 << order; 7698 struct scan_control sc = { 7699 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 7700 .gfp_mask = current_gfp_context(gfp_mask), 7701 .order = order, 7702 .priority = NODE_RECLAIM_PRIORITY, 7703 .may_writepage = !!(node_reclaim_mode & RECLAIM_WRITE), 7704 .may_unmap = !!(node_reclaim_mode & RECLAIM_UNMAP), 7705 .may_swap = 1, 7706 .reclaim_idx = gfp_zone(gfp_mask), 7707 }; 7708 7709 /* 7710 * Node reclaim reclaims unmapped file backed pages and 7711 * slab pages if we are over the defined limits. 7712 * 7713 * A small portion of unmapped file backed pages is needed for 7714 * file I/O otherwise pages read by file I/O will be immediately 7715 * thrown out if the node is overallocated. So we do not reclaim 7716 * if less than a specified percentage of the node is used by 7717 * unmapped file backed pages. 7718 */ 7719 if (node_pagecache_reclaimable(pgdat) <= pgdat->min_unmapped_pages && 7720 node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B) <= 7721 pgdat->min_slab_pages) 7722 return NODE_RECLAIM_FULL; 7723 7724 /* 7725 * Do not scan if the allocation should not be delayed. 7726 */ 7727 if (!gfpflags_allow_blocking(gfp_mask) || (current->flags & PF_MEMALLOC)) 7728 return NODE_RECLAIM_NOSCAN; 7729 7730 /* 7731 * Only run node reclaim on the local node or on nodes that do not 7732 * have associated processors. This will favor the local processor 7733 * over remote processors and spread off node memory allocations 7734 * as wide as possible. 7735 */ 7736 if (node_state(pgdat->node_id, N_CPU) && pgdat->node_id != numa_node_id()) 7737 return NODE_RECLAIM_NOSCAN; 7738 7739 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, &pgdat->flags)) 7740 return NODE_RECLAIM_NOSCAN; 7741 7742 ret = __node_reclaim(pgdat, gfp_mask, nr_pages, &sc) >= nr_pages; 7743 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7744 7745 if (ret) 7746 count_vm_event(PGSCAN_ZONE_RECLAIM_SUCCESS); 7747 else 7748 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED); 7749 7750 return ret; 7751 } 7752 7753 #else 7754 7755 static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, 7756 unsigned long nr_pages, 7757 struct scan_control *sc) 7758 { 7759 return 0; 7760 } 7761 7762 #endif 7763 7764 enum { 7765 MEMORY_RECLAIM_SWAPPINESS = 0, 7766 MEMORY_RECLAIM_SWAPPINESS_MAX, 7767 MEMORY_RECLAIM_NULL, 7768 }; 7769 static const match_table_t tokens = { 7770 { MEMORY_RECLAIM_SWAPPINESS, "swappiness=%d"}, 7771 { MEMORY_RECLAIM_SWAPPINESS_MAX, "swappiness=max"}, 7772 { MEMORY_RECLAIM_NULL, NULL }, 7773 }; 7774 7775 int user_proactive_reclaim(char *buf, 7776 struct mem_cgroup *memcg, pg_data_t *pgdat) 7777 { 7778 unsigned int nr_retries = MAX_RECLAIM_RETRIES; 7779 unsigned long nr_to_reclaim, nr_reclaimed = 0; 7780 int swappiness = -1; 7781 char *old_buf, *start; 7782 substring_t args[MAX_OPT_ARGS]; 7783 gfp_t gfp_mask = GFP_KERNEL; 7784 7785 if (!buf || (!memcg && !pgdat) || (memcg && pgdat)) 7786 return -EINVAL; 7787 7788 buf = strstrip(buf); 7789 7790 old_buf = buf; 7791 nr_to_reclaim = memparse(buf, &buf) / PAGE_SIZE; 7792 if (buf == old_buf) 7793 return -EINVAL; 7794 7795 buf = strstrip(buf); 7796 7797 while ((start = strsep(&buf, " ")) != NULL) { 7798 if (!strlen(start)) 7799 continue; 7800 switch (match_token(start, tokens, args)) { 7801 case MEMORY_RECLAIM_SWAPPINESS: 7802 if (match_int(&args[0], &swappiness)) 7803 return -EINVAL; 7804 if (swappiness < MIN_SWAPPINESS || 7805 swappiness > MAX_SWAPPINESS) 7806 return -EINVAL; 7807 break; 7808 case MEMORY_RECLAIM_SWAPPINESS_MAX: 7809 swappiness = SWAPPINESS_ANON_ONLY; 7810 break; 7811 default: 7812 return -EINVAL; 7813 } 7814 } 7815 7816 while (nr_reclaimed < nr_to_reclaim) { 7817 /* Will converge on zero, but reclaim enforces a minimum */ 7818 unsigned long batch_size = (nr_to_reclaim - nr_reclaimed) / 4; 7819 unsigned long reclaimed; 7820 7821 if (signal_pending(current)) 7822 return -EINTR; 7823 7824 /* 7825 * This is the final attempt, drain percpu lru caches in the 7826 * hope of introducing more evictable pages. 7827 */ 7828 if (!nr_retries) 7829 lru_add_drain_all(); 7830 7831 if (memcg) { 7832 unsigned int reclaim_options; 7833 7834 reclaim_options = MEMCG_RECLAIM_MAY_SWAP | 7835 MEMCG_RECLAIM_PROACTIVE; 7836 reclaimed = try_to_free_mem_cgroup_pages(memcg, 7837 batch_size, gfp_mask, 7838 reclaim_options, 7839 swappiness == -1 ? NULL : &swappiness); 7840 } else { 7841 struct scan_control sc = { 7842 .gfp_mask = current_gfp_context(gfp_mask), 7843 .reclaim_idx = gfp_zone(gfp_mask), 7844 .proactive_swappiness = swappiness == -1 ? NULL : &swappiness, 7845 .priority = DEF_PRIORITY, 7846 .may_writepage = !laptop_mode, 7847 .nr_to_reclaim = max(batch_size, SWAP_CLUSTER_MAX), 7848 .may_unmap = 1, 7849 .may_swap = 1, 7850 .proactive = 1, 7851 }; 7852 7853 if (test_and_set_bit_lock(PGDAT_RECLAIM_LOCKED, 7854 &pgdat->flags)) 7855 return -EBUSY; 7856 7857 reclaimed = __node_reclaim(pgdat, gfp_mask, 7858 batch_size, &sc); 7859 clear_bit_unlock(PGDAT_RECLAIM_LOCKED, &pgdat->flags); 7860 } 7861 7862 if (!reclaimed && !nr_retries--) 7863 return -EAGAIN; 7864 7865 nr_reclaimed += reclaimed; 7866 } 7867 7868 return 0; 7869 } 7870 7871 /** 7872 * check_move_unevictable_folios - Move evictable folios to appropriate zone 7873 * lru list 7874 * @fbatch: Batch of lru folios to check. 7875 * 7876 * Checks folios for evictability, if an evictable folio is in the unevictable 7877 * lru list, moves it to the appropriate evictable lru list. This function 7878 * should be only used for lru folios. 7879 */ 7880 void check_move_unevictable_folios(struct folio_batch *fbatch) 7881 { 7882 struct lruvec *lruvec = NULL; 7883 int pgscanned = 0; 7884 int pgrescued = 0; 7885 int i; 7886 7887 for (i = 0; i < fbatch->nr; i++) { 7888 struct folio *folio = fbatch->folios[i]; 7889 int nr_pages = folio_nr_pages(folio); 7890 7891 pgscanned += nr_pages; 7892 7893 /* block memcg migration while the folio moves between lrus */ 7894 if (!folio_test_clear_lru(folio)) 7895 continue; 7896 7897 lruvec = folio_lruvec_relock_irq(folio, lruvec); 7898 if (folio_evictable(folio) && folio_test_unevictable(folio)) { 7899 lruvec_del_folio(lruvec, folio); 7900 folio_clear_unevictable(folio); 7901 lruvec_add_folio(lruvec, folio); 7902 pgrescued += nr_pages; 7903 } 7904 folio_set_lru(folio); 7905 } 7906 7907 if (lruvec) { 7908 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); 7909 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7910 unlock_page_lruvec_irq(lruvec); 7911 } else if (pgscanned) { 7912 count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); 7913 } 7914 } 7915 EXPORT_SYMBOL_GPL(check_move_unevictable_folios); 7916 7917 #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) 7918 static ssize_t reclaim_store(struct device *dev, 7919 struct device_attribute *attr, 7920 const char *buf, size_t count) 7921 { 7922 int ret, nid = dev->id; 7923 7924 ret = user_proactive_reclaim((char *)buf, NULL, NODE_DATA(nid)); 7925 return ret ? -EAGAIN : count; 7926 } 7927 7928 static DEVICE_ATTR_WO(reclaim); 7929 int reclaim_register_node(struct node *node) 7930 { 7931 return device_create_file(&node->dev, &dev_attr_reclaim); 7932 } 7933 7934 void reclaim_unregister_node(struct node *node) 7935 { 7936 return device_remove_file(&node->dev, &dev_attr_reclaim); 7937 } 7938 #endif