개념 설명 전체 · v6.18.37 / arch/arm64/include/asm/tlbflush.h
1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/tlbflush.h 4 * 5 * Copyright (C) 1999-2003 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASM_TLBFLUSH_H 9 #define __ASM_TLBFLUSH_H 10 11 #ifndef __ASSEMBLY__ 12 13 #include <linux/bitfield.h> 14 #include <linux/mm_types.h> 15 #include <linux/sched.h> 16 #include <linux/mmu_notifier.h> 17 #include <asm/cputype.h> 18 #include <asm/mmu.h> 19 20 /* 21 * Raw TLBI operations. 22 * 23 * Where necessary, use the __tlbi() macro to avoid asm() 24 * boilerplate. Drivers and most kernel code should use the TLB 25 * management routines in preference to the macro below. 26 * 27 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending 28 * on whether a particular TLBI operation takes an argument or 29 * not. The macros handles invoking the asm with or without the 30 * register argument as appropriate. 31 */ 32 #define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE \ 33 "tlbi " #op "\n" \ 34 : : ) 35 36 #define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE \ 37 "tlbi " #op ", %x0\n" \ 38 : : "rZ" (arg)) 39 40 #define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg) 41 42 #define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0) 43 44 #define __tlbi_user(op, arg) do { \ 45 if (arm64_kernel_unmapped_at_el0()) \ 46 __tlbi(op, (arg) | USER_ASID_FLAG); \ 47 } while (0) 48 49 /* This macro creates a properly formatted VA operand for the TLBI */ 50 #define __TLBI_VADDR(addr, asid) \ 51 ({ \ 52 unsigned long __ta = (addr) >> 12; \ 53 __ta &= GENMASK_ULL(43, 0); \ 54 __ta |= (unsigned long)(asid) << 48; \ 55 __ta; \ 56 }) 57 58 /* 59 * Get translation granule of the system, which is decided by 60 * PAGE_SIZE. Used by TTL. 61 * - 4KB : 1 62 * - 16KB : 2 63 * - 64KB : 3 64 */ 65 #define TLBI_TTL_TG_4K 1 66 #define TLBI_TTL_TG_16K 2 67 #define TLBI_TTL_TG_64K 3 68 69 static inline unsigned long get_trans_granule(void) 70 { 71 switch (PAGE_SIZE) { 72 case SZ_4K: 73 return TLBI_TTL_TG_4K; 74 case SZ_16K: 75 return TLBI_TTL_TG_16K; 76 case SZ_64K: 77 return TLBI_TTL_TG_64K; 78 default: 79 return 0; 80 } 81 } 82 83 #ifdef CONFIG_ARM64_ERRATUM_4193714 84 85 void sme_do_dvmsync(const struct cpumask *mask); 86 87 static inline void sme_dvmsync(struct mm_struct *mm) 88 { 89 if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) 90 return; 91 92 sme_do_dvmsync(mm_cpumask(mm)); 93 } 94 95 static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, 96 struct mm_struct *mm) 97 { 98 if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) 99 return; 100 101 /* 102 * Order the mm_cpumask() read after the hardware DVMSync. 103 */ 104 dsb(ish); 105 if (cpumask_empty(mm_cpumask(mm))) 106 return; 107 108 /* 109 * Allocate the batch cpumask on first use. Fall back to an immediate 110 * IPI for this mm in case of failure. 111 */ 112 if (!cpumask_available(batch->cpumask) && 113 !zalloc_cpumask_var(&batch->cpumask, GFP_ATOMIC)) { 114 sme_do_dvmsync(mm_cpumask(mm)); 115 return; 116 } 117 118 cpumask_or(batch->cpumask, batch->cpumask, mm_cpumask(mm)); 119 } 120 121 static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) 122 { 123 if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_4193714)) 124 return; 125 126 if (!cpumask_available(batch->cpumask)) 127 return; 128 129 sme_do_dvmsync(batch->cpumask); 130 cpumask_clear(batch->cpumask); 131 } 132 133 #else 134 135 static inline void sme_dvmsync(struct mm_struct *mm) 136 { 137 } 138 static inline void sme_dvmsync_add_pending(struct arch_tlbflush_unmap_batch *batch, 139 struct mm_struct *mm) 140 { 141 } 142 static inline void sme_dvmsync_batch(struct arch_tlbflush_unmap_batch *batch) 143 { 144 } 145 146 #endif /* CONFIG_ARM64_ERRATUM_4193714 */ 147 148 /* 149 * Level-based TLBI operations. 150 * 151 * When ARMv8.4-TTL exists, TLBI operations take an additional hint for 152 * the level at which the invalidation must take place. If the level is 153 * wrong, no invalidation may take place. In the case where the level 154 * cannot be easily determined, the value TLBI_TTL_UNKNOWN will perform 155 * a non-hinted invalidation. Any provided level outside the hint range 156 * will also cause fall-back to non-hinted invalidation. 157 * 158 * For Stage-2 invalidation, use the level values provided to that effect 159 * in asm/stage2_pgtable.h. 160 */ 161 #define TLBI_TTL_MASK GENMASK_ULL(47, 44) 162 163 #define TLBI_TTL_UNKNOWN INT_MAX 164 165 #define __tlbi_level(op, addr, level) do { \ 166 u64 arg = addr; \ 167 \ 168 if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && \ 169 level >= 0 && level <= 3) { \ 170 u64 ttl = level & 3; \ 171 ttl |= get_trans_granule() << 2; \ 172 arg &= ~TLBI_TTL_MASK; \ 173 arg |= FIELD_PREP(TLBI_TTL_MASK, ttl); \ 174 } \ 175 \ 176 __tlbi(op, arg); \ 177 } while(0) 178 179 #define __tlbi_user_level(op, arg, level) do { \ 180 if (arm64_kernel_unmapped_at_el0()) \ 181 __tlbi_level(op, (arg | USER_ASID_FLAG), level); \ 182 } while (0) 183 184 /* 185 * This macro creates a properly formatted VA operand for the TLB RANGE. The 186 * value bit assignments are: 187 * 188 * +----------+------+-------+-------+-------+----------------------+ 189 * | ASID | TG | SCALE | NUM | TTL | BADDR | 190 * +-----------------+-------+-------+-------+----------------------+ 191 * |63 48|47 46|45 44|43 39|38 37|36 0| 192 * 193 * The address range is determined by below formula: [BADDR, BADDR + (NUM + 1) * 194 * 2^(5*SCALE + 1) * PAGESIZE) 195 * 196 * Note that the first argument, baddr, is pre-shifted; If LPA2 is in use, BADDR 197 * holds addr[52:16]. Else BADDR holds page number. See for example ARM DDI 198 * 0487J.a section C5.5.60 "TLBI VAE1IS, TLBI VAE1ISNXS, TLB Invalidate by VA, 199 * EL1, Inner Shareable". 200 * 201 */ 202 #define TLBIR_ASID_MASK GENMASK_ULL(63, 48) 203 #define TLBIR_TG_MASK GENMASK_ULL(47, 46) 204 #define TLBIR_SCALE_MASK GENMASK_ULL(45, 44) 205 #define TLBIR_NUM_MASK GENMASK_ULL(43, 39) 206 #define TLBIR_TTL_MASK GENMASK_ULL(38, 37) 207 #define TLBIR_BADDR_MASK GENMASK_ULL(36, 0) 208 209 #define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl) \ 210 ({ \ 211 unsigned long __ta = 0; \ 212 unsigned long __ttl = (ttl >= 1 && ttl <= 3) ? ttl : 0; \ 213 __ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr); \ 214 __ta |= FIELD_PREP(TLBIR_TTL_MASK, __ttl); \ 215 __ta |= FIELD_PREP(TLBIR_NUM_MASK, num); \ 216 __ta |= FIELD_PREP(TLBIR_SCALE_MASK, scale); \ 217 __ta |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); \ 218 __ta |= FIELD_PREP(TLBIR_ASID_MASK, asid); \ 219 __ta; \ 220 }) 221 222 /* These macros are used by the TLBI RANGE feature. */ 223 #define __TLBI_RANGE_PAGES(num, scale) \ 224 ((unsigned long)((num) + 1) << (5 * (scale) + 1)) 225 #define MAX_TLBI_RANGE_PAGES __TLBI_RANGE_PAGES(31, 3) 226 227 /* 228 * Generate 'num' values from -1 to 31 with -1 rejected by the 229 * __flush_tlb_range() loop below. Its return value is only 230 * significant for a maximum of MAX_TLBI_RANGE_PAGES pages. If 231 * 'pages' is more than that, you must iterate over the overall 232 * range. 233 */ 234 #define __TLBI_RANGE_NUM(pages, scale) \ 235 ({ \ 236 int __pages = min((pages), \ 237 __TLBI_RANGE_PAGES(31, (scale))); \ 238 (__pages >> (5 * (scale) + 1)) - 1; \ 239 }) 240 241 #define __repeat_tlbi_sync(op, arg...) \ 242 do { \ 243 if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) \ 244 break; \ 245 __tlbi(op, ##arg); \ 246 dsb(ish); \ 247 } while (0) 248 249 /* 250 * Complete broadcast TLB maintenance issued by the host which invalidates 251 * stage 1 information in the host's own translation regime. 252 */ 253 static inline void __tlbi_sync_s1ish(struct mm_struct *mm) 254 { 255 dsb(ish); 256 __repeat_tlbi_sync(vale1is, 0); 257 sme_dvmsync(mm); 258 } 259 260 static inline void __tlbi_sync_s1ish_batch(struct arch_tlbflush_unmap_batch *batch) 261 { 262 dsb(ish); 263 __repeat_tlbi_sync(vale1is, 0); 264 sme_dvmsync_batch(batch); 265 } 266 267 static inline void __tlbi_sync_s1ish_kernel(void) 268 { 269 dsb(ish); 270 __repeat_tlbi_sync(vale1is, 0); 271 } 272 273 /* 274 * Complete broadcast TLB maintenance issued by hyp code which invalidates 275 * stage 1 translation information in any translation regime. 276 */ 277 static inline void __tlbi_sync_s1ish_hyp(void) 278 { 279 dsb(ish); 280 __repeat_tlbi_sync(vale2is, 0); 281 } 282 283 /* 284 * TLB Invalidation 285 * ================ 286 * 287 * This header file implements the low-level TLB invalidation routines 288 * (sometimes referred to as "flushing" in the kernel) for arm64. 289 * 290 * Every invalidation operation uses the following template: 291 * 292 * DSB ISHST // Ensure prior page-table updates have completed 293 * TLBI ... // Invalidate the TLB 294 * DSB ISH // Ensure the TLB invalidation has completed 295 * if (invalidated kernel mappings) 296 * ISB // Discard any instructions fetched from the old mapping 297 * 298 * 299 * The following functions form part of the "core" TLB invalidation API, 300 * as documented in Documentation/core-api/cachetlb.rst: 301 * 302 * flush_tlb_all() 303 * Invalidate the entire TLB (kernel + user) on all CPUs 304 * 305 * flush_tlb_mm(mm) 306 * Invalidate an entire user address space on all CPUs. 307 * The 'mm' argument identifies the ASID to invalidate. 308 * 309 * flush_tlb_range(vma, start, end) 310 * Invalidate the virtual-address range '[start, end)' on all 311 * CPUs for the user address space corresponding to 'vma->mm'. 312 * Note that this operation also invalidates any walk-cache 313 * entries associated with translations for the specified address 314 * range. 315 * 316 * flush_tlb_kernel_range(start, end) 317 * Same as flush_tlb_range(..., start, end), but applies to 318 * kernel mappings rather than a particular user address space. 319 * Whilst not explicitly documented, this function is used when 320 * unmapping pages from vmalloc/io space. 321 * 322 * flush_tlb_page(vma, addr) 323 * Invalidate a single user mapping for address 'addr' in the 324 * address space corresponding to 'vma->mm'. Note that this 325 * operation only invalidates a single, last-level page-table 326 * entry and therefore does not affect any walk-caches. 327 * 328 * 329 * Next, we have some undocumented invalidation routines that you probably 330 * don't want to call unless you know what you're doing: 331 * 332 * local_flush_tlb_all() 333 * Same as flush_tlb_all(), but only applies to the calling CPU. 334 * 335 * __flush_tlb_kernel_pgtable(addr) 336 * Invalidate a single kernel mapping for address 'addr' on all 337 * CPUs, ensuring that any walk-cache entries associated with the 338 * translation are also invalidated. 339 * 340 * __flush_tlb_range(vma, start, end, stride, last_level, tlb_level) 341 * Invalidate the virtual-address range '[start, end)' on all 342 * CPUs for the user address space corresponding to 'vma->mm'. 343 * The invalidation operations are issued at a granularity 344 * determined by 'stride' and only affect any walk-cache entries 345 * if 'last_level' is equal to false. tlb_level is the level at 346 * which the invalidation must take place. If the level is wrong, 347 * no invalidation may take place. In the case where the level 348 * cannot be easily determined, the value TLBI_TTL_UNKNOWN will 349 * perform a non-hinted invalidation. 350 * 351 * 352 * Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented 353 * on top of these routines, since that is our interface to the mmu_gather 354 * API as used by munmap() and friends. 355 */ 356 static inline void local_flush_tlb_all(void) 357 { 358 dsb(nshst); 359 __tlbi(vmalle1); 360 dsb(nsh); 361 isb(); 362 } 363 364 static inline void flush_tlb_all(void) 365 { 366 dsb(ishst); 367 __tlbi(vmalle1is); 368 __tlbi_sync_s1ish_kernel(); 369 isb(); 370 } 371 372 static inline void flush_tlb_mm(struct mm_struct *mm) 373 { 374 unsigned long asid; 375 376 dsb(ishst); 377 asid = __TLBI_VADDR(0, ASID(mm)); 378 __tlbi(aside1is, asid); 379 __tlbi_user(aside1is, asid); 380 __tlbi_sync_s1ish(mm); 381 mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL); 382 } 383 384 static inline void __flush_tlb_page_nosync(struct mm_struct *mm, 385 unsigned long uaddr) 386 { 387 unsigned long addr; 388 389 dsb(ishst); 390 addr = __TLBI_VADDR(uaddr, ASID(mm)); 391 __tlbi(vale1is, addr); 392 __tlbi_user(vale1is, addr); 393 mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK, 394 (uaddr & PAGE_MASK) + PAGE_SIZE); 395 } 396 397 static inline void flush_tlb_page_nosync(struct vm_area_struct *vma, 398 unsigned long uaddr) 399 { 400 return __flush_tlb_page_nosync(vma->vm_mm, uaddr); 401 } 402 403 static inline void flush_tlb_page(struct vm_area_struct *vma, 404 unsigned long uaddr) 405 { 406 flush_tlb_page_nosync(vma, uaddr); 407 __tlbi_sync_s1ish(vma->vm_mm); 408 } 409 410 static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm) 411 { 412 return true; 413 } 414 415 /* 416 * To support TLB batched flush for multiple pages unmapping, we only send 417 * the TLBI for each page in arch_tlbbatch_add_pending() and wait for the 418 * completion at the end in arch_tlbbatch_flush(). Since we've already issued 419 * TLBI for each page so only a DSB is needed to synchronise its effect on the 420 * other CPUs. 421 * 422 * This will save the time waiting on DSB comparing issuing a TLBI;DSB sequence 423 * for each page. 424 */ 425 static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) 426 { 427 __tlbi_sync_s1ish_batch(batch); 428 } 429 430 /* 431 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not 432 * necessarily a performance improvement. 433 */ 434 #define MAX_DVM_OPS PTRS_PER_PTE 435 436 /* 437 * __flush_tlb_range_op - Perform TLBI operation upon a range 438 * 439 * @op: TLBI instruction that operates on a range (has 'r' prefix) 440 * @start: The start address of the range 441 * @pages: Range as the number of pages from 'start' 442 * @stride: Flush granularity 443 * @asid: The ASID of the task (0 for IPA instructions) 444 * @tlb_level: Translation Table level hint, if known 445 * @tlbi_user: If 'true', call an additional __tlbi_user() 446 * (typically for user ASIDs). 'flase' for IPA instructions 447 * @lpa2: If 'true', the lpa2 scheme is used as set out below 448 * 449 * When the CPU does not support TLB range operations, flush the TLB 450 * entries one by one at the granularity of 'stride'. If the TLB 451 * range ops are supported, then: 452 * 453 * 1. If FEAT_LPA2 is in use, the start address of a range operation must be 454 * 64KB aligned, so flush pages one by one until the alignment is reached 455 * using the non-range operations. This step is skipped if LPA2 is not in 456 * use. 457 * 458 * 2. The minimum range granularity is decided by 'scale', so multiple range 459 * TLBI operations may be required. Start from scale = 3, flush the largest 460 * possible number of pages ((num+1)*2^(5*scale+1)) that fit into the 461 * requested range, then decrement scale and continue until one or zero pages 462 * are left. We must start from highest scale to ensure 64KB start alignment 463 * is maintained in the LPA2 case. 464 * 465 * 3. If there is 1 page remaining, flush it through non-range operations. Range 466 * operations can only span an even number of pages. We save this for last to 467 * ensure 64KB start alignment is maintained for the LPA2 case. 468 */ 469 #define __flush_tlb_range_op(op, start, pages, stride, \ 470 asid, tlb_level, tlbi_user, lpa2) \ 471 do { \ 472 typeof(start) __flush_start = start; \ 473 typeof(pages) __flush_pages = pages; \ 474 int num = 0; \ 475 int scale = 3; \ 476 int shift = lpa2 ? 16 : PAGE_SHIFT; \ 477 unsigned long addr; \ 478 \ 479 while (__flush_pages > 0) { \ 480 if (!system_supports_tlb_range() || \ 481 __flush_pages == 1 || \ 482 (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) { \ 483 addr = __TLBI_VADDR(__flush_start, asid); \ 484 __tlbi_level(op, addr, tlb_level); \ 485 if (tlbi_user) \ 486 __tlbi_user_level(op, addr, tlb_level); \ 487 __flush_start += stride; \ 488 __flush_pages -= stride >> PAGE_SHIFT; \ 489 continue; \ 490 } \ 491 \ 492 num = __TLBI_RANGE_NUM(__flush_pages, scale); \ 493 if (num >= 0) { \ 494 addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ 495 scale, num, tlb_level); \ 496 __tlbi(r##op, addr); \ 497 if (tlbi_user) \ 498 __tlbi_user(r##op, addr); \ 499 __flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ 500 __flush_pages -= __TLBI_RANGE_PAGES(num, scale);\ 501 } \ 502 scale--; \ 503 } \ 504 } while (0) 505 506 #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ 507 __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_lpa2_is_enabled()); 508 509 static inline bool __flush_tlb_range_limit_excess(unsigned long start, 510 unsigned long end, unsigned long pages, unsigned long stride) 511 { 512 /* 513 * When the system does not support TLB range based flush 514 * operation, (MAX_DVM_OPS - 1) pages can be handled. But 515 * with TLB range based operation, MAX_TLBI_RANGE_PAGES 516 * pages can be handled. 517 */ 518 if ((!system_supports_tlb_range() && 519 (end - start) >= (MAX_DVM_OPS * stride)) || 520 pages > MAX_TLBI_RANGE_PAGES) 521 return true; 522 523 return false; 524 } 525 526 static inline void __flush_tlb_range_nosync(struct mm_struct *mm, 527 unsigned long start, unsigned long end, 528 unsigned long stride, bool last_level, 529 int tlb_level) 530 { 531 unsigned long asid, pages; 532 533 start = round_down(start, stride); 534 end = round_up(end, stride); 535 pages = (end - start) >> PAGE_SHIFT; 536 537 if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { 538 flush_tlb_mm(mm); 539 return; 540 } 541 542 dsb(ishst); 543 asid = ASID(mm); 544 545 if (last_level) 546 __flush_tlb_range_op(vale1is, start, pages, stride, asid, 547 tlb_level, true, lpa2_is_enabled()); 548 else 549 __flush_tlb_range_op(vae1is, start, pages, stride, asid, 550 tlb_level, true, lpa2_is_enabled()); 551 552 mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); 553 } 554 555 static inline void __flush_tlb_range(struct vm_area_struct *vma, 556 unsigned long start, unsigned long end, 557 unsigned long stride, bool last_level, 558 int tlb_level) 559 { 560 __flush_tlb_range_nosync(vma->vm_mm, start, end, stride, 561 last_level, tlb_level); 562 __tlbi_sync_s1ish(vma->vm_mm); 563 } 564 565 static inline void flush_tlb_range(struct vm_area_struct *vma, 566 unsigned long start, unsigned long end) 567 { 568 /* 569 * We cannot use leaf-only invalidation here, since we may be invalidating 570 * table entries as part of collapsing hugepages or moving page tables. 571 * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough 572 * information here. 573 */ 574 __flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN); 575 } 576 577 static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) 578 { 579 const unsigned long stride = PAGE_SIZE; 580 unsigned long pages; 581 582 start = round_down(start, stride); 583 end = round_up(end, stride); 584 pages = (end - start) >> PAGE_SHIFT; 585 586 if (__flush_tlb_range_limit_excess(start, end, pages, stride)) { 587 flush_tlb_all(); 588 return; 589 } 590 591 dsb(ishst); 592 __flush_tlb_range_op(vaale1is, start, pages, stride, 0, 593 TLBI_TTL_UNKNOWN, false, lpa2_is_enabled()); 594 __tlbi_sync_s1ish_kernel(); 595 isb(); 596 } 597 598 /* 599 * Used to invalidate the TLB (walk caches) corresponding to intermediate page 600 * table levels (pgd/pud/pmd). 601 */ 602 static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr) 603 { 604 unsigned long addr = __TLBI_VADDR(kaddr, 0); 605 606 dsb(ishst); 607 __tlbi(vaae1is, addr); 608 __tlbi_sync_s1ish_kernel(); 609 isb(); 610 } 611 612 static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch, 613 struct mm_struct *mm, unsigned long start, unsigned long end) 614 { 615 __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3); 616 sme_dvmsync_add_pending(batch, mm); 617 } 618 #endif 619 620 #endif