Linux v6.18.37 · 원본 파일 · 온라인 원본
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Based on arch/arm/mm/proc.S
4 *
5 * Copyright (C) 2001 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 * Author: Catalin Marinas <[email protected]>
8 */
9
10#include <linux/init.h>
11#include <linux/linkage.h>
12#include <linux/pgtable.h>
13#include <linux/cfi_types.h>
14#include <asm/assembler.h>
15#include <asm/asm-offsets.h>
16#include <asm/asm_pointer_auth.h>
17#include <asm/hwcap.h>
18#include <asm/kernel-pgtable.h>
19#include <asm/pgtable-hwdef.h>
20#include <asm/cpufeature.h>
21#include <asm/alternative.h>
22#include <asm/smp.h>
23#include <asm/sysreg.h>
24
25#ifdef CONFIG_ARM64_64K_PAGES
26#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
27#elif defined(CONFIG_ARM64_16K_PAGES)
28#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
29#else /* CONFIG_ARM64_4K_PAGES */
30#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
31#endif
32
33#ifdef CONFIG_RANDOMIZE_BASE
34#define TCR_KASLR_FLAGS TCR_NFD1
35#else
36#define TCR_KASLR_FLAGS 0
37#endif
38
39/* PTWs cacheable, inner/outer WBWA */
40#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA
41
42#ifdef CONFIG_KASAN_SW_TAGS
43#define TCR_KASAN_SW_FLAGS TCR_TBI1 | TCR_TBID1
44#else
45#define TCR_KASAN_SW_FLAGS 0
46#endif
47
48#ifdef CONFIG_KASAN_HW_TAGS
49#define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1
50#elif defined(CONFIG_ARM64_MTE)
51/*
52 * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on
53 * TBI being enabled at EL1.
54 */
55#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1
56#else
57#define TCR_MTE_FLAGS 0
58#endif
59
60/*
61 * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and
62 * changed during mte_cpu_setup to Normal Tagged if the system supports MTE.
63 */
64#define MAIR_EL1_SET \
65 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \
66 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \
67 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \
68 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \
69 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED))
70
71#ifdef CONFIG_CPU_PM
72/**
73 * cpu_do_suspend - save CPU registers context
74 *
75 * x0: virtual address of context pointer
76 *
77 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
78 */
79SYM_FUNC_START(cpu_do_suspend)
80 mrs x2, tpidr_el0
81 mrs x3, tpidrro_el0
82 mrs x4, contextidr_el1
83 mrs x5, osdlr_el1
84 mrs x6, cpacr_el1
85 mrs x7, tcr_el1
86 mrs x8, vbar_el1
87 mrs x9, mdscr_el1
88 mrs x10, oslsr_el1
89 mrs x11, sctlr_el1
90 get_this_cpu_offset x12
91 mrs x13, sp_el0
92 stp x2, x3, [x0]
93 stp x4, x5, [x0, #16]
94 stp x6, x7, [x0, #32]
95 stp x8, x9, [x0, #48]
96 stp x10, x11, [x0, #64]
97 stp x12, x13, [x0, #80]
98 /*
99 * Save x18 as it may be used as a platform register, e.g. by shadow
100 * call stack.
101 */
102 str x18, [x0, #96]
103alternative_if ARM64_HAS_TCR2
104 mrs x2, REG_TCR2_EL1
105 str x2, [x0, #104]
106alternative_else_nop_endif
107 ret
108SYM_FUNC_END(cpu_do_suspend)
109
110/**
111 * cpu_do_resume - restore CPU register context
112 *
113 * x0: Address of context pointer
114 */
115SYM_FUNC_START(cpu_do_resume)
116 ldp x2, x3, [x0]
117 ldp x4, x5, [x0, #16]
118 ldp x6, x8, [x0, #32]
119 ldp x9, x10, [x0, #48]
120 ldp x11, x12, [x0, #64]
121 ldp x13, x14, [x0, #80]
122 /*
123 * Restore x18, as it may be used as a platform register, and clear
124 * the buffer to minimize the risk of exposure when used for shadow
125 * call stack.
126 */
127 ldr x18, [x0, #96]
128 str xzr, [x0, #96]
129 msr tpidr_el0, x2
130 msr tpidrro_el0, x3
131 msr contextidr_el1, x4
132 msr cpacr_el1, x6
133
134 /* Don't change t0sz here, mask those bits when restoring */
135 mrs x7, tcr_el1
136 bfi x8, x7, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH
137
138 msr tcr_el1, x8
139 msr vbar_el1, x9
140 msr mdscr_el1, x10
141alternative_if ARM64_HAS_TCR2
142 ldr x2, [x0, #104]
143 msr REG_TCR2_EL1, x2
144alternative_else_nop_endif
145
146 msr sctlr_el1, x12
147 set_this_cpu_offset x13
148 msr sp_el0, x14
149 /*
150 * Restore oslsr_el1 by writing oslar_el1
151 */
152 msr osdlr_el1, x5
153 ubfx x11, x11, #1, #1
154 msr oslar_el1, x11
155 reset_pmuserenr_el0 x0 // Disable PMU access from EL0
156 reset_amuserenr_el0 x0 // Disable AMU access from EL0
157
158alternative_if ARM64_HAS_RAS_EXTN
159 msr_s SYS_DISR_EL1, xzr
160alternative_else_nop_endif
161
162 ptrauth_keys_install_kernel_nosync x14, x1, x2, x3
163 isb
164 ret
165SYM_FUNC_END(cpu_do_resume)
166#endif
167
168 .pushsection ".idmap.text", "a"
169
170.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
171 adrp \tmp1, reserved_pg_dir
172 phys_to_ttbr \tmp2, \tmp1
173 offset_ttbr1 \tmp2, \tmp1
174 msr ttbr1_el1, \tmp2
175 isb
176 tlbi vmalle1
177 dsb nsh
178 isb
179.endm
180
181/*
182 * void idmap_cpu_replace_ttbr1(phys_addr_t ttbr1)
183 *
184 * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
185 * called by anything else. It can only be executed from a TTBR0 mapping.
186 */
187SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
188 __idmap_cpu_set_reserved_ttbr1 x1, x3
189
190 offset_ttbr1 x0, x3
191 msr ttbr1_el1, x0
192 isb
193
194 ret
195SYM_FUNC_END(idmap_cpu_replace_ttbr1)
196SYM_FUNC_ALIAS(__pi_idmap_cpu_replace_ttbr1, idmap_cpu_replace_ttbr1)
197 .popsection
198
199#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
200
201#define KPTI_NG_PTE_FLAGS (PTE_ATTRINDX(MT_NORMAL) | PTE_TYPE_PAGE | \
202 PTE_AF | PTE_SHARED | PTE_UXN | PTE_WRITE)
203
204 .pushsection ".idmap.text", "a"
205
206 .macro pte_to_phys, phys, pte
207 and \phys, \pte, #PTE_ADDR_LOW
208#ifdef CONFIG_ARM64_PA_BITS_52
209 and \pte, \pte, #PTE_ADDR_HIGH
210 orr \phys, \phys, \pte, lsl #PTE_ADDR_HIGH_SHIFT
211#endif
212 .endm
213
214 .macro kpti_mk_tbl_ng, type, num_entries
215 add end_\type\()p, cur_\type\()p, #\num_entries * 8
216.Ldo_\type:
217 ldr \type, [cur_\type\()p], #8 // Load the entry and advance
218 tbz \type, #0, .Lnext_\type // Skip invalid and
219 tbnz \type, #11, .Lnext_\type // non-global entries
220 orr \type, \type, #PTE_NG // Same bit for blocks and pages
221 str \type, [cur_\type\()p, #-8] // Update the entry
222 .ifnc \type, pte
223 tbnz \type, #1, .Lderef_\type
224 .endif
225.Lnext_\type:
226 cmp cur_\type\()p, end_\type\()p
227 b.ne .Ldo_\type
228 .endm
229
230 /*
231 * Dereference the current table entry and map it into the temporary
232 * fixmap slot associated with the current level.
233 */
234 .macro kpti_map_pgtbl, type, level
235 str xzr, [temp_pte, #8 * (\level + 2)] // break before make
236 dsb nshst
237 add pte, temp_pte, #PAGE_SIZE * (\level + 2)
238 lsr pte, pte, #12
239 tlbi vaae1, pte
240 dsb nsh
241 isb
242
243 phys_to_pte pte, cur_\type\()p
244 add cur_\type\()p, temp_pte, #PAGE_SIZE * (\level + 2)
245 orr pte, pte, pte_flags
246 str pte, [temp_pte, #8 * (\level + 2)]
247 dsb nshst
248 .endm
249
250/*
251 * void __kpti_install_ng_mappings(int cpu, int num_secondaries, phys_addr_t temp_pgd,
252 * unsigned long temp_pte_va)
253 *
254 * Called exactly once from stop_machine context by each CPU found during boot.
255 */
256SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
257 cpu .req w0
258 temp_pte .req x0
259 num_cpus .req w1
260 pte_flags .req x1
261 temp_pgd_phys .req x2
262 swapper_ttb .req x3
263 flag_ptr .req x4
264 cur_pgdp .req x5
265 end_pgdp .req x6
266 pgd .req x7
267 cur_pudp .req x8
268 end_pudp .req x9
269 cur_pmdp .req x11
270 end_pmdp .req x12
271 cur_ptep .req x14
272 end_ptep .req x15
273 pte .req x16
274 valid .req x17
275 cur_p4dp .req x19
276 end_p4dp .req x20
277
278 mov x5, x3 // preserve temp_pte arg
279 mrs swapper_ttb, ttbr1_el1
280 adr_l flag_ptr, idmap_kpti_bbml2_flag
281
282 cbnz cpu, __idmap_kpti_secondary
283
284#if CONFIG_PGTABLE_LEVELS > 4
285 stp x29, x30, [sp, #-32]!
286 mov x29, sp
287 stp x19, x20, [sp, #16]
288#endif
289
290 /* We're the boot CPU. Wait for the others to catch up */
291 sevl
2921: wfe
293 ldaxr w17, [flag_ptr]
294 eor w17, w17, num_cpus
295 cbnz w17, 1b
296
297 /* Switch to the temporary page tables on this CPU only */
298 __idmap_cpu_set_reserved_ttbr1 x8, x9
299 offset_ttbr1 temp_pgd_phys, x8
300 msr ttbr1_el1, temp_pgd_phys
301 isb
302
303 mov temp_pte, x5
304 mov_q pte_flags, KPTI_NG_PTE_FLAGS
305
306 /* Everybody is enjoying the idmap, so we can rewrite swapper. */
307
308#ifdef CONFIG_ARM64_LPA2
309 /*
310 * If LPA2 support is configured, but 52-bit virtual addressing is not
311 * enabled at runtime, we will fall back to one level of paging less,
312 * and so we have to walk swapper_pg_dir as if we dereferenced its
313 * address from a PGD level entry, and terminate the PGD level loop
314 * right after.
315 */
316 adrp pgd, swapper_pg_dir // walk &swapper_pg_dir at the next level
317 mov cur_pgdp, end_pgdp // must be equal to terminate the PGD loop
318alternative_if_not ARM64_HAS_VA52
319 b .Lderef_pgd // skip to the next level
320alternative_else_nop_endif
321 /*
322 * LPA2 based 52-bit virtual addressing requires 52-bit physical
323 * addressing to be enabled as well. In this case, the shareability
324 * bits are repurposed as physical address bits, and should not be
325 * set in pte_flags.
326 */
327 bic pte_flags, pte_flags, #PTE_SHARED
328#endif
329
330 /* PGD */
331 adrp cur_pgdp, swapper_pg_dir
332 kpti_map_pgtbl pgd, -1
333 kpti_mk_tbl_ng pgd, PTRS_PER_PGD
334
335 /* Ensure all the updated entries are visible to secondary CPUs */
336 dsb ishst
337
338 /* We're done: fire up swapper_pg_dir again */
339 __idmap_cpu_set_reserved_ttbr1 x8, x9
340 msr ttbr1_el1, swapper_ttb
341 isb
342
343 /* Set the flag to zero to indicate that we're all done */
344 str wzr, [flag_ptr]
345#if CONFIG_PGTABLE_LEVELS > 4
346 ldp x19, x20, [sp, #16]
347 ldp x29, x30, [sp], #32
348#endif
349 ret
350
351.Lderef_pgd:
352 /* P4D */
353 .if CONFIG_PGTABLE_LEVELS > 4
354 p4d .req x30
355 pte_to_phys cur_p4dp, pgd
356 kpti_map_pgtbl p4d, 0
357 kpti_mk_tbl_ng p4d, PTRS_PER_P4D
358 b .Lnext_pgd
359 .else /* CONFIG_PGTABLE_LEVELS <= 4 */
360 p4d .req pgd
361 .set .Lnext_p4d, .Lnext_pgd
362 .endif
363
364.Lderef_p4d:
365 /* PUD */
366 .if CONFIG_PGTABLE_LEVELS > 3
367 pud .req x10
368 pte_to_phys cur_pudp, p4d
369 kpti_map_pgtbl pud, 1
370 kpti_mk_tbl_ng pud, PTRS_PER_PUD
371 b .Lnext_p4d
372 .else /* CONFIG_PGTABLE_LEVELS <= 3 */
373 pud .req pgd
374 .set .Lnext_pud, .Lnext_pgd
375 .endif
376
377.Lderef_pud:
378 /* PMD */
379 .if CONFIG_PGTABLE_LEVELS > 2
380 pmd .req x13
381 pte_to_phys cur_pmdp, pud
382 kpti_map_pgtbl pmd, 2
383 kpti_mk_tbl_ng pmd, PTRS_PER_PMD
384 b .Lnext_pud
385 .else /* CONFIG_PGTABLE_LEVELS <= 2 */
386 pmd .req pgd
387 .set .Lnext_pmd, .Lnext_pgd
388 .endif
389
390.Lderef_pmd:
391 /* PTE */
392 pte_to_phys cur_ptep, pmd
393 kpti_map_pgtbl pte, 3
394 kpti_mk_tbl_ng pte, PTRS_PER_PTE
395 b .Lnext_pmd
396
397 .unreq cpu
398 .unreq temp_pte
399 .unreq num_cpus
400 .unreq pte_flags
401 .unreq temp_pgd_phys
402 .unreq cur_pgdp
403 .unreq end_pgdp
404 .unreq pgd
405 .unreq cur_pudp
406 .unreq end_pudp
407 .unreq pud
408 .unreq cur_pmdp
409 .unreq end_pmdp
410 .unreq pmd
411 .unreq cur_ptep
412 .unreq end_ptep
413 .unreq pte
414 .unreq valid
415 .unreq cur_p4dp
416 .unreq end_p4dp
417 .unreq p4d
418
419 /* Secondary CPUs end up here */
420__idmap_kpti_secondary:
421 /* Uninstall swapper before surgery begins */
422 __idmap_cpu_set_reserved_ttbr1 x16, x17
423 b scondary_cpu_wait
424
425 .unreq swapper_ttb
426 .unreq flag_ptr
427SYM_FUNC_END(idmap_kpti_install_ng_mappings)
428 .popsection
429#endif
430
431 .pushsection ".idmap.text", "a"
432SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes)
433 /* Must be same registers as in idmap_kpti_install_ng_mappings */
434 swapper_ttb .req x3
435 flag_ptr .req x4
436
437 mrs swapper_ttb, ttbr1_el1
438 adr_l flag_ptr, idmap_kpti_bbml2_flag
439 __idmap_cpu_set_reserved_ttbr1 x16, x17
440
441scondary_cpu_wait:
442 /* Increment the flag to let the boot CPU we're ready */
4431: ldxr w16, [flag_ptr]
444 add w16, w16, #1
445 stxr w17, w16, [flag_ptr]
446 cbnz w17, 1b
447
448 /* Wait for the boot CPU to finish messing around with swapper */
449 sevl
4501: wfe
451 ldxr w16, [flag_ptr]
452 cbnz w16, 1b
453
454 /* All done, act like nothing happened */
455 msr ttbr1_el1, swapper_ttb
456 isb
457 ret
458
459 .unreq swapper_ttb
460 .unreq flag_ptr
461SYM_FUNC_END(wait_linear_map_split_to_ptes)
462 .popsection
463
464/*
465 * __cpu_setup
466 *
467 * Initialise the processor for turning the MMU on.
468 *
469 * Output:
470 * Return in x0 the value of the SCTLR_EL1 register.
471 */
472 .pushsection ".idmap.text", "a"
473SYM_FUNC_START(__cpu_setup)
474 tlbi vmalle1 // Invalidate local TLB
475 dsb nsh
476
477 msr cpacr_el1, xzr // Reset cpacr_el1
478 mov x1, MDSCR_EL1_TDCC // Reset mdscr_el1 and disable
479 msr mdscr_el1, x1 // access to the DCC from EL0
480 reset_pmuserenr_el0 x1 // Disable PMU access from EL0
481 reset_amuserenr_el0 x1 // Disable AMU access from EL0
482
483 /*
484 * Default values for VMSA control registers. These will be adjusted
485 * below depending on detected CPU features.
486 */
487 mair .req x17
488 tcr .req x16
489 tcr2 .req x15
490 mov_q mair, MAIR_EL1_SET
491 mov_q tcr, TCR_T0SZ(IDMAP_VA_BITS) | TCR_T1SZ(VA_BITS_MIN) | TCR_CACHE_FLAGS | \
492 TCR_SHARED | TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
493 TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS
494 mov tcr2, xzr
495
496 tcr_clear_errata_bits tcr, x9, x5
497
498#ifdef CONFIG_ARM64_VA_BITS_52
499 mov x9, #64 - VA_BITS
500alternative_if ARM64_HAS_VA52
501 tcr_set_t1sz tcr, x9
502#ifdef CONFIG_ARM64_LPA2
503 orr tcr, tcr, #TCR_DS
504#endif
505alternative_else_nop_endif
506#endif
507
508 /*
509 * Set the IPS bits in TCR_EL1.
510 */
511 tcr_compute_pa_size tcr, #TCR_IPS_SHIFT, x5, x6
512#ifdef CONFIG_ARM64_HW_AFDBM
513 /*
514 * Enable hardware update of the Access Flags bit.
515 * Hardware dirty bit management is enabled later,
516 * via capabilities.
517 */
518 mrs x9, ID_AA64MMFR1_EL1
519 ubfx x9, x9, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, #4
520 cbz x9, 1f
521 orr tcr, tcr, #TCR_HA // hardware Access flag update
522#ifdef CONFIG_ARM64_HAFT
523 cmp x9, ID_AA64MMFR1_EL1_HAFDBS_HAFT
524 b.lt 1f
525 orr tcr2, tcr2, TCR2_EL1_HAFT
526#endif /* CONFIG_ARM64_HAFT */
5271:
528#endif /* CONFIG_ARM64_HW_AFDBM */
529 msr mair_el1, mair
530 msr tcr_el1, tcr
531
532 mrs_s x1, SYS_ID_AA64MMFR3_EL1
533 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4
534 cbz x1, .Lskip_indirection
535
536 mov_q x0, PIE_E0_ASM
537 msr REG_PIRE0_EL1, x0
538 mov_q x0, PIE_E1_ASM
539 msr REG_PIR_EL1, x0
540
541 orr tcr2, tcr2, TCR2_EL1_PIE
542
543.Lskip_indirection:
544
545 mrs_s x1, SYS_ID_AA64MMFR3_EL1
546 ubfx x1, x1, #ID_AA64MMFR3_EL1_TCRX_SHIFT, #4
547 cbz x1, 1f
548 msr REG_TCR2_EL1, tcr2
5491:
550
551 /*
552 * Prepare SCTLR
553 */
554 mov_q x0, INIT_SCTLR_EL1_MMU_ON
555 ret // return to head.S
556
557 .unreq mair
558 .unreq tcr
559 .unreq tcr2
560SYM_FUNC_END(__cpu_setup)