개념 설명 전체 · v6.18.37 / arch/arm64/kernel/entry-common.c
1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Exception handling code 4 * 5 * Copyright (C) 2019 ARM Ltd. 6 */ 7 8 #include <linux/context_tracking.h> 9 #include <linux/irq-entry-common.h> 10 #include <linux/kasan.h> 11 #include <linux/linkage.h> 12 #include <linux/livepatch.h> 13 #include <linux/lockdep.h> 14 #include <linux/ptrace.h> 15 #include <linux/resume_user_mode.h> 16 #include <linux/sched.h> 17 #include <linux/sched/debug.h> 18 #include <linux/thread_info.h> 19 20 #include <asm/cpufeature.h> 21 #include <asm/daifflags.h> 22 #include <asm/esr.h> 23 #include <asm/exception.h> 24 #include <asm/fpsimd.h> 25 #include <asm/irq_regs.h> 26 #include <asm/kprobes.h> 27 #include <asm/mmu.h> 28 #include <asm/processor.h> 29 #include <asm/sdei.h> 30 #include <asm/stacktrace.h> 31 #include <asm/sysreg.h> 32 #include <asm/system_misc.h> 33 34 /* 35 * Handle IRQ/context state management when entering from kernel mode. 36 * Before this function is called it is not safe to call regular kernel code, 37 * instrumentable code, or any code which may trigger an exception. 38 * 39 * This is intended to match the logic in irqentry_enter(), handling the kernel 40 * mode transitions only. 41 */ 42 static __always_inline irqentry_state_t __enter_from_kernel_mode(struct pt_regs *regs) 43 { 44 return irqentry_enter(regs); 45 } 46 47 static noinstr irqentry_state_t enter_from_kernel_mode(struct pt_regs *regs) 48 { 49 irqentry_state_t state; 50 51 state = __enter_from_kernel_mode(regs); 52 mte_check_tfsr_entry(); 53 mte_disable_tco_entry(current); 54 55 return state; 56 } 57 58 /* 59 * Handle IRQ/context state management when exiting to kernel mode. 60 * After this function returns it is not safe to call regular kernel code, 61 * instrumentable code, or any code which may trigger an exception. 62 * 63 * This is intended to match the logic in irqentry_exit(), handling the kernel 64 * mode transitions only, and with preemption handled elsewhere. 65 */ 66 static __always_inline void __exit_to_kernel_mode(struct pt_regs *regs, 67 irqentry_state_t state) 68 { 69 irqentry_exit(regs, state); 70 } 71 72 static void noinstr exit_to_kernel_mode(struct pt_regs *regs, 73 irqentry_state_t state) 74 { 75 mte_check_tfsr_exit(); 76 __exit_to_kernel_mode(regs, state); 77 } 78 79 /* 80 * Handle IRQ/context state management when entering from user mode. 81 * Before this function is called it is not safe to call regular kernel code, 82 * instrumentable code, or any code which may trigger an exception. 83 */ 84 static __always_inline void __enter_from_user_mode(struct pt_regs *regs) 85 { 86 enter_from_user_mode(regs); 87 mte_disable_tco_entry(current); 88 sme_enter_from_user_mode(); 89 } 90 91 static __always_inline void arm64_enter_from_user_mode(struct pt_regs *regs) 92 { 93 __enter_from_user_mode(regs); 94 } 95 96 /* 97 * Handle IRQ/context state management when exiting to user mode. 98 * After this function returns it is not safe to call regular kernel code, 99 * instrumentable code, or any code which may trigger an exception. 100 */ 101 102 static __always_inline void arm64_exit_to_user_mode(struct pt_regs *regs) 103 { 104 local_irq_disable(); 105 exit_to_user_mode_prepare(regs); 106 local_daif_mask(); 107 sme_exit_to_user_mode(); 108 mte_check_tfsr_exit(); 109 exit_to_user_mode(); 110 } 111 112 asmlinkage void noinstr asm_exit_to_user_mode(struct pt_regs *regs) 113 { 114 arm64_exit_to_user_mode(regs); 115 } 116 117 /* 118 * Handle IRQ/context state management when entering a debug exception from 119 * kernel mode. Before this function is called it is not safe to call regular 120 * kernel code, instrumentable code, or any code which may trigger an exception. 121 */ 122 static noinstr irqentry_state_t arm64_enter_el1_dbg(struct pt_regs *regs) 123 { 124 irqentry_state_t state; 125 126 state.lockdep = lockdep_hardirqs_enabled(); 127 128 lockdep_hardirqs_off(CALLER_ADDR0); 129 ct_nmi_enter(); 130 131 trace_hardirqs_off_finish(); 132 133 return state; 134 } 135 136 /* 137 * Handle IRQ/context state management when exiting a debug exception from 138 * kernel mode. After this function returns it is not safe to call regular 139 * kernel code, instrumentable code, or any code which may trigger an exception. 140 */ 141 static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs, 142 irqentry_state_t state) 143 { 144 if (state.lockdep) { 145 trace_hardirqs_on_prepare(); 146 lockdep_hardirqs_on_prepare(); 147 } 148 149 ct_nmi_exit(); 150 if (state.lockdep) 151 lockdep_hardirqs_on(CALLER_ADDR0); 152 } 153 154 static void do_interrupt_handler(struct pt_regs *regs, 155 void (*handler)(struct pt_regs *)) 156 { 157 struct pt_regs *old_regs = set_irq_regs(regs); 158 159 if (on_thread_stack()) 160 call_on_irq_stack(regs, handler); 161 else 162 handler(regs); 163 164 set_irq_regs(old_regs); 165 } 166 167 extern void (*handle_arch_irq)(struct pt_regs *); 168 extern void (*handle_arch_fiq)(struct pt_regs *); 169 170 static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, 171 unsigned long esr) 172 { 173 irqentry_nmi_enter(regs); 174 175 console_verbose(); 176 177 pr_crit("Unhandled %s exception on CPU%d, ESR 0x%016lx -- %s\n", 178 vector, smp_processor_id(), esr, 179 esr_get_class_string(esr)); 180 181 __show_regs(regs); 182 panic("Unhandled exception"); 183 } 184 185 #define UNHANDLED(el, regsize, vector) \ 186 asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ 187 { \ 188 const char *desc = #regsize "-bit " #el " " #vector; \ 189 __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ 190 } 191 192 #ifdef CONFIG_ARM64_ERRATUM_1463225 193 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); 194 195 static void cortex_a76_erratum_1463225_svc_handler(void) 196 { 197 u64 reg, val; 198 199 if (!unlikely(test_thread_flag(TIF_SINGLESTEP))) 200 return; 201 202 if (!unlikely(this_cpu_has_cap(ARM64_WORKAROUND_1463225))) 203 return; 204 205 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 1); 206 reg = read_sysreg(mdscr_el1); 207 val = reg | MDSCR_EL1_SS | MDSCR_EL1_KDE; 208 write_sysreg(val, mdscr_el1); 209 asm volatile("msr daifclr, #8"); 210 isb(); 211 212 /* We will have taken a single-step exception by this point */ 213 214 write_sysreg(reg, mdscr_el1); 215 __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); 216 } 217 218 static __always_inline bool 219 cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 220 { 221 if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) 222 return false; 223 224 /* 225 * We've taken a dummy step exception from the kernel to ensure 226 * that interrupts are re-enabled on the syscall path. Return back 227 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions 228 * masked so that we can safely restore the mdscr and get on with 229 * handling the syscall. 230 */ 231 regs->pstate |= PSR_D_BIT; 232 return true; 233 } 234 #else /* CONFIG_ARM64_ERRATUM_1463225 */ 235 static void cortex_a76_erratum_1463225_svc_handler(void) { } 236 static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) 237 { 238 return false; 239 } 240 #endif /* CONFIG_ARM64_ERRATUM_1463225 */ 241 242 /* 243 * As per the ABI exit SME streaming mode and clear the SVE state not 244 * shared with FPSIMD on syscall entry. 245 */ 246 static inline void fpsimd_syscall_enter(void) 247 { 248 /* Ensure PSTATE.SM is clear, but leave PSTATE.ZA as-is. */ 249 if (system_supports_sme()) 250 sme_smstop_sm(); 251 252 /* 253 * The CPU is not in streaming mode. If non-streaming SVE is not 254 * supported, there is no SVE state that needs to be discarded. 255 */ 256 if (!system_supports_sve()) 257 return; 258 259 if (test_thread_flag(TIF_SVE)) { 260 unsigned int sve_vq_minus_one; 261 262 sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1; 263 sve_flush_live(true, sve_vq_minus_one); 264 } 265 266 /* 267 * Any live non-FPSIMD SVE state has been zeroed. Allow 268 * fpsimd_save_user_state() to lazily discard SVE state until either 269 * the live state is unbound or fpsimd_syscall_exit() is called. 270 */ 271 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_FPSIMD); 272 } 273 274 static __always_inline void fpsimd_syscall_exit(void) 275 { 276 if (!system_supports_sve()) 277 return; 278 279 /* 280 * The current task's user FPSIMD/SVE/SME state is now bound to this 281 * CPU. The fpsimd_last_state.to_save value is either: 282 * 283 * - FP_STATE_FPSIMD, if the state has not been reloaded on this CPU 284 * since fpsimd_syscall_enter(). 285 * 286 * - FP_STATE_CURRENT, if the state has been reloaded on this CPU at 287 * any point. 288 * 289 * Reset this to FP_STATE_CURRENT to stop lazy discarding. 290 */ 291 __this_cpu_write(fpsimd_last_state.to_save, FP_STATE_CURRENT); 292 } 293 294 /* 295 * In debug exception context, we explicitly disable preemption despite 296 * having interrupts disabled. 297 * This serves two purposes: it makes it much less likely that we would 298 * accidentally schedule in exception context and it will force a warning 299 * if we somehow manage to schedule by accident. 300 */ 301 static void debug_exception_enter(struct pt_regs *regs) 302 { 303 preempt_disable(); 304 305 /* This code is a bit fragile. Test it. */ 306 RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work"); 307 } 308 NOKPROBE_SYMBOL(debug_exception_enter); 309 310 static void debug_exception_exit(struct pt_regs *regs) 311 { 312 preempt_enable_no_resched(); 313 } 314 NOKPROBE_SYMBOL(debug_exception_exit); 315 316 UNHANDLED(el1t, 64, sync) 317 UNHANDLED(el1t, 64, irq) 318 UNHANDLED(el1t, 64, fiq) 319 UNHANDLED(el1t, 64, error) 320 321 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) 322 { 323 unsigned long far = read_sysreg(far_el1); 324 irqentry_state_t state; 325 326 state = enter_from_kernel_mode(regs); 327 local_daif_inherit(regs); 328 do_mem_abort(far, esr, regs); 329 local_daif_mask(); 330 exit_to_kernel_mode(regs, state); 331 } 332 333 static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr) 334 { 335 unsigned long far = read_sysreg(far_el1); 336 irqentry_state_t state; 337 338 state = enter_from_kernel_mode(regs); 339 local_daif_inherit(regs); 340 do_sp_pc_abort(far, esr, regs); 341 local_daif_mask(); 342 exit_to_kernel_mode(regs, state); 343 } 344 345 static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr) 346 { 347 irqentry_state_t state; 348 349 state = enter_from_kernel_mode(regs); 350 local_daif_inherit(regs); 351 do_el1_undef(regs, esr); 352 local_daif_mask(); 353 exit_to_kernel_mode(regs, state); 354 } 355 356 static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr) 357 { 358 irqentry_state_t state; 359 360 state = enter_from_kernel_mode(regs); 361 local_daif_inherit(regs); 362 do_el1_bti(regs, esr); 363 local_daif_mask(); 364 exit_to_kernel_mode(regs, state); 365 } 366 367 static void noinstr el1_gcs(struct pt_regs *regs, unsigned long esr) 368 { 369 irqentry_state_t state; 370 371 state = enter_from_kernel_mode(regs); 372 local_daif_inherit(regs); 373 do_el1_gcs(regs, esr); 374 local_daif_mask(); 375 exit_to_kernel_mode(regs, state); 376 } 377 378 static void noinstr el1_mops(struct pt_regs *regs, unsigned long esr) 379 { 380 irqentry_state_t state; 381 382 state = enter_from_kernel_mode(regs); 383 local_daif_inherit(regs); 384 do_el1_mops(regs, esr); 385 local_daif_mask(); 386 exit_to_kernel_mode(regs, state); 387 } 388 389 static void noinstr el1_breakpt(struct pt_regs *regs, unsigned long esr) 390 { 391 irqentry_state_t state; 392 393 state = arm64_enter_el1_dbg(regs); 394 debug_exception_enter(regs); 395 do_breakpoint(esr, regs); 396 debug_exception_exit(regs); 397 arm64_exit_el1_dbg(regs, state); 398 } 399 400 static void noinstr el1_softstp(struct pt_regs *regs, unsigned long esr) 401 { 402 irqentry_state_t state; 403 404 state = arm64_enter_el1_dbg(regs); 405 if (!cortex_a76_erratum_1463225_debug_handler(regs)) { 406 debug_exception_enter(regs); 407 /* 408 * After handling a breakpoint, we suspend the breakpoint 409 * and use single-step to move to the next instruction. 410 * If we are stepping a suspended breakpoint there's nothing more to do: 411 * the single-step is complete. 412 */ 413 if (!try_step_suspended_breakpoints(regs)) 414 do_el1_softstep(esr, regs); 415 debug_exception_exit(regs); 416 } 417 arm64_exit_el1_dbg(regs, state); 418 } 419 420 static void noinstr el1_watchpt(struct pt_regs *regs, unsigned long esr) 421 { 422 /* Watchpoints are the only debug exception to write FAR_EL1 */ 423 unsigned long far = read_sysreg(far_el1); 424 irqentry_state_t state; 425 426 state = arm64_enter_el1_dbg(regs); 427 debug_exception_enter(regs); 428 do_watchpoint(far, esr, regs); 429 debug_exception_exit(regs); 430 arm64_exit_el1_dbg(regs, state); 431 } 432 433 static void noinstr el1_brk64(struct pt_regs *regs, unsigned long esr) 434 { 435 irqentry_state_t state; 436 437 state = arm64_enter_el1_dbg(regs); 438 debug_exception_enter(regs); 439 do_el1_brk64(esr, regs); 440 debug_exception_exit(regs); 441 arm64_exit_el1_dbg(regs, state); 442 } 443 444 static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) 445 { 446 irqentry_state_t state; 447 448 state = enter_from_kernel_mode(regs); 449 local_daif_inherit(regs); 450 do_el1_fpac(regs, esr); 451 local_daif_mask(); 452 exit_to_kernel_mode(regs, state); 453 } 454 455 asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) 456 { 457 unsigned long esr = read_sysreg(esr_el1); 458 459 switch (ESR_ELx_EC(esr)) { 460 case ESR_ELx_EC_DABT_CUR: 461 case ESR_ELx_EC_IABT_CUR: 462 el1_abort(regs, esr); 463 break; 464 /* 465 * We don't handle ESR_ELx_EC_SP_ALIGN, since we will have hit a 466 * recursive exception when trying to push the initial pt_regs. 467 */ 468 case ESR_ELx_EC_PC_ALIGN: 469 el1_pc(regs, esr); 470 break; 471 case ESR_ELx_EC_SYS64: 472 case ESR_ELx_EC_UNKNOWN: 473 el1_undef(regs, esr); 474 break; 475 case ESR_ELx_EC_BTI: 476 el1_bti(regs, esr); 477 break; 478 case ESR_ELx_EC_GCS: 479 el1_gcs(regs, esr); 480 break; 481 case ESR_ELx_EC_MOPS: 482 el1_mops(regs, esr); 483 break; 484 case ESR_ELx_EC_BREAKPT_CUR: 485 el1_breakpt(regs, esr); 486 break; 487 case ESR_ELx_EC_SOFTSTP_CUR: 488 el1_softstp(regs, esr); 489 break; 490 case ESR_ELx_EC_WATCHPT_CUR: 491 el1_watchpt(regs, esr); 492 break; 493 case ESR_ELx_EC_BRK64: 494 el1_brk64(regs, esr); 495 break; 496 case ESR_ELx_EC_FPAC: 497 el1_fpac(regs, esr); 498 break; 499 default: 500 __panic_unhandled(regs, "64-bit el1h sync", esr); 501 } 502 } 503 504 static __always_inline void __el1_pnmi(struct pt_regs *regs, 505 void (*handler)(struct pt_regs *)) 506 { 507 irqentry_state_t state; 508 509 state = irqentry_nmi_enter(regs); 510 do_interrupt_handler(regs, handler); 511 irqentry_nmi_exit(regs, state); 512 } 513 514 static __always_inline void __el1_irq(struct pt_regs *regs, 515 void (*handler)(struct pt_regs *)) 516 { 517 irqentry_state_t state; 518 519 state = enter_from_kernel_mode(regs); 520 521 irq_enter_rcu(); 522 do_interrupt_handler(regs, handler); 523 irq_exit_rcu(); 524 525 exit_to_kernel_mode(regs, state); 526 } 527 static void noinstr el1_interrupt(struct pt_regs *regs, 528 void (*handler)(struct pt_regs *)) 529 { 530 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 531 532 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && regs_irqs_disabled(regs)) 533 __el1_pnmi(regs, handler); 534 else 535 __el1_irq(regs, handler); 536 } 537 538 asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) 539 { 540 el1_interrupt(regs, handle_arch_irq); 541 } 542 543 asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) 544 { 545 el1_interrupt(regs, handle_arch_fiq); 546 } 547 548 asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) 549 { 550 unsigned long esr = read_sysreg(esr_el1); 551 irqentry_state_t state; 552 553 local_daif_restore(DAIF_ERRCTX); 554 state = irqentry_nmi_enter(regs); 555 do_serror(regs, esr); 556 irqentry_nmi_exit(regs, state); 557 } 558 559 static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) 560 { 561 unsigned long far = read_sysreg(far_el1); 562 563 arm64_enter_from_user_mode(regs); 564 local_daif_restore(DAIF_PROCCTX); 565 do_mem_abort(far, esr, regs); 566 arm64_exit_to_user_mode(regs); 567 } 568 569 static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr) 570 { 571 unsigned long far = read_sysreg(far_el1); 572 573 /* 574 * We've taken an instruction abort from userspace and not yet 575 * re-enabled IRQs. If the address is a kernel address, apply 576 * BP hardening prior to enabling IRQs and pre-emption. 577 */ 578 if (!is_ttbr0_addr(far)) 579 arm64_apply_bp_hardening(); 580 581 arm64_enter_from_user_mode(regs); 582 local_daif_restore(DAIF_PROCCTX); 583 do_mem_abort(far, esr, regs); 584 arm64_exit_to_user_mode(regs); 585 } 586 587 static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr) 588 { 589 arm64_enter_from_user_mode(regs); 590 local_daif_restore(DAIF_PROCCTX); 591 do_fpsimd_acc(esr, regs); 592 arm64_exit_to_user_mode(regs); 593 } 594 595 static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr) 596 { 597 arm64_enter_from_user_mode(regs); 598 local_daif_restore(DAIF_PROCCTX); 599 do_sve_acc(esr, regs); 600 arm64_exit_to_user_mode(regs); 601 } 602 603 static void noinstr el0_sme_acc(struct pt_regs *regs, unsigned long esr) 604 { 605 arm64_enter_from_user_mode(regs); 606 local_daif_restore(DAIF_PROCCTX); 607 do_sme_acc(esr, regs); 608 arm64_exit_to_user_mode(regs); 609 } 610 611 static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr) 612 { 613 arm64_enter_from_user_mode(regs); 614 local_daif_restore(DAIF_PROCCTX); 615 do_fpsimd_exc(esr, regs); 616 arm64_exit_to_user_mode(regs); 617 } 618 619 static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr) 620 { 621 arm64_enter_from_user_mode(regs); 622 local_daif_restore(DAIF_PROCCTX); 623 do_el0_sys(esr, regs); 624 arm64_exit_to_user_mode(regs); 625 } 626 627 static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr) 628 { 629 unsigned long far = read_sysreg(far_el1); 630 631 if (!is_ttbr0_addr(instruction_pointer(regs))) 632 arm64_apply_bp_hardening(); 633 634 arm64_enter_from_user_mode(regs); 635 local_daif_restore(DAIF_PROCCTX); 636 do_sp_pc_abort(far, esr, regs); 637 arm64_exit_to_user_mode(regs); 638 } 639 640 static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr) 641 { 642 arm64_enter_from_user_mode(regs); 643 local_daif_restore(DAIF_PROCCTX); 644 do_sp_pc_abort(regs->sp, esr, regs); 645 arm64_exit_to_user_mode(regs); 646 } 647 648 static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr) 649 { 650 arm64_enter_from_user_mode(regs); 651 local_daif_restore(DAIF_PROCCTX); 652 do_el0_undef(regs, esr); 653 arm64_exit_to_user_mode(regs); 654 } 655 656 static void noinstr el0_bti(struct pt_regs *regs) 657 { 658 arm64_enter_from_user_mode(regs); 659 local_daif_restore(DAIF_PROCCTX); 660 do_el0_bti(regs); 661 arm64_exit_to_user_mode(regs); 662 } 663 664 static void noinstr el0_mops(struct pt_regs *regs, unsigned long esr) 665 { 666 arm64_enter_from_user_mode(regs); 667 local_daif_restore(DAIF_PROCCTX); 668 do_el0_mops(regs, esr); 669 arm64_exit_to_user_mode(regs); 670 } 671 672 static void noinstr el0_gcs(struct pt_regs *regs, unsigned long esr) 673 { 674 arm64_enter_from_user_mode(regs); 675 local_daif_restore(DAIF_PROCCTX); 676 do_el0_gcs(regs, esr); 677 arm64_exit_to_user_mode(regs); 678 } 679 680 static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr) 681 { 682 arm64_enter_from_user_mode(regs); 683 local_daif_restore(DAIF_PROCCTX); 684 bad_el0_sync(regs, 0, esr); 685 arm64_exit_to_user_mode(regs); 686 } 687 688 static void noinstr el0_breakpt(struct pt_regs *regs, unsigned long esr) 689 { 690 if (!is_ttbr0_addr(regs->pc)) 691 arm64_apply_bp_hardening(); 692 693 arm64_enter_from_user_mode(regs); 694 debug_exception_enter(regs); 695 do_breakpoint(esr, regs); 696 debug_exception_exit(regs); 697 local_daif_restore(DAIF_PROCCTX); 698 arm64_exit_to_user_mode(regs); 699 } 700 701 static void noinstr el0_softstp(struct pt_regs *regs, unsigned long esr) 702 { 703 bool step_done; 704 705 if (!is_ttbr0_addr(regs->pc)) 706 arm64_apply_bp_hardening(); 707 708 arm64_enter_from_user_mode(regs); 709 /* 710 * After handling a breakpoint, we suspend the breakpoint 711 * and use single-step to move to the next instruction. 712 * If we are stepping a suspended breakpoint there's nothing more to do: 713 * the single-step is complete. 714 */ 715 step_done = try_step_suspended_breakpoints(regs); 716 local_daif_restore(DAIF_PROCCTX); 717 if (!step_done) 718 do_el0_softstep(esr, regs); 719 arm64_exit_to_user_mode(regs); 720 } 721 722 static void noinstr el0_watchpt(struct pt_regs *regs, unsigned long esr) 723 { 724 /* Watchpoints are the only debug exception to write FAR_EL1 */ 725 unsigned long far = read_sysreg(far_el1); 726 727 arm64_enter_from_user_mode(regs); 728 debug_exception_enter(regs); 729 do_watchpoint(far, esr, regs); 730 debug_exception_exit(regs); 731 local_daif_restore(DAIF_PROCCTX); 732 arm64_exit_to_user_mode(regs); 733 } 734 735 static void noinstr el0_brk64(struct pt_regs *regs, unsigned long esr) 736 { 737 arm64_enter_from_user_mode(regs); 738 local_daif_restore(DAIF_PROCCTX); 739 do_el0_brk64(esr, regs); 740 arm64_exit_to_user_mode(regs); 741 } 742 743 static void noinstr el0_svc(struct pt_regs *regs) 744 { 745 arm64_enter_from_user_mode(regs); 746 cortex_a76_erratum_1463225_svc_handler(); 747 fpsimd_syscall_enter(); 748 local_daif_restore(DAIF_PROCCTX); 749 do_el0_svc(regs); 750 arm64_exit_to_user_mode(regs); 751 fpsimd_syscall_exit(); 752 } 753 754 static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) 755 { 756 arm64_enter_from_user_mode(regs); 757 local_daif_restore(DAIF_PROCCTX); 758 do_el0_fpac(regs, esr); 759 arm64_exit_to_user_mode(regs); 760 } 761 762 asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) 763 { 764 unsigned long esr = read_sysreg(esr_el1); 765 766 switch (ESR_ELx_EC(esr)) { 767 case ESR_ELx_EC_SVC64: 768 el0_svc(regs); 769 break; 770 case ESR_ELx_EC_DABT_LOW: 771 el0_da(regs, esr); 772 break; 773 case ESR_ELx_EC_IABT_LOW: 774 el0_ia(regs, esr); 775 break; 776 case ESR_ELx_EC_FP_ASIMD: 777 el0_fpsimd_acc(regs, esr); 778 break; 779 case ESR_ELx_EC_SVE: 780 el0_sve_acc(regs, esr); 781 break; 782 case ESR_ELx_EC_SME: 783 el0_sme_acc(regs, esr); 784 break; 785 case ESR_ELx_EC_FP_EXC64: 786 el0_fpsimd_exc(regs, esr); 787 break; 788 case ESR_ELx_EC_SYS64: 789 case ESR_ELx_EC_WFx: 790 el0_sys(regs, esr); 791 break; 792 case ESR_ELx_EC_SP_ALIGN: 793 el0_sp(regs, esr); 794 break; 795 case ESR_ELx_EC_PC_ALIGN: 796 el0_pc(regs, esr); 797 break; 798 case ESR_ELx_EC_UNKNOWN: 799 el0_undef(regs, esr); 800 break; 801 case ESR_ELx_EC_BTI: 802 el0_bti(regs); 803 break; 804 case ESR_ELx_EC_MOPS: 805 el0_mops(regs, esr); 806 break; 807 case ESR_ELx_EC_GCS: 808 el0_gcs(regs, esr); 809 break; 810 case ESR_ELx_EC_BREAKPT_LOW: 811 el0_breakpt(regs, esr); 812 break; 813 case ESR_ELx_EC_SOFTSTP_LOW: 814 el0_softstp(regs, esr); 815 break; 816 case ESR_ELx_EC_WATCHPT_LOW: 817 el0_watchpt(regs, esr); 818 break; 819 case ESR_ELx_EC_BRK64: 820 el0_brk64(regs, esr); 821 break; 822 case ESR_ELx_EC_FPAC: 823 el0_fpac(regs, esr); 824 break; 825 default: 826 el0_inv(regs, esr); 827 } 828 } 829 830 static void noinstr el0_interrupt(struct pt_regs *regs, 831 void (*handler)(struct pt_regs *)) 832 { 833 arm64_enter_from_user_mode(regs); 834 835 write_sysreg(DAIF_PROCCTX_NOIRQ, daif); 836 837 if (regs->pc & BIT(55)) 838 arm64_apply_bp_hardening(); 839 840 irq_enter_rcu(); 841 do_interrupt_handler(regs, handler); 842 irq_exit_rcu(); 843 844 arm64_exit_to_user_mode(regs); 845 } 846 847 static void noinstr __el0_irq_handler_common(struct pt_regs *regs) 848 { 849 el0_interrupt(regs, handle_arch_irq); 850 } 851 852 asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) 853 { 854 __el0_irq_handler_common(regs); 855 } 856 857 static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) 858 { 859 el0_interrupt(regs, handle_arch_fiq); 860 } 861 862 asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) 863 { 864 __el0_fiq_handler_common(regs); 865 } 866 867 static void noinstr __el0_error_handler_common(struct pt_regs *regs) 868 { 869 unsigned long esr = read_sysreg(esr_el1); 870 irqentry_state_t state; 871 872 arm64_enter_from_user_mode(regs); 873 local_daif_restore(DAIF_ERRCTX); 874 state = irqentry_nmi_enter(regs); 875 do_serror(regs, esr); 876 irqentry_nmi_exit(regs, state); 877 local_daif_restore(DAIF_PROCCTX); 878 arm64_exit_to_user_mode(regs); 879 } 880 881 asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) 882 { 883 __el0_error_handler_common(regs); 884 } 885 886 #ifdef CONFIG_COMPAT 887 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) 888 { 889 arm64_enter_from_user_mode(regs); 890 local_daif_restore(DAIF_PROCCTX); 891 do_el0_cp15(esr, regs); 892 arm64_exit_to_user_mode(regs); 893 } 894 895 static void noinstr el0_svc_compat(struct pt_regs *regs) 896 { 897 arm64_enter_from_user_mode(regs); 898 cortex_a76_erratum_1463225_svc_handler(); 899 local_daif_restore(DAIF_PROCCTX); 900 do_el0_svc_compat(regs); 901 arm64_exit_to_user_mode(regs); 902 } 903 904 static void noinstr el0_bkpt32(struct pt_regs *regs, unsigned long esr) 905 { 906 arm64_enter_from_user_mode(regs); 907 local_daif_restore(DAIF_PROCCTX); 908 do_bkpt32(esr, regs); 909 arm64_exit_to_user_mode(regs); 910 } 911 912 asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) 913 { 914 unsigned long esr = read_sysreg(esr_el1); 915 916 switch (ESR_ELx_EC(esr)) { 917 case ESR_ELx_EC_SVC32: 918 el0_svc_compat(regs); 919 break; 920 case ESR_ELx_EC_DABT_LOW: 921 el0_da(regs, esr); 922 break; 923 case ESR_ELx_EC_IABT_LOW: 924 el0_ia(regs, esr); 925 break; 926 case ESR_ELx_EC_FP_ASIMD: 927 el0_fpsimd_acc(regs, esr); 928 break; 929 case ESR_ELx_EC_FP_EXC32: 930 el0_fpsimd_exc(regs, esr); 931 break; 932 case ESR_ELx_EC_PC_ALIGN: 933 el0_pc(regs, esr); 934 break; 935 case ESR_ELx_EC_UNKNOWN: 936 case ESR_ELx_EC_CP14_MR: 937 case ESR_ELx_EC_CP14_LS: 938 case ESR_ELx_EC_CP14_64: 939 el0_undef(regs, esr); 940 break; 941 case ESR_ELx_EC_CP15_32: 942 case ESR_ELx_EC_CP15_64: 943 el0_cp15(regs, esr); 944 break; 945 case ESR_ELx_EC_BREAKPT_LOW: 946 el0_breakpt(regs, esr); 947 break; 948 case ESR_ELx_EC_SOFTSTP_LOW: 949 el0_softstp(regs, esr); 950 break; 951 case ESR_ELx_EC_WATCHPT_LOW: 952 el0_watchpt(regs, esr); 953 break; 954 case ESR_ELx_EC_BKPT32: 955 el0_bkpt32(regs, esr); 956 break; 957 default: 958 el0_inv(regs, esr); 959 } 960 } 961 962 asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) 963 { 964 __el0_irq_handler_common(regs); 965 } 966 967 asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) 968 { 969 __el0_fiq_handler_common(regs); 970 } 971 972 asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) 973 { 974 __el0_error_handler_common(regs); 975 } 976 #else /* CONFIG_COMPAT */ 977 UNHANDLED(el0t, 32, sync) 978 UNHANDLED(el0t, 32, irq) 979 UNHANDLED(el0t, 32, fiq) 980 UNHANDLED(el0t, 32, error) 981 #endif /* CONFIG_COMPAT */ 982 983 asmlinkage void noinstr __noreturn handle_bad_stack(struct pt_regs *regs) 984 { 985 unsigned long esr = read_sysreg(esr_el1); 986 unsigned long far = read_sysreg(far_el1); 987 988 irqentry_nmi_enter(regs); 989 panic_bad_stack(regs, esr, far); 990 } 991 992 #ifdef CONFIG_ARM_SDE_INTERFACE 993 asmlinkage noinstr unsigned long 994 __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) 995 { 996 irqentry_state_t state; 997 unsigned long ret; 998 999 /* 1000 * We didn't take an exception to get here, so the HW hasn't 1001 * set/cleared bits in PSTATE that we may rely on. 1002 * 1003 * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to 1004 * whether PSTATE bits are inherited unchanged or generated from 1005 * scratch, and the TF-A implementation always clears PAN and always 1006 * clears UAO. There are no other known implementations. 1007 * 1008 * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how 1009 * PSTATE is modified upon architectural exceptions, and so PAN is 1010 * either inherited or set per SCTLR_ELx.SPAN, and UAO is always 1011 * cleared. 1012 * 1013 * We must explicitly reset PAN to the expected state, including 1014 * clearing it when the host isn't using it, in case a VM had it set. 1015 */ 1016 if (system_uses_hw_pan()) 1017 set_pstate_pan(1); 1018 else if (cpu_has_pan()) 1019 set_pstate_pan(0); 1020 1021 state = irqentry_nmi_enter(regs); 1022 ret = do_sdei_event(regs, arg); 1023 irqentry_nmi_exit(regs, state); 1024 1025 return ret; 1026 } 1027 #endif /* CONFIG_ARM_SDE_INTERFACE */