Linux v6.18.37 · 원본 파일 · 온라인 원본
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * FP/SIMD context switching and fault handling
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <[email protected]>
7 */
8
9#include <linux/bitmap.h>
10#include <linux/bitops.h>
11#include <linux/bottom_half.h>
12#include <linux/bug.h>
13#include <linux/cache.h>
14#include <linux/compat.h>
15#include <linux/compiler.h>
16#include <linux/cpu.h>
17#include <linux/cpu_pm.h>
18#include <linux/cpumask.h>
19#include <linux/ctype.h>
20#include <linux/kernel.h>
21#include <linux/linkage.h>
22#include <linux/irqflags.h>
23#include <linux/init.h>
24#include <linux/percpu.h>
25#include <linux/prctl.h>
26#include <linux/preempt.h>
27#include <linux/ptrace.h>
28#include <linux/sched/signal.h>
29#include <linux/sched/task_stack.h>
30#include <linux/signal.h>
31#include <linux/slab.h>
32#include <linux/smp.h>
33#include <linux/stddef.h>
34#include <linux/sysctl.h>
35#include <linux/swab.h>
36
37#include <asm/esr.h>
38#include <asm/exception.h>
39#include <asm/fpsimd.h>
40#include <asm/cpufeature.h>
41#include <asm/cputype.h>
42#include <asm/neon.h>
43#include <asm/processor.h>
44#include <asm/simd.h>
45#include <asm/sigcontext.h>
46#include <asm/sysreg.h>
47#include <asm/traps.h>
48#include <asm/virt.h>
49
50#define FPEXC_IOF (1 << 0)
51#define FPEXC_DZF (1 << 1)
52#define FPEXC_OFF (1 << 2)
53#define FPEXC_UFF (1 << 3)
54#define FPEXC_IXF (1 << 4)
55#define FPEXC_IDF (1 << 7)
56
57/*
58 * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
59 *
60 * In order to reduce the number of times the FPSIMD state is needlessly saved
61 * and restored, we need to keep track of two things:
62 * (a) for each task, we need to remember which CPU was the last one to have
63 * the task's FPSIMD state loaded into its FPSIMD registers;
64 * (b) for each CPU, we need to remember which task's userland FPSIMD state has
65 * been loaded into its FPSIMD registers most recently, or whether it has
66 * been used to perform kernel mode NEON in the meantime.
67 *
68 * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to
69 * the id of the current CPU every time the state is loaded onto a CPU. For (b),
70 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
71 * address of the userland FPSIMD state of the task that was loaded onto the CPU
72 * the most recently, or NULL if kernel mode NEON has been performed after that.
73 *
74 * With this in place, we no longer have to restore the next FPSIMD state right
75 * when switching between tasks. Instead, we can defer this check to userland
76 * resume, at which time we verify whether the CPU's fpsimd_last_state and the
77 * task's fpsimd_cpu are still mutually in sync. If this is the case, we
78 * can omit the FPSIMD restore.
79 *
80 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
81 * indicate whether or not the userland FPSIMD state of the current task is
82 * present in the registers. The flag is set unless the FPSIMD registers of this
83 * CPU currently contain the most recent userland FPSIMD state of the current
84 * task. If the task is behaving as a VMM, then this is will be managed by
85 * KVM which will clear it to indicate that the vcpu FPSIMD state is currently
86 * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
87 * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and
88 * flag the register state as invalid.
89 *
90 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may be
91 * called from softirq context, which will save the task's FPSIMD context back
92 * to task_struct. To prevent this from racing with the manipulation of the
93 * task's FPSIMD state from task context and thereby corrupting the state, it
94 * is necessary to protect any manipulation of a task's fpsimd_state or
95 * TIF_FOREIGN_FPSTATE flag with get_cpu_fpsimd_context(), which will suspend
96 * softirq servicing entirely until put_cpu_fpsimd_context() is called.
97 *
98 * For a certain task, the sequence may look something like this:
99 * - the task gets scheduled in; if both the task's fpsimd_cpu field
100 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
101 * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
102 * cleared, otherwise it is set;
103 *
104 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
105 * userland FPSIMD state is copied from memory to the registers, the task's
106 * fpsimd_cpu field is set to the id of the current CPU, the current
107 * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
108 * TIF_FOREIGN_FPSTATE flag is cleared;
109 *
110 * - the task executes an ordinary syscall; upon return to userland, the
111 * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
112 * restored;
113 *
114 * - the task executes a syscall which executes some NEON instructions; this is
115 * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
116 * register contents to memory, clears the fpsimd_last_state per-cpu variable
117 * and sets the TIF_FOREIGN_FPSTATE flag;
118 *
119 * - the task gets preempted after kernel_neon_end() is called; as we have not
120 * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
121 * whatever is in the FPSIMD registers is not saved to memory, but discarded.
122 */
123
124DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
125
126__ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = {
127#ifdef CONFIG_ARM64_SVE
128 [ARM64_VEC_SVE] = {
129 .type = ARM64_VEC_SVE,
130 .name = "SVE",
131 .min_vl = SVE_VL_MIN,
132 .max_vl = SVE_VL_MIN,
133 .max_virtualisable_vl = SVE_VL_MIN,
134 },
135#endif
136#ifdef CONFIG_ARM64_SME
137 [ARM64_VEC_SME] = {
138 .type = ARM64_VEC_SME,
139 .name = "SME",
140 },
141#endif
142};
143
144static unsigned int vec_vl_inherit_flag(enum vec_type type)
145{
146 switch (type) {
147 case ARM64_VEC_SVE:
148 return TIF_SVE_VL_INHERIT;
149 case ARM64_VEC_SME:
150 return TIF_SME_VL_INHERIT;
151 default:
152 WARN_ON_ONCE(1);
153 return 0;
154 }
155}
156
157struct vl_config {
158 int __default_vl; /* Default VL for tasks */
159};
160
161static struct vl_config vl_config[ARM64_VEC_MAX];
162
163static inline int get_default_vl(enum vec_type type)
164{
165 return READ_ONCE(vl_config[type].__default_vl);
166}
167
168#ifdef CONFIG_ARM64_SVE
169
170static inline int get_sve_default_vl(void)
171{
172 return get_default_vl(ARM64_VEC_SVE);
173}
174
175static inline void set_default_vl(enum vec_type type, int val)
176{
177 WRITE_ONCE(vl_config[type].__default_vl, val);
178}
179
180static inline void set_sve_default_vl(int val)
181{
182 set_default_vl(ARM64_VEC_SVE, val);
183}
184
185static u8 *efi_sve_state;
186
187#else /* ! CONFIG_ARM64_SVE */
188
189/* Dummy declaration for code that will be optimised out: */
190extern u8 *efi_sve_state;
191
192#endif /* ! CONFIG_ARM64_SVE */
193
194#ifdef CONFIG_ARM64_SME
195
196static int get_sme_default_vl(void)
197{
198 return get_default_vl(ARM64_VEC_SME);
199}
200
201static void set_sme_default_vl(int val)
202{
203 set_default_vl(ARM64_VEC_SME, val);
204}
205
206static void sme_free(struct task_struct *);
207
208#else
209
210static inline void sme_free(struct task_struct *t) { }
211
212#endif
213
214static void fpsimd_bind_task_to_cpu(void);
215
216/*
217 * Claim ownership of the CPU FPSIMD context for use by the calling context.
218 *
219 * The caller may freely manipulate the FPSIMD context metadata until
220 * put_cpu_fpsimd_context() is called.
221 *
222 * On RT kernels local_bh_disable() is not sufficient because it only
223 * serializes soft interrupt related sections via a local lock, but stays
224 * preemptible. Disabling preemption is the right choice here as bottom
225 * half processing is always in thread context on RT kernels so it
226 * implicitly prevents bottom half processing as well.
227 */
228static void get_cpu_fpsimd_context(void)
229{
230 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
231 local_bh_disable();
232 else
233 preempt_disable();
234}
235
236/*
237 * Release the CPU FPSIMD context.
238 *
239 * Must be called from a context in which get_cpu_fpsimd_context() was
240 * previously called, with no call to put_cpu_fpsimd_context() in the
241 * meantime.
242 */
243static void put_cpu_fpsimd_context(void)
244{
245 if (!IS_ENABLED(CONFIG_PREEMPT_RT))
246 local_bh_enable();
247 else
248 preempt_enable();
249}
250
251unsigned int task_get_vl(const struct task_struct *task, enum vec_type type)
252{
253 return task->thread.vl[type];
254}
255
256void task_set_vl(struct task_struct *task, enum vec_type type,
257 unsigned long vl)
258{
259 task->thread.vl[type] = vl;
260}
261
262unsigned int task_get_vl_onexec(const struct task_struct *task,
263 enum vec_type type)
264{
265 return task->thread.vl_onexec[type];
266}
267
268void task_set_vl_onexec(struct task_struct *task, enum vec_type type,
269 unsigned long vl)
270{
271 task->thread.vl_onexec[type] = vl;
272}
273
274/*
275 * TIF_SME controls whether a task can use SME without trapping while
276 * in userspace, when TIF_SME is set then we must have storage
277 * allocated in sve_state and sme_state to store the contents of both ZA
278 * and the SVE registers for both streaming and non-streaming modes.
279 *
280 * If both SVCR.ZA and SVCR.SM are disabled then at any point we
281 * may disable TIF_SME and reenable traps.
282 */
283
284
285/*
286 * TIF_SVE controls whether a task can use SVE without trapping while
287 * in userspace, and also (together with TIF_SME) the way a task's
288 * FPSIMD/SVE state is stored in thread_struct.
289 *
290 * The kernel uses this flag to track whether a user task is actively
291 * using SVE, and therefore whether full SVE register state needs to
292 * be tracked. If not, the cheaper FPSIMD context handling code can
293 * be used instead of the more costly SVE equivalents.
294 *
295 * * TIF_SVE or SVCR.SM set:
296 *
297 * The task can execute SVE instructions while in userspace without
298 * trapping to the kernel.
299 *
300 * During any syscall, the kernel may optionally clear TIF_SVE and
301 * discard the vector state except for the FPSIMD subset.
302 *
303 * * TIF_SVE clear:
304 *
305 * An attempt by the user task to execute an SVE instruction causes
306 * do_sve_acc() to be called, which does some preparation and then
307 * sets TIF_SVE.
308 *
309 * During any syscall, the kernel may optionally clear TIF_SVE and
310 * discard the vector state except for the FPSIMD subset.
311 *
312 * The data will be stored in one of two formats:
313 *
314 * * FPSIMD only - FP_STATE_FPSIMD:
315 *
316 * When the FPSIMD only state stored task->thread.fp_type is set to
317 * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
318 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
319 * logically zero but not stored anywhere; P0-P15 and FFR are not
320 * stored and have unspecified values from userspace's point of
321 * view. For hygiene purposes, the kernel zeroes them on next use,
322 * but userspace is discouraged from relying on this.
323 *
324 * task->thread.sve_state does not need to be non-NULL, valid or any
325 * particular size: it must not be dereferenced and any data stored
326 * there should be considered stale and not referenced.
327 *
328 * * SVE state - FP_STATE_SVE:
329 *
330 * When the full SVE state is stored task->thread.fp_type is set to
331 * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
332 * corresponding Zn), P0-P15 and FFR are encoded in in
333 * task->thread.sve_state, formatted appropriately for vector
334 * length task->thread.sve_vl or, if SVCR.SM is set,
335 * task->thread.sme_vl. The storage for the vector registers in
336 * task->thread.uw.fpsimd_state should be ignored.
337 *
338 * task->thread.sve_state must point to a valid buffer at least
339 * sve_state_size(task) bytes in size. The data stored in
340 * task->thread.uw.fpsimd_state.vregs should be considered stale
341 * and not referenced.
342 *
343 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
344 * irrespective of whether TIF_SVE is clear or set, since these are
345 * not vector length dependent.
346 */
347
348/*
349 * Update current's FPSIMD/SVE registers from thread_struct.
350 *
351 * This function should be called only when the FPSIMD/SVE state in
352 * thread_struct is known to be up to date, when preparing to enter
353 * userspace.
354 */
355static void task_fpsimd_load(void)
356{
357 bool restore_sve_regs = false;
358 bool restore_ffr;
359
360 WARN_ON(!system_supports_fpsimd());
361 WARN_ON(preemptible());
362 WARN_ON(test_thread_flag(TIF_KERNEL_FPSTATE));
363
364 if (system_supports_sve() || system_supports_sme()) {
365 switch (current->thread.fp_type) {
366 case FP_STATE_FPSIMD:
367 /* Stop tracking SVE for this task until next use. */
368 clear_thread_flag(TIF_SVE);
369 break;
370 case FP_STATE_SVE:
371 if (!thread_sm_enabled(¤t->thread))
372 WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE));
373
374 if (test_thread_flag(TIF_SVE))
375 sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1);
376
377 restore_sve_regs = true;
378 restore_ffr = true;
379 break;
380 default:
381 /*
382 * This indicates either a bug in
383 * fpsimd_save_user_state() or memory corruption, we
384 * should always record an explicit format
385 * when we save. We always at least have the
386 * memory allocated for FPSIMD registers so
387 * try that and hope for the best.
388 */
389 WARN_ON_ONCE(1);
390 clear_thread_flag(TIF_SVE);
391 break;
392 }
393 }
394
395 /* Restore SME, override SVE register configuration if needed */
396 if (system_supports_sme()) {
397 unsigned long sme_vl = task_get_sme_vl(current);
398
399 /* Ensure VL is set up for restoring data */
400 if (test_thread_flag(TIF_SME))
401 sme_set_vq(sve_vq_from_vl(sme_vl) - 1);
402
403 write_sysreg_s(current->thread.svcr, SYS_SVCR);
404
405 if (thread_za_enabled(¤t->thread))
406 sme_load_state(current->thread.sme_state,
407 system_supports_sme2());
408
409 if (thread_sm_enabled(¤t->thread))
410 restore_ffr = system_supports_fa64();
411 }
412
413 if (system_supports_fpmr())
414 write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR);
415
416 if (restore_sve_regs) {
417 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE);
418 sve_load_state(sve_pffr(¤t->thread),
419 ¤t->thread.uw.fpsimd_state.fpsr,
420 restore_ffr);
421 } else {
422 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD);
423 fpsimd_load_state(¤t->thread.uw.fpsimd_state);
424 }
425}
426
427/*
428 * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
429 * date with respect to the CPU registers. Note carefully that the
430 * current context is the context last bound to the CPU stored in
431 * last, if KVM is involved this may be the guest VM context rather
432 * than the host thread for the VM pointed to by current. This means
433 * that we must always reference the state storage via last rather
434 * than via current, if we are saving KVM state then it will have
435 * ensured that the type of registers to save is set in last->to_save.
436 */
437static void fpsimd_save_user_state(void)
438{
439 struct cpu_fp_state const *last =
440 this_cpu_ptr(&fpsimd_last_state);
441 /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
442 bool save_sve_regs = false;
443 bool save_ffr;
444 unsigned int vl;
445
446 WARN_ON(!system_supports_fpsimd());
447 WARN_ON(preemptible());
448
449 if (test_thread_flag(TIF_FOREIGN_FPSTATE))
450 return;
451
452 if (system_supports_fpmr())
453 *(last->fpmr) = read_sysreg_s(SYS_FPMR);
454
455 /*
456 * Save SVE state if it is live.
457 *
458 * The syscall ABI discards live SVE state at syscall entry. When
459 * entering a syscall, fpsimd_syscall_enter() sets to_save to
460 * FP_STATE_FPSIMD to allow the SVE state to be lazily discarded until
461 * either new SVE state is loaded+bound or fpsimd_syscall_exit() is
462 * called prior to a return to userspace.
463 */
464 if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) ||
465 last->to_save == FP_STATE_SVE) {
466 save_sve_regs = true;
467 save_ffr = true;
468 vl = last->sve_vl;
469 }
470
471 if (system_supports_sme()) {
472 u64 *svcr = last->svcr;
473
474 *svcr = read_sysreg_s(SYS_SVCR);
475
476 if (*svcr & SVCR_ZA_MASK)
477 sme_save_state(last->sme_state,
478 system_supports_sme2());
479
480 /* If we are in streaming mode override regular SVE. */
481 if (*svcr & SVCR_SM_MASK) {
482 save_sve_regs = true;
483 save_ffr = system_supports_fa64();
484 vl = last->sme_vl;
485 }
486 }
487
488 if (IS_ENABLED(CONFIG_ARM64_SVE) && save_sve_regs) {
489 /* Get the configured VL from RDVL, will account for SM */
490 if (WARN_ON(sve_get_vl() != vl)) {
491 /*
492 * Can't save the user regs, so current would
493 * re-enter user with corrupt state.
494 * There's no way to recover, so kill it:
495 */
496 force_signal_inject(SIGKILL, SI_KERNEL, 0, 0);
497 return;
498 }
499
500 sve_save_state((char *)last->sve_state +
501 sve_ffr_offset(vl),
502 &last->st->fpsr, save_ffr);
503 *last->fp_type = FP_STATE_SVE;
504 } else {
505 fpsimd_save_state(last->st);
506 *last->fp_type = FP_STATE_FPSIMD;
507 }
508}
509
510/*
511 * All vector length selection from userspace comes through here.
512 * We're on a slow path, so some sanity-checks are included.
513 * If things go wrong there's a bug somewhere, but try to fall back to a
514 * safe choice.
515 */
516static unsigned int find_supported_vector_length(enum vec_type type,
517 unsigned int vl)
518{
519 struct vl_info *info = &vl_info[type];
520 int bit;
521 int max_vl = info->max_vl;
522
523 if (WARN_ON(!sve_vl_valid(vl)))
524 vl = info->min_vl;
525
526 if (WARN_ON(!sve_vl_valid(max_vl)))
527 max_vl = info->min_vl;
528
529 if (vl > max_vl)
530 vl = max_vl;
531 if (vl < info->min_vl)
532 vl = info->min_vl;
533
534 bit = find_next_bit(info->vq_map, SVE_VQ_MAX,
535 __vq_to_bit(sve_vq_from_vl(vl)));
536 return sve_vl_from_vq(__bit_to_vq(bit));
537}
538
539#if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL)
540
541static int vec_proc_do_default_vl(const struct ctl_table *table, int write,
542 void *buffer, size_t *lenp, loff_t *ppos)
543{
544 struct vl_info *info = table->extra1;
545 enum vec_type type = info->type;
546 int ret;
547 int vl = get_default_vl(type);
548 struct ctl_table tmp_table = {
549 .data = &vl,
550 .maxlen = sizeof(vl),
551 };
552
553 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
554 if (ret || !write)
555 return ret;
556
557 /* Writing -1 has the special meaning "set to max": */
558 if (vl == -1)
559 vl = info->max_vl;
560
561 if (!sve_vl_valid(vl))
562 return -EINVAL;
563
564 set_default_vl(type, find_supported_vector_length(type, vl));
565 return 0;
566}
567
568static const struct ctl_table sve_default_vl_table[] = {
569 {
570 .procname = "sve_default_vector_length",
571 .mode = 0644,
572 .proc_handler = vec_proc_do_default_vl,
573 .extra1 = &vl_info[ARM64_VEC_SVE],
574 },
575};
576
577static int __init sve_sysctl_init(void)
578{
579 if (system_supports_sve())
580 if (!register_sysctl("abi", sve_default_vl_table))
581 return -EINVAL;
582
583 return 0;
584}
585
586#else /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
587static int __init sve_sysctl_init(void) { return 0; }
588#endif /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */
589
590#if defined(CONFIG_ARM64_SME) && defined(CONFIG_SYSCTL)
591static const struct ctl_table sme_default_vl_table[] = {
592 {
593 .procname = "sme_default_vector_length",
594 .mode = 0644,
595 .proc_handler = vec_proc_do_default_vl,
596 .extra1 = &vl_info[ARM64_VEC_SME],
597 },
598};
599
600static int __init sme_sysctl_init(void)
601{
602 if (system_supports_sme())
603 if (!register_sysctl("abi", sme_default_vl_table))
604 return -EINVAL;
605
606 return 0;
607}
608
609#else /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */
610static int __init sme_sysctl_init(void) { return 0; }
611#endif /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */
612
613#define ZREG(sve_state, vq, n) ((char *)(sve_state) + \
614 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
615
616#ifdef CONFIG_CPU_BIG_ENDIAN
617static __uint128_t arm64_cpu_to_le128(__uint128_t x)
618{
619 u64 a = swab64(x);
620 u64 b = swab64(x >> 64);
621
622 return ((__uint128_t)a << 64) | b;
623}
624#else
625static __uint128_t arm64_cpu_to_le128(__uint128_t x)
626{
627 return x;
628}
629#endif
630
631#define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
632
633static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
634 unsigned int vq)
635{
636 unsigned int i;
637 __uint128_t *p;
638
639 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
640 p = (__uint128_t *)ZREG(sst, vq, i);
641 *p = arm64_cpu_to_le128(fst->vregs[i]);
642 }
643}
644
645/*
646 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
647 * task->thread.sve_state.
648 *
649 * Task can be a non-runnable task, or current. In the latter case,
650 * the caller must have ownership of the cpu FPSIMD context before calling
651 * this function.
652 * task->thread.sve_state must point to at least sve_state_size(task)
653 * bytes of allocated kernel memory.
654 * task->thread.uw.fpsimd_state must be up to date before calling this
655 * function.
656 */
657static inline void fpsimd_to_sve(struct task_struct *task)
658{
659 unsigned int vq;
660 void *sst = task->thread.sve_state;
661 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
662
663 if (!system_supports_sve() && !system_supports_sme())
664 return;
665
666 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));
667 __fpsimd_to_sve(sst, fst, vq);
668}
669
670/*
671 * Transfer the SVE state in task->thread.sve_state to
672 * task->thread.uw.fpsimd_state.
673 *
674 * Task can be a non-runnable task, or current. In the latter case,
675 * the caller must have ownership of the cpu FPSIMD context before calling
676 * this function.
677 * task->thread.sve_state must point to at least sve_state_size(task)
678 * bytes of allocated kernel memory.
679 * task->thread.sve_state must be up to date before calling this function.
680 */
681static inline void sve_to_fpsimd(struct task_struct *task)
682{
683 unsigned int vq, vl;
684 void const *sst = task->thread.sve_state;
685 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
686 unsigned int i;
687 __uint128_t const *p;
688
689 if (!system_supports_sve() && !system_supports_sme())
690 return;
691
692 vl = thread_get_cur_vl(&task->thread);
693 vq = sve_vq_from_vl(vl);
694 for (i = 0; i < SVE_NUM_ZREGS; ++i) {
695 p = (__uint128_t const *)ZREG(sst, vq, i);
696 fst->vregs[i] = arm64_le128_to_cpu(*p);
697 }
698}
699
700static inline void __fpsimd_zero_vregs(struct user_fpsimd_state *fpsimd)
701{
702 memset(&fpsimd->vregs, 0, sizeof(fpsimd->vregs));
703}
704
705/*
706 * Simulate the effects of an SMSTOP SM instruction.
707 */
708void task_smstop_sm(struct task_struct *task)
709{
710 if (!thread_sm_enabled(&task->thread))
711 return;
712
713 __fpsimd_zero_vregs(&task->thread.uw.fpsimd_state);
714 task->thread.uw.fpsimd_state.fpsr = 0x0800009f;
715 if (system_supports_fpmr())
716 task->thread.uw.fpmr = 0;
717
718 task->thread.svcr &= ~SVCR_SM_MASK;
719 task->thread.fp_type = FP_STATE_FPSIMD;
720}
721
722void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p)
723{
724 write_sysreg_s(read_sysreg_s(SYS_SCTLR_EL1) | SCTLR_EL1_EnFPM_MASK,
725 SYS_SCTLR_EL1);
726}
727
728#ifdef CONFIG_ARM64_SVE
729static void sve_free(struct task_struct *task)
730{
731 kfree(task->thread.sve_state);
732 task->thread.sve_state = NULL;
733}
734
735/*
736 * Ensure that task->thread.sve_state is allocated and sufficiently large.
737 *
738 * This function should be used only in preparation for replacing
739 * task->thread.sve_state with new data. The memory is always zeroed
740 * here to prevent stale data from showing through: this is done in
741 * the interest of testability and predictability: except in the
742 * do_sve_acc() case, there is no ABI requirement to hide stale data
743 * written previously be task.
744 */
745void sve_alloc(struct task_struct *task, bool flush)
746{
747 if (task->thread.sve_state) {
748 if (flush)
749 memset(task->thread.sve_state, 0,
750 sve_state_size(task));
751 return;
752 }
753
754 /* This is a small allocation (maximum ~8KB) and Should Not Fail. */
755 task->thread.sve_state =
756 kzalloc(sve_state_size(task), GFP_KERNEL);
757}
758
759/*
760 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the
761 * task's currently effective FPSIMD/SVE state.
762 *
763 * The task's FPSIMD/SVE/SME state must not be subject to concurrent
764 * manipulation.
765 */
766void fpsimd_sync_from_effective_state(struct task_struct *task)
767{
768 if (task->thread.fp_type == FP_STATE_SVE)
769 sve_to_fpsimd(task);
770}
771
772/*
773 * Ensure that the task's currently effective FPSIMD/SVE state is up to date
774 * with respect to task->thread.uw.fpsimd_state, zeroing any effective
775 * non-FPSIMD (S)SVE state.
776 *
777 * The task's FPSIMD/SVE/SME state must not be subject to concurrent
778 * manipulation.
779 */
780void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task)
781{
782 unsigned int vq;
783 void *sst = task->thread.sve_state;
784 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
785
786 if (task->thread.fp_type != FP_STATE_SVE)
787 return;
788
789 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread));
790
791 memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
792 __fpsimd_to_sve(sst, fst, vq);
793}
794
795static int change_live_vector_length(struct task_struct *task,
796 enum vec_type type,
797 unsigned long vl)
798{
799 unsigned int sve_vl = task_get_sve_vl(task);
800 unsigned int sme_vl = task_get_sme_vl(task);
801 void *sve_state = NULL, *sme_state = NULL;
802
803 if (type == ARM64_VEC_SME)
804 sme_vl = vl;
805 else
806 sve_vl = vl;
807
808 /*
809 * Allocate the new sve_state and sme_state before freeing the old
810 * copies so that allocation failure can be handled without needing to
811 * mutate the task's state in any way.
812 *
813 * Changes to the SVE vector length must not discard live ZA state or
814 * clear PSTATE.ZA, as userspace code which is unaware of the AAPCS64
815 * ZA lazy saving scheme may attempt to change the SVE vector length
816 * while unsaved/dormant ZA state exists.
817 */
818 sve_state = kzalloc(__sve_state_size(sve_vl, sme_vl), GFP_KERNEL);
819 if (!sve_state)
820 goto out_mem;
821
822 if (type == ARM64_VEC_SME) {
823 sme_state = kzalloc(__sme_state_size(sme_vl), GFP_KERNEL);
824 if (!sme_state)
825 goto out_mem;
826 }
827
828 if (task == current)
829 fpsimd_save_and_flush_current_state();
830 else
831 fpsimd_flush_task_state(task);
832
833 /*
834 * Always preserve PSTATE.SM and the effective FPSIMD state, zeroing
835 * other SVE state.
836 */
837 fpsimd_sync_from_effective_state(task);
838 task_set_vl(task, type, vl);
839 kfree(task->thread.sve_state);
840 task->thread.sve_state = sve_state;
841 fpsimd_sync_to_effective_state_zeropad(task);
842
843 if (type == ARM64_VEC_SME) {
844 task->thread.svcr &= ~SVCR_ZA_MASK;
845 kfree(task->thread.sme_state);
846 task->thread.sme_state = sme_state;
847 }
848
849 return 0;
850
851out_mem:
852 kfree(sve_state);
853 kfree(sme_state);
854 return -ENOMEM;
855}
856
857int vec_set_vector_length(struct task_struct *task, enum vec_type type,
858 unsigned long vl, unsigned long flags)
859{
860 bool onexec = flags & PR_SVE_SET_VL_ONEXEC;
861 bool inherit = flags & PR_SVE_VL_INHERIT;
862
863 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
864 PR_SVE_SET_VL_ONEXEC))
865 return -EINVAL;
866
867 if (!sve_vl_valid(vl))
868 return -EINVAL;
869
870 /*
871 * Clamp to the maximum vector length that VL-agnostic code
872 * can work with. A flag may be assigned in the future to
873 * allow setting of larger vector lengths without confusing
874 * older software.
875 */
876 if (vl > VL_ARCH_MAX)
877 vl = VL_ARCH_MAX;
878
879 vl = find_supported_vector_length(type, vl);
880
881 if (!onexec && vl != task_get_vl(task, type)) {
882 if (change_live_vector_length(task, type, vl))
883 return -ENOMEM;
884 }
885
886 if (onexec || inherit)
887 task_set_vl_onexec(task, type, vl);
888 else
889 /* Reset VL to system default on next exec: */
890 task_set_vl_onexec(task, type, 0);
891
892 update_tsk_thread_flag(task, vec_vl_inherit_flag(type),
893 flags & PR_SVE_VL_INHERIT);
894
895 return 0;
896}
897
898/*
899 * Encode the current vector length and flags for return.
900 * This is only required for prctl(): ptrace has separate fields.
901 * SVE and SME use the same bits for _ONEXEC and _INHERIT.
902 *
903 * flags are as for vec_set_vector_length().
904 */
905static int vec_prctl_status(enum vec_type type, unsigned long flags)
906{
907 int ret;
908
909 if (flags & PR_SVE_SET_VL_ONEXEC)
910 ret = task_get_vl_onexec(current, type);
911 else
912 ret = task_get_vl(current, type);
913
914 if (test_thread_flag(vec_vl_inherit_flag(type)))
915 ret |= PR_SVE_VL_INHERIT;
916
917 return ret;
918}
919
920/* PR_SVE_SET_VL */
921int sve_set_current_vl(unsigned long arg)
922{
923 unsigned long vl, flags;
924 int ret;
925
926 vl = arg & PR_SVE_VL_LEN_MASK;
927 flags = arg & ~vl;
928
929 if (!system_supports_sve() || is_compat_task())
930 return -EINVAL;
931
932 ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags);
933 if (ret)
934 return ret;
935
936 return vec_prctl_status(ARM64_VEC_SVE, flags);
937}
938
939/* PR_SVE_GET_VL */
940int sve_get_current_vl(void)
941{
942 if (!system_supports_sve() || is_compat_task())
943 return -EINVAL;
944
945 return vec_prctl_status(ARM64_VEC_SVE, 0);
946}
947
948#ifdef CONFIG_ARM64_SME
949/* PR_SME_SET_VL */
950int sme_set_current_vl(unsigned long arg)
951{
952 unsigned long vl, flags;
953 int ret;
954
955 vl = arg & PR_SME_VL_LEN_MASK;
956 flags = arg & ~vl;
957
958 if (!system_supports_sme() || is_compat_task())
959 return -EINVAL;
960
961 ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags);
962 if (ret)
963 return ret;
964
965 return vec_prctl_status(ARM64_VEC_SME, flags);
966}
967
968/* PR_SME_GET_VL */
969int sme_get_current_vl(void)
970{
971 if (!system_supports_sme() || is_compat_task())
972 return -EINVAL;
973
974 return vec_prctl_status(ARM64_VEC_SME, 0);
975}
976#endif /* CONFIG_ARM64_SME */
977
978static void vec_probe_vqs(struct vl_info *info,
979 DECLARE_BITMAP(map, SVE_VQ_MAX))
980{
981 unsigned int vq, vl;
982
983 bitmap_zero(map, SVE_VQ_MAX);
984
985 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
986 write_vl(info->type, vq - 1); /* self-syncing */
987
988 switch (info->type) {
989 case ARM64_VEC_SVE:
990 vl = sve_get_vl();
991 break;
992 case ARM64_VEC_SME:
993 vl = sme_get_vl();
994 break;
995 default:
996 vl = 0;
997 break;
998 }
999
1000 /* Minimum VL identified? */
1001 if (sve_vq_from_vl(vl) > vq)
1002 break;
1003
1004 vq = sve_vq_from_vl(vl); /* skip intervening lengths */
1005 set_bit(__vq_to_bit(vq), map);
1006 }
1007}
1008
1009/*
1010 * Initialise the set of known supported VQs for the boot CPU.
1011 * This is called during kernel boot, before secondary CPUs are brought up.
1012 */
1013void __init vec_init_vq_map(enum vec_type type)
1014{
1015 struct vl_info *info = &vl_info[type];
1016 vec_probe_vqs(info, info->vq_map);
1017 bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX);
1018}
1019
1020/*
1021 * If we haven't committed to the set of supported VQs yet, filter out
1022 * those not supported by the current CPU.
1023 * This function is called during the bring-up of early secondary CPUs only.
1024 */
1025void vec_update_vq_map(enum vec_type type)
1026{
1027 struct vl_info *info = &vl_info[type];
1028 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1029
1030 vec_probe_vqs(info, tmp_map);
1031 bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX);
1032 bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map,
1033 SVE_VQ_MAX);
1034}
1035
1036/*
1037 * Check whether the current CPU supports all VQs in the committed set.
1038 * This function is called during the bring-up of late secondary CPUs only.
1039 */
1040int vec_verify_vq_map(enum vec_type type)
1041{
1042 struct vl_info *info = &vl_info[type];
1043 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1044 unsigned long b;
1045
1046 vec_probe_vqs(info, tmp_map);
1047
1048 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
1049 if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) {
1050 pr_warn("%s: cpu%d: Required vector length(s) missing\n",
1051 info->name, smp_processor_id());
1052 return -EINVAL;
1053 }
1054
1055 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
1056 return 0;
1057
1058 /*
1059 * For KVM, it is necessary to ensure that this CPU doesn't
1060 * support any vector length that guests may have probed as
1061 * unsupported.
1062 */
1063
1064 /* Recover the set of supported VQs: */
1065 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
1066 /* Find VQs supported that are not globally supported: */
1067 bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX);
1068
1069 /* Find the lowest such VQ, if any: */
1070 b = find_last_bit(tmp_map, SVE_VQ_MAX);
1071 if (b >= SVE_VQ_MAX)
1072 return 0; /* no mismatches */
1073
1074 /*
1075 * Mismatches above sve_max_virtualisable_vl are fine, since
1076 * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
1077 */
1078 if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) {
1079 pr_warn("%s: cpu%d: Unsupported vector length(s) present\n",
1080 info->name, smp_processor_id());
1081 return -EINVAL;
1082 }
1083
1084 return 0;
1085}
1086
1087static void __init sve_efi_setup(void)
1088{
1089 int max_vl = 0;
1090 int i;
1091
1092 if (!IS_ENABLED(CONFIG_EFI))
1093 return;
1094
1095 for (i = 0; i < ARRAY_SIZE(vl_info); i++)
1096 max_vl = max(vl_info[i].max_vl, max_vl);
1097
1098 /*
1099 * alloc_percpu() warns and prints a backtrace if this goes wrong.
1100 * This is evidence of a crippled system and we are returning void,
1101 * so no attempt is made to handle this situation here.
1102 */
1103 if (!sve_vl_valid(max_vl))
1104 goto fail;
1105
1106 efi_sve_state = kmalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(max_vl)),
1107 GFP_KERNEL);
1108 if (!efi_sve_state)
1109 goto fail;
1110
1111 return;
1112
1113fail:
1114 panic("Cannot allocate memory for EFI SVE save/restore");
1115}
1116
1117void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p)
1118{
1119 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
1120 isb();
1121
1122 write_sysreg_s(0, SYS_ZCR_EL1);
1123}
1124
1125void __init sve_setup(void)
1126{
1127 struct vl_info *info = &vl_info[ARM64_VEC_SVE];
1128 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
1129 unsigned long b;
1130 int max_bit;
1131
1132 if (!system_supports_sve())
1133 return;
1134
1135 /*
1136 * The SVE architecture mandates support for 128-bit vectors,
1137 * so sve_vq_map must have at least SVE_VQ_MIN set.
1138 * If something went wrong, at least try to patch it up:
1139 */
1140 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map)))
1141 set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map);
1142
1143 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX);
1144 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit));
1145
1146 /*
1147 * For the default VL, pick the maximum supported value <= 64.
1148 * VL == 64 is guaranteed not to grow the signal frame.
1149 */
1150 set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64));
1151
1152 bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map,
1153 SVE_VQ_MAX);
1154
1155 b = find_last_bit(tmp_map, SVE_VQ_MAX);
1156 if (b >= SVE_VQ_MAX)
1157 /* No non-virtualisable VLs found */
1158 info->max_virtualisable_vl = SVE_VQ_MAX;
1159 else if (WARN_ON(b == SVE_VQ_MAX - 1))
1160 /* No virtualisable VLs? This is architecturally forbidden. */
1161 info->max_virtualisable_vl = SVE_VQ_MIN;
1162 else /* b + 1 < SVE_VQ_MAX */
1163 info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
1164
1165 if (info->max_virtualisable_vl > info->max_vl)
1166 info->max_virtualisable_vl = info->max_vl;
1167
1168 pr_info("%s: maximum available vector length %u bytes per vector\n",
1169 info->name, info->max_vl);
1170 pr_info("%s: default vector length %u bytes per vector\n",
1171 info->name, get_sve_default_vl());
1172
1173 /* KVM decides whether to support mismatched systems. Just warn here: */
1174 if (sve_max_virtualisable_vl() < sve_max_vl())
1175 pr_warn("%s: unvirtualisable vector lengths present\n",
1176 info->name);
1177
1178 sve_efi_setup();
1179}
1180
1181/*
1182 * Called from the put_task_struct() path, which cannot get here
1183 * unless dead_task is really dead and not schedulable.
1184 */
1185void fpsimd_release_task(struct task_struct *dead_task)
1186{
1187 sve_free(dead_task);
1188 sme_free(dead_task);
1189}
1190
1191#endif /* CONFIG_ARM64_SVE */
1192
1193#ifdef CONFIG_ARM64_SME
1194
1195/*
1196 * Ensure that task->thread.sme_state is allocated and sufficiently large.
1197 *
1198 * This function should be used only in preparation for replacing
1199 * task->thread.sme_state with new data. The memory is always zeroed
1200 * here to prevent stale data from showing through: this is done in
1201 * the interest of testability and predictability, the architecture
1202 * guarantees that when ZA is enabled it will be zeroed.
1203 */
1204void sme_alloc(struct task_struct *task, bool flush)
1205{
1206 if (task->thread.sme_state) {
1207 if (flush)
1208 memset(task->thread.sme_state, 0,
1209 sme_state_size(task));
1210 return;
1211 }
1212
1213 /* This could potentially be up to 64K. */
1214 task->thread.sme_state =
1215 kzalloc(sme_state_size(task), GFP_KERNEL);
1216}
1217
1218static void sme_free(struct task_struct *task)
1219{
1220 kfree(task->thread.sme_state);
1221 task->thread.sme_state = NULL;
1222}
1223
1224void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p)
1225{
1226 /* Set priority for all PEs to architecturally defined minimum */
1227 write_sysreg_s(read_sysreg_s(SYS_SMPRI_EL1) & ~SMPRI_EL1_PRIORITY_MASK,
1228 SYS_SMPRI_EL1);
1229
1230 /* Allow SME in kernel */
1231 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_SMEN_EL1EN, CPACR_EL1);
1232 isb();
1233
1234 /* Ensure all bits in SMCR are set to known values */
1235 write_sysreg_s(0, SYS_SMCR_EL1);
1236
1237 /* Allow EL0 to access TPIDR2 */
1238 write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1);
1239 isb();
1240}
1241
1242void cpu_enable_sme2(const struct arm64_cpu_capabilities *__always_unused p)
1243{
1244 /* This must be enabled after SME */
1245 BUILD_BUG_ON(ARM64_SME2 <= ARM64_SME);
1246
1247 /* Allow use of ZT0 */
1248 write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_EZT0_MASK,
1249 SYS_SMCR_EL1);
1250}
1251
1252void cpu_enable_fa64(const struct arm64_cpu_capabilities *__always_unused p)
1253{
1254 /* This must be enabled after SME */
1255 BUILD_BUG_ON(ARM64_SME_FA64 <= ARM64_SME);
1256
1257 /* Allow use of FA64 */
1258 write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_FA64_MASK,
1259 SYS_SMCR_EL1);
1260}
1261
1262void __init sme_setup(void)
1263{
1264 struct vl_info *info = &vl_info[ARM64_VEC_SME];
1265 int min_bit, max_bit;
1266
1267 if (!system_supports_sme())
1268 return;
1269
1270 min_bit = find_last_bit(info->vq_map, SVE_VQ_MAX);
1271
1272 /*
1273 * SME doesn't require any particular vector length be
1274 * supported but it does require at least one. We should have
1275 * disabled the feature entirely while bringing up CPUs but
1276 * let's double check here. The bitmap is SVE_VQ_MAP sized for
1277 * sharing with SVE.
1278 */
1279 WARN_ON(min_bit >= SVE_VQ_MAX);
1280
1281 info->min_vl = sve_vl_from_vq(__bit_to_vq(min_bit));
1282
1283 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX);
1284 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit));
1285
1286 WARN_ON(info->min_vl > info->max_vl);
1287
1288 /*
1289 * For the default VL, pick the maximum supported value <= 32
1290 * (256 bits) if there is one since this is guaranteed not to
1291 * grow the signal frame when in streaming mode, otherwise the
1292 * minimum available VL will be used.
1293 */
1294 set_sme_default_vl(find_supported_vector_length(ARM64_VEC_SME, 32));
1295
1296 pr_info("SME: minimum available vector length %u bytes per vector\n",
1297 info->min_vl);
1298 pr_info("SME: maximum available vector length %u bytes per vector\n",
1299 info->max_vl);
1300 pr_info("SME: default vector length %u bytes per vector\n",
1301 get_sme_default_vl());
1302}
1303
1304void sme_suspend_exit(void)
1305{
1306 u64 smcr = 0;
1307
1308 if (!system_supports_sme())
1309 return;
1310
1311 if (system_supports_fa64())
1312 smcr |= SMCR_ELx_FA64;
1313 if (system_supports_sme2())
1314 smcr |= SMCR_ELx_EZT0;
1315
1316 write_sysreg_s(smcr, SYS_SMCR_EL1);
1317 write_sysreg_s(0, SYS_SMPRI_EL1);
1318}
1319
1320#endif /* CONFIG_ARM64_SME */
1321
1322static void sve_init_regs(void)
1323{
1324 /*
1325 * Convert the FPSIMD state to SVE, zeroing all the state that
1326 * is not shared with FPSIMD. If (as is likely) the current
1327 * state is live in the registers then do this there and
1328 * update our metadata for the current task including
1329 * disabling the trap, otherwise update our in-memory copy.
1330 * We are guaranteed to not be in streaming mode, we can only
1331 * take a SVE trap when not in streaming mode and we can't be
1332 * in streaming mode when taking a SME trap.
1333 */
1334 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
1335 unsigned long vq_minus_one =
1336 sve_vq_from_vl(task_get_sve_vl(current)) - 1;
1337 sve_set_vq(vq_minus_one);
1338 sve_flush_live(true, vq_minus_one);
1339 fpsimd_bind_task_to_cpu();
1340 } else {
1341 fpsimd_to_sve(current);
1342 current->thread.fp_type = FP_STATE_SVE;
1343 fpsimd_flush_task_state(current);
1344 }
1345}
1346
1347/*
1348 * Trapped SVE access
1349 *
1350 * Storage is allocated for the full SVE state, the current FPSIMD
1351 * register contents are migrated across, and the access trap is
1352 * disabled.
1353 *
1354 * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state()
1355 * would have disabled the SVE access trap for userspace during
1356 * ret_to_user, making an SVE access trap impossible in that case.
1357 */
1358void do_sve_acc(unsigned long esr, struct pt_regs *regs)
1359{
1360 /* Even if we chose not to use SVE, the hardware could still trap: */
1361 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
1362 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1363 return;
1364 }
1365
1366 sve_alloc(current, true);
1367 if (!current->thread.sve_state) {
1368 force_sig(SIGKILL);
1369 return;
1370 }
1371
1372 get_cpu_fpsimd_context();
1373
1374 if (test_and_set_thread_flag(TIF_SVE))
1375 WARN_ON(1); /* SVE access shouldn't have trapped */
1376
1377 /*
1378 * Even if the task can have used streaming mode we can only
1379 * generate SVE access traps in normal SVE mode and
1380 * transitioning out of streaming mode may discard any
1381 * streaming mode state. Always clear the high bits to avoid
1382 * any potential errors tracking what is properly initialised.
1383 */
1384 sve_init_regs();
1385
1386 put_cpu_fpsimd_context();
1387}
1388
1389#ifdef CONFIG_ARM64_ERRATUM_4193714
1390
1391/*
1392 * SME/CME erratum handling.
1393 */
1394static cpumask_t sme_dvmsync_cpus;
1395
1396/*
1397 * These helpers are only called from non-preemptible contexts, so
1398 * smp_processor_id() is safe here.
1399 */
1400void sme_set_active(void)
1401{
1402 unsigned int cpu = smp_processor_id();
1403
1404 if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus))
1405 return;
1406
1407 cpumask_set_cpu(cpu, mm_cpumask(current->mm));
1408
1409 /*
1410 * A subsequent (post ERET) SME access may use a stale address
1411 * translation. On C1-Pro, a TLBI+DSB on a different CPU will wait for
1412 * the completion of cpumask_set_cpu() above as it appears in program
1413 * order before the SME access. The post-TLBI+DSB read of mm_cpumask()
1414 * will lead to the IPI being issued.
1415 *
1416 * https://lore.kernel.org/r/ablEXwhfKyJW1i7l@J2N7QTR9R3
1417 */
1418}
1419
1420void sme_clear_active(void)
1421{
1422 unsigned int cpu = smp_processor_id();
1423
1424 if (!cpumask_test_cpu(cpu, &sme_dvmsync_cpus))
1425 return;
1426
1427 /*
1428 * With SCTLR_EL1.IESB enabled, the SME memory transactions are
1429 * completed on entering EL1.
1430 */
1431 cpumask_clear_cpu(cpu, mm_cpumask(current->mm));
1432}
1433
1434static void sme_dvmsync_ipi(void *unused)
1435{
1436 /*
1437 * With SCTLR_EL1.IESB on, taking an exception is sufficient to ensure
1438 * the completion of the SME memory accesses, so no need for an
1439 * explicit DSB.
1440 */
1441}
1442
1443void sme_do_dvmsync(const struct cpumask *mask)
1444{
1445 /*
1446 * This is called from the TLB maintenance functions after the DSB ISH
1447 * to send the hardware DVMSync message. If this CPU sees the mask as
1448 * empty, the remote CPU executing sme_set_active() would have seen
1449 * the DVMSync and no IPI required.
1450 */
1451 if (cpumask_empty(mask))
1452 return;
1453
1454 preempt_disable();
1455 smp_call_function_many(mask, sme_dvmsync_ipi, NULL, true);
1456 preempt_enable();
1457}
1458
1459void sme_enable_dvmsync(void)
1460{
1461 cpumask_set_cpu(smp_processor_id(), &sme_dvmsync_cpus);
1462}
1463
1464#endif /* CONFIG_ARM64_ERRATUM_4193714 */
1465
1466/*
1467 * Trapped SME access
1468 *
1469 * Storage is allocated for the full SVE and SME state, the current
1470 * FPSIMD register contents are migrated to SVE if SVE is not already
1471 * active, and the access trap is disabled.
1472 *
1473 * TIF_SME should be clear on entry: otherwise, fpsimd_restore_current_state()
1474 * would have disabled the SME access trap for userspace during
1475 * ret_to_user, making an SME access trap impossible in that case.
1476 */
1477void do_sme_acc(unsigned long esr, struct pt_regs *regs)
1478{
1479 /* Even if we chose not to use SME, the hardware could still trap: */
1480 if (unlikely(!system_supports_sme()) || WARN_ON(is_compat_task())) {
1481 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1482 return;
1483 }
1484
1485 /*
1486 * If this not a trap due to SME being disabled then something
1487 * is being used in the wrong mode, report as SIGILL.
1488 */
1489 if (ESR_ELx_SME_ISS_SMTC(esr) != ESR_ELx_SME_ISS_SMTC_SME_DISABLED) {
1490 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1491 return;
1492 }
1493
1494 sve_alloc(current, false);
1495 sme_alloc(current, true);
1496 if (!current->thread.sve_state || !current->thread.sme_state) {
1497 force_sig(SIGKILL);
1498 return;
1499 }
1500
1501 get_cpu_fpsimd_context();
1502
1503 /* With TIF_SME userspace shouldn't generate any traps */
1504 if (test_and_set_thread_flag(TIF_SME))
1505 WARN_ON(1);
1506
1507 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
1508 unsigned long vq_minus_one =
1509 sve_vq_from_vl(task_get_sme_vl(current)) - 1;
1510 sme_set_vq(vq_minus_one);
1511
1512 fpsimd_bind_task_to_cpu();
1513 } else {
1514 fpsimd_flush_task_state(current);
1515 }
1516
1517 put_cpu_fpsimd_context();
1518}
1519
1520/*
1521 * Trapped FP/ASIMD access.
1522 */
1523void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs)
1524{
1525 /* Even if we chose not to use FPSIMD, the hardware could still trap: */
1526 if (!system_supports_fpsimd()) {
1527 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
1528 return;
1529 }
1530
1531 /*
1532 * When FPSIMD is enabled, we should never take a trap unless something
1533 * has gone very wrong.
1534 */
1535 BUG();
1536}
1537
1538/*
1539 * Raise a SIGFPE for the current process.
1540 */
1541void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs)
1542{
1543 unsigned int si_code = FPE_FLTUNK;
1544
1545 if (esr & ESR_ELx_FP_EXC_TFV) {
1546 if (esr & FPEXC_IOF)
1547 si_code = FPE_FLTINV;
1548 else if (esr & FPEXC_DZF)
1549 si_code = FPE_FLTDIV;
1550 else if (esr & FPEXC_OFF)
1551 si_code = FPE_FLTOVF;
1552 else if (esr & FPEXC_UFF)
1553 si_code = FPE_FLTUND;
1554 else if (esr & FPEXC_IXF)
1555 si_code = FPE_FLTRES;
1556 }
1557
1558 send_sig_fault(SIGFPE, si_code,
1559 (void __user *)instruction_pointer(regs),
1560 current);
1561}
1562
1563static void fpsimd_load_kernel_state(struct task_struct *task)
1564{
1565 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1566
1567 /*
1568 * Elide the load if this CPU holds the most recent kernel mode
1569 * FPSIMD context of the current task.
1570 */
1571 if (last->st == &task->thread.kernel_fpsimd_state &&
1572 task->thread.kernel_fpsimd_cpu == smp_processor_id())
1573 return;
1574
1575 fpsimd_load_state(&task->thread.kernel_fpsimd_state);
1576}
1577
1578static void fpsimd_save_kernel_state(struct task_struct *task)
1579{
1580 struct cpu_fp_state cpu_fp_state = {
1581 .st = &task->thread.kernel_fpsimd_state,
1582 .to_save = FP_STATE_FPSIMD,
1583 };
1584
1585 fpsimd_save_state(&task->thread.kernel_fpsimd_state);
1586 fpsimd_bind_state_to_cpu(&cpu_fp_state);
1587
1588 task->thread.kernel_fpsimd_cpu = smp_processor_id();
1589}
1590
1591/*
1592 * Invalidate any task's FPSIMD state that is present on this cpu.
1593 * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1594 * before calling this function.
1595 */
1596static void fpsimd_flush_cpu_state(void)
1597{
1598 WARN_ON(!system_supports_fpsimd());
1599 __this_cpu_write(fpsimd_last_state.st, NULL);
1600
1601 /*
1602 * Leaving streaming mode enabled will cause issues for any kernel
1603 * NEON and leaving streaming mode or ZA enabled may increase power
1604 * consumption.
1605 */
1606 if (system_supports_sme())
1607 sme_smstop();
1608
1609 set_thread_flag(TIF_FOREIGN_FPSTATE);
1610}
1611
1612void fpsimd_thread_switch(struct task_struct *next)
1613{
1614 bool wrong_task, wrong_cpu;
1615
1616 if (!system_supports_fpsimd())
1617 return;
1618
1619 WARN_ON_ONCE(!irqs_disabled());
1620
1621 /* Save unsaved fpsimd state, if any: */
1622 if (test_thread_flag(TIF_KERNEL_FPSTATE))
1623 fpsimd_save_kernel_state(current);
1624 else
1625 fpsimd_save_user_state();
1626
1627 if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) {
1628 fpsimd_flush_cpu_state();
1629 fpsimd_load_kernel_state(next);
1630 } else {
1631 /*
1632 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
1633 * state. For kernel threads, FPSIMD registers are never
1634 * loaded with user mode FPSIMD state and so wrong_task and
1635 * wrong_cpu will always be true.
1636 */
1637 wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
1638 &next->thread.uw.fpsimd_state;
1639 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
1640
1641 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
1642 wrong_task || wrong_cpu);
1643 }
1644}
1645
1646static void fpsimd_flush_thread_vl(enum vec_type type)
1647{
1648 int vl, supported_vl;
1649
1650 /*
1651 * Reset the task vector length as required. This is where we
1652 * ensure that all user tasks have a valid vector length
1653 * configured: no kernel task can become a user task without
1654 * an exec and hence a call to this function. By the time the
1655 * first call to this function is made, all early hardware
1656 * probing is complete, so __sve_default_vl should be valid.
1657 * If a bug causes this to go wrong, we make some noise and
1658 * try to fudge thread.sve_vl to a safe value here.
1659 */
1660 vl = task_get_vl_onexec(current, type);
1661 if (!vl)
1662 vl = get_default_vl(type);
1663
1664 if (WARN_ON(!sve_vl_valid(vl)))
1665 vl = vl_info[type].min_vl;
1666
1667 supported_vl = find_supported_vector_length(type, vl);
1668 if (WARN_ON(supported_vl != vl))
1669 vl = supported_vl;
1670
1671 task_set_vl(current, type, vl);
1672
1673 /*
1674 * If the task is not set to inherit, ensure that the vector
1675 * length will be reset by a subsequent exec:
1676 */
1677 if (!test_thread_flag(vec_vl_inherit_flag(type)))
1678 task_set_vl_onexec(current, type, 0);
1679}
1680
1681void fpsimd_flush_thread(void)
1682{
1683 void *sve_state = NULL;
1684 void *sme_state = NULL;
1685
1686 if (!system_supports_fpsimd())
1687 return;
1688
1689 get_cpu_fpsimd_context();
1690
1691 fpsimd_flush_task_state(current);
1692 memset(¤t->thread.uw.fpsimd_state, 0,
1693 sizeof(current->thread.uw.fpsimd_state));
1694
1695 if (system_supports_sve()) {
1696 clear_thread_flag(TIF_SVE);
1697
1698 /* Defer kfree() while in atomic context */
1699 sve_state = current->thread.sve_state;
1700 current->thread.sve_state = NULL;
1701
1702 fpsimd_flush_thread_vl(ARM64_VEC_SVE);
1703 }
1704
1705 if (system_supports_sme()) {
1706 clear_thread_flag(TIF_SME);
1707
1708 /* Defer kfree() while in atomic context */
1709 sme_state = current->thread.sme_state;
1710 current->thread.sme_state = NULL;
1711
1712 fpsimd_flush_thread_vl(ARM64_VEC_SME);
1713 current->thread.svcr = 0;
1714 }
1715
1716 if (system_supports_fpmr())
1717 current->thread.uw.fpmr = 0;
1718
1719 current->thread.fp_type = FP_STATE_FPSIMD;
1720
1721 put_cpu_fpsimd_context();
1722 kfree(sve_state);
1723 kfree(sme_state);
1724}
1725
1726/*
1727 * Save the userland FPSIMD state of 'current' to memory, but only if the state
1728 * currently held in the registers does in fact belong to 'current'
1729 */
1730void fpsimd_preserve_current_state(void)
1731{
1732 if (!system_supports_fpsimd())
1733 return;
1734
1735 get_cpu_fpsimd_context();
1736 fpsimd_save_user_state();
1737 put_cpu_fpsimd_context();
1738}
1739
1740/*
1741 * Associate current's FPSIMD context with this cpu
1742 * The caller must have ownership of the cpu FPSIMD context before calling
1743 * this function.
1744 */
1745static void fpsimd_bind_task_to_cpu(void)
1746{
1747 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1748
1749 WARN_ON(!system_supports_fpsimd());
1750 last->st = ¤t->thread.uw.fpsimd_state;
1751 last->sve_state = current->thread.sve_state;
1752 last->sme_state = current->thread.sme_state;
1753 last->sve_vl = task_get_sve_vl(current);
1754 last->sme_vl = task_get_sme_vl(current);
1755 last->svcr = ¤t->thread.svcr;
1756 last->fpmr = ¤t->thread.uw.fpmr;
1757 last->fp_type = ¤t->thread.fp_type;
1758 last->to_save = FP_STATE_CURRENT;
1759 current->thread.fpsimd_cpu = smp_processor_id();
1760
1761 /*
1762 * Toggle SVE and SME trapping for userspace if needed, these
1763 * are serialsied by ret_to_user().
1764 */
1765 if (system_supports_sme()) {
1766 if (test_thread_flag(TIF_SME))
1767 sme_user_enable();
1768 else
1769 sme_user_disable();
1770 }
1771
1772 if (system_supports_sve()) {
1773 if (test_thread_flag(TIF_SVE))
1774 sve_user_enable();
1775 else
1776 sve_user_disable();
1777 }
1778}
1779
1780void fpsimd_bind_state_to_cpu(struct cpu_fp_state *state)
1781{
1782 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state);
1783
1784 WARN_ON(!system_supports_fpsimd());
1785 WARN_ON(!in_softirq() && !irqs_disabled());
1786
1787 *last = *state;
1788}
1789
1790/*
1791 * Load the userland FPSIMD state of 'current' from memory, but only if the
1792 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
1793 * state of 'current'. This is called when we are preparing to return to
1794 * userspace to ensure that userspace sees a good register state.
1795 */
1796void fpsimd_restore_current_state(void)
1797{
1798 /*
1799 * TIF_FOREIGN_FPSTATE is set on the init task and copied by
1800 * arch_dup_task_struct() regardless of whether FP/SIMD is detected.
1801 * Thus user threads can have this set even when FP/SIMD hasn't been
1802 * detected.
1803 *
1804 * When FP/SIMD is detected, begin_new_exec() will set
1805 * TIF_FOREIGN_FPSTATE via flush_thread() -> fpsimd_flush_thread(),
1806 * and fpsimd_thread_switch() will set TIF_FOREIGN_FPSTATE when
1807 * switching tasks. We detect FP/SIMD before we exec the first user
1808 * process, ensuring this has TIF_FOREIGN_FPSTATE set and
1809 * do_notify_resume() will call fpsimd_restore_current_state() to
1810 * install the user FP/SIMD context.
1811 *
1812 * When FP/SIMD is not detected, nothing else will clear or set
1813 * TIF_FOREIGN_FPSTATE prior to the first return to userspace, and
1814 * we must clear TIF_FOREIGN_FPSTATE to avoid do_notify_resume()
1815 * looping forever calling fpsimd_restore_current_state().
1816 */
1817 if (!system_supports_fpsimd()) {
1818 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1819 return;
1820 }
1821
1822 get_cpu_fpsimd_context();
1823
1824 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1825 task_fpsimd_load();
1826 fpsimd_bind_task_to_cpu();
1827 }
1828
1829 put_cpu_fpsimd_context();
1830}
1831
1832void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1833{
1834 if (WARN_ON(!system_supports_fpsimd()))
1835 return;
1836
1837 current->thread.uw.fpsimd_state = *state;
1838 if (current->thread.fp_type == FP_STATE_SVE)
1839 fpsimd_to_sve(current);
1840}
1841
1842/*
1843 * Invalidate live CPU copies of task t's FPSIMD state
1844 *
1845 * This function may be called with preemption enabled. The barrier()
1846 * ensures that the assignment to fpsimd_cpu is visible to any
1847 * preemption/softirq that could race with set_tsk_thread_flag(), so
1848 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1849 *
1850 * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1851 * subsequent code.
1852 */
1853void fpsimd_flush_task_state(struct task_struct *t)
1854{
1855 t->thread.fpsimd_cpu = NR_CPUS;
1856 /*
1857 * If we don't support fpsimd, bail out after we have
1858 * reset the fpsimd_cpu for this task and clear the
1859 * FPSTATE.
1860 */
1861 if (!system_supports_fpsimd())
1862 return;
1863 barrier();
1864 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1865
1866 barrier();
1867}
1868
1869void fpsimd_save_and_flush_current_state(void)
1870{
1871 if (!system_supports_fpsimd())
1872 return;
1873
1874 get_cpu_fpsimd_context();
1875 fpsimd_save_user_state();
1876 fpsimd_flush_task_state(current);
1877 put_cpu_fpsimd_context();
1878}
1879
1880/*
1881 * Save the FPSIMD state to memory and invalidate cpu view.
1882 * This function must be called with preemption disabled.
1883 */
1884void fpsimd_save_and_flush_cpu_state(void)
1885{
1886 unsigned long flags;
1887
1888 if (!system_supports_fpsimd())
1889 return;
1890 WARN_ON(preemptible());
1891 local_irq_save(flags);
1892 fpsimd_save_user_state();
1893 fpsimd_flush_cpu_state();
1894 local_irq_restore(flags);
1895}
1896
1897#ifdef CONFIG_KERNEL_MODE_NEON
1898
1899/*
1900 * Kernel-side NEON support functions
1901 */
1902
1903/*
1904 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
1905 * context
1906 *
1907 * Must not be called unless may_use_simd() returns true.
1908 * Task context in the FPSIMD registers is saved back to memory as necessary.
1909 *
1910 * A matching call to kernel_neon_end() must be made before returning from the
1911 * calling context.
1912 *
1913 * The caller may freely use the FPSIMD registers until kernel_neon_end() is
1914 * called.
1915 */
1916void kernel_neon_begin(void)
1917{
1918 if (WARN_ON(!system_supports_fpsimd()))
1919 return;
1920
1921 BUG_ON(!may_use_simd());
1922
1923 get_cpu_fpsimd_context();
1924
1925 /* Save unsaved fpsimd state, if any: */
1926 if (test_thread_flag(TIF_KERNEL_FPSTATE)) {
1927 BUG_ON(IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq());
1928 fpsimd_save_kernel_state(current);
1929 } else {
1930 fpsimd_save_user_state();
1931
1932 /*
1933 * Set the thread flag so that the kernel mode FPSIMD state
1934 * will be context switched along with the rest of the task
1935 * state.
1936 *
1937 * On non-PREEMPT_RT, softirqs may interrupt task level kernel
1938 * mode FPSIMD, but the task will not be preemptible so setting
1939 * TIF_KERNEL_FPSTATE for those would be both wrong (as it
1940 * would mark the task context FPSIMD state as requiring a
1941 * context switch) and unnecessary.
1942 *
1943 * On PREEMPT_RT, softirqs are serviced from a separate thread,
1944 * which is scheduled as usual, and this guarantees that these
1945 * softirqs are not interrupting use of the FPSIMD in kernel
1946 * mode in task context. So in this case, setting the flag here
1947 * is always appropriate.
1948 */
1949 if (IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq())
1950 set_thread_flag(TIF_KERNEL_FPSTATE);
1951 }
1952
1953 /* Invalidate any task state remaining in the fpsimd regs: */
1954 fpsimd_flush_cpu_state();
1955
1956 put_cpu_fpsimd_context();
1957}
1958EXPORT_SYMBOL_GPL(kernel_neon_begin);
1959
1960/*
1961 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1962 *
1963 * Must be called from a context in which kernel_neon_begin() was previously
1964 * called, with no call to kernel_neon_end() in the meantime.
1965 *
1966 * The caller must not use the FPSIMD registers after this function is called,
1967 * unless kernel_neon_begin() is called again in the meantime.
1968 */
1969void kernel_neon_end(void)
1970{
1971 if (!system_supports_fpsimd())
1972 return;
1973
1974 /*
1975 * If we are returning from a nested use of kernel mode FPSIMD, restore
1976 * the task context kernel mode FPSIMD state. This can only happen when
1977 * running in softirq context on non-PREEMPT_RT.
1978 */
1979 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && in_serving_softirq() &&
1980 test_thread_flag(TIF_KERNEL_FPSTATE))
1981 fpsimd_load_kernel_state(current);
1982 else
1983 clear_thread_flag(TIF_KERNEL_FPSTATE);
1984}
1985EXPORT_SYMBOL_GPL(kernel_neon_end);
1986
1987#ifdef CONFIG_EFI
1988
1989static struct user_fpsimd_state efi_fpsimd_state;
1990static bool efi_fpsimd_state_used;
1991static bool efi_sve_state_used;
1992static bool efi_sm_state;
1993
1994/*
1995 * EFI runtime services support functions
1996 *
1997 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
1998 * This means that for EFI (and only for EFI), we have to assume that FPSIMD
1999 * is always used rather than being an optional accelerator.
2000 *
2001 * These functions provide the necessary support for ensuring FPSIMD
2002 * save/restore in the contexts from which EFI is used.
2003 *
2004 * Do not use them for any other purpose -- if tempted to do so, you are
2005 * either doing something wrong or you need to propose some refactoring.
2006 */
2007
2008/*
2009 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
2010 */
2011void __efi_fpsimd_begin(void)
2012{
2013 if (!system_supports_fpsimd())
2014 return;
2015
2016 WARN_ON(preemptible());
2017
2018 if (may_use_simd()) {
2019 kernel_neon_begin();
2020 } else {
2021 /*
2022 * If !efi_sve_state, SVE can't be in use yet and doesn't need
2023 * preserving:
2024 */
2025 if (system_supports_sve() && efi_sve_state != NULL) {
2026 bool ffr = true;
2027 u64 svcr;
2028
2029 efi_sve_state_used = true;
2030
2031 if (system_supports_sme()) {
2032 svcr = read_sysreg_s(SYS_SVCR);
2033
2034 efi_sm_state = svcr & SVCR_SM_MASK;
2035
2036 /*
2037 * Unless we have FA64 FFR does not
2038 * exist in streaming mode.
2039 */
2040 if (!system_supports_fa64())
2041 ffr = !(svcr & SVCR_SM_MASK);
2042 }
2043
2044 sve_save_state(efi_sve_state + sve_ffr_offset(sve_max_vl()),
2045 &efi_fpsimd_state.fpsr, ffr);
2046
2047 if (system_supports_sme())
2048 sysreg_clear_set_s(SYS_SVCR,
2049 SVCR_SM_MASK, 0);
2050
2051 } else {
2052 fpsimd_save_state(&efi_fpsimd_state);
2053 }
2054
2055 efi_fpsimd_state_used = true;
2056 }
2057}
2058
2059/*
2060 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
2061 */
2062void __efi_fpsimd_end(void)
2063{
2064 if (!system_supports_fpsimd())
2065 return;
2066
2067 if (!efi_fpsimd_state_used) {
2068 kernel_neon_end();
2069 } else {
2070 if (system_supports_sve() && efi_sve_state_used) {
2071 bool ffr = true;
2072
2073 /*
2074 * Restore streaming mode; EFI calls are
2075 * normal function calls so should not return in
2076 * streaming mode.
2077 */
2078 if (system_supports_sme()) {
2079 if (efi_sm_state) {
2080 sysreg_clear_set_s(SYS_SVCR,
2081 0,
2082 SVCR_SM_MASK);
2083
2084 /*
2085 * Unless we have FA64 FFR does not
2086 * exist in streaming mode.
2087 */
2088 if (!system_supports_fa64())
2089 ffr = false;
2090 }
2091 }
2092
2093 sve_load_state(efi_sve_state + sve_ffr_offset(sve_max_vl()),
2094 &efi_fpsimd_state.fpsr, ffr);
2095
2096 efi_sve_state_used = false;
2097 } else {
2098 fpsimd_load_state(&efi_fpsimd_state);
2099 }
2100
2101 efi_fpsimd_state_used = false;
2102 }
2103}
2104
2105#endif /* CONFIG_EFI */
2106
2107#endif /* CONFIG_KERNEL_MODE_NEON */
2108
2109#ifdef CONFIG_CPU_PM
2110static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
2111 unsigned long cmd, void *v)
2112{
2113 switch (cmd) {
2114 case CPU_PM_ENTER:
2115 fpsimd_save_and_flush_cpu_state();
2116 break;
2117 case CPU_PM_EXIT:
2118 break;
2119 case CPU_PM_ENTER_FAILED:
2120 default:
2121 return NOTIFY_DONE;
2122 }
2123 return NOTIFY_OK;
2124}
2125
2126static struct notifier_block fpsimd_cpu_pm_notifier_block = {
2127 .notifier_call = fpsimd_cpu_pm_notifier,
2128};
2129
2130static void __init fpsimd_pm_init(void)
2131{
2132 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
2133}
2134
2135#else
2136static inline void fpsimd_pm_init(void) { }
2137#endif /* CONFIG_CPU_PM */
2138
2139#ifdef CONFIG_HOTPLUG_CPU
2140static int fpsimd_cpu_dead(unsigned int cpu)
2141{
2142 per_cpu(fpsimd_last_state.st, cpu) = NULL;
2143 return 0;
2144}
2145
2146static inline void fpsimd_hotplug_init(void)
2147{
2148 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
2149 NULL, fpsimd_cpu_dead);
2150}
2151
2152#else
2153static inline void fpsimd_hotplug_init(void) { }
2154#endif
2155
2156void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__always_unused p)
2157{
2158 unsigned long enable = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN;
2159 write_sysreg(read_sysreg(CPACR_EL1) | enable, CPACR_EL1);
2160 isb();
2161}
2162
2163/*
2164 * FP/SIMD support code initialisation.
2165 */
2166static int __init fpsimd_init(void)
2167{
2168 if (cpu_have_named_feature(FP)) {
2169 fpsimd_pm_init();
2170 fpsimd_hotplug_init();
2171 } else {
2172 pr_notice("Floating-point is not implemented\n");
2173 }
2174
2175 if (!cpu_have_named_feature(ASIMD))
2176 pr_notice("Advanced SIMD is not implemented\n");
2177
2178
2179 sve_sysctl_init();
2180 sme_sysctl_init();
2181
2182 return 0;
2183}
2184core_initcall(fpsimd_init);