개념 설명 전체
arch/arm64/kvm/arm.c · v6.18.37
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
4 * Author: Christoffer Dall <[email protected]>
5 */
6
7 #include <linux/bug.h>
8 #include <linux/cpu_pm.h>
9 #include <linux/errno.h>
10 #include <linux/err.h>
11 #include <linux/kvm_host.h>
12 #include <linux/list.h>
13 #include <linux/module.h>
14 #include <linux/vmalloc.h>
15 #include <linux/fs.h>
16 #include <linux/mman.h>
17 #include <linux/sched.h>
18 #include <linux/kvm.h>
19 #include <linux/kvm_irqfd.h>
20 #include <linux/irqbypass.h>
21 #include <linux/sched/stat.h>
22 #include <linux/psci.h>
23 #include <trace/events/kvm.h>
24
25 #define CREATE_TRACE_POINTS
26 #include "trace_arm.h"
27
28 #include <linux/uaccess.h>
29 #include <asm/ptrace.h>
30 #include <asm/mman.h>
31 #include <asm/tlbflush.h>
32 #include <asm/cacheflush.h>
33 #include <asm/cpufeature.h>
34 #include <asm/virt.h>
35 #include <asm/kvm_arm.h>
36 #include <asm/kvm_asm.h>
37 #include <asm/kvm_emulate.h>
38 #include <asm/kvm_mmu.h>
39 #include <asm/kvm_nested.h>
40 #include <asm/kvm_pkvm.h>
41 #include <asm/kvm_ptrauth.h>
42 #include <asm/sections.h>
43
44 #include <kvm/arm_hypercalls.h>
45 #include <kvm/arm_pmu.h>
46 #include <kvm/arm_psci.h>
47
48 #include "sys_regs.h"
49
50 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
51
52 enum kvm_wfx_trap_policy {
53 KVM_WFX_NOTRAP_SINGLE_TASK, /* Default option */
54 KVM_WFX_NOTRAP,
55 KVM_WFX_TRAP,
56 };
57
58 static enum kvm_wfx_trap_policy kvm_wfi_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
59 static enum kvm_wfx_trap_policy kvm_wfe_trap_policy __read_mostly = KVM_WFX_NOTRAP_SINGLE_TASK;
60
61 DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
62
63 DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
64 DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
65
66 DECLARE_KVM_NVHE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
67
68 static bool vgic_present, kvm_arm_initialised;
69
70 static DEFINE_PER_CPU(unsigned char, kvm_hyp_initialized);
71
72 bool is_kvm_arm_initialised(void)
73 {
74 return kvm_arm_initialised;
75 }
76
77 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
78 {
79 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
80 }
81
82 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
83 struct kvm_enable_cap *cap)
84 {
85 int r = -EINVAL;
86
87 if (cap->flags)
88 return -EINVAL;
89
90 if (kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(cap->cap))
91 return -EINVAL;
92
93 switch (cap->cap) {
94 case KVM_CAP_ARM_NISV_TO_USER:
95 r = 0;
96 set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
97 &kvm->arch.flags);
98 break;
99 case KVM_CAP_ARM_MTE:
100 mutex_lock(&kvm->lock);
101 if (system_supports_mte() && !kvm->created_vcpus) {
102 r = 0;
103 set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags);
104 }
105 mutex_unlock(&kvm->lock);
106 break;
107 case KVM_CAP_ARM_SYSTEM_SUSPEND:
108 r = 0;
109 set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags);
110 break;
111 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
112 mutex_lock(&kvm->slots_lock);
113 /*
114 * To keep things simple, allow changing the chunk
115 * size only when no memory slots have been created.
116 */
117 if (kvm_are_all_memslots_empty(kvm)) {
118 u64 new_cap = cap->args[0];
119
120 if (!new_cap || kvm_is_block_size_supported(new_cap)) {
121 r = 0;
122 kvm->arch.mmu.split_page_chunk_size = new_cap;
123 }
124 }
125 mutex_unlock(&kvm->slots_lock);
126 break;
127 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
128 mutex_lock(&kvm->lock);
129 if (!kvm->created_vcpus) {
130 r = 0;
131 set_bit(KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, &kvm->arch.flags);
132 }
133 mutex_unlock(&kvm->lock);
134 break;
135 default:
136 break;
137 }
138
139 return r;
140 }
141
142 static int kvm_arm_default_max_vcpus(void)
143 {
144 return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
145 }
146
147 /**
148 * kvm_arch_init_vm - initializes a VM data structure
149 * @kvm: pointer to the KVM struct
150 * @type: kvm device type
151 */
152 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
153 {
154 int ret;
155
156 mutex_init(&kvm->arch.config_lock);
157
158 #ifdef CONFIG_LOCKDEP
159 /* Clue in lockdep that the config_lock must be taken inside kvm->lock */
160 mutex_lock(&kvm->lock);
161 mutex_lock(&kvm->arch.config_lock);
162 mutex_unlock(&kvm->arch.config_lock);
163 mutex_unlock(&kvm->lock);
164 #endif
165
166 kvm_init_nested(kvm);
167
168 ret = kvm_share_hyp(kvm, kvm + 1);
169 if (ret)
170 return ret;
171
172 if (!zalloc_cpumask_var(&kvm->arch.supported_cpus, GFP_KERNEL_ACCOUNT)) {
173 ret = -ENOMEM;
174 goto err_unshare_kvm;
175 }
176 cpumask_copy(kvm->arch.supported_cpus, cpu_possible_mask);
177
178 ret = kvm_init_stage2_mmu(kvm, &kvm->arch.mmu, type);
179 if (ret)
180 goto err_free_cpumask;
181
182 if (is_protected_kvm_enabled()) {
183 /*
184 * If any failures occur after this is successful, make sure to
185 * call __pkvm_unreserve_vm to unreserve the VM in hyp.
186 */
187 ret = pkvm_init_host_vm(kvm);
188 if (ret)
189 goto err_free_cpumask;
190 }
191
192 kvm_vgic_early_init(kvm);
193
194 kvm_timer_init_vm(kvm);
195
196 /* The maximum number of VCPUs is limited by the host's GIC model */
197 kvm->max_vcpus = kvm_arm_default_max_vcpus();
198
199 kvm_arm_init_hypercalls(kvm);
200
201 bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES);
202
203 return 0;
204
205 err_free_cpumask:
206 free_cpumask_var(kvm->arch.supported_cpus);
207 err_unshare_kvm:
208 kvm_unshare_hyp(kvm, kvm + 1);
209 return ret;
210 }
211
212 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
213 {
214 return VM_FAULT_SIGBUS;
215 }
216
217 void kvm_arch_create_vm_debugfs(struct kvm *kvm)
218 {
219 kvm_sys_regs_create_debugfs(kvm);
220 kvm_s2_ptdump_create_debugfs(kvm);
221 }
222
223 static void kvm_destroy_mpidr_data(struct kvm *kvm)
224 {
225 struct kvm_mpidr_data *data;
226
227 mutex_lock(&kvm->arch.config_lock);
228
229 data = rcu_dereference_protected(kvm->arch.mpidr_data,
230 lockdep_is_held(&kvm->arch.config_lock));
231 if (data) {
232 rcu_assign_pointer(kvm->arch.mpidr_data, NULL);
233 synchronize_rcu();
234 kfree(data);
235 }
236
237 mutex_unlock(&kvm->arch.config_lock);
238 }
239
240 /**
241 * kvm_arch_destroy_vm - destroy the VM data structure
242 * @kvm: pointer to the KVM struct
243 */
244 void kvm_arch_destroy_vm(struct kvm *kvm)
245 {
246 bitmap_free(kvm->arch.pmu_filter);
247 free_cpumask_var(kvm->arch.supported_cpus);
248
249 kvm_vgic_destroy(kvm);
250
251 if (is_protected_kvm_enabled())
252 pkvm_destroy_hyp_vm(kvm);
253
254 kvm_destroy_mpidr_data(kvm);
255
256 kfree(kvm->arch.sysreg_masks);
257 kvm_destroy_vcpus(kvm);
258
259 kvm_unshare_hyp(kvm, kvm + 1);
260
261 kvm_arm_teardown_hypercalls(kvm);
262 }
263
264 static bool kvm_has_full_ptr_auth(void)
265 {
266 bool apa, gpa, api, gpi, apa3, gpa3;
267 u64 isar1, isar2, val;
268
269 /*
270 * Check that:
271 *
272 * - both Address and Generic auth are implemented for a given
273 * algorithm (Q5, IMPDEF or Q3)
274 * - only a single algorithm is implemented.
275 */
276 if (!system_has_full_ptr_auth())
277 return false;
278
279 isar1 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
280 isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
281
282 apa = !!FIELD_GET(ID_AA64ISAR1_EL1_APA_MASK, isar1);
283 val = FIELD_GET(ID_AA64ISAR1_EL1_GPA_MASK, isar1);
284 gpa = (val == ID_AA64ISAR1_EL1_GPA_IMP);
285
286 api = !!FIELD_GET(ID_AA64ISAR1_EL1_API_MASK, isar1);
287 val = FIELD_GET(ID_AA64ISAR1_EL1_GPI_MASK, isar1);
288 gpi = (val == ID_AA64ISAR1_EL1_GPI_IMP);
289
290 apa3 = !!FIELD_GET(ID_AA64ISAR2_EL1_APA3_MASK, isar2);
291 val = FIELD_GET(ID_AA64ISAR2_EL1_GPA3_MASK, isar2);
292 gpa3 = (val == ID_AA64ISAR2_EL1_GPA3_IMP);
293
294 return (apa == gpa && api == gpi && apa3 == gpa3 &&
295 (apa + api + apa3) == 1);
296 }
297
298 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
299 {
300 int r;
301
302 if (kvm && kvm_vm_is_protected(kvm) && !kvm_pvm_ext_allowed(ext))
303 return 0;
304
305 switch (ext) {
306 case KVM_CAP_IRQCHIP:
307 r = vgic_present;
308 break;
309 case KVM_CAP_IOEVENTFD:
310 case KVM_CAP_USER_MEMORY:
311 case KVM_CAP_SYNC_MMU:
312 case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
313 case KVM_CAP_ONE_REG:
314 case KVM_CAP_ARM_PSCI:
315 case KVM_CAP_ARM_PSCI_0_2:
316 case KVM_CAP_READONLY_MEM:
317 case KVM_CAP_MP_STATE:
318 case KVM_CAP_IMMEDIATE_EXIT:
319 case KVM_CAP_VCPU_EVENTS:
320 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
321 case KVM_CAP_ARM_NISV_TO_USER:
322 case KVM_CAP_ARM_INJECT_EXT_DABT:
323 case KVM_CAP_SET_GUEST_DEBUG:
324 case KVM_CAP_VCPU_ATTRIBUTES:
325 case KVM_CAP_PTP_KVM:
326 case KVM_CAP_ARM_SYSTEM_SUSPEND:
327 case KVM_CAP_IRQFD_RESAMPLE:
328 case KVM_CAP_COUNTER_OFFSET:
329 case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
330 r = 1;
331 break;
332 case KVM_CAP_SET_GUEST_DEBUG2:
333 return KVM_GUESTDBG_VALID_MASK;
334 case KVM_CAP_ARM_SET_DEVICE_ADDR:
335 r = 1;
336 break;
337 case KVM_CAP_NR_VCPUS:
338 /*
339 * ARM64 treats KVM_CAP_NR_CPUS differently from all other
340 * architectures, as it does not always bound it to
341 * KVM_CAP_MAX_VCPUS. It should not matter much because
342 * this is just an advisory value.
343 */
344 r = min_t(unsigned int, num_online_cpus(),
345 kvm_arm_default_max_vcpus());
346 break;
347 case KVM_CAP_MAX_VCPUS:
348 case KVM_CAP_MAX_VCPU_ID:
349 if (kvm)
350 r = kvm->max_vcpus;
351 else
352 r = kvm_arm_default_max_vcpus();
353 break;
354 case KVM_CAP_MSI_DEVID:
355 if (!kvm)
356 r = -EINVAL;
357 else
358 r = kvm->arch.vgic.msis_require_devid;
359 break;
360 case KVM_CAP_ARM_USER_IRQ:
361 /*
362 * 1: EL1_VTIMER, EL1_PTIMER, and PMU.
363 * (bump this number if adding more devices)
364 */
365 r = 1;
366 break;
367 case KVM_CAP_ARM_MTE:
368 r = system_supports_mte();
369 break;
370 case KVM_CAP_STEAL_TIME:
371 r = kvm_arm_pvtime_supported();
372 break;
373 case KVM_CAP_ARM_EL1_32BIT:
374 r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
375 break;
376 case KVM_CAP_ARM_EL2:
377 r = cpus_have_final_cap(ARM64_HAS_NESTED_VIRT);
378 break;
379 case KVM_CAP_ARM_EL2_E2H0:
380 r = cpus_have_final_cap(ARM64_HAS_HCR_NV1);
381 break;
382 case KVM_CAP_GUEST_DEBUG_HW_BPS:
383 r = get_num_brps();
384 break;
385 case KVM_CAP_GUEST_DEBUG_HW_WPS:
386 r = get_num_wrps();
387 break;
388 case KVM_CAP_ARM_PMU_V3:
389 r = kvm_supports_guest_pmuv3();
390 break;
391 case KVM_CAP_ARM_INJECT_SERROR_ESR:
392 r = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
393 break;
394 case KVM_CAP_ARM_VM_IPA_SIZE:
395 r = get_kvm_ipa_limit();
396 break;
397 case KVM_CAP_ARM_SVE:
398 r = system_supports_sve();
399 break;
400 case KVM_CAP_ARM_PTRAUTH_ADDRESS:
401 case KVM_CAP_ARM_PTRAUTH_GENERIC:
402 r = kvm_has_full_ptr_auth();
403 break;
404 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
405 if (kvm)
406 r = kvm->arch.mmu.split_page_chunk_size;
407 else
408 r = KVM_ARM_EAGER_SPLIT_CHUNK_SIZE_DEFAULT;
409 break;
410 case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
411 r = kvm_supported_block_sizes();
412 break;
413 case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
414 r = BIT(0);
415 break;
416 case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED:
417 if (!kvm)
418 r = -EINVAL;
419 else
420 r = kvm_supports_cacheable_pfnmap();
421 break;
422
423 default:
424 r = 0;
425 }
426
427 return r;
428 }
429
430 long kvm_arch_dev_ioctl(struct file *filp,
431 unsigned int ioctl, unsigned long arg)
432 {
433 return -EINVAL;
434 }
435
436 struct kvm *kvm_arch_alloc_vm(void)
437 {
438 size_t sz = sizeof(struct kvm);
439
440 if (!has_vhe())
441 return kzalloc(sz, GFP_KERNEL_ACCOUNT);
442
443 return __vmalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_HIGHMEM | __GFP_ZERO);
444 }
445
446 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
447 {
448 if (irqchip_in_kernel(kvm) && vgic_initialized(kvm))
449 return -EBUSY;
450
451 if (id >= kvm->max_vcpus)
452 return -EINVAL;
453
454 return 0;
455 }
456
457 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
458 {
459 int err;
460
461 spin_lock_init(&vcpu->arch.mp_state_lock);
462
463 #ifdef CONFIG_LOCKDEP
464 /* Inform lockdep that the config_lock is acquired after vcpu->mutex */
465 mutex_lock(&vcpu->mutex);
466 mutex_lock(&vcpu->kvm->arch.config_lock);
467 mutex_unlock(&vcpu->kvm->arch.config_lock);
468 mutex_unlock(&vcpu->mutex);
469 #endif
470
471 /* Force users to call KVM_ARM_VCPU_INIT */
472 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
473
474 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO;
475
476 /* Set up the timer */
477 kvm_timer_vcpu_init(vcpu);
478
479 kvm_pmu_vcpu_init(vcpu);
480
481 kvm_arm_pvtime_vcpu_init(&vcpu->arch);
482
483 vcpu->arch.hw_mmu = &vcpu->kvm->arch.mmu;
484
485 /*
486 * This vCPU may have been created after mpidr_data was initialized.
487 * Throw out the pre-computed mappings if that is the case which forces
488 * KVM to fall back to iteratively searching the vCPUs.
489 */
490 kvm_destroy_mpidr_data(vcpu->kvm);
491
492 err = kvm_vgic_vcpu_init(vcpu);
493 if (err) {
494 kvm_vgic_vcpu_destroy(vcpu);
495 return err;
496 }
497
498 err = kvm_share_hyp(vcpu, vcpu + 1);
499 if (err)
500 kvm_vgic_vcpu_destroy(vcpu);
501
502 return err;
503 }
504
505 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
506 {
507 }
508
509 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
510 {
511 if (!is_protected_kvm_enabled())
512 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
513 else
514 free_hyp_memcache(&vcpu->arch.pkvm_memcache);
515 kvm_timer_vcpu_terminate(vcpu);
516 kvm_pmu_vcpu_destroy(vcpu);
517 kvm_vgic_vcpu_destroy(vcpu);
518 kvm_arm_vcpu_destroy(vcpu);
519 }
520
521 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu)
522 {
523
524 }
525
526 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
527 {
528
529 }
530
531 static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
532 {
533 if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
534 /*
535 * Either we're running an L2 guest, and the API/APK bits come
536 * from L1's HCR_EL2, or API/APK are both set.
537 */
538 if (unlikely(is_nested_ctxt(vcpu))) {
539 u64 val;
540
541 val = __vcpu_sys_reg(vcpu, HCR_EL2);
542 val &= (HCR_API | HCR_APK);
543 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
544 vcpu->arch.hcr_el2 |= val;
545 } else {
546 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
547 }
548
549 /*
550 * Save the host keys if there is any chance for the guest
551 * to use pauth, as the entry code will reload the guest
552 * keys in that case.
553 */
554 if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
555 struct kvm_cpu_context *ctxt;
556
557 ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
558 ptrauth_save_keys(ctxt);
559 }
560 }
561 }
562
563 static bool kvm_vcpu_should_clear_twi(struct kvm_vcpu *vcpu)
564 {
565 if (unlikely(kvm_wfi_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
566 return kvm_wfi_trap_policy == KVM_WFX_NOTRAP;
567
568 return single_task_running() &&
569 (atomic_read(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe.vlpi_count) ||
570 vcpu->kvm->arch.vgic.nassgireq);
571 }
572
573 static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
574 {
575 if (unlikely(kvm_wfe_trap_policy != KVM_WFX_NOTRAP_SINGLE_TASK))
576 return kvm_wfe_trap_policy == KVM_WFX_NOTRAP;
577
578 return single_task_running();
579 }
580
581 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
582 {
583 struct kvm_s2_mmu *mmu;
584 int *last_ran;
585
586 if (is_protected_kvm_enabled())
587 goto nommu;
588
589 if (vcpu_has_nv(vcpu))
590 kvm_vcpu_load_hw_mmu(vcpu);
591
592 mmu = vcpu->arch.hw_mmu;
593 last_ran = this_cpu_ptr(mmu->last_vcpu_ran);
594
595 /*
596 * Ensure a VMID is allocated for the MMU before programming VTTBR_EL2,
597 * which happens eagerly in VHE.
598 *
599 * Also, the VMID allocator only preserves VMIDs that are active at the
600 * time of rollover, so KVM might need to grab a new VMID for the MMU if
601 * this is called from kvm_sched_in().
602 */
603 kvm_arm_vmid_update(&mmu->vmid);
604
605 /*
606 * We guarantee that both TLBs and I-cache are private to each
607 * vcpu. If detecting that a vcpu from the same VM has
608 * previously run on the same physical CPU, call into the
609 * hypervisor code to nuke the relevant contexts.
610 *
611 * We might get preempted before the vCPU actually runs, but
612 * over-invalidation doesn't affect correctness.
613 */
614 if (*last_ran != vcpu->vcpu_idx) {
615 kvm_call_hyp(__kvm_flush_cpu_context, mmu);
616 *last_ran = vcpu->vcpu_idx;
617 }
618
619 nommu:
620 vcpu->cpu = cpu;
621
622 /*
623 * The timer must be loaded before the vgic to correctly set up physical
624 * interrupt deactivation in nested state (e.g. timer interrupt).
625 */
626 kvm_timer_vcpu_load(vcpu);
627 kvm_vgic_load(vcpu);
628 kvm_vcpu_load_debug(vcpu);
629 kvm_vcpu_load_fgt(vcpu);
630 if (has_vhe())
631 kvm_vcpu_load_vhe(vcpu);
632 kvm_arch_vcpu_load_fp(vcpu);
633 kvm_vcpu_pmu_restore_guest(vcpu);
634 if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
635 kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
636
637 if (kvm_vcpu_should_clear_twe(vcpu))
638 vcpu->arch.hcr_el2 &= ~HCR_TWE;
639 else
640 vcpu->arch.hcr_el2 |= HCR_TWE;
641
642 if (kvm_vcpu_should_clear_twi(vcpu))
643 vcpu->arch.hcr_el2 &= ~HCR_TWI;
644 else
645 vcpu->arch.hcr_el2 |= HCR_TWI;
646
647 vcpu_set_pauth_traps(vcpu);
648
649 if (is_protected_kvm_enabled()) {
650 kvm_call_hyp_nvhe(__pkvm_vcpu_load,
651 vcpu->kvm->arch.pkvm.handle,
652 vcpu->vcpu_idx, vcpu->arch.hcr_el2);
653 kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
654 &vcpu->arch.vgic_cpu.vgic_v3);
655 }
656
657 if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
658 vcpu_set_on_unsupported_cpu(vcpu);
659 }
660
661 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
662 {
663 if (is_protected_kvm_enabled()) {
664 kvm_call_hyp(__vgic_v3_save_vmcr_aprs,
665 &vcpu->arch.vgic_cpu.vgic_v3);
666 kvm_call_hyp_nvhe(__pkvm_vcpu_put);
667 }
668
669 kvm_vcpu_put_debug(vcpu);
670 kvm_arch_vcpu_put_fp(vcpu);
671 if (has_vhe())
672 kvm_vcpu_put_vhe(vcpu);
673 kvm_timer_vcpu_put(vcpu);
674 kvm_vgic_put(vcpu);
675 kvm_vcpu_pmu_restore_host(vcpu);
676 if (vcpu_has_nv(vcpu))
677 kvm_vcpu_put_hw_mmu(vcpu);
678 kvm_arm_vmid_clear_active();
679
680 vcpu_clear_on_unsupported_cpu(vcpu);
681 vcpu->cpu = -1;
682 }
683
684 static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
685 {
686 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
687 kvm_make_request(KVM_REQ_SLEEP, vcpu);
688 kvm_vcpu_kick(vcpu);
689 }
690
691 void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
692 {
693 spin_lock(&vcpu->arch.mp_state_lock);
694 __kvm_arm_vcpu_power_off(vcpu);
695 spin_unlock(&vcpu->arch.mp_state_lock);
696 }
697
698 bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
699 {
700 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
701 }
702
703 static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
704 {
705 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_SUSPENDED);
706 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
707 kvm_vcpu_kick(vcpu);
708 }
709
710 static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
711 {
712 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
713 }
714
715 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
716 struct kvm_mp_state *mp_state)
717 {
718 *mp_state = READ_ONCE(vcpu->arch.mp_state);
719
720 return 0;
721 }
722
723 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
724 struct kvm_mp_state *mp_state)
725 {
726 int ret = 0;
727
728 spin_lock(&vcpu->arch.mp_state_lock);
729
730 switch (mp_state->mp_state) {
731 case KVM_MP_STATE_RUNNABLE:
732 WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
733 break;
734 case KVM_MP_STATE_STOPPED:
735 __kvm_arm_vcpu_power_off(vcpu);
736 break;
737 case KVM_MP_STATE_SUSPENDED:
738 kvm_arm_vcpu_suspend(vcpu);
739 break;
740 default:
741 ret = -EINVAL;
742 }
743
744 spin_unlock(&vcpu->arch.mp_state_lock);
745
746 return ret;
747 }
748
749 /**
750 * kvm_arch_vcpu_runnable - determine if the vcpu can be scheduled
751 * @v: The VCPU pointer
752 *
753 * If the guest CPU is not waiting for interrupts or an interrupt line is
754 * asserted, the CPU is by definition runnable.
755 */
756 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
757 {
758 bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
759
760 irq_lines |= (!irqchip_in_kernel(v->kvm) &&
761 (kvm_timer_should_notify_user(v) ||
762 kvm_pmu_should_notify_user(v)));
763
764 return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
765 && !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
766 }
767
768 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
769 {
770 return vcpu_mode_priv(vcpu);
771 }
772
773 #ifdef CONFIG_GUEST_PERF_EVENTS
774 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu)
775 {
776 return *vcpu_pc(vcpu);
777 }
778 #endif
779
780 static void kvm_init_mpidr_data(struct kvm *kvm)
781 {
782 struct kvm_mpidr_data *data = NULL;
783 unsigned long c, mask, nr_entries;
784 u64 aff_set = 0, aff_clr = ~0UL;
785 struct kvm_vcpu *vcpu;
786
787 mutex_lock(&kvm->arch.config_lock);
788
789 if (rcu_access_pointer(kvm->arch.mpidr_data) ||
790 atomic_read(&kvm->online_vcpus) == 1)
791 goto out;
792
793 kvm_for_each_vcpu(c, vcpu, kvm) {
794 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
795 aff_set |= aff;
796 aff_clr &= aff;
797 }
798
799 /*
800 * A significant bit can be either 0 or 1, and will only appear in
801 * aff_set. Use aff_clr to weed out the useless stuff.
802 */
803 mask = aff_set ^ aff_clr;
804 nr_entries = BIT_ULL(hweight_long(mask));
805
806 /*
807 * Don't let userspace fool us. If we need more than a single page
808 * to describe the compressed MPIDR array, just fall back to the
809 * iterative method. Single vcpu VMs do not need this either.
810 */
811 if (struct_size(data, cmpidr_to_idx, nr_entries) <= PAGE_SIZE)
812 data = kzalloc(struct_size(data, cmpidr_to_idx, nr_entries),
813 GFP_KERNEL_ACCOUNT);
814
815 if (!data)
816 goto out;
817
818 data->mpidr_mask = mask;
819
820 kvm_for_each_vcpu(c, vcpu, kvm) {
821 u64 aff = kvm_vcpu_get_mpidr_aff(vcpu);
822 u16 index = kvm_mpidr_index(data, aff);
823
824 data->cmpidr_to_idx[index] = c;
825 }
826
827 rcu_assign_pointer(kvm->arch.mpidr_data, data);
828 out:
829 mutex_unlock(&kvm->arch.config_lock);
830 }
831
832 /*
833 * Handle both the initialisation that is being done when the vcpu is
834 * run for the first time, as well as the updates that must be
835 * performed each time we get a new thread dealing with this vcpu.
836 */
837 int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
838 {
839 struct kvm *kvm = vcpu->kvm;
840 int ret;
841
842 if (!kvm_vcpu_initialized(vcpu))
843 return -ENOEXEC;
844
845 if (!kvm_arm_vcpu_is_finalized(vcpu))
846 return -EPERM;
847
848 if (likely(vcpu_has_run_once(vcpu)))
849 return 0;
850
851 kvm_init_mpidr_data(kvm);
852
853 if (likely(irqchip_in_kernel(kvm))) {
854 /*
855 * Map the VGIC hardware resources before running a vcpu the
856 * first time on this VM.
857 */
858 ret = kvm_vgic_map_resources(kvm);
859 if (ret)
860 return ret;
861 }
862
863 ret = kvm_finalize_sys_regs(vcpu);
864 if (ret)
865 return ret;
866
867 if (vcpu_has_nv(vcpu)) {
868 ret = kvm_vcpu_allocate_vncr_tlb(vcpu);
869 if (ret)
870 return ret;
871
872 ret = kvm_vgic_vcpu_nv_init(vcpu);
873 if (ret)
874 return ret;
875 }
876
877 /*
878 * This needs to happen after any restriction has been applied
879 * to the feature set.
880 */
881 kvm_calculate_traps(vcpu);
882
883 ret = kvm_timer_enable(vcpu);
884 if (ret)
885 return ret;
886
887 if (kvm_vcpu_has_pmu(vcpu)) {
888 ret = kvm_arm_pmu_v3_enable(vcpu);
889 if (ret)
890 return ret;
891 }
892
893 if (is_protected_kvm_enabled()) {
894 ret = pkvm_create_hyp_vm(kvm);
895 if (ret)
896 return ret;
897
898 ret = pkvm_create_hyp_vcpu(vcpu);
899 if (ret)
900 return ret;
901 }
902
903 mutex_lock(&kvm->arch.config_lock);
904 set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags);
905 mutex_unlock(&kvm->arch.config_lock);
906
907 return ret;
908 }
909
910 bool kvm_arch_intc_initialized(struct kvm *kvm)
911 {
912 return vgic_initialized(kvm);
913 }
914
915 void kvm_arm_halt_guest(struct kvm *kvm)
916 {
917 unsigned long i;
918 struct kvm_vcpu *vcpu;
919
920 kvm_for_each_vcpu(i, vcpu, kvm)
921 vcpu->arch.pause = true;
922 kvm_make_all_cpus_request(kvm, KVM_REQ_SLEEP);
923 }
924
925 void kvm_arm_resume_guest(struct kvm *kvm)
926 {
927 unsigned long i;
928 struct kvm_vcpu *vcpu;
929
930 kvm_for_each_vcpu(i, vcpu, kvm) {
931 vcpu->arch.pause = false;
932 __kvm_vcpu_wake_up(vcpu);
933 }
934 }
935
936 static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu)
937 {
938 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
939
940 rcuwait_wait_event(wait,
941 (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause),
942 TASK_INTERRUPTIBLE);
943
944 if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) {
945 /* Awaken to handle a signal, request we sleep again later. */
946 kvm_make_request(KVM_REQ_SLEEP, vcpu);
947 }
948
949 /*
950 * Make sure we will observe a potential reset request if we've
951 * observed a change to the power state. Pairs with the smp_wmb() in
952 * kvm_psci_vcpu_on().
953 */
954 smp_rmb();
955 }
956
957 /**
958 * kvm_vcpu_wfi - emulate Wait-For-Interrupt behavior
959 * @vcpu: The VCPU pointer
960 *
961 * Suspend execution of a vCPU until a valid wake event is detected, i.e. until
962 * the vCPU is runnable. The vCPU may or may not be scheduled out, depending
963 * on when a wake event arrives, e.g. there may already be a pending wake event.
964 */
965 void kvm_vcpu_wfi(struct kvm_vcpu *vcpu)
966 {
967 /*
968 * Sync back the state of the GIC CPU interface so that we have
969 * the latest PMR and group enables. This ensures that
970 * kvm_arch_vcpu_runnable has up-to-date data to decide whether
971 * we have pending interrupts, e.g. when determining if the
972 * vCPU should block.
973 *
974 * For the same reason, we want to tell GICv4 that we need
975 * doorbells to be signalled, should an interrupt become pending.
976 */
977 preempt_disable();
978 vcpu_set_flag(vcpu, IN_WFI);
979 kvm_vgic_put(vcpu);
980 preempt_enable();
981
982 kvm_vcpu_halt(vcpu);
983 vcpu_clear_flag(vcpu, IN_WFIT);
984
985 preempt_disable();
986 vcpu_clear_flag(vcpu, IN_WFI);
987 kvm_vgic_load(vcpu);
988 preempt_enable();
989 }
990
991 static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu)
992 {
993 if (!kvm_arm_vcpu_suspended(vcpu))
994 return 1;
995
996 kvm_vcpu_wfi(vcpu);
997
998 /*
999 * The suspend state is sticky; we do not leave it until userspace
1000 * explicitly marks the vCPU as runnable. Request that we suspend again
1001 * later.
1002 */
1003 kvm_make_request(KVM_REQ_SUSPEND, vcpu);
1004
1005 /*
1006 * Check to make sure the vCPU is actually runnable. If so, exit to
1007 * userspace informing it of the wakeup condition.
1008 */
1009 if (kvm_arch_vcpu_runnable(vcpu)) {
1010 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event));
1011 vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP;
1012 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT;
1013 return 0;
1014 }
1015
1016 /*
1017 * Otherwise, we were unblocked to process a different event, such as a
1018 * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to
1019 * process the event.
1020 */
1021 return 1;
1022 }
1023
1024 /**
1025 * check_vcpu_requests - check and handle pending vCPU requests
1026 * @vcpu: the VCPU pointer
1027 *
1028 * Return: 1 if we should enter the guest
1029 * 0 if we should exit to userspace
1030 * < 0 if we should exit to userspace, where the return value indicates
1031 * an error
1032 */
1033 static int check_vcpu_requests(struct kvm_vcpu *vcpu)
1034 {
1035 if (kvm_request_pending(vcpu)) {
1036 if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu))
1037 return -EIO;
1038
1039 if (kvm_check_request(KVM_REQ_SLEEP, vcpu))
1040 kvm_vcpu_sleep(vcpu);
1041
1042 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1043 kvm_reset_vcpu(vcpu);
1044
1045 /*
1046 * Clear IRQ_PENDING requests that were made to guarantee
1047 * that a VCPU sees new virtual interrupts.
1048 */
1049 kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu);
1050
1051 if (kvm_check_request(KVM_REQ_RECORD_STEAL, vcpu))
1052 kvm_update_stolen_time(vcpu);
1053
1054 if (kvm_check_request(KVM_REQ_RELOAD_GICv4, vcpu)) {
1055 /* The distributor enable bits were changed */
1056 preempt_disable();
1057 vgic_v4_put(vcpu);
1058 vgic_v4_load(vcpu);
1059 preempt_enable();
1060 }
1061
1062 if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
1063 kvm_vcpu_reload_pmu(vcpu);
1064
1065 if (kvm_check_request(KVM_REQ_RESYNC_PMU_EL0, vcpu))
1066 kvm_vcpu_pmu_restore_guest(vcpu);
1067
1068 if (kvm_check_request(KVM_REQ_SUSPEND, vcpu))
1069 return kvm_vcpu_suspend(vcpu);
1070
1071 if (kvm_dirty_ring_check_request(vcpu))
1072 return 0;
1073
1074 check_nested_vcpu_requests(vcpu);
1075 }
1076
1077 return 1;
1078 }
1079
1080 static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
1081 {
1082 if (likely(!vcpu_mode_is_32bit(vcpu)))
1083 return false;
1084
1085 if (vcpu_has_nv(vcpu))
1086 return true;
1087
1088 return !kvm_supports_32bit_el0();
1089 }
1090
1091 /**
1092 * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
1093 * @vcpu: The VCPU pointer
1094 * @ret: Pointer to write optional return code
1095 *
1096 * Returns: true if the VCPU needs to return to a preemptible + interruptible
1097 * and skip guest entry.
1098 *
1099 * This function disambiguates between two different types of exits: exits to a
1100 * preemptible + interruptible kernel context and exits to userspace. For an
1101 * exit to userspace, this function will write the return code to ret and return
1102 * true. For an exit to preemptible + interruptible kernel context (i.e. check
1103 * for pending work and re-enter), return true without writing to ret.
1104 */
1105 static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
1106 {
1107 struct kvm_run *run = vcpu->run;
1108
1109 /*
1110 * If we're using a userspace irqchip, then check if we need
1111 * to tell a userspace irqchip about timer or PMU level
1112 * changes and if so, exit to userspace (the actual level
1113 * state gets updated in kvm_timer_update_run and
1114 * kvm_pmu_update_run below).
1115 */
1116 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1117 if (kvm_timer_should_notify_user(vcpu) ||
1118 kvm_pmu_should_notify_user(vcpu)) {
1119 *ret = -EINTR;
1120 run->exit_reason = KVM_EXIT_INTR;
1121 return true;
1122 }
1123 }
1124
1125 if (unlikely(vcpu_on_unsupported_cpu(vcpu))) {
1126 run->exit_reason = KVM_EXIT_FAIL_ENTRY;
1127 run->fail_entry.hardware_entry_failure_reason = KVM_EXIT_FAIL_ENTRY_CPU_UNSUPPORTED;
1128 run->fail_entry.cpu = smp_processor_id();
1129 *ret = 0;
1130 return true;
1131 }
1132
1133 return kvm_request_pending(vcpu) ||
1134 xfer_to_guest_mode_work_pending();
1135 }
1136
1137 /*
1138 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while
1139 * the vCPU is running.
1140 *
1141 * This must be noinstr as instrumentation may make use of RCU, and this is not
1142 * safe during the EQS.
1143 */
1144 static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
1145 {
1146 int ret;
1147
1148 guest_state_enter_irqoff();
1149 ret = kvm_call_hyp_ret(__kvm_vcpu_run, vcpu);
1150 guest_state_exit_irqoff();
1151
1152 return ret;
1153 }
1154
1155 /**
1156 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
1157 * @vcpu: The VCPU pointer
1158 *
1159 * This function is called through the VCPU_RUN ioctl called from user space. It
1160 * will execute VM code in a loop until the time slice for the process is used
1161 * or some emulation is needed from user space in which case the function will
1162 * return with return value 0 and with the kvm_run structure filled in with the
1163 * required data for the requested emulation.
1164 */
1165 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1166 {
1167 struct kvm_run *run = vcpu->run;
1168 int ret;
1169
1170 if (run->exit_reason == KVM_EXIT_MMIO) {
1171 ret = kvm_handle_mmio_return(vcpu);
1172 if (ret <= 0)
1173 return ret;
1174 }
1175
1176 vcpu_load(vcpu);
1177
1178 if (!vcpu->wants_to_run) {
1179 ret = -EINTR;
1180 goto out;
1181 }
1182
1183 kvm_sigset_activate(vcpu);
1184
1185 ret = 1;
1186 run->exit_reason = KVM_EXIT_UNKNOWN;
1187 run->flags = 0;
1188 while (ret > 0) {
1189 /*
1190 * Check conditions before entering the guest
1191 */
1192 ret = kvm_xfer_to_guest_mode_handle_work(vcpu);
1193 if (!ret)
1194 ret = 1;
1195
1196 if (ret > 0)
1197 ret = check_vcpu_requests(vcpu);
1198
1199 /*
1200 * Preparing the interrupts to be injected also
1201 * involves poking the GIC, which must be done in a
1202 * non-preemptible context.
1203 */
1204 preempt_disable();
1205
1206 kvm_nested_flush_hwstate(vcpu);
1207
1208 if (kvm_vcpu_has_pmu(vcpu))
1209 kvm_pmu_flush_hwstate(vcpu);
1210
1211 local_irq_disable();
1212
1213 kvm_vgic_flush_hwstate(vcpu);
1214
1215 kvm_pmu_update_vcpu_events(vcpu);
1216
1217 /*
1218 * Ensure we set mode to IN_GUEST_MODE after we disable
1219 * interrupts and before the final VCPU requests check.
1220 * See the comment in kvm_vcpu_exiting_guest_mode() and
1221 * Documentation/virt/kvm/vcpu-requests.rst
1222 */
1223 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
1224
1225 if (ret <= 0 || kvm_vcpu_exit_request(vcpu, &ret)) {
1226 vcpu->mode = OUTSIDE_GUEST_MODE;
1227 isb(); /* Ensure work in x_flush_hwstate is committed */
1228 if (kvm_vcpu_has_pmu(vcpu))
1229 kvm_pmu_sync_hwstate(vcpu);
1230 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1231 kvm_timer_sync_user(vcpu);
1232 kvm_vgic_sync_hwstate(vcpu);
1233 local_irq_enable();
1234 preempt_enable();
1235 continue;
1236 }
1237
1238 kvm_arch_vcpu_ctxflush_fp(vcpu);
1239
1240 /**************************************************************
1241 * Enter the guest
1242 */
1243 trace_kvm_entry(*vcpu_pc(vcpu));
1244 guest_timing_enter_irqoff();
1245
1246 ret = kvm_arm_vcpu_enter_exit(vcpu);
1247
1248 vcpu->mode = OUTSIDE_GUEST_MODE;
1249 vcpu->stat.exits++;
1250 /*
1251 * Back from guest
1252 *************************************************************/
1253
1254 /*
1255 * We must sync the PMU state before the vgic state so
1256 * that the vgic can properly sample the updated state of the
1257 * interrupt line.
1258 */
1259 if (kvm_vcpu_has_pmu(vcpu))
1260 kvm_pmu_sync_hwstate(vcpu);
1261
1262 /*
1263 * Sync the vgic state before syncing the timer state because
1264 * the timer code needs to know if the virtual timer
1265 * interrupts are active.
1266 */
1267 kvm_vgic_sync_hwstate(vcpu);
1268
1269 /*
1270 * Sync the timer hardware state before enabling interrupts as
1271 * we don't want vtimer interrupts to race with syncing the
1272 * timer virtual interrupt state.
1273 */
1274 if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
1275 kvm_timer_sync_user(vcpu);
1276
1277 if (is_hyp_ctxt(vcpu))
1278 kvm_timer_sync_nested(vcpu);
1279
1280 kvm_arch_vcpu_ctxsync_fp(vcpu);
1281
1282 /*
1283 * We must ensure that any pending interrupts are taken before
1284 * we exit guest timing so that timer ticks are accounted as
1285 * guest time. Transiently unmask interrupts so that any
1286 * pending interrupts are taken.
1287 *
1288 * Per ARM DDI 0487G.b section D1.13.4, an ISB (or other
1289 * context synchronization event) is necessary to ensure that
1290 * pending interrupts are taken.
1291 */
1292 if (ARM_EXCEPTION_CODE(ret) == ARM_EXCEPTION_IRQ) {
1293 local_irq_enable();
1294 isb();
1295 local_irq_disable();
1296 }
1297
1298 guest_timing_exit_irqoff();
1299
1300 local_irq_enable();
1301
1302 trace_kvm_exit(ret, kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu));
1303
1304 /* Exit types that need handling before we can be preempted */
1305 handle_exit_early(vcpu, ret);
1306
1307 kvm_nested_sync_hwstate(vcpu);
1308
1309 preempt_enable();
1310
1311 /*
1312 * The ARMv8 architecture doesn't give the hypervisor
1313 * a mechanism to prevent a guest from dropping to AArch32 EL0
1314 * if implemented by the CPU. If we spot the guest in such
1315 * state and that we decided it wasn't supposed to do so (like
1316 * with the asymmetric AArch32 case), return to userspace with
1317 * a fatal error.
1318 */
1319 if (vcpu_mode_is_bad_32bit(vcpu)) {
1320 /*
1321 * As we have caught the guest red-handed, decide that
1322 * it isn't fit for purpose anymore by making the vcpu
1323 * invalid. The VMM can try and fix it by issuing a
1324 * KVM_ARM_VCPU_INIT if it really wants to.
1325 */
1326 vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
1327 ret = ARM_EXCEPTION_IL;
1328 }
1329
1330 ret = handle_exit(vcpu, ret);
1331 }
1332
1333 /* Tell userspace about in-kernel device output levels */
1334 if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
1335 kvm_timer_update_run(vcpu);
1336 kvm_pmu_update_run(vcpu);
1337 }
1338
1339 kvm_sigset_deactivate(vcpu);
1340
1341 out:
1342 /*
1343 * In the unlikely event that we are returning to userspace
1344 * with pending exceptions or PC adjustment, commit these
1345 * adjustments in order to give userspace a consistent view of
1346 * the vcpu state. Note that this relies on __kvm_adjust_pc()
1347 * being preempt-safe on VHE.
1348 */
1349 if (unlikely(vcpu_get_flag(vcpu, PENDING_EXCEPTION) ||
1350 vcpu_get_flag(vcpu, INCREMENT_PC)))
1351 kvm_call_hyp(__kvm_adjust_pc, vcpu);
1352
1353 vcpu_put(vcpu);
1354 return ret;
1355 }
1356
1357 static int vcpu_interrupt_line(struct kvm_vcpu *vcpu, int number, bool level)
1358 {
1359 int bit_index;
1360 bool set;
1361 unsigned long *hcr;
1362
1363 if (number == KVM_ARM_IRQ_CPU_IRQ)
1364 bit_index = __ffs(HCR_VI);
1365 else /* KVM_ARM_IRQ_CPU_FIQ */
1366 bit_index = __ffs(HCR_VF);
1367
1368 hcr = vcpu_hcr(vcpu);
1369 if (level)
1370 set = test_and_set_bit(bit_index, hcr);
1371 else
1372 set = test_and_clear_bit(bit_index, hcr);
1373
1374 /*
1375 * If we didn't change anything, no need to wake up or kick other CPUs
1376 */
1377 if (set == level)
1378 return 0;
1379
1380 /*
1381 * The vcpu irq_lines field was updated, wake up sleeping VCPUs and
1382 * trigger a world-switch round on the running physical CPU to set the
1383 * virtual IRQ/FIQ fields in the HCR appropriately.
1384 */
1385 kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
1386 kvm_vcpu_kick(vcpu);
1387
1388 return 0;
1389 }
1390
1391 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
1392 bool line_status)
1393 {
1394 u32 irq = irq_level->irq;
1395 unsigned int irq_type, vcpu_id, irq_num;
1396 struct kvm_vcpu *vcpu = NULL;
1397 bool level = irq_level->level;
1398
1399 irq_type = (irq >> KVM_ARM_IRQ_TYPE_SHIFT) & KVM_ARM_IRQ_TYPE_MASK;
1400 vcpu_id = (irq >> KVM_ARM_IRQ_VCPU_SHIFT) & KVM_ARM_IRQ_VCPU_MASK;
1401 vcpu_id += ((irq >> KVM_ARM_IRQ_VCPU2_SHIFT) & KVM_ARM_IRQ_VCPU2_MASK) * (KVM_ARM_IRQ_VCPU_MASK + 1);
1402 irq_num = (irq >> KVM_ARM_IRQ_NUM_SHIFT) & KVM_ARM_IRQ_NUM_MASK;
1403
1404 trace_kvm_irq_line(irq_type, vcpu_id, irq_num, irq_level->level);
1405
1406 switch (irq_type) {
1407 case KVM_ARM_IRQ_TYPE_CPU:
1408 if (irqchip_in_kernel(kvm))
1409 return -ENXIO;
1410
1411 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1412 if (!vcpu)
1413 return -EINVAL;
1414
1415 if (irq_num > KVM_ARM_IRQ_CPU_FIQ)
1416 return -EINVAL;
1417
1418 return vcpu_interrupt_line(vcpu, irq_num, level);
1419 case KVM_ARM_IRQ_TYPE_PPI:
1420 if (!irqchip_in_kernel(kvm))
1421 return -ENXIO;
1422
1423 vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
1424 if (!vcpu)
1425 return -EINVAL;
1426
1427 if (irq_num < VGIC_NR_SGIS || irq_num >= VGIC_NR_PRIVATE_IRQS)
1428 return -EINVAL;
1429
1430 return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
1431 case KVM_ARM_IRQ_TYPE_SPI:
1432 if (!irqchip_in_kernel(kvm))
1433 return -ENXIO;
1434
1435 if (irq_num < VGIC_NR_PRIVATE_IRQS)
1436 return -EINVAL;
1437
1438 return kvm_vgic_inject_irq(kvm, NULL, irq_num, level, NULL);
1439 }
1440
1441 return -EINVAL;
1442 }
1443
1444 static unsigned long system_supported_vcpu_features(void)
1445 {
1446 unsigned long features = KVM_VCPU_VALID_FEATURES;
1447
1448 if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
1449 clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
1450
1451 if (!kvm_supports_guest_pmuv3())
1452 clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
1453
1454 if (!system_supports_sve())
1455 clear_bit(KVM_ARM_VCPU_SVE, &features);
1456
1457 if (!kvm_has_full_ptr_auth()) {
1458 clear_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features);
1459 clear_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features);
1460 }
1461
1462 if (!cpus_have_final_cap(ARM64_HAS_NESTED_VIRT))
1463 clear_bit(KVM_ARM_VCPU_HAS_EL2, &features);
1464
1465 return features;
1466 }
1467
1468 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
1469 const struct kvm_vcpu_init *init)
1470 {
1471 unsigned long features = init->features[0];
1472 int i;
1473
1474 if (features & ~KVM_VCPU_VALID_FEATURES)
1475 return -ENOENT;
1476
1477 for (i = 1; i < ARRAY_SIZE(init->features); i++) {
1478 if (init->features[i])
1479 return -ENOENT;
1480 }
1481
1482 if (features & ~system_supported_vcpu_features())
1483 return -EINVAL;
1484
1485 /*
1486 * For now make sure that both address/generic pointer authentication
1487 * features are requested by the userspace together.
1488 */
1489 if (test_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, &features) !=
1490 test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
1491 return -EINVAL;
1492
1493 if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
1494 return 0;
1495
1496 /* MTE is incompatible with AArch32 */
1497 if (kvm_has_mte(vcpu->kvm))
1498 return -EINVAL;
1499
1500 /* NV is incompatible with AArch32 */
1501 if (test_bit(KVM_ARM_VCPU_HAS_EL2, &features))
1502 return -EINVAL;
1503
1504 return 0;
1505 }
1506
1507 static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
1508 const struct kvm_vcpu_init *init)
1509 {
1510 unsigned long features = init->features[0];
1511
1512 return !bitmap_equal(vcpu->kvm->arch.vcpu_features, &features,
1513 KVM_VCPU_MAX_FEATURES);
1514 }
1515
1516 static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
1517 {
1518 struct kvm *kvm = vcpu->kvm;
1519 int ret = 0;
1520
1521 /*
1522 * When the vCPU has a PMU, but no PMU is set for the guest
1523 * yet, set the default one.
1524 */
1525 if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
1526 ret = kvm_arm_set_default_pmu(kvm);
1527
1528 /* Prepare for nested if required */
1529 if (!ret && vcpu_has_nv(vcpu))
1530 ret = kvm_vcpu_init_nested(vcpu);
1531
1532 return ret;
1533 }
1534
1535 static int __kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1536 const struct kvm_vcpu_init *init)
1537 {
1538 unsigned long features = init->features[0];
1539 struct kvm *kvm = vcpu->kvm;
1540 int ret = -EINVAL;
1541
1542 mutex_lock(&kvm->arch.config_lock);
1543
1544 if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) &&
1545 kvm_vcpu_init_changed(vcpu, init))
1546 goto out_unlock;
1547
1548 bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES);
1549
1550 ret = kvm_setup_vcpu(vcpu);
1551 if (ret)
1552 goto out_unlock;
1553
1554 /* Now we know what it is, we can reset it. */
1555 kvm_reset_vcpu(vcpu);
1556
1557 set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags);
1558 vcpu_set_flag(vcpu, VCPU_INITIALIZED);
1559 ret = 0;
1560 out_unlock:
1561 mutex_unlock(&kvm->arch.config_lock);
1562 return ret;
1563 }
1564
1565 static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu,
1566 const struct kvm_vcpu_init *init)
1567 {
1568 int ret;
1569
1570 if (init->target != KVM_ARM_TARGET_GENERIC_V8 &&
1571 init->target != kvm_target_cpu())
1572 return -EINVAL;
1573
1574 ret = kvm_vcpu_init_check_features(vcpu, init);
1575 if (ret)
1576 return ret;
1577
1578 if (!kvm_vcpu_initialized(vcpu))
1579 return __kvm_vcpu_set_target(vcpu, init);
1580
1581 if (kvm_vcpu_init_changed(vcpu, init))
1582 return -EINVAL;
1583
1584 kvm_reset_vcpu(vcpu);
1585 return 0;
1586 }
1587
1588 static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
1589 struct kvm_vcpu_init *init)
1590 {
1591 bool power_off = false;
1592 int ret;
1593
1594 /*
1595 * Treat the power-off vCPU feature as ephemeral. Clear the bit to avoid
1596 * reflecting it in the finalized feature set, thus limiting its scope
1597 * to a single KVM_ARM_VCPU_INIT call.
1598 */
1599 if (init->features[0] & BIT(KVM_ARM_VCPU_POWER_OFF)) {
1600 init->features[0] &= ~BIT(KVM_ARM_VCPU_POWER_OFF);
1601 power_off = true;
1602 }
1603
1604 ret = kvm_vcpu_set_target(vcpu, init);
1605 if (ret)
1606 return ret;
1607
1608 /*
1609 * Ensure a rebooted VM will fault in RAM pages and detect if the
1610 * guest MMU is turned off and flush the caches as needed.
1611 *
1612 * S2FWB enforces all memory accesses to RAM being cacheable,
1613 * ensuring that the data side is always coherent. We still
1614 * need to invalidate the I-cache though, as FWB does *not*
1615 * imply CTR_EL0.DIC.
1616 */
1617 if (vcpu_has_run_once(vcpu)) {
1618 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
1619 stage2_unmap_vm(vcpu->kvm);
1620 else
1621 icache_inval_all_pou();
1622 }
1623
1624 vcpu_reset_hcr(vcpu);
1625
1626 /*
1627 * Handle the "start in power-off" case.
1628 */
1629 spin_lock(&vcpu->arch.mp_state_lock);
1630
1631 if (power_off)
1632 __kvm_arm_vcpu_power_off(vcpu);
1633 else
1634 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
1635
1636 spin_unlock(&vcpu->arch.mp_state_lock);
1637
1638 return 0;
1639 }
1640
1641 static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu,
1642 struct kvm_device_attr *attr)
1643 {
1644 int ret = -ENXIO;
1645
1646 switch (attr->group) {
1647 default:
1648 ret = kvm_arm_vcpu_arch_set_attr(vcpu, attr);
1649 break;
1650 }
1651
1652 return ret;
1653 }
1654
1655 static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu,
1656 struct kvm_device_attr *attr)
1657 {
1658 int ret = -ENXIO;
1659
1660 switch (attr->group) {
1661 default:
1662 ret = kvm_arm_vcpu_arch_get_attr(vcpu, attr);
1663 break;
1664 }
1665
1666 return ret;
1667 }
1668
1669 static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu,
1670 struct kvm_device_attr *attr)
1671 {
1672 int ret = -ENXIO;
1673
1674 switch (attr->group) {
1675 default:
1676 ret = kvm_arm_vcpu_arch_has_attr(vcpu, attr);
1677 break;
1678 }
1679
1680 return ret;
1681 }
1682
1683 static int kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
1684 struct kvm_vcpu_events *events)
1685 {
1686 memset(events, 0, sizeof(*events));
1687
1688 return __kvm_arm_vcpu_get_events(vcpu, events);
1689 }
1690
1691 static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
1692 struct kvm_vcpu_events *events)
1693 {
1694 int i;
1695
1696 /* check whether the reserved field is zero */
1697 for (i = 0; i < ARRAY_SIZE(events->reserved); i++)
1698 if (events->reserved[i])
1699 return -EINVAL;
1700
1701 /* check whether the pad field is zero */
1702 for (i = 0; i < ARRAY_SIZE(events->exception.pad); i++)
1703 if (events->exception.pad[i])
1704 return -EINVAL;
1705
1706 return __kvm_arm_vcpu_set_events(vcpu, events);
1707 }
1708
1709 long kvm_arch_vcpu_ioctl(struct file *filp,
1710 unsigned int ioctl, unsigned long arg)
1711 {
1712 struct kvm_vcpu *vcpu = filp->private_data;
1713 void __user *argp = (void __user *)arg;
1714 struct kvm_device_attr attr;
1715 long r;
1716
1717 switch (ioctl) {
1718 case KVM_ARM_VCPU_INIT: {
1719 struct kvm_vcpu_init init;
1720
1721 r = -EFAULT;
1722 if (copy_from_user(&init, argp, sizeof(init)))
1723 break;
1724
1725 r = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init);
1726 break;
1727 }
1728 case KVM_SET_ONE_REG:
1729 case KVM_GET_ONE_REG: {
1730 struct kvm_one_reg reg;
1731
1732 r = -ENOEXEC;
1733 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1734 break;
1735
1736 r = -EFAULT;
1737 if (copy_from_user(®, argp, sizeof(reg)))
1738 break;
1739
1740 /*
1741 * We could owe a reset due to PSCI. Handle the pending reset
1742 * here to ensure userspace register accesses are ordered after
1743 * the reset.
1744 */
1745 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu))
1746 kvm_reset_vcpu(vcpu);
1747
1748 if (ioctl == KVM_SET_ONE_REG)
1749 r = kvm_arm_set_reg(vcpu, ®);
1750 else
1751 r = kvm_arm_get_reg(vcpu, ®);
1752 break;
1753 }
1754 case KVM_GET_REG_LIST: {
1755 struct kvm_reg_list __user *user_list = argp;
1756 struct kvm_reg_list reg_list;
1757 unsigned n;
1758
1759 r = -ENOEXEC;
1760 if (unlikely(!kvm_vcpu_initialized(vcpu)))
1761 break;
1762
1763 r = -EPERM;
1764 if (!kvm_arm_vcpu_is_finalized(vcpu))
1765 break;
1766
1767 r = -EFAULT;
1768 if (copy_from_user(®_list, user_list, sizeof(reg_list)))
1769 break;
1770 n = reg_list.n;
1771 reg_list.n = kvm_arm_num_regs(vcpu);
1772 if (copy_to_user(user_list, ®_list, sizeof(reg_list)))
1773 break;
1774 r = -E2BIG;
1775 if (n < reg_list.n)
1776 break;
1777 r = kvm_arm_copy_reg_indices(vcpu, user_list->reg);
1778 break;
1779 }
1780 case KVM_SET_DEVICE_ATTR: {
1781 r = -EFAULT;
1782 if (copy_from_user(&attr, argp, sizeof(attr)))
1783 break;
1784 r = kvm_arm_vcpu_set_attr(vcpu, &attr);
1785 break;
1786 }
1787 case KVM_GET_DEVICE_ATTR: {
1788 r = -EFAULT;
1789 if (copy_from_user(&attr, argp, sizeof(attr)))
1790 break;
1791 r = kvm_arm_vcpu_get_attr(vcpu, &attr);
1792 break;
1793 }
1794 case KVM_HAS_DEVICE_ATTR: {
1795 r = -EFAULT;
1796 if (copy_from_user(&attr, argp, sizeof(attr)))
1797 break;
1798 r = kvm_arm_vcpu_has_attr(vcpu, &attr);
1799 break;
1800 }
1801 case KVM_GET_VCPU_EVENTS: {
1802 struct kvm_vcpu_events events;
1803
1804 if (!kvm_vcpu_initialized(vcpu))
1805 return -ENOEXEC;
1806
1807 if (kvm_arm_vcpu_get_events(vcpu, &events))
1808 return -EINVAL;
1809
1810 if (copy_to_user(argp, &events, sizeof(events)))
1811 return -EFAULT;
1812
1813 return 0;
1814 }
1815 case KVM_SET_VCPU_EVENTS: {
1816 struct kvm_vcpu_events events;
1817
1818 if (!kvm_vcpu_initialized(vcpu))
1819 return -ENOEXEC;
1820
1821 if (copy_from_user(&events, argp, sizeof(events)))
1822 return -EFAULT;
1823
1824 return kvm_arm_vcpu_set_events(vcpu, &events);
1825 }
1826 case KVM_ARM_VCPU_FINALIZE: {
1827 int what;
1828
1829 if (!kvm_vcpu_initialized(vcpu))
1830 return -ENOEXEC;
1831
1832 if (get_user(what, (const int __user *)argp))
1833 return -EFAULT;
1834
1835 return kvm_arm_vcpu_finalize(vcpu, what);
1836 }
1837 default:
1838 r = -EINVAL;
1839 }
1840
1841 return r;
1842 }
1843
1844 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
1845 {
1846
1847 }
1848
1849 static int kvm_vm_ioctl_set_device_addr(struct kvm *kvm,
1850 struct kvm_arm_device_addr *dev_addr)
1851 {
1852 switch (FIELD_GET(KVM_ARM_DEVICE_ID_MASK, dev_addr->id)) {
1853 case KVM_ARM_DEVICE_VGIC_V2:
1854 if (!vgic_present)
1855 return -ENXIO;
1856 return kvm_set_legacy_vgic_v2_addr(kvm, dev_addr);
1857 default:
1858 return -ENODEV;
1859 }
1860 }
1861
1862 static int kvm_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1863 {
1864 switch (attr->group) {
1865 case KVM_ARM_VM_SMCCC_CTRL:
1866 return kvm_vm_smccc_has_attr(kvm, attr);
1867 default:
1868 return -ENXIO;
1869 }
1870 }
1871
1872 static int kvm_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1873 {
1874 switch (attr->group) {
1875 case KVM_ARM_VM_SMCCC_CTRL:
1876 return kvm_vm_smccc_set_attr(kvm, attr);
1877 default:
1878 return -ENXIO;
1879 }
1880 }
1881
1882 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
1883 {
1884 struct kvm *kvm = filp->private_data;
1885 void __user *argp = (void __user *)arg;
1886 struct kvm_device_attr attr;
1887
1888 switch (ioctl) {
1889 case KVM_CREATE_IRQCHIP: {
1890 int ret;
1891 if (!vgic_present)
1892 return -ENXIO;
1893 mutex_lock(&kvm->lock);
1894 ret = kvm_vgic_create(kvm, KVM_DEV_TYPE_ARM_VGIC_V2);
1895 mutex_unlock(&kvm->lock);
1896 return ret;
1897 }
1898 case KVM_ARM_SET_DEVICE_ADDR: {
1899 struct kvm_arm_device_addr dev_addr;
1900
1901 if (copy_from_user(&dev_addr, argp, sizeof(dev_addr)))
1902 return -EFAULT;
1903 return kvm_vm_ioctl_set_device_addr(kvm, &dev_addr);
1904 }
1905 case KVM_ARM_PREFERRED_TARGET: {
1906 struct kvm_vcpu_init init = {
1907 .target = KVM_ARM_TARGET_GENERIC_V8,
1908 };
1909
1910 if (copy_to_user(argp, &init, sizeof(init)))
1911 return -EFAULT;
1912
1913 return 0;
1914 }
1915 case KVM_ARM_MTE_COPY_TAGS: {
1916 struct kvm_arm_copy_mte_tags copy_tags;
1917
1918 if (copy_from_user(©_tags, argp, sizeof(copy_tags)))
1919 return -EFAULT;
1920 return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags);
1921 }
1922 case KVM_ARM_SET_COUNTER_OFFSET: {
1923 struct kvm_arm_counter_offset offset;
1924
1925 if (copy_from_user(&offset, argp, sizeof(offset)))
1926 return -EFAULT;
1927 return kvm_vm_ioctl_set_counter_offset(kvm, &offset);
1928 }
1929 case KVM_HAS_DEVICE_ATTR: {
1930 if (copy_from_user(&attr, argp, sizeof(attr)))
1931 return -EFAULT;
1932
1933 return kvm_vm_has_attr(kvm, &attr);
1934 }
1935 case KVM_SET_DEVICE_ATTR: {
1936 if (copy_from_user(&attr, argp, sizeof(attr)))
1937 return -EFAULT;
1938
1939 return kvm_vm_set_attr(kvm, &attr);
1940 }
1941 case KVM_ARM_GET_REG_WRITABLE_MASKS: {
1942 struct reg_mask_range range;
1943
1944 if (copy_from_user(&range, argp, sizeof(range)))
1945 return -EFAULT;
1946 return kvm_vm_ioctl_get_reg_writable_masks(kvm, &range);
1947 }
1948 default:
1949 return -EINVAL;
1950 }
1951 }
1952
1953 static unsigned long nvhe_percpu_size(void)
1954 {
1955 return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
1956 (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_start);
1957 }
1958
1959 static unsigned long nvhe_percpu_order(void)
1960 {
1961 unsigned long size = nvhe_percpu_size();
1962
1963 return size ? get_order(size) : 0;
1964 }
1965
1966 static size_t pkvm_host_sve_state_order(void)
1967 {
1968 return get_order(pkvm_host_sve_state_size());
1969 }
1970
1971 /* A lookup table holding the hypervisor VA for each vector slot */
1972 static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
1973
1974 static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
1975 {
1976 hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
1977 }
1978
1979 static int kvm_init_vector_slots(void)
1980 {
1981 int err;
1982 void *base;
1983
1984 base = kern_hyp_va(kvm_ksym_ref(__kvm_hyp_vector));
1985 kvm_init_vector_slot(base, HYP_VECTOR_DIRECT);
1986
1987 base = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
1988 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_DIRECT);
1989
1990 if (kvm_system_needs_idmapped_vectors() &&
1991 !is_protected_kvm_enabled()) {
1992 err = create_hyp_exec_mappings(__pa_symbol(__bp_harden_hyp_vecs),
1993 __BP_HARDEN_HYP_VECS_SZ, &base);
1994 if (err)
1995 return err;
1996 }
1997
1998 kvm_init_vector_slot(base, HYP_VECTOR_INDIRECT);
1999 kvm_init_vector_slot(base, HYP_VECTOR_SPECTRE_INDIRECT);
2000 return 0;
2001 }
2002
2003 static void __init cpu_prepare_hyp_mode(int cpu, u32 hyp_va_bits)
2004 {
2005 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2006 unsigned long tcr;
2007
2008 /*
2009 * Calculate the raw per-cpu offset without a translation from the
2010 * kernel's mapping to the linear mapping, and store it in tpidr_el2
2011 * so that we can use adr_l to access per-cpu variables in EL2.
2012 * Also drop the KASAN tag which gets in the way...
2013 */
2014 params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
2015 (unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
2016
2017 params->mair_el2 = read_sysreg(mair_el1);
2018
2019 tcr = read_sysreg(tcr_el1);
2020 if (cpus_have_final_cap(ARM64_KVM_HVHE)) {
2021 tcr &= ~(TCR_HD | TCR_HA | TCR_A1 | TCR_T0SZ_MASK);
2022 tcr |= TCR_EPD1_MASK;
2023 } else {
2024 unsigned long ips = FIELD_GET(TCR_IPS_MASK, tcr);
2025
2026 tcr &= TCR_EL2_MASK;
2027 tcr |= TCR_EL2_RES1 | FIELD_PREP(TCR_EL2_PS_MASK, ips);
2028 if (lpa2_is_enabled())
2029 tcr |= TCR_EL2_DS;
2030 }
2031 tcr |= TCR_T0SZ(hyp_va_bits);
2032 params->tcr_el2 = tcr;
2033
2034 params->pgd_pa = kvm_mmu_get_httbr();
2035 if (is_protected_kvm_enabled())
2036 params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
2037 else
2038 params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
2039 if (cpus_have_final_cap(ARM64_KVM_HVHE))
2040 params->hcr_el2 |= HCR_E2H;
2041 params->vttbr = params->vtcr = 0;
2042
2043 /*
2044 * Flush the init params from the data cache because the struct will
2045 * be read while the MMU is off.
2046 */
2047 kvm_flush_dcache_to_poc(params, sizeof(*params));
2048 }
2049
2050 static void hyp_install_host_vector(void)
2051 {
2052 struct kvm_nvhe_init_params *params;
2053 struct arm_smccc_res res;
2054
2055 /* Switch from the HYP stub to our own HYP init vector */
2056 __hyp_set_vectors(kvm_get_idmap_vector());
2057
2058 /*
2059 * Call initialization code, and switch to the full blown HYP code.
2060 * If the cpucaps haven't been finalized yet, something has gone very
2061 * wrong, and hyp will crash and burn when it uses any
2062 * cpus_have_*_cap() wrapper.
2063 */
2064 BUG_ON(!system_capabilities_finalized());
2065 params = this_cpu_ptr_nvhe_sym(kvm_init_params);
2066 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
2067 WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
2068 }
2069
2070 static void cpu_init_hyp_mode(void)
2071 {
2072 hyp_install_host_vector();
2073
2074 /*
2075 * Disabling SSBD on a non-VHE system requires us to enable SSBS
2076 * at EL2.
2077 */
2078 if (this_cpu_has_cap(ARM64_SSBS) &&
2079 arm64_get_spectre_v4_state() == SPECTRE_VULNERABLE) {
2080 kvm_call_hyp_nvhe(__kvm_enable_ssbs);
2081 }
2082 }
2083
2084 static void cpu_hyp_reset(void)
2085 {
2086 if (!is_kernel_in_hyp_mode())
2087 __hyp_reset_vectors();
2088 }
2089
2090 /*
2091 * EL2 vectors can be mapped and rerouted in a number of ways,
2092 * depending on the kernel configuration and CPU present:
2093 *
2094 * - If the CPU is affected by Spectre-v2, the hardening sequence is
2095 * placed in one of the vector slots, which is executed before jumping
2096 * to the real vectors.
2097 *
2098 * - If the CPU also has the ARM64_SPECTRE_V3A cap, the slot
2099 * containing the hardening sequence is mapped next to the idmap page,
2100 * and executed before jumping to the real vectors.
2101 *
2102 * - If the CPU only has the ARM64_SPECTRE_V3A cap, then an
2103 * empty slot is selected, mapped next to the idmap page, and
2104 * executed before jumping to the real vectors.
2105 *
2106 * Note that ARM64_SPECTRE_V3A is somewhat incompatible with
2107 * VHE, as we don't have hypervisor-specific mappings. If the system
2108 * is VHE and yet selects this capability, it will be ignored.
2109 */
2110 static void cpu_set_hyp_vector(void)
2111 {
2112 struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
2113 void *vector = hyp_spectre_vector_selector[data->slot];
2114
2115 if (!is_protected_kvm_enabled())
2116 *this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
2117 else
2118 kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
2119 }
2120
2121 static void cpu_hyp_init_context(void)
2122 {
2123 kvm_init_host_cpu_context(host_data_ptr(host_ctxt));
2124 kvm_init_host_debug_data();
2125
2126 if (!is_kernel_in_hyp_mode())
2127 cpu_init_hyp_mode();
2128 }
2129
2130 static void cpu_hyp_init_features(void)
2131 {
2132 cpu_set_hyp_vector();
2133
2134 if (is_kernel_in_hyp_mode()) {
2135 kvm_timer_init_vhe();
2136 kvm_debug_init_vhe();
2137 }
2138
2139 if (vgic_present)
2140 kvm_vgic_init_cpu_hardware();
2141 }
2142
2143 static void cpu_hyp_reinit(void)
2144 {
2145 cpu_hyp_reset();
2146 cpu_hyp_init_context();
2147 cpu_hyp_init_features();
2148 }
2149
2150 static void cpu_hyp_init(void *discard)
2151 {
2152 if (!__this_cpu_read(kvm_hyp_initialized)) {
2153 cpu_hyp_reinit();
2154 __this_cpu_write(kvm_hyp_initialized, 1);
2155 }
2156 }
2157
2158 static void cpu_hyp_uninit(void *discard)
2159 {
2160 if (!is_protected_kvm_enabled() && __this_cpu_read(kvm_hyp_initialized)) {
2161 cpu_hyp_reset();
2162 __this_cpu_write(kvm_hyp_initialized, 0);
2163 }
2164 }
2165
2166 int kvm_arch_enable_virtualization_cpu(void)
2167 {
2168 /*
2169 * Most calls to this function are made with migration
2170 * disabled, but not with preemption disabled. The former is
2171 * enough to ensure correctness, but most of the helpers
2172 * expect the later and will throw a tantrum otherwise.
2173 */
2174 preempt_disable();
2175
2176 cpu_hyp_init(NULL);
2177
2178 kvm_vgic_cpu_up();
2179 kvm_timer_cpu_up();
2180
2181 preempt_enable();
2182
2183 return 0;
2184 }
2185
2186 void kvm_arch_disable_virtualization_cpu(void)
2187 {
2188 kvm_timer_cpu_down();
2189 kvm_vgic_cpu_down();
2190
2191 if (!is_protected_kvm_enabled())
2192 cpu_hyp_uninit(NULL);
2193 }
2194
2195 #ifdef CONFIG_CPU_PM
2196 static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
2197 unsigned long cmd,
2198 void *v)
2199 {
2200 /*
2201 * kvm_hyp_initialized is left with its old value over
2202 * PM_ENTER->PM_EXIT. It is used to indicate PM_EXIT should
2203 * re-enable hyp.
2204 */
2205 switch (cmd) {
2206 case CPU_PM_ENTER:
2207 if (__this_cpu_read(kvm_hyp_initialized))
2208 /*
2209 * don't update kvm_hyp_initialized here
2210 * so that the hyp will be re-enabled
2211 * when we resume. See below.
2212 */
2213 cpu_hyp_reset();
2214
2215 return NOTIFY_OK;
2216 case CPU_PM_ENTER_FAILED:
2217 case CPU_PM_EXIT:
2218 if (__this_cpu_read(kvm_hyp_initialized))
2219 /* The hyp was enabled before suspend. */
2220 cpu_hyp_reinit();
2221
2222 return NOTIFY_OK;
2223
2224 default:
2225 return NOTIFY_DONE;
2226 }
2227 }
2228
2229 static struct notifier_block hyp_init_cpu_pm_nb = {
2230 .notifier_call = hyp_init_cpu_pm_notifier,
2231 };
2232
2233 static void __init hyp_cpu_pm_init(void)
2234 {
2235 if (!is_protected_kvm_enabled())
2236 cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
2237 }
2238 static void __init hyp_cpu_pm_exit(void)
2239 {
2240 if (!is_protected_kvm_enabled())
2241 cpu_pm_unregister_notifier(&hyp_init_cpu_pm_nb);
2242 }
2243 #else
2244 static inline void __init hyp_cpu_pm_init(void)
2245 {
2246 }
2247 static inline void __init hyp_cpu_pm_exit(void)
2248 {
2249 }
2250 #endif
2251
2252 static void __init init_cpu_logical_map(void)
2253 {
2254 unsigned int cpu;
2255
2256 /*
2257 * Copy the MPIDR <-> logical CPU ID mapping to hyp.
2258 * Only copy the set of online CPUs whose features have been checked
2259 * against the finalized system capabilities. The hypervisor will not
2260 * allow any other CPUs from the `possible` set to boot.
2261 */
2262 for_each_online_cpu(cpu)
2263 hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
2264 }
2265
2266 #define init_psci_0_1_impl_state(config, what) \
2267 config.psci_0_1_ ## what ## _implemented = psci_ops.what
2268
2269 static bool __init init_psci_relay(void)
2270 {
2271 /*
2272 * If PSCI has not been initialized, protected KVM cannot install
2273 * itself on newly booted CPUs.
2274 */
2275 if (!psci_ops.get_version) {
2276 kvm_err("Cannot initialize protected mode without PSCI\n");
2277 return false;
2278 }
2279
2280 kvm_host_psci_config.version = psci_ops.get_version();
2281 kvm_host_psci_config.smccc_version = arm_smccc_get_version();
2282
2283 if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
2284 kvm_host_psci_config.function_ids_0_1 = get_psci_0_1_function_ids();
2285 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_suspend);
2286 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_on);
2287 init_psci_0_1_impl_state(kvm_host_psci_config, cpu_off);
2288 init_psci_0_1_impl_state(kvm_host_psci_config, migrate);
2289 }
2290 return true;
2291 }
2292
2293 static int __init init_subsystems(void)
2294 {
2295 int err = 0;
2296
2297 /*
2298 * Enable hardware so that subsystem initialisation can access EL2.
2299 */
2300 on_each_cpu(cpu_hyp_init, NULL, 1);
2301
2302 /*
2303 * Register CPU lower-power notifier
2304 */
2305 hyp_cpu_pm_init();
2306
2307 /*
2308 * Init HYP view of VGIC
2309 */
2310 err = kvm_vgic_hyp_init();
2311 switch (err) {
2312 case 0:
2313 vgic_present = true;
2314 break;
2315 case -ENODEV:
2316 case -ENXIO:
2317 /*
2318 * No VGIC? No pKVM for you.
2319 *
2320 * Protected mode assumes that VGICv3 is present, so no point
2321 * in trying to hobble along if vgic initialization fails.
2322 */
2323 if (is_protected_kvm_enabled())
2324 goto out;
2325
2326 /*
2327 * Otherwise, userspace could choose to implement a GIC for its
2328 * guest on non-cooperative hardware.
2329 */
2330 vgic_present = false;
2331 err = 0;
2332 break;
2333 default:
2334 goto out;
2335 }
2336
2337 if (kvm_mode == KVM_MODE_NV &&
2338 !(vgic_present && (kvm_vgic_global_state.type == VGIC_V3 ||
2339 kvm_vgic_global_state.has_gcie_v3_compat))) {
2340 kvm_err("NV support requires GICv3 or GICv5 with legacy support, giving up\n");
2341 err = -EINVAL;
2342 goto out;
2343 }
2344
2345 /*
2346 * Init HYP architected timer support
2347 */
2348 err = kvm_timer_hyp_init(vgic_present);
2349 if (err)
2350 goto out;
2351
2352 kvm_register_perf_callbacks(NULL);
2353
2354 out:
2355 if (err)
2356 hyp_cpu_pm_exit();
2357
2358 if (err || !is_protected_kvm_enabled())
2359 on_each_cpu(cpu_hyp_uninit, NULL, 1);
2360
2361 return err;
2362 }
2363
2364 static void __init teardown_subsystems(void)
2365 {
2366 kvm_unregister_perf_callbacks();
2367 hyp_cpu_pm_exit();
2368 }
2369
2370 static void __init teardown_hyp_mode(void)
2371 {
2372 bool free_sve = system_supports_sve() && is_protected_kvm_enabled();
2373 int cpu;
2374
2375 free_hyp_pgds();
2376 for_each_possible_cpu(cpu) {
2377 if (per_cpu(kvm_hyp_initialized, cpu))
2378 continue;
2379
2380 free_pages(per_cpu(kvm_arm_hyp_stack_base, cpu), NVHE_STACK_SHIFT - PAGE_SHIFT);
2381
2382 if (!kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu])
2383 continue;
2384
2385 if (free_sve) {
2386 struct cpu_sve_state *sve_state;
2387
2388 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2389 free_pages((unsigned long) sve_state, pkvm_host_sve_state_order());
2390 }
2391
2392 free_pages(kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu], nvhe_percpu_order());
2393
2394 }
2395 }
2396
2397 static int __init do_pkvm_init(u32 hyp_va_bits)
2398 {
2399 void *per_cpu_base = kvm_ksym_ref(kvm_nvhe_sym(kvm_arm_hyp_percpu_base));
2400 int ret;
2401
2402 preempt_disable();
2403 cpu_hyp_init_context();
2404 ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
2405 num_possible_cpus(), kern_hyp_va(per_cpu_base),
2406 hyp_va_bits);
2407 cpu_hyp_init_features();
2408
2409 /*
2410 * The stub hypercalls are now disabled, so set our local flag to
2411 * prevent a later re-init attempt in kvm_arch_enable_virtualization_cpu().
2412 */
2413 __this_cpu_write(kvm_hyp_initialized, 1);
2414 preempt_enable();
2415
2416 return ret;
2417 }
2418
2419 static u64 get_hyp_id_aa64pfr0_el1(void)
2420 {
2421 /*
2422 * Track whether the system isn't affected by spectre/meltdown in the
2423 * hypervisor's view of id_aa64pfr0_el1, used for protected VMs.
2424 * Although this is per-CPU, we make it global for simplicity, e.g., not
2425 * to have to worry about vcpu migration.
2426 *
2427 * Unlike for non-protected VMs, userspace cannot override this for
2428 * protected VMs.
2429 */
2430 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
2431
2432 val &= ~(ID_AA64PFR0_EL1_CSV2 |
2433 ID_AA64PFR0_EL1_CSV3);
2434
2435 val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV2,
2436 arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED);
2437 val |= FIELD_PREP(ID_AA64PFR0_EL1_CSV3,
2438 arm64_get_meltdown_state() == SPECTRE_UNAFFECTED);
2439
2440 return val;
2441 }
2442
2443 static void kvm_hyp_init_symbols(void)
2444 {
2445 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val) = get_hyp_id_aa64pfr0_el1();
2446 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
2447 kvm_nvhe_sym(id_aa64isar0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR0_EL1);
2448 kvm_nvhe_sym(id_aa64isar1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2449 kvm_nvhe_sym(id_aa64isar2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
2450 kvm_nvhe_sym(id_aa64mmfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
2451 kvm_nvhe_sym(id_aa64mmfr1_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
2452 kvm_nvhe_sym(id_aa64mmfr2_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64MMFR2_EL1);
2453 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val) = read_sanitised_ftr_reg(SYS_ID_AA64SMFR0_EL1);
2454 kvm_nvhe_sym(__icache_flags) = __icache_flags;
2455 kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
2456
2457 /* Propagate the FGT state to the the nVHE side */
2458 kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks;
2459 kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks;
2460 kvm_nvhe_sym(hfgitr_masks) = hfgitr_masks;
2461 kvm_nvhe_sym(hdfgrtr_masks) = hdfgrtr_masks;
2462 kvm_nvhe_sym(hdfgwtr_masks) = hdfgwtr_masks;
2463 kvm_nvhe_sym(hafgrtr_masks) = hafgrtr_masks;
2464 kvm_nvhe_sym(hfgrtr2_masks) = hfgrtr2_masks;
2465 kvm_nvhe_sym(hfgwtr2_masks) = hfgwtr2_masks;
2466 kvm_nvhe_sym(hfgitr2_masks) = hfgitr2_masks;
2467 kvm_nvhe_sym(hdfgrtr2_masks)= hdfgrtr2_masks;
2468 kvm_nvhe_sym(hdfgwtr2_masks)= hdfgwtr2_masks;
2469
2470 /*
2471 * Flush entire BSS since part of its data containing init symbols is read
2472 * while the MMU is off.
2473 */
2474 kvm_flush_dcache_to_poc(kvm_ksym_ref(__hyp_bss_start),
2475 kvm_ksym_ref(__hyp_bss_end) - kvm_ksym_ref(__hyp_bss_start));
2476 }
2477
2478 static int __init kvm_hyp_init_protection(u32 hyp_va_bits)
2479 {
2480 void *addr = phys_to_virt(hyp_mem_base);
2481 int ret;
2482
2483 ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
2484 if (ret)
2485 return ret;
2486
2487 ret = do_pkvm_init(hyp_va_bits);
2488 if (ret)
2489 return ret;
2490
2491 free_hyp_pgds();
2492
2493 return 0;
2494 }
2495
2496 static int init_pkvm_host_sve_state(void)
2497 {
2498 int cpu;
2499
2500 if (!system_supports_sve())
2501 return 0;
2502
2503 /* Allocate pages for host sve state in protected mode. */
2504 for_each_possible_cpu(cpu) {
2505 struct page *page = alloc_pages(GFP_KERNEL, pkvm_host_sve_state_order());
2506
2507 if (!page)
2508 return -ENOMEM;
2509
2510 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state = page_address(page);
2511 }
2512
2513 /*
2514 * Don't map the pages in hyp since these are only used in protected
2515 * mode, which will (re)create its own mapping when initialized.
2516 */
2517
2518 return 0;
2519 }
2520
2521 /*
2522 * Finalizes the initialization of hyp mode, once everything else is initialized
2523 * and the initialziation process cannot fail.
2524 */
2525 static void finalize_init_hyp_mode(void)
2526 {
2527 int cpu;
2528
2529 if (system_supports_sve() && is_protected_kvm_enabled()) {
2530 for_each_possible_cpu(cpu) {
2531 struct cpu_sve_state *sve_state;
2532
2533 sve_state = per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state;
2534 per_cpu_ptr_nvhe_sym(kvm_host_data, cpu)->sve_state =
2535 kern_hyp_va(sve_state);
2536 }
2537 }
2538 }
2539
2540 static void pkvm_hyp_init_ptrauth(void)
2541 {
2542 struct kvm_cpu_context *hyp_ctxt;
2543 int cpu;
2544
2545 for_each_possible_cpu(cpu) {
2546 hyp_ctxt = per_cpu_ptr_nvhe_sym(kvm_hyp_ctxt, cpu);
2547 hyp_ctxt->sys_regs[APIAKEYLO_EL1] = get_random_long();
2548 hyp_ctxt->sys_regs[APIAKEYHI_EL1] = get_random_long();
2549 hyp_ctxt->sys_regs[APIBKEYLO_EL1] = get_random_long();
2550 hyp_ctxt->sys_regs[APIBKEYHI_EL1] = get_random_long();
2551 hyp_ctxt->sys_regs[APDAKEYLO_EL1] = get_random_long();
2552 hyp_ctxt->sys_regs[APDAKEYHI_EL1] = get_random_long();
2553 hyp_ctxt->sys_regs[APDBKEYLO_EL1] = get_random_long();
2554 hyp_ctxt->sys_regs[APDBKEYHI_EL1] = get_random_long();
2555 hyp_ctxt->sys_regs[APGAKEYLO_EL1] = get_random_long();
2556 hyp_ctxt->sys_regs[APGAKEYHI_EL1] = get_random_long();
2557 }
2558 }
2559
2560 /* Inits Hyp-mode on all online CPUs */
2561 static int __init init_hyp_mode(void)
2562 {
2563 u32 hyp_va_bits;
2564 int cpu;
2565 int err = -ENOMEM;
2566
2567 /*
2568 * The protected Hyp-mode cannot be initialized if the memory pool
2569 * allocation has failed.
2570 */
2571 if (is_protected_kvm_enabled() && !hyp_mem_base)
2572 goto out_err;
2573
2574 /*
2575 * Allocate Hyp PGD and setup Hyp identity mapping
2576 */
2577 err = kvm_mmu_init(&hyp_va_bits);
2578 if (err)
2579 goto out_err;
2580
2581 /*
2582 * Allocate stack pages for Hypervisor-mode
2583 */
2584 for_each_possible_cpu(cpu) {
2585 unsigned long stack_base;
2586
2587 stack_base = __get_free_pages(GFP_KERNEL, NVHE_STACK_SHIFT - PAGE_SHIFT);
2588 if (!stack_base) {
2589 err = -ENOMEM;
2590 goto out_err;
2591 }
2592
2593 per_cpu(kvm_arm_hyp_stack_base, cpu) = stack_base;
2594 }
2595
2596 /*
2597 * Allocate and initialize pages for Hypervisor-mode percpu regions.
2598 */
2599 for_each_possible_cpu(cpu) {
2600 struct page *page;
2601 void *page_addr;
2602
2603 page = alloc_pages(GFP_KERNEL, nvhe_percpu_order());
2604 if (!page) {
2605 err = -ENOMEM;
2606 goto out_err;
2607 }
2608
2609 page_addr = page_address(page);
2610 memcpy(page_addr, CHOOSE_NVHE_SYM(__per_cpu_start), nvhe_percpu_size());
2611 kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu] = (unsigned long)page_addr;
2612 }
2613
2614 /*
2615 * Map the Hyp-code called directly from the host
2616 */
2617 err = create_hyp_mappings(kvm_ksym_ref(__hyp_text_start),
2618 kvm_ksym_ref(__hyp_text_end), PAGE_HYP_EXEC);
2619 if (err) {
2620 kvm_err("Cannot map world-switch code\n");
2621 goto out_err;
2622 }
2623
2624 err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start),
2625 kvm_ksym_ref(__hyp_data_end), PAGE_HYP);
2626 if (err) {
2627 kvm_err("Cannot map .hyp.data section\n");
2628 goto out_err;
2629 }
2630
2631 err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
2632 kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
2633 if (err) {
2634 kvm_err("Cannot map .hyp.rodata section\n");
2635 goto out_err;
2636 }
2637
2638 err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
2639 kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
2640 if (err) {
2641 kvm_err("Cannot map rodata section\n");
2642 goto out_err;
2643 }
2644
2645 /*
2646 * .hyp.bss is guaranteed to be placed at the beginning of the .bss
2647 * section thanks to an assertion in the linker script. Map it RW and
2648 * the rest of .bss RO.
2649 */
2650 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
2651 kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
2652 if (err) {
2653 kvm_err("Cannot map hyp bss section: %d\n", err);
2654 goto out_err;
2655 }
2656
2657 err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
2658 kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
2659 if (err) {
2660 kvm_err("Cannot map bss section\n");
2661 goto out_err;
2662 }
2663
2664 /*
2665 * Map the Hyp stack pages
2666 */
2667 for_each_possible_cpu(cpu) {
2668 struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
2669 char *stack_base = (char *)per_cpu(kvm_arm_hyp_stack_base, cpu);
2670
2671 err = create_hyp_stack(__pa(stack_base), ¶ms->stack_hyp_va);
2672 if (err) {
2673 kvm_err("Cannot map hyp stack\n");
2674 goto out_err;
2675 }
2676
2677 /*
2678 * Save the stack PA in nvhe_init_params. This will be needed
2679 * to recreate the stack mapping in protected nVHE mode.
2680 * __hyp_pa() won't do the right thing there, since the stack
2681 * has been mapped in the flexible private VA space.
2682 */
2683 params->stack_pa = __pa(stack_base);
2684 }
2685
2686 for_each_possible_cpu(cpu) {
2687 char *percpu_begin = (char *)kvm_nvhe_sym(kvm_arm_hyp_percpu_base)[cpu];
2688 char *percpu_end = percpu_begin + nvhe_percpu_size();
2689
2690 /* Map Hyp percpu pages */
2691 err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
2692 if (err) {
2693 kvm_err("Cannot map hyp percpu region\n");
2694 goto out_err;
2695 }
2696
2697 /* Prepare the CPU initialization parameters */
2698 cpu_prepare_hyp_mode(cpu, hyp_va_bits);
2699 }
2700
2701 kvm_hyp_init_symbols();
2702
2703 if (is_protected_kvm_enabled()) {
2704 if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL) &&
2705 cpus_have_final_cap(ARM64_HAS_ADDRESS_AUTH))
2706 pkvm_hyp_init_ptrauth();
2707
2708 init_cpu_logical_map();
2709
2710 if (!init_psci_relay()) {
2711 err = -ENODEV;
2712 goto out_err;
2713 }
2714
2715 err = init_pkvm_host_sve_state();
2716 if (err)
2717 goto out_err;
2718
2719 err = kvm_hyp_init_protection(hyp_va_bits);
2720 if (err) {
2721 kvm_err("Failed to init hyp memory protection\n");
2722 goto out_err;
2723 }
2724 }
2725
2726 return 0;
2727
2728 out_err:
2729 teardown_hyp_mode();
2730 kvm_err("error initializing Hyp mode: %d\n", err);
2731 return err;
2732 }
2733
2734 struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr)
2735 {
2736 struct kvm_vcpu *vcpu = NULL;
2737 struct kvm_mpidr_data *data;
2738 unsigned long i;
2739
2740 mpidr &= MPIDR_HWID_BITMASK;
2741
2742 rcu_read_lock();
2743 data = rcu_dereference(kvm->arch.mpidr_data);
2744
2745 if (data) {
2746 u16 idx = kvm_mpidr_index(data, mpidr);
2747
2748 vcpu = kvm_get_vcpu(kvm, data->cmpidr_to_idx[idx]);
2749 if (mpidr != kvm_vcpu_get_mpidr_aff(vcpu))
2750 vcpu = NULL;
2751 }
2752
2753 rcu_read_unlock();
2754
2755 if (vcpu)
2756 return vcpu;
2757
2758 kvm_for_each_vcpu(i, vcpu, kvm) {
2759 if (mpidr == kvm_vcpu_get_mpidr_aff(vcpu))
2760 return vcpu;
2761 }
2762 return NULL;
2763 }
2764
2765 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2766 {
2767 return irqchip_in_kernel(kvm);
2768 }
2769
2770 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
2771 struct irq_bypass_producer *prod)
2772 {
2773 struct kvm_kernel_irqfd *irqfd =
2774 container_of(cons, struct kvm_kernel_irqfd, consumer);
2775 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2776
2777 /*
2778 * The only thing we have a chance of directly-injecting is LPIs. Maybe
2779 * one day...
2780 */
2781 if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2782 return 0;
2783
2784 return kvm_vgic_v4_set_forwarding(irqfd->kvm, prod->irq,
2785 &irqfd->irq_entry);
2786 }
2787
2788 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
2789 struct irq_bypass_producer *prod)
2790 {
2791 struct kvm_kernel_irqfd *irqfd =
2792 container_of(cons, struct kvm_kernel_irqfd, consumer);
2793 struct kvm_kernel_irq_routing_entry *irq_entry = &irqfd->irq_entry;
2794
2795 if (irq_entry->type != KVM_IRQ_ROUTING_MSI)
2796 return;
2797
2798 kvm_vgic_v4_unset_forwarding(irqfd->kvm, prod->irq);
2799 }
2800
2801 void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd,
2802 struct kvm_kernel_irq_routing_entry *old,
2803 struct kvm_kernel_irq_routing_entry *new)
2804 {
2805 if (old->type == KVM_IRQ_ROUTING_MSI &&
2806 new->type == KVM_IRQ_ROUTING_MSI &&
2807 !memcmp(&old->msi, &new->msi, sizeof(new->msi)))
2808 return;
2809
2810 /*
2811 * Remapping the vLPI requires taking the its_lock mutex to resolve
2812 * the new translation. We're in spinlock land at this point, so no
2813 * chance of resolving the translation.
2814 *
2815 * Unmap the vLPI and fall back to software LPI injection.
2816 */
2817 return kvm_vgic_v4_unset_forwarding(irqfd->kvm, irqfd->producer->irq);
2818 }
2819
2820 void kvm_arch_irq_bypass_stop(struct irq_bypass_consumer *cons)
2821 {
2822 struct kvm_kernel_irqfd *irqfd =
2823 container_of(cons, struct kvm_kernel_irqfd, consumer);
2824
2825 kvm_arm_halt_guest(irqfd->kvm);
2826 }
2827
2828 void kvm_arch_irq_bypass_start(struct irq_bypass_consumer *cons)
2829 {
2830 struct kvm_kernel_irqfd *irqfd =
2831 container_of(cons, struct kvm_kernel_irqfd, consumer);
2832
2833 kvm_arm_resume_guest(irqfd->kvm);
2834 }
2835
2836 /* Initialize Hyp-mode and memory mappings on all CPUs */
2837 static __init int kvm_arm_init(void)
2838 {
2839 int err;
2840 bool in_hyp_mode;
2841
2842 if (!is_hyp_mode_available()) {
2843 kvm_info("HYP mode not available\n");
2844 return -ENODEV;
2845 }
2846
2847 if (kvm_get_mode() == KVM_MODE_NONE) {
2848 kvm_info("KVM disabled from command line\n");
2849 return -ENODEV;
2850 }
2851
2852 err = kvm_sys_reg_table_init();
2853 if (err) {
2854 kvm_info("Error initializing system register tables");
2855 return err;
2856 }
2857
2858 in_hyp_mode = is_kernel_in_hyp_mode();
2859
2860 if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
2861 cpus_have_final_cap(ARM64_WORKAROUND_1508412))
2862 kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
2863 "Only trusted guests should be used on this system.\n");
2864
2865 err = kvm_set_ipa_limit();
2866 if (err)
2867 return err;
2868
2869 err = kvm_arm_init_sve();
2870 if (err)
2871 return err;
2872
2873 err = kvm_arm_vmid_alloc_init();
2874 if (err) {
2875 kvm_err("Failed to initialize VMID allocator.\n");
2876 return err;
2877 }
2878
2879 if (!in_hyp_mode) {
2880 err = init_hyp_mode();
2881 if (err)
2882 goto out_err;
2883 }
2884
2885 err = kvm_init_vector_slots();
2886 if (err) {
2887 kvm_err("Cannot initialise vector slots\n");
2888 goto out_hyp;
2889 }
2890
2891 err = init_subsystems();
2892 if (err)
2893 goto out_hyp;
2894
2895 kvm_info("%s%sVHE%s mode initialized successfully\n",
2896 in_hyp_mode ? "" : (is_protected_kvm_enabled() ?
2897 "Protected " : "Hyp "),
2898 in_hyp_mode ? "" : (cpus_have_final_cap(ARM64_KVM_HVHE) ?
2899 "h" : "n"),
2900 cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) ? "+NV2": "");
2901
2902 /*
2903 * FIXME: Do something reasonable if kvm_init() fails after pKVM
2904 * hypervisor protection is finalized.
2905 */
2906 err = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
2907 if (err)
2908 goto out_subs;
2909
2910 /*
2911 * This should be called after initialization is done and failure isn't
2912 * possible anymore.
2913 */
2914 if (!in_hyp_mode)
2915 finalize_init_hyp_mode();
2916
2917 kvm_arm_initialised = true;
2918
2919 return 0;
2920
2921 out_subs:
2922 teardown_subsystems();
2923 out_hyp:
2924 if (!in_hyp_mode)
2925 teardown_hyp_mode();
2926 out_err:
2927 kvm_arm_vmid_alloc_free();
2928 return err;
2929 }
2930
2931 static int __init early_kvm_mode_cfg(char *arg)
2932 {
2933 if (!arg)
2934 return -EINVAL;
2935
2936 if (strcmp(arg, "none") == 0) {
2937 kvm_mode = KVM_MODE_NONE;
2938 return 0;
2939 }
2940
2941 if (!is_hyp_mode_available()) {
2942 pr_warn_once("KVM is not available. Ignoring kvm-arm.mode\n");
2943 return 0;
2944 }
2945
2946 if (strcmp(arg, "protected") == 0) {
2947 if (!is_kernel_in_hyp_mode())
2948 kvm_mode = KVM_MODE_PROTECTED;
2949 else
2950 pr_warn_once("Protected KVM not available with VHE\n");
2951
2952 return 0;
2953 }
2954
2955 if (strcmp(arg, "nvhe") == 0 && !WARN_ON(is_kernel_in_hyp_mode())) {
2956 kvm_mode = KVM_MODE_DEFAULT;
2957 return 0;
2958 }
2959
2960 if (strcmp(arg, "nested") == 0 && !WARN_ON(!is_kernel_in_hyp_mode())) {
2961 kvm_mode = KVM_MODE_NV;
2962 return 0;
2963 }
2964
2965 return -EINVAL;
2966 }
2967 early_param("kvm-arm.mode", early_kvm_mode_cfg);
2968
2969 static int __init early_kvm_wfx_trap_policy_cfg(char *arg, enum kvm_wfx_trap_policy *p)
2970 {
2971 if (!arg)
2972 return -EINVAL;
2973
2974 if (strcmp(arg, "trap") == 0) {
2975 *p = KVM_WFX_TRAP;
2976 return 0;
2977 }
2978
2979 if (strcmp(arg, "notrap") == 0) {
2980 *p = KVM_WFX_NOTRAP;
2981 return 0;
2982 }
2983
2984 return -EINVAL;
2985 }
2986
2987 static int __init early_kvm_wfi_trap_policy_cfg(char *arg)
2988 {
2989 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfi_trap_policy);
2990 }
2991 early_param("kvm-arm.wfi_trap_policy", early_kvm_wfi_trap_policy_cfg);
2992
2993 static int __init early_kvm_wfe_trap_policy_cfg(char *arg)
2994 {
2995 return early_kvm_wfx_trap_policy_cfg(arg, &kvm_wfe_trap_policy);
2996 }
2997 early_param("kvm-arm.wfe_trap_policy", early_kvm_wfe_trap_policy_cfg);
2998
2999 enum kvm_mode kvm_get_mode(void)
3000 {
3001 return kvm_mode;
3002 }
3003
3004 module_init(kvm_arm_init);