.. | .. |
---|
9 | 9 | #include <linux/sched.h> |
---|
10 | 10 | #include <linux/thread_info.h> |
---|
11 | 11 | #include <linux/kvm_host.h> |
---|
| 12 | +#include <asm/fpsimd.h> |
---|
12 | 13 | #include <asm/kvm_asm.h> |
---|
13 | | -#include <asm/kvm_host.h> |
---|
| 14 | +#include <asm/kvm_hyp.h> |
---|
14 | 15 | #include <asm/kvm_mmu.h> |
---|
15 | 16 | #include <asm/sysreg.h> |
---|
16 | 17 | |
---|
.. | .. |
---|
41 | 42 | ret = create_hyp_mappings(fpsimd, fpsimd + 1, PAGE_HYP); |
---|
42 | 43 | if (ret) |
---|
43 | 44 | goto error; |
---|
| 45 | + |
---|
| 46 | + if (vcpu->arch.sve_state) { |
---|
| 47 | + void *sve_end; |
---|
| 48 | + |
---|
| 49 | + sve_end = vcpu->arch.sve_state + vcpu_sve_state_size(vcpu); |
---|
| 50 | + |
---|
| 51 | + ret = create_hyp_mappings(vcpu->arch.sve_state, sve_end, |
---|
| 52 | + PAGE_HYP); |
---|
| 53 | + if (ret) |
---|
| 54 | + goto error; |
---|
| 55 | + } |
---|
44 | 56 | |
---|
45 | 57 | vcpu->arch.host_thread_info = kern_hyp_va(ti); |
---|
46 | 58 | vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd); |
---|
.. | .. |
---|
85 | 97 | WARN_ON_ONCE(!irqs_disabled()); |
---|
86 | 98 | |
---|
87 | 99 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { |
---|
88 | | - fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.gp_regs.fp_regs); |
---|
| 100 | + fpsimd_bind_state_to_cpu(&vcpu->arch.ctxt.fp_regs, |
---|
| 101 | + vcpu->arch.sve_state, |
---|
| 102 | + vcpu->arch.sve_max_vl); |
---|
| 103 | + |
---|
89 | 104 | clear_thread_flag(TIF_FOREIGN_FPSTATE); |
---|
90 | | - clear_thread_flag(TIF_SVE); |
---|
| 105 | + update_thread_flag(TIF_SVE, vcpu_has_sve(vcpu)); |
---|
91 | 106 | } |
---|
92 | 107 | } |
---|
93 | 108 | |
---|
.. | .. |
---|
100 | 115 | void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu) |
---|
101 | 116 | { |
---|
102 | 117 | unsigned long flags; |
---|
| 118 | + bool host_has_sve = system_supports_sve(); |
---|
| 119 | + bool guest_has_sve = vcpu_has_sve(vcpu); |
---|
103 | 120 | |
---|
104 | 121 | local_irq_save(flags); |
---|
105 | 122 | |
---|
106 | 123 | if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) { |
---|
107 | | - /* Clean guest FP state to memory and invalidate cpu view */ |
---|
108 | | - fpsimd_save(); |
---|
109 | | - fpsimd_flush_cpu_state(); |
---|
110 | | - } else if (system_supports_sve()) { |
---|
| 124 | + if (guest_has_sve) { |
---|
| 125 | + __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); |
---|
| 126 | + |
---|
| 127 | + /* Restore the VL that was saved when bound to the CPU */ |
---|
| 128 | + if (!has_vhe()) |
---|
| 129 | + sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, |
---|
| 130 | + SYS_ZCR_EL1); |
---|
| 131 | + } |
---|
| 132 | + |
---|
| 133 | + fpsimd_save_and_flush_cpu_state(); |
---|
| 134 | + } else if (has_vhe() && host_has_sve) { |
---|
111 | 135 | /* |
---|
112 | 136 | * The FPSIMD/SVE state in the CPU has not been touched, and we |
---|
113 | 137 | * have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been |
---|