diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 40d93a5daa62..1a55a5b7d3f3 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -118,6 +118,11 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus, void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); #endif +#ifdef __KVM_NVHE_HYPERVISOR__ +struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu); +struct kvm_host_sve_state *get_host_sve_state(struct kvm_vcpu *vcpu); +#endif + extern u64 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64isar0_el1_sys_val); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06c94f8dde96..4fce0949b0f4 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -411,14 +411,6 @@ static const hyp_entry_exit_handler_fn exit_hyp_vm_handlers[] = { [ESR_ELx_EC_DABT_LOW] = handle_vm_exit_abt, }; -static struct user_fpsimd_state *get_host_fpsimd_state(void) -{ - char *state = (char *) host_fp_state + - sizeof(struct user_fpsimd_state) * hyp_smp_processor_id(); - - return (struct user_fpsimd_state *) state; -} - static void flush_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -687,16 +679,25 @@ static void fpsimd_host_restore(void) if (unlikely(is_protected_kvm_enabled())) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - struct user_fpsimd_state *host_fpsimd_state; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; - host_fpsimd_state = get_host_fpsimd_state(); - - if (vcpu_has_sve(&hyp_vcpu->vcpu)) + if (vcpu_has_sve(vcpu)) __hyp_sve_save_guest(hyp_vcpu); else __fpsimd_save_state(&hyp_vcpu->vcpu.arch.ctxt.fp_regs); - __fpsimd_restore_state(host_fpsimd_state); + if (system_supports_sve()) { + struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); + u64 vq_len = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; + + write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); + sve_cond_update_zcr_vq(vq_len, SYS_ZCR_EL2); + __sve_restore_state(sve_state->sve_regs + + sve_ffr_offset(kvm_host_sve_max_vl), + &sve_state->fpsr); + } else { + __fpsimd_restore_state(get_host_fpsimd_state(vcpu)); + } hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; } @@ -733,7 +734,6 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) *last_ran = hyp_vcpu->vcpu.vcpu_id; } - hyp_vcpu->vcpu.arch.host_fpsimd_state = get_host_fpsimd_state(); hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index d7ab28852994..a6cc23d91d81 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -43,6 +43,33 @@ static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); */ void *host_fp_state; +static void *__get_host_fpsimd_bytes(void) +{ + void *state = host_fp_state + + size_mul(pkvm_host_fp_state_size(), hyp_smp_processor_id()); + + if (state < host_fp_state) + return NULL; + + return state; +} + +struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu) +{ + if (likely(!is_protected_kvm_enabled())) + return vcpu->arch.host_fpsimd_state; + + WARN_ON(system_supports_sve()); + return __get_host_fpsimd_bytes(); +} + +struct kvm_host_sve_state *get_host_sve_state(struct kvm_vcpu *vcpu) +{ + WARN_ON(!system_supports_sve()); + WARN_ON(!is_protected_kvm_enabled()); + return __get_host_fpsimd_bytes(); +} + /* * Set trap register values based on features in ID_AA64PFR0. */ diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 553db97cab8e..c98dcd1b05d6 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -112,8 +112,10 @@ static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu) { u64 reg = CPTR_EL2_TFP; - if (vcpu_has_sve(vcpu)) + if (vcpu_has_sve(vcpu) || + (is_protected_kvm_enabled() && system_supports_sve())) { reg |= CPTR_EL2_TZ; + } sysreg_clear_set(cptr_el2, reg, 0); } @@ -190,7 +192,27 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) static void kvm_hyp_handle_fpsimd_host(struct kvm_vcpu *vcpu) { - __fpsimd_save_state(vcpu->arch.host_fpsimd_state); + /* + * Non-protected kvm relies on the host restoring its sve state. + * Protected kvm restores the host's sve state as not to reveal that + * fpsimd was used by a guest nor leak upper sve bits. + */ + if (unlikely(is_protected_kvm_enabled() && system_supports_sve())) { + struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); + u64 vq_len = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; + + sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); + sve_cond_update_zcr_vq(vq_len, SYS_ZCR_EL2); + __sve_save_state(sve_state->sve_regs + + sve_ffr_offset(kvm_host_sve_max_vl), + &sve_state->fpsr); + + /* Still trap SVE since it's handled by hyp in pKVM. */ + if (!vcpu_has_sve(vcpu)) + sysreg_clear_set(cptr_el2, 0, CPTR_EL2_TZ); + } else { + __fpsimd_save_state(get_host_fpsimd_state(vcpu)); + } } static const exit_handler_fn hyp_exit_handlers[] = {