From c96bd2e36f7dd95af9e2b7ee0b9b25ea990d72d6 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 14 Feb 2023 13:13:27 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Save and restore host sve state in pKVM Non-protected mode relies on the host to restore its SVE state if necessary. However, protected VMs shouldn't reveal any information to the host, including whether they have potentially dirtied the host's sve state. Therefore, save and restore the host's sve state at hyp in protected mode. Currently this behavior applies to protected and non-protected VMs in protected mode. It could be optimised for non-protected VMs by applying the same behavior as non-protected mode, which is to inform the host that it should restore its sve state. But for now it's kept this way to maintain the same behavior for all VMs in protected mode. Signed-off-by: Fuad Tabba Bug: 267291591 Change-Id: Ifbcc64b387c3f821a6c1047e8c843f6250a3f690 --- arch/arm64/include/asm/kvm_hyp.h | 5 +++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 28 ++++++++++++++-------------- arch/arm64/kvm/hyp/nvhe/pkvm.c | 27 +++++++++++++++++++++++++++ arch/arm64/kvm/hyp/nvhe/switch.c | 26 ++++++++++++++++++++++++-- 4 files changed, 70 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 40d93a5daa62..1a55a5b7d3f3 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -118,6 +118,11 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus, void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); #endif +#ifdef __KVM_NVHE_HYPERVISOR__ +struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu); +struct kvm_host_sve_state *get_host_sve_state(struct kvm_vcpu *vcpu); +#endif + extern u64 kvm_nvhe_sym(id_aa64pfr0_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64pfr1_el1_sys_val); extern u64 kvm_nvhe_sym(id_aa64isar0_el1_sys_val); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06c94f8dde96..4fce0949b0f4 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -411,14 +411,6 @@ static const hyp_entry_exit_handler_fn exit_hyp_vm_handlers[] = { [ESR_ELx_EC_DABT_LOW] = handle_vm_exit_abt, }; -static struct user_fpsimd_state *get_host_fpsimd_state(void) -{ - char *state = (char *) host_fp_state + - sizeof(struct user_fpsimd_state) * hyp_smp_processor_id(); - - return (struct user_fpsimd_state *) state; -} - static void flush_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -687,16 +679,25 @@ static void fpsimd_host_restore(void) if (unlikely(is_protected_kvm_enabled())) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - struct user_fpsimd_state *host_fpsimd_state; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; - host_fpsimd_state = get_host_fpsimd_state(); - - if (vcpu_has_sve(&hyp_vcpu->vcpu)) + if (vcpu_has_sve(vcpu)) __hyp_sve_save_guest(hyp_vcpu); else __fpsimd_save_state(&hyp_vcpu->vcpu.arch.ctxt.fp_regs); - __fpsimd_restore_state(host_fpsimd_state); + if (system_supports_sve()) { + struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); + u64 vq_len = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; + + write_sysreg_el1(sve_state->zcr_el1, SYS_ZCR); + sve_cond_update_zcr_vq(vq_len, SYS_ZCR_EL2); + __sve_restore_state(sve_state->sve_regs + + sve_ffr_offset(kvm_host_sve_max_vl), + &sve_state->fpsr); + } else { + __fpsimd_restore_state(get_host_fpsimd_state(vcpu)); + } hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; } @@ -733,7 +734,6 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) *last_ran = hyp_vcpu->vcpu.vcpu_id; } - hyp_vcpu->vcpu.arch.host_fpsimd_state = get_host_fpsimd_state(); hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index d7ab28852994..a6cc23d91d81 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -43,6 +43,33 @@ static DEFINE_PER_CPU(struct pkvm_hyp_vcpu *, loaded_hyp_vcpu); */ void *host_fp_state; +static void *__get_host_fpsimd_bytes(void) +{ + void *state = host_fp_state + + size_mul(pkvm_host_fp_state_size(), hyp_smp_processor_id()); + + if (state < host_fp_state) + return NULL; + + return state; +} + +struct user_fpsimd_state *get_host_fpsimd_state(struct kvm_vcpu *vcpu) +{ + if (likely(!is_protected_kvm_enabled())) + return vcpu->arch.host_fpsimd_state; + + WARN_ON(system_supports_sve()); + return __get_host_fpsimd_bytes(); +} + +struct kvm_host_sve_state *get_host_sve_state(struct kvm_vcpu *vcpu) +{ + WARN_ON(!system_supports_sve()); + WARN_ON(!is_protected_kvm_enabled()); + return __get_host_fpsimd_bytes(); +} + /* * Set trap register values based on features in ID_AA64PFR0. */ diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 553db97cab8e..c98dcd1b05d6 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -112,8 +112,10 @@ static void __deactivate_fpsimd_traps(struct kvm_vcpu *vcpu) { u64 reg = CPTR_EL2_TFP; - if (vcpu_has_sve(vcpu)) + if (vcpu_has_sve(vcpu) || + (is_protected_kvm_enabled() && system_supports_sve())) { reg |= CPTR_EL2_TZ; + } sysreg_clear_set(cptr_el2, reg, 0); } @@ -190,7 +192,27 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) static void kvm_hyp_handle_fpsimd_host(struct kvm_vcpu *vcpu) { - __fpsimd_save_state(vcpu->arch.host_fpsimd_state); + /* + * Non-protected kvm relies on the host restoring its sve state. + * Protected kvm restores the host's sve state as not to reveal that + * fpsimd was used by a guest nor leak upper sve bits. + */ + if (unlikely(is_protected_kvm_enabled() && system_supports_sve())) { + struct kvm_host_sve_state *sve_state = get_host_sve_state(vcpu); + u64 vq_len = sve_vq_from_vl(kvm_host_sve_max_vl) - 1; + + sve_state->zcr_el1 = read_sysreg_el1(SYS_ZCR); + sve_cond_update_zcr_vq(vq_len, SYS_ZCR_EL2); + __sve_save_state(sve_state->sve_regs + + sve_ffr_offset(kvm_host_sve_max_vl), + &sve_state->fpsr); + + /* Still trap SVE since it's handled by hyp in pKVM. */ + if (!vcpu_has_sve(vcpu)) + sysreg_clear_set(cptr_el2, 0, CPTR_EL2_TZ); + } else { + __fpsimd_save_state(get_host_fpsimd_state(vcpu)); + } } static const exit_handler_fn hyp_exit_handlers[] = {