From e93b1b4738378ae1fdb9784afd83327044981af6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 26 Apr 2022 09:22:50 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Lazy host FP save/restore Implement lazy save/restore of the host FPSIMD register state at EL2. This allows us to save/restore guest FPSIMD registers without involving the host and means that we can avoid having to repopulate the hyp vCPU register state on every flush. Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon Bug: 233587962 Change-Id: I7e9827d7bf52656df69ece1844fc1b8bd7884175 --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 76 ++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 71b92996f177..cfff5a19af19 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -20,6 +20,14 @@ #include +/* + * Host FPSIMD state. Written to when the guest accesses its own FPSIMD state, + * and read when the guest state is live and we need to switch back to the host. + * + * Only valid when (fp_state == FP_STATE_GUEST_OWNED) in the hyp vCPU structure. + */ +static DEFINE_PER_CPU(struct user_fpsimd_state, loaded_host_fpsimd_state); + DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); @@ -185,12 +193,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.hcr_el2 = host_vcpu->arch.hcr_el2; hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; - hyp_vcpu->vcpu.arch.cptr_el2 = host_vcpu->arch.cptr_el2; - - hyp_vcpu->vcpu.arch.fp_state = host_vcpu->arch.fp_state; hyp_vcpu->vcpu.arch.debug_ptr = kern_hyp_va(host_vcpu->arch.debug_ptr); - hyp_vcpu->vcpu.arch.host_fpsimd_state = host_vcpu->arch.host_fpsimd_state; hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; @@ -224,9 +228,6 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2; - host_vcpu->arch.cptr_el2 = hyp_vcpu->vcpu.arch.cptr_el2; - - host_vcpu->arch.fp_state = hyp_vcpu->vcpu.arch.fp_state; sync_hyp_vgic_state(hyp_vcpu); sync_hyp_timer_state(hyp_vcpu); @@ -251,6 +252,40 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) hyp_vcpu->exit_code = exit_reason; } +static void __hyp_sve_save_guest(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + __sve_save_state(vcpu_sve_pffr(vcpu), &vcpu->arch.ctxt.fp_regs.fpsr); + __vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR); + sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL1); +} + +static void fpsimd_host_restore(void) +{ + sysreg_clear_set(cptr_el2, CPTR_EL2_TZ | CPTR_EL2_TFP, 0); + isb(); + + if (unlikely(is_protected_kvm_enabled())) { + struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + struct user_fpsimd_state *host_fpsimd_state; + + host_fpsimd_state = this_cpu_ptr(&loaded_host_fpsimd_state); + + if (vcpu_has_sve(&hyp_vcpu->vcpu)) + __hyp_sve_save_guest(hyp_vcpu); + else + __fpsimd_save_state(&hyp_vcpu->vcpu.arch.ctxt.fp_regs); + + __fpsimd_restore_state(host_fpsimd_state); + + hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; + } + + if (system_supports_sve()) + sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2); +} + static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); @@ -277,6 +312,9 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) *last_ran = hyp_vcpu->vcpu.vcpu_id; } + hyp_vcpu->vcpu.arch.host_fpsimd_state = this_cpu_ptr(&loaded_host_fpsimd_state); + hyp_vcpu->vcpu.arch.fp_state = FP_STATE_HOST_OWNED; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { /* Propagate WFx trapping flags, trap ptrauth */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI | @@ -296,6 +334,9 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) if (hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) + fpsimd_host_restore(); + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu) && !vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) { __sync_hyp_vcpu(hyp_vcpu); @@ -316,6 +357,9 @@ static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) return; + if (hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) + fpsimd_host_restore(); + __sync_hyp_vcpu(hyp_vcpu); } @@ -362,6 +406,19 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); sync_hyp_vcpu(hyp_vcpu, ret); + + if (hyp_vcpu->vcpu.arch.fp_state == FP_STATE_GUEST_OWNED) { + /* + * The guest has used the FP, trap all accesses + * from the host (both FP and SVE). + */ + u64 reg = CPTR_EL2_TFP; + + if (system_supports_sve()) + reg |= CPTR_EL2_TZ; + + sysreg_clear_set(cptr_el2, 0, reg); + } } else { /* The host is fully trusted, run its vCPU directly. */ ret = __kvm_vcpu_run(host_vcpu); @@ -706,10 +763,9 @@ void handle_trap(struct kvm_cpu_context *host_ctxt) case ESR_ELx_EC_SMC64: handle_host_smc(host_ctxt); break; + case ESR_ELx_EC_FP_ASIMD: case ESR_ELx_EC_SVE: - sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0); - isb(); - sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2); + fpsimd_host_restore(); break; case ESR_ELx_EC_IABT_LOW: case ESR_ELx_EC_DABT_LOW: