diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5dcb0a85ef77..8b2f2598fe96 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -79,6 +79,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_teardown_shadow, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, + __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index eba8c310b784..23083f49c070 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -550,6 +550,7 @@ struct kvm_vcpu_arch { #define KVM_ARM64_DEBUG_STATE_SAVE_SPE (1 << 12) /* Save SPE context if active */ #define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */ #define KVM_ARM64_FP_FOREIGN_FPSTATE (1 << 14) +#define KVM_ARM64_PKVM_STATE_DIRTY (1 << 15) #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ KVM_GUESTDBG_USE_SW_BP | \ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 96940095fa07..ff0a4f9fb09f 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -475,6 +475,28 @@ static void sync_timer_state(struct pkvm_loaded_state *state) __vcpu_sys_reg(shadow_vcpu, CNTV_CTL_EL0) = read_sysreg_el0(SYS_CNTV_CTL); } +static void __sync_vcpu_state(struct kvm_vcpu *from_vcpu, + struct kvm_vcpu *to_vcpu) +{ + int i; + + to_vcpu->arch.ctxt.regs = from_vcpu->arch.ctxt.regs; + to_vcpu->arch.ctxt.spsr_abt = from_vcpu->arch.ctxt.spsr_abt; + to_vcpu->arch.ctxt.spsr_und = from_vcpu->arch.ctxt.spsr_und; + to_vcpu->arch.ctxt.spsr_irq = from_vcpu->arch.ctxt.spsr_irq; + to_vcpu->arch.ctxt.spsr_fiq = from_vcpu->arch.ctxt.spsr_fiq; + + /* + * Copy the sysregs, but don't mess with the timer state which + * is directly handled by EL1 and is expected to be preserved. + */ + for (i = 1; i < NR_SYS_REGS; i++) { + if (i >= CNTVOFF_EL2 && i <= CNTP_CTL_EL0) + continue; + to_vcpu->arch.ctxt.sys_regs[i] = from_vcpu->arch.ctxt.sys_regs[i]; + } +} + static void flush_shadow_state(struct pkvm_loaded_state *state) { struct kvm_vcpu *shadow_vcpu = state->vcpu; @@ -485,6 +507,19 @@ static void flush_shadow_state(struct pkvm_loaded_state *state) if (READ_ONCE(shadow_vcpu->arch.pkvm.power_state) == PSCI_0_2_AFFINITY_LEVEL_ON_PENDING) pkvm_reset_vcpu(shadow_vcpu); + /* + * If we deal with a non-protected guest and that the state is + * dirty (from a host perspective), copy the state back into + * the shadow. + */ + if (!state->is_protected) { + if (READ_ONCE(host_vcpu->arch.flags) & KVM_ARM64_PKVM_STATE_DIRTY) + __sync_vcpu_state(host_vcpu, shadow_vcpu); + + state->vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS & ~(HCR_RW | HCR_TWI | HCR_TWE); + state->vcpu->arch.hcr_el2 |= host_vcpu->arch.hcr_el2; + } + flush_vgic_state(host_vcpu, shadow_vcpu); flush_timer_state(state); @@ -518,6 +553,10 @@ static void sync_shadow_state(struct pkvm_loaded_state *state, u32 exit_reason) u8 esr_ec; shadow_entry_exit_handler_fn ec_handler; + /* + * Don't sync the vcpu GPR/sysreg state after a run. Instead, + * leave it in the shadow until someone actually requires it. + */ sync_vgic_state(host_vcpu, shadow_vcpu); sync_timer_state(state); @@ -616,6 +655,10 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) if (state->vcpu->arch.flags & KVM_ARM64_FP_ENABLED) fpsimd_host_restore(); + if (!state->is_protected && + !(READ_ONCE(vcpu->arch.flags) & KVM_ARM64_PKVM_STATE_DIRTY)) + __sync_vcpu_state(state->vcpu, vcpu); + put_shadow_vcpu(state->vcpu); /* "It's over and done with..." */ @@ -624,6 +667,23 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) } } +static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); + + if (unlikely(is_protected_kvm_enabled())) { + struct pkvm_loaded_state *state = this_cpu_ptr(&loaded_state); + + vcpu = kern_hyp_va(vcpu); + + if (!state->vcpu || state->is_protected || + state->vcpu->arch.pkvm.host_vcpu != vcpu) + return; + + __sync_vcpu_state(state->vcpu, vcpu); + } +} + static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); @@ -932,6 +992,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_teardown_shadow), HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), + HANDLE_FUNC(__pkvm_vcpu_sync_state), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)