diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 3bfb5c916d09..c73b4ce1abce 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -75,8 +75,6 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, __KVM_HOST_SMCCC_FUNC___pkvm_init_shadow, __KVM_HOST_SMCCC_FUNC___pkvm_teardown_shadow, - __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, - __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index b858ef02ced1..b65a2d3dcc23 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -450,22 +450,10 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) if (vcpu_has_ptrauth(vcpu)) vcpu_ptrauth_disable(vcpu); kvm_arch_vcpu_load_debug_state_flags(vcpu); - - if (is_protected_kvm_enabled()) { - kvm_call_hyp_nvhe(__pkvm_vcpu_load, vcpu); - kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, - &vcpu->arch.vgic_cpu.vgic_v3); - } } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) { - if (is_protected_kvm_enabled()) { - kvm_call_hyp(__vgic_v3_save_vmcr_aprs, - &vcpu->arch.vgic_cpu.vgic_v3); - kvm_call_hyp_nvhe(__pkvm_vcpu_put, vcpu); - } - kvm_arch_vcpu_put_debug_state_flags(vcpu); kvm_arch_vcpu_put_fp(vcpu); if (has_vhe()) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 6c6a9be69290..13870bc5d4d0 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -275,7 +275,7 @@ static void sync_timer_state(struct kvm_vcpu *shadow_vcpu) __vcpu_sys_reg(shadow_vcpu, CNTV_CTL_EL0) = read_sysreg_el0(SYS_CNTV_CTL); } -static void flush_shadow_state(struct kvm_vcpu *shadow_vcpu) +static bool handle_shadow_entry(struct kvm_vcpu *shadow_vcpu) { struct kvm_vcpu *host_vcpu = shadow_vcpu->arch.pkvm.host_vcpu; u8 esr_ec; @@ -286,8 +286,6 @@ static void flush_shadow_state(struct kvm_vcpu *shadow_vcpu) switch (ARM_EXCEPTION_CODE(shadow_vcpu->arch.pkvm.exit_code)) { case ARM_EXCEPTION_IRQ: - case ARM_EXCEPTION_EL1_SERROR: - case ARM_EXCEPTION_IL: break; case ARM_EXCEPTION_TRAP: esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(shadow_vcpu)); @@ -298,13 +296,13 @@ static void flush_shadow_state(struct kvm_vcpu *shadow_vcpu) break; default: - BUG(); + return false; } - shadow_vcpu->arch.pkvm.exit_code = 0; + return true; } -static void sync_shadow_state(struct kvm_vcpu *shadow_vcpu, u32 exit_reason) +static void handle_shadow_exit(struct kvm_vcpu *shadow_vcpu) { struct kvm_vcpu *host_vcpu = shadow_vcpu->arch.pkvm.host_vcpu; u8 esr_ec; @@ -313,7 +311,7 @@ static void sync_shadow_state(struct kvm_vcpu *shadow_vcpu, u32 exit_reason) sync_vgic_state(host_vcpu, shadow_vcpu); sync_timer_state(shadow_vcpu); - switch (ARM_EXCEPTION_CODE(exit_reason)) { + switch (shadow_vcpu->arch.pkvm.exit_code) { case ARM_EXCEPTION_IRQ: break; case ARM_EXCEPTION_TRAP: @@ -323,83 +321,47 @@ static void sync_shadow_state(struct kvm_vcpu *shadow_vcpu, u32 exit_reason) if (ec_handler) ec_handler(host_vcpu, shadow_vcpu); - break; - case ARM_EXCEPTION_EL1_SERROR: - case ARM_EXCEPTION_IL: break; default: - BUG(); - } - - host_vcpu->arch.flags &= ~(KVM_ARM64_PENDING_EXCEPTION | KVM_ARM64_INCREMENT_PC); - shadow_vcpu->arch.pkvm.exit_code = exit_reason; -} - -struct pkvm_loaded_state { - /* loaded vcpu is HYP VA */ - struct kvm_vcpu *vcpu; - bool is_shadow; -}; - -static DEFINE_PER_CPU(struct pkvm_loaded_state, loaded_state); - -static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) -{ - DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); - struct pkvm_loaded_state *state; - - /* Why did you bother? */ - if (!is_protected_kvm_enabled()) - return; - - state = this_cpu_ptr(&loaded_state); - - /* Nice try */ - if (state->vcpu) - return; - - vcpu = kern_hyp_va(vcpu); - - state->vcpu = hyp_get_shadow_vcpu(vcpu) ?: vcpu; - state->is_shadow = state->vcpu != vcpu; - - if (state->is_shadow) { - /* FIXME: we can't trust the validity of these pointers */ - state->vcpu->arch.host_fpsimd_state = vcpu->arch.host_fpsimd_state; - - /* Propagate WFx trapping flags, trap ptrauth */ - state->vcpu->arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI | - HCR_API | HCR_APK); - state->vcpu->arch.hcr_el2 |= vcpu->arch.hcr_el2 & (HCR_TWE | HCR_TWI); + break; } } -static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) +static struct kvm_vcpu *get_shadow_vcpu(struct kvm_vcpu *host_vcpu) { - struct pkvm_loaded_state *state = this_cpu_ptr(&loaded_state); + struct kvm_vcpu *shadow_vcpu; - /* "It's over and done with..." */ - state->vcpu = NULL; + host_vcpu = kern_hyp_va(host_vcpu); + shadow_vcpu = hyp_get_shadow_vcpu(host_vcpu); + + if (shadow_vcpu) { + if (!handle_shadow_entry(shadow_vcpu)) + return NULL; + + shadow_vcpu->arch.pkvm.exit_code = 0; + return shadow_vcpu; + } + + return host_vcpu; +} + +static void put_shadow_vcpu(struct kvm_vcpu *shadow_vcpu, int exit_code) +{ + shadow_vcpu->arch.pkvm.exit_code = exit_code; + handle_shadow_exit(shadow_vcpu); } static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); + struct kvm_vcpu *shadow_vcpu; int ret; - if (unlikely(is_protected_kvm_enabled())) { - struct pkvm_loaded_state *state = this_cpu_ptr(&loaded_state); + shadow_vcpu = get_shadow_vcpu(vcpu); + ret = __kvm_vcpu_run(shadow_vcpu); - if (state->is_shadow) - flush_shadow_state(state->vcpu); - - ret = __kvm_vcpu_run(state->vcpu); - - if (state->is_shadow) - sync_shadow_state(state->vcpu, ret); - } else { - ret = __kvm_vcpu_run(kern_hyp_va(vcpu)); - } + if (shadow_vcpu != kern_hyp_va(vcpu)) + put_shadow_vcpu(shadow_vcpu, ret); cpu_reg(host_ctxt, 1) = ret; } @@ -408,19 +370,13 @@ static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); - vcpu = kern_hyp_va(vcpu); - - if (unlikely(is_protected_kvm_enabled())) { - struct pkvm_loaded_state *state = this_cpu_ptr(&loaded_state); - - /* - * A shadow vcpu can never be updated from EL1, and we - * must have a vcpu loaded when protected mode is - * enabled. - */ - if (!state->vcpu || state->is_shadow || state->vcpu != vcpu) - return; - } + /* + * This get_shadow_vcpu() shouldn't exist, as we would never + * commit a pending update before returning to userspace, and + * this is an actual attack vector (it leaves EL1 in full + * control of PC). + */ + vcpu = get_shadow_vcpu(vcpu); __kvm_adjust_pc(vcpu); } @@ -484,29 +440,13 @@ static void handle___kvm_get_mdcr_el2(struct kvm_cpu_context *host_ctxt) static struct vgic_v3_cpu_if *get_shadow_vgic_v3_cpu_if(struct vgic_v3_cpu_if *cpu_if) { - if (unlikely(is_protected_kvm_enabled())) { - struct pkvm_loaded_state *state = this_cpu_ptr(&loaded_state); - struct kvm_vcpu *host_vcpu; + struct kvm_vcpu *vcpu, *shadow_vcpu; - if (!state->vcpu) - return NULL; - - if (state->is_shadow) { - host_vcpu = state->vcpu->arch.pkvm.host_vcpu; - - if (&host_vcpu->arch.vgic_cpu.vgic_v3 != cpu_if) - return NULL; - - return &state->vcpu->arch.vgic_cpu.vgic_v3; - } else { - if (&state->vcpu->arch.vgic_cpu.vgic_v3 != cpu_if) - return NULL; - - return cpu_if; - } - } - - return cpu_if; + vcpu = container_of(cpu_if, struct kvm_vcpu, arch.vgic_cpu.vgic_v3); + shadow_vcpu = hyp_get_shadow_vcpu(vcpu); + if (!shadow_vcpu) + return cpu_if; + return &shadow_vcpu->arch.vgic_cpu.vgic_v3; } static void handle___vgic_v3_save_vmcr_aprs(struct kvm_cpu_context *host_ctxt) @@ -653,8 +593,6 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), HANDLE_FUNC(__pkvm_init_shadow), HANDLE_FUNC(__pkvm_teardown_shadow), - HANDLE_FUNC(__pkvm_vcpu_load), - HANDLE_FUNC(__pkvm_vcpu_put), }; static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)