diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index c9d357028fd9..65e09a589eaa 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -75,7 +75,7 @@ static inline bool pkvm_hyp_vcpu_is_protected(struct pkvm_hyp_vcpu *hyp_vcpu) void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva); + unsigned long pgd_hva, unsigned long last_ran_hva); int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, unsigned long vcpu_hva); int __pkvm_teardown_vm(pkvm_handle_t handle); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 167c9ad98e19..76a96814e47b 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -144,6 +144,7 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) DECLARE_REG(unsigned int, vcpu_idx, host_ctxt, 2); DECLARE_REG(u64, hcr_el2, host_ctxt, 3); struct pkvm_hyp_vcpu *hyp_vcpu; + int *last_ran; if (!is_protected_kvm_enabled()) return; @@ -152,6 +153,17 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) if (!hyp_vcpu) return; + /* + * Guarantee that both TLBs and I-cache are private to each vcpu. If a + * vcpu from the same VM has previously run on the same physical CPU, + * nuke the relevant contexts. + */ + last_ran = &hyp_vcpu->vcpu.arch.hw_mmu->last_vcpu_ran[hyp_smp_processor_id()]; + if (*last_ran != hyp_vcpu->vcpu.vcpu_id) { + __kvm_flush_cpu_context(hyp_vcpu->vcpu.arch.hw_mmu); + *last_ran = hyp_vcpu->vcpu.vcpu_id; + } + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { /* Propagate WFx trapping flags, trap ptrauth */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI | @@ -435,9 +447,11 @@ static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2); DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3); + DECLARE_REG(unsigned long, last_ran_hva, host_ctxt, 4); host_kvm = kern_hyp_va(host_kvm); - cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva); + cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva, + last_ran_hva); } static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 3ce85f9caf92..12d1f1a1f48c 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -310,12 +310,19 @@ static void unpin_host_vcpus(struct pkvm_hyp_vcpu *hyp_vcpus[], unpin_host_vcpu(hyp_vcpus[i]->host_vcpu); } +static size_t pkvm_get_last_ran_size(void) +{ + return array_size(hyp_nr_cpus, sizeof(int)); +} + static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, - unsigned int nr_vcpus) + int *last_ran, unsigned int nr_vcpus) { hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; hyp_vm->kvm.arch.vtcr = host_mmu.arch.vtcr; + hyp_vm->kvm.arch.mmu.last_vcpu_ran = last_ran; + memset(hyp_vm->kvm.arch.mmu.last_vcpu_ran, -1, pkvm_get_last_ran_size()); } static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, @@ -471,15 +478,17 @@ static void unmap_donated_memory_noclear(void *va, size_t size) * pgd_hva: The host va of the area being donated for the stage-2 PGD for * the VM. Must be page aligned. Its size is implied by the VM's * VTCR. - * + * last_ran_hva: The host va of the area being donated for hyp to use to track + * the most recent physical cpu on which each vcpu has run. * Return a unique handle to the protected VM on success, * negative error code on failure. */ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva) + unsigned long pgd_hva, unsigned long last_ran_hva) { struct pkvm_hyp_vm *hyp_vm = NULL; - size_t vm_size, pgd_size; + void *last_ran = NULL; + size_t vm_size, pgd_size, last_ran_size; unsigned int nr_vcpus; void *pgd = NULL; int ret; @@ -495,6 +504,7 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, } vm_size = pkvm_get_hyp_vm_size(nr_vcpus); + last_ran_size = pkvm_get_last_ran_size(); pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.vtcr); ret = -ENOMEM; @@ -503,11 +513,15 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, if (!hyp_vm) goto err_remove_mappings; + last_ran = map_donated_memory(last_ran_hva, last_ran_size); + if (!last_ran) + goto err_remove_mappings; + pgd = map_donated_memory_noclear(pgd_hva, pgd_size); if (!pgd) goto err_remove_mappings; - init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus); + init_pkvm_hyp_vm(host_kvm, hyp_vm, last_ran, nr_vcpus); hyp_spin_lock(&vm_table_lock); ret = insert_vm_table_entry(host_kvm, hyp_vm); @@ -527,6 +541,7 @@ err_unlock: hyp_spin_unlock(&vm_table_lock); err_remove_mappings: unmap_donated_memory(hyp_vm, vm_size); + unmap_donated_memory(last_ran, last_ran_size); unmap_donated_memory(pgd, pgd_size); err_unpin_kvm: hyp_unpin_shared_mem(host_kvm, host_kvm + 1); @@ -601,10 +616,10 @@ teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size) int __pkvm_teardown_vm(pkvm_handle_t handle) { + size_t vm_size, last_ran_size; struct kvm_hyp_memcache *mc; struct pkvm_hyp_vm *hyp_vm; unsigned int idx; - size_t vm_size; int err; hyp_spin_lock(&vm_table_lock); @@ -638,6 +653,10 @@ int __pkvm_teardown_vm(pkvm_handle_t handle) teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); } + last_ran_size = pkvm_get_last_ran_size(); + teardown_donated_memory(mc, hyp_vm->kvm.arch.mmu.last_vcpu_ran, + last_ran_size); + vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus); teardown_donated_memory(mc, hyp_vm, vm_size); return 0; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index aea560873028..d97d17c96321 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -109,10 +109,10 @@ void __init kvm_hyp_reserve(void) */ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) { - size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz; + size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz, last_ran_sz; struct kvm_vcpu *host_vcpu; pkvm_handle_t handle; - void *pgd, *hyp_vm; + void *pgd, *hyp_vm, *last_ran; unsigned long idx; int ret; @@ -140,10 +140,18 @@ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) goto free_pgd; } - /* Donate the VM memory to hyp and let hyp initialize it. */ - ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd); - if (ret < 0) + /* Allocate memory to donate to hyp for tracking mmu->last_vcpu_ran. */ + last_ran_sz = PAGE_ALIGN(array_size(num_possible_cpus(), sizeof(int))); + last_ran = alloc_pages_exact(last_ran_sz, GFP_KERNEL_ACCOUNT); + if (!last_ran) { + ret = -ENOMEM; goto free_vm; + } + + /* Donate the VM memory to hyp and let hyp initialize it. */ + ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd, last_ran); + if (ret < 0) + goto free_last_ran; handle = ret; @@ -179,6 +187,8 @@ static int __pkvm_create_hyp_vm(struct kvm *host_kvm) destroy_vm: pkvm_destroy_hyp_vm(host_kvm); return ret; +free_last_ran: + free_pages_exact(last_ran, last_ran_sz); free_vm: free_pages_exact(hyp_vm, hyp_vm_sz); free_pgd: