diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index bec69219b989..86adf6778108 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -21,6 +21,7 @@ int pkvm_vm_ioctl_enable_cap(struct kvm *kvm,struct kvm_enable_cap *cap); int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); int pkvm_create_hyp_vm(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); +void pkvm_host_reclaim_page(struct kvm *host_kvm, phys_addr_t ipa); /* * Definitions for features to be allowed or restricted for guest virtual diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 38e5e9b259fc..81a2c9c9450a 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -72,6 +72,8 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu); int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu); int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa); int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa); +int __pkvm_guest_relinquish_to_host(struct pkvm_hyp_vcpu *vcpu, + u64 ipa, u64 *ppa); bool addr_is_memory(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index eea4c60ffb75..c880d6605453 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -112,6 +112,7 @@ int kvm_check_pvm_sysreg_table(void); void pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code); +bool kvm_hyp_handle_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code); struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *vm, u64 mpidr); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 877dbb21da3a..2c9ac32daf11 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -92,6 +92,8 @@ static void handle_pvm_entry_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) case ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID: fallthrough; case ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID: + fallthrough; + case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID: vcpu_set_reg(&hyp_vcpu->vcpu, 0, SMCCC_RET_SUCCESS); break; default: @@ -260,6 +262,8 @@ static void handle_pvm_exit_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) case ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID: fallthrough; case ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID: + fallthrough; + case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID: n = 4; break; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 5ac5ad879419..2e934988adb2 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -265,6 +265,7 @@ static int reclaim_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, { kvm_pte_t pte = *ptep; struct hyp_page *page; + u64 *pa = arg; if (!kvm_pte_valid(pte)) return 0; @@ -276,6 +277,8 @@ static int reclaim_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, fallthrough; case PKVM_PAGE_SHARED_BORROWED: case PKVM_PAGE_SHARED_OWNED: + if (pa) + *pa = kvm_pte_to_phys(pte); page->flags |= HOST_PAGE_PENDING_RECLAIM; break; default: @@ -315,6 +318,36 @@ void reclaim_guest_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc) } } +int __pkvm_guest_relinquish_to_host(struct pkvm_hyp_vcpu *vcpu, + u64 ipa, u64 *ppa) +{ + struct kvm_pgtable_walker walker = { + .cb = reclaim_walker, + .arg = ppa, + .flags = KVM_PGTABLE_WALK_LEAF + }; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + int ret; + + host_lock_component(); + guest_lock_component(vm); + + /* Set default pa value to "not found". */ + *ppa = 0; + + /* If ipa is mapped: sets page flags, and gets the pa. */ + ret = kvm_pgtable_walk(&vm->pgt, ipa, PAGE_SIZE, &walker); + + /* Zap the guest stage2 pte. */ + if (!ret) + kvm_pgtable_stage2_unmap(&vm->pgt, ipa, PAGE_SIZE); + + guest_unlock_component(vm); + host_unlock_component(); + + return ret; +} + int __pkvm_prot_finalize(void) { struct kvm_s2_mmu *mmu = &host_mmu.arch.mmu; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 4754ca282161..f2728e556174 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -1277,6 +1277,54 @@ out_guest_err: return true; } +static bool pkvm_meminfo_call(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 arg1 = smccc_get_arg1(vcpu); + u64 arg2 = smccc_get_arg2(vcpu); + u64 arg3 = smccc_get_arg3(vcpu); + + if (arg1 || arg2 || arg3) + goto out_guest_err; + + smccc_set_retval(vcpu, PAGE_SIZE, 0, 0, 0); + return true; + +out_guest_err: + smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0); + return true; +} + +static bool pkvm_memrelinquish_call(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 ipa = smccc_get_arg1(vcpu); + u64 arg2 = smccc_get_arg2(vcpu); + u64 arg3 = smccc_get_arg3(vcpu); + u64 pa = 0; + int ret; + + if (arg2 || arg3) + goto out_guest_err; + + ret = __pkvm_guest_relinquish_to_host(hyp_vcpu, ipa, &pa); + if (ret) + goto out_guest_err; + + if (pa != 0) { + /* Now pass to host. */ + return false; + } + + /* This was a NOP as no page was actually mapped at the IPA. */ + smccc_set_retval(vcpu, 0, 0, 0, 0); + return true; + +out_guest_err: + smccc_set_retval(vcpu, SMCCC_RET_INVALID_PARAMETER, 0, 0, 0); + return true; +} + bool smccc_trng_available; static bool pkvm_forward_trng(struct kvm_vcpu *vcpu) @@ -1336,20 +1384,16 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_SHARE); val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_UNSHARE); + val[0] |= BIT(ARM_SMCCC_KVM_FUNC_MEM_RELINQUISH); break; case ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID: - if (smccc_get_arg1(vcpu) || - smccc_get_arg2(vcpu) || - smccc_get_arg3(vcpu)) { - val[0] = SMCCC_RET_INVALID_PARAMETER; - } else { - val[0] = PAGE_SIZE; - } - break; + return pkvm_meminfo_call(hyp_vcpu); case ARM_SMCCC_VENDOR_HYP_KVM_MEM_SHARE_FUNC_ID: return pkvm_memshare_call(hyp_vcpu, exit_code); case ARM_SMCCC_VENDOR_HYP_KVM_MEM_UNSHARE_FUNC_ID: return pkvm_memunshare_call(hyp_vcpu); + case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID: + return pkvm_memrelinquish_call(hyp_vcpu); case ARM_SMCCC_TRNG_VERSION ... ARM_SMCCC_TRNG_RND32: case ARM_SMCCC_TRNG_RND64: if (smccc_trng_available) @@ -1362,3 +1406,26 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]); return true; } + +/* + * Handler for non-protected VM HVC calls. + * + * Returns true if the hypervisor has handled the exit, and control should go + * back to the guest, or false if it hasn't. + */ +bool kvm_hyp_handle_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + u32 fn = smccc_get_function(vcpu); + struct pkvm_hyp_vcpu *hyp_vcpu; + + hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); + + switch (fn) { + case ARM_SMCCC_VENDOR_HYP_KVM_HYP_MEMINFO_FUNC_ID: + return pkvm_meminfo_call(hyp_vcpu); + case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID: + return pkvm_memrelinquish_call(hyp_vcpu); + } + + return false; +} diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 4b904252e1db..310aaf493909 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -181,6 +181,7 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) static const exit_handler_fn hyp_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] = NULL, [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, + [ESR_ELx_EC_HVC64] = kvm_hyp_handle_hvc64, [ESR_ELx_EC_SYS64] = kvm_hyp_handle_sysreg, [ESR_ELx_EC_SVE] = kvm_hyp_handle_fpsimd, [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index c9f401fa01a9..2a92b9bd2a46 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -13,8 +14,15 @@ GENMASK(KVM_REG_ARM_STD_BMAP_BIT_COUNT - 1, 0) #define KVM_ARM_SMCCC_STD_HYP_FEATURES \ GENMASK(KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT - 1, 0) -#define KVM_ARM_SMCCC_VENDOR_HYP_FEATURES \ - GENMASK(KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT - 1, 0) +#define KVM_ARM_SMCCC_VENDOR_HYP_FEATURES ({ \ + unsigned long f; \ + f = GENMASK(KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT - 1, 0); \ + if (is_protected_kvm_enabled()) { \ + f |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); \ + f |= BIT(ARM_SMCCC_KVM_FUNC_MEM_RELINQUISH); \ + } \ + f; \ +}) static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) { @@ -116,6 +124,9 @@ static bool kvm_hvc_call_allowed(struct kvm_vcpu *vcpu, u32 func_id) case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_PTP, &smccc_feat->vendor_hyp_bmap); + case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID: + return test_bit(ARM_SMCCC_KVM_FUNC_MEM_RELINQUISH, + &smccc_feat->vendor_hyp_bmap); default: return kvm_hvc_call_default_allowed(func_id); } @@ -213,6 +224,10 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: kvm_ptp_get_time(vcpu, val); break; + case ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID: + pkvm_host_reclaim_page(vcpu->kvm, smccc_get_arg1(vcpu)); + val[0] = SMCCC_RET_SUCCESS; + break; case ARM_SMCCC_TRNG_VERSION: case ARM_SMCCC_TRNG_FEATURES: case ARM_SMCCC_TRNG_GET_UUID: diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 1ef33dc9b0da..ea2ea5b61424 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -259,6 +259,41 @@ int pkvm_init_host_vm(struct kvm *host_kvm, unsigned long type) return 0; } +static int rb_ppage_cmp(const void *key, const struct rb_node *node) +{ + struct kvm_pinned_page *p = container_of(node, struct kvm_pinned_page, node); + phys_addr_t ipa = (phys_addr_t)key; + + return (ipa < p->ipa) ? -1 : (ipa > p->ipa); +} + +void pkvm_host_reclaim_page(struct kvm *host_kvm, phys_addr_t ipa) +{ + struct kvm_pinned_page *ppage; + struct mm_struct *mm = current->mm; + struct rb_node *node; + + write_lock(&host_kvm->mmu_lock); + node = rb_find((void *)ipa, &host_kvm->arch.pkvm.pinned_pages, + rb_ppage_cmp); + if (node) + rb_erase(node, &host_kvm->arch.pkvm.pinned_pages); + write_unlock(&host_kvm->mmu_lock); + + WARN_ON(!node); + if (!node) + return; + + ppage = container_of(node, struct kvm_pinned_page, node); + + WARN_ON(kvm_call_hyp_nvhe(__pkvm_host_reclaim_page, + page_to_pfn(ppage->page))); + + account_locked_vm(mm, 1, false); + unpin_user_pages_dirty_lock(&ppage->page, 1, true); + kfree(ppage); +} + static int __init pkvm_firmware_rmem_err(struct reserved_mem *rmem, const char *reason) { diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index 25c576a910df..8224bed759ca 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -115,6 +115,7 @@ #define ARM_SMCCC_KVM_FUNC_HYP_MEMINFO 2 #define ARM_SMCCC_KVM_FUNC_MEM_SHARE 3 #define ARM_SMCCC_KVM_FUNC_MEM_UNSHARE 4 +#define ARM_SMCCC_KVM_FUNC_MEM_RELINQUISH 9 #define ARM_SMCCC_KVM_FUNC_FEATURES_2 127 #define ARM_SMCCC_KVM_NUM_FUNCS 128 @@ -155,6 +156,12 @@ ARM_SMCCC_OWNER_VENDOR_HYP, \ ARM_SMCCC_KVM_FUNC_MEM_UNSHARE) +#define ARM_SMCCC_VENDOR_HYP_KVM_MEM_RELINQUISH_FUNC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_VENDOR_HYP, \ + ARM_SMCCC_KVM_FUNC_MEM_RELINQUISH) + /* ptp_kvm counter type ID */ #define KVM_PTP_VIRT_COUNTER 0 #define KVM_PTP_PHYS_COUNTER 1