diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 4d14b4cfb3b7..38e5e9b259fc 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -70,6 +70,8 @@ int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages); int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages); int __pkvm_host_share_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu); int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu); +int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa); +int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa); bool addr_is_memory(phys_addr_t phys); int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot prot); diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 0130d137173e..85289934cccd 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -779,11 +779,41 @@ static int __host_ack_transition(u64 addr, const struct pkvm_mem_transition *tx, return __host_check_page_state_range(addr, size, state); } +static int host_ack_share(u64 addr, const struct pkvm_mem_transition *tx, + enum kvm_pgtable_prot perms) +{ + if (perms != PKVM_HOST_MEM_PROT) + return -EPERM; + + return __host_ack_transition(addr, tx, PKVM_NOPAGE); +} + static int host_ack_donation(u64 addr, const struct pkvm_mem_transition *tx) { return __host_ack_transition(addr, tx, PKVM_NOPAGE); } +static int host_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx) +{ + return __host_ack_transition(addr, tx, PKVM_PAGE_SHARED_BORROWED); +} + +static int host_complete_share(u64 addr, const struct pkvm_mem_transition *tx, + enum kvm_pgtable_prot perms) +{ + u64 size = tx->nr_pages * PAGE_SIZE; + + return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_BORROWED); +} + +static int host_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx) +{ + u8 owner_id = tx->initiator.id; + u64 size = tx->nr_pages * PAGE_SIZE; + + return host_stage2_set_owner_locked(addr, size, owner_id); +} + static int host_complete_donation(u64 addr, const struct pkvm_mem_transition *tx) { u64 size = tx->nr_pages * PAGE_SIZE; @@ -969,6 +999,120 @@ static int guest_complete_donation(u64 addr, const struct pkvm_mem_transition *t prot, &vcpu->vcpu.arch.pkvm_memcache); } +static int __guest_get_completer_addr(u64 *completer_addr, phys_addr_t phys, + const struct pkvm_mem_transition *tx) +{ + switch (tx->completer.id) { + case PKVM_ID_HOST: + *completer_addr = phys; + break; + case PKVM_ID_HYP: + *completer_addr = (u64)__hyp_va(phys); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int __guest_request_page_transition(u64 *completer_addr, + const struct pkvm_mem_transition *tx, + enum pkvm_page_state desired) +{ + struct pkvm_hyp_vcpu *vcpu = tx->initiator.guest.hyp_vcpu; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + enum pkvm_page_state state; + phys_addr_t phys; + kvm_pte_t pte; + u32 level; + int ret; + + if (tx->nr_pages != 1) + return -E2BIG; + + ret = kvm_pgtable_get_leaf(&vm->pgt, tx->initiator.addr, &pte, &level); + if (ret) + return ret; + + state = guest_get_page_state(pte); + if (state == PKVM_NOPAGE) + return -EFAULT; + + if (state != desired) + return -EPERM; + + /* + * We only deal with page granular mappings in the guest for now as + * the pgtable code relies on being able to recreate page mappings + * lazily after zapping a block mapping, which doesn't work once the + * pages have been donated. + */ + if (level != KVM_PGTABLE_MAX_LEVELS - 1) + return -EINVAL; + + phys = kvm_pte_to_phys(pte); + if (!addr_is_allowed_memory(phys)) + return -EINVAL; + + return __guest_get_completer_addr(completer_addr, phys, tx); +} + +static int guest_request_share(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + return __guest_request_page_transition(completer_addr, tx, + PKVM_PAGE_OWNED); +} + +static int guest_request_unshare(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + return __guest_request_page_transition(completer_addr, tx, + PKVM_PAGE_SHARED_OWNED); +} + +static int __guest_initiate_page_transition(u64 *completer_addr, + const struct pkvm_mem_transition *tx, + enum pkvm_page_state state) +{ + struct pkvm_hyp_vcpu *vcpu = tx->initiator.guest.hyp_vcpu; + struct kvm_hyp_memcache *mc = &vcpu->vcpu.arch.pkvm_memcache; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + u64 size = tx->nr_pages * PAGE_SIZE; + u64 addr = tx->initiator.addr; + enum kvm_pgtable_prot prot; + phys_addr_t phys; + kvm_pte_t pte; + int ret; + + ret = kvm_pgtable_get_leaf(&vm->pgt, addr, &pte, NULL); + if (ret) + return ret; + + phys = kvm_pte_to_phys(pte); + prot = pkvm_mkstate(kvm_pgtable_stage2_pte_prot(pte), state); + ret = kvm_pgtable_stage2_map(&vm->pgt, addr, size, phys, prot, mc); + if (ret) + return ret; + + return __guest_get_completer_addr(completer_addr, phys, tx); +} + +static int guest_initiate_share(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + return __guest_initiate_page_transition(completer_addr, tx, + PKVM_PAGE_SHARED_OWNED); +} + +static int guest_initiate_unshare(u64 *completer_addr, + const struct pkvm_mem_transition *tx) +{ + return __guest_initiate_page_transition(completer_addr, tx, + PKVM_PAGE_OWNED); +} + static int check_share(struct pkvm_mem_share *share) { const struct pkvm_mem_transition *tx = &share->tx; @@ -979,6 +1123,9 @@ static int check_share(struct pkvm_mem_share *share) case PKVM_ID_HOST: ret = host_request_owned_transition(&completer_addr, tx); break; + case PKVM_ID_GUEST: + ret = guest_request_share(&completer_addr, tx); + break; default: ret = -EINVAL; } @@ -987,6 +1134,9 @@ static int check_share(struct pkvm_mem_share *share) return ret; switch (tx->completer.id) { + case PKVM_ID_HOST: + ret = host_ack_share(completer_addr, tx, share->completer_prot); + break; case PKVM_ID_HYP: ret = hyp_ack_share(completer_addr, tx, share->completer_prot); break; @@ -1010,6 +1160,9 @@ static int __do_share(struct pkvm_mem_share *share) case PKVM_ID_HOST: ret = host_initiate_share(&completer_addr, tx); break; + case PKVM_ID_GUEST: + ret = guest_initiate_share(&completer_addr, tx); + break; default: ret = -EINVAL; } @@ -1018,6 +1171,9 @@ static int __do_share(struct pkvm_mem_share *share) return ret; switch (tx->completer.id) { + case PKVM_ID_HOST: + ret = host_complete_share(completer_addr, tx, share->completer_prot); + break; case PKVM_ID_HYP: ret = hyp_complete_share(completer_addr, tx, share->completer_prot); break; @@ -1061,6 +1217,9 @@ static int check_unshare(struct pkvm_mem_share *share) case PKVM_ID_HOST: ret = host_request_unshare(&completer_addr, tx); break; + case PKVM_ID_GUEST: + ret = guest_request_unshare(&completer_addr, tx); + break; default: ret = -EINVAL; } @@ -1069,6 +1228,9 @@ static int check_unshare(struct pkvm_mem_share *share) return ret; switch (tx->completer.id) { + case PKVM_ID_HOST: + ret = host_ack_unshare(completer_addr, tx); + break; case PKVM_ID_HYP: ret = hyp_ack_unshare(completer_addr, tx); break; @@ -1089,6 +1251,9 @@ static int __do_unshare(struct pkvm_mem_share *share) case PKVM_ID_HOST: ret = host_initiate_unshare(&completer_addr, tx); break; + case PKVM_ID_GUEST: + ret = guest_initiate_unshare(&completer_addr, tx); + break; default: ret = -EINVAL; } @@ -1097,6 +1262,9 @@ static int __do_unshare(struct pkvm_mem_share *share) return ret; switch (tx->completer.id) { + case PKVM_ID_HOST: + ret = host_complete_unshare(completer_addr, tx); + break; case PKVM_ID_HYP: ret = hyp_complete_unshare(completer_addr, tx); break; @@ -1254,6 +1422,70 @@ int __pkvm_host_share_hyp(u64 pfn) return ret; } +int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa) +{ + int ret; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + struct pkvm_mem_share share = { + .tx = { + .nr_pages = 1, + .initiator = { + .id = PKVM_ID_GUEST, + .addr = ipa, + .guest = { + .hyp_vcpu = vcpu, + }, + }, + .completer = { + .id = PKVM_ID_HOST, + }, + }, + .completer_prot = PKVM_HOST_MEM_PROT, + }; + + host_lock_component(); + guest_lock_component(vm); + + ret = do_share(&share); + + guest_unlock_component(vm); + host_unlock_component(); + + return ret; +} + +int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa) +{ + int ret; + struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); + struct pkvm_mem_share share = { + .tx = { + .nr_pages = 1, + .initiator = { + .id = PKVM_ID_GUEST, + .addr = ipa, + .guest = { + .hyp_vcpu = vcpu, + }, + }, + .completer = { + .id = PKVM_ID_HOST, + }, + }, + .completer_prot = PKVM_HOST_MEM_PROT, + }; + + host_lock_component(); + guest_lock_component(vm); + + ret = do_unshare(&share); + + guest_unlock_component(vm); + host_unlock_component(); + + return ret; +} + int __pkvm_host_unshare_hyp(u64 pfn) { int ret;