From a0f22aaf985346fca36ce38c58dc7e4b77409884 Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Fri, 4 Mar 2022 16:48:32 +0000 Subject: [PATCH] ANDROID: KVM: arm64: Flag pages needing poisoning in hyp_vmemmap During teardown, we currently walk the guest stage-2 page-table and annotate all of its pages as 'pending poisoning' in the host stage-2. Sadly, this requires a host stage-2 walk for every guest page, which is rather inefficient and can lead to a long non-preemptible amount of time spent at EL2. This gets particularly bad with IOMMUs as, in its current form, the host stage-2 annotation triggers IOMMU updates. To avoid the host stage-2 walks, let's annotate the pages pending poisoning using a flag in the hyp_vmemmap instead. Bug: 219180169 Signed-off-by: Quentin Perret Change-Id: I8894bd8e0b10ea8817763479412b540c0291e8f5 --- arch/arm64/kvm/hyp/include/nvhe/mem_protect.h | 1 - arch/arm64/kvm/hyp/include/nvhe/memory.h | 6 ++++ arch/arm64/kvm/hyp/nvhe/mem_protect.c | 33 +++++-------------- 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 0118a3b6b943..e9eef7c2cebb 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -58,7 +58,6 @@ typedef u32 pkvm_id; static const pkvm_id pkvm_host_id = 0; static const pkvm_id pkvm_hyp_id = (1 << 16); static const pkvm_id pkvm_ffa_id = pkvm_hyp_id + 1; /* Secure world */ -static const pkvm_id pkvm_host_poison = pkvm_ffa_id + 1; extern unsigned long hyp_nr_cpus; diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/include/nvhe/memory.h index 97da521cf332..0e64650e2f10 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/memory.h +++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h @@ -7,6 +7,12 @@ #include +/* + * Accesses to struct hyp_page flags must be serialized by the host stage-2 + * page-table lock due to the lack of atomics at EL2. + */ +#define HOST_PAGE_NEED_POISONING BIT(0) + struct hyp_page { unsigned short refcount; u8 order; diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 603d9e78ca1e..8758bc56de56 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -286,17 +286,13 @@ static int reclaim_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, void * const arg) { kvm_pte_t pte = *ptep; - phys_addr_t phys; + struct hyp_page *page; if (!kvm_pte_valid(pte)) return 0; - /* - * Only update the host stage-2 -- we're about to tear-down the guest - * stage-2 so no need to waste effort trying to keep it in sync. - */ - phys = kvm_pte_to_phys(pte); - BUG_ON(host_stage2_set_owner_locked(phys, PAGE_SIZE, pkvm_host_poison)); + page = hyp_phys_to_page(kvm_pte_to_phys(pte)); + page->flags |= HOST_PAGE_NEED_POISONING; return 0; } @@ -520,11 +516,6 @@ static kvm_pte_t kvm_init_invalid_leaf_owner(pkvm_id owner_id) return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id); } -static pkvm_id kvm_get_owner_id(kvm_pte_t pte) -{ - return FIELD_GET(KVM_INVALID_PTE_OWNER_MASK, pte); -} - int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, pkvm_id owner_id) { @@ -1864,7 +1855,7 @@ static int hyp_zero_page(phys_addr_t phys) int __pkvm_host_reclaim_page(u64 pfn) { u64 addr = hyp_pfn_to_phys(pfn); - enum pkvm_page_state state; + struct hyp_page *page; kvm_pte_t pte; int ret; @@ -1874,23 +1865,17 @@ int __pkvm_host_reclaim_page(u64 pfn) if (ret) goto unlock; - if (kvm_pte_valid(pte)) { - state = host_get_page_state(pte); - ret = (state == PKVM_PAGE_OWNED) ? 0 : -EPERM; + if (host_get_page_state(pte) == PKVM_PAGE_OWNED) goto unlock; - } - switch (kvm_get_owner_id(pte)) { - case pkvm_host_id: - ret = 0; - break; - case pkvm_host_poison: + page = hyp_phys_to_page(addr); + if (page->flags & HOST_PAGE_NEED_POISONING) { ret = hyp_zero_page(addr); if (ret) goto unlock; + page->flags &= ~HOST_PAGE_NEED_POISONING; ret = host_stage2_set_owner_locked(addr, PAGE_SIZE, pkvm_host_id); - break; - default: + } else { ret = -EPERM; }