From 8b29f4a6263f77dee0e4113061c7532cd181b2ef Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 7 Jul 2022 15:51:41 +0100 Subject: [PATCH] Revert "ANDROID: KVM: arm64: Back hyp_vmemmap for all of memory" This reverts commit b047bd2325af8c450029dc2bac3d9a691ef5c513. Bug: 233587962 Signed-off-by: Will Deacon Change-Id: If3f226371bae6419cbff922b13090061f5767d65 --- arch/arm64/include/asm/kvm_pkvm.h | 26 ----------------------- arch/arm64/kvm/hyp/include/nvhe/mm.h | 14 ++++++++++++- arch/arm64/kvm/hyp/nvhe/mm.c | 31 ++++------------------------ arch/arm64/kvm/hyp/nvhe/page_alloc.c | 4 +++- arch/arm64/kvm/hyp/nvhe/setup.c | 7 ++++--- arch/arm64/kvm/pkvm.c | 18 ++++++++++++++-- 6 files changed, 40 insertions(+), 60 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 8f7b8a2314bb..9f4ad2a8df59 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -14,32 +14,6 @@ extern struct memblock_region kvm_nvhe_sym(hyp_memory)[]; extern unsigned int kvm_nvhe_sym(hyp_memblock_nr); -static inline unsigned long -hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size) -{ - unsigned long nr_pages = reg->size >> PAGE_SHIFT; - unsigned long start, end; - - start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size; - end = start + nr_pages * vmemmap_entry_size; - start = ALIGN_DOWN(start, PAGE_SIZE); - end = ALIGN(end, PAGE_SIZE); - - return end - start; -} - -static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size) -{ - unsigned long res = 0, i; - - for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) { - res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i], - vmemmap_entry_size); - } - - return res >> PAGE_SHIFT; -} - static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages) { unsigned long total = 0, i; diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 73309ccc192e..2d08510c6cc1 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -15,11 +15,23 @@ extern hyp_spinlock_t pkvm_pgd_lock; int hyp_create_idmap(u32 hyp_va_bits); int hyp_map_vectors(void); -int hyp_back_vmemmap(phys_addr_t back); +int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back); int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot); unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot); +static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size, + unsigned long *start, unsigned long *end) +{ + unsigned long nr_pages = size >> PAGE_SHIFT; + struct hyp_page *p = hyp_phys_to_page(phys); + + *start = (unsigned long)p; + *end = *start + nr_pages * sizeof(struct hyp_page); + *start = ALIGN_DOWN(*start, PAGE_SIZE); + *end = ALIGN(*end, PAGE_SIZE); +} + #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 168e7fbe9a3c..cdbe8e246418 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -105,36 +105,13 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) return ret; } -int hyp_back_vmemmap(phys_addr_t back) +int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back) { - unsigned long i, start, size, end = 0; - int ret; + unsigned long start, end; - for (i = 0; i < hyp_memblock_nr; i++) { - start = hyp_memory[i].base; - start = ALIGN_DOWN((u64)hyp_phys_to_page(start), PAGE_SIZE); - /* - * The begining of the hyp_vmemmap region for the current - * memblock may already be backed by the page backing the end - * the previous region, so avoid mapping it twice. - */ - start = max(start, end); + hyp_vmemmap_range(phys, size, &start, &end); - end = hyp_memory[i].base + hyp_memory[i].size; - end = PAGE_ALIGN((u64)hyp_phys_to_page(end)); - if (start >= end) - continue; - - size = end - start; - ret = __pkvm_create_mappings(start, size, back, PAGE_HYP); - if (ret) - return ret; - - memset(hyp_phys_to_virt(back), 0, size); - back += size; - } - - return 0; + return __pkvm_create_mappings(start, end - start, back, PAGE_HYP); } static void *__hyp_bp_vect_base; diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index 31c073dcf478..38ea4e1aa72f 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -230,8 +230,10 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, /* Init the vmemmap portion */ p = hyp_phys_to_page(phys); - for (i = 0; i < nr_pages; i++) + for (i = 0; i < nr_pages; i++) { + p[i].order = 0; hyp_set_page_refcounted(&p[i]); + } /* Attach the unused pages to the buddy tree */ for (i = reserved_pages; i < nr_pages; i++) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 4c5c9dc11f93..0f58e7ea6610 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -31,11 +31,12 @@ static struct hyp_pool hpool; static int divide_memory_pool(void *virt, unsigned long size) { - unsigned long nr_pages; + unsigned long vstart, vend, nr_pages; hyp_early_alloc_init(virt, size); - nr_pages = hyp_vmemmap_pages(sizeof(struct hyp_page)); + hyp_vmemmap_range(__hyp_pa(virt), size, &vstart, &vend); + nr_pages = (vend - vstart) >> PAGE_SHIFT; vmemmap_base = hyp_early_alloc_contig(nr_pages); if (!vmemmap_base) return -ENOMEM; @@ -77,7 +78,7 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, if (ret) return ret; - ret = hyp_back_vmemmap(hyp_virt_to_phys(vmemmap_base)); + ret = hyp_back_vmemmap(phys, size, hyp_virt_to_phys(vmemmap_base)); if (ret) return ret; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 34229425b25d..ebecb7c045f4 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -53,7 +53,7 @@ static int __init register_memblock_regions(void) void __init kvm_hyp_reserve(void) { - u64 hyp_mem_pages = 0; + u64 nr_pages, prev, hyp_mem_pages = 0; int ret; if (!is_hyp_mode_available() || is_kernel_in_hyp_mode()) @@ -71,7 +71,21 @@ void __init kvm_hyp_reserve(void) hyp_mem_pages += hyp_s1_pgtable_pages(); hyp_mem_pages += host_s2_pgtable_pages(); - hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE); + + /* + * The hyp_vmemmap needs to be backed by pages, but these pages + * themselves need to be present in the vmemmap, so compute the number + * of pages needed by looking for a fixed point. + */ + nr_pages = 0; + do { + prev = nr_pages; + nr_pages = hyp_mem_pages + prev; + nr_pages = DIV_ROUND_UP(nr_pages * STRUCT_HYP_PAGE_SIZE, + PAGE_SIZE); + nr_pages += __hyp_pgtable_max_pages(nr_pages); + } while (nr_pages != prev); + hyp_mem_pages += nr_pages; /* * Try to allocate a PMD-aligned region to reduce TLB pressure once