Revert "ANDROID: KVM: arm64: Back hyp_vmemmap for all of memory"

This reverts commit b047bd2325.

Bug: 233587962
Signed-off-by: Will Deacon <willdeacon@google.com>
Change-Id: If3f226371bae6419cbff922b13090061f5767d65

arch/arm64/include/asm/kvm_pkvm.h

@@ -14,32 +14,6 @@
 extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
 extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
 
-static inline unsigned long
-hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size)
-{
-	unsigned long nr_pages = reg->size >> PAGE_SHIFT;
-	unsigned long start, end;
-
-	start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size;
-	end = start + nr_pages * vmemmap_entry_size;
-	start = ALIGN_DOWN(start, PAGE_SIZE);
-	end = ALIGN(end, PAGE_SIZE);
-
-	return end - start;
-}
-
-static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size)
-{
-	unsigned long res = 0, i;
-
-	for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
-		res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i],
-						 vmemmap_entry_size);
-	}
-
-	return res >> PAGE_SHIFT;
-}
-
 static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
 {
 	unsigned long total = 0, i;

arch/arm64/kvm/hyp/include/nvhe/mm.h

@@ -15,11 +15,23 @@ extern hyp_spinlock_t pkvm_pgd_lock;
 
 int hyp_create_idmap(u32 hyp_va_bits);
 int hyp_map_vectors(void);
-int hyp_back_vmemmap(phys_addr_t back);
+int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back);
 int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);
 int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
 int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);
 unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
 					    enum kvm_pgtable_prot prot);
 
+static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size,
+				     unsigned long *start, unsigned long *end)
+{
+	unsigned long nr_pages = size >> PAGE_SHIFT;
+	struct hyp_page *p = hyp_phys_to_page(phys);
+
+	*start = (unsigned long)p;
+	*end = *start + nr_pages * sizeof(struct hyp_page);
+	*start = ALIGN_DOWN(*start, PAGE_SIZE);
+	*end = ALIGN(*end, PAGE_SIZE);
+}
+
 #endif /* __KVM_HYP_MM_H */

arch/arm64/kvm/hyp/nvhe/mm.c

@@ -105,36 +105,13 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
 	return ret;
 }
 
-int hyp_back_vmemmap(phys_addr_t back)
+int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back)
 {
-	unsigned long i, start, size, end = 0;
-	int ret;
+	unsigned long start, end;
 
-	for (i = 0; i < hyp_memblock_nr; i++) {
-		start = hyp_memory[i].base;
-		start = ALIGN_DOWN((u64)hyp_phys_to_page(start), PAGE_SIZE);
-		/*
-		 * The begining of the hyp_vmemmap region for the current
-		 * memblock may already be backed by the page backing the end
-		 * the previous region, so avoid mapping it twice.
-		 */
-		start = max(start, end);
+	hyp_vmemmap_range(phys, size, &start, &end);
 
-		end = hyp_memory[i].base + hyp_memory[i].size;
-		end = PAGE_ALIGN((u64)hyp_phys_to_page(end));
-		if (start >= end)
-			continue;
-
-		size = end - start;
-		ret = __pkvm_create_mappings(start, size, back, PAGE_HYP);
-		if (ret)
-			return ret;
-
-		memset(hyp_phys_to_virt(back), 0, size);
-		back += size;
-	}
-
-	return 0;
+	return __pkvm_create_mappings(start, end - start, back, PAGE_HYP);
 }
 
 static void *__hyp_bp_vect_base;

arch/arm64/kvm/hyp/nvhe/page_alloc.c

@@ -230,8 +230,10 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages,
 
 	/* Init the vmemmap portion */
 	p = hyp_phys_to_page(phys);
-	for (i = 0; i < nr_pages; i++)
+	for (i = 0; i < nr_pages; i++) {
+		p[i].order = 0;
 		hyp_set_page_refcounted(&p[i]);
+	}
 
 	/* Attach the unused pages to the buddy tree */
 	for (i = reserved_pages; i < nr_pages; i++)

arch/arm64/kvm/hyp/nvhe/setup.c

@@ -31,11 +31,12 @@ static struct hyp_pool hpool;
 
 static int divide_memory_pool(void *virt, unsigned long size)
 {
-	unsigned long nr_pages;
+	unsigned long vstart, vend, nr_pages;
 
 	hyp_early_alloc_init(virt, size);
 
-	nr_pages = hyp_vmemmap_pages(sizeof(struct hyp_page));
+	hyp_vmemmap_range(__hyp_pa(virt), size, &vstart, &vend);
+	nr_pages = (vend - vstart) >> PAGE_SHIFT;
 	vmemmap_base = hyp_early_alloc_contig(nr_pages);
 	if (!vmemmap_base)
 		return -ENOMEM;
@@ -77,7 +78,7 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
 	if (ret)
 		return ret;
 
-	ret = hyp_back_vmemmap(hyp_virt_to_phys(vmemmap_base));
+	ret = hyp_back_vmemmap(phys, size, hyp_virt_to_phys(vmemmap_base));
 	if (ret)
 		return ret;
 

arch/arm64/kvm/pkvm.c

@@ -53,7 +53,7 @@ static int __init register_memblock_regions(void)
 
 void __init kvm_hyp_reserve(void)
 {
-	u64 hyp_mem_pages = 0;
+	u64 nr_pages, prev, hyp_mem_pages = 0;
 	int ret;
 
 	if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
@@ -71,7 +71,21 @@ void __init kvm_hyp_reserve(void)
 
 	hyp_mem_pages += hyp_s1_pgtable_pages();
 	hyp_mem_pages += host_s2_pgtable_pages();
-	hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE);
+
+	/*
+	 * The hyp_vmemmap needs to be backed by pages, but these pages
+	 * themselves need to be present in the vmemmap, so compute the number
+	 * of pages needed by looking for a fixed point.
+	 */
+	nr_pages = 0;
+	do {
+		prev = nr_pages;
+		nr_pages = hyp_mem_pages + prev;
+		nr_pages = DIV_ROUND_UP(nr_pages * STRUCT_HYP_PAGE_SIZE,
+					PAGE_SIZE);
+		nr_pages += __hyp_pgtable_max_pages(nr_pages);
+	} while (nr_pages != prev);
+	hyp_mem_pages += nr_pages;
 
 	/*
 	 * Try to allocate a PMD-aligned region to reduce TLB pressure once