diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index c5855bf53446..14b822a187d0 100644
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1189,7 +1189,22 @@ static int pkvm_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
 	ret = pin_user_pages(hva, 1, flags, &page, NULL);
 	mmap_read_unlock(mm);
 
-	if (ret == -EHWPOISON) {
+	/*
+	 * We really can't deal with page-cache pages returned by GUP
+	 * because (a) we may trigger writeback of a page for which we
+	 * no longer have access and (b) page_mkclean() won't find the
+	 * stage-2 mapping in the rmap so we can get out-of-whack with
+	 * the filesystem when marking the page dirty during unpinning.
+	 *
+	 * Ideally we'd just restrict ourselves to anonymous pages, but
+	 * we also want to allow memfd (i.e. shmem) pages, so check for
+	 * pages backed by swap in the knowledge that the GUP pin will
+	 * prevent try_to_unmap() from succeeding.
+	 */
+	if (!PageSwapBacked(page)) {
+		ret = -EIO;
+		goto dec_account;
+	} else if (ret == -EHWPOISON) {
 		kvm_send_hwpoison_signal(hva, PAGE_SHIFT);
 		ret = 0;
 		goto dec_account;