diff --git a/mm/memory.c b/mm/memory.c
index bfb8513e9530..c34e002b4241 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -3567,6 +3567,11 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 	vm_fault_t ret;
 	void *shadow = NULL;
 
+	if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
+		pte_unmap(vmf->pte);
+		return VM_FAULT_RETRY;
+	}
+
 	ret = pte_unmap_same(vmf);
 	if (ret) {
 		/*