diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
index e91a79c5e3df..e0cc7b5bf4a3 100644
--- a/arch/arm64/mm/ioremap.c
+++ b/arch/arm64/mm/ioremap.c
@@ -114,9 +114,6 @@ void ioremap_phys_range_hook(phys_addr_t phys_addr, size_t size, pgprot_t prot)
 	if (!static_branch_unlikely(&ioremap_guard_key))
 		return;
 
-	if (pfn_valid(__phys_to_pfn(phys_addr)))
-		return;
-
 	mutex_lock(&ioremap_guard_lock);
 
 	while (size) {
@@ -124,6 +121,9 @@ void ioremap_phys_range_hook(phys_addr_t phys_addr, size_t size, pgprot_t prot)
 		struct ioremap_guard_ref *ref;
 		struct arm_smccc_res res;
 
+		if (pfn_valid(__phys_to_pfn(phys_addr)))
+			goto next;
+
 		ref = xa_load(&ioremap_guard_array, pfn);
 		if (ref) {
 			refcount_inc(&ref->count);