diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c
index 224c7d7e7dc9..21957337a45e 100644
--- a/arch/arm64/mm/ioremap.c
+++ b/arch/arm64/mm/ioremap.c
@@ -125,9 +125,6 @@ void ioremap_phys_range_hook(phys_addr_t phys_addr, size_t size, pgprot_t prot)
 	if (!static_branch_unlikely(&ioremap_guard_key))
 		return;
 
-	if (pfn_valid(__phys_to_pfn(phys_addr)))
-		return;
-
 	mutex_lock(&ioremap_guard_lock);
 
 	while (size) {
@@ -135,6 +132,9 @@ void ioremap_phys_range_hook(phys_addr_t phys_addr, size_t size, pgprot_t prot)
 		struct ioremap_guard_ref *ref;
 		struct arm_smccc_res res;
 
+		if (pfn_valid(__phys_to_pfn(phys_addr)))
+			goto next;
+
 		ref = xa_load(&ioremap_guard_array, pfn);
 		if (ref) {
 			refcount_inc(&ref->count);