diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 0b5ec89a8ec4..bb92d3875791 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -784,7 +784,7 @@ config RELOCATABLE
 
 config RANDOMIZE_BASE
 	bool "Randomize the address of the kernel image"
-	select ARM64_MODULE_PLTS
+	select ARM64_MODULE_PLTS if MODULES
 	select RELOCATABLE
 	help
 	  Randomizes the virtual address at which the kernel image is