diff --git a/arch/arm64/src/Toolchain.defs b/arch/arm64/src/Toolchain.defs
index 3de82652a5f..40862575419 100644
--- a/arch/arm64/src/Toolchain.defs
+++ b/arch/arm64/src/Toolchain.defs
@@ -100,6 +100,13 @@ ifeq ($(CONFIG_MM_KASAN_DISABLE_WRITES_CHECK),y)
   ARCHOPTIMIZATION += --param asan-instrument-writes=0
 endif
 
+# If we don't use MMU, all memory is treated as device memory and can't handle
+# alignment faults. Some compiler builtins like memset() can introduce
+# alignment faults. We need to disable them in that case.
+ifneq ($(CONFIG_ARCH_USE_MMU),y)
+  ARCHOPTIMIZATION += -fno-builtin
+endif
+
 # Instrumentation options
 
 ifeq ($(CONFIG_ARCH_INSTRUMENT_ALL),y)
diff --git a/arch/arm64/src/cmake/Toolchain.cmake b/arch/arm64/src/cmake/Toolchain.cmake
index f86506374e9..973f8c88973 100644
--- a/arch/arm64/src/cmake/Toolchain.cmake
+++ b/arch/arm64/src/cmake/Toolchain.cmake
@@ -195,6 +195,10 @@ if(CONFIG_DEBUG_SYMBOLS)
   add_compile_options(${CONFIG_DEBUG_SYMBOLS_LEVEL})
 endif()
 
+if(NOT CONFIG_ARCH_USE_MMU)
+  add_compile_options(-fno-builtin)
+endif()
+
 if(CONFIG_ARCH_TOOLCHAIN_GNU AND NOT CONFIG_ARCH_TOOLCHAIN_CLANG)
   if(NOT GCCVER)
     execute_process(COMMAND ${CMAKE_C_COMPILER} --version
diff --git a/arch/arm64/src/imx9/imx9_boot.c b/arch/arm64/src/imx9/imx9_boot.c
index 259c3cbb239..f0ed552d8c9 100644
--- a/arch/arm64/src/imx9/imx9_boot.c
+++ b/arch/arm64/src/imx9/imx9_boot.c
@@ -142,9 +142,11 @@ void arm64_chip_boot(void)
 #endif
 #endif
 
+#ifdef CONFIG_ARCH_USE_MMU
   /* MAP IO and DRAM, enable MMU. */
 
   arm64_mmu_init(true);
+#endif
 
   /* Do UART early initialization & pin muxing */