diff --git a/CMakeLists.txt b/CMakeLists.txt
index b6718fb..ee68b4b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -68,7 +68,9 @@ endif()
 option(NEON "Use ARM Neon SIMD instructions if in ARM CPU" ON)
 if(${NEON} AND ${NEON_CPU})
   target_compile_definitions(SoundTouch PRIVATE SOUNDTOUCH_USE_NEON)
-  target_compile_options(SoundTouch PRIVATE -mfpu=neon)
+  if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64.*$")
+    target_compile_options(SoundTouch PRIVATE -mfpu=neon)
+  endif()
 endif()
 
 find_package(OpenMP)