diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
index 4efb3d82..cba645f3 100644
--- a/benchmarks/CMakeLists.txt
+++ b/benchmarks/CMakeLists.txt
@@ -32,6 +32,7 @@ macro(sfizz_add_benchmark TARGET)
         target_link_libraries ("${TARGET}" PRIVATE atomic)
     endif()
 	target_include_directories("${TARGET}" PRIVATE ../src/sfizz ../src/external)
+	sfizz_enable_fast_math("${TARGET}")
 endmacro()
 
 sfizz_add_benchmark(bm_opf_high_vs_low BM_OPF_high_vs_low.cpp)
diff --git a/cmake/SfizzConfig.cmake b/cmake/SfizzConfig.cmake
index 16c805bd..fc69cbbd 100644
--- a/cmake/SfizzConfig.cmake
+++ b/cmake/SfizzConfig.cmake
@@ -32,7 +32,6 @@ endif()
 if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
     add_compile_options(-Wall)
     add_compile_options(-Wextra)
-    add_compile_options(-ffast-math)
     add_compile_options(-fno-omit-frame-pointer) # For debugging purposes
     if (SFIZZ_SYSTEM_PROCESSOR MATCHES "^(i.86|x86_64)$")
         add_compile_options(-msse2)
@@ -43,6 +42,12 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
     set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
 endif()
 
+function(sfizz_enable_fast_math NAME)
+    if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
+        target_compile_options("${NAME}" PRIVATE "-ffast-math")
+    endif()
+endfunction()
+
 add_library(sfizz-sndfile INTERFACE)
 
 if (SFIZZ_USE_VCPKG OR CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ed19f1b6..0837e2ac 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -217,6 +217,7 @@ endif()
 if (SFIZZ_RELEASE_ASSERTS)
     target_compile_definitions (sfizz_static PRIVATE "SFIZZ_ENABLE_RELEASE_ASSERT=1")
 endif()
+sfizz_enable_fast_math(sfizz_static)
 
 if (NOT MSVC)
     install (TARGETS sfizz_static
@@ -257,6 +258,7 @@ if (SFIZZ_SHARED)
     target_compile_definitions(sfizz_shared PRIVATE SFIZZ_EXPORT_SYMBOLS)
     set_target_properties (sfizz_shared PROPERTIES SOVERSION ${PROJECT_VERSION_MAJOR} OUTPUT_NAME sfizz)
     sfizz_enable_lto_if_needed(sfizz_shared)
+    sfizz_enable_fast_math(sfizz_shared)
 
     if (NOT MSVC)
         install (TARGETS sfizz_shared
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 76cb7ae9..61b16360 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -47,6 +47,7 @@ set(SFIZZ_TEST_SOURCES
 add_executable(sfizz_tests ${SFIZZ_TEST_SOURCES})
 target_link_libraries(sfizz_tests PRIVATE sfizz::sfizz)
 sfizz_enable_lto_if_needed(sfizz_tests)
+sfizz_enable_fast_math(sfizz_tests)
 # target_link_libraries(sfizz_tests PRIVATE absl::strings absl::str_format absl::flat_hash_map cnpy absl::span absl::algorithm)
 
 find_package(PkgConfig)