diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fa261547b..d5f774d77 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -713,6 +713,11 @@ else()
 		set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWIN32_LEAN_AND_MEAN")
 	endif()
 
+	# Use a safe subset of flags to speed up math calculations:
+	# - we don't need errno or math exceptions
+	# - we don't deal with Inf/NaN or signed zero
+	set(MATH_FLAGS "-fno-math-errno -fno-trapping-math -ffinite-math-only -fno-signed-zeros")
+
 	set(CMAKE_CXX_FLAGS_RELEASE "-DNDEBUG ${RELEASE_WARNING_FLAGS} ${WARNING_FLAGS} ${OTHER_FLAGS} -Wall -pipe -funroll-loops")
 	if(CMAKE_SYSTEM_NAME MATCHES "(Darwin|BSD|DragonFly)")
 		set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Os")
@@ -723,7 +728,7 @@ else()
 				AND CMAKE_CXX_COMPILER_VERSION MATCHES "^9\\.")
 			# Clang 9 has broken -ffast-math on glibc
 		else()
-			set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -ffast-math")
+			set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${MATH_FLAGS}")
 		endif()
 	endif(CMAKE_SYSTEM_NAME MATCHES "(Darwin|BSD|DragonFly)")
 	set(CMAKE_CXX_FLAGS_SEMIDEBUG "-g -O1 -Wall -Wabi ${WARNING_FLAGS} ${OTHER_FLAGS}")
diff --git a/src/collision.cpp b/src/collision.cpp
index 6d24bc699..a089f3377 100644
--- a/src/collision.cpp
+++ b/src/collision.cpp
@@ -32,6 +32,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "util/timetaker.h"
 #include "profiler.h"
 
+#ifdef __FAST_MATH__
+#warning "-ffast-math is known to cause bugs in collision code, do not use!"
+#endif
 
 struct NearbyCollisionInfo {
 	NearbyCollisionInfo(bool is_ul, bool is_obj, int bouncy,