diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 788516dedc..66931ac976 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -97,6 +97,7 @@ add_custom_command(OUTPUT scm_rev.cpp
 add_library(common STATIC
     algorithm.h
     alignment.h
+    assert.cpp
     assert.h
     atomic_ops.h
     detached_tasks.cpp
diff --git a/src/common/assert.cpp b/src/common/assert.cpp
new file mode 100644
index 0000000000..d7d91b96b0
--- /dev/null
+++ b/src/common/assert.cpp
@@ -0,0 +1,11 @@
+// Copyright 2021 yuzu Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "common/assert.h"
+
+#include "common/common_funcs.h"
+
+void assert_handle_failure() {
+    Crash();
+}
diff --git a/src/common/assert.h b/src/common/assert.h
index 06d7b5612b..b3ba35c0f9 100644
--- a/src/common/assert.h
+++ b/src/common/assert.h
@@ -4,10 +4,13 @@
 
 #pragma once
 
-#include <cstdlib>
-#include "common/common_funcs.h"
 #include "common/logging/log.h"
 
+// Sometimes we want to try to continue even after hitting an assert.
+// However touching this file yields a global recompilation as this header is included almost
+// everywhere. So let's just move the handling of the failed assert to a single cpp file.
+void assert_handle_failure();
+
 // For asserts we'd like to keep all the junk executed when an assert happens away from the
 // important code in the function. One way of doing this is to put all the relevant code inside a
 // lambda and force the compiler to not inline it. Unfortunately, MSVC seems to have no syntax to
@@ -17,15 +20,14 @@
 // enough for our purposes.
 template <typename Fn>
 #if defined(_MSC_VER)
-[[msvc::noinline, noreturn]]
+[[msvc::noinline]]
 #elif defined(__GNUC__)
-[[gnu::cold, gnu::noinline, noreturn]]
+[[gnu::cold, gnu::noinline]]
 #endif
 static void
 assert_noinline_call(const Fn& fn) {
     fn();
-    Crash();
-    exit(1); // Keeps GCC's mouth shut about this actually returning
+    assert_handle_failure();
 }
 
 #define ASSERT(_a_)                                                                                \