diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt
index 4a4ba1101..88cc52588 100644
--- a/externals/CMakeLists.txt
+++ b/externals/CMakeLists.txt
@@ -11,11 +11,13 @@ target_include_directories(catch-single-include INTERFACE catch/single_include)
 add_subdirectory(cryptopp)
 
 # Dynarmic
-# Dynarmic will skip defining xbyak if it's already defined, we then define it below
-add_library(xbyak INTERFACE)
-option(DYNARMIC_TESTS OFF)
-set(DYNARMIC_NO_BUNDLED_FMT ON)
-add_subdirectory(dynarmic)
+if (ARCHITECTURE_x86_64)
+    # Dynarmic will skip defining xbyak if it's already defined, we then define it below
+    add_library(xbyak INTERFACE)
+    option(DYNARMIC_TESTS OFF)
+    set(DYNARMIC_NO_BUNDLED_FMT ON)
+    add_subdirectory(dynarmic)
+endif()
 
 # libfmt
 add_subdirectory(fmt)
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 6057b183f..f6d6fd3d4 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -1,10 +1,6 @@
 add_library(core STATIC
     3ds.h
     arm/arm_interface.h
-    arm/dynarmic/arm_dynarmic.cpp
-    arm/dynarmic/arm_dynarmic.h
-    arm/dynarmic/arm_dynarmic_cp15.cpp
-    arm/dynarmic/arm_dynarmic_cp15.h
     arm/dyncom/arm_dyncom.cpp
     arm/dyncom/arm_dyncom.h
     arm/dyncom/arm_dyncom_dec.cpp
@@ -404,7 +400,17 @@ add_library(core STATIC
 create_target_directory_groups(core)
 
 target_link_libraries(core PUBLIC common PRIVATE audio_core network video_core)
-target_link_libraries(core PUBLIC Boost::boost PRIVATE cryptopp dynarmic fmt)
+target_link_libraries(core PUBLIC Boost::boost PRIVATE cryptopp fmt)
 if (ENABLE_WEB_SERVICE)
     target_link_libraries(core PUBLIC json-headers web_service)
 endif()
+
+if (ARCHITECTURE_x86_64)
+    add_library(core STATIC
+        arm/dynarmic/arm_dynarmic.cpp
+        arm/dynarmic/arm_dynarmic.h
+        arm/dynarmic/arm_dynarmic_cp15.cpp
+        arm/dynarmic/arm_dynarmic_cp15.h
+    )
+    target_link_libraries(core PRIVATE dynarmic)
+endif()