diff --git a/src/core/arm/arm_interface.h b/src/core/arm/arm_interface.h
index 40bfc5543..649aa9219 100644
--- a/src/core/arm/arm_interface.h
+++ b/src/core/arm/arm_interface.h
@@ -228,8 +228,12 @@ public:
 
     virtual void PurgeState() = 0;
 
-    std::shared_ptr<Core::Timing::Timer> GetTimer() {
-        return timer;
+    Core::Timing::Timer& GetTimer() {
+        return *timer;
+    }
+
+    const Core::Timing::Timer& GetTimer() const {
+        return *timer;
     }
 
     u32 GetID() const {
diff --git a/src/core/arm/dynarmic/arm_dynarmic.cpp b/src/core/arm/dynarmic/arm_dynarmic.cpp
index 039601a54..64cb0524e 100644
--- a/src/core/arm/dynarmic/arm_dynarmic.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic.cpp
@@ -137,10 +137,10 @@ public:
     }
 
     void AddTicks(std::uint64_t ticks) override {
-        parent.GetTimer()->AddTicks(ticks);
+        parent.GetTimer().AddTicks(ticks);
     }
     std::uint64_t GetTicksRemaining() override {
-        s64 ticks = parent.GetTimer()->GetDowncount();
+        s64 ticks = parent.GetTimer().GetDowncount();
         return static_cast<u64>(ticks <= 0 ? 0 : ticks);
     }
 
diff --git a/src/core/arm/dyncom/arm_dyncom_interpreter.cpp b/src/core/arm/dyncom/arm_dyncom_interpreter.cpp
index 706e0092b..7ba67320b 100644
--- a/src/core/arm/dyncom/arm_dyncom_interpreter.cpp
+++ b/src/core/arm/dyncom/arm_dyncom_interpreter.cpp
@@ -3865,7 +3865,7 @@ SWI_INST : {
     if (inst_base->cond == ConditionCode::AL || CondPassed(cpu, inst_base->cond)) {
         DEBUG_ASSERT(cpu->system != nullptr);
         swi_inst* const inst_cream = (swi_inst*)inst_base->component;
-        cpu->system->GetRunningCore().GetTimer()->AddTicks(num_instrs);
+        cpu->system->GetRunningCore().GetTimer().AddTicks(num_instrs);
         cpu->NumInstrsToExecute =
             num_instrs >= cpu->NumInstrsToExecute ? 0 : cpu->NumInstrsToExecute - num_instrs;
         num_instrs = 0;
diff --git a/src/core/core.cpp b/src/core/core.cpp
index 6a20dc374..47fafebe6 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -144,14 +144,14 @@ System::ResultStatus System::RunLoop(bool tight_loop) {
     // So we have to get those cores to the same global time first
     u64 global_ticks = timing->GetGlobalTicks();
     s64 max_delay = 0;
-    std::shared_ptr<ARM_Interface> current_core_to_execute = nullptr;
+    ARM_Interface* current_core_to_execute = nullptr;
     for (auto& cpu_core : cpu_cores) {
-        if (cpu_core->GetTimer()->GetTicks() < global_ticks) {
-            s64 delay = global_ticks - cpu_core->GetTimer()->GetTicks();
-            cpu_core->GetTimer()->Advance(delay);
+        if (cpu_core->GetTimer().GetTicks() < global_ticks) {
+            s64 delay = global_ticks - cpu_core->GetTimer().GetTicks();
+            cpu_core->GetTimer().Advance(delay);
             if (max_delay < delay) {
                 max_delay = delay;
-                current_core_to_execute = cpu_core;
+                current_core_to_execute = cpu_core.get();
             }
         }
     }
@@ -159,12 +159,14 @@ System::ResultStatus System::RunLoop(bool tight_loop) {
     if (max_delay > 0) {
         LOG_TRACE(Core_ARM11, "Core {} running (delayed) for {} ticks",
                   current_core_to_execute->GetID(),
-                  current_core_to_execute->GetTimer()->GetDowncount());
-        running_core = current_core_to_execute.get();
-        kernel->SetRunningCPU(current_core_to_execute);
+                  current_core_to_execute->GetTimer().GetDowncount());
+        if (running_core != current_core_to_execute) {
+            running_core = current_core_to_execute;
+            kernel->SetRunningCPU(running_core);
+        }
         if (kernel->GetCurrentThreadManager().GetCurrentThread() == nullptr) {
             LOG_TRACE(Core_ARM11, "Core {} idling", current_core_to_execute->GetID());
-            current_core_to_execute->GetTimer()->Idle();
+            current_core_to_execute->GetTimer().Idle();
             PrepareReschedule();
         } else {
             if (tight_loop) {
@@ -179,21 +181,21 @@ System::ResultStatus System::RunLoop(bool tight_loop) {
         // TODO: Make special check for idle since we can easily revert the time of idle cores
         s64 max_slice = Timing::MAX_SLICE_LENGTH;
         for (const auto& cpu_core : cpu_cores) {
-            max_slice = std::min(max_slice, cpu_core->GetTimer()->GetMaxSliceLength());
+            max_slice = std::min(max_slice, cpu_core->GetTimer().GetMaxSliceLength());
         }
         for (auto& cpu_core : cpu_cores) {
-            cpu_core->GetTimer()->Advance(max_slice);
+            cpu_core->GetTimer().Advance(max_slice);
         }
         for (auto& cpu_core : cpu_cores) {
             LOG_TRACE(Core_ARM11, "Core {} running for {} ticks", cpu_core->GetID(),
-                      cpu_core->GetTimer()->GetDowncount());
+                      cpu_core->GetTimer().GetDowncount());
             running_core = cpu_core.get();
-            kernel->SetRunningCPU(cpu_core);
+            kernel->SetRunningCPU(running_core);
             // If we don't have a currently active thread then don't execute instructions,
             // instead advance to the next event and try to yield to the next thread
             if (kernel->GetCurrentThreadManager().GetCurrentThread() == nullptr) {
                 LOG_TRACE(Core_ARM11, "Core {} idling", cpu_core->GetID());
-                cpu_core->GetTimer()->Idle();
+                cpu_core->GetTimer().Idle();
                 PrepareReschedule();
             } else {
                 if (tight_loop) {
@@ -371,7 +373,7 @@ System::ResultStatus System::Init(Frontend::EmuWindow& emu_window, u32 system_mo
     running_core = cpu_cores[0].get();
 
     kernel->SetCPUs(cpu_cores);
-    kernel->SetRunningCPU(cpu_cores[0]);
+    kernel->SetRunningCPU(cpu_cores[0].get());
 
     if (Settings::values.enable_dsp_lle) {
         dsp_core = std::make_unique<AudioCore::DspLle>(*memory,
diff --git a/src/core/core_timing.cpp b/src/core/core_timing.cpp
index 963926596..875412e83 100644
--- a/src/core/core_timing.cpp
+++ b/src/core/core_timing.cpp
@@ -26,7 +26,7 @@ Timing::Timing(std::size_t num_cores, u32 cpu_clock_percentage) {
         timers[i] = std::make_shared<Timer>();
     }
     UpdateClockSpeed(cpu_clock_percentage);
-    current_timer = timers[0];
+    current_timer = timers[0].get();
 }
 
 void Timing::UpdateClockSpeed(u32 cpu_clock_percentage) {
@@ -50,12 +50,12 @@ TimingEventType* Timing::RegisterEvent(const std::string& name, TimedCallback ca
 void Timing::ScheduleEvent(s64 cycles_into_future, const TimingEventType* event_type, u64 userdata,
                            std::size_t core_id) {
     ASSERT(event_type != nullptr);
-    std::shared_ptr<Timing::Timer> timer;
+    Timing::Timer* timer = nullptr;
     if (core_id == std::numeric_limits<std::size_t>::max()) {
         timer = current_timer;
     } else {
         ASSERT(core_id < timers.size());
-        timer = timers.at(core_id);
+        timer = timers.at(core_id).get();
     }
 
     s64 timeout = timer->GetTicks() + cycles_into_future;
@@ -103,7 +103,7 @@ void Timing::RemoveEvent(const TimingEventType* event_type) {
 }
 
 void Timing::SetCurrentTimer(std::size_t core_id) {
-    current_timer = timers[core_id];
+    current_timer = timers[core_id].get();
 }
 
 s64 Timing::GetTicks() const {
diff --git a/src/core/core_timing.h b/src/core/core_timing.h
index 74437c9b1..73b226cde 100644
--- a/src/core/core_timing.h
+++ b/src/core/core_timing.h
@@ -281,20 +281,28 @@ private:
     std::unordered_map<std::string, TimingEventType> event_types = {};
 
     std::vector<std::shared_ptr<Timer>> timers;
-    std::shared_ptr<Timer> current_timer;
+    Timer* current_timer = nullptr;
 
     // Stores a scaling for the internal clockspeed. Changing this number results in
     // under/overclocking the guest cpu
     double cpu_clock_scale = 1.0;
 
     template <class Archive>
-    void serialize(Archive& ar, const unsigned int) {
+    void serialize(Archive& ar, const unsigned int file_version) {
         // event_types set during initialization of other things
         ar& global_timer;
         ar& timers;
-        ar& current_timer;
+        if (file_version == 0) {
+            std::shared_ptr<Timer> x;
+            ar& x;
+            current_timer = x.get();
+        } else {
+            ar& current_timer;
+        }
     }
     friend class boost::serialization::access;
 };
 
 } // namespace Core
+
+BOOST_CLASS_VERSION(Core::Timing, 1)
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index d36c8a4c7..c9c65a111 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -93,7 +93,7 @@ void KernelSystem::SetCPUs(std::vector<std::shared_ptr<ARM_Interface>> cpus) {
     }
 }
 
-void KernelSystem::SetRunningCPU(std::shared_ptr<ARM_Interface> cpu) {
+void KernelSystem::SetRunningCPU(ARM_Interface* cpu) {
     if (current_process) {
         stored_processes[current_cpu->GetID()] = current_process;
     }
diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h
index f0a8368d9..5d595f674 100644
--- a/src/core/hle/kernel/kernel.h
+++ b/src/core/hle/kernel/kernel.h
@@ -218,7 +218,7 @@ public:
 
     void SetCPUs(std::vector<std::shared_ptr<ARM_Interface>> cpu);
 
-    void SetRunningCPU(std::shared_ptr<ARM_Interface> cpu);
+    void SetRunningCPU(ARM_Interface* cpu);
 
     ThreadManager& GetThreadManager(u32 core_id);
     const ThreadManager& GetThreadManager(u32 core_id) const;
@@ -257,7 +257,7 @@ public:
     /// Map of named ports managed by the kernel, which can be retrieved using the ConnectToPort
     std::unordered_map<std::string, std::shared_ptr<ClientPort>> named_ports;
 
-    std::shared_ptr<ARM_Interface> current_cpu;
+    ARM_Interface* current_cpu = nullptr;
 
     Memory::MemorySystem& memory;
 
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 25eb9bb4e..ee0eae626 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -1254,10 +1254,10 @@ void SVC::SleepThread(s64 nanoseconds) {
 /// This returns the total CPU ticks elapsed since the CPU was powered-on
 s64 SVC::GetSystemTick() {
     // TODO: Use globalTicks here?
-    s64 result = system.GetRunningCore().GetTimer()->GetTicks();
+    s64 result = system.GetRunningCore().GetTimer().GetTicks();
     // Advance time to defeat dumb games (like Cubic Ninja) that busy-wait for the frame to end.
     // Measured time between two calls on a 9.2 o3DS with Ninjhax 1.1b
-    system.GetRunningCore().GetTimer()->AddTicks(150);
+    system.GetRunningCore().GetTimer().AddTicks(150);
     return result;
 }