diff --git a/src/common/threadsafe_queue.h b/src/common/threadsafe_queue.h
index d69ab1289..2754d585c 100644
--- a/src/common/threadsafe_queue.h
+++ b/src/common/threadsafe_queue.h
@@ -48,15 +48,13 @@ public:
         ElementPtr* new_ptr = new ElementPtr();
         write_ptr->next.store(new_ptr, std::memory_order_release);
         write_ptr = new_ptr;
+        ++size;
 
-        const size_t previous_size{size++};
-
-        // Acquire the mutex and then immediately release it as a fence.
+        // cv_mutex must be held or else there will be a missed wakeup if the other thread is in the
+        // line before cv.wait
         // TODO(bunnei): This can be replaced with C++20 waitable atomics when properly supported.
         // See discussion on https://github.com/yuzu-emu/yuzu/pull/3173 for details.
-        if (previous_size == 0) {
-            std::lock_guard lock{cv_mutex};
-        }
+        std::lock_guard lock{cv_mutex};
         cv.notify_one();
     }