diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h
index 19cec0f66a..036f3996c3 100644
--- a/src/video_core/fence_manager.h
+++ b/src/video_core/fence_manager.h
@@ -53,7 +53,10 @@ public:
     void WaitPendingFences() {
         while (!fences.empty()) {
             TFence& current_fence = fences.front();
-            WaitFence(current_fence);
+            bool should_wait = texture_cache.ShouldWaitAsyncFlushes();
+            if (should_wait) {
+                WaitFence(current_fence);
+            }
             texture_cache.PopAsyncFlushes();
             auto& gpu{system.GPU()};
             auto& memory_manager{gpu.MemoryManager()};
@@ -80,7 +83,8 @@ private:
     void TryReleasePendingFences() {
         while (!fences.empty()) {
             TFence& current_fence = fences.front();
-            if (!IsFenceSignaled(current_fence)) {
+            bool should_wait = texture_cache.ShouldWaitAsyncFlushes();
+            if (should_wait && !IsFenceSignaled(current_fence)) {
                 return;
             }
             texture_cache.PopAsyncFlushes();
diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 6629c59ed2..04fe69c11d 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -322,6 +322,17 @@ public:
         uncommited_flushes.reset();
     }
 
+    bool ShouldWaitAsyncFlushes() {
+        if (commited_flushes.empty()) {
+            return false;
+        }
+        auto& flush_list = commited_flushes.front();
+        if (!flush_list) {
+            return false;
+        }
+        return true;
+    }
+
     void PopAsyncFlushes() {
         if (commited_flushes.empty()) {
             return;