diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
index e2c6bd1f3..3b894444f 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
@@ -495,7 +495,9 @@ static bool FillSurface(const Surface& surface, const u8* fill_data,
 }
 
 CachedSurface::~CachedSurface() {
-    owner.host_texture_recycler.emplace(*this, std::move(texture));
+    if (texture.handle) {
+        owner.host_texture_recycler.emplace(*this, std::move(texture));
+    }
 }
 
 bool CachedSurface::CanFill(const SurfaceParams& dest_surface,