diff --git a/src/core/core.cpp b/src/core/core.cpp
index d199b16e2..aeab0d2b5 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -437,7 +437,9 @@ void System::Reset() {
 
 template <class Archive>
 void System::serialize(Archive& ar, const unsigned int file_version) {
-    Memory::RasterizerFlushAndInvalidateRegion(0, 0xFFFFFFFF);
+    // flush on save, don't flush on load
+    bool should_flush = !Archive::is_loading::value;
+    Memory::RasterizerClearAll(should_flush);
     ar&* timing.get();
     ar&* cpu_core.get();
     ar&* service_manager.get();
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 4c083d8c0..a0fa2a738 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -556,6 +556,16 @@ void RasterizerFlushAndInvalidateRegion(PAddr start, u32 size) {
     VideoCore::g_renderer->Rasterizer()->FlushAndInvalidateRegion(start, size);
 }
 
+void RasterizerClearAll(bool flush) {
+    // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be
+    // null here
+    if (VideoCore::g_renderer == nullptr) {
+        return;
+    }
+
+    VideoCore::g_renderer->Rasterizer()->ClearAll(flush);
+}
+
 void RasterizerFlushVirtualRegion(VAddr start, u32 size, FlushMode mode) {
     // Since pages are unmapped on shutdown after video core is shutdown, the renderer may be
     // null here
diff --git a/src/core/memory.h b/src/core/memory.h
index 348fee3c9..eb5b6d69c 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -283,6 +283,12 @@ enum class FlushMode {
     FlushAndInvalidate,
 };
 
+/**
+ * Flushes and invalidates all memory in the rasterizer cache and removes any leftover state
+ * If flush is true, the rasterizer should flush any cached resources to RAM before clearing
+ */
+void RasterizerClearAll(bool flush);
+
 /**
  * Flushes and invalidates any externally cached rasterizer resources touching the given virtual
  * address region.
diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h
index 468d84084..a2510292e 100644
--- a/src/video_core/rasterizer_interface.h
+++ b/src/video_core/rasterizer_interface.h
@@ -55,6 +55,9 @@ public:
     /// and invalidated
     virtual void FlushAndInvalidateRegion(PAddr addr, u32 size) = 0;
 
+    /// Removes as much state as possible from the rasterizer in preparation for a save/load state
+    virtual void ClearAll(bool flush) = 0;
+
     /// Attempt to use a faster method to perform a display transfer with is_texture_copy = 0
     virtual bool AccelerateDisplayTransfer(const GPU::Regs::DisplayTransferConfig& config) {
         return false;
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 862dcac00..34bac4f87 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -1367,6 +1367,10 @@ void RasterizerOpenGL::FlushAndInvalidateRegion(PAddr addr, u32 size) {
     res_cache.InvalidateRegion(addr, size, nullptr);
 }
 
+void RasterizerOpenGL::ClearAll(bool flush) {
+    res_cache.ClearAll(flush);
+}
+
 bool RasterizerOpenGL::AccelerateDisplayTransfer(const GPU::Regs::DisplayTransferConfig& config) {
     MICROPROFILE_SCOPE(OpenGL_Blits);
 
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h
index b3356a69b..92cca2e4e 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.h
+++ b/src/video_core/renderer_opengl/gl_rasterizer.h
@@ -53,6 +53,7 @@ public:
     void FlushRegion(PAddr addr, u32 size) override;
     void InvalidateRegion(PAddr addr, u32 size) override;
     void FlushAndInvalidateRegion(PAddr addr, u32 size) override;
+    void ClearAll(bool flush) override;
     bool AccelerateDisplayTransfer(const GPU::Regs::DisplayTransferConfig& config) override;
     bool AccelerateTextureCopy(const GPU::Regs::DisplayTransferConfig& config) override;
     bool AccelerateFill(const GPU::Regs::MemoryFillConfig& config) override;
diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
index af2a918a6..4dcc64832 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
@@ -1276,9 +1276,7 @@ void main() {
 }
 
 RasterizerCacheOpenGL::~RasterizerCacheOpenGL() {
-    FlushAll();
-    while (!surface_cache.empty())
-        UnregisterSurface(*surface_cache.begin()->second.begin());
+    ClearAll(false);
 }
 
 MICROPROFILE_DEFINE(OpenGL_BlitSurface, "OpenGL", "BlitSurface", MP_RGB(128, 192, 64));
@@ -1927,6 +1925,31 @@ void RasterizerCacheOpenGL::ValidateSurface(const Surface& surface, PAddr addr,
     }
 }
 
+void RasterizerCacheOpenGL::ClearAll(bool flush) {
+    const SurfaceInterval flush_interval(0x0, 0xFFFFFFFF);
+    // Force flush all surfaces from the cache
+    if (flush) {
+        FlushRegion(0x0, 0xFFFFFFFF);
+    }
+    // Unmark all of the marked pages
+    for (auto& pair : RangeFromInterval(cached_pages, flush_interval)) {
+        const auto interval = pair.first & flush_interval;
+        const int count = pair.second;
+
+        const PAddr interval_start_addr = boost::icl::first(interval) << Memory::PAGE_BITS;
+        const PAddr interval_end_addr = boost::icl::last_next(interval) << Memory::PAGE_BITS;
+        const u32 interval_size = interval_end_addr - interval_start_addr;
+
+        VideoCore::g_memory->RasterizerMarkRegionCached(interval_start_addr, interval_size, false);
+    }
+
+    // Remove the whole cache without really looking at it.
+    cached_pages -= flush_interval;
+    dirty_regions -= flush_interval;
+    surface_cache -= flush_interval;
+    remove_surfaces.clear();
+}
+
 void RasterizerCacheOpenGL::FlushRegion(PAddr addr, u32 size, Surface flush_surface) {
     if (size == 0)
         return;
diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.h b/src/video_core/renderer_opengl/gl_rasterizer_cache.h
index cd601ef29..69322f713 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer_cache.h
+++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.h
@@ -482,6 +482,9 @@ public:
     /// Flush all cached resources tracked by this cache manager
     void FlushAll();
 
+    /// Clear all cached resources tracked by this cache manager
+    void ClearAll(bool flush);
+
 private:
     void DuplicateSurface(const Surface& src_surface, const Surface& dest_surface);
 
diff --git a/src/video_core/swrasterizer/swrasterizer.h b/src/video_core/swrasterizer/swrasterizer.h
index e2292f4a4..9e7a140f1 100644
--- a/src/video_core/swrasterizer/swrasterizer.h
+++ b/src/video_core/swrasterizer/swrasterizer.h
@@ -22,6 +22,7 @@ class SWRasterizer : public RasterizerInterface {
     void FlushRegion(PAddr addr, u32 size) override {}
     void InvalidateRegion(PAddr addr, u32 size) override {}
     void FlushAndInvalidateRegion(PAddr addr, u32 size) override {}
+    void ClearAll(bool flush) override {}
 };
 
 } // namespace VideoCore