diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h
index 6d7d8226f..27c82cd20 100644
--- a/src/video_core/texture_cache/texture_cache.h
+++ b/src/video_core/texture_cache/texture_cache.h
@@ -42,8 +42,8 @@ TextureCache<P>::TextureCache(Runtime& runtime_, VideoCore::RasterizerInterface&
     // These values were chosen based on typical peak swizzle data sizes seen in some titles
     static constexpr size_t SWIZZLE_DATA_BUFFER_INITIAL_CAPACITY = 8_MiB;
     static constexpr size_t UNSWIZZLE_DATA_BUFFER_INITIAL_CAPACITY = 1_MiB;
-    swizzle_data_buffer.reserve(SWIZZLE_DATA_BUFFER_INITIAL_CAPACITY);
-    unswizzle_data_buffer.reserve(UNSWIZZLE_DATA_BUFFER_INITIAL_CAPACITY);
+    swizzle_data_buffer.resize_destructive(SWIZZLE_DATA_BUFFER_INITIAL_CAPACITY);
+    unswizzle_data_buffer.resize_destructive(UNSWIZZLE_DATA_BUFFER_INITIAL_CAPACITY);
 
     // Make sure the first index is reserved for the null resources
     // This way the null resource becomes a compile time constant
@@ -746,11 +746,11 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
         return;
     }
     const size_t guest_size_bytes = image.guest_size_bytes;
-    swizzle_data_buffer.resize(guest_size_bytes);
+    swizzle_data_buffer.resize_destructive(guest_size_bytes);
     gpu_memory->ReadBlockUnsafe(gpu_addr, swizzle_data_buffer.data(), guest_size_bytes);
 
     if (True(image.flags & ImageFlagBits::Converted)) {
-        unswizzle_data_buffer.resize(image.unswizzled_size_bytes);
+        unswizzle_data_buffer.resize_destructive(image.unswizzled_size_bytes);
         auto copies = UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data_buffer,
                                      unswizzle_data_buffer);
         ConvertImage(unswizzle_data_buffer, image.info, mapped_span, copies);
diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h
index 67e8acf25..4fd677a80 100644
--- a/src/video_core/texture_cache/texture_cache_base.h
+++ b/src/video_core/texture_cache/texture_cache_base.h
@@ -17,6 +17,7 @@
 #include "common/literals.h"
 #include "common/lru_cache.h"
 #include "common/polyfill_ranges.h"
+#include "common/scratch_buffer.h"
 #include "video_core/compatible_formats.h"
 #include "video_core/control/channel_state_cache.h"
 #include "video_core/delayed_destruction_ring.h"
@@ -417,8 +418,8 @@ private:
 
     std::unordered_map<GPUVAddr, ImageAllocId> image_allocs_table;
 
-    std::vector<u8> swizzle_data_buffer;
-    std::vector<u8> unswizzle_data_buffer;
+    Common::ScratchBuffer<u8> swizzle_data_buffer;
+    Common::ScratchBuffer<u8> unswizzle_data_buffer;
 
     u64 modification_tick = 0;
     u64 frame_tick = 0;
diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp
index 7999a7f06..03acc68d9 100644
--- a/src/video_core/texture_cache/util.cpp
+++ b/src/video_core/texture_cache/util.cpp
@@ -505,7 +505,7 @@ void SwizzlePitchLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr
 
 void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr,
                              const ImageInfo& info, const BufferImageCopy& copy,
-                             std::span<const u8> input, std::vector<u8>& tmp_buffer) {
+                             std::span<const u8> input, Common::ScratchBuffer<u8>& tmp_buffer) {
     const Extent3D size = info.size;
     const LevelInfo level_info = MakeLevelInfo(info);
     const Extent2D tile_size = DefaultBlockSize(info.format);
@@ -534,7 +534,7 @@ void SwizzleBlockLinearImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr
                        tile_size.height, info.tile_width_spacing);
     const size_t subresource_size = sizes[level];
 
-    tmp_buffer.resize(subresource_size);
+    tmp_buffer.resize_destructive(subresource_size);
     const std::span<u8> dst(tmp_buffer);
 
     for (s32 layer = 0; layer < info.resources.layers; ++layer) {
@@ -978,7 +978,7 @@ std::vector<SwizzleParameters> FullUploadSwizzles(const ImageInfo& info) {
 
 void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
                   std::span<const BufferImageCopy> copies, std::span<const u8> memory,
-                  std::vector<u8>& tmp_buffer) {
+                  Common::ScratchBuffer<u8>& tmp_buffer) {
     const bool is_pitch_linear = info.type == ImageType::Linear;
     for (const BufferImageCopy& copy : copies) {
         if (is_pitch_linear) {
diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h
index 2c991b4d2..d103db8ae 100644
--- a/src/video_core/texture_cache/util.h
+++ b/src/video_core/texture_cache/util.h
@@ -7,6 +7,7 @@
 #include <span>
 
 #include "common/common_types.h"
+#include "common/scratch_buffer.h"
 
 #include "video_core/surface.h"
 #include "video_core/texture_cache/image_base.h"
@@ -78,7 +79,7 @@ void ConvertImage(std::span<const u8> input, const ImageInfo& info, std::span<u8
 
 void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const ImageInfo& info,
                   std::span<const BufferImageCopy> copies, std::span<const u8> memory,
-                  std::vector<u8>& tmp_buffer);
+                  Common::ScratchBuffer<u8>& tmp_buffer);
 
 [[nodiscard]] bool IsBlockLinearSizeCompatible(const ImageInfo& new_info,
                                                const ImageInfo& overlap_info, u32 new_level,