diff --git a/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs b/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs
index febabdad84..726b97ea17 100644
--- a/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs
+++ b/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs
@@ -43,7 +43,7 @@ namespace Ryujinx.Graphics.Gpu.Image
 
                 oldestTexture.SynchronizeMemory();
 
-                if (oldestTexture.IsModified && !oldestTexture.ConsumeModified())
+                if (oldestTexture.IsModified && !oldestTexture.CheckModified(true))
                 {
                     // The texture must be flushed if it falls out of the auto delete cache.
                     // Flushes out of the auto delete cache do not trigger write tracking,
diff --git a/Ryujinx.Graphics.Gpu/Image/Texture.cs b/Ryujinx.Graphics.Gpu/Image/Texture.cs
index e156ff5eda..0a083ebc33 100644
--- a/Ryujinx.Graphics.Gpu/Image/Texture.cs
+++ b/Ryujinx.Graphics.Gpu/Image/Texture.cs
@@ -252,7 +252,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 if (!isView)
                 {
                     // Don't update this texture the next time we synchronize.
-                    ConsumeModified();
+                    CheckModified(true);
 
                     if (ScaleMode == TextureScaleMode.Scaled)
                     {
@@ -599,12 +599,13 @@ namespace Ryujinx.Graphics.Gpu.Image
 
         /// <summary>
         /// Checks if the memory for this texture was modified, and returns true if it was.
-        /// The modified flags are consumed as a result.
+        /// The modified flags are optionally consumed as a result.
         /// </summary>
+        /// <param name="consume">True to consume the dirty flags and reprotect, false to leave them as is</param>
         /// <returns>True if the texture was modified, false otherwise.</returns>
-        public bool ConsumeModified()
+        public bool CheckModified(bool consume)
         {
-            return Group.ConsumeDirty(this);
+            return Group.CheckDirty(this, consume);
         }
 
         /// <summary>
@@ -634,7 +635,7 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
             else
             {
-                Group.ConsumeDirty(this);
+                Group.CheckDirty(this, true);
                 SynchronizeFull();
             }
         }
@@ -698,7 +699,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             BlacklistScale();
 
-            Group.ConsumeDirty(this);
+            Group.CheckDirty(this, true);
 
             IsModified = false;
 
diff --git a/Ryujinx.Graphics.Gpu/Image/TextureCache.cs b/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
index 58cd3a2f7c..37682b655f 100644
--- a/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
+++ b/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
@@ -623,29 +623,49 @@ namespace Ryujinx.Graphics.Gpu.Image
                         hasLayerViews |= overlap.Info.GetSlices() < texture.Info.GetSlices();
                         hasMipViews |= overlap.Info.Levels < texture.Info.Levels;
                     }
-                    else if (overlapInCache || !setData)
+                    else
                     {
-                        if (info.GobBlocksInZ > 1 && info.GobBlocksInZ == overlap.Info.GobBlocksInZ)
+                        bool removeOverlap;
+                        bool modified = overlap.CheckModified(false);
+
+                        if (overlapInCache || !setData)
                         {
-                            // Allow overlapping slices of 3D textures. Could be improved in future by making sure the textures don't overlap.
-                            continue;
+                            if (info.GobBlocksInZ > 1 && info.GobBlocksInZ == overlap.Info.GobBlocksInZ)
+                            {
+                                // Allow overlapping slices of 3D textures. Could be improved in future by making sure the textures don't overlap.
+                                continue;
+                            }
+
+                            // The overlap texture is going to contain garbage data after we draw, or is generally incompatible.
+                            // If the texture cannot be entirely contained in the new address space, and one of its view children is compatible with us,
+                            // it must be flushed before removal, so that the data is not lost.
+
+                            // If the texture was modified since its last use, then that data is probably meant to go into this texture.
+                            // If the data has been modified by the CPU, then it also shouldn't be flushed.
+
+                            bool viewCompatibleChild = overlap.HasViewCompatibleChild(texture);
+
+                            bool flush = overlapInCache && !modified && !texture.Range.Contains(overlap.Range) && viewCompatibleChild;
+
+                            setData |= modified || flush;
+
+                            if (overlapInCache)
+                            {
+                                _cache.Remove(overlap, flush);
+                            }
+
+                            removeOverlap = modified && !viewCompatibleChild;
+                        }
+                        else
+                        {
+                            // If an incompatible overlapping texture has been modified, then it's data is likely destined for this texture,
+                            // and the overlapped texture will contain garbage. In this case, it should be removed to save memory.
+                            removeOverlap = modified;
                         }
 
-                        // The overlap texture is going to contain garbage data after we draw, or is generally incompatible.
-                        // If the texture cannot be entirely contained in the new address space, and one of its view children is compatible with us,
-                        // it must be flushed before removal, so that the data is not lost.
-
-                        // If the texture was modified since its last use, then that data is probably meant to go into this texture.
-                        // If the data has been modified by the CPU, then it also shouldn't be flushed.
-                        bool modified = overlap.ConsumeModified();
-
-                        bool flush = overlapInCache && !modified && !texture.Range.Contains(overlap.Range) && overlap.HasViewCompatibleChild(texture);
-
-                        setData |= modified || flush;
-
-                        if (overlapInCache)
+                        if (removeOverlap && overlap.Info.Target != Target.TextureBuffer)
                         {
-                            _cache.Remove(overlap, flush);
+                            overlap.RemoveFromPools(false);
                         }
                     }
                 }
diff --git a/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs b/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
index c4be1cecab..1fe0bbf7a7 100644
--- a/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
+++ b/Ryujinx.Graphics.Gpu/Image/TextureGroup.cs
@@ -83,11 +83,13 @@ namespace Ryujinx.Graphics.Gpu.Image
         }
 
         /// <summary>
-        /// Consume the dirty flags for a given texture. The state is shared between views of the same layers and levels.
+        /// Check and optionally consume the dirty flags for a given texture.
+        /// The state is shared between views of the same layers and levels.
         /// </summary>
         /// <param name="texture">The texture being used</param>
+        /// <param name="consume">True to consume the dirty flags and reprotect, false to leave them as is</param>
         /// <returns>True if a flag was dirty, false otherwise</returns>
-        public bool ConsumeDirty(Texture texture)
+        public bool CheckDirty(Texture texture, bool consume)
         {
             bool dirty = false;
 
@@ -101,7 +103,11 @@ namespace Ryujinx.Graphics.Gpu.Image
                     {
                         if (handle.Dirty)
                         {
-                            handle.Reprotect();
+                            if (consume)
+                            {
+                                handle.Reprotect();
+                            }
+
                             dirty = true;
                         }
                     }