diff --git a/src/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs b/src/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs
index a0b9f57bd9..2465efb0f9 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/AutoDeleteCache.cs
@@ -1,5 +1,4 @@
 using System.Collections;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
 
 namespace Ryujinx.Graphics.Gpu.Image
@@ -9,6 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Image
     /// </summary>
     class ShortTextureCacheEntry
     {
+        public bool IsAutoDelete;
         public readonly TextureDescriptor Descriptor;
         public readonly int InvalidatedSequence;
         public readonly Texture Texture;
@@ -24,6 +24,17 @@ namespace Ryujinx.Graphics.Gpu.Image
             InvalidatedSequence = texture.InvalidatedSequence;
             Texture = texture;
         }
+
+        /// <summary>
+        /// Create a new entry on the short duration texture cache from the auto delete cache.
+        /// </summary>
+        /// <param name="texture">The texture</param>
+        public ShortTextureCacheEntry(Texture texture)
+        {
+            IsAutoDelete = true;
+            InvalidatedSequence = texture.InvalidatedSequence;
+            Texture = texture;
+        }
     }
 
     /// <summary>
@@ -199,7 +210,11 @@ namespace Ryujinx.Graphics.Gpu.Image
             {
                 texture.DecrementReferenceCount();
 
-                _shortCacheLookup.Remove(texture.ShortCacheEntry.Descriptor);
+                if (!texture.ShortCacheEntry.IsAutoDelete)
+                {
+                    _shortCacheLookup.Remove(texture.ShortCacheEntry.Descriptor);
+                }
+
                 texture.ShortCacheEntry = null;
             }
         }
@@ -222,6 +237,25 @@ namespace Ryujinx.Graphics.Gpu.Image
             texture.IncrementReferenceCount();
         }
 
+        /// <summary>
+        /// Adds a texture to the short duration cache without a descriptor. This typically keeps it alive for two ticks.
+        /// On expiry, it will be removed from the AutoDeleteCache.
+        /// </summary>
+        /// <param name="texture">Texture to add to the short cache</param>
+        public void AddShortCache(Texture texture)
+        {
+            if (texture.ShortCacheEntry != null)
+            {
+                var entry = new ShortTextureCacheEntry(texture);
+
+                _shortCacheBuilder.Add(entry);
+
+                texture.ShortCacheEntry = entry;
+
+                texture.IncrementReferenceCount();
+            }
+        }
+
         /// <summary>
         /// Delete textures from the short duration cache.
         /// Moves the builder set to be deleted on next process.
@@ -234,7 +268,15 @@ namespace Ryujinx.Graphics.Gpu.Image
             {
                 entry.Texture.DecrementReferenceCount();
 
-                _shortCacheLookup.Remove(entry.Descriptor);
+                if (entry.IsAutoDelete)
+                {
+                    Remove(entry.Texture, false);
+                }
+                else
+                {
+                    _shortCacheLookup.Remove(entry.Descriptor);
+                }
+
                 entry.Texture.ShortCacheEntry = null;
             }
 
diff --git a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
index 3b25798894..6c9de8d6a8 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/Texture.cs
@@ -144,6 +144,11 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         public ShortTextureCacheEntry ShortCacheEntry { get; set; }
 
+        /// <summary>
+        /// Whether this texture has ever been referenced by a pool.
+        /// </summary>
+        public bool HadPoolOwner { get; private set; }
+
         /// Physical memory ranges where the texture data is located.
         /// </summary>
         public MultiRange Range { get; private set; }
@@ -1506,10 +1511,13 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="gpuVa">GPU VA of the pool reference</param>
         public void IncrementReferenceCount(TexturePool pool, int id, ulong gpuVa)
         {
+            HadPoolOwner = true;
+
             lock (_poolOwners)
             {
                 _poolOwners.Add(new TexturePoolOwner { Pool = pool, ID = id, GpuAddress = gpuVa });
             }
+
             _referenceCount++;
 
             if (ShortCacheEntry != null)
@@ -1594,7 +1602,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                 _poolOwners.Clear();
             }
 
-            if (ShortCacheEntry != null && _context.IsGpuThread())
+            if (ShortCacheEntry != null && !ShortCacheEntry.IsAutoDelete && _context.IsGpuThread())
             {
                 // If this is called from another thread (unmapped), the short cache will
                 // have to remove this texture on a future tick.
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureCache.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
index 97d78a3496..fc2c07e559 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/TextureCache.cs
@@ -848,7 +848,17 @@ namespace Ryujinx.Graphics.Gpu.Image
 
                             if (overlapInCache)
                             {
-                                _cache.Remove(overlap, flush);
+                                if (flush || overlap.HadPoolOwner || overlap.IsView)
+                                {
+                                    _cache.Remove(overlap, flush);
+                                }
+                                else
+                                {
+                                    // This texture has only ever been referenced in the AutoDeleteCache.
+                                    // Keep this texture alive with the short duration cache, as it may be used often but not sampled.
+
+                                    _cache.AddShortCache(overlap);
+                                }
                             }
 
                             removeOverlap = modified;
@@ -1198,6 +1208,16 @@ namespace Ryujinx.Graphics.Gpu.Image
             _cache.AddShortCache(texture, ref descriptor);
         }
 
+        /// <summary>
+        /// Adds a texture to the short duration cache without a descriptor. This typically keeps it alive for two ticks.
+        /// On expiry, it will be removed from the AutoDeleteCache.
+        /// </summary>
+        /// <param name="texture">Texture to add to the short cache</param>
+        public void AddShortCache(Texture texture)
+        {
+            _cache.AddShortCache(texture);
+        }
+
         /// <summary>
         /// Removes a texture from the short duration cache.
         /// </summary>