diff --git a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
index 76125e31a0..0eaf012357 100644
--- a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
@@ -259,10 +259,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="size">The size of the region</param>
         public void ClearModified(ulong address, ulong size)
         {
-            if (_modifiedRanges != null)
-            {
-                _modifiedRanges.Clear(address, size);
-            }
+            _modifiedRanges?.Clear(address, size);
         }
 
         /// <summary>
@@ -275,7 +272,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
             if (_useGranular)
             {
-                _modifiedRanges.GetRanges(Address, Size, (address, size) =>
+                _modifiedRanges?.GetRanges(Address, Size, (address, size) =>
                 {
                     _memoryTrackingGranular.RegisterAction(address, size, _externalFlushDelegate);
                     SynchronizeMemory(address, size);
@@ -302,18 +299,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
                     _syncActionRegistered = true;
                 }
 
-                EnsureRangeList();
-                _modifiedRanges.InheritRanges(from._modifiedRanges, (ulong address, ulong size) =>
+                if (_modifiedRanges == null)
                 {
-                    if (_useGranular)
+                    _modifiedRanges = from._modifiedRanges;
+                    from._modifiedRanges = null;
+                }
+                else
+                {
+                    _modifiedRanges.InheritRanges(from._modifiedRanges, (ulong address, ulong size) =>
                     {
-                        _memoryTrackingGranular.RegisterAction(address, size, _externalFlushDelegate);
-                    }
-                    else
-                    {
-                        _memoryTracking.RegisterAction(_externalFlushDelegate);
-                    }
-                });
+                        if (_useGranular)
+                        {
+                            _memoryTrackingGranular.RegisterAction(address, size, _externalFlushDelegate);
+                        }
+                        else
+                        {
+                            _memoryTracking.RegisterAction(_externalFlushDelegate);
+                        }
+                    });
+                }
             }
         }
 
@@ -381,10 +385,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="mSize">Size of the region to force dirty</param>
         public void ForceDirty(ulong mAddress, ulong mSize)
         {
-            if (_modifiedRanges != null)
-            {
-                _modifiedRanges.Clear(mAddress, mSize);
-            }
+            _modifiedRanges?.Clear(mAddress, mSize);
 
             if (_useGranular)
             {
@@ -497,7 +498,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="size">Size of the unmapped region</param>
         public void Unmapped(ulong address, ulong size)
         {
-            _modifiedRanges?.Clear(address, size);
+            BufferModifiedRangeList modifiedRanges = _modifiedRanges;
+
+            modifiedRanges?.Clear(address, size);
 
             UnmappedSequence++;
         }
diff --git a/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs b/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
index 6322743161..84cf0dd8bb 100644
--- a/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
+++ b/Ryujinx.Graphics.Gpu/Memory/BufferModifiedRangeList.cs
@@ -60,6 +60,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
     /// </summary>
     class BufferModifiedRangeList : RangeList<BufferModifiedRange>
     {
+        private const int BackingInitialSize = 8;
+
         private GpuContext _context;
 
         private object _lock = new object();
@@ -68,7 +70,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// Creates a new instance of a modified range list.
         /// </summary>
         /// <param name="context">GPU context that the buffer range list belongs to</param>
-        public BufferModifiedRangeList(GpuContext context)
+        public BufferModifiedRangeList(GpuContext context) : base(BackingInitialSize)
         {
             _context = context;
         }
diff --git a/Ryujinx.Memory/Range/RangeList.cs b/Ryujinx.Memory/Range/RangeList.cs
index d82a05c5ef..7278e7eb44 100644
--- a/Ryujinx.Memory/Range/RangeList.cs
+++ b/Ryujinx.Memory/Range/RangeList.cs
@@ -33,18 +33,22 @@ namespace Ryujinx.Memory.Range
             }
         }
 
+        private const int BackingInitialSize = 1024;
         private const int ArrayGrowthSize = 32;
-        private const int BackingGrowthSize = 1024;
 
         private RangeItem<T>[] _items;
+        private readonly int _backingGrowthSize;
+
         public int Count { get; protected set; }
 
         /// <summary>
         /// Creates a new range list.
         /// </summary>
-        public RangeList()
+        /// <param name="backingInitialSize">The initial size of the backing array</param>
+        public RangeList(int backingInitialSize = BackingInitialSize)
         {
-            _items = new RangeItem<T>[BackingGrowthSize];
+            _backingGrowthSize = backingInitialSize;
+            _items = new RangeItem<T>[backingInitialSize];
         }
 
         /// <summary>
@@ -68,7 +72,7 @@ namespace Ryujinx.Memory.Range
         {
             if (Count + 1 > _items.Length)
             {
-                Array.Resize(ref _items, _items.Length + ArrayGrowthSize);
+                Array.Resize(ref _items, _items.Length + _backingGrowthSize);
             }
 
             if (index >= Count)