R/Ryujinx.Memory/Tracking/MultiRegionHandle.cs

using System;
using System.Collections.Generic;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Threading;

namespace Ryujinx.Memory.Tracking
{
    /// <summary>
    /// A region handle that tracks a large region using many smaller handles, to provide
    /// granular tracking that can be used to track partial updates. Backed by a bitmap
    /// to improve performance when scanning large regions.
    /// </summary>
    public class MultiRegionHandle : IMultiRegionHandle
    {
        /// <summary>
        /// A list of region handles for each granularity sized chunk of the whole region.
        /// </summary>
        private readonly RegionHandle[] _handles;
        private readonly ulong Address;
        private readonly ulong Granularity;
        private readonly ulong Size;

        private ConcurrentBitmap _dirtyBitmap;

        private int _sequenceNumber;
        private BitMap _sequenceNumberBitmap;
        private int _uncheckedHandles;

        public bool Dirty { get; private set; } = true;

        internal MultiRegionHandle(MemoryTracking tracking, ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity)
        {
            _handles = new RegionHandle[size / granularity];
            Granularity = granularity;

            _dirtyBitmap = new ConcurrentBitmap(_handles.Length, true);
            _sequenceNumberBitmap = new BitMap(_handles.Length);

            int i = 0;

            if (handles != null)
            {
                // Inherit from the handles we were given. Any gaps must be filled with new handles,
                // and old handles larger than our granularity must copy their state onto new granular handles and dispose.
                // It is assumed that the provided handles do not overlap, in order, are on page boundaries,
                // and don't extend past the requested range.

                foreach (RegionHandle handle in handles)
                {
                    int startIndex = (int)((handle.Address - address) / granularity);

                    // Fill any gap left before this handle.
                    while (i < startIndex)
                    {
                        RegionHandle fillHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
                        fillHandle.Parent = this;
                        _handles[i++] = fillHandle;
                    }

                    lock (tracking.TrackingLock)
                    {
                        if (handle is RegionHandle bitHandle && handle.Size == granularity)
                        {
                            handle.Parent = this;

                            bitHandle.ReplaceBitmap(_dirtyBitmap, i);

                            _handles[i++] = bitHandle;
                        }
                        else
                        {
                            int endIndex = (int)((handle.EndAddress - address) / granularity);

                            while (i < endIndex)
                            {
                                RegionHandle splitHandle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
                                splitHandle.Parent = this;

                                splitHandle.Reprotect(handle.Dirty);

                                RegionSignal signal = handle.PreAction;
                                if (signal != null)
                                {
                                    splitHandle.RegisterAction(signal);
                                }

                                _handles[i++] = splitHandle;
                            }

                            handle.Dispose();
                        }
                    }
                }
            }

            // Fill any remaining space with new handles.
            while (i < _handles.Length)
            {
                RegionHandle handle = tracking.BeginTrackingBitmap(address + (ulong)i * granularity, granularity, _dirtyBitmap, i);
                handle.Parent = this;
                _handles[i++] = handle;
            }

            _uncheckedHandles = _handles.Length;

            Address = address;
            Size = size;
        }

        public void SignalWrite()
        {
            Dirty = true;
        }

        public IEnumerable<RegionHandle> GetHandles()
        {
            return _handles;
        }

        public void ForceDirty(ulong address, ulong size)
        {
            Dirty = true;

            int startHandle = (int)((address - Address) / Granularity);
            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);

            for (int i = startHandle; i <= lastHandle; i++)
            {
                if (_sequenceNumberBitmap.Clear(i))
                {
                    _uncheckedHandles++;
                }

                _handles[i].ForceDirty();
            }
        }

        public void QueryModified(Action<ulong, ulong> modifiedAction)
        {
            if (!Dirty)
            {
                return;
            }

            Dirty = false;

            QueryModified(Address, Size, modifiedAction);
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private void ParseDirtyBits(long dirtyBits, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
        {
            while (dirtyBits != 0)
            {
                int bit = BitOperations.TrailingZeroCount(dirtyBits);

                dirtyBits &= ~(1L << bit);

                int handleIndex = baseBit + bit;

                RegionHandle handle = _handles[handleIndex];

                if (handleIndex != prevHandle + 1)
                {
                    // Submit handles scanned until the gap as dirty
                    if (rgSize != 0)
                    {
                        modifiedAction(rgStart, rgSize);
                        rgSize = 0;
                    }
                    rgStart = handle.Address;
                }

                if (handle.Dirty)
                {
                    rgSize += handle.Size;
                    handle.Reprotect();
                }

                prevHandle = handleIndex;
            }

            baseBit += ConcurrentBitmap.IntSize;
        }

        public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction)
        {
            int startHandle = (int)((address - Address) / Granularity);
            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);

            ulong rgStart = _handles[startHandle].Address;

            if (startHandle == lastHandle)
            {
                RegionHandle handle = _handles[startHandle];

                if (handle.Dirty)
                {
                    handle.Reprotect();
                    modifiedAction(rgStart, handle.Size);
                }

                return;
            }

            ulong rgSize = 0;

            long[] masks = _dirtyBitmap.Masks;

            int startIndex = startHandle >> ConcurrentBitmap.IntShift;
            int startBit = startHandle & ConcurrentBitmap.IntMask;
            long startMask = -1L << startBit;

            int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
            int endBit = lastHandle & ConcurrentBitmap.IntMask;
            long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));

            long startValue = Volatile.Read(ref masks[startIndex]);

            int baseBit = startIndex << ConcurrentBitmap.IntShift;
            int prevHandle = startHandle - 1;

            if (startIndex == endIndex)
            {
                ParseDirtyBits(startValue & startMask & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
            }
            else
            {
                ParseDirtyBits(startValue & startMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);

                for (int i = startIndex + 1; i < endIndex; i++)
                {
                    ParseDirtyBits(Volatile.Read(ref masks[i]), ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
                }

                long endValue = Volatile.Read(ref masks[endIndex]);

                ParseDirtyBits(endValue & endMask, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
            }

            if (rgSize != 0)
            {
                modifiedAction(rgStart, rgSize);
            }
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        private void ParseDirtyBits(long dirtyBits, long mask, int index, long[] seqMasks, ref int baseBit, ref int prevHandle, ref ulong rgStart, ref ulong rgSize, Action<ulong, ulong> modifiedAction)
        {
            long seqMask = mask & ~seqMasks[index];
            dirtyBits &= seqMask;

            while (dirtyBits != 0)
            {
                int bit = BitOperations.TrailingZeroCount(dirtyBits);

                dirtyBits &= ~(1L << bit);

                int handleIndex = baseBit + bit;

                RegionHandle handle = _handles[handleIndex];

                if (handleIndex != prevHandle + 1)
                {
                    // Submit handles scanned until the gap as dirty
                    if (rgSize != 0)
                    {
                        modifiedAction(rgStart, rgSize);
                        rgSize = 0;
                    }
                    rgStart = handle.Address;
                }

                rgSize += handle.Size;
                handle.Reprotect();

                prevHandle = handleIndex;
            }

            seqMasks[index] |= mask;
            _uncheckedHandles -= BitOperations.PopCount((ulong)seqMask);

            baseBit += ConcurrentBitmap.IntSize;
        }

        public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber)
        {
            int startHandle = (int)((address - Address) / Granularity);
            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);

            ulong rgStart = Address + (ulong)startHandle * Granularity;

            if (sequenceNumber != _sequenceNumber)
            {
                if (_uncheckedHandles != _handles.Length)
                {
                    _sequenceNumberBitmap.Clear();
                    _uncheckedHandles = _handles.Length;
                }

                _sequenceNumber = sequenceNumber;
            }

            if (startHandle == lastHandle)
            {
                var handle = _handles[startHandle];
                if (_sequenceNumberBitmap.Set(startHandle))
                {
                    _uncheckedHandles--;

                    if (handle.DirtyOrVolatile())
                    {
                        handle.Reprotect();

                        modifiedAction(rgStart, handle.Size);
                    }
                }

                return;
            }

            if (_uncheckedHandles == 0)
            {
                return;
            }

            ulong rgSize = 0;

            long[] seqMasks = _sequenceNumberBitmap.Masks;
            long[] masks = _dirtyBitmap.Masks;

            int startIndex = startHandle >> ConcurrentBitmap.IntShift;
            int startBit = startHandle & ConcurrentBitmap.IntMask;
            long startMask = -1L << startBit;

            int endIndex = lastHandle >> ConcurrentBitmap.IntShift;
            int endBit = lastHandle & ConcurrentBitmap.IntMask;
            long endMask = (long)(ulong.MaxValue >> (ConcurrentBitmap.IntMask - endBit));

            long startValue = Volatile.Read(ref masks[startIndex]);

            int baseBit = startIndex << ConcurrentBitmap.IntShift;
            int prevHandle = startHandle - 1;

            if (startIndex == endIndex)
            {
                ParseDirtyBits(startValue, startMask & endMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
            }
            else
            {
                ParseDirtyBits(startValue, startMask, startIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);

                for (int i = startIndex + 1; i < endIndex; i++)
                {
                    ParseDirtyBits(Volatile.Read(ref masks[i]), -1L, i, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
                }

                long endValue = Volatile.Read(ref masks[endIndex]);

                ParseDirtyBits(endValue, endMask, endIndex, seqMasks, ref baseBit, ref prevHandle, ref rgStart, ref rgSize, modifiedAction);
            }

            if (rgSize != 0)
            {
                modifiedAction(rgStart, rgSize);
            }
        }

        public void RegisterAction(ulong address, ulong size, RegionSignal action)
        {
            int startHandle = (int)((address - Address) / Granularity);
            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);

            for (int i = startHandle; i <= lastHandle; i++)
            {
                _handles[i].RegisterAction(action);
            }
        }

        public void RegisterPreciseAction(ulong address, ulong size, PreciseRegionSignal action)
        {
            int startHandle = (int)((address - Address) / Granularity);
            int lastHandle = (int)((address + (size - 1) - Address) / Granularity);

            for (int i = startHandle; i <= lastHandle; i++)
            {
                _handles[i].RegisterPreciseAction(action);
            }
        }

        public void Dispose()
        {
            foreach (var handle in _handles)
            {
                handle.Dispose();
            }
        }
    }
}