Add inlined on translation call counting (#2190)

* Add EntryTable<TEntry>

* Add on translation call counting

* Add Counter

* Add PPTC support

* Make Counter a generic & use a 32-bit counter instead

* Return false on overflow

* Set PPTC version

* Print more information about the rejit queue

* Make Counter<T> disposable

* Remove Block.TailCall since it is not used anymore

* Apply suggestions from code review

Address gdkchan's feedback

Co-authored-by: gdkchan <gab.dark.100@gmail.com>

* Fix more stale docs

* Remove rejit requests queue logging

* Make Counter<T> finalizable

Most certainly quite an odd use case.

* Make EntryTable<T>.TryAllocate set entry to default

* Re-trigger CI

* Dispose Counters before they hit the finalizer queue

* Re-trigger CI

Just for good measure...

* Make EntryTable<T> expandable

* EntryTable is now expandable instead of being a fixed slab.
* Remove EntryTable<T>.TryAllocate
* Remove Counter<T>.TryCreate

Address LDj3SNuD's feedback

* Apply suggestions from code review

Address LDj3SNuD's feedback

Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com>

* Remove useless return

* POH approach, but the sequel

* Revert "POH approach, but the sequel"

This reverts commit 5f5abaa247.

The sequel got shelved

* Add extra documentation

Co-authored-by: gdkchan <gab.dark.100@gmail.com>
Co-authored-by: LDj3SNuD <35856442+LDj3SNuD@users.noreply.github.com>

ARMeilleure/Common/Counter.cs

@@ -0,0 +1,99 @@
+using System;
+
+namespace ARMeilleure.Common
+{
+    /// <summary>
+    /// Represents a numeric counter which can be used for instrumentation of compiled code.
+    /// </summary>
+    /// <typeparam name="T">Type of the counter</typeparam>
+    class Counter<T> : IDisposable where T : unmanaged
+    {
+        private bool _disposed;
+        private readonly int _index;
+        private readonly EntryTable<T> _countTable;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Counter{T}"/> class from the specified
+        /// <see cref="EntryTable{T}"/> instance and index.
+        /// </summary>
+        /// <param name="countTable"><see cref="EntryTable{T}"/> instance</param>
+        /// <param name="index">Index in the <see cref="EntryTable{T}"/></param>
+        /// <exception cref="ArgumentNullException"><paramref name="countTable"/> is <see langword="null"/></exception>
+        /// <exception cref="ArgumentException"><typeparamref name="T"/> is unsupported</exception>
+        public Counter(EntryTable<T> countTable)
+        {
+            if (typeof(T) != typeof(byte) && typeof(T) != typeof(sbyte) &&
+                typeof(T) != typeof(short) && typeof(T) != typeof(ushort) &&
+                typeof(T) != typeof(int) && typeof(T) != typeof(uint) &&
+                typeof(T) != typeof(long) && typeof(T) != typeof(ulong) &&
+                typeof(T) != typeof(nint) && typeof(T) != typeof(nuint) &&
+                typeof(T) != typeof(float) && typeof(T) != typeof(double))
+            {
+                throw new ArgumentException("Counter does not support the specified type.");
+            }
+
+            _countTable = countTable ?? throw new ArgumentNullException(nameof(countTable));
+            _index = countTable.Allocate();
+        }
+
+        /// <summary>
+        /// Gets a reference to the value of the counter.
+        /// </summary>
+        /// <exception cref="ObjectDisposedException"><see cref="Counter{T}"/> instance was disposed</exception>
+        /// <remarks>
+        /// This can refer to freed memory if the owning <see cref="EntryTable{TEntry}"/> is disposed.
+        /// </remarks>
+        public ref T Value
+        {
+            get
+            {
+                if (_disposed)
+                {
+                    throw new ObjectDisposedException(null);
+                }
+
+                return ref _countTable.GetValue(_index);
+            }
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="Counter{T}"/> instance.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged and optionally managed resources used by the <see cref="Counter{T}"/> instance.
+        /// </summary>
+        /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                try
+                {
+                    // The index into the EntryTable is essentially an unmanaged resource since we allocate and free the
+                    // resource ourselves.
+                    _countTable.Free(_index);
+                }
+                catch (ObjectDisposedException)
+                {
+                    // Can happen because _countTable may be disposed before the Counter instance.
+                }
+
+                _disposed = true;
+            }
+        }
+
+        /// <summary>
+        /// Frees resources used by the <see cref="Counter{T}"/> instance.
+        /// </summary>
+        ~Counter()
+        {
+            Dispose(false);
+        }
+    }
+}

ARMeilleure/Common/EntryTable.cs

@@ -0,0 +1,196 @@
+using System;
+using System.Collections.Generic;
+using System.Numerics;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Common
+{
+    /// <summary>
+    /// Represents an expandable table of the type <typeparamref name="TEntry"/>, whose entries will remain at the same
+    /// address through out the table's lifetime.
+    /// </summary>
+    /// <typeparam name="TEntry">Type of the entry in the table</typeparam>
+    class EntryTable<TEntry> : IDisposable where TEntry : unmanaged
+    {
+        private bool _disposed;
+        private int _freeHint;
+        private readonly int _pageCapacity; // Number of entries per page.
+        private readonly int _pageLogCapacity;
+        private readonly Dictionary<int, IntPtr> _pages;
+        private readonly BitMap _allocated;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="EntryTable{TEntry}"/> class with the desired page size in
+        /// bytes.
+        /// </summary>
+        /// <param name="pageSize">Desired page size in bytes</param>
+        /// <exception cref="ArgumentOutOfRangeException"><paramref name="pageSize"/> is less than 0</exception>
+        /// <exception cref="ArgumentException"><typeparamref name="TEntry"/>'s size is zero</exception>
+        /// <remarks>
+        /// The actual page size may be smaller or larger depending on the size of <typeparamref name="TEntry"/>.
+        /// </remarks>
+        public unsafe EntryTable(int pageSize = 4096)
+        {
+            if (pageSize < 0)
+            {
+                throw new ArgumentOutOfRangeException(nameof(pageSize), "Page size cannot be negative.");
+            }
+
+            if (sizeof(TEntry) == 0)
+            {
+                throw new ArgumentException("Size of TEntry cannot be zero.");
+            }
+
+            _allocated = new BitMap();
+            _pages = new Dictionary<int, IntPtr>();
+            _pageLogCapacity = BitOperations.Log2((uint)(pageSize / sizeof(TEntry)));
+            _pageCapacity = 1 << _pageLogCapacity;
+        }
+
+        /// <summary>
+        /// Allocates an entry in the <see cref="EntryTable{TEntry}"/>.
+        /// </summary>
+        /// <returns>Index of entry allocated in the table</returns>
+        /// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
+        public int Allocate()
+        {
+            if (_disposed)
+            {
+                throw new ObjectDisposedException(null);
+            }
+
+            lock (_allocated)
+            {
+                if (_allocated.IsSet(_freeHint))
+                {
+                    _freeHint = _allocated.FindFirstUnset();
+                }
+
+                int index = _freeHint++;
+                var page = GetPage(index);
+
+                _allocated.Set(index);
+
+                GetValue(page, index) = default;
+
+                return index;
+            }
+        }
+
+        /// <summary>
+        /// Frees the entry at the specified <paramref name="index"/>.
+        /// </summary>
+        /// <param name="index">Index of entry to free</param>
+        /// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
+        public void Free(int index)
+        {
+            if (_disposed)
+            {
+                throw new ObjectDisposedException(null);
+            }
+
+            lock (_allocated)
+            {
+                if (_allocated.IsSet(index))
+                {
+                    _allocated.Clear(index);
+
+                    _freeHint = index;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets a reference to the entry at the specified allocated <paramref name="index"/>.
+        /// </summary>
+        /// <param name="index">Index of the entry</param>
+        /// <returns>Reference to the entry at the specified <paramref name="index"/></returns>
+        /// <exception cref="ObjectDisposedException"><see cref="EntryTable{TEntry}"/> instance was disposed</exception>
+        /// <exception cref="ArgumentException">Entry at <paramref name="index"/> is not allocated</exception>
+        public ref TEntry GetValue(int index)
+        {
+            if (_disposed)
+            {
+                throw new ObjectDisposedException(null);
+            }
+
+            lock (_allocated)
+            {
+                if (!_allocated.IsSet(index))
+                {
+                    throw new ArgumentException("Entry at the specified index was not allocated", nameof(index));
+                }
+
+                var page = GetPage(index);
+
+                return ref GetValue(page, index);
+            }
+        }
+
+        /// <summary>
+        /// Gets a reference to the entry at using the specified <paramref name="index"/> from the specified
+        /// <paramref name="page"/>.
+        /// </summary>
+        /// <param name="page">Page to use</param>
+        /// <param name="index">Index to use</param>
+        /// <returns>Reference to the entry</returns>
+        private ref TEntry GetValue(Span<TEntry> page, int index)
+        {
+            return ref page[index & (_pageCapacity - 1)];
+        }
+
+        /// <summary>
+        /// Gets the page for the specified <see cref="index"/>.
+        /// </summary>
+        /// <param name="index">Index to use</param>
+        /// <returns>Page for the specified <see cref="index"/></returns>
+        private unsafe Span<TEntry> GetPage(int index)
+        {
+            var pageIndex = (int)((uint)(index & ~(_pageCapacity - 1)) >> _pageLogCapacity);
+
+            if (!_pages.TryGetValue(pageIndex, out IntPtr page))
+            {
+                page = Marshal.AllocHGlobal(sizeof(TEntry) * _pageCapacity);
+
+                _pages.Add(pageIndex, page);
+            }
+
+            return new Span<TEntry>((void*)page, _pageCapacity);
+        }
+
+        /// <summary>
+        /// Releases all resources used by the <see cref="EntryTable{TEntry}"/> instance.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases all unmanaged and optionally managed resources used by the <see cref="EntryTable{TEntry}{T}"/>
+        /// instance.
+        /// </summary>
+        /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!_disposed)
+            {
+                foreach (var page in _pages.Values)
+                {
+                    Marshal.FreeHGlobal(page);
+                }
+
+                _disposed = true;
+            }
+        }
+
+        /// <summary>
+        /// Frees resources used by the <see cref="EntryTable{TEntry}"/> instance.
+        /// </summary>
+        ~EntryTable()
+        {
+            Dispose(false);
+        }
+    }
+}

ARMeilleure/Decoders/Block.cs

@@ -11,7 +11,6 @@ namespace ARMeilleure.Decoders
         public Block Next   { get; set; }
         public Block Branch { get; set; }
 
-        public bool TailCall { get; set; }
         public bool Exit { get; set; }
 
         public List<OpCode> OpCodes { get; }

ARMeilleure/Decoders/Optimizations/TailCallRemover.cs

@@ -58,7 +58,7 @@ namespace ARMeilleure.Decoders.Optimizations
                 return blocks.ToArray(); // Nothing to do here.
             }
             
-            // Mark branches outside of contiguous region as exit blocks.
+            // Mark branches whose target is outside of the contiguous region as an exit block.
             for (int i = startBlockIndex; i <= endBlockIndex; i++)
             {
                 Block block = blocks[i];
@@ -66,7 +66,6 @@ namespace ARMeilleure.Decoders.Optimizations
                 if (block.Branch != null && (block.Branch.Address > endBlock.EndAddress || block.Branch.EndAddress < startBlock.Address))
                 {
                     block.Branch.Exit = true;
-                    block.Branch.TailCall = true;
                 }
             }
 

ARMeilleure/Instructions/InstEmitFlowHelper.cs

@@ -200,7 +200,7 @@ namespace ARMeilleure.Instructions
             }
         }
 
-        public static void EmitTailContinue(ArmEmitterContext context, Operand address, bool allowRejit)
+        public static void EmitTailContinue(ArmEmitterContext context, Operand address)
         {
             // Left option here as it may be useful if we need to return to managed rather than tail call in future.
             // (eg. for debug)
@@ -218,9 +218,7 @@ namespace ARMeilleure.Instructions
                 {
                     context.StoreToContext();
 
-                    Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(allowRejit
-                        ? nameof(NativeInterface.GetFunctionAddress)
-                        : nameof(NativeInterface.GetFunctionAddressWithoutRejit)), address);
+                    Operand fallbackAddr = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)), address);
 
                     EmitNativeCall(context, fallbackAddr, isJump: true);
                 }

ARMeilleure/Instructions/NativeInterface.cs

@@ -220,6 +220,11 @@ namespace ARMeilleure.Instructions
         }
         #endregion
 
+        public static void EnqueueForRejit(ulong address)
+        {
+            Context.Translator.EnqueueForRejit(address, GetContext().ExecutionMode);
+        }
+
         public static void SignalMemoryTracking(ulong address, ulong size, bool write)
         {
             GetMemoryManager().SignalMemoryTracking(address, size, write);
@@ -232,24 +237,14 @@ namespace ARMeilleure.Instructions
 
         public static ulong GetFunctionAddress(ulong address)
         {
-            return GetFunctionAddressWithHint(address, true);
-        }
-
-        public static ulong GetFunctionAddressWithoutRejit(ulong address)
-        {
-            return GetFunctionAddressWithHint(address, false);
-        }
-
-        private static ulong GetFunctionAddressWithHint(ulong address, bool hintRejit)
-        {
-            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit);
+            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
 
             return (ulong)function.FuncPtr.ToInt64();
         }
 
         public static ulong GetIndirectFunctionAddress(ulong address, ulong entryAddress)
         {
-            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode, hintRejit: true);
+            TranslatedFunction function = Context.Translator.GetOrTranslate(address, GetContext().ExecutionMode);
 
             ulong ptr = (ulong)function.FuncPtr.ToInt64();
 

ARMeilleure/IntermediateRepresentation/OperandHelper.cs

@@ -1,4 +1,5 @@
 using ARMeilleure.Common;
+using System.Runtime.CompilerServices;
 
 namespace ARMeilleure.IntermediateRepresentation
 {
@@ -34,6 +35,11 @@ namespace ARMeilleure.IntermediateRepresentation
             return Operand().With(value);
         }
 
+        public static unsafe Operand Const<T>(ref T reference, int? index = null)
+        {
+            return Operand().With((long)Unsafe.AsPointer(ref reference), index != null, index);
+        }
+
         public static Operand ConstF(float value)
         {
             return Operand().With(value);

ARMeilleure/Translation/ArmEmitterContext.cs

@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
 using ARMeilleure.Decoders;
 using ARMeilleure.Instructions;
 using ARMeilleure.IntermediateRepresentation;
@@ -41,15 +42,23 @@ namespace ARMeilleure.Translation
         public IMemoryManager Memory { get; }
 
         public JumpTable JumpTable { get; }
+        public EntryTable<uint> CountTable { get; }
 
         public ulong EntryAddress { get; }
         public bool HighCq { get; }
         public Aarch32Mode Mode { get; }
 
-        public ArmEmitterContext(IMemoryManager memory, JumpTable jumpTable, ulong entryAddress, bool highCq, Aarch32Mode mode)
+        public ArmEmitterContext(
+            IMemoryManager memory,
+            JumpTable jumpTable,
+            EntryTable<uint> countTable,
+            ulong entryAddress,
+            bool highCq,
+            Aarch32Mode mode)
         {
             Memory = memory;
             JumpTable = jumpTable;
+            CountTable = countTable;
             EntryAddress = entryAddress;
             HighCq = highCq;
             Mode = mode;

ARMeilleure/Translation/Delegates.cs

@@ -103,6 +103,7 @@ namespace ARMeilleure.Translation
 
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.Break)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
+            SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntfrqEl0)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntpctEl0)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetCntvctEl0)));
@@ -113,7 +114,6 @@ namespace ARMeilleure.Translation
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpscr))); // A32 only.
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpsr)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddress)));
-            SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFunctionAddressWithoutRejit)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetIndirectFunctionAddress)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr)));
             SetDelegateInfo(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetTpidr32))); // A32 only.

ARMeilleure/Translation/PTC/Ptc.cs

@@ -1,6 +1,7 @@
 using ARMeilleure.CodeGen;
 using ARMeilleure.CodeGen.Unwinding;
 using ARMeilleure.CodeGen.X86;
+using ARMeilleure.Common;
 using ARMeilleure.Memory;
 using ARMeilleure.Translation.Cache;
 using Ryujinx.Common;
@@ -27,7 +28,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 2169; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 2190; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";
@@ -38,6 +39,7 @@ namespace ARMeilleure.Translation.PTC
         internal const int PageTablePointerIndex = -1; // Must be a negative value.
         internal const int JumpPointerIndex = -2; // Must be a negative value.
         internal const int DynamicPointerIndex = -3; // Must be a negative value.
+        internal const int CountTableIndex = -4; // Must be a negative value.
 
         private const byte FillingByte = 0x00;
         private const CompressionLevel SaveCompressionLevel = CompressionLevel.Fastest;
@@ -538,7 +540,11 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        internal static void LoadTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable)
+        internal static void LoadTranslations(
+            ConcurrentDictionary<ulong, TranslatedFunction> funcs,
+            IMemoryManager memory,
+            JumpTable jumpTable,
+            EntryTable<uint> countTable)
         {
             if (AreCarriersEmpty())
             {
@@ -567,16 +573,18 @@ namespace ARMeilleure.Translation.PTC
                     {
                         byte[] code = ReadCode(index, infoEntry.CodeLength);
 
+                        Counter<uint> callCounter = null;
+
                         if (infoEntry.RelocEntriesCount != 0)
                         {
                             RelocEntry[] relocEntries = GetRelocEntries(relocsReader, infoEntry.RelocEntriesCount);
 
-                            PatchCode(code.AsSpan(), relocEntries, memory.PageTablePointer, jumpTable);
+                            PatchCode(code, relocEntries, memory.PageTablePointer, jumpTable, countTable, out callCounter);
                         }
 
                         UnwindInfo unwindInfo = ReadUnwindInfo(unwindInfosReader);
 
-                        TranslatedFunction func = FastTranslate(code, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
+                        TranslatedFunction func = FastTranslate(code, callCounter, infoEntry.GuestSize, unwindInfo, infoEntry.HighCq);
 
                         bool isAddressUnique = funcs.TryAdd(infoEntry.Address, func);
 
@@ -670,8 +678,16 @@ namespace ARMeilleure.Translation.PTC
             return relocEntries;
         }
 
-        private static void PatchCode(Span<byte> code, RelocEntry[] relocEntries, IntPtr pageTablePointer, JumpTable jumpTable)
+        private static void PatchCode(
+            Span<byte> code,
+            RelocEntry[] relocEntries,
+            IntPtr pageTablePointer,
+            JumpTable jumpTable,
+            EntryTable<uint> countTable,
+            out Counter<uint> callCounter)
         {
+            callCounter = null;
+
             foreach (RelocEntry relocEntry in relocEntries)
             {
                 ulong imm;
@@ -688,6 +704,12 @@ namespace ARMeilleure.Translation.PTC
                 {
                     imm = (ulong)jumpTable.DynamicPointer.ToInt64();
                 }
+                else if (relocEntry.Index == CountTableIndex)
+                {
+                    callCounter = new Counter<uint>(countTable);
+
+                    unsafe { imm = (ulong)Unsafe.AsPointer(ref callCounter.Value); }
+                }
                 else if (Delegates.TryGetDelegateFuncPtrByIndex(relocEntry.Index, out IntPtr funcPtr))
                 {
                     imm = (ulong)funcPtr.ToInt64();
@@ -722,7 +744,12 @@ namespace ARMeilleure.Translation.PTC
             return new UnwindInfo(pushEntries, prologueSize);
         }
 
-        private static TranslatedFunction FastTranslate(byte[] code, ulong guestSize, UnwindInfo unwindInfo, bool highCq)
+        private static TranslatedFunction FastTranslate(
+            byte[] code,
+            Counter<uint> callCounter,
+            ulong guestSize,
+            UnwindInfo unwindInfo,
+            bool highCq)
         {
             CompiledFunction cFunc = new CompiledFunction(code, unwindInfo);
 
@@ -730,7 +757,7 @@ namespace ARMeilleure.Translation.PTC
 
             GuestFunction gFunc = Marshal.GetDelegateForFunctionPointer<GuestFunction>(codePtr);
 
-            TranslatedFunction tFunc = new TranslatedFunction(gFunc, guestSize, highCq);
+            TranslatedFunction tFunc = new TranslatedFunction(gFunc, callCounter, guestSize, highCq);
 
             return tFunc;
         }
@@ -771,7 +798,11 @@ namespace ARMeilleure.Translation.PTC
             }
         }
 
-        internal static void MakeAndSaveTranslations(ConcurrentDictionary<ulong, TranslatedFunction> funcs, IMemoryManager memory, JumpTable jumpTable)
+        internal static void MakeAndSaveTranslations(
+            ConcurrentDictionary<ulong, TranslatedFunction> funcs,
+            IMemoryManager memory,
+            JumpTable jumpTable,
+            EntryTable<uint> countTable)
         {
             var profiledFuncsToTranslate = PtcProfiler.GetProfiledFuncsToTranslate(funcs);
 
@@ -813,7 +844,7 @@ namespace ARMeilleure.Translation.PTC
 
                     Debug.Assert(PtcProfiler.IsAddressInStaticCodeRange(address));
 
-                    TranslatedFunction func = Translator.Translate(memory, jumpTable, address, item.mode, item.highCq);
+                    TranslatedFunction func = Translator.Translate(memory, jumpTable, countTable, address, item.mode, item.highCq);
 
                     bool isAddressUnique = funcs.TryAdd(address, func);
 

ARMeilleure/Translation/TranslatedFunction.cs

@@ -1,24 +1,22 @@
+using ARMeilleure.Common;
 using System;
 using System.Runtime.InteropServices;
-using System.Threading;
 
 namespace ARMeilleure.Translation
 {
     class TranslatedFunction
     {
-        private const int MinCallsForRejit = 100;
-
         private readonly GuestFunction _func; // Ensure that this delegate will not be garbage collected.
 
-        private int _callCount;
-
+        public Counter<uint> CallCounter { get; }
         public ulong GuestSize { get; }
         public bool HighCq { get; }
         public IntPtr FuncPtr { get; }
 
-        public TranslatedFunction(GuestFunction func, ulong guestSize, bool highCq)
+        public TranslatedFunction(GuestFunction func, Counter<uint> callCounter, ulong guestSize, bool highCq)
         {
             _func = func;
+            CallCounter = callCounter;
             GuestSize = guestSize;
             HighCq = highCq;
             FuncPtr = Marshal.GetFunctionPointerForDelegate(func);
@@ -28,15 +26,5 @@ namespace ARMeilleure.Translation
         {
             return _func(context.NativeContextPtr);
         }
-
-        public bool ShouldRejit()
-        {
-            return !HighCq && Interlocked.Increment(ref _callCount) == MinCallsForRejit;
-        }
-
-        public void ResetCallCount()
-        {
-            Interlocked.Exchange(ref _callCount, 0);
-        }
     }
 }

ARMeilleure/Translation/Translator.cs

@@ -1,3 +1,4 @@
+using ARMeilleure.Common;
 using ARMeilleure.Decoders;
 using ARMeilleure.Diagnostics;
 using ARMeilleure.Instructions;
@@ -10,7 +11,6 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
 using System.Runtime;
 using System.Threading;
 
@@ -22,23 +22,27 @@ namespace ARMeilleure.Translation
 {
     public class Translator
     {
+        private const int CountTableCapacity = 4 * 1024 * 1024;
+
         private readonly IJitMemoryAllocator _allocator;
         private readonly IMemoryManager _memory;
 
         private readonly ConcurrentDictionary<ulong, TranslatedFunction> _funcs;
-        private readonly ConcurrentQueue<KeyValuePair<ulong, IntPtr>> _oldFuncs;
+        private readonly ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>> _oldFuncs;
 
+        private readonly ConcurrentDictionary<ulong, object> _backgroundSet;
         private readonly ConcurrentStack<RejitRequest> _backgroundStack;
         private readonly AutoResetEvent _backgroundTranslatorEvent;
         private readonly ReaderWriterLock _backgroundTranslatorLock;
 
         private JumpTable _jumpTable;
         internal JumpTable JumpTable => _jumpTable;
+        internal EntryTable<uint> CountTable { get; }
 
         private volatile int _threadCount;
 
         // FIXME: Remove this once the init logic of the emulator will be redone.
-        public static ManualResetEvent IsReadyForTranslation = new ManualResetEvent(false);
+        public static readonly ManualResetEvent IsReadyForTranslation = new(false);
 
         public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
         {
@@ -46,12 +50,15 @@ namespace ARMeilleure.Translation
             _memory = memory;
 
             _funcs = new ConcurrentDictionary<ulong, TranslatedFunction>();
-            _oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, IntPtr>>();
+            _oldFuncs = new ConcurrentQueue<KeyValuePair<ulong, TranslatedFunction>>();
 
+            _backgroundSet = new ConcurrentDictionary<ulong, object>();
             _backgroundStack = new ConcurrentStack<RejitRequest>();
             _backgroundTranslatorEvent = new AutoResetEvent(false);
             _backgroundTranslatorLock = new ReaderWriterLock();
 
+            CountTable = new EntryTable<uint>();
+
             JitCache.Initialize(allocator);
 
             DirectCallStubs.InitializeStubs();
@@ -63,9 +70,16 @@ namespace ARMeilleure.Translation
             {
                 _backgroundTranslatorLock.AcquireReaderLock(Timeout.Infinite);
 
-                if (_backgroundStack.TryPop(out RejitRequest request))
+                if (_backgroundStack.TryPop(out RejitRequest request) && 
+                    _backgroundSet.TryRemove(request.Address, out _))
                 {
-                    TranslatedFunction func = Translate(_memory, _jumpTable, request.Address, request.Mode, highCq: true);
+                    TranslatedFunction func = Translate(
+                        _memory,
+                        _jumpTable,
+                        CountTable,
+                        request.Address,
+                        request.Mode,
+                        highCq: true);
 
                     _funcs.AddOrUpdate(request.Address, func, (key, oldFunc) =>
                     {
@@ -89,7 +103,8 @@ namespace ARMeilleure.Translation
                 }
             }
 
-            _backgroundTranslatorEvent.Set(); // Wake up any other background translator threads, to encourage them to exit.
+             // Wake up any other background translator threads, to encourage them to exit.
+            _backgroundTranslatorEvent.Set();
         }
 
         public void Execute(State.ExecutionContext context, ulong address)
@@ -104,18 +119,21 @@ namespace ARMeilleure.Translation
                 if (Ptc.State == PtcState.Enabled)
                 {
                     Debug.Assert(_funcs.Count == 0);
-                    Ptc.LoadTranslations(_funcs, _memory, _jumpTable);
-                    Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable);
+                    Ptc.LoadTranslations(_funcs, _memory, _jumpTable, CountTable);
+                    Ptc.MakeAndSaveTranslations(_funcs, _memory, _jumpTable, CountTable);
                 }
 
                 PtcProfiler.Start();
 
                 Ptc.Disable();
 
-                // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core+ht etc).
-                // All threads are normal priority except from the last, which just fills as much of the last core as the os lets it with a low priority.
-                // If we only have one rejit thread, it should be normal priority as highCq code is performance critical.
-                // TODO: Use physical cores rather than logical. This only really makes sense for processors with hyperthreading. Requires OS specific code.
+                // Simple heuristic, should be user configurable in future. (1 for 4 core/ht or less, 2 for 6 core + ht
+                // etc). All threads are normal priority except from the last, which just fills as much of the last core
+                // as the os lets it with a low priority. If we only have one rejit thread, it should be normal priority
+                // as highCq code is performance critical.
+                //
+                // TODO: Use physical cores rather than logical. This only really makes sense for processors with
+                // hyperthreading. Requires OS specific code.
                 int unboundedThreadCount = Math.Max(1, (Environment.ProcessorCount - 6) / 3);
                 int threadCount          = Math.Min(4, unboundedThreadCount);
 
@@ -156,6 +174,8 @@ namespace ARMeilleure.Translation
                 _jumpTable.Dispose();
                 _jumpTable = null;
 
+                CountTable.Dispose();
+
                 GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
             }
         }
@@ -173,11 +193,11 @@ namespace ARMeilleure.Translation
             return nextAddr;
         }
 
-        internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode, bool hintRejit = false)
+        internal TranslatedFunction GetOrTranslate(ulong address, ExecutionMode mode)
         {
             if (!_funcs.TryGetValue(address, out TranslatedFunction func))
             {
-                func = Translate(_memory, _jumpTable, address, mode, highCq: false);
+                func = Translate(_memory, _jumpTable, CountTable, address, mode, highCq: false);
 
                 TranslatedFunction getFunc = _funcs.GetOrAdd(address, func);
 
@@ -193,18 +213,18 @@ namespace ARMeilleure.Translation
                 }
             }
 
-            if (hintRejit && func.ShouldRejit())
-            {
-                _backgroundStack.Push(new RejitRequest(address, mode));
-                _backgroundTranslatorEvent.Set();
-            }
-
             return func;
         }
 
-        internal static TranslatedFunction Translate(IMemoryManager memory, JumpTable jumpTable, ulong address, ExecutionMode mode, bool highCq)
+        internal static TranslatedFunction Translate(
+            IMemoryManager memory,
+            JumpTable jumpTable,
+            EntryTable<uint> countTable,
+            ulong address,
+            ExecutionMode mode,
+            bool highCq)
         {
-            ArmEmitterContext context = new ArmEmitterContext(memory, jumpTable, address, highCq, Aarch32Mode.User);
+            var context = new ArmEmitterContext(memory, jumpTable, countTable, address, highCq, Aarch32Mode.User);
 
             Logger.StartPass(PassName.Decoding);
 
@@ -216,6 +236,13 @@ namespace ARMeilleure.Translation
 
             Logger.StartPass(PassName.Translation);
 
+            Counter<uint> counter = null;
+
+            if (!context.HighCq)
+            {
+                EmitRejitCheck(context, out counter);
+            }
+
             EmitSynchronization(context);
 
             if (blocks[0].Address != address)
@@ -258,7 +285,7 @@ namespace ARMeilleure.Translation
                 Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, highCq, ptcInfo);
             }
 
-            return new TranslatedFunction(func, funcSize, highCq);
+            return new TranslatedFunction(func, counter, funcSize, highCq);
         }
 
         internal static void PreparePool(int groupId = 0)
@@ -320,7 +347,7 @@ namespace ARMeilleure.Translation
 
                 if (block.Exit)
                 {
-                    InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address), block.TailCall);
+                    InstEmitFlowHelper.EmitTailContinue(context, Const(block.Address));
                 }
                 else
                 {
@@ -368,29 +395,43 @@ namespace ARMeilleure.Translation
             return context.GetControlFlowGraph();
         }
 
+        internal static void EmitRejitCheck(ArmEmitterContext context, out Counter<uint> counter)
+        {
+            const int MinsCallForRejit = 100;
+
+            counter = new Counter<uint>(context.CountTable);
+
+            Operand lblEnd = Label();
+
+            Operand address = Const(ref counter.Value, Ptc.CountTableIndex);
+            Operand curCount = context.Load(OperandType.I32, address);
+            Operand count = context.Add(curCount, Const(1));
+            context.Store(address, count);
+            context.BranchIf(lblEnd, curCount, Const(MinsCallForRejit), Comparison.NotEqual, BasicBlockFrequency.Cold);
+
+            context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.EnqueueForRejit)), Const(context.EntryAddress));
+
+            context.MarkLabel(lblEnd);
+        }
+
         internal static void EmitSynchronization(EmitterContext context)
         {
             long countOffs = NativeContext.GetCounterOffset();
 
-            Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
-
-            Operand count = context.Load(OperandType.I32, countAddr);
-
             Operand lblNonZero = Label();
             Operand lblExit = Label();
 
+            Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs));
+            Operand count = context.Load(OperandType.I32, countAddr);
             context.BranchIfTrue(lblNonZero, count, BasicBlockFrequency.Cold);
 
             Operand running = context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.CheckSynchronization)));
-
             context.BranchIfTrue(lblExit, running, BasicBlockFrequency.Cold);
 
             context.Return(Const(0L));
 
             context.MarkLabel(lblNonZero);
-
             count = context.Subtract(count, Const(1));
-
             context.Store(countAddr, count);
 
             context.MarkLabel(lblExit);
@@ -404,9 +445,18 @@ namespace ARMeilleure.Translation
             // TODO: Completely remove functions overlapping the specified range from the cache.
         }
 
+        internal void EnqueueForRejit(ulong guestAddress, ExecutionMode mode)
+        {
+            if (_backgroundSet.TryAdd(guestAddress, null))
+            {
+                _backgroundStack.Push(new RejitRequest(guestAddress, mode));
+                _backgroundTranslatorEvent.Set();
+            }
+        }
+
         private void EnqueueForDeletion(ulong guestAddress, TranslatedFunction func)
         {
-            _oldFuncs.Enqueue(new KeyValuePair<ulong, IntPtr>(guestAddress, func.FuncPtr));
+            _oldFuncs.Enqueue(new(guestAddress, func));
         }
 
         private void ClearJitCache()
@@ -414,16 +464,20 @@ namespace ARMeilleure.Translation
             // Ensure no attempt will be made to compile new functions due to rejit.
             ClearRejitQueue(allowRequeue: false);
 
-            foreach (var kv in _funcs)
+            foreach (var func in _funcs.Values)
             {
-                JitCache.Unmap(kv.Value.FuncPtr);
+                JitCache.Unmap(func.FuncPtr);
+
+                func.CallCounter?.Dispose();
             }
 
             _funcs.Clear();
 
             while (_oldFuncs.TryDequeue(out var kv))
             {
-                JitCache.Unmap(kv.Value);
+                JitCache.Unmap(kv.Value.FuncPtr);
+
+                kv.Value.CallCounter?.Dispose();
             }
         }
 
@@ -435,10 +489,12 @@ namespace ARMeilleure.Translation
             {
                 while (_backgroundStack.TryPop(out var request))
                 {
-                    if (_funcs.TryGetValue(request.Address, out var func))
+                    if (_funcs.TryGetValue(request.Address, out var func) && func.CallCounter != null)
                     {
-                        func.ResetCallCount();
+                        Volatile.Write(ref func.CallCounter.Value, 0);
                     }
+
+                    _backgroundSet.TryRemove(request.Address, out _);
                 }
             }
             else