From 22b2cb39af00fb8881e908fd671fbf57a6e2db2a Mon Sep 17 00:00:00 2001 From: FICTURE7 Date: Tue, 17 Aug 2021 22:08:34 +0400 Subject: [PATCH] Reduce JIT GC allocations (#2515) * Turn `MemoryOperand` into a struct * Remove `IntrinsicOperation` * Remove `PhiNode` * Remove `Node` * Turn `Operand` into a struct * Turn `Operation` into a struct * Clean up pool management methods * Add `Arena` allocator * Move `OperationHelper` to `Operation.Factory` * Move `OperandHelper` to `Operand.Factory` * Optimize `Operation` a bit * Fix `Arena` initialization * Rename `NativeList` to `ArenaList` * Reduce `Operand` size from 88 to 56 bytes * Reduce `Operation` size from 56 to 40 bytes * Add optimistic interning of Register & Constant operands * Optimize `RegisterUsage` pass a bit * Optimize `RemoveUnusedNodes` pass a bit Iterating in reverse-order allows killing dependency chains in a single pass. * Fix PPTC symbols * Optimize `BasicBlock` a bit Reduce allocations from `_successor` & `DominanceFrontiers` * Fix `Operation` resize * Make `Arena` expandable Change the arena allocator to be expandable by allocating in pages, with some of them being pooled. Currently 32 pages are pooled. An LRU removal mechanism should probably be added to it. Apparently MHR can allocate bitmaps large enough to exceed the 16MB limit for the type. * Move `Arena` & `ArenaList` to `Common` * Remove `ThreadStaticPool` & co * Add `PhiOperation` * Reduce `Operand` size from 56 from 48 bytes * Add linear-probing to `Operand` intern table * Optimize `HybridAllocator` a bit * Add `Allocators` class * Tune `ArenaAllocator` sizes * Add page removal mechanism to `ArenaAllocator` Remove pages which have not been used for more than 5s after each reset. I am on fence if this would be better using a Gen2 callback object like the one in System.Buffers.ArrayPool, to trim the pool. Because right now if a large translation happens, the pages will be freed only after a reset. This reset may not happen for a while because no new translation is hit, but the arena base sizes are rather small. * Fix `OOM` when allocating larger than page size in `ArenaAllocator` Tweak resizing mechanism for Operand.Uses and Assignemnts. * Optimize `Optimizer` a bit * Optimize `Operand.Add/Remove` a bit * Clean up `PreAllocator` * Fix phi insertion order Reduce codegen diffs. * Fix code alignment * Use new heuristics for degree of parallelism * Suppress warnings * Address gdkchan's feedback Renamed `GetValue()` to `GetValueUnsafe()` to make it more clear that `Operand.Value` should usually not be modified directly. * Add fast path to `ArenaAllocator` * Assembly for `ArenaAllocator.Allocate(ulong)`: .L0: mov rax, [rcx+0x18] lea r8, [rax+rdx] cmp r8, [rcx+0x10] ja short .L2 .L1: mov rdx, [rcx+8] add rax, [rdx+8] mov [rcx+0x18], r8 ret .L2: jmp ArenaAllocator.AllocateSlow(UInt64) A few variable/field had to be changed to ulong so that RyuJIT avoids emitting zero-extends. * Implement a new heuristic to free pooled pages. If an arena is used often, it is more likely that its pages will be needed, so the pages are kept for longer (e.g: during PPTC rebuild or burst sof compilations). If is not used often, then it is more likely that its pages will not be needed (e.g: after PPTC rebuild or bursts of compilations). * Address riperiperi's feedback * Use `EqualityComparer` in `IntrusiveList` Avoids a potential GC hole in `Equals(T, T)`. --- ARMeilleure/Allocators.cs | 38 ++ .../CodeGen/Optimizations/BlockPlacement.cs | 9 +- .../CodeGen/Optimizations/ConstantFolding.cs | 5 +- .../CodeGen/Optimizations/Optimizer.cs | 130 ++--- .../CodeGen/Optimizations/Simplification.cs | 3 +- .../RegisterAllocators/CopyResolver.cs | 7 +- .../RegisterAllocators/HybridAllocator.cs | 246 +++++---- .../RegisterAllocators/LinearScanAllocator.cs | 70 ++- .../RegisterAllocators/LiveInterval.cs | 2 +- ARMeilleure/CodeGen/X86/Assembler.cs | 141 ++--- ARMeilleure/CodeGen/X86/CodeGenerator.cs | 72 ++- ARMeilleure/CodeGen/X86/PreAllocator.cs | 395 +++++++------ ARMeilleure/CodeGen/X86/X86Optimizer.cs | 71 +-- ARMeilleure/Common/AddressTable.cs | 2 +- ARMeilleure/Common/Allocator.cs | 24 + ARMeilleure/Common/ArenaAllocator.cs | 188 +++++++ ARMeilleure/Common/BitMap.cs | 157 +++--- ARMeilleure/Common/BitMapPool.cs | 32 -- ARMeilleure/Common/EntryTable.cs | 11 +- ARMeilleure/Common/NativeAllocator.cs | 27 + ARMeilleure/Common/ThreadStaticPool.cs | 219 -------- ARMeilleure/Common/ThreadStaticPoolEnums.cs | 14 - ARMeilleure/Diagnostics/IRDumper.cs | 73 +-- ARMeilleure/Instructions/InstEmitAlu.cs | 2 +- ARMeilleure/Instructions/InstEmitAlu32.cs | 2 +- ARMeilleure/Instructions/InstEmitAluHelper.cs | 2 +- ARMeilleure/Instructions/InstEmitBfm.cs | 2 +- ARMeilleure/Instructions/InstEmitCcmp.cs | 2 +- ARMeilleure/Instructions/InstEmitCsel.cs | 2 +- ARMeilleure/Instructions/InstEmitDiv.cs | 2 +- ARMeilleure/Instructions/InstEmitException.cs | 2 +- .../Instructions/InstEmitException32.cs | 2 +- ARMeilleure/Instructions/InstEmitFlow.cs | 2 +- ARMeilleure/Instructions/InstEmitFlow32.cs | 2 +- .../Instructions/InstEmitFlowHelper.cs | 4 +- .../Instructions/InstEmitHashHelper.cs | 2 +- ARMeilleure/Instructions/InstEmitHelper.cs | 2 +- ARMeilleure/Instructions/InstEmitMemory.cs | 4 +- ARMeilleure/Instructions/InstEmitMemory32.cs | 4 +- ARMeilleure/Instructions/InstEmitMemoryEx.cs | 2 +- .../Instructions/InstEmitMemoryEx32.cs | 2 +- .../Instructions/InstEmitMemoryExHelper.cs | 6 +- .../Instructions/InstEmitMemoryHelper.cs | 14 +- ARMeilleure/Instructions/InstEmitMove.cs | 2 +- ARMeilleure/Instructions/InstEmitMul32.cs | 2 +- .../Instructions/InstEmitSimdArithmetic.cs | 64 +-- .../Instructions/InstEmitSimdArithmetic32.cs | 5 +- ARMeilleure/Instructions/InstEmitSimdCmp.cs | 2 +- ARMeilleure/Instructions/InstEmitSimdCmp32.cs | 2 +- ARMeilleure/Instructions/InstEmitSimdCvt.cs | 2 +- ARMeilleure/Instructions/InstEmitSimdCvt32.cs | 14 +- .../Instructions/InstEmitSimdHelper.cs | 12 +- .../Instructions/InstEmitSimdHelper32.cs | 4 +- .../Instructions/InstEmitSimdLogical.cs | 2 +- .../Instructions/InstEmitSimdLogical32.cs | 2 +- .../Instructions/InstEmitSimdMemory.cs | 2 +- .../Instructions/InstEmitSimdMemory32.cs | 2 +- ARMeilleure/Instructions/InstEmitSimdMove.cs | 6 +- .../Instructions/InstEmitSimdMove32.cs | 8 +- ARMeilleure/Instructions/InstEmitSimdShift.cs | 4 +- .../Instructions/InstEmitSimdShift32.cs | 2 +- ARMeilleure/Instructions/InstEmitSystem.cs | 2 +- ARMeilleure/Instructions/InstEmitSystem32.cs | 2 +- .../IntermediateRepresentation/BasicBlock.cs | 120 +++- .../IIntrusiveListNode.cs | 2 +- .../IntermediateRepresentation/Instruction.cs | 3 +- .../IntermediateRepresentation/Intrinsic.cs | 2 +- .../IntrinsicOperation.cs | 12 - .../IntrusiveList.cs | 86 ++- .../MemoryOperand.cs | 69 ++- .../IntermediateRepresentation/Node.cs | 309 ----------- .../IntermediateRepresentation/Operand.cs | 522 +++++++++++++++--- .../OperandHelper.cs | 119 ---- .../IntermediateRepresentation/OperandKind.cs | 1 + .../IntermediateRepresentation/Operation.cs | 422 +++++++++++--- .../OperationHelper.cs | 66 --- .../IntermediateRepresentation/PhiNode.cs | 22 - .../PhiOperation.cs | 37 ++ ARMeilleure/Signal/NativeSignalHandler.cs | 8 +- ARMeilleure/Translation/ArmEmitterContext.cs | 6 +- ARMeilleure/Translation/Compiler.cs | 2 +- ARMeilleure/Translation/ControlFlowGraph.cs | 19 +- ARMeilleure/Translation/EmitterContext.cs | 44 +- .../Translation/PTC/DegreeOfParallelism.cs | 50 -- ARMeilleure/Translation/PTC/Ptc.cs | 23 +- ARMeilleure/Translation/RegisterToLocal.cs | 6 +- ARMeilleure/Translation/RegisterUsage.cs | 220 ++++---- ARMeilleure/Translation/SsaConstruction.cs | 112 ++-- ARMeilleure/Translation/SsaDeconstruction.cs | 28 +- ARMeilleure/Translation/Translator.cs | 43 +- ARMeilleure/Translation/TranslatorStubs.cs | 4 +- 91 files changed, 2354 insertions(+), 2142 deletions(-) create mode 100644 ARMeilleure/Allocators.cs create mode 100644 ARMeilleure/Common/Allocator.cs create mode 100644 ARMeilleure/Common/ArenaAllocator.cs delete mode 100644 ARMeilleure/Common/BitMapPool.cs create mode 100644 ARMeilleure/Common/NativeAllocator.cs delete mode 100644 ARMeilleure/Common/ThreadStaticPool.cs delete mode 100644 ARMeilleure/Common/ThreadStaticPoolEnums.cs delete mode 100644 ARMeilleure/IntermediateRepresentation/IntrinsicOperation.cs delete mode 100644 ARMeilleure/IntermediateRepresentation/Node.cs delete mode 100644 ARMeilleure/IntermediateRepresentation/OperandHelper.cs delete mode 100644 ARMeilleure/IntermediateRepresentation/OperationHelper.cs delete mode 100644 ARMeilleure/IntermediateRepresentation/PhiNode.cs create mode 100644 ARMeilleure/IntermediateRepresentation/PhiOperation.cs delete mode 100644 ARMeilleure/Translation/PTC/DegreeOfParallelism.cs diff --git a/ARMeilleure/Allocators.cs b/ARMeilleure/Allocators.cs new file mode 100644 index 0000000000..df762f4c48 --- /dev/null +++ b/ARMeilleure/Allocators.cs @@ -0,0 +1,38 @@ +using ARMeilleure.Common; +using System; +using System.Runtime.CompilerServices; + +namespace ARMeilleure +{ + static class Allocators + { + [ThreadStatic] private static ArenaAllocator _default; + [ThreadStatic] private static ArenaAllocator _operands; + [ThreadStatic] private static ArenaAllocator _operations; + [ThreadStatic] private static ArenaAllocator _references; + + public static ArenaAllocator Default => GetAllocator(ref _default, 256 * 1024, 4); + public static ArenaAllocator Operands => GetAllocator(ref _operands, 64 * 1024, 8); + public static ArenaAllocator Operations => GetAllocator(ref _operations, 64 * 1024, 8); + public static ArenaAllocator References => GetAllocator(ref _references, 64 * 1024, 8); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ArenaAllocator GetAllocator(ref ArenaAllocator alloc, uint pageSize, uint pageCount) + { + if (alloc == null) + { + alloc = new ArenaAllocator(pageSize, pageCount); + } + + return alloc; + } + + public static void ResetAll() + { + Default.Reset(); + Operands.Reset(); + Operations.Reset(); + References.Reset(); + } + } +} diff --git a/ARMeilleure/CodeGen/Optimizations/BlockPlacement.cs b/ARMeilleure/CodeGen/Optimizations/BlockPlacement.cs index a200f54e09..c06ed5205f 100644 --- a/ARMeilleure/CodeGen/Optimizations/BlockPlacement.cs +++ b/ARMeilleure/CodeGen/Optimizations/BlockPlacement.cs @@ -1,8 +1,7 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using System.Diagnostics; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.CodeGen.Optimizations { @@ -33,8 +32,10 @@ namespace ARMeilleure.CodeGen.Optimizations { nextBlock = block.ListNext; - if (block.SuccessorCount == 2 && block.Operations.Last is Operation branchOp) + if (block.SuccessorsCount == 2) { + Operation branchOp = block.Operations.Last; + Debug.Assert(branchOp.Instruction == Instruction.BranchIf); BasicBlock falseSucc = block.GetSuccessor(0); @@ -59,7 +60,7 @@ namespace ARMeilleure.CodeGen.Optimizations if (update) { - cfg.Update(removeUnreachableBlocks: false); + cfg.Update(); } } } diff --git a/ARMeilleure/CodeGen/Optimizations/ConstantFolding.cs b/ARMeilleure/CodeGen/Optimizations/ConstantFolding.cs index 412f6ea4c9..0423c25592 100644 --- a/ARMeilleure/CodeGen/Optimizations/ConstantFolding.cs +++ b/ARMeilleure/CodeGen/Optimizations/ConstantFolding.cs @@ -1,7 +1,6 @@ using ARMeilleure.IntermediateRepresentation; using System; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.CodeGen.Optimizations { @@ -9,7 +8,7 @@ namespace ARMeilleure.CodeGen.Optimizations { public static void RunPass(Operation operation) { - if (operation.Destination == null || operation.SourcesCount == 0) + if (operation.Destination == default || operation.SourcesCount == 0) { return; } diff --git a/ARMeilleure/CodeGen/Optimizations/Optimizer.cs b/ARMeilleure/CodeGen/Optimizations/Optimizer.cs index 438010a2ee..919e996bdd 100644 --- a/ARMeilleure/CodeGen/Optimizations/Optimizer.cs +++ b/ARMeilleure/CodeGen/Optimizations/Optimizer.cs @@ -1,8 +1,8 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; +using System; using System.Diagnostics; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.CodeGen.Optimizations { @@ -10,62 +10,60 @@ namespace ARMeilleure.CodeGen.Optimizations { public static void RunPass(ControlFlowGraph cfg) { + // Scratch buffer used to store uses. + Span buffer = default; + bool modified; do { modified = false; - for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) + for (BasicBlock block = cfg.Blocks.Last; block != null; block = block.ListPrevious) { - Node node = block.Operations.First; + Operation node; + Operation prevNode; - while (node != null) + for (node = block.Operations.Last; node != default; node = prevNode) { - Node nextNode = node.ListNext; + prevNode = node.ListPrevious; - bool isUnused = IsUnused(node); - - if (!(node is Operation operation) || isUnused) + if (IsUnused(node)) { - if (isUnused) - { - RemoveNode(block, node); + RemoveNode(block, node); - modified = true; - } - - node = nextNode; + modified = true; continue; } + else if (node.Instruction == Instruction.Phi) + { + continue; + } - ConstantFolding.RunPass(operation); + ConstantFolding.RunPass(node); + Simplification.RunPass(node); - Simplification.RunPass(operation); - - if (DestIsLocalVar(operation)) + if (DestIsLocalVar(node)) { - if (IsPropagableCompare(operation)) + if (IsPropagableCompare(node)) { - modified |= PropagateCompare(operation); + modified |= PropagateCompare(ref buffer, node); - if (modified && IsUnused(operation)) + if (modified && IsUnused(node)) { RemoveNode(block, node); } } - else if (IsPropagableCopy(operation)) + else if (IsPropagableCopy(node)) { - PropagateCopy(operation); + PropagateCopy(ref buffer, node); RemoveNode(block, node); modified = true; } } - - node = nextNode; } } } @@ -80,13 +78,14 @@ namespace ARMeilleure.CodeGen.Optimizations { modified = false; - for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) + for (BasicBlock block = cfg.Blocks.Last; block != null; block = block.ListPrevious) { - Node node = block.Operations.First; + Operation node; + Operation prevNode; - while (node != null) + for (node = block.Operations.Last; node != default; node = prevNode) { - Node nextNode = node.ListNext; + prevNode = node.ListPrevious; if (IsUnused(node)) { @@ -94,15 +93,27 @@ namespace ARMeilleure.CodeGen.Optimizations modified = true; } - - node = nextNode; } } } while (modified); } - private static bool PropagateCompare(Operation compOp) + private static Span GetUses(ref Span buffer, Operand operand) + { + ReadOnlySpan uses = operand.Uses; + + if (buffer.Length < uses.Length) + { + buffer = Allocators.Default.AllocateSpan((uint)uses.Length); + } + + uses.CopyTo(buffer); + + return buffer.Slice(0, uses.Length); + } + + private static bool PropagateCompare(ref Span buffer, Operation compOp) { // Try to propagate Compare operations into their BranchIf uses, when these BranchIf uses are in the form // of: @@ -149,17 +160,12 @@ namespace ARMeilleure.CodeGen.Optimizations Comparison compType = (Comparison)comp.AsInt32(); - Node[] uses = dest.Uses.ToArray(); + Span uses = GetUses(ref buffer, dest); - foreach (Node use in uses) + foreach (Operation use in uses) { - if (!(use is Operation operation)) - { - continue; - } - // If operation is a BranchIf and has a constant value 0 in its RHS or LHS source operands. - if (IsZeroBranch(operation, out Comparison otherCompType)) + if (IsZeroBranch(use, out Comparison otherCompType)) { Comparison propCompType; @@ -176,9 +182,9 @@ namespace ARMeilleure.CodeGen.Optimizations continue; } - operation.SetSource(0, src1); - operation.SetSource(1, src2); - operation.SetSource(2, Const((int)propCompType)); + use.SetSource(0, src1); + use.SetSource(1, src2); + use.SetSource(2, Const((int)propCompType)); modified = true; } @@ -187,15 +193,15 @@ namespace ARMeilleure.CodeGen.Optimizations return modified; } - private static void PropagateCopy(Operation copyOp) + private static void PropagateCopy(ref Span buffer, Operation copyOp) { // Propagate copy source operand to all uses of the destination operand. Operand dest = copyOp.Destination; Operand source = copyOp.GetSource(0); - Node[] uses = dest.Uses.ToArray(); + Span uses = GetUses(ref buffer, dest); - foreach (Node use in uses) + foreach (Operation use in uses) { for (int index = 0; index < use.SourcesCount; index++) { @@ -207,7 +213,7 @@ namespace ARMeilleure.CodeGen.Optimizations } } - private static void RemoveNode(BasicBlock block, Node node) + private static void RemoveNode(BasicBlock block, Operation node) { // Remove a node from the nodes list, and also remove itself // from all the use lists on the operands that this node uses. @@ -215,31 +221,31 @@ namespace ARMeilleure.CodeGen.Optimizations for (int index = 0; index < node.SourcesCount; index++) { - node.SetSource(index, null); + node.SetSource(index, default); } - Debug.Assert(node.Destination == null || node.Destination.Uses.Count == 0); + Debug.Assert(node.Destination == default || node.Destination.UsesCount == 0); - node.Destination = null; + node.Destination = default; } - private static bool IsUnused(Node node) + private static bool IsUnused(Operation node) { - return DestIsLocalVar(node) && node.Destination.Uses.Count == 0 && !HasSideEffects(node); + return DestIsLocalVar(node) && node.Destination.UsesCount == 0 && !HasSideEffects(node); } - private static bool DestIsLocalVar(Node node) + private static bool DestIsLocalVar(Operation node) { - return node.Destination != null && node.Destination.Kind == OperandKind.LocalVariable; + return node.Destination != default && node.Destination.Kind == OperandKind.LocalVariable; } - private static bool HasSideEffects(Node node) + private static bool HasSideEffects(Operation node) { - return (node is Operation operation) && (operation.Instruction == Instruction.Call - || operation.Instruction == Instruction.Tailcall - || operation.Instruction == Instruction.CompareAndSwap - || operation.Instruction == Instruction.CompareAndSwap16 - || operation.Instruction == Instruction.CompareAndSwap8); + return node.Instruction == Instruction.Call + || node.Instruction == Instruction.Tailcall + || node.Instruction == Instruction.CompareAndSwap + || node.Instruction == Instruction.CompareAndSwap16 + || node.Instruction == Instruction.CompareAndSwap8; } private static bool IsPropagableCompare(Operation operation) diff --git a/ARMeilleure/CodeGen/Optimizations/Simplification.cs b/ARMeilleure/CodeGen/Optimizations/Simplification.cs index db32e99321..341143d861 100644 --- a/ARMeilleure/CodeGen/Optimizations/Simplification.cs +++ b/ARMeilleure/CodeGen/Optimizations/Simplification.cs @@ -1,7 +1,6 @@ using ARMeilleure.IntermediateRepresentation; using System; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.CodeGen.Optimizations { diff --git a/ARMeilleure/CodeGen/RegisterAllocators/CopyResolver.cs b/ARMeilleure/CodeGen/RegisterAllocators/CopyResolver.cs index 417f3bae68..cc731b74fe 100644 --- a/ARMeilleure/CodeGen/RegisterAllocators/CopyResolver.cs +++ b/ARMeilleure/CodeGen/RegisterAllocators/CopyResolver.cs @@ -1,9 +1,8 @@ using ARMeilleure.IntermediateRepresentation; using System; using System.Collections.Generic; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; +using static ARMeilleure.IntermediateRepresentation.Operation.Factory; namespace ARMeilleure.CodeGen.RegisterAllocators { @@ -210,7 +209,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators Operand register = GetRegister(left.Register, type); - _spillQueue.Enqueue(Operation(Instruction.Spill, null, offset, register)); + _spillQueue.Enqueue(Operation(Instruction.Spill, default, offset, register)); HasCopy = true; } diff --git a/ARMeilleure/CodeGen/RegisterAllocators/HybridAllocator.cs b/ARMeilleure/CodeGen/RegisterAllocators/HybridAllocator.cs index 2f68c43f87..7b339efb3d 100644 --- a/ARMeilleure/CodeGen/RegisterAllocators/HybridAllocator.cs +++ b/ARMeilleure/CodeGen/RegisterAllocators/HybridAllocator.cs @@ -1,11 +1,10 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; -using System.Collections.Generic; +using System; using System.Diagnostics; using System.Numerics; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; +using static ARMeilleure.IntermediateRepresentation.Operation.Factory; namespace ARMeilleure.CodeGen.RegisterAllocators { @@ -29,19 +28,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } } - private class LocalInfo + private struct LocalInfo { - public int Uses { get; set; } - public int UseCount { get; set; } - - public bool PreAllocated { get; set; } - public int Register { get; set; } - public int SpillOffset { get; set; } - + public int Uses { get; set; } + public int UsesAllocated { get; set; } + public int Register { get; set; } + public int SpillOffset { get; set; } public int Sequence { get; set; } - public Operand Temp { get; set; } - public OperandType Type { get; } private int _first; @@ -49,13 +43,21 @@ namespace ARMeilleure.CodeGen.RegisterAllocators public bool IsBlockLocal => _first == _last; - public LocalInfo(OperandType type, int uses) + public LocalInfo(OperandType type, int uses, int blkIndex) { Uses = uses; Type = type; + UsesAllocated = 0; + Register = 0; + SpillOffset = 0; + Sequence = 0; + Temp = default; + _first = -1; _last = -1; + + SetBlockIndex(blkIndex); } public void SetBlockIndex(int blkIndex) @@ -72,10 +74,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } } - public AllocationResult RunPass( - ControlFlowGraph cfg, - StackAllocator stackAlloc, - RegisterMasks regMasks) + public AllocationResult RunPass(ControlFlowGraph cfg, StackAllocator stackAlloc, RegisterMasks regMasks) { int intUsedRegisters = 0; int vecUsedRegisters = 0; @@ -84,9 +83,33 @@ namespace ARMeilleure.CodeGen.RegisterAllocators int vecFreeRegisters = regMasks.VecAvailableRegisters; var blockInfo = new BlockInfo[cfg.Blocks.Count]; + var localInfo = new LocalInfo[cfg.Blocks.Count * 3]; + int localInfoCount = 0; - var locInfo = new List(); - var locVisited = new HashSet(); + // The "visited" state is stored in the MSB of the local's value. + const ulong VisitedMask = 1ul << 63; + + bool IsVisited(Operand local) + { + return (local.GetValueUnsafe() & VisitedMask) != 0; + } + + void SetVisited(Operand local) + { + local.GetValueUnsafe() |= VisitedMask | (uint)++localInfoCount; + } + + ref LocalInfo GetLocalInfo(Operand local) + { + Debug.Assert(local.Kind == OperandKind.LocalVariable); + + if (!IsVisited(local)) + { + throw new InvalidOperationException("Local was not visisted yet. Used before defined?"); + } + + return ref localInfo[(uint)local.GetValueUnsafe() - 1]; + } for (int index = cfg.PostOrderBlocks.Length - 1; index >= 0; index--) { @@ -97,59 +120,58 @@ namespace ARMeilleure.CodeGen.RegisterAllocators bool hasCall = false; - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { - if (node is Operation operation && operation.Instruction == Instruction.Call) + if (node.Instruction == Instruction.Call) { hasCall = true; } - for (int srcIndex = 0; srcIndex < node.SourcesCount; srcIndex++) + for (int i = 0; i < node.SourcesCount; i++) { - Operand source = node.GetSource(srcIndex); + Operand source = node.GetSource(i); if (source.Kind == OperandKind.LocalVariable) { - locInfo[source.GetLocalNumber() - 1].SetBlockIndex(block.Index); + GetLocalInfo(source).SetBlockIndex(block.Index); } else if (source.Kind == OperandKind.Memory) { - MemoryOperand memOp = (MemoryOperand)source; + MemoryOperand memOp = source.GetMemory(); - if (memOp.BaseAddress != null) + if (memOp.BaseAddress != default) { - locInfo[memOp.BaseAddress.GetLocalNumber() - 1].SetBlockIndex(block.Index); + GetLocalInfo(memOp.BaseAddress).SetBlockIndex(block.Index); } - if (memOp.Index != null) + if (memOp.Index != default) { - locInfo[memOp.Index.GetLocalNumber() - 1].SetBlockIndex(block.Index); + GetLocalInfo(memOp.Index).SetBlockIndex(block.Index); } } } - for (int dstIndex = 0; dstIndex < node.DestinationsCount; dstIndex++) + for (int i = 0; i < node.DestinationsCount; i++) { - Operand dest = node.GetDestination(dstIndex); + Operand dest = node.GetDestination(i); if (dest.Kind == OperandKind.LocalVariable) { - LocalInfo info; - - if (!locVisited.Add(dest)) + if (IsVisited(dest)) { - info = locInfo[dest.GetLocalNumber() - 1]; + GetLocalInfo(dest).SetBlockIndex(block.Index); } else { - dest.NumberLocal(locInfo.Count + 1); + SetVisited(dest); - info = new LocalInfo(dest.Type, UsesCount(dest)); + if (localInfoCount > localInfo.Length) + { + Array.Resize(ref localInfo, localInfoCount * 2); + } - locInfo.Add(info); + GetLocalInfo(dest) = new LocalInfo(dest.Type, UsesCount(dest), block.Index); } - - info.SetBlockIndex(block.Index); } else if (dest.Kind == OperandKind.Register) { @@ -192,42 +214,26 @@ namespace ARMeilleure.CodeGen.RegisterAllocators intLocalFreeRegisters &= ~(intSpillTempRegisters | intCallerSavedRegisters); vecLocalFreeRegisters &= ~(vecSpillTempRegisters | vecCallerSavedRegisters); - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { int intLocalUse = 0; int vecLocalUse = 0; - void AllocateRegister(Operand source, MemoryOperand memOp, int srcIndex) + Operand AllocateRegister(Operand local) { - LocalInfo info = locInfo[source.GetLocalNumber() - 1]; + ref LocalInfo info = ref GetLocalInfo(local); - info.UseCount++; + info.UsesAllocated++; - Debug.Assert(info.UseCount <= info.Uses); + Debug.Assert(info.UsesAllocated <= info.Uses); if (info.Register != -1) { - Operand reg = Register(info.Register, source.Type.ToRegisterType(), source.Type); + Operand reg = Register(info.Register, local.Type.ToRegisterType(), local.Type); - if (memOp != null) + if (info.UsesAllocated == info.Uses) { - if (srcIndex == 0) - { - memOp.BaseAddress = reg; - } - else /* if (srcIndex == 1) */ - { - memOp.Index = reg; - } - } - else - { - node.SetSource(srcIndex, reg); - } - - if (info.UseCount == info.Uses && !info.PreAllocated) - { - if (source.Type.IsInteger()) + if (local.Type.IsInteger()) { intLocalFreeRegisters |= 1 << info.Register; } @@ -236,72 +242,80 @@ namespace ARMeilleure.CodeGen.RegisterAllocators vecLocalFreeRegisters |= 1 << info.Register; } } - } - else if (node is Operation operation && operation.Instruction == Instruction.Copy) - { - Operation fillOp = Operation(Instruction.Fill, node.Destination, Const(info.SpillOffset)); - block.Operations.AddBefore(node, fillOp); - block.Operations.Remove(node); - - node = fillOp; + return reg; } else { Operand temp = info.Temp; - if (temp == null || info.Sequence != sequence) + if (temp == default || info.Sequence != sequence) { - temp = source.Type.IsInteger() - ? GetSpillTemp(source, intSpillTempRegisters, ref intLocalUse) - : GetSpillTemp(source, vecSpillTempRegisters, ref vecLocalUse); + temp = local.Type.IsInteger() + ? GetSpillTemp(local, intSpillTempRegisters, ref intLocalUse) + : GetSpillTemp(local, vecSpillTempRegisters, ref vecLocalUse); info.Sequence = sequence; info.Temp = temp; } - if (memOp != null) - { - if (srcIndex == 0) - { - memOp.BaseAddress = temp; - } - else /* if (srcIndex == 1) */ - { - memOp.Index = temp; - } - } - else - { - node.SetSource(srcIndex, temp); - } - Operation fillOp = Operation(Instruction.Fill, temp, Const(info.SpillOffset)); block.Operations.AddBefore(node, fillOp); + + return temp; } } - for (int srcIndex = 0; srcIndex < node.SourcesCount; srcIndex++) + bool folded = false; + + // If operation is a copy of a local and that local is living on the stack, we turn the copy into + // a fill, instead of inserting a fill before it. + if (node.Instruction == Instruction.Copy) { - Operand source = node.GetSource(srcIndex); + Operand source = node.GetSource(0); if (source.Kind == OperandKind.LocalVariable) { - AllocateRegister(source, null, srcIndex); - } - else if (source.Kind == OperandKind.Memory) - { - MemoryOperand memOp = (MemoryOperand)source; + ref LocalInfo info = ref GetLocalInfo(source); - if (memOp.BaseAddress != null) + if (info.Register == -1) { - AllocateRegister(memOp.BaseAddress, memOp, 0); + Operation fillOp = Operation(Instruction.Fill, node.Destination, Const(info.SpillOffset)); + + block.Operations.AddBefore(node, fillOp); + block.Operations.Remove(node); + + node = fillOp; + + folded = true; } + } + } - if (memOp.Index != null) + if (!folded) + { + for (int i = 0; i < node.SourcesCount; i++) + { + Operand source = node.GetSource(i); + + if (source.Kind == OperandKind.LocalVariable) { - AllocateRegister(memOp.Index, memOp, 1); + node.SetSource(i, AllocateRegister(source)); + } + else if (source.Kind == OperandKind.Memory) + { + MemoryOperand memOp = source.GetMemory(); + + if (memOp.BaseAddress != default) + { + memOp.BaseAddress = AllocateRegister(memOp.BaseAddress); + } + + if (memOp.Index != default) + { + memOp.Index = AllocateRegister(memOp.Index); + } } } } @@ -309,18 +323,18 @@ namespace ARMeilleure.CodeGen.RegisterAllocators int intLocalAsg = 0; int vecLocalAsg = 0; - for (int dstIndex = 0; dstIndex < node.DestinationsCount; dstIndex++) + for (int i = 0; i < node.DestinationsCount; i++) { - Operand dest = node.GetDestination(dstIndex); + Operand dest = node.GetDestination(i); if (dest.Kind != OperandKind.LocalVariable) { continue; } - LocalInfo info = locInfo[dest.GetLocalNumber() - 1]; + ref LocalInfo info = ref GetLocalInfo(dest); - if (info.UseCount == 0 && !info.PreAllocated) + if (info.UsesAllocated == 0) { int mask = dest.Type.IsInteger() ? intLocalFreeRegisters @@ -350,19 +364,19 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } } - info.UseCount++; + info.UsesAllocated++; - Debug.Assert(info.UseCount <= info.Uses); + Debug.Assert(info.UsesAllocated <= info.Uses); if (info.Register != -1) { - node.SetDestination(dstIndex, Register(info.Register, dest.Type.ToRegisterType(), dest.Type)); + node.SetDestination(i, Register(info.Register, dest.Type.ToRegisterType(), dest.Type)); } else { Operand temp = info.Temp; - if (temp == null || info.Sequence != sequence) + if (temp == default || info.Sequence != sequence) { temp = dest.Type.IsInteger() ? GetSpillTemp(dest, intSpillTempRegisters, ref intLocalAsg) @@ -372,9 +386,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators info.Temp = temp; } - node.SetDestination(dstIndex, temp); + node.SetDestination(i, temp); - Operation spillOp = Operation(Instruction.Spill, null, Const(info.SpillOffset), temp); + Operation spillOp = Operation(Instruction.Spill, default, Const(info.SpillOffset), temp); block.Operations.AddAfter(node, spillOp); @@ -435,7 +449,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators private static int UsesCount(Operand local) { - return local.Assignments.Count + local.Uses.Count; + return local.AssignmentsCount + local.UsesCount; } } } \ No newline at end of file diff --git a/ARMeilleure/CodeGen/RegisterAllocators/LinearScanAllocator.cs b/ARMeilleure/CodeGen/RegisterAllocators/LinearScanAllocator.cs index 88adeeb0f3..fd1420a2e9 100644 --- a/ARMeilleure/CodeGen/RegisterAllocators/LinearScanAllocator.cs +++ b/ARMeilleure/CodeGen/RegisterAllocators/LinearScanAllocator.cs @@ -29,11 +29,11 @@ namespace ARMeilleure.CodeGen.RegisterAllocators private LiveInterval[] _parentIntervals; - private List<(IntrusiveList, Node)> _operationNodes; + private List<(IntrusiveList, Operation)> _operationNodes; private int _operationsCount; - private class AllocationContext : IDisposable + private class AllocationContext { public RegisterMasks Masks { get; } @@ -50,10 +50,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators StackAlloc = stackAlloc; Masks = masks; - BitMapPool.PrepareBitMapPool(); - - Active = BitMapPool.Allocate(intervalsCount); - Inactive = BitMapPool.Allocate(intervalsCount); + Active = new BitMap(Allocators.Default, intervalsCount); + Inactive = new BitMap(Allocators.Default, intervalsCount); } public void MoveActiveToInactive(int bit) @@ -72,11 +70,6 @@ namespace ARMeilleure.CodeGen.RegisterAllocators dest.Set(bit); } - - public void Dispose() - { - BitMapPool.ResetBitMapPool(); - } } public AllocationResult RunPass( @@ -86,7 +79,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators { NumberLocals(cfg); - using AllocationContext context = new AllocationContext(stackAlloc, regMasks, _intervals.Count); + var context = new AllocationContext(stackAlloc, regMasks, _intervals.Count); BuildIntervals(cfg, context); @@ -588,7 +581,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators int splitPosition = kv.Key; - (IntrusiveList nodes, Node node) = GetOperationNode(splitPosition); + (IntrusiveList nodes, Operation node) = GetOperationNode(splitPosition); Operation[] sequence = copyResolver.Sequence(); @@ -621,9 +614,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators continue; } - bool hasSingleOrNoSuccessor = block.SuccessorCount <= 1; + bool hasSingleOrNoSuccessor = block.SuccessorsCount <= 1; - for (int i = 0; i < block.SuccessorCount; i++) + for (int i = 0; i < block.SuccessorsCount; i++) { BasicBlock successor = block.GetSuccessor(i); @@ -677,7 +670,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators { successor.Operations.AddFirst(sequence[0]); - Node prependNode = sequence[0]; + Operation prependNode = sequence[0]; for (int index = 1; index < sequence.Length; index++) { @@ -710,7 +703,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators for (int i = usePositions.Count - 1; i >= 0; i--) { int usePosition = -usePositions[i]; - (_, Node operation) = GetOperationNode(usePosition); + (_, Operation operation) = GetOperationNode(usePosition); for (int index = 0; index < operation.SourcesCount; index++) { @@ -722,7 +715,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } else if (source.Kind == OperandKind.Memory) { - MemoryOperand memOp = (MemoryOperand)source; + MemoryOperand memOp = source.GetMemory(); if (memOp.BaseAddress == current.Local) { @@ -752,20 +745,20 @@ namespace ARMeilleure.CodeGen.RegisterAllocators { Debug.Assert(!interval.IsSpilled, "Spilled intervals are not allowed."); - return OperandHelper.Register( + return Operand.Factory.Register( interval.Register.Index, interval.Register.Type, interval.Local.Type); } - private (IntrusiveList, Node) GetOperationNode(int position) + private (IntrusiveList, Operation) GetOperationNode(int position) { return _operationNodes[position / InstructionGap]; } private void NumberLocals(ControlFlowGraph cfg) { - _operationNodes = new List<(IntrusiveList, Node)>(); + _operationNodes = new List<(IntrusiveList, Operation)>(); _intervals = new List(); @@ -783,13 +776,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators { BasicBlock block = cfg.PostOrderBlocks[index]; - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { _operationNodes.Add((block.Operations, node)); for (int i = 0; i < node.DestinationsCount; i++) { Operand dest = node.GetDestination(i); + if (dest.Kind == OperandKind.LocalVariable && visited.Add(dest)) { dest.NumberLocal(_intervals.Count); @@ -804,7 +798,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators if (block.Operations.Count == 0) { // Pretend we have a dummy instruction on the empty block. - _operationNodes.Add((null, null)); + _operationNodes.Add((default, default)); _operationsCount += InstructionGap; } @@ -825,10 +819,10 @@ namespace ARMeilleure.CodeGen.RegisterAllocators // Compute local live sets. for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { - BitMap liveGen = BitMapPool.Allocate(mapSize); - BitMap liveKill = BitMapPool.Allocate(mapSize); + BitMap liveGen = new BitMap(Allocators.Default, mapSize); + BitMap liveKill = new BitMap(Allocators.Default, mapSize); - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { Sources(node, (source) => { @@ -857,8 +851,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators for (int index = 0; index < cfg.Blocks.Count; index++) { - blkLiveIn [index] = BitMapPool.Allocate(mapSize); - blkLiveOut[index] = BitMapPool.Allocate(mapSize); + blkLiveIn [index] = new BitMap(Allocators.Default, mapSize); + blkLiveOut[index] = new BitMap(Allocators.Default, mapSize); } bool modified; @@ -873,7 +867,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators BitMap liveOut = blkLiveOut[block.Index]; - for (int i = 0; i < block.SuccessorCount; i++) + for (int i = 0; i < block.SuccessorsCount; i++) { BasicBlock succ = block.GetSuccessor(i); @@ -926,7 +920,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators continue; } - foreach (Node node in BottomOperations(block)) + foreach (Operation node in BottomOperations(block)) { operationPos -= InstructionGap; @@ -947,7 +941,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators interval.AddUsePosition(operationPos); }); - if (node is Operation operation && operation.Instruction == Instruction.Call) + if (node.Instruction == Instruction.Call) { AddIntervalCallerSavedReg(context.Masks.IntCallerSavedRegisters, operationPos, RegisterType.Integer); AddIntervalCallerSavedReg(context.Masks.VecCallerSavedRegisters, operationPos, RegisterType.Vector); @@ -993,11 +987,11 @@ namespace ARMeilleure.CodeGen.RegisterAllocators return (register.Index << 1) | (register.Type == RegisterType.Vector ? 1 : 0); } - private static IEnumerable BottomOperations(BasicBlock block) + private static IEnumerable BottomOperations(BasicBlock block) { - Node node = block.Operations.Last; + Operation node = block.Operations.Last; - while (node != null && !(node is PhiNode)) + while (node != default) { yield return node; @@ -1005,7 +999,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } } - private static void Sources(Node node, Action action) + private static void Sources(Operation node, Action action) { for (int index = 0; index < node.SourcesCount; index++) { @@ -1017,14 +1011,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } else if (source.Kind == OperandKind.Memory) { - MemoryOperand memOp = (MemoryOperand)source; + MemoryOperand memOp = source.GetMemory(); - if (memOp.BaseAddress != null) + if (memOp.BaseAddress != default) { action(memOp.BaseAddress); } - if (memOp.Index != null) + if (memOp.Index != default) { action(memOp.Index); } diff --git a/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs b/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs index 309c5ba306..be5876521b 100644 --- a/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs +++ b/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs @@ -34,7 +34,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators public bool IsEmpty => _ranges.Count == 0; - public LiveInterval(Operand local = null, LiveInterval parent = null) + public LiveInterval(Operand local = default, LiveInterval parent = null) { Local = local; _parent = parent ?? this; diff --git a/ARMeilleure/CodeGen/X86/Assembler.cs b/ARMeilleure/CodeGen/X86/Assembler.cs index 39aeb7c9df..044f604726 100644 --- a/ARMeilleure/CodeGen/X86/Assembler.cs +++ b/ARMeilleure/CodeGen/X86/Assembler.cs @@ -329,12 +329,12 @@ namespace ARMeilleure.CodeGen.X86 public void Bswap(Operand dest) { - WriteInstruction(dest, null, dest.Type, X86Instruction.Bswap); + WriteInstruction(dest, default, dest.Type, X86Instruction.Bswap); } public void Call(Operand dest) { - WriteInstruction(dest, null, OperandType.None, X86Instruction.Call); + WriteInstruction(dest, default, OperandType.None, X86Instruction.Call); } public void Cdq() @@ -346,7 +346,7 @@ namespace ARMeilleure.CodeGen.X86 { InstructionInfo info = _instTable[(int)X86Instruction.Cmovcc]; - WriteOpCode(dest, null, source, type, info.Flags, info.OpRRM | (int)condition, rrm: true); + WriteOpCode(dest, default, source, type, info.Flags, info.OpRRM | (int)condition, rrm: true); } public void Cmp(Operand src1, Operand src2, OperandType type) @@ -360,30 +360,38 @@ namespace ARMeilleure.CodeGen.X86 WriteByte(0x99); } - public void Cmpxchg(MemoryOperand memOp, Operand src) + public void Cmpxchg(Operand memOp, Operand src) { + Debug.Assert(memOp.Kind == OperandKind.Memory); + WriteByte(LockPrefix); WriteInstruction(memOp, src, src.Type, X86Instruction.Cmpxchg); } - public void Cmpxchg16(MemoryOperand memOp, Operand src) + public void Cmpxchg16(Operand memOp, Operand src) { + Debug.Assert(memOp.Kind == OperandKind.Memory); + WriteByte(LockPrefix); WriteByte(0x66); WriteInstruction(memOp, src, src.Type, X86Instruction.Cmpxchg); } - public void Cmpxchg16b(MemoryOperand memOp) + public void Cmpxchg16b(Operand memOp) { + Debug.Assert(memOp.Kind == OperandKind.Memory); + WriteByte(LockPrefix); - WriteInstruction(memOp, null, OperandType.None, X86Instruction.Cmpxchg16b); + WriteInstruction(memOp, default, OperandType.None, X86Instruction.Cmpxchg16b); } - public void Cmpxchg8(MemoryOperand memOp, Operand src) + public void Cmpxchg8(Operand memOp, Operand src) { + Debug.Assert(memOp.Kind == OperandKind.Memory); + WriteByte(LockPrefix); WriteInstruction(memOp, src, src.Type, X86Instruction.Cmpxchg8); @@ -391,12 +399,12 @@ namespace ARMeilleure.CodeGen.X86 public void Comisd(Operand src1, Operand src2) { - WriteInstruction(src1, null, src2, X86Instruction.Comisd); + WriteInstruction(src1, default, src2, X86Instruction.Comisd); } public void Comiss(Operand src1, Operand src2) { - WriteInstruction(src1, null, src2, X86Instruction.Comiss); + WriteInstruction(src1, default, src2, X86Instruction.Comiss); } public void Cvtsd2ss(Operand dest, Operand src1, Operand src2) @@ -421,7 +429,7 @@ namespace ARMeilleure.CodeGen.X86 public void Div(Operand source) { - WriteInstruction(null, source, source.Type, X86Instruction.Div); + WriteInstruction(default, source, source.Type, X86Instruction.Div); } public void Divsd(Operand dest, Operand src1, Operand src2) @@ -436,12 +444,12 @@ namespace ARMeilleure.CodeGen.X86 public void Idiv(Operand source) { - WriteInstruction(null, source, source.Type, X86Instruction.Idiv); + WriteInstruction(default, source, source.Type, X86Instruction.Idiv); } public void Imul(Operand source) { - WriteInstruction(null, source, source.Type, X86Instruction.Imul128); + WriteInstruction(default, source, source.Type, X86Instruction.Imul128); } public void Imul(Operand dest, Operand source, OperandType type) @@ -465,13 +473,13 @@ namespace ARMeilleure.CodeGen.X86 if (IsImm8(src2.Value, src2.Type) && info.OpRMImm8 != BadOp) { - WriteOpCode(dest, null, src1, type, info.Flags, info.OpRMImm8, rrm: true); + WriteOpCode(dest, default, src1, type, info.Flags, info.OpRMImm8, rrm: true); WriteByte(src2.AsByte()); } else if (IsImm32(src2.Value, src2.Type) && info.OpRMImm32 != BadOp) { - WriteOpCode(dest, null, src1, type, info.Flags, info.OpRMImm32, rrm: true); + WriteOpCode(dest, default, src1, type, info.Flags, info.OpRMImm32, rrm: true); WriteInt32(src2.AsInt32()); } @@ -531,12 +539,12 @@ namespace ARMeilleure.CodeGen.X86 public void Jmp(Operand dest) { - WriteInstruction(dest, null, OperandType.None, X86Instruction.Jmp); + WriteInstruction(dest, default, OperandType.None, X86Instruction.Jmp); } public void Ldmxcsr(Operand dest) { - WriteInstruction(dest, null, OperandType.I32, X86Instruction.Ldmxcsr); + WriteInstruction(dest, default, OperandType.I32, X86Instruction.Ldmxcsr); } public void Lea(Operand dest, Operand source, OperandType type) @@ -565,17 +573,17 @@ namespace ARMeilleure.CodeGen.X86 if (source.Type.IsInteger() || source.Kind == OperandKind.Memory) { - WriteOpCode(dest, null, source, OperandType.None, info.Flags, info.OpRRM, rrm: true); + WriteOpCode(dest, default, source, OperandType.None, info.Flags, info.OpRRM, rrm: true); } else { - WriteOpCode(dest, null, source, OperandType.None, info.Flags, info.OpRMR); + WriteOpCode(dest, default, source, OperandType.None, info.Flags, info.OpRMR); } } public void Movdqu(Operand dest, Operand source) { - WriteInstruction(dest, null, source, X86Instruction.Movdqu); + WriteInstruction(dest, default, source, X86Instruction.Movdqu); } public void Movhlps(Operand dest, Operand src1, Operand src2) @@ -596,11 +604,11 @@ namespace ARMeilleure.CodeGen.X86 if (source.Type.IsInteger() || source.Kind == OperandKind.Memory) { - WriteOpCode(dest, null, source, OperandType.None, flags, info.OpRRM, rrm: true); + WriteOpCode(dest, default, source, OperandType.None, flags, info.OpRRM, rrm: true); } else if (dest.Type.IsInteger() || dest.Kind == OperandKind.Memory) { - WriteOpCode(dest, null, source, OperandType.None, flags, info.OpRMR); + WriteOpCode(dest, default, source, OperandType.None, flags, info.OpRMR); } else { @@ -645,7 +653,7 @@ namespace ARMeilleure.CodeGen.X86 public void Mul(Operand source) { - WriteInstruction(null, source, source.Type, X86Instruction.Mul128); + WriteInstruction(default, source, source.Type, X86Instruction.Mul128); } public void Mulsd(Operand dest, Operand src1, Operand src2) @@ -660,12 +668,12 @@ namespace ARMeilleure.CodeGen.X86 public void Neg(Operand dest) { - WriteInstruction(dest, null, dest.Type, X86Instruction.Neg); + WriteInstruction(dest, default, dest.Type, X86Instruction.Neg); } public void Not(Operand dest) { - WriteInstruction(dest, null, dest.Type, X86Instruction.Not); + WriteInstruction(dest, default, dest.Type, X86Instruction.Not); } public void Or(Operand dest, Operand source, OperandType type) @@ -675,7 +683,7 @@ namespace ARMeilleure.CodeGen.X86 public void Pclmulqdq(Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, X86Instruction.Pclmulqdq); + WriteInstruction(dest, default, source, X86Instruction.Pclmulqdq); WriteByte(imm); } @@ -687,28 +695,28 @@ namespace ARMeilleure.CodeGen.X86 public void Pextrb(Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, X86Instruction.Pextrb); + WriteInstruction(dest, default, source, X86Instruction.Pextrb); WriteByte(imm); } public void Pextrd(Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, X86Instruction.Pextrd); + WriteInstruction(dest, default, source, X86Instruction.Pextrd); WriteByte(imm); } public void Pextrq(Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, X86Instruction.Pextrq); + WriteInstruction(dest, default, source, X86Instruction.Pextrq); WriteByte(imm); } public void Pextrw(Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, X86Instruction.Pextrw); + WriteInstruction(dest, default, source, X86Instruction.Pextrw); WriteByte(imm); } @@ -749,7 +757,7 @@ namespace ARMeilleure.CodeGen.X86 } else { - WriteInstruction(dest, null, dest.Type, X86Instruction.Pop); + WriteInstruction(dest, default, dest.Type, X86Instruction.Pop); } } @@ -760,7 +768,7 @@ namespace ARMeilleure.CodeGen.X86 public void Pshufd(Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, X86Instruction.Pshufd); + WriteInstruction(dest, default, source, X86Instruction.Pshufd); WriteByte(imm); } @@ -773,7 +781,7 @@ namespace ARMeilleure.CodeGen.X86 } else { - WriteInstruction(null, source, source.Type, X86Instruction.Push); + WriteInstruction(default, source, source.Type, X86Instruction.Push); } } @@ -806,12 +814,12 @@ namespace ARMeilleure.CodeGen.X86 { InstructionInfo info = _instTable[(int)X86Instruction.Setcc]; - WriteOpCode(dest, null, null, OperandType.None, info.Flags, info.OpRRM | (int)condition); + WriteOpCode(dest, default, default, OperandType.None, info.Flags, info.OpRRM | (int)condition); } public void Stmxcsr(Operand dest) { - WriteInstruction(dest, null, OperandType.I32, X86Instruction.Stmxcsr); + WriteInstruction(dest, default, OperandType.I32, X86Instruction.Stmxcsr); } public void Sub(Operand dest, Operand source, OperandType type) @@ -850,7 +858,7 @@ namespace ARMeilleure.CodeGen.X86 Operand source, OperandType type = OperandType.None) { - WriteInstruction(dest, null, source, inst, type); + WriteInstruction(dest, default, source, inst, type); } public void WriteInstruction(X86Instruction inst, Operand dest, Operand src1, Operand src2) @@ -877,7 +885,7 @@ namespace ARMeilleure.CodeGen.X86 public void WriteInstruction(X86Instruction inst, Operand dest, Operand source, byte imm) { - WriteInstruction(dest, null, source, inst); + WriteInstruction(dest, default, source, inst); WriteByte(imm); } @@ -917,11 +925,11 @@ namespace ARMeilleure.CodeGen.X86 Debug.Assert(shiftReg == X86Register.Rcx, $"Invalid shift register \"{shiftReg}\"."); - source = null; + source = default; } else if (source.Kind == OperandKind.Constant) { - source = source.With((int)source.Value & (dest.Type == OperandType.I32 ? 0x1f : 0x3f)); + source = Operand.Factory.Const((int)source.Value & (dest.Type == OperandType.I32 ? 0x1f : 0x3f)); } WriteInstruction(dest, source, type, inst); @@ -931,7 +939,7 @@ namespace ARMeilleure.CodeGen.X86 { InstructionInfo info = _instTable[(int)inst]; - if (source != null) + if (source != default) { if (source.Kind == OperandKind.Constant) { @@ -939,29 +947,29 @@ namespace ARMeilleure.CodeGen.X86 if (inst == X86Instruction.Mov8) { - WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm8); + WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm8); WriteByte((byte)imm); } else if (inst == X86Instruction.Mov16) { - WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm32); + WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm32); WriteInt16((short)imm); } else if (IsImm8(imm, type) && info.OpRMImm8 != BadOp) { - WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm8); + WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm8); WriteByte((byte)imm); } else if (!source.Relocatable && IsImm32(imm, type) && info.OpRMImm32 != BadOp) { - WriteOpCode(dest, null, null, type, info.Flags, info.OpRMImm32); + WriteOpCode(dest, default, default, type, info.Flags, info.OpRMImm32); WriteInt32((int)imm); } - else if (dest?.Kind == OperandKind.Register && info.OpRImm64 != BadOp) + else if (dest != default && dest.Kind == OperandKind.Register && info.OpRImm64 != BadOp) { int rexPrefix = GetRexPrefix(dest, source, type, rrm: false); @@ -972,7 +980,7 @@ namespace ARMeilleure.CodeGen.X86 WriteByte((byte)(info.OpRImm64 + (dest.GetRegister().Index & 0b111))); - if (_ptcInfo != null && source.Relocatable) + if (_ptcInfo != default && source.Relocatable) { _ptcInfo.WriteRelocEntry(new RelocEntry((int)_stream.Position, source.Symbol)); } @@ -986,11 +994,11 @@ namespace ARMeilleure.CodeGen.X86 } else if (source.Kind == OperandKind.Register && info.OpRMR != BadOp) { - WriteOpCode(dest, null, source, type, info.Flags, info.OpRMR); + WriteOpCode(dest, default, source, type, info.Flags, info.OpRMR); } else if (info.OpRRM != BadOp) { - WriteOpCode(dest, null, source, type, info.Flags, info.OpRRM, rrm: true); + WriteOpCode(dest, default, source, type, info.Flags, info.OpRRM, rrm: true); } else { @@ -999,11 +1007,11 @@ namespace ARMeilleure.CodeGen.X86 } else if (info.OpRRM != BadOp) { - WriteOpCode(dest, null, source, type, info.Flags, info.OpRRM, rrm: true); + WriteOpCode(dest, default, source, type, info.Flags, info.OpRRM, rrm: true); } else if (info.OpRMR != BadOp) { - WriteOpCode(dest, null, source, type, info.Flags, info.OpRMR); + WriteOpCode(dest, default, source, type, info.Flags, info.OpRMR); } else { @@ -1020,7 +1028,7 @@ namespace ARMeilleure.CodeGen.X86 { InstructionInfo info = _instTable[(int)inst]; - if (src2 != null) + if (src2 != default) { if (src2.Kind == OperandKind.Constant) { @@ -1028,7 +1036,7 @@ namespace ARMeilleure.CodeGen.X86 if ((byte)imm == imm && info.OpRMImm8 != BadOp) { - WriteOpCode(dest, src1, null, type, info.Flags, info.OpRMImm8); + WriteOpCode(dest, src1, default, type, info.Flags, info.OpRMImm8); WriteByte((byte)imm); } @@ -1082,9 +1090,10 @@ namespace ARMeilleure.CodeGen.X86 int modRM = (opCode >> OpModRMBits) << 3; - MemoryOperand memOp = null; + MemoryOperand memOp = default; + bool hasMemOp = false; - if (dest != null) + if (dest != default) { if (dest.Kind == OperandKind.Register) { @@ -1099,7 +1108,8 @@ namespace ARMeilleure.CodeGen.X86 } else if (dest.Kind == OperandKind.Memory) { - memOp = dest as MemoryOperand; + memOp = dest.GetMemory(); + hasMemOp = true; } else { @@ -1107,7 +1117,7 @@ namespace ARMeilleure.CodeGen.X86 } } - if (src2 != null) + if (src2 != default) { if (src2.Kind == OperandKind.Register) { @@ -1120,9 +1130,10 @@ namespace ARMeilleure.CodeGen.X86 rexPrefix |= RexPrefix; } } - else if (src2.Kind == OperandKind.Memory && memOp == null) + else if (src2.Kind == OperandKind.Memory && !hasMemOp) { - memOp = src2 as MemoryOperand; + memOp = src2.GetMemory(); + hasMemOp = true; } else { @@ -1135,14 +1146,14 @@ namespace ARMeilleure.CodeGen.X86 int sib = 0; - if (memOp != null) + if (hasMemOp) { // Either source or destination is a memory operand. Register baseReg = memOp.BaseAddress.GetRegister(); X86Register baseRegLow = (X86Register)(baseReg.Index & 0b111); - needsSibByte = memOp.Index != null || baseRegLow == X86Register.Rsp; + needsSibByte = memOp.Index != default || baseRegLow == X86Register.Rsp; needsDisplacement = memOp.Displacement != 0 || baseRegLow == X86Register.Rbp; if (needsDisplacement) @@ -1168,7 +1179,7 @@ namespace ARMeilleure.CodeGen.X86 { sib = (int)baseRegLow; - if (memOp.Index != null) + if (memOp.Index != default) { int indexReg = memOp.Index.GetRegister().Index; @@ -1217,7 +1228,7 @@ namespace ARMeilleure.CodeGen.X86 _ => 0 }; - if (src1 != null) + if (src1 != default) { vexByte2 |= (src1.GetRegister().Index ^ 0xf) << 3; } @@ -1284,7 +1295,7 @@ namespace ARMeilleure.CodeGen.X86 } } - if (dest != null && (flags & InstructionFlags.RegOnly) != 0) + if (dest != default && (flags & InstructionFlags.RegOnly) != 0) { opCode += dest.GetRegister().Index & 7; } @@ -1353,12 +1364,12 @@ namespace ARMeilleure.CodeGen.X86 } } - if (dest != null && dest.Kind == OperandKind.Register) + if (dest != default && dest.Kind == OperandKind.Register) { SetRegisterHighBit(dest.GetRegister(), rrm ? 2 : 0); } - if (source != null && source.Kind == OperandKind.Register) + if (source != default && source.Kind == OperandKind.Register) { SetRegisterHighBit(source.GetRegister(), rrm ? 0 : 2); } diff --git a/ARMeilleure/CodeGen/X86/CodeGenerator.cs b/ARMeilleure/CodeGen/X86/CodeGenerator.cs index 3c14594db6..5818eb2eeb 100644 --- a/ARMeilleure/CodeGen/X86/CodeGenerator.cs +++ b/ARMeilleure/CodeGen/X86/CodeGenerator.cs @@ -11,8 +11,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Numerics; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.CodeGen.X86 { @@ -162,20 +161,18 @@ namespace ARMeilleure.CodeGen.X86 { context.EnterBlock(block); - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { - if (node is Operation operation) - { - GenerateOperation(context, operation); - } + GenerateOperation(context, node); } - if (block.SuccessorCount == 0) + if (block.SuccessorsCount == 0) { // The only blocks which can have 0 successors are exit blocks. - Debug.Assert(block.Operations.Last is Operation operation && - (operation.Instruction == Instruction.Tailcall || - operation.Instruction == Instruction.Return)); + Operation last = block.Operations.Last; + + Debug.Assert(last.Instruction == Instruction.Tailcall || + last.Instruction == Instruction.Return); } else { @@ -205,9 +202,7 @@ namespace ARMeilleure.CodeGen.X86 { if (operation.Instruction == Instruction.Extended) { - IntrinsicOperation intrinOp = (IntrinsicOperation)operation; - - IntrinsicInfo info = IntrinsicTable.GetInfo(intrinOp.Intrinsic); + IntrinsicInfo info = IntrinsicTable.GetInfo(operation.Intrinsic); switch (info.Type) { @@ -217,7 +212,7 @@ namespace ARMeilleure.CodeGen.X86 Operand src1 = operation.GetSource(0); Operand src2 = operation.GetSource(1); - switch (intrinOp.Intrinsic) + switch (operation.Intrinsic) { case Intrinsic.X86Comisdeq: context.Assembler.Comisd(src1, src2); @@ -266,14 +261,13 @@ namespace ARMeilleure.CodeGen.X86 int offs = offset.AsInt32() + context.CallArgsRegionSize; Operand rsp = Register(X86Register.Rsp); - - MemoryOperand memOp = MemoryOp(OperandType.I32, rsp, null, Multiplier.x1, offs); + Operand memOp = MemoryOp(OperandType.I32, rsp, default, Multiplier.x1, offs); Debug.Assert(HardwareCapabilities.SupportsSse || HardwareCapabilities.SupportsVexEncoding); context.Assembler.Stmxcsr(memOp); - if (intrinOp.Intrinsic == Intrinsic.X86Mxcsrmb) + if (operation.Intrinsic == Intrinsic.X86Mxcsrmb) { context.Assembler.Or(memOp, bits, OperandType.I32); } @@ -324,7 +318,7 @@ namespace ARMeilleure.CodeGen.X86 Debug.Assert(dest.Type.IsInteger() && !source.Type.IsInteger()); - if (intrinOp.Intrinsic == Intrinsic.X86Cvtsi2si) + if (operation.Intrinsic == Intrinsic.X86Cvtsi2si) { if (dest.Type == OperandType.I32) { @@ -538,7 +532,7 @@ namespace ARMeilleure.CodeGen.X86 if (src2.Kind == OperandKind.Constant) { offset = src2.AsInt32(); - index = null; + index = default; } else { @@ -546,7 +540,7 @@ namespace ARMeilleure.CodeGen.X86 index = src2; } - MemoryOperand memOp = MemoryOp(dest.Type, src1, index, Multiplier.x1, offset); + Operand memOp = MemoryOp(dest.Type, src1, index, Multiplier.x1, offset); context.Assembler.Lea(dest, memOp, dest.Type); } @@ -720,7 +714,7 @@ namespace ARMeilleure.CodeGen.X86 if (operation.SourcesCount == 5) // CompareAndSwap128 has 5 sources, compared to CompareAndSwap64/32's 3. { - MemoryOperand memOp = MemoryOp(OperandType.I64, src1); + Operand memOp = MemoryOp(OperandType.I64, src1); context.Assembler.Cmpxchg16b(memOp); } @@ -731,7 +725,7 @@ namespace ARMeilleure.CodeGen.X86 EnsureSameType(src2, src3); - MemoryOperand memOp = MemoryOp(src3.Type, src1); + Operand memOp = MemoryOp(src3.Type, src1); context.Assembler.Cmpxchg(memOp, src3); } @@ -745,7 +739,7 @@ namespace ARMeilleure.CodeGen.X86 EnsureSameType(src2, src3); - MemoryOperand memOp = MemoryOp(src3.Type, src1); + Operand memOp = MemoryOp(src3.Type, src1); context.Assembler.Cmpxchg16(memOp, src3); } @@ -758,7 +752,7 @@ namespace ARMeilleure.CodeGen.X86 EnsureSameType(src2, src3); - MemoryOperand memOp = MemoryOp(src3.Type, src1); + Operand memOp = MemoryOp(src3.Type, src1); context.Assembler.Cmpxchg8(memOp, src3); } @@ -954,7 +948,7 @@ namespace ARMeilleure.CodeGen.X86 Operand rsp = Register(X86Register.Rsp); - MemoryOperand memOp = MemoryOp(dest.Type, rsp, null, Multiplier.x1, offs); + Operand memOp = MemoryOp(dest.Type, rsp, default, Multiplier.x1, offs); GenerateLoad(context, memOp, dest); } @@ -1153,7 +1147,7 @@ namespace ARMeilleure.CodeGen.X86 Operand rsp = Register(X86Register.Rsp); - MemoryOperand memOp = MemoryOp(source.Type, rsp, null, Multiplier.x1, offs); + Operand memOp = MemoryOp(source.Type, rsp, default, Multiplier.x1, offs); GenerateStore(context, memOp, source); } @@ -1169,7 +1163,7 @@ namespace ARMeilleure.CodeGen.X86 Operand rsp = Register(X86Register.Rsp); - MemoryOperand memOp = MemoryOp(OperandType.I64, rsp, null, Multiplier.x1, offs); + Operand memOp = MemoryOp(OperandType.I64, rsp, default, Multiplier.x1, offs); context.Assembler.Lea(dest, memOp, OperandType.I64); } @@ -1644,19 +1638,19 @@ namespace ARMeilleure.CodeGen.X86 context.Assembler.Pshufd(dest, dest, 0xfc); } - private static bool MatchOperation(Node node, Instruction inst, OperandType destType, Register destReg) + private static bool MatchOperation(Operation node, Instruction inst, OperandType destType, Register destReg) { - if (!(node is Operation operation) || node.DestinationsCount == 0) + if (node == default || node.DestinationsCount == 0) { return false; } - if (operation.Instruction != inst) + if (node.Instruction != inst) { return false; } - Operand dest = operation.Destination; + Operand dest = node.Destination; return dest.Kind == OperandKind.Register && dest.Type == destType && @@ -1761,7 +1755,7 @@ namespace ARMeilleure.CodeGen.X86 offset -= 16; - MemoryOperand memOp = MemoryOp(OperandType.V128, rsp, null, Multiplier.x1, offset); + Operand memOp = MemoryOp(OperandType.V128, rsp, default, Multiplier.x1, offset); context.Assembler.Movdqu(memOp, Xmm((X86Register)bit)); @@ -1791,7 +1785,7 @@ namespace ARMeilleure.CodeGen.X86 offset -= 16; - MemoryOperand memOp = MemoryOp(OperandType.V128, rsp, null, Multiplier.x1, offset); + Operand memOp = MemoryOp(OperandType.V128, rsp, default, Multiplier.x1, offset); context.Assembler.Movdqu(Xmm((X86Register)bit), memOp); @@ -1832,17 +1826,17 @@ namespace ARMeilleure.CodeGen.X86 for (int offset = PageSize; offset < size; offset += PageSize) { - Operand memOp = MemoryOp(OperandType.I32, rsp, null, Multiplier.x1, -offset); + Operand memOp = MemoryOp(OperandType.I32, rsp, default, Multiplier.x1, -offset); context.Assembler.Mov(temp, memOp, OperandType.I32); } } - private static MemoryOperand Memory(Operand operand, OperandType type) + private static Operand Memory(Operand operand, OperandType type) { if (operand.Kind == OperandKind.Memory) { - return operand as MemoryOperand; + return operand; } return MemoryOp(type, operand); @@ -1850,12 +1844,12 @@ namespace ARMeilleure.CodeGen.X86 private static Operand Register(X86Register register, OperandType type = OperandType.I64) { - return OperandHelper.Register((int)register, RegisterType.Integer, type); + return Operand.Factory.Register((int)register, RegisterType.Integer, type); } private static Operand Xmm(X86Register register) { - return OperandHelper.Register((int)register, RegisterType.Vector, OperandType.V128); + return Operand.Factory.Register((int)register, RegisterType.Vector, OperandType.V128); } } } \ No newline at end of file diff --git a/ARMeilleure/CodeGen/X86/PreAllocator.cs b/ARMeilleure/CodeGen/X86/PreAllocator.cs index 3b3fd68374..334f8f7e35 100644 --- a/ARMeilleure/CodeGen/X86/PreAllocator.cs +++ b/ARMeilleure/CodeGen/X86/PreAllocator.cs @@ -4,9 +4,8 @@ using ARMeilleure.Translation; using System; using System.Collections.Generic; using System.Diagnostics; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; +using static ARMeilleure.IntermediateRepresentation.Operation.Factory; namespace ARMeilleure.CodeGen.X86 { @@ -22,31 +21,31 @@ namespace ARMeilleure.CodeGen.X86 for (BasicBlock block = cctx.Cfg.Blocks.First; block != null; block = block.ListNext) { - Node nextNode; + Operation nextNode; - for (Node node = block.Operations.First; node != null; node = nextNode) + for (Operation node = block.Operations.First; node != default; node = nextNode) { nextNode = node.ListNext; - if (node is not Operation operation) + if (node.Instruction == Instruction.Phi) { continue; } - HandleConstantRegCopy(block.Operations, node, operation); - HandleDestructiveRegCopy(block.Operations, node, operation); - HandleConstrainedRegCopy(block.Operations, node, operation); + HandleConstantRegCopy(block.Operations, node); + HandleDestructiveRegCopy(block.Operations, node); + HandleConstrainedRegCopy(block.Operations, node); - switch (operation.Instruction) + switch (node.Instruction) { case Instruction.Call: // Get the maximum number of arguments used on a call. // On windows, when a struct is returned from the call, // we also need to pass the pointer where the struct // should be written on the first argument. - int argsCount = operation.SourcesCount - 1; + int argsCount = node.SourcesCount - 1; - if (operation.Destination != null && operation.Destination.Type == OperandType.V128) + if (node.Destination != default && node.Destination.Type == OperandType.V128) { argsCount++; } @@ -60,72 +59,71 @@ namespace ARMeilleure.CodeGen.X86 // being called, as mandated by the ABI. if (callConv == CallConvName.Windows) { - HandleCallWindowsAbi(block.Operations, stackAlloc, node, operation); + HandleCallWindowsAbi(block.Operations, stackAlloc, node); } else /* if (callConv == CallConvName.SystemV) */ { - HandleCallSystemVAbi(block.Operations, node, operation); + HandleCallSystemVAbi(block.Operations, node); } break; case Instruction.ConvertToFPUI: - HandleConvertToFPUI(block.Operations, node, operation); + HandleConvertToFPUI(block.Operations, node); break; case Instruction.LoadArgument: if (callConv == CallConvName.Windows) { - nextNode = HandleLoadArgumentWindowsAbi(cctx, block.Operations, node, preservedArgs, operation); + nextNode = HandleLoadArgumentWindowsAbi(cctx, block.Operations, preservedArgs, node); } else /* if (callConv == CallConvName.SystemV) */ { - nextNode = HandleLoadArgumentSystemVAbi(cctx, block.Operations, node, preservedArgs, operation); + nextNode = HandleLoadArgumentSystemVAbi(cctx, block.Operations, preservedArgs, node); } break; case Instruction.Negate: - if (!operation.GetSource(0).Type.IsInteger()) + if (!node.GetSource(0).Type.IsInteger()) { - HandleNegate(block.Operations, node, operation); + HandleNegate(block.Operations, node); } break; case Instruction.Return: if (callConv == CallConvName.Windows) { - HandleReturnWindowsAbi(cctx, block.Operations, node, preservedArgs, operation); + HandleReturnWindowsAbi(cctx, block.Operations, preservedArgs, node); } else /* if (callConv == CallConvName.SystemV) */ { - HandleReturnSystemVAbi(block.Operations, node, operation); + HandleReturnSystemVAbi(block.Operations, node); } break; case Instruction.Tailcall: if (callConv == CallConvName.Windows) { - HandleTailcallWindowsAbi(block.Operations, stackAlloc, node, operation); + HandleTailcallWindowsAbi(block.Operations, stackAlloc, node); } else { - HandleTailcallSystemVAbi(block.Operations, stackAlloc, node, operation); + HandleTailcallSystemVAbi(block.Operations, stackAlloc, node); } break; case Instruction.VectorInsert8: if (!HardwareCapabilities.SupportsSse41) { - HandleVectorInsert8(block.Operations, node, operation); + HandleVectorInsert8(block.Operations, node); } break; case Instruction.Extended: - IntrinsicOperation intrinOp = (IntrinsicOperation)operation; - - if (intrinOp.Intrinsic == Intrinsic.X86Mxcsrmb || intrinOp.Intrinsic == Intrinsic.X86Mxcsrub) + if (node.Intrinsic == Intrinsic.X86Mxcsrmb || node.Intrinsic == Intrinsic.X86Mxcsrub) { int stackOffset = stackAlloc.Allocate(OperandType.I32); - operation.SetSources(new Operand[] { Const(stackOffset), operation.GetSource(0) }); + + node.SetSources(new Operand[] { Const(stackOffset), node.GetSource(0) }); } break; } @@ -133,16 +131,16 @@ namespace ARMeilleure.CodeGen.X86 } } - private static void HandleConstantRegCopy(IntrusiveList nodes, Node node, Operation operation) + private static void HandleConstantRegCopy(IntrusiveList nodes, Operation node) { - if (operation.SourcesCount == 0 || IsXmmIntrinsic(operation)) + if (node.SourcesCount == 0 || IsXmmIntrinsic(node)) { return; } - Instruction inst = operation.Instruction; + Instruction inst = node.Instruction; - Operand src1 = operation.GetSource(0); + Operand src1 = node.GetSource(0); Operand src2; if (src1.Kind == OperandKind.Constant) @@ -156,7 +154,7 @@ namespace ARMeilleure.CodeGen.X86 // - Replace the constant use with the XMM register. src1 = AddXmmCopy(nodes, node, src1); - operation.SetSource(0, src1); + node.SetSource(0, src1); } else if (!HasConstSrc1(inst)) { @@ -168,34 +166,34 @@ namespace ARMeilleure.CodeGen.X86 // -- Doing so may allow us to encode the constant as operand 2 and avoid a copy. // - If the constant is on operand 2, we check if the instruction supports it, // if not, we also add a copy. 64-bits constants are usually not supported. - if (IsCommutative(operation)) + if (IsCommutative(node)) { - src2 = operation.GetSource(1); + src2 = node.GetSource(1); Operand temp = src1; src1 = src2; src2 = temp; - operation.SetSource(0, src1); - operation.SetSource(1, src2); + node.SetSource(0, src1); + node.SetSource(1, src2); } if (src1.Kind == OperandKind.Constant) { src1 = AddCopy(nodes, node, src1); - operation.SetSource(0, src1); + node.SetSource(0, src1); } } } - if (operation.SourcesCount < 2) + if (node.SourcesCount < 2) { return; } - src2 = operation.GetSource(1); + src2 = node.GetSource(1); if (src2.Kind == OperandKind.Constant) { @@ -203,28 +201,28 @@ namespace ARMeilleure.CodeGen.X86 { src2 = AddXmmCopy(nodes, node, src2); - operation.SetSource(1, src2); + node.SetSource(1, src2); } else if (!HasConstSrc2(inst) || CodeGenCommon.IsLongConst(src2)) { src2 = AddCopy(nodes, node, src2); - operation.SetSource(1, src2); + node.SetSource(1, src2); } } } - private static void HandleConstrainedRegCopy(IntrusiveList nodes, Node node, Operation operation) + private static void HandleConstrainedRegCopy(IntrusiveList nodes, Operation node) { - Operand dest = operation.Destination; + Operand dest = node.Destination; - switch (operation.Instruction) + switch (node.Instruction) { case Instruction.CompareAndSwap: case Instruction.CompareAndSwap16: case Instruction.CompareAndSwap8: { - OperandType type = operation.GetSource(1).Type; + OperandType type = node.GetSource(1).Type; if (type == OperandType.V128) { @@ -243,14 +241,15 @@ namespace ARMeilleure.CodeGen.X86 Operand rcx = Gpr(X86Register.Rcx, OperandType.I64); Operand rdx = Gpr(X86Register.Rdx, OperandType.I64); - SplitOperand(operation.GetSource(1), rax, rdx); - SplitOperand(operation.GetSource(2), rbx, rcx); + SplitOperand(node.GetSource(1), rax, rdx); + SplitOperand(node.GetSource(2), rbx, rcx); + + Operation operation = node; node = nodes.AddAfter(node, Operation(Instruction.VectorCreateScalar, dest, rax)); nodes.AddAfter(node, Operation(Instruction.VectorInsert, dest, dest, rdx, Const(1))); operation.SetDestinations(new Operand[] { rdx, rax }); - operation.SetSources(new Operand[] { operation.GetSource(0), rdx, rax, rcx, rbx }); } else @@ -258,8 +257,8 @@ namespace ARMeilleure.CodeGen.X86 // Handle the many restrictions of the compare and exchange (32/64) instruction: // - The expected value should be in (E/R)AX. // - The value at the memory location is loaded to (E/R)AX. - Operand expected = operation.GetSource(1); - Operand newValue = operation.GetSource(2); + Operand expected = node.GetSource(1); + Operand newValue = node.GetSource(2); Operand rax = Gpr(X86Register.Rax, expected.Type); @@ -271,11 +270,11 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddBefore(node, Operation(Instruction.Copy, temp, newValue)); - operation.SetSources(new Operand[] { operation.GetSource(0), rax, temp }); + node.SetSources(new Operand[] { node.GetSource(0), rax, temp }); nodes.AddAfter(node, Operation(Instruction.Copy, dest, rax)); - operation.Destination = rax; + node.Destination = rax; } break; @@ -290,7 +289,7 @@ namespace ARMeilleure.CodeGen.X86 // - Additionally it also writes the remainder in RDX. if (dest.Type.IsInteger()) { - Operand src1 = operation.GetSource(0); + Operand src1 = node.GetSource(0); Operand rax = Gpr(X86Register.Rax, src1.Type); Operand rdx = Gpr(X86Register.Rdx, src1.Type); @@ -300,11 +299,8 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddAfter(node, Operation(Instruction.Copy, dest, rax)); - operation.SetDestinations(new Operand[] { rdx, rax }); - - operation.SetSources(new Operand[] { rdx, rax, operation.GetSource(1) }); - - operation.Destination = rax; + node.SetSources(new Operand[] { rdx, rax, node.GetSource(1) }); + node.Destination = rax; } break; @@ -312,19 +308,17 @@ namespace ARMeilleure.CodeGen.X86 case Instruction.Extended: { - IntrinsicOperation intrinOp = (IntrinsicOperation)operation; - // BLENDVPD, BLENDVPS, PBLENDVB last operand is always implied to be XMM0 when VEX is not supported. - if ((intrinOp.Intrinsic == Intrinsic.X86Blendvpd || - intrinOp.Intrinsic == Intrinsic.X86Blendvps || - intrinOp.Intrinsic == Intrinsic.X86Pblendvb) && + if ((node.Intrinsic == Intrinsic.X86Blendvpd || + node.Intrinsic == Intrinsic.X86Blendvps || + node.Intrinsic == Intrinsic.X86Pblendvb) && !HardwareCapabilities.SupportsVexEncoding) { Operand xmm0 = Xmm(X86Register.Xmm0, OperandType.V128); - nodes.AddBefore(node, Operation(Instruction.Copy, xmm0, operation.GetSource(2))); + nodes.AddBefore(node, Operation(Instruction.Copy, xmm0, node.GetSource(2))); - operation.SetSource(2, xmm0); + node.SetSource(2, xmm0); } break; @@ -337,18 +331,18 @@ namespace ARMeilleure.CodeGen.X86 // - The multiplicand is always in RAX. // - The lower 64-bits of the result is always in RAX. // - The higher 64-bits of the result is always in RDX. - Operand src1 = operation.GetSource(0); + Operand src1 = node.GetSource(0); Operand rax = Gpr(X86Register.Rax, src1.Type); Operand rdx = Gpr(X86Register.Rdx, src1.Type); nodes.AddBefore(node, Operation(Instruction.Copy, rax, src1)); - operation.SetSource(0, rax); + node.SetSource(0, rax); nodes.AddAfter(node, Operation(Instruction.Copy, dest, rdx)); - operation.SetDestinations(new Operand[] { rdx, rax }); + node.SetDestinations(new Operand[] { rdx, rax }); break; } @@ -359,13 +353,13 @@ namespace ARMeilleure.CodeGen.X86 case Instruction.ShiftRightUI: { // The shift register is always implied to be CL (low 8-bits of RCX or ECX). - if (operation.GetSource(1).Kind == OperandKind.LocalVariable) + if (node.GetSource(1).Kind == OperandKind.LocalVariable) { Operand rcx = Gpr(X86Register.Rcx, OperandType.I32); - nodes.AddBefore(node, Operation(Instruction.Copy, rcx, operation.GetSource(1))); + nodes.AddBefore(node, Operation(Instruction.Copy, rcx, node.GetSource(1))); - operation.SetSource(1, rcx); + node.SetSource(1, rcx); } break; @@ -373,29 +367,29 @@ namespace ARMeilleure.CodeGen.X86 } } - private static void HandleDestructiveRegCopy(IntrusiveList nodes, Node node, Operation operation) + private static void HandleDestructiveRegCopy(IntrusiveList nodes, Operation node) { - if (operation.Destination == null || operation.SourcesCount == 0) + if (node.Destination == default || node.SourcesCount == 0) { return; } - Instruction inst = operation.Instruction; + Instruction inst = node.Instruction; - Operand dest = operation.Destination; - Operand src1 = operation.GetSource(0); + Operand dest = node.Destination; + Operand src1 = node.GetSource(0); // The multiply instruction (that maps to IMUL) is somewhat special, it has // a three operand form where the second source is a immediate value. - bool threeOperandForm = inst == Instruction.Multiply && operation.GetSource(1).Kind == OperandKind.Constant; + bool threeOperandForm = inst == Instruction.Multiply && node.GetSource(1).Kind == OperandKind.Constant; - if (IsSameOperandDestSrc1(operation) && src1.Kind == OperandKind.LocalVariable && !threeOperandForm) + if (IsSameOperandDestSrc1(node) && src1.Kind == OperandKind.LocalVariable && !threeOperandForm) { bool useNewLocal = false; - for (int srcIndex = 1; srcIndex < operation.SourcesCount; srcIndex++) + for (int srcIndex = 1; srcIndex < node.SourcesCount; srcIndex++) { - if (operation.GetSource(srcIndex) == dest) + if (node.GetSource(srcIndex) == dest) { useNewLocal = true; @@ -412,23 +406,23 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddBefore(node, Operation(Instruction.Copy, temp, src1)); - operation.SetSource(0, temp); + node.SetSource(0, temp); nodes.AddAfter(node, Operation(Instruction.Copy, dest, temp)); - operation.Destination = temp; + node.Destination = temp; } else { nodes.AddBefore(node, Operation(Instruction.Copy, dest, src1)); - operation.SetSource(0, dest); + node.SetSource(0, dest); } } else if (inst == Instruction.ConditionalSelect) { - Operand src2 = operation.GetSource(1); - Operand src3 = operation.GetSource(2); + Operand src2 = node.GetSource(1); + Operand src3 = node.GetSource(2); if (src1 == dest || src2 == dest) { @@ -436,32 +430,32 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddBefore(node, Operation(Instruction.Copy, temp, src3)); - operation.SetSource(2, temp); + node.SetSource(2, temp); nodes.AddAfter(node, Operation(Instruction.Copy, dest, temp)); - operation.Destination = temp; + node.Destination = temp; } else { nodes.AddBefore(node, Operation(Instruction.Copy, dest, src3)); - operation.SetSource(2, dest); + node.SetSource(2, dest); } } } - private static void HandleConvertToFPUI(IntrusiveList nodes, Node node, Operation operation) + private static void HandleConvertToFPUI(IntrusiveList nodes, Operation node) { // Unsigned integer to FP conversions are not supported on X86. // We need to turn them into signed integer to FP conversions, and // adjust the final result. - Operand dest = operation.Destination; - Operand source = operation.GetSource(0); + Operand dest = node.Destination; + Operand source = node.GetSource(0); Debug.Assert(source.Type.IsInteger(), $"Invalid source type \"{source.Type}\"."); - Node currentNode = node; + Operation currentNode = node; if (source.Type == OperandType.I32) { @@ -502,21 +496,21 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddAfter(node, Operation(Instruction.Add, dest, dest, lsbF)); } - Delete(nodes, currentNode, operation); + Delete(nodes, currentNode); } - private static void HandleNegate(IntrusiveList nodes, Node node, Operation operation) + private static void HandleNegate(IntrusiveList nodes, Operation node) { // There's no SSE FP negate instruction, so we need to transform that into // a XOR of the value to be negated with a mask with the highest bit set. // This also produces -0 for a negation of the value 0. - Operand dest = operation.Destination; - Operand source = operation.GetSource(0); + Operand dest = node.Destination; + Operand source = node.GetSource(0); Debug.Assert(dest.Type == OperandType.FP32 || dest.Type == OperandType.FP64, $"Invalid destination type \"{dest.Type}\"."); - Node currentNode = node; + Operation currentNode = node; Operand res = Local(dest.Type); @@ -524,27 +518,27 @@ namespace ARMeilleure.CodeGen.X86 if (dest.Type == OperandType.FP32) { - node = nodes.AddAfter(node, new IntrinsicOperation(Intrinsic.X86Pslld, res, res, Const(31))); + node = nodes.AddAfter(node, Operation(Intrinsic.X86Pslld, res, res, Const(31))); } else /* if (dest.Type == OperandType.FP64) */ { - node = nodes.AddAfter(node, new IntrinsicOperation(Intrinsic.X86Psllq, res, res, Const(63))); + node = nodes.AddAfter(node, Operation(Intrinsic.X86Psllq, res, res, Const(63))); } - node = nodes.AddAfter(node, new IntrinsicOperation(Intrinsic.X86Xorps, res, res, source)); + node = nodes.AddAfter(node, Operation(Intrinsic.X86Xorps, res, res, source)); nodes.AddAfter(node, Operation(Instruction.Copy, dest, res)); - Delete(nodes, currentNode, operation); + Delete(nodes, currentNode); } - private static void HandleVectorInsert8(IntrusiveList nodes, Node node, Operation operation) + private static void HandleVectorInsert8(IntrusiveList nodes, Operation node) { // Handle vector insertion, when SSE 4.1 is not supported. - Operand dest = operation.Destination; - Operand src1 = operation.GetSource(0); // Vector - Operand src2 = operation.GetSource(1); // Value - Operand src3 = operation.GetSource(2); // Index + Operand dest = node.Destination; + Operand src1 = node.GetSource(0); // Vector + Operand src2 = node.GetSource(1); // Value + Operand src3 = node.GetSource(2); // Index Debug.Assert(src3.Kind == OperandKind.Constant); @@ -552,7 +546,7 @@ namespace ARMeilleure.CodeGen.X86 Debug.Assert(index < 16); - Node currentNode = node; + Operation currentNode = node; Operand temp1 = Local(OperandType.I32); Operand temp2 = Local(OperandType.I32); @@ -580,16 +574,15 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddAfter(node, vinsOp); - Delete(nodes, currentNode, operation); + Delete(nodes, currentNode); } - private static void HandleCallWindowsAbi(IntrusiveList nodes, StackAllocator stackAlloc, Node node, Operation operation) + private static void HandleCallWindowsAbi(IntrusiveList nodes, StackAllocator stackAlloc, Operation node) { - Operand dest = operation.Destination; + Operand dest = node.Destination; // Handle struct arguments. int retArgs = 0; - int stackAllocOffset = 0; int AllocateOnStack(int size) @@ -610,9 +603,9 @@ namespace ARMeilleure.CodeGen.X86 return offset; } - Operand arg0Reg = null; + Operand arg0Reg = default; - if (dest != null && dest.Type == OperandType.V128) + if (dest != default && dest.Type == OperandType.V128) { int stackOffset = AllocateOnStack(dest.Type.GetSizeInBytes()); @@ -625,8 +618,7 @@ namespace ARMeilleure.CodeGen.X86 retArgs = 1; } - int argsCount = operation.SourcesCount - 1; - + int argsCount = node.SourcesCount - 1; int maxArgs = CallingConvention.GetArgumentsOnRegsCount() - retArgs; if (argsCount > maxArgs) @@ -636,16 +628,16 @@ namespace ARMeilleure.CodeGen.X86 Operand[] sources = new Operand[1 + retArgs + argsCount]; - sources[0] = operation.GetSource(0); + sources[0] = node.GetSource(0); - if (arg0Reg != null) + if (arg0Reg != default) { sources[1] = arg0Reg; } - for (int index = 1; index < operation.SourcesCount; index++) + for (int index = 1; index < node.SourcesCount; index++) { - Operand source = operation.GetSource(index); + Operand source = node.GetSource(index); if (source.Type == OperandType.V128) { @@ -655,19 +647,18 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddBefore(node, Operation(Instruction.StackAlloc, stackAddr, Const(stackOffset))); - Operation storeOp = Operation(Instruction.Store, null, stackAddr, source); + Operation storeOp = Operation(Instruction.Store, default, stackAddr, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, storeOp), storeOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, storeOp)); - operation.SetSource(index, stackAddr); + node.SetSource(index, stackAddr); } } // Handle arguments passed on registers. for (int index = 0; index < argsCount; index++) { - Operand source = operation.GetSource(index + 1); - + Operand source = node.GetSource(index + 1); Operand argReg; int argIndex = index + retArgs; @@ -683,25 +674,24 @@ namespace ARMeilleure.CodeGen.X86 Operation copyOp = Operation(Instruction.Copy, argReg, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp), copyOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp)); sources[1 + retArgs + index] = argReg; } // The remaining arguments (those that are not passed on registers) // should be passed on the stack, we write them to the stack with "SpillArg". - for (int index = argsCount; index < operation.SourcesCount - 1; index++) + for (int index = argsCount; index < node.SourcesCount - 1; index++) { - Operand source = operation.GetSource(index + 1); - + Operand source = node.GetSource(index + 1); Operand offset = Const((index + retArgs) * 8); - Operation spillOp = Operation(Instruction.SpillArg, null, offset, source); + Operation spillOp = Operation(Instruction.SpillArg, default, offset, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, spillOp), spillOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, spillOp)); } - if (dest != null) + if (dest != default) { if (dest.Type == OperandType.V128) { @@ -713,7 +703,7 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddAfter(node, loadOp); - operation.Destination = null; + node.Destination = default; } else { @@ -725,23 +715,23 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddAfter(node, copyOp); - operation.Destination = retReg; + node.Destination = retReg; } } - operation.SetSources(sources); + node.SetSources(sources); } - private static void HandleCallSystemVAbi(IntrusiveList nodes, Node node, Operation operation) + private static void HandleCallSystemVAbi(IntrusiveList nodes, Operation node) { - Operand dest = operation.Destination; + Operand dest = node.Destination; List sources = new List { - operation.GetSource(0) + node.GetSource(0) }; - int argsCount = operation.SourcesCount - 1; + int argsCount = node.SourcesCount - 1; int intMax = CallingConvention.GetIntArgumentsOnRegsCount(); int vecMax = CallingConvention.GetVecArgumentsOnRegsCount(); @@ -753,7 +743,7 @@ namespace ARMeilleure.CodeGen.X86 for (int index = 0; index < argsCount; index++) { - Operand source = operation.GetSource(index + 1); + Operand source = node.GetSource(index + 1); bool passOnReg; @@ -790,7 +780,7 @@ namespace ARMeilleure.CodeGen.X86 Operation copyOp = Operation(Instruction.Copy, argReg, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp), copyOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp)); sources.Add(argReg); } @@ -798,25 +788,27 @@ namespace ARMeilleure.CodeGen.X86 { Operand offset = Const(stackOffset); - Operation spillOp = Operation(Instruction.SpillArg, null, offset, source); + Operation spillOp = Operation(Instruction.SpillArg, default, offset, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, spillOp), spillOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, spillOp)); stackOffset += source.Type.GetSizeInBytes(); } } - if (dest != null) + if (dest != default) { if (dest.Type == OperandType.V128) { Operand retLReg = Gpr(CallingConvention.GetIntReturnRegister(), OperandType.I64); Operand retHReg = Gpr(CallingConvention.GetIntReturnRegisterHigh(), OperandType.I64); + Operation operation = node; + node = nodes.AddAfter(node, Operation(Instruction.VectorCreateScalar, dest, retLReg)); nodes.AddAfter(node, Operation(Instruction.VectorInsert, dest, dest, retHReg, Const(1))); - operation.Destination = null; + operation.Destination = default; } else { @@ -828,21 +820,21 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddAfter(node, copyOp); - operation.Destination = retReg; + node.Destination = retReg; } } - operation.SetSources(sources.ToArray()); + node.SetSources(sources.ToArray()); } - private static void HandleTailcallSystemVAbi(IntrusiveList nodes, StackAllocator stackAlloc, Node node, Operation operation) + private static void HandleTailcallSystemVAbi(IntrusiveList nodes, StackAllocator stackAlloc, Operation node) { List sources = new List { - operation.GetSource(0) + node.GetSource(0) }; - int argsCount = operation.SourcesCount - 1; + int argsCount = node.SourcesCount - 1; int intMax = CallingConvention.GetIntArgumentsOnRegsCount(); int vecMax = CallingConvention.GetVecArgumentsOnRegsCount(); @@ -853,7 +845,7 @@ namespace ARMeilleure.CodeGen.X86 // Handle arguments passed on registers. for (int index = 0; index < argsCount; index++) { - Operand source = operation.GetSource(1 + index); + Operand source = node.GetSource(1 + index); bool passOnReg; @@ -886,7 +878,7 @@ namespace ARMeilleure.CodeGen.X86 Operation copyOp = Operation(Instruction.Copy, argReg, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp), copyOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp)); sources.Add(argReg); } @@ -901,19 +893,18 @@ namespace ARMeilleure.CodeGen.X86 // callee saved register (which would be trashed on the epilogue). Operand retReg = Gpr(CallingConvention.GetIntReturnRegister(), OperandType.I64); - Operation addrCopyOp = Operation(Instruction.Copy, retReg, operation.GetSource(0)); + Operation addrCopyOp = Operation(Instruction.Copy, retReg, node.GetSource(0)); nodes.AddBefore(node, addrCopyOp); sources[0] = retReg; - operation.SetSources(sources.ToArray()); + node.SetSources(sources.ToArray()); } - private static void HandleTailcallWindowsAbi(IntrusiveList nodes, StackAllocator stackAlloc, Node node, Operation operation) + private static void HandleTailcallWindowsAbi(IntrusiveList nodes, StackAllocator stackAlloc, Operation node) { - int argsCount = operation.SourcesCount - 1; - + int argsCount = node.SourcesCount - 1; int maxArgs = CallingConvention.GetArgumentsOnRegsCount(); if (argsCount > maxArgs) @@ -926,15 +917,14 @@ namespace ARMeilleure.CodeGen.X86 // Handle arguments passed on registers. for (int index = 0; index < argsCount; index++) { - Operand source = operation.GetSource(1 + index); - + Operand source = node.GetSource(1 + index); Operand argReg = source.Type.IsInteger() ? Gpr(CallingConvention.GetIntArgumentRegister(index), source.Type) : Xmm(CallingConvention.GetVecArgumentRegister(index), source.Type); Operation copyOp = Operation(Instruction.Copy, argReg, source); - HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp), copyOp); + HandleConstantRegCopy(nodes, nodes.AddBefore(node, copyOp)); sources[1 + index] = argReg; } @@ -944,23 +934,22 @@ namespace ARMeilleure.CodeGen.X86 // callee saved register (which would be trashed on the epilogue). Operand retReg = Gpr(CallingConvention.GetIntReturnRegister(), OperandType.I64); - Operation addrCopyOp = Operation(Instruction.Copy, retReg, operation.GetSource(0)); + Operation addrCopyOp = Operation(Instruction.Copy, retReg, node.GetSource(0)); nodes.AddBefore(node, addrCopyOp); sources[0] = retReg; - operation.SetSources(sources); + node.SetSources(sources); } - private static Node HandleLoadArgumentWindowsAbi( + private static Operation HandleLoadArgumentWindowsAbi( CompilerContext cctx, - IntrusiveList nodes, - Node node, + IntrusiveList nodes, Operand[] preservedArgs, - Operation operation) + Operation node) { - Operand source = operation.GetSource(0); + Operand source = node.GetSource(0); Debug.Assert(source.Kind == OperandKind.Constant, "Non-constant LoadArgument source kind."); @@ -970,28 +959,25 @@ namespace ARMeilleure.CodeGen.X86 if (index < CallingConvention.GetArgumentsOnRegsCount()) { - Operand dest = operation.Destination; + Operand dest = node.Destination; - if (preservedArgs[index] == null) + if (preservedArgs[index] == default) { Operand argReg, pArg; if (dest.Type.IsInteger()) { argReg = Gpr(CallingConvention.GetIntArgumentRegister(index), dest.Type); - pArg = Local(dest.Type); } else if (dest.Type == OperandType.V128) { argReg = Gpr(CallingConvention.GetIntArgumentRegister(index), OperandType.I64); - pArg = Local(OperandType.I64); } else { argReg = Xmm(CallingConvention.GetVecArgumentRegister(index), dest.Type); - pArg = Local(dest.Type); } @@ -1006,9 +992,9 @@ namespace ARMeilleure.CodeGen.X86 ? Instruction.Load : Instruction.Copy, dest, preservedArgs[index]); - Node newNode = nodes.AddBefore(node, argCopyOp); + Operation newNode = nodes.AddBefore(node, argCopyOp); - Delete(nodes, node, operation); + Delete(nodes, node); return newNode; } @@ -1019,14 +1005,13 @@ namespace ARMeilleure.CodeGen.X86 } } - private static Node HandleLoadArgumentSystemVAbi( + private static Operation HandleLoadArgumentSystemVAbi( CompilerContext cctx, - IntrusiveList nodes, - Node node, + IntrusiveList nodes, Operand[] preservedArgs, - Operation operation) + Operation node) { - Operand source = operation.GetSource(0); + Operand source = node.GetSource(0); Debug.Assert(source.Kind == OperandKind.Constant, "Non-constant LoadArgument source kind."); @@ -1070,9 +1055,9 @@ namespace ARMeilleure.CodeGen.X86 if (passOnReg) { - Operand dest = operation.Destination; + Operand dest = node.Destination; - if (preservedArgs[index] == null) + if (preservedArgs[index] == default) { if (dest.Type == OperandType.V128) { @@ -1108,9 +1093,9 @@ namespace ARMeilleure.CodeGen.X86 Operation argCopyOp = Operation(Instruction.Copy, dest, preservedArgs[index]); - Node newNode = nodes.AddBefore(node, argCopyOp); + Operation newNode = nodes.AddBefore(node, argCopyOp); - Delete(nodes, node, operation); + Delete(nodes, node); return newNode; } @@ -1123,18 +1108,16 @@ namespace ARMeilleure.CodeGen.X86 private static void HandleReturnWindowsAbi( CompilerContext cctx, - IntrusiveList nodes, - Node node, + IntrusiveList nodes, Operand[] preservedArgs, - Operation operation) + Operation node) { - if (operation.SourcesCount == 0) + if (node.SourcesCount == 0) { return; } - Operand source = operation.GetSource(0); - + Operand source = node.GetSource(0); Operand retReg; if (source.Type.IsInteger()) @@ -1143,10 +1126,9 @@ namespace ARMeilleure.CodeGen.X86 } else if (source.Type == OperandType.V128) { - if (preservedArgs[0] == null) + if (preservedArgs[0] == default) { Operand preservedArg = Local(OperandType.I64); - Operand arg0 = Gpr(CallingConvention.GetIntArgumentRegister(0), OperandType.I64); Operation copyOp = Operation(Instruction.Copy, preservedArg, arg0); @@ -1165,7 +1147,7 @@ namespace ARMeilleure.CodeGen.X86 if (source.Type == OperandType.V128) { - Operation retStoreOp = Operation(Instruction.Store, null, retReg, source); + Operation retStoreOp = Operation(Instruction.Store, default, retReg, source); nodes.AddBefore(node, retStoreOp); } @@ -1176,17 +1158,17 @@ namespace ARMeilleure.CodeGen.X86 nodes.AddBefore(node, retCopyOp); } - operation.SetSources(Array.Empty()); + node.SetSources(Array.Empty()); } - private static void HandleReturnSystemVAbi(IntrusiveList nodes, Node node, Operation operation) + private static void HandleReturnSystemVAbi(IntrusiveList nodes, Operation node) { - if (operation.SourcesCount == 0) + if (node.SourcesCount == 0) { return; } - Operand source = operation.GetSource(0); + Operand source = node.GetSource(0); if (source.Type == OperandType.V128) { @@ -1208,10 +1190,9 @@ namespace ARMeilleure.CodeGen.X86 } } - private static Operand AddXmmCopy(IntrusiveList nodes, Node node, Operand source) + private static Operand AddXmmCopy(IntrusiveList nodes, Operation node, Operand source) { Operand temp = Local(source.Type); - Operand intConst = AddCopy(nodes, node, GetIntConst(source)); Operation copyOp = Operation(Instruction.VectorCreateScalar, temp, intConst); @@ -1221,7 +1202,7 @@ namespace ARMeilleure.CodeGen.X86 return temp; } - private static Operand AddCopy(IntrusiveList nodes, Node node, Operand source) + private static Operand AddCopy(IntrusiveList nodes, Operation node, Operand source) { Operand temp = Local(source.Type); @@ -1246,13 +1227,13 @@ namespace ARMeilleure.CodeGen.X86 return value; } - private static void Delete(IntrusiveList nodes, Node node, Operation operation) + private static void Delete(IntrusiveList nodes, Operation node) { - operation.Destination = null; + node.Destination = default; - for (int index = 0; index < operation.SourcesCount; index++) + for (int index = 0; index < node.SourcesCount; index++) { - operation.SetSource(index, null); + node.SetSource(index, default); } nodes.Remove(node); @@ -1307,8 +1288,7 @@ namespace ARMeilleure.CodeGen.X86 private static bool IsIntrinsicSameOperandDestSrc1(Operation operation) { - IntrinsicOperation intrinOp = (IntrinsicOperation)operation; - IntrinsicInfo info = IntrinsicTable.GetInfo(intrinOp.Intrinsic); + IntrinsicInfo info = IntrinsicTable.GetInfo(operation.Intrinsic); return info.Type == IntrinsicType.Crc32 || info.Type == IntrinsicType.Fma || IsVexSameOperandDestSrc1(operation); } @@ -1319,7 +1299,7 @@ namespace ARMeilleure.CodeGen.X86 { bool isUnary = operation.SourcesCount < 2; - bool hasVecDest = operation.Destination != null && operation.Destination.Type == OperandType.V128; + bool hasVecDest = operation.Destination != default && operation.Destination.Type == OperandType.V128; return !HardwareCapabilities.SupportsVexEncoding && !isUnary && hasVecDest; } @@ -1408,8 +1388,7 @@ namespace ARMeilleure.CodeGen.X86 return false; } - IntrinsicOperation intrinOp = (IntrinsicOperation)operation; - IntrinsicInfo info = IntrinsicTable.GetInfo(intrinOp.Intrinsic); + IntrinsicInfo info = IntrinsicTable.GetInfo(operation.Intrinsic); return info.Type != IntrinsicType.Crc32; } diff --git a/ARMeilleure/CodeGen/X86/X86Optimizer.cs b/ARMeilleure/CodeGen/X86/X86Optimizer.cs index fa8b54e840..ed040e150c 100644 --- a/ARMeilleure/CodeGen/X86/X86Optimizer.cs +++ b/ARMeilleure/CodeGen/X86/X86Optimizer.cs @@ -2,9 +2,8 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using System.Collections.Generic; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; +using static ARMeilleure.IntermediateRepresentation.Operation.Factory; namespace ARMeilleure.CodeGen.X86 { @@ -34,32 +33,27 @@ namespace ARMeilleure.CodeGen.X86 { constants.Clear(); - Node nextNode; + Operation nextNode; - for (Node node = block.Operations.First; node != null; node = nextNode) + for (Operation node = block.Operations.First; node != default; node = nextNode) { nextNode = node.ListNext; - if (!(node is Operation operation)) - { - continue; - } - // Insert copies for constants that can't fit on a 32-bits immediate. // Doing this early unblocks a few optimizations. - if (operation.Instruction == Instruction.Add) + if (node.Instruction == Instruction.Add) { - Operand src1 = operation.GetSource(0); - Operand src2 = operation.GetSource(1); + Operand src1 = node.GetSource(0); + Operand src2 = node.GetSource(1); if (src1.Kind == OperandKind.Constant && (src1.Relocatable || CodeGenCommon.IsLongConst(src1))) { - operation.SetSource(0, GetConstantCopy(block, operation, src1)); + node.SetSource(0, GetConstantCopy(block, node, src1)); } if (src2.Kind == OperandKind.Constant && (src2.Relocatable || CodeGenCommon.IsLongConst(src2))) { - operation.SetSource(1, GetConstantCopy(block, operation, src2)); + node.SetSource(1, GetConstantCopy(block, node, src2)); } } @@ -70,24 +64,24 @@ namespace ARMeilleure.CodeGen.X86 // mov rax, [rax] // Into: // mov rax, [rax+rbx*4+0xcafe] - if (IsMemoryLoadOrStore(operation.Instruction)) + if (IsMemoryLoadOrStore(node.Instruction)) { OperandType type; - if (operation.Destination != null) + if (node.Destination != default) { - type = operation.Destination.Type; + type = node.Destination.Type; } else { - type = operation.GetSource(1).Type; + type = node.GetSource(1).Type; } - MemoryOperand memOp = GetMemoryOperandOrNull(operation.GetSource(0), type); + Operand memOp = GetMemoryOperandOrNull(node.GetSource(0), type); - if (memOp != null) + if (memOp != default) { - operation.SetSource(0, memOp); + node.SetSource(0, memOp); } } } @@ -96,7 +90,7 @@ namespace ARMeilleure.CodeGen.X86 Optimizer.RemoveUnusedNodes(cfg); } - private static MemoryOperand GetMemoryOperandOrNull(Operand addr, OperandType type) + private static Operand GetMemoryOperandOrNull(Operand addr, OperandType type) { Operand baseOp = addr; @@ -117,10 +111,10 @@ namespace ARMeilleure.CodeGen.X86 // If baseOp is still equal to address, then there's nothing that can be optimized. if (baseOp == addr) { - return null; + return default; } - if (imm == 0 && scale == Multiplier.x1 && indexOp != null) + if (imm == 0 && scale == Multiplier.x1 && indexOp != default) { imm = GetConstOp(ref indexOp); } @@ -132,7 +126,7 @@ namespace ARMeilleure.CodeGen.X86 { Operation operation = GetAsgOpWithInst(baseOp, Instruction.Add); - if (operation == null) + if (operation == default) { return 0; } @@ -172,13 +166,13 @@ namespace ARMeilleure.CodeGen.X86 private static (Operand, Multiplier) GetIndexOp(ref Operand baseOp) { - Operand indexOp = null; + Operand indexOp = default; Multiplier scale = Multiplier.x1; Operation addOp = GetAsgOpWithInst(baseOp, Instruction.Add); - if (addOp == null) + if (addOp == default) { return (indexOp, scale); } @@ -198,14 +192,14 @@ namespace ARMeilleure.CodeGen.X86 bool indexOnSrc2 = false; - if (shlOp == null) + if (shlOp == default) { shlOp = GetAsgOpWithInst(src2, Instruction.ShiftLeft); indexOnSrc2 = true; } - if (shlOp != null) + if (shlOp != default) { Operand shSrc = shlOp.GetSource(0); Operand shift = shlOp.GetSource(1); @@ -233,24 +227,19 @@ namespace ARMeilleure.CodeGen.X86 // If we have multiple assignments, folding is not safe // as the value may be different depending on the // control flow path. - if (op.Assignments.Count != 1) + if (op.AssignmentsCount != 1) { - return null; + return default; } - Node asgOp = op.Assignments[0]; + Operation asgOp = op.Assignments[0]; - if (!(asgOp is Operation operation)) + if (asgOp.Instruction != inst) { - return null; + return default; } - if (operation.Instruction != inst) - { - return null; - } - - return operation; + return asgOp; } private static bool IsMemoryLoadOrStore(Instruction inst) diff --git a/ARMeilleure/Common/AddressTable.cs b/ARMeilleure/Common/AddressTable.cs index 4af1dc3a0a..60586a3512 100644 --- a/ARMeilleure/Common/AddressTable.cs +++ b/ARMeilleure/Common/AddressTable.cs @@ -211,7 +211,7 @@ namespace ARMeilleure.Common private IntPtr Allocate(int length, T fill, bool leaf) where T : unmanaged { var size = sizeof(T) * length; - var page = Marshal.AllocHGlobal(size); + var page = (IntPtr)NativeAllocator.Instance.Allocate((uint)size); var span = new Span((void*)page, length); span.Fill(fill); diff --git a/ARMeilleure/Common/Allocator.cs b/ARMeilleure/Common/Allocator.cs new file mode 100644 index 0000000000..247a8e8bc1 --- /dev/null +++ b/ARMeilleure/Common/Allocator.cs @@ -0,0 +1,24 @@ +using System; + +namespace ARMeilleure.Common +{ + unsafe abstract class Allocator : IDisposable + { + public T* Allocate(ulong count = 1) where T : unmanaged + { + return (T*)Allocate(count * (uint)sizeof(T)); + } + + public abstract void* Allocate(ulong size); + + public abstract void Free(void* block); + + protected virtual void Dispose(bool disposing) { } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/ARMeilleure/Common/ArenaAllocator.cs b/ARMeilleure/Common/ArenaAllocator.cs new file mode 100644 index 0000000000..4ac7020da5 --- /dev/null +++ b/ARMeilleure/Common/ArenaAllocator.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace ARMeilleure.Common +{ + unsafe sealed class ArenaAllocator : Allocator + { + private class PageInfo + { + public byte* Pointer; + public byte Unused; + public int UnusedCounter; + } + + private int _lastReset; + private ulong _index; + private int _pageIndex; + private PageInfo _page; + private List _pages; + private readonly ulong _pageSize; + private readonly uint _pageCount; + private readonly List _extras; + + public ArenaAllocator(uint pageSize, uint pageCount) + { + _lastReset = Environment.TickCount; + + // Set _index to pageSize so that the first allocation goes through the slow path. + _index = pageSize; + _pageIndex = -1; + + _page = null; + _pages = new List(); + _pageSize = pageSize; + _pageCount = pageCount; + + _extras = new List(); + } + + public Span AllocateSpan(ulong count) where T : unmanaged + { + return new Span(Allocate(count), (int)count); + } + + public override void* Allocate(ulong size) + { + if (_index + size <= _pageSize) + { + byte* result = _page.Pointer + _index; + + _index += size; + + return result; + } + + return AllocateSlow(size); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void* AllocateSlow(ulong size) + { + if (size > _pageSize) + { + void* extra = NativeAllocator.Instance.Allocate(size); + + _extras.Add((IntPtr)extra); + + return extra; + } + + if (_index + size > _pageSize) + { + _index = 0; + _pageIndex++; + } + + if (_pageIndex < _pages.Count) + { + _page = _pages[_pageIndex]; + _page.Unused = 0; + } + else + { + _page = new PageInfo(); + _page.Pointer = (byte*)NativeAllocator.Instance.Allocate(_pageSize); + + _pages.Add(_page); + } + + byte* result = _page.Pointer + _index; + + _index += size; + + return result; + } + + public override void Free(void* block) { } + + public void Reset() + { + _index = _pageSize; + _pageIndex = -1; + _page = null; + + // Free excess pages that was allocated. + while (_pages.Count > _pageCount) + { + NativeAllocator.Instance.Free(_pages[_pages.Count - 1].Pointer); + + _pages.RemoveAt(_pages.Count - 1); + } + + // Free extra blocks that are not page-sized + foreach (IntPtr ptr in _extras) + { + NativeAllocator.Instance.Free((void*)ptr); + } + + _extras.Clear(); + + // Free pooled pages that has not been used in a while. Remove pages at the back first, because we try to + // keep the pages at the front alive, since they're more likely to be hot and in the d-cache. + bool removing = true; + + // If arena is used frequently, keep pages for longer. Otherwise keep pages for a shorter amount of time. + int now = Environment.TickCount; + int count = (now - _lastReset) switch { + >= 5000 => 0, + >= 2500 => 50, + >= 1000 => 100, + >= 10 => 1500, + _ => 5000 + }; + + for (int i = _pages.Count - 1; i >= 0; i--) + { + PageInfo page = _pages[i]; + + if (page.Unused == 0) + { + page.UnusedCounter = 0; + } + + page.UnusedCounter += page.Unused; + page.Unused = 1; + + // If page not used after `count` resets, remove it. + if (removing && page.UnusedCounter >= count) + { + NativeAllocator.Instance.Free(page.Pointer); + + _pages.RemoveAt(i); + } + else + { + removing = false; + } + } + + _lastReset = now; + } + + protected override void Dispose(bool disposing) + { + if (_pages != null) + { + foreach (PageInfo info in _pages) + { + NativeAllocator.Instance.Free(info.Pointer); + } + + foreach (IntPtr ptr in _extras) + { + NativeAllocator.Instance.Free((void*)ptr); + } + + _pages = null; + } + } + + ~ArenaAllocator() + { + Dispose(false); + } + } +} diff --git a/ARMeilleure/Common/BitMap.cs b/ARMeilleure/Common/BitMap.cs index f782ac8b97..4872c442ef 100644 --- a/ARMeilleure/Common/BitMap.cs +++ b/ARMeilleure/Common/BitMap.cs @@ -1,57 +1,27 @@ +using System; using System.Collections; using System.Collections.Generic; using System.Numerics; namespace ARMeilleure.Common { - class BitMap : IEnumerator, IEnumerable + unsafe class BitMap : IEnumerable, IDisposable { private const int IntSize = 64; private const int IntMask = IntSize - 1; - private readonly List _masks; + private int _count; + private long* _masks; + private readonly Allocator _allocator; - private int _enumIndex; - private long _enumMask; - private int _enumBit; - - public int Current => _enumIndex * IntSize + _enumBit; - object IEnumerator.Current => Current; - - public BitMap() + public BitMap(Allocator allocator) { - _masks = new List(0); + _allocator = allocator; } - public BitMap(int initialCapacity) + public BitMap(Allocator allocator, int capacity) : this(allocator) { - int count = (initialCapacity + IntMask) / IntSize; - - _masks = new List(count); - - while (count-- > 0) - { - _masks.Add(0); - } - } - - public BitMap Reset(int initialCapacity) - { - int count = (initialCapacity + IntMask) / IntSize; - - if (count > _masks.Capacity) - { - _masks.Capacity = count; - } - - _masks.Clear(); - - while (count-- > 0) - { - _masks.Add(0); - } - - return this; + EnsureCapacity(capacity); } public bool Set(int bit) @@ -97,7 +67,7 @@ namespace ARMeilleure.Common public int FindFirstUnset() { - for (int index = 0; index < _masks.Count; index++) + for (int index = 0; index < _count; index++) { long mask = _masks[index]; @@ -107,16 +77,16 @@ namespace ARMeilleure.Common } } - return _masks.Count * IntSize; + return _count * IntSize; } public bool Set(BitMap map) { - EnsureCapacity(map._masks.Count * IntSize); + EnsureCapacity(map._count * IntSize); bool modified = false; - for (int index = 0; index < _masks.Count; index++) + for (int index = 0; index < _count; index++) { long newValue = _masks[index] | map._masks[index]; @@ -133,11 +103,11 @@ namespace ARMeilleure.Common public bool Clear(BitMap map) { - EnsureCapacity(map._masks.Count * IntSize); + EnsureCapacity(map._count * IntSize); bool modified = false; - for (int index = 0; index < _masks.Count; index++) + for (int index = 0; index < _count; index++) { long newValue = _masks[index] & ~map._masks[index]; @@ -152,15 +122,34 @@ namespace ARMeilleure.Common return modified; } - #region IEnumerable Methods - - // Note: The bit enumerator is embedded in this class to avoid creating garbage when enumerating. - private void EnsureCapacity(int size) { - while (_masks.Count * IntSize < size) + int count = (size + IntMask) / IntSize; + + if (count > _count) { - _masks.Add(0); + var oldMask = _masks; + var oldSpan = new Span(_masks, _count); + + _masks = _allocator.Allocate((uint)count); + _count = count; + + var newSpan = new Span(_masks, _count); + + oldSpan.CopyTo(newSpan); + newSpan.Slice(oldSpan.Length).Clear(); + + _allocator.Free(oldMask); + } + } + + public void Dispose() + { + if (_masks != null) + { + _allocator.Free(_masks); + + _masks = null; } } @@ -169,39 +158,59 @@ namespace ARMeilleure.Common return GetEnumerator(); } - public IEnumerator GetEnumerator() + IEnumerator IEnumerable.GetEnumerator() { - Reset(); - return this; + return GetEnumerator(); } - public bool MoveNext() + public Enumerator GetEnumerator() { - if (_enumMask != 0) + return new Enumerator(this); + } + + public struct Enumerator : IEnumerator + { + private int _index; + private long _mask; + private int _bit; + private readonly BitMap _map; + + public int Current => _index * IntSize + _bit; + object IEnumerator.Current => Current; + + public Enumerator(BitMap map) { - _enumMask &= ~(1L << _enumBit); + _index = -1; + _mask = 0; + _bit = 0; + _map = map; } - while (_enumMask == 0) + + public bool MoveNext() { - if (++_enumIndex >= _masks.Count) + if (_mask != 0) { - return false; + _mask &= ~(1L << _bit); } - _enumMask = _masks[_enumIndex]; + + while (_mask == 0) + { + if (++_index >= _map._count) + { + return false; + } + + _mask = _map._masks[_index]; + } + + _bit = BitOperations.TrailingZeroCount(_mask); + + return true; } - _enumBit = BitOperations.TrailingZeroCount(_enumMask); - return true; + + public void Reset() { } + + public void Dispose() { } } - - public void Reset() - { - _enumIndex = -1; - _enumMask = 0; - _enumBit = 0; - } - - public void Dispose() { } - - #endregion } } \ No newline at end of file diff --git a/ARMeilleure/Common/BitMapPool.cs b/ARMeilleure/Common/BitMapPool.cs deleted file mode 100644 index d8d297faf0..0000000000 --- a/ARMeilleure/Common/BitMapPool.cs +++ /dev/null @@ -1,32 +0,0 @@ -namespace ARMeilleure.Common -{ - static class BitMapPool - { - public static BitMap Allocate(int initialCapacity) - { - return BitMap().Reset(initialCapacity); - } - - #region "ThreadStaticPool" - public static void PrepareBitMapPool(int groupId = 0) - { - ThreadStaticPool.PreparePool(groupId, ChunkSizeLimit.Small); - } - - private static BitMap BitMap() - { - return ThreadStaticPool.Instance.Allocate(); - } - - public static void ResetBitMapPool(int groupId = 0) - { - ThreadStaticPool.ResetPool(groupId); - } - - public static void DisposeBitMapPools() - { - ThreadStaticPool.DisposePools(); - } - #endregion - } -} diff --git a/ARMeilleure/Common/EntryTable.cs b/ARMeilleure/Common/EntryTable.cs index b61af8f81d..f3f3ce281a 100644 --- a/ARMeilleure/Common/EntryTable.cs +++ b/ARMeilleure/Common/EntryTable.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Numerics; -using System.Runtime.InteropServices; namespace ARMeilleure.Common { @@ -41,7 +40,7 @@ namespace ARMeilleure.Common throw new ArgumentException("Size of TEntry cannot be zero."); } - _allocated = new BitMap(); + _allocated = new BitMap(NativeAllocator.Instance); _pages = new Dictionary(); _pageLogCapacity = BitOperations.Log2((uint)(pageSize / sizeof(TEntry))); _pageCapacity = 1 << _pageLogCapacity; @@ -150,7 +149,7 @@ namespace ARMeilleure.Common if (!_pages.TryGetValue(pageIndex, out IntPtr page)) { - page = Marshal.AllocHGlobal(sizeof(TEntry) * _pageCapacity); + page = (IntPtr)NativeAllocator.Instance.Allocate((uint)sizeof(TEntry) * (uint)_pageCapacity); _pages.Add(pageIndex, page); } @@ -172,13 +171,15 @@ namespace ARMeilleure.Common /// instance. /// /// to dispose managed resources also; otherwise just unmanaged resouces - protected virtual void Dispose(bool disposing) + protected unsafe virtual void Dispose(bool disposing) { if (!_disposed) { + _allocated.Dispose(); + foreach (var page in _pages.Values) { - Marshal.FreeHGlobal(page); + NativeAllocator.Instance.Free((void*)page); } _disposed = true; diff --git a/ARMeilleure/Common/NativeAllocator.cs b/ARMeilleure/Common/NativeAllocator.cs new file mode 100644 index 0000000000..71c04a9b13 --- /dev/null +++ b/ARMeilleure/Common/NativeAllocator.cs @@ -0,0 +1,27 @@ +using System; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Common +{ + unsafe sealed class NativeAllocator : Allocator + { + public static NativeAllocator Instance { get; } = new(); + + public override void* Allocate(ulong size) + { + void* result = (void*)Marshal.AllocHGlobal((IntPtr)size); + + if (result == null) + { + throw new OutOfMemoryException(); + } + + return result; + } + + public override void Free(void* block) + { + Marshal.FreeHGlobal((IntPtr)block); + } + } +} diff --git a/ARMeilleure/Common/ThreadStaticPool.cs b/ARMeilleure/Common/ThreadStaticPool.cs deleted file mode 100644 index bbe662f827..0000000000 --- a/ARMeilleure/Common/ThreadStaticPool.cs +++ /dev/null @@ -1,219 +0,0 @@ -using ARMeilleure.Translation.PTC; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; - -namespace ARMeilleure.Common -{ - class ThreadStaticPool where T : class, new() - { - [ThreadStatic] - private static ThreadStaticPool _instance; - - public static ThreadStaticPool Instance - { - get - { - if (_instance == null) - { - PreparePool(); // So that we can still use a pool when blindly initializing one. - } - - return _instance; - } - } - - private static readonly ConcurrentDictionary>> _pools = new(); - - private static Stack> GetPools(int groupId) - { - return _pools.GetOrAdd(groupId, (groupId) => new()); - } - - public static void PreparePool( - int groupId = 0, - ChunkSizeLimit chunkSizeLimit = ChunkSizeLimit.Large, - PoolSizeIncrement poolSizeIncrement = PoolSizeIncrement.Default) - { - if (Ptc.State == PtcState.Disabled) - { - PreparePoolDefault(groupId, (int)chunkSizeLimit, (int)poolSizeIncrement); - } - else - { - PreparePoolSlim((int)chunkSizeLimit, (int)poolSizeIncrement); - } - } - - private static void PreparePoolDefault(int groupId, int chunkSizeLimit, int poolSizeIncrement) - { - // Prepare the pool for this thread, ideally using an existing one from the specified group. - - if (_instance == null) - { - var pools = GetPools(groupId); - lock (pools) - { - _instance = (pools.Count != 0) ? pools.Pop() : new(chunkSizeLimit, poolSizeIncrement); - } - } - } - - private static void PreparePoolSlim(int chunkSizeLimit, int poolSizeIncrement) - { - // Prepare the pool for this thread. - - if (_instance == null) - { - _instance = new(chunkSizeLimit, poolSizeIncrement); - } - } - - public static void ResetPool(int groupId = 0) - { - if (Ptc.State == PtcState.Disabled) - { - ResetPoolDefault(groupId); - } - else - { - ResetPoolSlim(); - } - } - - private static void ResetPoolDefault(int groupId) - { - // Reset, limit if necessary, and return the pool for this thread to the specified group. - - if (_instance != null) - { - var pools = GetPools(groupId); - lock (pools) - { - _instance.Clear(); - _instance.ChunkSizeLimiter(); - pools.Push(_instance); - - _instance = null; - } - } - } - - private static void ResetPoolSlim() - { - // Reset, limit if necessary, the pool for this thread. - - if (_instance != null) - { - _instance.Clear(); - _instance.ChunkSizeLimiter(); - } - } - - public static void DisposePools() - { - if (Ptc.State == PtcState.Disabled) - { - DisposePoolsDefault(); - } - else - { - DisposePoolSlim(); - } - } - - private static void DisposePoolsDefault() - { - // Resets any static references to the pools used by threads for each group, allowing them to be garbage collected. - - foreach (var pools in _pools.Values) - { - foreach (var instance in pools) - { - instance.Dispose(); - } - - pools.Clear(); - } - - _pools.Clear(); - } - - private static void DisposePoolSlim() - { - // Dispose the pool for this thread. - - if (_instance != null) - { - _instance.Dispose(); - - _instance = null; - } - } - - private List _pool; - private int _chunkIndex = -1; - private int _poolIndex = -1; - private int _chunkSizeLimit; - private int _poolSizeIncrement; - - private ThreadStaticPool(int chunkSizeLimit, int poolSizeIncrement) - { - _chunkSizeLimit = chunkSizeLimit; - _poolSizeIncrement = poolSizeIncrement; - - _pool = new(chunkSizeLimit * 2); - - AddChunkIfNeeded(); - } - - public T Allocate() - { - if (++_poolIndex >= _poolSizeIncrement) - { - AddChunkIfNeeded(); - - _poolIndex = 0; - } - - return _pool[_chunkIndex][_poolIndex]; - } - - private void AddChunkIfNeeded() - { - if (++_chunkIndex >= _pool.Count) - { - T[] pool = new T[_poolSizeIncrement]; - - for (int i = 0; i < _poolSizeIncrement; i++) - { - pool[i] = new T(); - } - - _pool.Add(pool); - } - } - - public void Clear() - { - _chunkIndex = 0; - _poolIndex = -1; - } - - private void ChunkSizeLimiter() - { - if (_pool.Count >= _chunkSizeLimit) - { - int newChunkSize = _chunkSizeLimit / 2; - - _pool.RemoveRange(newChunkSize, _pool.Count - newChunkSize); - _pool.Capacity = _chunkSizeLimit * 2; - } - } - - private void Dispose() - { - _pool = null; - } - } -} diff --git a/ARMeilleure/Common/ThreadStaticPoolEnums.cs b/ARMeilleure/Common/ThreadStaticPoolEnums.cs deleted file mode 100644 index 0d1d98d3cb..0000000000 --- a/ARMeilleure/Common/ThreadStaticPoolEnums.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace ARMeilleure.Common -{ - public enum PoolSizeIncrement - { - Default = 200 - } - - public enum ChunkSizeLimit - { - Large = 200000 / PoolSizeIncrement.Default, - Medium = 100000 / PoolSizeIncrement.Default, - Small = 50000 / PoolSizeIncrement.Default - } -} \ No newline at end of file diff --git a/ARMeilleure/Diagnostics/IRDumper.cs b/ARMeilleure/Diagnostics/IRDumper.cs index 1a2ffae9c7..3d1a60e585 100644 --- a/ARMeilleure/Diagnostics/IRDumper.cs +++ b/ARMeilleure/Diagnostics/IRDumper.cs @@ -18,7 +18,7 @@ namespace ARMeilleure.Diagnostics private readonly Dictionary _localNames; private readonly Dictionary _symbolNames; - private IRDumper(int indent) + public IRDumper(int indent) { _indentLevel = indent; @@ -62,15 +62,15 @@ namespace ARMeilleure.Diagnostics _builder.Append(" cold"); } - if (block.SuccessorCount > 0) + if (block.SuccessorsCount > 0) { _builder.Append(" ("); - for (int i = 0; i < block.SuccessorCount; i++) + for (int i = 0; i < block.SuccessorsCount; i++) { DumpBlockName(block.GetSuccessor(i)); - if (i < block.SuccessorCount - 1) + if (i < block.SuccessorsCount - 1) { _builder.Append(", "); } @@ -84,7 +84,7 @@ namespace ARMeilleure.Diagnostics private void DumpOperand(Operand operand) { - if (operand == null) + if (operand == default) { _builder.Append(""); return; @@ -131,13 +131,13 @@ namespace ARMeilleure.Diagnostics break; case OperandKind.Memory: - var memOp = (MemoryOperand)operand; + var memOp = operand.GetMemory(); _builder.Append('['); DumpOperand(memOp.BaseAddress); - if (memOp.Index != null) + if (memOp.Index != default) { _builder.Append(" + "); @@ -165,7 +165,7 @@ namespace ARMeilleure.Diagnostics } } - private void DumpNode(Node node) + private void DumpNode(ControlFlowGraph cfg, Operation node) { for (int index = 0; index < node.DestinationsCount; index++) { @@ -183,38 +183,41 @@ namespace ARMeilleure.Diagnostics switch (node) { - case PhiNode phi: - _builder.Append("Phi "); - - for (int index = 0; index < phi.SourcesCount; index++) - { - _builder.Append('('); - - DumpBlockName(phi.GetBlock(index)); - - _builder.Append(": "); - - DumpOperand(phi.GetSource(index)); - - _builder.Append(')'); - - if (index < phi.SourcesCount - 1) - { - _builder.Append(", "); - } - } - break; - case Operation operation: + if (operation.Instruction == Instruction.Phi) + { + PhiOperation phi = operation.AsPhi(); + + _builder.Append("Phi "); + + for (int index = 0; index < phi.SourcesCount; index++) + { + _builder.Append('('); + + DumpBlockName(phi.GetBlock(cfg, index)); + + _builder.Append(": "); + + DumpOperand(phi.GetSource(index)); + + _builder.Append(')'); + + if (index < phi.SourcesCount - 1) + { + _builder.Append(", "); + } + } + + break; + } + bool comparison = false; _builder.Append(operation.Instruction); if (operation.Instruction == Instruction.Extended) { - var intrinOp = (IntrinsicOperation)operation; - - _builder.Append('.').Append(intrinOp.Intrinsic); + _builder.Append('.').Append(operation.Intrinsic); } else if (operation.Instruction == Instruction.BranchIf || operation.Instruction == Instruction.Compare) @@ -277,10 +280,10 @@ namespace ARMeilleure.Diagnostics dumper.IncreaseIndentation(); - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { dumper.Indent(); - dumper.DumpNode(node); + dumper.DumpNode(cfg, node); dumper._builder.AppendLine(); } diff --git a/ARMeilleure/Instructions/InstEmitAlu.cs b/ARMeilleure/Instructions/InstEmitAlu.cs index 6e2875e649..e0d10e77db 100644 --- a/ARMeilleure/Instructions/InstEmitAlu.cs +++ b/ARMeilleure/Instructions/InstEmitAlu.cs @@ -6,7 +6,7 @@ using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitAluHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitAlu32.cs b/ARMeilleure/Instructions/InstEmitAlu32.cs index f3da121c2c..49fce31d40 100644 --- a/ARMeilleure/Instructions/InstEmitAlu32.cs +++ b/ARMeilleure/Instructions/InstEmitAlu32.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitAluHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitAluHelper.cs b/ARMeilleure/Instructions/InstEmitAluHelper.cs index caef66c285..32440283f6 100644 --- a/ARMeilleure/Instructions/InstEmitAluHelper.cs +++ b/ARMeilleure/Instructions/InstEmitAluHelper.cs @@ -6,7 +6,7 @@ using System; using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitBfm.cs b/ARMeilleure/Instructions/InstEmitBfm.cs index 8fdbf6cfd0..46a7ddddd6 100644 --- a/ARMeilleure/Instructions/InstEmitBfm.cs +++ b/ARMeilleure/Instructions/InstEmitBfm.cs @@ -3,7 +3,7 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitCcmp.cs b/ARMeilleure/Instructions/InstEmitCcmp.cs index b1b0a2a1c8..7f0beb6cb1 100644 --- a/ARMeilleure/Instructions/InstEmitCcmp.cs +++ b/ARMeilleure/Instructions/InstEmitCcmp.cs @@ -6,7 +6,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitAluHelper; using static ARMeilleure.Instructions.InstEmitFlowHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitCsel.cs b/ARMeilleure/Instructions/InstEmitCsel.cs index 60baf0bc29..926b9a9edf 100644 --- a/ARMeilleure/Instructions/InstEmitCsel.cs +++ b/ARMeilleure/Instructions/InstEmitCsel.cs @@ -4,7 +4,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitFlowHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitDiv.cs b/ARMeilleure/Instructions/InstEmitDiv.cs index 0c21dd1bac..39a5c32e60 100644 --- a/ARMeilleure/Instructions/InstEmitDiv.cs +++ b/ARMeilleure/Instructions/InstEmitDiv.cs @@ -3,7 +3,7 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitException.cs b/ARMeilleure/Instructions/InstEmitException.cs index c04ba6f25b..8819824bd9 100644 --- a/ARMeilleure/Instructions/InstEmitException.cs +++ b/ARMeilleure/Instructions/InstEmitException.cs @@ -1,7 +1,7 @@ using ARMeilleure.Decoders; using ARMeilleure.Translation; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitException32.cs b/ARMeilleure/Instructions/InstEmitException32.cs index 5357e8a962..76dbbf744e 100644 --- a/ARMeilleure/Instructions/InstEmitException32.cs +++ b/ARMeilleure/Instructions/InstEmitException32.cs @@ -2,7 +2,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitFlowHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitFlow.cs b/ARMeilleure/Instructions/InstEmitFlow.cs index 2fcf50dbfd..c40eb55cf7 100644 --- a/ARMeilleure/Instructions/InstEmitFlow.cs +++ b/ARMeilleure/Instructions/InstEmitFlow.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitFlowHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitFlow32.cs b/ARMeilleure/Instructions/InstEmitFlow32.cs index d7ebc94503..6665ca5124 100644 --- a/ARMeilleure/Instructions/InstEmitFlow32.cs +++ b/ARMeilleure/Instructions/InstEmitFlow32.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitFlowHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitFlowHelper.cs b/ARMeilleure/Instructions/InstEmitFlowHelper.cs index 808d15c805..2d73745316 100644 --- a/ARMeilleure/Instructions/InstEmitFlowHelper.cs +++ b/ARMeilleure/Instructions/InstEmitFlowHelper.cs @@ -6,7 +6,7 @@ using ARMeilleure.Translation.Cache; using ARMeilleure.Translation.PTC; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -28,7 +28,7 @@ namespace ARMeilleure.Instructions { Operand cmpResult = context.TryGetComparisonResult(condition); - if (cmpResult != null) + if (cmpResult != default) { return cmpResult; } diff --git a/ARMeilleure/Instructions/InstEmitHashHelper.cs b/ARMeilleure/Instructions/InstEmitHashHelper.cs index 9206e6d5b9..1dfe771c46 100644 --- a/ARMeilleure/Instructions/InstEmitHashHelper.cs +++ b/ARMeilleure/Instructions/InstEmitHashHelper.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using System; using System.Diagnostics; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; using static ARMeilleure.Instructions.InstEmitSimdHelper; namespace ARMeilleure.Instructions diff --git a/ARMeilleure/Instructions/InstEmitHelper.cs b/ARMeilleure/Instructions/InstEmitHelper.cs index cd515c0c8a..0479c380c2 100644 --- a/ARMeilleure/Instructions/InstEmitHelper.cs +++ b/ARMeilleure/Instructions/InstEmitHelper.cs @@ -4,7 +4,7 @@ using ARMeilleure.State; using ARMeilleure.Translation; using System; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitMemory.cs b/ARMeilleure/Instructions/InstEmitMemory.cs index 87564fdcdb..b507938d55 100644 --- a/ARMeilleure/Instructions/InstEmitMemory.cs +++ b/ARMeilleure/Instructions/InstEmitMemory.cs @@ -4,7 +4,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitMemoryHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -121,7 +121,7 @@ namespace ARMeilleure.Instructions private static Operand GetAddress(ArmEmitterContext context, long addend = 0) { - Operand address = null; + Operand address = default; switch (context.CurrOp) { diff --git a/ARMeilleure/Instructions/InstEmitMemory32.cs b/ARMeilleure/Instructions/InstEmitMemory32.cs index ffd816b299..64189134cf 100644 --- a/ARMeilleure/Instructions/InstEmitMemory32.cs +++ b/ARMeilleure/Instructions/InstEmitMemory32.cs @@ -6,7 +6,7 @@ using System; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitMemoryHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -156,7 +156,7 @@ namespace ARMeilleure.Instructions Operand n = context.Copy(GetIntA32(context, op.Rn)); Operand m = GetMemM(context, setCarry: false); - Operand temp = null; + Operand temp = default; if (op.Index || op.WBack) { diff --git a/ARMeilleure/Instructions/InstEmitMemoryEx.cs b/ARMeilleure/Instructions/InstEmitMemoryEx.cs index 95be4fcfe9..522b2a4708 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryEx.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryEx.cs @@ -6,7 +6,7 @@ using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitMemoryExHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitMemoryEx32.cs b/ARMeilleure/Instructions/InstEmitMemoryEx32.cs index abe61cd87d..28fe000d14 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryEx32.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryEx32.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitMemoryExHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs b/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs index 15f5e2abc9..9a69442a69 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryExHelper.cs @@ -3,7 +3,7 @@ using ARMeilleure.State; using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -20,7 +20,7 @@ namespace ARMeilleure.Instructions if (size == 4) { // Only 128-bit CAS is guaranteed to have a atomic load. - Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: false, 4); + Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, default, write: false, 4); Operand zero = context.VectorZero(); @@ -109,7 +109,7 @@ namespace ARMeilleure.Instructions context.BranchIfTrue(lblExit, exFailed); // STEP 2: We have exclusive access and the address is valid, attempt the store using CAS. - Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, null, write: true, size); + Operand physAddr = InstEmitMemoryHelper.EmitPtPointerLoad(context, address, default, write: true, size); Operand exValuePtr = context.Add(arg0, Const((long)NativeContext.GetExclusiveValueOffset())); Operand exValue = size switch diff --git a/ARMeilleure/Instructions/InstEmitMemoryHelper.cs b/ARMeilleure/Instructions/InstEmitMemoryHelper.cs index 3bac685591..570fb02a4e 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryHelper.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryHelper.cs @@ -7,7 +7,7 @@ using System; using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -130,7 +130,7 @@ namespace ARMeilleure.Instructions Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size); - Operand value = null; + Operand value = default; switch (size) { @@ -161,7 +161,7 @@ namespace ARMeilleure.Instructions throw new ArgumentOutOfRangeException(nameof(size)); } - Operand physAddr = EmitPtPointerLoad(context, address, null, write: false, size); + Operand physAddr = EmitPtPointerLoad(context, address, default, write: false, size); return size switch { @@ -186,7 +186,7 @@ namespace ARMeilleure.Instructions Operand physAddr = EmitPtPointerLoad(context, address, lblSlowPath, write: false, size); - Operand value = null; + Operand value = default; switch (size) { @@ -257,7 +257,7 @@ namespace ARMeilleure.Instructions throw new ArgumentOutOfRangeException(nameof(size)); } - Operand physAddr = EmitPtPointerLoad(context, address, null, write: true, size); + Operand physAddr = EmitPtPointerLoad(context, address, default, write: true, size); if (size < 3 && value.Type == OperandType.I64) { @@ -348,7 +348,7 @@ namespace ARMeilleure.Instructions // If the VA is out of range, or not aligned to the access size, force PTE to 0 by masking it. pte = context.BitwiseAnd(pte, context.ShiftRightSI(context.Add(addrShifted, Const(-(long)ptLevelSize)), Const(63))); - if (lblSlowPath != null) + if (lblSlowPath != default) { if (write) { @@ -505,7 +505,7 @@ namespace ARMeilleure.Instructions case 4: info = typeof(NativeInterface).GetMethod(nameof(NativeInterface.WriteVector128)); break; } - Operand value = null; + Operand value = default; if (size < 4) { diff --git a/ARMeilleure/Instructions/InstEmitMove.cs b/ARMeilleure/Instructions/InstEmitMove.cs index bf051f3294..d551bf2da6 100644 --- a/ARMeilleure/Instructions/InstEmitMove.cs +++ b/ARMeilleure/Instructions/InstEmitMove.cs @@ -3,7 +3,7 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitMul32.cs b/ARMeilleure/Instructions/InstEmitMul32.cs index fa744d2528..92ed477285 100644 --- a/ARMeilleure/Instructions/InstEmitMul32.cs +++ b/ARMeilleure/Instructions/InstEmitMul32.cs @@ -6,7 +6,7 @@ using System; using static ARMeilleure.Instructions.InstEmitAluHelper; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs index 9d118c6757..e290e706e5 100644 --- a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs +++ b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs @@ -11,7 +11,7 @@ using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -126,10 +126,10 @@ namespace ARMeilleure.Instructions 8 => Clz_V_I8 (context, GetVec(op.Rn)), 16 => Clz_V_I16(context, GetVec(op.Rn)), 32 => Clz_V_I32(context, GetVec(op.Rn)), - _ => null + _ => default }; - if (res != null) + if (res != default) { if (op.RegisterSize == RegisterSize.Simd64) { @@ -159,7 +159,7 @@ namespace ARMeilleure.Instructions { if (!Optimizations.UseSsse3) { - return null; + return default; } // CLZ nibble table. @@ -189,7 +189,7 @@ namespace ARMeilleure.Instructions { if (!Optimizations.UseSsse3) { - return null; + return default; } Operand maskSwap = X86GetElements(context, 0x80_0f_80_0d_80_0b_80_09, 0x80_07_80_05_80_03_80_01); @@ -215,7 +215,7 @@ namespace ARMeilleure.Instructions // TODO: Use vplzcntd when AVX-512 is supported. if (!Optimizations.UseSse2) { - return null; + return default; } Operand AddVectorI32(Operand op0, Operand op1) => context.AddIntrinsic(Intrinsic.X86Paddd, op0, op1); @@ -3684,8 +3684,8 @@ namespace ARMeilleure.Instructions Operand mask2 = context.AddIntrinsic(Intrinsic.X86Pand, opF, qMask); mask2 = context.AddIntrinsic(Intrinsic.X86Cmpps, mask2, qMask, Const((int)CmpCondition.Equal)); - qNaNMask = isQNaN == null || (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andps, mask2, mask1) : null; - sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnps, mask2, mask1) : null; + qNaNMask = isQNaN == null || (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andps, mask2, mask1) : default; + sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnps, mask2, mask1) : default; } else /* if ((op.Size & 1) == 1) */ { @@ -3698,8 +3698,8 @@ namespace ARMeilleure.Instructions Operand mask2 = context.AddIntrinsic(Intrinsic.X86Pand, opF, qMask); mask2 = context.AddIntrinsic(Intrinsic.X86Cmppd, mask2, qMask, Const((int)CmpCondition.Equal)); - qNaNMask = isQNaN == null || (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andpd, mask2, mask1) : null; - sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnpd, mask2, mask1) : null; + qNaNMask = isQNaN == null || (bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andpd, mask2, mask1) : default; + sNaNMask = isQNaN == null || !(bool)isQNaN ? context.AddIntrinsic(Intrinsic.X86Andnpd, mask2, mask1) : default; } } @@ -3707,11 +3707,11 @@ namespace ARMeilleure.Instructions ArmEmitterContext context, Func2I emit, bool scalar, - Operand n = null, - Operand m = null) + Operand n = default, + Operand m = default) { - Operand nCopy = n ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)); - Operand mCopy = m ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)); + Operand nCopy = n == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)) : n; + Operand mCopy = m == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)) : m; EmitSse2VectorIsNaNOpF(context, nCopy, out Operand nQNaNMask, out Operand nSNaNMask); EmitSse2VectorIsNaNOpF(context, mCopy, out _, out Operand mSNaNMask, isQNaN: false); @@ -3734,7 +3734,7 @@ namespace ARMeilleure.Instructions Operand res = context.AddIntrinsic(Intrinsic.X86Blendvps, resNaN, emit(nCopy, mCopy), resMask); - if (n != null || m != null) + if (n != default || m != default) { return res; } @@ -3750,7 +3750,7 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res); - return null; + return default; } else /* if (sizeF == 1) */ { @@ -3768,7 +3768,7 @@ namespace ARMeilleure.Instructions Operand res = context.AddIntrinsic(Intrinsic.X86Blendvpd, resNaN, emit(nCopy, mCopy), resMask); - if (n != null || m != null) + if (n != default || m != default) { return res; } @@ -3780,7 +3780,7 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res); - return null; + return default; } } @@ -3788,11 +3788,11 @@ namespace ARMeilleure.Instructions ArmEmitterContext context, Func2I emit, bool scalar, - Operand n = null, - Operand m = null) + Operand n = default, + Operand m = default) { - Operand nCopy = n ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)); - Operand mCopy = m ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)); + Operand nCopy = n == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)) : n; + Operand mCopy = m == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)) : m; EmitSseOrAvxEnterFtzAndDazModesOpF(context, out Operand isTrue); @@ -3800,7 +3800,7 @@ namespace ARMeilleure.Instructions EmitSseOrAvxExitFtzAndDazModesOpF(context, isTrue); - if (n != null || m != null) + if (n != default || m != default) { return res; } @@ -3828,7 +3828,7 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res); - return null; + return default; } private static Operand EmitSse2VectorMaxMinOpF(ArmEmitterContext context, Operand n, Operand m, bool isMax) @@ -3865,11 +3865,11 @@ namespace ARMeilleure.Instructions ArmEmitterContext context, bool isMaxNum, bool scalar, - Operand n = null, - Operand m = null) + Operand n = default, + Operand m = default) { - Operand nCopy = n ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)); - Operand mCopy = m ?? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)); + Operand nCopy = n == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rn)) : n; + Operand mCopy = m == default ? context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rm)) : m; EmitSse2VectorIsNaNOpF(context, nCopy, out Operand nQNaNMask, out _, isQNaN: true); EmitSse2VectorIsNaNOpF(context, mCopy, out Operand mQNaNMask, out _, isQNaN: true); @@ -3896,7 +3896,7 @@ namespace ARMeilleure.Instructions }, scalar: scalar, op1, op2); }, scalar: scalar, nCopy, mCopy); - if (n != null || m != null) + if (n != default || m != default) { return res; } @@ -3912,7 +3912,7 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res); - return null; + return default; } else /* if (sizeF == 1) */ { @@ -3934,7 +3934,7 @@ namespace ARMeilleure.Instructions }, scalar: scalar, op1, op2); }, scalar: scalar, nCopy, mCopy); - if (n != null || m != null) + if (n != default || m != default) { return res; } @@ -3946,7 +3946,7 @@ namespace ARMeilleure.Instructions context.Copy(GetVec(((OpCodeSimdReg)context.CurrOp).Rd), res); - return null; + return default; } } diff --git a/ARMeilleure/Instructions/InstEmitSimdArithmetic32.cs b/ARMeilleure/Instructions/InstEmitSimdArithmetic32.cs index 0d26a90f8f..68cd8d848c 100644 --- a/ARMeilleure/Instructions/InstEmitSimdArithmetic32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdArithmetic32.cs @@ -8,7 +8,7 @@ using static ARMeilleure.Instructions.InstEmitFlowHelper; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -1275,7 +1275,8 @@ namespace ARMeilleure.Instructions { OpCode32SimdSel op = (OpCode32SimdSel)context.CurrOp; - Operand condition = null; + Operand condition = default; + switch (op.Cc) { case OpCode32SimdSelMode.Eq: diff --git a/ARMeilleure/Instructions/InstEmitSimdCmp.cs b/ARMeilleure/Instructions/InstEmitSimdCmp.cs index 22cf9f21d1..71055155c7 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCmp.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCmp.cs @@ -6,7 +6,7 @@ using System; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdCmp32.cs b/ARMeilleure/Instructions/InstEmitSimdCmp32.cs index 290cc17ee4..1acc74657e 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCmp32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCmp32.cs @@ -8,7 +8,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdCvt.cs b/ARMeilleure/Instructions/InstEmitSimdCvt.cs index 0350427cb1..a5b472ec53 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCvt.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCvt.cs @@ -8,7 +8,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdCvt32.cs b/ARMeilleure/Instructions/InstEmitSimdCvt32.cs index cddeda5117..ec1ead486f 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCvt32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCvt32.cs @@ -9,7 +9,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -391,7 +391,8 @@ namespace ARMeilleure.Instructions Operand zero = context.VectorZero(); Operand nCmp; - Operand nIntOrLong2 = null; + Operand nIntOrLong2 = default; + if (!signed) { nCmp = context.AddIntrinsic(Intrinsic.X86Cmpss, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); @@ -441,7 +442,8 @@ namespace ARMeilleure.Instructions Operand zero = context.VectorZero(); Operand nCmp; - Operand nIntOrLong2 = null; + Operand nIntOrLong2 = default; + if (!signed) { nCmp = context.AddIntrinsic(Intrinsic.X86Cmpsd, nRes, zero, Const((int)CmpCondition.NotLessThanOrEqual)); @@ -510,7 +512,8 @@ namespace ARMeilleure.Instructions Operand fpMaxValMask = X86GetAllElements(context, 0x4F000000); // 2.14748365E9f (2147483648) Operand nInt = context.AddIntrinsic(Intrinsic.X86Cvtps2dq, nRes); - Operand nInt2 = null; + Operand nInt2 = default; + if (!signed) { nRes = context.AddIntrinsic(Intrinsic.X86Subps, nRes, fpMaxValMask); @@ -551,7 +554,8 @@ namespace ARMeilleure.Instructions Operand fpMaxValMask = X86GetAllElements(context, 0x43E0000000000000L); // 9.2233720368547760E18d (9223372036854775808) Operand nLong = InstEmit.EmitSse2CvtDoubleToInt64OpF(context, nRes, false); - Operand nLong2 = null; + Operand nLong2 = default; + if (!signed) { nRes = context.AddIntrinsic(Intrinsic.X86Subpd, nRes, fpMaxValMask); diff --git a/ARMeilleure/Instructions/InstEmitSimdHelper.cs b/ARMeilleure/Instructions/InstEmitSimdHelper.cs index 36602f2551..736d16a30e 100644 --- a/ARMeilleure/Instructions/InstEmitSimdHelper.cs +++ b/ARMeilleure/Instructions/InstEmitSimdHelper.cs @@ -7,7 +7,7 @@ using System.Diagnostics; using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -1279,9 +1279,11 @@ namespace ARMeilleure.Instructions context.MarkLabel(lblTrue); } - public static void EmitSseOrAvxExitFtzAndDazModesOpF(ArmEmitterContext context, Operand isTrue = null) + public static void EmitSseOrAvxExitFtzAndDazModesOpF(ArmEmitterContext context, Operand isTrue = default) { - isTrue ??= context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpcrFz))); + isTrue = isTrue == default + ? context.Call(typeof(NativeInterface).GetMethod(nameof(NativeInterface.GetFpcrFz))) + : isTrue; Operand lblTrue = Label(); context.BranchIfFalse(lblTrue, isTrue); @@ -1455,7 +1457,7 @@ namespace ARMeilleure.Instructions } else { - Operand me = null; + Operand me = default; if (byElem) { @@ -1625,7 +1627,7 @@ namespace ARMeilleure.Instructions { ThrowIfInvalid(index, size); - Operand res = null; + Operand res = default; switch (size) { diff --git a/ARMeilleure/Instructions/InstEmitSimdHelper32.cs b/ARMeilleure/Instructions/InstEmitSimdHelper32.cs index 59d3dc294f..07ff481c1d 100644 --- a/ARMeilleure/Instructions/InstEmitSimdHelper32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdHelper32.cs @@ -7,7 +7,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -1127,7 +1127,7 @@ namespace ARMeilleure.Instructions { ThrowIfInvalid(index, size); - Operand res = null; + Operand res = default; switch (size) { diff --git a/ARMeilleure/Instructions/InstEmitSimdLogical.cs b/ARMeilleure/Instructions/InstEmitSimdLogical.cs index 52a9a57629..dbd1a1a00a 100644 --- a/ARMeilleure/Instructions/InstEmitSimdLogical.cs +++ b/ARMeilleure/Instructions/InstEmitSimdLogical.cs @@ -6,7 +6,7 @@ using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdLogical32.cs b/ARMeilleure/Instructions/InstEmitSimdLogical32.cs index 48bf18bc75..dd686d4dd4 100644 --- a/ARMeilleure/Instructions/InstEmitSimdLogical32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdLogical32.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdMemory.cs b/ARMeilleure/Instructions/InstEmitSimdMemory.cs index 22e9ef7a80..9b19872afd 100644 --- a/ARMeilleure/Instructions/InstEmitSimdMemory.cs +++ b/ARMeilleure/Instructions/InstEmitSimdMemory.cs @@ -6,7 +6,7 @@ using System.Diagnostics; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitMemoryHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdMemory32.cs b/ARMeilleure/Instructions/InstEmitSimdMemory32.cs index fb9931d811..1e8f7ccd3b 100644 --- a/ARMeilleure/Instructions/InstEmitSimdMemory32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdMemory32.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitMemoryHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSimdMove.cs b/ARMeilleure/Instructions/InstEmitSimdMove.cs index 12fc71c990..b58a32f699 100644 --- a/ARMeilleure/Instructions/InstEmitSimdMove.cs +++ b/ARMeilleure/Instructions/InstEmitSimdMove.cs @@ -6,7 +6,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -639,7 +639,7 @@ namespace ARMeilleure.Instructions if (Optimizations.UseSsse3) { - Operand mask = null; + Operand mask = default; if (op.Size < 3) { @@ -707,7 +707,7 @@ namespace ARMeilleure.Instructions { if (op.RegisterSize == RegisterSize.Simd128) { - Operand mask = null; + Operand mask = default; if (op.Size < 3) { diff --git a/ARMeilleure/Instructions/InstEmitSimdMove32.cs b/ARMeilleure/Instructions/InstEmitSimdMove32.cs index 5229224240..7da180fc9c 100644 --- a/ARMeilleure/Instructions/InstEmitSimdMove32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdMove32.cs @@ -6,7 +6,7 @@ using System; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -267,7 +267,7 @@ namespace ARMeilleure.Instructions Operand selectedIndex = context.ZeroExtend8(OperandType.I32, context.VectorExtract8(m, index + op.Im)); Operand inRange = context.ICompareLess(selectedIndex, Const(byteLength)); - Operand elemRes = null; // Note: This is I64 for ease of calculation. + Operand elemRes = default; // Note: This is I64 for ease of calculation. // TODO: Branching rather than conditional select. @@ -325,7 +325,7 @@ namespace ARMeilleure.Instructions { EmitVectorShuffleOpSimd32(context, (m, d) => { - Operand mask = null; + Operand mask = default; if (op.Size < 3) { @@ -467,7 +467,7 @@ namespace ARMeilleure.Instructions { if (op.RegisterSize == RegisterSize.Simd128) { - Operand mask = null; + Operand mask = default; if (op.Size < 3) { diff --git a/ARMeilleure/Instructions/InstEmitSimdShift.cs b/ARMeilleure/Instructions/InstEmitSimdShift.cs index 62363fdec3..0ee50f305c 100644 --- a/ARMeilleure/Instructions/InstEmitSimdShift.cs +++ b/ARMeilleure/Instructions/InstEmitSimdShift.cs @@ -9,7 +9,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { @@ -39,7 +39,7 @@ namespace ARMeilleure.Instructions Operand dLow = context.VectorZeroUpper64(d); - Operand mask = null; + Operand mask = default; switch (op.Size + 1) { diff --git a/ARMeilleure/Instructions/InstEmitSimdShift32.cs b/ARMeilleure/Instructions/InstEmitSimdShift32.cs index c904c0ee55..6dcfe065b1 100644 --- a/ARMeilleure/Instructions/InstEmitSimdShift32.cs +++ b/ARMeilleure/Instructions/InstEmitSimdShift32.cs @@ -8,7 +8,7 @@ using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper; using static ARMeilleure.Instructions.InstEmitSimdHelper32; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSystem.cs b/ARMeilleure/Instructions/InstEmitSystem.cs index 827c3a7930..a5278a27bc 100644 --- a/ARMeilleure/Instructions/InstEmitSystem.cs +++ b/ARMeilleure/Instructions/InstEmitSystem.cs @@ -6,7 +6,7 @@ using System; using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/Instructions/InstEmitSystem32.cs b/ARMeilleure/Instructions/InstEmitSystem32.cs index fcd6fc8a11..9e28a1a16a 100644 --- a/ARMeilleure/Instructions/InstEmitSystem32.cs +++ b/ARMeilleure/Instructions/InstEmitSystem32.cs @@ -6,7 +6,7 @@ using System; using System.Reflection; using static ARMeilleure.Instructions.InstEmitHelper; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Instructions { diff --git a/ARMeilleure/IntermediateRepresentation/BasicBlock.cs b/ARMeilleure/IntermediateRepresentation/BasicBlock.cs index 056a9d469d..7cee52e582 100644 --- a/ARMeilleure/IntermediateRepresentation/BasicBlock.cs +++ b/ARMeilleure/IntermediateRepresentation/BasicBlock.cs @@ -1,37 +1,47 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; namespace ARMeilleure.IntermediateRepresentation { - class BasicBlock : IIntrusiveListNode + class BasicBlock : IEquatable, IIntrusiveListNode { - private readonly List _successors; + private const uint MaxSuccessors = 2; + + private int _succCount; + private BasicBlock _succ0; + private BasicBlock _succ1; + private HashSet _domFrontiers; public int Index { get; set; } - public BasicBlockFrequency Frequency { get; set; } - public BasicBlock ListPrevious { get; set; } public BasicBlock ListNext { get; set; } - - public IntrusiveList Operations { get; } - + public IntrusiveList Operations { get; } public List Predecessors { get; } - - public HashSet DominanceFrontiers { get; } public BasicBlock ImmediateDominator { get; set; } - public int SuccessorCount => _successors.Count; + public int SuccessorsCount => _succCount; + + public HashSet DominanceFrontiers + { + get + { + if (_domFrontiers == null) + { + _domFrontiers = new HashSet(); + } + + return _domFrontiers; + } + } public BasicBlock() : this(index: -1) { } public BasicBlock(int index) { - _successors = new List(); - - Operations = new IntrusiveList(); + Operations = new IntrusiveList(); Predecessors = new List(); - DominanceFrontiers = new HashSet(); Index = index; } @@ -40,54 +50,92 @@ namespace ARMeilleure.IntermediateRepresentation { if (block == null) { - throw new ArgumentNullException(nameof(block)); + ThrowNull(nameof(block)); + } + + if ((uint)_succCount + 1 > MaxSuccessors) + { + ThrowSuccessorOverflow(); } block.Predecessors.Add(this); - _successors.Add(block); + GetSuccessorUnsafe(_succCount++) = block; } public void RemoveSuccessor(int index) { - BasicBlock oldBlock = _successors[index]; + if ((uint)index >= (uint)_succCount) + { + ThrowOutOfRange(nameof(index)); + } + + ref BasicBlock oldBlock = ref GetSuccessorUnsafe(index); oldBlock.Predecessors.Remove(this); + oldBlock = null; - _successors.RemoveAt(index); + if (index == 0) + { + _succ0 = _succ1; + } + + _succCount--; } public BasicBlock GetSuccessor(int index) { - return _successors[index]; + if ((uint)index >= (uint)_succCount) + { + ThrowOutOfRange(nameof(index)); + } + + return GetSuccessorUnsafe(index); + } + + private ref BasicBlock GetSuccessorUnsafe(int index) + { + return ref Unsafe.Add(ref _succ0, index); } public void SetSuccessor(int index, BasicBlock block) { if (block == null) { - throw new ArgumentNullException(nameof(block)); + ThrowNull(nameof(block)); } - BasicBlock oldBlock = _successors[index]; + if ((uint)index >= (uint)_succCount) + { + ThrowOutOfRange(nameof(index)); + } + + ref BasicBlock oldBlock = ref GetSuccessorUnsafe(index); oldBlock.Predecessors.Remove(this); block.Predecessors.Add(this); - - _successors[index] = block; + + oldBlock = block; } - public void Append(Node node) + public void Append(Operation node) { - var lastOp = Operations.Last as Operation; + Operation last = Operations.Last; // Append node before terminal or to end if no terminal. - switch (lastOp?.Instruction) + if (last == default) + { + Operations.AddLast(node); + + return; + } + + switch (last.Instruction) { case Instruction.Return: case Instruction.Tailcall: case Instruction.BranchIf: - Operations.AddBefore(lastOp, node); + Operations.AddBefore(last, node); break; default: @@ -96,9 +144,23 @@ namespace ARMeilleure.IntermediateRepresentation } } - public Node GetLastOp() + private static void ThrowNull(string name) => throw new ArgumentNullException(name); + private static void ThrowOutOfRange(string name) => throw new ArgumentOutOfRangeException(name); + private static void ThrowSuccessorOverflow() => throw new OverflowException($"BasicBlock can only have {MaxSuccessors} successors."); + + public bool Equals(BasicBlock other) { - return Operations.Last; + return other == this; + } + + public override bool Equals(object obj) + { + return Equals(obj as BasicBlock); + } + + public override int GetHashCode() + { + return base.GetHashCode(); } } } \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/IIntrusiveListNode.cs b/ARMeilleure/IntermediateRepresentation/IIntrusiveListNode.cs index 797e789177..caa9b83fe3 100644 --- a/ARMeilleure/IntermediateRepresentation/IIntrusiveListNode.cs +++ b/ARMeilleure/IntermediateRepresentation/IIntrusiveListNode.cs @@ -1,6 +1,6 @@ namespace ARMeilleure.IntermediateRepresentation { - interface IIntrusiveListNode where T : class + interface IIntrusiveListNode { T ListPrevious { get; set; } T ListNext { get; set; } diff --git a/ARMeilleure/IntermediateRepresentation/Instruction.cs b/ARMeilleure/IntermediateRepresentation/Instruction.cs index 938a528dff..b675ed1cc5 100644 --- a/ARMeilleure/IntermediateRepresentation/Instruction.cs +++ b/ARMeilleure/IntermediateRepresentation/Instruction.cs @@ -1,6 +1,6 @@ namespace ARMeilleure.IntermediateRepresentation { - enum Instruction + enum Instruction : ushort { Add, BitwiseAnd, @@ -63,6 +63,7 @@ namespace ARMeilleure.IntermediateRepresentation Extended, Fill, LoadFromContext, + Phi, Spill, SpillArg, StoreToContext diff --git a/ARMeilleure/IntermediateRepresentation/Intrinsic.cs b/ARMeilleure/IntermediateRepresentation/Intrinsic.cs index 1ddf93e5b1..f5c5f3d7cb 100644 --- a/ARMeilleure/IntermediateRepresentation/Intrinsic.cs +++ b/ARMeilleure/IntermediateRepresentation/Intrinsic.cs @@ -1,6 +1,6 @@ namespace ARMeilleure.IntermediateRepresentation { - enum Intrinsic + enum Intrinsic : ushort { X86Addpd, X86Addps, diff --git a/ARMeilleure/IntermediateRepresentation/IntrinsicOperation.cs b/ARMeilleure/IntermediateRepresentation/IntrinsicOperation.cs deleted file mode 100644 index 34781b7005..0000000000 --- a/ARMeilleure/IntermediateRepresentation/IntrinsicOperation.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace ARMeilleure.IntermediateRepresentation -{ - class IntrinsicOperation : Operation - { - public Intrinsic Intrinsic { get; } - - public IntrinsicOperation(Intrinsic intrin, Operand dest, params Operand[] sources) : base(Instruction.Extended, dest, sources) - { - Intrinsic = intrin; - } - } -} \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/IntrusiveList.cs b/ARMeilleure/IntermediateRepresentation/IntrusiveList.cs index a7b0f7a7e3..184df87c88 100644 --- a/ARMeilleure/IntermediateRepresentation/IntrusiveList.cs +++ b/ARMeilleure/IntermediateRepresentation/IntrusiveList.cs @@ -1,4 +1,6 @@ -using System.Diagnostics; +using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.CompilerServices; namespace ARMeilleure.IntermediateRepresentation @@ -7,7 +9,7 @@ namespace ARMeilleure.IntermediateRepresentation /// Represents a efficient linked list that stores the pointer on the object directly and does not allocate. /// /// Type of the list items - class IntrusiveList where T : class, IIntrusiveListNode + class IntrusiveList where T : IEquatable, IIntrusiveListNode { /// /// First item of the list, or null if empty. @@ -24,22 +26,34 @@ namespace ARMeilleure.IntermediateRepresentation /// public int Count { get; private set; } + /// + /// Initializes a new instance of the class. + /// + /// is not pointer sized. + public IntrusiveList() + { + if (Unsafe.SizeOf() != IntPtr.Size) + { + throw new ArgumentException("T must be a reference type or a pointer sized struct."); + } + } + /// /// Adds a item as the first item of the list. /// /// Item to be added [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void AddFirst(T newNode) + public T AddFirst(T newNode) { - if (First != null) + if (!EqualsNull(First)) { - AddBefore(First, newNode); + return AddBefore(First, newNode); } else { - Debug.Assert(newNode.ListPrevious == null); - Debug.Assert(newNode.ListNext == null); - Debug.Assert(Last == null); + Debug.Assert(EqualsNull(newNode.ListPrevious)); + Debug.Assert(EqualsNull(newNode.ListNext)); + Debug.Assert(EqualsNull(Last)); First = newNode; Last = newNode; @@ -47,6 +61,8 @@ namespace ARMeilleure.IntermediateRepresentation Debug.Assert(Count == 0); Count = 1; + + return newNode; } } @@ -55,17 +71,17 @@ namespace ARMeilleure.IntermediateRepresentation /// /// Item to be added [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void AddLast(T newNode) + public T AddLast(T newNode) { - if (Last != null) + if (!EqualsNull(Last)) { - AddAfter(Last, newNode); + return AddAfter(Last, newNode); } else { - Debug.Assert(newNode.ListPrevious == null); - Debug.Assert(newNode.ListNext == null); - Debug.Assert(First == null); + Debug.Assert(EqualsNull(newNode.ListPrevious)); + Debug.Assert(EqualsNull(newNode.ListNext)); + Debug.Assert(EqualsNull(First)); First = newNode; Last = newNode; @@ -73,6 +89,8 @@ namespace ARMeilleure.IntermediateRepresentation Debug.Assert(Count == 0); Count = 1; + + return newNode; } } @@ -85,20 +103,20 @@ namespace ARMeilleure.IntermediateRepresentation [MethodImpl(MethodImplOptions.AggressiveInlining)] public T AddBefore(T node, T newNode) { - Debug.Assert(newNode.ListPrevious == null); - Debug.Assert(newNode.ListNext == null); + Debug.Assert(EqualsNull(newNode.ListPrevious)); + Debug.Assert(EqualsNull(newNode.ListNext)); newNode.ListPrevious = node.ListPrevious; newNode.ListNext = node; node.ListPrevious = newNode; - if (newNode.ListPrevious != null) + if (!EqualsNull(newNode.ListPrevious)) { newNode.ListPrevious.ListNext = newNode; } - if (First == node) + if (Equals(First, node)) { First = newNode; } @@ -117,20 +135,20 @@ namespace ARMeilleure.IntermediateRepresentation [MethodImpl(MethodImplOptions.AggressiveInlining)] public T AddAfter(T node, T newNode) { - Debug.Assert(newNode.ListPrevious == null); - Debug.Assert(newNode.ListNext == null); + Debug.Assert(EqualsNull(newNode.ListPrevious)); + Debug.Assert(EqualsNull(newNode.ListNext)); newNode.ListPrevious = node; newNode.ListNext = node.ListNext; node.ListNext = newNode; - if (newNode.ListNext != null) + if (!EqualsNull(newNode.ListNext)) { newNode.ListNext.ListPrevious = newNode; } - if (Last == node) + if (Equals(Last, node)) { Last = newNode; } @@ -147,32 +165,44 @@ namespace ARMeilleure.IntermediateRepresentation [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Remove(T node) { - if (node.ListPrevious != null) + if (!EqualsNull(node.ListPrevious)) { node.ListPrevious.ListNext = node.ListNext; } else { - Debug.Assert(First == node); + Debug.Assert(Equals(First, node)); First = node.ListNext; } - if (node.ListNext != null) + if (!EqualsNull(node.ListNext)) { node.ListNext.ListPrevious = node.ListPrevious; } else { - Debug.Assert(Last == node); + Debug.Assert(Equals(Last, node)); Last = node.ListPrevious; } - node.ListPrevious = null; - node.ListNext = null; + node.ListPrevious = default; + node.ListNext = default; Count--; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool EqualsNull(T a) + { + return EqualityComparer.Default.Equals(a, default); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool Equals(T a, T b) + { + return EqualityComparer.Default.Equals(a, b); + } } } diff --git a/ARMeilleure/IntermediateRepresentation/MemoryOperand.cs b/ARMeilleure/IntermediateRepresentation/MemoryOperand.cs index 56d07288a0..07d2633b42 100644 --- a/ARMeilleure/IntermediateRepresentation/MemoryOperand.cs +++ b/ARMeilleure/IntermediateRepresentation/MemoryOperand.cs @@ -1,29 +1,54 @@ +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; + namespace ARMeilleure.IntermediateRepresentation { - class MemoryOperand : Operand + unsafe struct MemoryOperand { - public Operand BaseAddress { get; set; } - public Operand Index { get; set; } - - public Multiplier Scale { get; private set; } - - public int Displacement { get; private set; } - - public MemoryOperand() { } - - public MemoryOperand With( - OperandType type, - Operand baseAddress, - Operand index = null, - Multiplier scale = Multiplier.x1, - int displacement = 0) + private struct Data { - With(OperandKind.Memory, type); - BaseAddress = baseAddress; - Index = index; - Scale = scale; - Displacement = displacement; - return this; +#pragma warning disable CS0649 + public byte Kind; + public byte Type; +#pragma warning restore CS0649 + public byte Scale; + public Operand BaseAddress; + public Operand Index; + public int Displacement; + } + + private Data* _data; + + public MemoryOperand(Operand operand) + { + Debug.Assert(operand.Kind == OperandKind.Memory); + + _data = (Data*)Unsafe.As(ref operand); + } + + public Operand BaseAddress + { + get => _data->BaseAddress; + set => _data->BaseAddress = value; + } + + public Operand Index + { + get => _data->Index; + set => _data->Index = value; + } + + public Multiplier Scale + { + get => (Multiplier)_data->Scale; + set => _data->Scale = (byte)value; + } + + public int Displacement + { + get => _data->Displacement; + set => _data->Displacement = value; } } } \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/Node.cs b/ARMeilleure/IntermediateRepresentation/Node.cs deleted file mode 100644 index 3f41d814ca..0000000000 --- a/ARMeilleure/IntermediateRepresentation/Node.cs +++ /dev/null @@ -1,309 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace ARMeilleure.IntermediateRepresentation -{ - class Node : IIntrusiveListNode - { - public Node ListPrevious { get; set; } - public Node ListNext { get; set; } - - public Operand Destination - { - get => _destinations.Count != 0 ? GetDestination(0) : null; - set => SetDestination(value); - } - - private readonly List _destinations; - private readonly List _sources; - private bool _clearedDest; - - public int DestinationsCount => _destinations.Count; - public int SourcesCount => _sources.Count; - - private void Resize(List list, int size) - { - if (list.Count > size) - { - list.RemoveRange(size, list.Count - size); - } - else - { - while (list.Count < size) - { - list.Add(null); - } - } - } - - public Node() - { - _destinations = new List(); - _sources = new List(); - } - - public Node(Operand destination, int sourcesCount) : this() - { - Destination = destination; - - Resize(_sources, sourcesCount); - } - - private void Reset(int sourcesCount) - { - _clearedDest = true; - _sources.Clear(); - ListPrevious = null; - ListNext = null; - - Resize(_sources, sourcesCount); - } - - public Node With(Operand destination, int sourcesCount) - { - Reset(sourcesCount); - Destination = destination; - - return this; - } - - public Node With(Operand[] destinations, int sourcesCount) - { - Reset(sourcesCount); - SetDestinations(destinations ?? throw new ArgumentNullException(nameof(destinations))); - - return this; - } - - public Operand GetDestination(int index) - { - return _destinations[index]; - } - - public Operand GetSource(int index) - { - return _sources[index]; - } - - public void SetDestination(int index, Operand destination) - { - if (!_clearedDest) - { - RemoveAssignment(_destinations[index]); - } - - AddAssignment(destination); - - _clearedDest = false; - - _destinations[index] = destination; - } - - public void SetSource(int index, Operand source) - { - RemoveUse(_sources[index]); - - AddUse(source); - - _sources[index] = source; - } - - private void RemoveOldDestinations() - { - if (!_clearedDest) - { - for (int index = 0; index < _destinations.Count; index++) - { - RemoveAssignment(_destinations[index]); - } - } - - _clearedDest = false; - } - - public void SetDestination(Operand destination) - { - RemoveOldDestinations(); - - if (destination == null) - { - _destinations.Clear(); - _clearedDest = true; - } - else - { - Resize(_destinations, 1); - - _destinations[0] = destination; - - AddAssignment(destination); - } - } - - public void SetDestinations(Operand[] destinations) - { - RemoveOldDestinations(); - - Resize(_destinations, destinations.Length); - - for (int index = 0; index < destinations.Length; index++) - { - Operand newOp = destinations[index]; - - _destinations[index] = newOp; - - AddAssignment(newOp); - } - } - - private void RemoveOldSources() - { - for (int index = 0; index < _sources.Count; index++) - { - RemoveUse(_sources[index]); - } - } - - public void SetSource(Operand source) - { - RemoveOldSources(); - - if (source == null) - { - _sources.Clear(); - } - else - { - Resize(_sources, 1); - - _sources[0] = source; - - AddUse(source); - } - } - - public void SetSources(Operand[] sources) - { - RemoveOldSources(); - - Resize(_sources, sources.Length); - - for (int index = 0; index < sources.Length; index++) - { - Operand newOp = sources[index]; - - _sources[index] = newOp; - - AddUse(newOp); - } - } - - private void AddAssignment(Operand op) - { - if (op == null) - { - return; - } - - if (op.Kind == OperandKind.LocalVariable) - { - op.Assignments.Add(this); - } - else if (op.Kind == OperandKind.Memory) - { - MemoryOperand memOp = (MemoryOperand)op; - - if (memOp.BaseAddress != null) - { - memOp.BaseAddress.Assignments.Add(this); - } - - if (memOp.Index != null) - { - memOp.Index.Assignments.Add(this); - } - } - } - - private void RemoveAssignment(Operand op) - { - if (op == null) - { - return; - } - - if (op.Kind == OperandKind.LocalVariable) - { - op.Assignments.Remove(this); - } - else if (op.Kind == OperandKind.Memory) - { - MemoryOperand memOp = (MemoryOperand)op; - - if (memOp.BaseAddress != null) - { - memOp.BaseAddress.Assignments.Remove(this); - } - - if (memOp.Index != null) - { - memOp.Index.Assignments.Remove(this); - } - } - } - - private void AddUse(Operand op) - { - if (op == null) - { - return; - } - - if (op.Kind == OperandKind.LocalVariable) - { - op.Uses.Add(this); - } - else if (op.Kind == OperandKind.Memory) - { - MemoryOperand memOp = (MemoryOperand)op; - - if (memOp.BaseAddress != null) - { - memOp.BaseAddress.Uses.Add(this); - } - - if (memOp.Index != null) - { - memOp.Index.Uses.Add(this); - } - } - } - - private void RemoveUse(Operand op) - { - if (op == null) - { - return; - } - - if (op.Kind == OperandKind.LocalVariable) - { - op.Uses.Remove(this); - } - else if (op.Kind == OperandKind.Memory) - { - MemoryOperand memOp = (MemoryOperand)op; - - if (memOp.BaseAddress != null) - { - memOp.BaseAddress.Uses.Remove(this); - } - - if (memOp.Index != null) - { - memOp.Index.Uses.Remove(this); - } - } - } - } -} \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/Operand.cs b/ARMeilleure/IntermediateRepresentation/Operand.cs index 64df416f54..2e0b649c05 100644 --- a/ARMeilleure/IntermediateRepresentation/Operand.cs +++ b/ARMeilleure/IntermediateRepresentation/Operand.cs @@ -1,3 +1,4 @@ +using ARMeilleure.Common; using ARMeilleure.Translation.PTC; using System; using System.Collections.Generic; @@ -6,96 +7,108 @@ using System.Runtime.CompilerServices; namespace ARMeilleure.IntermediateRepresentation { - class Operand + unsafe struct Operand : IEquatable { - public OperandKind Kind { get; private set; } - public OperandType Type { get; private set; } + internal struct Data + { + public byte Kind; + public byte Type; + public byte SymbolType; + public ushort AssignmentsCount; + public ushort AssignmentsCapacity; + public ushort UsesCount; + public ushort UsesCapacity; + public Operation* Assignments; + public Operation* Uses; + public ulong Value; + public ulong SymbolValue; + } - public ulong Value { get; private set; } + private Data* _data; - public List Assignments { get; } - public List Uses { get; } + public OperandKind Kind + { + get => (OperandKind)_data->Kind; + private set => _data->Kind = (byte)value; + } + + public OperandType Type + { + get => (OperandType)_data->Type; + private set => _data->Type = (byte)value; + } + + public ulong Value + { + get => _data->Value; + private set => _data->Value = value; + } + + public Symbol Symbol + { + get + { + Debug.Assert(Kind != OperandKind.Memory); + + return new Symbol((SymbolType)_data->SymbolType, _data->SymbolValue); + } + private set + { + Debug.Assert(Kind != OperandKind.Memory); + + if (value.Type == SymbolType.None) + { + _data->SymbolType = (byte)SymbolType.None; + } + else + { + _data->SymbolType = (byte)value.Type; + _data->SymbolValue = value.Value; + } + } + } + + public ReadOnlySpan Assignments + { + get + { + Debug.Assert(Kind != OperandKind.Memory); + + return new ReadOnlySpan(_data->Assignments, _data->AssignmentsCount); + } + } + + public ReadOnlySpan Uses + { + get + { + Debug.Assert(Kind != OperandKind.Memory); + + return new ReadOnlySpan(_data->Uses, _data->UsesCount); + } + } + + public int UsesCount => _data->UsesCount; + public int AssignmentsCount => _data->AssignmentsCount; - public Symbol Symbol { get; private set; } public bool Relocatable => Symbol.Type != SymbolType.None; - public Operand() - { - Assignments = new List(); - Uses = new List(); - } - - public Operand(OperandKind kind, OperandType type = OperandType.None) : this() - { - Kind = kind; - Type = type; - } - - public Operand With( - OperandKind kind, - OperandType type = OperandType.None, - ulong value = 0, - Symbol symbol = default) - { - Kind = kind; - Type = type; - - Value = value; - - Symbol = symbol; - - Assignments.Clear(); - Uses.Clear(); - - return this; - } - - public Operand With(int value) - { - return With(OperandKind.Constant, OperandType.I32, (uint)value); - } - - public Operand With(uint value) - { - return With(OperandKind.Constant, OperandType.I32, value); - } - - public Operand With(long value) - { - return With(OperandKind.Constant, OperandType.I64, (ulong)value); - } - - public Operand With(long value, Symbol symbol) - { - return With(OperandKind.Constant, OperandType.I64, (ulong)value, symbol); - } - - public Operand With(ulong value) - { - return With(OperandKind.Constant, OperandType.I64, value); - } - - public Operand With(float value) - { - return With(OperandKind.Constant, OperandType.FP32, (ulong)BitConverter.SingleToInt32Bits(value)); - } - - public Operand With(double value) - { - return With(OperandKind.Constant, OperandType.FP64, (ulong)BitConverter.DoubleToInt64Bits(value)); - } - - public Operand With(int index, RegisterType regType, OperandType type) - { - return With(OperandKind.Register, type, (ulong)((int)regType << 24 | index)); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public Register GetRegister() { + Debug.Assert(Kind == OperandKind.Register); + return new Register((int)Value & 0xffffff, (RegisterType)(Value >> 24)); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public MemoryOperand GetMemory() + { + Debug.Assert(Kind == OperandKind.Memory); + + return new MemoryOperand(this); + } + public int GetLocalNumber() { Debug.Assert(Kind == OperandKind.LocalVariable); @@ -133,6 +146,11 @@ namespace ARMeilleure.IntermediateRepresentation return BitConverter.Int64BitsToDouble((long)Value); } + internal ref ulong GetValueUnsafe() + { + return ref _data->Value; + } + internal void NumberLocal(int number) { if (Kind != OperandKind.LocalVariable) @@ -143,6 +161,158 @@ namespace ARMeilleure.IntermediateRepresentation Value = (ulong)number; } + public void AddAssignment(Operation operation) + { + if (Kind == OperandKind.LocalVariable) + { + Add(operation, ref _data->Assignments, ref _data->AssignmentsCount, ref _data->AssignmentsCapacity); + } + else if (Kind == OperandKind.Memory) + { + MemoryOperand memOp = GetMemory(); + Operand addr = memOp.BaseAddress; + Operand index = memOp.Index; + + if (addr != default) + { + Add(operation, ref addr._data->Assignments, ref addr._data->AssignmentsCount, ref addr._data->AssignmentsCapacity); + } + + if (index != default) + { + Add(operation, ref index._data->Assignments, ref index._data->AssignmentsCount, ref index._data->AssignmentsCapacity); + } + } + } + + public void RemoveAssignment(Operation operation) + { + if (Kind == OperandKind.LocalVariable) + { + Remove(operation, ref _data->Assignments, ref _data->AssignmentsCount); + } + else if (Kind == OperandKind.Memory) + { + MemoryOperand memOp = GetMemory(); + Operand addr = memOp.BaseAddress; + Operand index = memOp.Index; + + if (addr != default) + { + Remove(operation, ref addr._data->Assignments, ref addr._data->AssignmentsCount); + } + + if (index != default) + { + Remove(operation, ref index._data->Assignments, ref index._data->AssignmentsCount); + } + } + } + + public void AddUse(Operation operation) + { + if (Kind == OperandKind.LocalVariable) + { + Add(operation, ref _data->Uses, ref _data->UsesCount, ref _data->UsesCapacity); + } + else if (Kind == OperandKind.Memory) + { + MemoryOperand memOp = GetMemory(); + Operand addr = memOp.BaseAddress; + Operand index = memOp.Index; + + if (addr != default) + { + Add(operation, ref addr._data->Uses, ref addr._data->UsesCount, ref addr._data->UsesCapacity); + } + + if (index != default) + { + Add(operation, ref index._data->Uses, ref index._data->UsesCount, ref index._data->UsesCapacity); + } + } + } + + public void RemoveUse(Operation operation) + { + if (Kind == OperandKind.LocalVariable) + { + Remove(operation, ref _data->Uses, ref _data->UsesCount); + } + else if (Kind == OperandKind.Memory) + { + MemoryOperand memOp = GetMemory(); + Operand addr = memOp.BaseAddress; + Operand index = memOp.Index; + + if (addr != default) + { + Remove(operation, ref addr._data->Uses, ref addr._data->UsesCount); + } + + if (index != default) + { + Remove(operation, ref index._data->Uses, ref index._data->UsesCount); + } + } + } + + private static void New(ref T* data, ref ushort count, ref ushort capacity, ushort initialCapacity) where T : unmanaged + { + count = 0; + capacity = initialCapacity; + data = Allocators.References.Allocate(initialCapacity); + } + + private static void Add(T item, ref T* data, ref ushort count, ref ushort capacity) where T : unmanaged + { + if (count < capacity) + { + data[(uint)count++] = item; + + return; + } + + // Could not add item in the fast path, fallback onto the slow path. + ExpandAdd(item, ref data, ref count, ref capacity); + + static void ExpandAdd(T item, ref T* data, ref ushort count, ref ushort capacity) + { + ushort newCount = checked((ushort)(count + 1)); + ushort newCapacity = (ushort)Math.Min(capacity * 2, ushort.MaxValue); + + var oldSpan = new Span(data, count); + + capacity = newCapacity; + data = Allocators.References.Allocate(capacity); + + oldSpan.CopyTo(new Span(data, count)); + + data[count] = item; + count = newCount; + } + } + + private static void Remove(in T item, ref T* data, ref ushort count) where T : unmanaged + { + var span = new Span(data, count); + + for (int i = 0; i < span.Length; i++) + { + if (EqualityComparer.Default.Equals(span[i], item)) + { + if (i + 1 < count) + { + span.Slice(i + 1).CopyTo(span.Slice(i)); + } + + count--; + + return; + } + } + } + public override int GetHashCode() { if (Kind == OperandKind.LocalVariable) @@ -154,5 +324,201 @@ namespace ARMeilleure.IntermediateRepresentation return (int)Value ^ ((int)Kind << 16) ^ ((int)Type << 20); } } + + public bool Equals(Operand operand) + { + return operand._data == _data; + } + + public override bool Equals(object obj) + { + return obj is Operand operand && Equals(operand); + } + + public static bool operator ==(Operand a, Operand b) + { + return a.Equals(b); + } + + public static bool operator !=(Operand a, Operand b) + { + return !a.Equals(b); + } + + public static class Factory + { + private const int InternTableSize = 256; + private const int InternTableProbeLength = 8; + + [ThreadStatic] + private static Data* _internTable; + + private static Data* InternTable + { + get + { + if (_internTable == null) + { + _internTable = (Data*)NativeAllocator.Instance.Allocate((uint)sizeof(Data) * InternTableSize); + + // Make sure the table is zeroed. + new Span(_internTable, InternTableSize).Clear(); + } + + return _internTable; + } + } + + private static Operand Make(OperandKind kind, OperandType type, ulong value, Symbol symbol = default) + { + Debug.Assert(kind != OperandKind.None); + + Data* data = null; + + // If constant or register, then try to look up in the intern table before allocating. + if (kind == OperandKind.Constant || kind == OperandKind.Register) + { + uint hash = (uint)HashCode.Combine(kind, type, value); + + // Look in the next InternTableProbeLength slots for a match. + for (uint i = 0; i < InternTableProbeLength; i++) + { + Operand interned = new(); + interned._data = &InternTable[(hash + i) % InternTableSize]; + + // If slot matches the allocation request then return that slot. + if (interned.Kind == kind && interned.Type == type && interned.Value == value && interned.Symbol == symbol) + { + return interned; + } + // Otherwise if the slot is not occupied, we store in that slot. + else if (interned.Kind == OperandKind.None) + { + data = interned._data; + + break; + } + } + } + + // If we could not get a slot from the intern table, we allocate somewhere else and store there. + if (data == null) + { + data = Allocators.Operands.Allocate(); + } + + *data = default; + + Operand result = new(); + result._data = data; + result.Value = value; + result.Kind = kind; + result.Type = type; + + if (kind != OperandKind.Memory) + { + result.Symbol = symbol; + } + + // If local variable, then the use and def list is initialized with default sizes. + if (kind == OperandKind.LocalVariable) + { + New(ref result._data->Assignments, ref result._data->AssignmentsCount, ref result._data->AssignmentsCapacity, 1); + New(ref result._data->Uses, ref result._data->UsesCount, ref result._data->UsesCapacity, 4); + } + + return result; + } + + public static Operand Const(OperandType type, long value) + { + Debug.Assert(type is OperandType.I32 or OperandType.I64); + + return type == OperandType.I32 ? Const((int)value) : Const(value); + } + + public static Operand Const(bool value) + { + return Const(value ? 1 : 0); + } + + public static Operand Const(int value) + { + return Const((uint)value); + } + + public static Operand Const(uint value) + { + return Make(OperandKind.Constant, OperandType.I32, value); + } + + public static Operand Const(long value) + { + return Const(value, symbol: default); + } + + public static Operand Const(ref T reference, Symbol symbol = default) + { + return Const((long)Unsafe.AsPointer(ref reference), symbol); + } + + public static Operand Const(long value, Symbol symbol) + { + return Make(OperandKind.Constant, OperandType.I64, (ulong)value, symbol); + } + + public static Operand Const(ulong value) + { + return Make(OperandKind.Constant, OperandType.I64, value); + } + + public static Operand ConstF(float value) + { + return Make(OperandKind.Constant, OperandType.FP32, (ulong)BitConverter.SingleToInt32Bits(value)); + } + + public static Operand ConstF(double value) + { + return Make(OperandKind.Constant, OperandType.FP64, (ulong)BitConverter.DoubleToInt64Bits(value)); + } + + public static Operand Label() + { + return Make(OperandKind.Label, OperandType.None, 0); + } + + public static Operand Local(OperandType type) + { + return Make(OperandKind.LocalVariable, type, 0); + } + + public static Operand Register(int index, RegisterType regType, OperandType type) + { + return Make(OperandKind.Register, type, (ulong)((int)regType << 24 | index)); + } + + public static Operand Undef() + { + return Make(OperandKind.Undefined, OperandType.None, 0); + } + + public static Operand MemoryOp( + OperandType type, + Operand baseAddress, + Operand index = default, + Multiplier scale = Multiplier.x1, + int displacement = 0) + { + Operand result = Make(OperandKind.Memory, type, 0); + + MemoryOperand memory = result.GetMemory(); + memory.BaseAddress = baseAddress; + memory.Index = index; + memory.Scale = scale; + memory.Displacement = displacement; + + return result; + } + } } } \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs b/ARMeilleure/IntermediateRepresentation/OperandHelper.cs deleted file mode 100644 index 420555a785..0000000000 --- a/ARMeilleure/IntermediateRepresentation/OperandHelper.cs +++ /dev/null @@ -1,119 +0,0 @@ -using ARMeilleure.Common; -using ARMeilleure.Translation.PTC; -using System.Runtime.CompilerServices; - -namespace ARMeilleure.IntermediateRepresentation -{ - static class OperandHelper - { - public static Operand Const(OperandType type, long value) - { - return type == OperandType.I32 ? Operand().With((int)value) : Operand().With(value); - } - - public static Operand Const(bool value) - { - return Operand().With(value ? 1 : 0); - } - - public static Operand Const(int value) - { - return Operand().With(value); - } - - public static Operand Const(uint value) - { - return Operand().With(value); - } - - public static Operand Const(long value) - { - return Operand().With(value); - } - - public static Operand Const(long value, Symbol symbol) - { - return Operand().With(value, symbol); - } - - public static Operand Const(ulong value) - { - return Operand().With(value); - } - - public static unsafe Operand Const(ref T reference, Symbol symbol = default) - { - return Operand().With((long)Unsafe.AsPointer(ref reference), symbol); - } - - public static Operand ConstF(float value) - { - return Operand().With(value); - } - - public static Operand ConstF(double value) - { - return Operand().With(value); - } - - public static Operand Label() - { - return Operand().With(OperandKind.Label); - } - - public static Operand Local(OperandType type) - { - return Operand().With(OperandKind.LocalVariable, type); - } - - public static Operand Register(int index, RegisterType regType, OperandType type) - { - return Operand().With(index, regType, type); - } - - public static Operand Undef() - { - return Operand().With(OperandKind.Undefined); - } - - public static MemoryOperand MemoryOp( - OperandType type, - Operand baseAddress, - Operand index = null, - Multiplier scale = Multiplier.x1, - int displacement = 0) - { - return MemoryOperand().With(type, baseAddress, index, scale, displacement); - } - - #region "ThreadStaticPool" - public static void PrepareOperandPool(int groupId = 0) - { - ThreadStaticPool.PreparePool(groupId, ChunkSizeLimit.Large); - ThreadStaticPool.PreparePool(groupId, ChunkSizeLimit.Small); - } - - private static Operand Operand() - { - return ThreadStaticPool.Instance.Allocate(); - } - - private static MemoryOperand MemoryOperand() - { - return ThreadStaticPool.Instance.Allocate(); - } - - public static void ResetOperandPool(int groupId = 0) - { - ThreadStaticPool.ResetPool(groupId); - ThreadStaticPool.ResetPool(groupId); - } - - public static void DisposeOperandPools() - { - ThreadStaticPool.DisposePools(); - ThreadStaticPool.DisposePools(); - } - #endregion - } -} diff --git a/ARMeilleure/IntermediateRepresentation/OperandKind.cs b/ARMeilleure/IntermediateRepresentation/OperandKind.cs index 5761835344..adb8356141 100644 --- a/ARMeilleure/IntermediateRepresentation/OperandKind.cs +++ b/ARMeilleure/IntermediateRepresentation/OperandKind.cs @@ -2,6 +2,7 @@ namespace ARMeilleure.IntermediateRepresentation { enum OperandKind { + None, Constant, Label, LocalVariable, diff --git a/ARMeilleure/IntermediateRepresentation/Operation.cs b/ARMeilleure/IntermediateRepresentation/Operation.cs index 4cdbe3261a..08b874cf1a 100644 --- a/ARMeilleure/IntermediateRepresentation/Operation.cs +++ b/ARMeilleure/IntermediateRepresentation/Operation.cs @@ -1,89 +1,180 @@ +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; + namespace ARMeilleure.IntermediateRepresentation { - class Operation : Node + unsafe struct Operation : IEquatable, IIntrusiveListNode { - public Instruction Instruction { get; private set; } - - public Operation() : base() { } - - public Operation( - Instruction instruction, - Operand destination, - Operand[] sources) : base(destination, sources.Length) + internal struct Data { - Instruction = instruction; + public ushort Instruction; + public ushort Intrinsic; + public ushort SourcesCount; + public ushort DestinationsCount; + public Operation ListPrevious; + public Operation ListNext; + public Operand* Destinations; + public Operand* Sources; + } - for (int index = 0; index < sources.Length; index++) + private Data* _data; + + public Instruction Instruction + { + get => (Instruction)_data->Instruction; + private set => _data->Instruction = (ushort)value; + } + + public Intrinsic Intrinsic + { + get => (Intrinsic)_data->Intrinsic; + private set => _data->Intrinsic = (ushort)value; + } + + public Operation ListPrevious + { + get => _data->ListPrevious; + set => _data->ListPrevious = value; + } + + public Operation ListNext + { + get => _data->ListNext; + set => _data->ListNext = value; + } + + public Operand Destination + { + get => _data->DestinationsCount != 0 ? GetDestination(0) : default; + set => SetDestination(value); + } + + public int DestinationsCount => _data->DestinationsCount; + public int SourcesCount => _data->SourcesCount; + + private Span Destinations => new(_data->Destinations, _data->DestinationsCount); + private Span Sources => new(_data->Sources, _data->SourcesCount); + + public PhiOperation AsPhi() + { + Debug.Assert(Instruction == Instruction.Phi); + + return new PhiOperation(this); + } + + public Operand GetDestination(int index) + { + return Destinations[index]; + } + + public Operand GetSource(int index) + { + return Sources[index]; + } + + public void SetDestination(int index, Operand dest) + { + ref Operand curDest = ref Destinations[index]; + + RemoveAssignment(curDest); + AddAssignment(dest); + + curDest = dest; + } + + public void SetSource(int index, Operand src) + { + ref Operand curSrc = ref Sources[index]; + + RemoveUse(curSrc); + AddUse(src); + + curSrc = src; + } + + private void RemoveOldDestinations() + { + for (int i = 0; i < _data->DestinationsCount; i++) { - SetSource(index, sources[index]); + RemoveAssignment(_data->Destinations[i]); } } - public Operation With(Instruction instruction, Operand destination) + public void SetDestination(Operand dest) { - With(destination, 0); - Instruction = instruction; - return this; - } + RemoveOldDestinations(); - public Operation With(Instruction instruction, Operand destination, Operand[] sources) - { - With(destination, sources.Length); - Instruction = instruction; - - for (int index = 0; index < sources.Length; index++) + if (dest == default) { - SetSource(index, sources[index]); + _data->DestinationsCount = 0; } - return this; - } - - public Operation With(Instruction instruction, Operand destination, - Operand source0) - { - With(destination, 1); - Instruction = instruction; - - SetSource(0, source0); - return this; - } - - public Operation With(Instruction instruction, Operand destination, - Operand source0, Operand source1) - { - With(destination, 2); - Instruction = instruction; - - SetSource(0, source0); - SetSource(1, source1); - return this; - } - - public Operation With(Instruction instruction, Operand destination, - Operand source0, Operand source1, Operand source2) - { - With(destination, 3); - Instruction = instruction; - - SetSource(0, source0); - SetSource(1, source1); - SetSource(2, source2); - return this; - } - - public Operation With( - Instruction instruction, - Operand[] destinations, - Operand[] sources) - { - With(destinations, sources.Length); - Instruction = instruction; - - for (int index = 0; index < sources.Length; index++) + else { - SetSource(index, sources[index]); + EnsureCapacity(ref _data->Destinations, ref _data->DestinationsCount, 1); + + _data->Destinations[0] = dest; + + AddAssignment(dest); + } + } + + public void SetDestinations(Operand[] dests) + { + RemoveOldDestinations(); + + EnsureCapacity(ref _data->Destinations, ref _data->DestinationsCount, dests.Length); + + for (int index = 0; index < dests.Length; index++) + { + Operand newOp = dests[index]; + + _data->Destinations[index] = newOp; + + AddAssignment(newOp); + } + } + + private void RemoveOldSources() + { + for (int index = 0; index < _data->SourcesCount; index++) + { + RemoveUse(_data->Sources[index]); + } + } + + public void SetSource(Operand src) + { + RemoveOldSources(); + + if (src == default) + { + _data->SourcesCount = 0; + } + else + { + EnsureCapacity(ref _data->Sources, ref _data->SourcesCount, 1); + + _data->Sources[0] = src; + + AddUse(src); + } + } + + public void SetSources(Operand[] srcs) + { + RemoveOldSources(); + + EnsureCapacity(ref _data->Sources, ref _data->SourcesCount, srcs.Length); + + for (int index = 0; index < srcs.Length; index++) + { + Operand newOp = srcs[index]; + + _data->Sources[index] = newOp; + + AddUse(newOp); } - return this; } public void TurnIntoCopy(Operand source) @@ -92,5 +183,194 @@ namespace ARMeilleure.IntermediateRepresentation SetSource(source); } + + private void AddAssignment(Operand op) + { + if (op != default) + { + op.AddAssignment(this); + } + } + + private void RemoveAssignment(Operand op) + { + if (op != default) + { + op.RemoveAssignment(this); + } + } + + private void AddUse(Operand op) + { + if (op != default) + { + op.AddUse(this); + } + } + + private void RemoveUse(Operand op) + { + if (op != default) + { + op.RemoveUse(this); + } + } + + public bool Equals(Operation operation) + { + return operation._data == _data; + } + + public override bool Equals(object obj) + { + return obj is Operation operation && Equals(operation); + } + + public override int GetHashCode() + { + return HashCode.Combine((IntPtr)_data); + } + + public static bool operator ==(Operation a, Operation b) + { + return a.Equals(b); + } + + public static bool operator !=(Operation a, Operation b) + { + return !a.Equals(b); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void EnsureCapacity(ref Operand* list, ref ushort capacity, int newCapacity) + { + if (newCapacity > ushort.MaxValue) + { + ThrowOverflow(newCapacity); + } + // We only need to allocate a new buffer if we're increasing the size. + else if (newCapacity > capacity) + { + list = Allocators.References.Allocate((uint)newCapacity); + } + + capacity = (ushort)newCapacity; + } + + private static void ThrowOverflow(int count) => + throw new OverflowException($"Exceeded maximum size for Source or Destinations. Required {count}."); + + public static class Factory + { + private static Operation Make(Instruction inst, int destCount, int srcCount) + { + Data* data = Allocators.Operations.Allocate(); + *data = default; + + Operation result = new(); + result._data = data; + result.Instruction = inst; + + EnsureCapacity(ref result._data->Destinations, ref result._data->DestinationsCount, destCount); + EnsureCapacity(ref result._data->Sources, ref result._data->SourcesCount, srcCount); + + result.Destinations.Clear(); + result.Sources.Clear(); + + return result; + } + + public static Operation Operation(Instruction inst, Operand dest) + { + Operation result = Make(inst, 0, 0); + result.SetDestination(dest); + return result; + } + + public static Operation Operation(Instruction inst, Operand dest, Operand src0) + { + Operation result = Make(inst, 0, 1); + result.SetDestination(dest); + result.SetSource(0, src0); + return result; + } + + public static Operation Operation(Instruction inst, Operand dest, Operand src0, Operand src1) + { + Operation result = Make(inst, 0, 2); + result.SetDestination(dest); + result.SetSource(0, src0); + result.SetSource(1, src1); + return result; + } + + public static Operation Operation(Instruction inst, Operand dest, Operand src0, Operand src1, Operand src2) + { + Operation result = Make(inst, 0, 3); + result.SetDestination(dest); + result.SetSource(0, src0); + result.SetSource(1, src1); + result.SetSource(2, src2); + return result; + } + + public static Operation Operation(Instruction inst, Operand dest, int srcCount) + { + Operation result = Make(inst, 0, srcCount); + result.SetDestination(dest); + return result; + } + + public static Operation Operation(Instruction inst, Operand dest, Operand[] srcs) + { + Operation result = Make(inst, 0, srcs.Length); + + result.SetDestination(dest); + + for (int index = 0; index < srcs.Length; index++) + { + result.SetSource(index, srcs[index]); + } + + return result; + } + + public static Operation Operation(Intrinsic intrin, Operand dest, params Operand[] srcs) + { + Operation result = Make(Instruction.Extended, 0, srcs.Length); + + result.Intrinsic = intrin; + result.SetDestination(dest); + + for (int index = 0; index < srcs.Length; index++) + { + result.SetSource(index, srcs[index]); + } + + return result; + } + + public static Operation Operation(Instruction inst, Operand[] dests, Operand[] srcs) + { + Operation result = Make(inst, dests.Length, srcs.Length); + + for (int index = 0; index < dests.Length; index++) + { + result.SetDestination(index, dests[index]); + } + + for (int index = 0; index < srcs.Length; index++) + { + result.SetSource(index, srcs[index]); + } + + return result; + } + + public static Operation PhiOperation(Operand dest, int srcCount) + { + return Operation(Instruction.Phi, dest, srcCount * 2); + } + } } } \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/OperationHelper.cs b/ARMeilleure/IntermediateRepresentation/OperationHelper.cs deleted file mode 100644 index 0e560ee0a2..0000000000 --- a/ARMeilleure/IntermediateRepresentation/OperationHelper.cs +++ /dev/null @@ -1,66 +0,0 @@ -using ARMeilleure.Common; - -namespace ARMeilleure.IntermediateRepresentation -{ - static class OperationHelper - { - public static Operation Operation(Instruction instruction, Operand destination) - { - return Operation().With(instruction, destination); - } - - public static Operation Operation(Instruction instruction, Operand destination, - Operand[] sources) - { - return Operation().With(instruction, destination, sources); - } - - public static Operation Operation(Instruction instruction, Operand destination, - Operand source0) - { - return Operation().With(instruction, destination, source0); - } - - public static Operation Operation(Instruction instruction, Operand destination, - Operand source0, Operand source1) - { - return Operation().With(instruction, destination, source0, source1); - } - - public static Operation Operation(Instruction instruction, Operand destination, - Operand source0, Operand source1, Operand source2) - { - return Operation().With(instruction, destination, source0, source1, source2); - } - - public static Operation Operation( - Instruction instruction, - Operand[] destinations, - Operand[] sources) - { - return Operation().With(instruction, destinations, sources); - } - - #region "ThreadStaticPool" - public static void PrepareOperationPool(int groupId = 0) - { - ThreadStaticPool.PreparePool(groupId, ChunkSizeLimit.Medium); - } - - private static Operation Operation() - { - return ThreadStaticPool.Instance.Allocate(); - } - - public static void ResetOperationPool(int groupId = 0) - { - ThreadStaticPool.ResetPool(groupId); - } - - public static void DisposeOperationPools() - { - ThreadStaticPool.DisposePools(); - } - #endregion - } -} diff --git a/ARMeilleure/IntermediateRepresentation/PhiNode.cs b/ARMeilleure/IntermediateRepresentation/PhiNode.cs deleted file mode 100644 index 30fc4d384c..0000000000 --- a/ARMeilleure/IntermediateRepresentation/PhiNode.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace ARMeilleure.IntermediateRepresentation -{ - class PhiNode : Node - { - private BasicBlock[] _blocks; - - public PhiNode(Operand destination, int predecessorsCount) : base(destination, predecessorsCount) - { - _blocks = new BasicBlock[predecessorsCount]; - } - - public BasicBlock GetBlock(int index) - { - return _blocks[index]; - } - - public void SetBlock(int index, BasicBlock block) - { - _blocks[index] = block; - } - } -} \ No newline at end of file diff --git a/ARMeilleure/IntermediateRepresentation/PhiOperation.cs b/ARMeilleure/IntermediateRepresentation/PhiOperation.cs new file mode 100644 index 0000000000..f24308820d --- /dev/null +++ b/ARMeilleure/IntermediateRepresentation/PhiOperation.cs @@ -0,0 +1,37 @@ +using ARMeilleure.Translation; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; + +namespace ARMeilleure.IntermediateRepresentation +{ + struct PhiOperation + { + private readonly Operation _operation; + + public PhiOperation(Operation operation) + { + _operation = operation; + } + + public int SourcesCount => _operation.SourcesCount / 2; + + public BasicBlock GetBlock(ControlFlowGraph cfg, int index) + { + return cfg.PostOrderBlocks[cfg.PostOrderMap[_operation.GetSource(index * 2).AsInt32()]]; + } + + public void SetBlock(int index, BasicBlock block) + { + _operation.SetSource(index * 2, Const(block.Index)); + } + + public Operand GetSource(int index) + { + return _operation.GetSource(index * 2 + 1); + } + + public void SetSource(int index, Operand operand) + { + _operation.SetSource(index * 2 + 1, operand); + } + } +} diff --git a/ARMeilleure/Signal/NativeSignalHandler.cs b/ARMeilleure/Signal/NativeSignalHandler.cs index e5387ca622..18d19b82fb 100644 --- a/ARMeilleure/Signal/NativeSignalHandler.cs +++ b/ARMeilleure/Signal/NativeSignalHandler.cs @@ -4,7 +4,7 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Signal { @@ -95,8 +95,6 @@ namespace ARMeilleure.Signal { if (_initialized) return; - Translator.PreparePool(); - bool unix = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX); ref SignalHandlerConfig config = ref GetConfigRef(); @@ -124,10 +122,6 @@ namespace ARMeilleure.Signal _signalHandlerHandle = WindowsSignalHandlerRegistration.RegisterExceptionHandler(_signalHandlerPtr); } - Translator.ResetPool(); - - Translator.DisposePools(); - _initialized = true; } } diff --git a/ARMeilleure/Translation/ArmEmitterContext.cs b/ARMeilleure/Translation/ArmEmitterContext.cs index 7a82b27b14..563775ee98 100644 --- a/ARMeilleure/Translation/ArmEmitterContext.cs +++ b/ARMeilleure/Translation/ArmEmitterContext.cs @@ -9,7 +9,7 @@ using ARMeilleure.Translation.PTC; using System; using System.Collections.Generic; using System.Reflection; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Translation { @@ -138,7 +138,7 @@ namespace ARMeilleure.Translation { if (_optOpLastCompare == null || _optOpLastCompare != _optOpLastFlagSet) { - return null; + return default; } Operand n = _optCmpTempN; @@ -193,7 +193,7 @@ namespace ARMeilleure.Translation } } - return null; + return default; } } } \ No newline at end of file diff --git a/ARMeilleure/Translation/Compiler.cs b/ARMeilleure/Translation/Compiler.cs index af718e21af..812144a13c 100644 --- a/ARMeilleure/Translation/Compiler.cs +++ b/ARMeilleure/Translation/Compiler.cs @@ -55,7 +55,7 @@ namespace ARMeilleure.Translation Logger.EndPass(PassName.SsaConstruction, cfg); - CompilerContext cctx = new CompilerContext(cfg, argTypes, retType, options); + CompilerContext cctx = new(cfg, argTypes, retType, options); return CodeGenerator.Generate(cctx, ptcInfo); } diff --git a/ARMeilleure/Translation/ControlFlowGraph.cs b/ARMeilleure/Translation/ControlFlowGraph.cs index 4c76d5dd18..3e7ff0c955 100644 --- a/ARMeilleure/Translation/ControlFlowGraph.cs +++ b/ARMeilleure/Translation/ControlFlowGraph.cs @@ -22,15 +22,12 @@ namespace ARMeilleure.Translation Blocks = blocks; LocalsCount = localsCount; - Update(removeUnreachableBlocks: true); + Update(); } - public void Update(bool removeUnreachableBlocks) + public void Update() { - if (removeUnreachableBlocks) - { - RemoveUnreachableBlocks(Blocks); - } + RemoveUnreachableBlocks(Blocks); var visited = new HashSet(); var blockStack = new Stack(); @@ -47,7 +44,7 @@ namespace ARMeilleure.Translation { bool visitedNew = false; - for (int i = 0; i < block.SuccessorCount; i++) + for (int i = 0; i < block.SuccessorsCount; i++) { BasicBlock succ = block.GetSuccessor(i); @@ -83,7 +80,7 @@ namespace ARMeilleure.Translation { Debug.Assert(block.Index != -1, "Invalid block index."); - for (int i = 0; i < block.SuccessorCount; i++) + for (int i = 0; i < block.SuccessorsCount; i++) { BasicBlock succ = block.GetSuccessor(i); @@ -105,9 +102,9 @@ namespace ARMeilleure.Translation if (!visited.Contains(block)) { - while (block.SuccessorCount > 0) + while (block.SuccessorsCount > 0) { - block.RemoveSuccessor(index: block.SuccessorCount - 1); + block.RemoveSuccessor(index: block.SuccessorsCount - 1); } blocks.Remove(block); @@ -126,7 +123,7 @@ namespace ARMeilleure.Translation { BasicBlock splitBlock = new BasicBlock(Blocks.Count); - for (int i = 0; i < predecessor.SuccessorCount; i++) + for (int i = 0; i < predecessor.SuccessorsCount; i++) { if (predecessor.GetSuccessor(i) == successor) { diff --git a/ARMeilleure/Translation/EmitterContext.cs b/ARMeilleure/Translation/EmitterContext.cs index fbd9e69137..7525a5d4e4 100644 --- a/ARMeilleure/Translation/EmitterContext.cs +++ b/ARMeilleure/Translation/EmitterContext.cs @@ -1,12 +1,10 @@ using ARMeilleure.Diagnostics; using ARMeilleure.IntermediateRepresentation; using ARMeilleure.State; -using ARMeilleure.Translation.PTC; using System; using System.Collections.Generic; using System.Reflection; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Translation { @@ -77,7 +75,7 @@ namespace ARMeilleure.Translation public void BranchIf(Operand label, Operand op1, Operand op2, Comparison comp, BasicBlockFrequency falseFreq = default) { - Add(Instruction.BranchIf, null, op1, op2, Const((int)comp)); + Add(Instruction.BranchIf, default, op1, op2, Const((int)comp)); BranchToLabel(label, uncond: false, falseFreq); } @@ -157,7 +155,7 @@ namespace ARMeilleure.Translation } else { - return Add(Instruction.Call, null, args); + return Add(Instruction.Call, default, args); } } @@ -169,7 +167,7 @@ namespace ARMeilleure.Translation Array.Copy(callArgs, 0, args, 1, callArgs.Length); - Add(Instruction.Tailcall, null, args); + Add(Instruction.Tailcall, default, args); _needsNewBlock = true; } @@ -356,7 +354,7 @@ namespace ARMeilleure.Translation public void Return(Operand op1) { - Add(Instruction.Return, null, op1); + Add(Instruction.Return, default, op1); _needsNewBlock = true; } @@ -398,17 +396,17 @@ namespace ARMeilleure.Translation public void Store(Operand address, Operand value) { - Add(Instruction.Store, null, address, value); + Add(Instruction.Store, default, address, value); } public void Store16(Operand address, Operand value) { - Add(Instruction.Store16, null, address, value); + Add(Instruction.Store16, default, address, value); } public void Store8(Operand address, Operand value) { - Add(Instruction.Store8, null, address, value); + Add(Instruction.Store8, default, address, value); } public void StoreToContext() @@ -501,11 +499,11 @@ namespace ARMeilleure.Translation } } - private Operand Add(Instruction inst, Operand dest = null) + private Operand Add(Instruction inst, Operand dest = default) { NewNextBlockIfNeeded(); - Operation operation = OperationHelper.Operation(inst, dest); + Operation operation = Operation.Factory.Operation(inst, dest); _irBlock.Operations.AddLast(operation); @@ -516,7 +514,7 @@ namespace ARMeilleure.Translation { NewNextBlockIfNeeded(); - Operation operation = OperationHelper.Operation(inst, dest, sources); + Operation operation = Operation.Factory.Operation(inst, dest, sources); _irBlock.Operations.AddLast(operation); @@ -527,7 +525,7 @@ namespace ARMeilleure.Translation { NewNextBlockIfNeeded(); - Operation operation = OperationHelper.Operation(inst, dest, source0); + Operation operation = Operation.Factory.Operation(inst, dest, source0); _irBlock.Operations.AddLast(operation); @@ -538,7 +536,7 @@ namespace ARMeilleure.Translation { NewNextBlockIfNeeded(); - Operation operation = OperationHelper.Operation(inst, dest, source0, source1); + Operation operation = Operation.Factory.Operation(inst, dest, source0, source1); _irBlock.Operations.AddLast(operation); @@ -549,7 +547,7 @@ namespace ARMeilleure.Translation { NewNextBlockIfNeeded(); - Operation operation = OperationHelper.Operation(inst, dest, source0, source1, source2); + Operation operation = Operation.Factory.Operation(inst, dest, source0, source1, source2); _irBlock.Operations.AddLast(operation); @@ -573,14 +571,14 @@ namespace ARMeilleure.Translation public void AddIntrinsicNoRet(Intrinsic intrin, params Operand[] args) { - Add(intrin, null, args); + Add(intrin, default, args); } private Operand Add(Intrinsic intrin, Operand dest, params Operand[] sources) { NewNextBlockIfNeeded(); - IntrinsicOperation operation = new IntrinsicOperation(intrin, dest, sources); + Operation operation = Operation.Factory.Operation(intrin, dest, sources); _irBlock.Operations.AddLast(operation); @@ -641,7 +639,7 @@ namespace ARMeilleure.Translation private void NextBlock(BasicBlock nextBlock) { - if (_irBlock?.SuccessorCount == 0 && !EndsWithUnconditional(_irBlock)) + if (_irBlock?.SuccessorsCount == 0 && !EndsWithUnconditional(_irBlock)) { _irBlock.AddSuccessor(nextBlock); @@ -662,9 +660,11 @@ namespace ARMeilleure.Translation private static bool EndsWithUnconditional(BasicBlock block) { - return block.Operations.Last is Operation lastOp && - (lastOp.Instruction == Instruction.Return || - lastOp.Instruction == Instruction.Tailcall); + Operation last = block.Operations.Last; + + return last != default && + (last.Instruction == Instruction.Return || + last.Instruction == Instruction.Tailcall); } public ControlFlowGraph GetControlFlowGraph() diff --git a/ARMeilleure/Translation/PTC/DegreeOfParallelism.cs b/ARMeilleure/Translation/PTC/DegreeOfParallelism.cs deleted file mode 100644 index e4752c5e3f..0000000000 --- a/ARMeilleure/Translation/PTC/DegreeOfParallelism.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; - -namespace ARMeilleure.Translation.PTC -{ - class DegreeOfParallelism - { - public double GiBRef { get; } // GiB. - public double WeightRef { get; } // %. - public double IncrementByGiB { get; } // %. - private double _coefficient; - - public DegreeOfParallelism(double gibRef, double weightRef, double incrementByGiB) - { - GiBRef = gibRef; - WeightRef = weightRef; - IncrementByGiB = incrementByGiB; - - _coefficient = weightRef - (incrementByGiB * gibRef); - } - - public int GetDegreeOfParallelism(int min, int max) - { - double degreeOfParallelism = (GetProcessorCount() * GetWeight(GetAvailableMemoryGiB())) / 100d; - - return Math.Clamp((int)Math.Round(degreeOfParallelism), min, max); - } - - public static double GetProcessorCount() - { - return (double)Environment.ProcessorCount; - } - - public double GetWeight(double gib) - { - return (IncrementByGiB * gib) + _coefficient; - } - - public static double GetAvailableMemoryGiB() - { - GCMemoryInfo gcMemoryInfo = GC.GetGCMemoryInfo(); - - return FromBytesToGiB(gcMemoryInfo.TotalAvailableMemoryBytes - gcMemoryInfo.MemoryLoadBytes); - } - - private static double FromBytesToGiB(long bytes) - { - return Math.ScaleB((double)bytes, -30); - } - } -} \ No newline at end of file diff --git a/ARMeilleure/Translation/PTC/Ptc.cs b/ARMeilleure/Translation/PTC/Ptc.cs index 9f07ca011d..1ed5494522 100644 --- a/ARMeilleure/Translation/PTC/Ptc.cs +++ b/ARMeilleure/Translation/PTC/Ptc.cs @@ -9,7 +9,6 @@ using Ryujinx.Common.Configuration; using Ryujinx.Common.Logging; using System; using System.Buffers.Binary; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -28,7 +27,7 @@ namespace ARMeilleure.Translation.PTC private const string OuterHeaderMagicString = "PTCohd\0\0"; private const string InnerHeaderMagicString = "PTCihd\0\0"; - private const uint InternalVersion = 2228; //! To be incremented manually for each change to the ARMeilleure project. + private const uint InternalVersion = 2515; //! To be incremented manually for each change to the ARMeilleure project. private const string ActualDir = "0"; private const string BackupDir = "1"; @@ -776,9 +775,7 @@ namespace ARMeilleure.Translation.PTC _translateCount = 0; _translateTotalCount = profiledFuncsToTranslate.Count; - int degreeOfParallelism = new DegreeOfParallelism(4d, 75d, 12.5d).GetDegreeOfParallelism(0, 32); - - if (_translateTotalCount == 0 || degreeOfParallelism == 0) + if (_translateTotalCount == 0) { ResetCarriersIfNeeded(); @@ -787,6 +784,14 @@ namespace ARMeilleure.Translation.PTC return; } + int degreeOfParallelism = Environment.ProcessorCount; + + // If there are enough cores lying around, we leave one alone for other tasks. + if (degreeOfParallelism > 4) + { + degreeOfParallelism--; + } + Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}"); PtcStateChanged?.Invoke(PtcLoadingState.Start, _translateCount, _translateTotalCount); @@ -825,8 +830,6 @@ namespace ARMeilleure.Translation.PTC break; } } - - Translator.DisposePools(); } List threads = new List(); @@ -839,6 +842,8 @@ namespace ARMeilleure.Translation.PTC threads.Add(thread); } + Stopwatch sw = Stopwatch.StartNew(); + threads.ForEach((thread) => thread.Start()); threads.ForEach((thread) => thread.Join()); @@ -847,9 +852,11 @@ namespace ARMeilleure.Translation.PTC progressReportEvent.Set(); progressReportThread.Join(); + sw.Stop(); + PtcStateChanged?.Invoke(PtcLoadingState.Loaded, _translateCount, _translateTotalCount); - Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism}"); + Logger.Info?.Print(LogClass.Ptc, $"{_translateCount} of {_translateTotalCount} functions translated | Thread count: {degreeOfParallelism} in {sw.Elapsed.TotalSeconds} s"); Thread preSaveThread = new Thread(PreSave); preSaveThread.IsBackground = true; diff --git a/ARMeilleure/Translation/RegisterToLocal.cs b/ARMeilleure/Translation/RegisterToLocal.cs index 088cec7ef3..abb9b373c4 100644 --- a/ARMeilleure/Translation/RegisterToLocal.cs +++ b/ARMeilleure/Translation/RegisterToLocal.cs @@ -1,7 +1,7 @@ using ARMeilleure.IntermediateRepresentation; using System.Collections.Generic; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Translation { @@ -27,11 +27,11 @@ namespace ARMeilleure.Translation for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { Operand dest = node.Destination; - if (dest != null && dest.Kind == OperandKind.Register) + if (dest != default && dest.Kind == OperandKind.Register) { node.Destination = GetLocal(dest); } diff --git a/ARMeilleure/Translation/RegisterUsage.cs b/ARMeilleure/Translation/RegisterUsage.cs index 1a97515f20..035d45409a 100644 --- a/ARMeilleure/Translation/RegisterUsage.cs +++ b/ARMeilleure/Translation/RegisterUsage.cs @@ -1,9 +1,11 @@ using ARMeilleure.IntermediateRepresentation; using ARMeilleure.State; using System; - -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using System.Numerics; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; +using static ARMeilleure.IntermediateRepresentation.Operation.Factory; namespace ARMeilleure.Translation { @@ -14,27 +16,48 @@ namespace ARMeilleure.Translation private struct RegisterMask : IEquatable { - public long IntMask { get; set; } - public long VecMask { get; set; } + public long IntMask => Mask.GetElement(0); + public long VecMask => Mask.GetElement(1); + + public Vector128 Mask { get; } + + public RegisterMask(Vector128 mask) + { + Mask = mask; + } public RegisterMask(long intMask, long vecMask) { - IntMask = intMask; - VecMask = vecMask; + Mask = Vector128.Create(intMask, vecMask); } public static RegisterMask operator &(RegisterMask x, RegisterMask y) { + if (Sse2.IsSupported) + { + return new RegisterMask(Sse2.And(x.Mask, y.Mask)); + } + return new RegisterMask(x.IntMask & y.IntMask, x.VecMask & y.VecMask); } public static RegisterMask operator |(RegisterMask x, RegisterMask y) { + if (Sse2.IsSupported) + { + return new RegisterMask(Sse2.Or(x.Mask, y.Mask)); + } + return new RegisterMask(x.IntMask | y.IntMask, x.VecMask | y.VecMask); } public static RegisterMask operator ~(RegisterMask x) { + if (Sse2.IsSupported) + { + return new RegisterMask(Sse2.AndNot(x.Mask, Vector128.AllBitsSet)); + } + return new RegisterMask(~x.IntMask, ~x.VecMask); } @@ -55,12 +78,12 @@ namespace ARMeilleure.Translation public bool Equals(RegisterMask other) { - return IntMask == other.IntMask && VecMask == other.VecMask; + return Mask.Equals(other.Mask); } public override int GetHashCode() { - return HashCode.Combine(IntMask, VecMask); + return Mask.GetHashCode(); } } @@ -72,27 +95,23 @@ namespace ARMeilleure.Translation for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { - Operation operation = node as Operation; - - for (int srcIndex = 0; srcIndex < operation.SourcesCount; srcIndex++) + for (int index = 0; index < node.SourcesCount; index++) { - Operand source = operation.GetSource(srcIndex); + Operand source = node.GetSource(index); - if (source.Kind != OperandKind.Register) + if (source.Kind == OperandKind.Register) { - continue; + Register register = source.GetRegister(); + + localInputs[block.Index] |= GetMask(register) & ~localOutputs[block.Index]; } - - Register register = source.GetRegister(); - - localInputs[block.Index] |= GetMask(register) & ~localOutputs[block.Index]; } - if (operation.Destination != null && operation.Destination.Kind == OperandKind.Register) + if (node.Destination != default && node.Destination.Kind == OperandKind.Register) { - localOutputs[block.Index] |= GetMask(operation.Destination.GetRegister()); + localOutputs[block.Index] |= GetMask(node.Destination.GetRegister()); } } } @@ -104,7 +123,6 @@ namespace ARMeilleure.Translation RegisterMask[] globalOutputs = new RegisterMask[cfg.Blocks.Count]; bool modified; - bool firstPass = true; do @@ -121,7 +139,6 @@ namespace ARMeilleure.Translation BasicBlock predecessor = block.Predecessors[0]; RegisterMask cmnOutputs = localOutputs[predecessor.Index] | globalCmnOutputs[predecessor.Index]; - RegisterMask outputs = globalOutputs[predecessor.Index]; for (int pIndex = 1; pIndex < block.Predecessors.Count; pIndex++) @@ -129,7 +146,6 @@ namespace ARMeilleure.Translation predecessor = block.Predecessors[pIndex]; cmnOutputs &= localOutputs[predecessor.Index] | globalCmnOutputs[predecessor.Index]; - outputs |= globalOutputs[predecessor.Index]; } @@ -140,21 +156,13 @@ namespace ARMeilleure.Translation cmnOutputs &= globalCmnOutputs[block.Index]; } - if (Exchange(globalCmnOutputs, block.Index, cmnOutputs)) - { - modified = true; - } - + modified |= Exchange(globalCmnOutputs, block.Index, cmnOutputs); outputs |= localOutputs[block.Index]; - - if (Exchange(globalOutputs, block.Index, globalOutputs[block.Index] | outputs)) - { - modified = true; - } + modified |= Exchange(globalOutputs, block.Index, globalOutputs[block.Index] | outputs); } - else if (Exchange(globalOutputs, block.Index, localOutputs[block.Index])) + else { - modified = true; + modified |= Exchange(globalOutputs, block.Index, localOutputs[block.Index]); } } @@ -165,17 +173,14 @@ namespace ARMeilleure.Translation RegisterMask inputs = localInputs[block.Index]; - for (int i = 0; i < block.SuccessorCount; i++) + for (int i = 0; i < block.SuccessorsCount; i++) { inputs |= globalInputs[block.GetSuccessor(i).Index]; } inputs &= ~globalCmnOutputs[block.Index]; - if (Exchange(globalInputs, block.Index, globalInputs[block.Index] | inputs)) - { - modified = true; - } + modified |= Exchange(globalInputs, block.Index, globalInputs[block.Index] | inputs); } firstPass = false; @@ -192,12 +197,18 @@ namespace ARMeilleure.Translation block.Operations.Remove(block.Operations.First); } + Operand arg = default; + // The only block without any predecessor should be the entry block. // It always needs a context load as it is the first block to run. if (block.Predecessors.Count == 0 || hasContextLoad) { - LoadLocals(block, globalInputs[block.Index].VecMask, RegisterType.Vector, mode); - LoadLocals(block, globalInputs[block.Index].IntMask, RegisterType.Integer, mode); + arg = Local(OperandType.I64); + + Operation loadArg = block.Operations.AddFirst(Operation(Instruction.LoadArgument, arg, Const(0))); + + LoadLocals(block, globalInputs[block.Index].VecMask, RegisterType.Vector, mode, loadArg, arg); + LoadLocals(block, globalInputs[block.Index].IntMask, RegisterType.Integer, mode, loadArg, arg); } bool hasContextStore = HasContextStore(block); @@ -209,8 +220,15 @@ namespace ARMeilleure.Translation if (EndsWithReturn(block) || hasContextStore) { - StoreLocals(block, globalOutputs[block.Index].IntMask, RegisterType.Integer, mode); - StoreLocals(block, globalOutputs[block.Index].VecMask, RegisterType.Vector, mode); + if (arg == default) + { + arg = Local(OperandType.I64); + + block.Append(Operation(Instruction.LoadArgument, arg, Const(0))); + } + + StoreLocals(block, globalOutputs[block.Index].IntMask, RegisterType.Integer, mode, arg); + StoreLocals(block, globalOutputs[block.Index].VecMask, RegisterType.Vector, mode, arg); } } } @@ -222,27 +240,31 @@ namespace ARMeilleure.Translation private static bool HasContextStore(BasicBlock block) { - return EndsWith(block, Instruction.StoreToContext) && block.GetLastOp().SourcesCount == 0; + return EndsWith(block, Instruction.StoreToContext) && block.Operations.Last.SourcesCount == 0; } private static bool StartsWith(BasicBlock block, Instruction inst) { - if (block.Operations.Count == 0) + if (block.Operations.Count > 0) { - return false; + Operation first = block.Operations.First; + + return first != default && first.Instruction == inst; } - return block.Operations.First is Operation operation && operation.Instruction == inst; + return false; } private static bool EndsWith(BasicBlock block, Instruction inst) { - if (block.Operations.Count == 0) + if (block.Operations.Count > 0) { - return false; + Operation last = block.Operations.Last; + + return last != default && last.Instruction == inst; } - return block.Operations.Last is Operation operation && operation.Instruction == inst; + return false; } private static RegisterMask GetMask(Register register) @@ -263,76 +285,57 @@ namespace ARMeilleure.Translation private static bool Exchange(RegisterMask[] masks, int blkIndex, RegisterMask value) { - RegisterMask oldValue = masks[blkIndex]; + ref RegisterMask curValue = ref masks[blkIndex]; - masks[blkIndex] = value; + bool changed = curValue != value; - return oldValue != value; + curValue = value; + + return changed; } - private static void LoadLocals(BasicBlock block, long inputs, RegisterType baseType, ExecutionMode mode) + private static void LoadLocals( + BasicBlock block, + long inputs, + RegisterType baseType, + ExecutionMode mode, + Operation loadArg, + Operand arg) { - Operand arg0 = Local(OperandType.I64); - - for (int bit = 63; bit >= 0; bit--) + while (inputs != 0) { - long mask = 1L << bit; - - if ((inputs & mask) == 0) - { - continue; - } + int bit = 63 - BitOperations.LeadingZeroCount((ulong)inputs); Operand dest = GetRegFromBit(bit, baseType, mode); - - long offset = NativeContext.GetRegisterOffset(dest.GetRegister()); - + Operand offset = Const((long)NativeContext.GetRegisterOffset(dest.GetRegister())); Operand addr = Local(OperandType.I64); - Operation loadOp = Operation(Instruction.Load, dest, addr); + block.Operations.AddAfter(loadArg, Operation(Instruction.Load, dest, addr)); + block.Operations.AddAfter(loadArg, Operation(Instruction.Add, addr, arg, offset)); - block.Operations.AddFirst(loadOp); - - Operation calcOffsOp = Operation(Instruction.Add, addr, arg0, Const(offset)); - - block.Operations.AddFirst(calcOffsOp); + inputs &= ~(1L << bit); } - - Operation loadArg0 = Operation(Instruction.LoadArgument, arg0, Const(0)); - - block.Operations.AddFirst(loadArg0); } - private static void StoreLocals(BasicBlock block, long outputs, RegisterType baseType, ExecutionMode mode) + private static void StoreLocals( + BasicBlock block, + long outputs, + RegisterType baseType, + ExecutionMode mode, + Operand arg) { - Operand arg0 = Local(OperandType.I64); - - Operation loadArg0 = Operation(Instruction.LoadArgument, arg0, Const(0)); - - block.Append(loadArg0); - - for (int bit = 0; bit < 64; bit++) + while (outputs != 0) { - long mask = 1L << bit; - - if ((outputs & mask) == 0) - { - continue; - } + int bit = BitOperations.TrailingZeroCount(outputs); Operand source = GetRegFromBit(bit, baseType, mode); - - long offset = NativeContext.GetRegisterOffset(source.GetRegister()); - + Operand offset = Const((long)NativeContext.GetRegisterOffset(source.GetRegister())); Operand addr = Local(OperandType.I64); - Operation calcOffsOp = Operation(Instruction.Add, addr, arg0, Const(offset)); + block.Append(Operation(Instruction.Add, addr, arg, offset)); + block.Append(Operation(Instruction.Store, default, addr, source)); - block.Append(calcOffsOp); - - Operation storeOp = Operation(Instruction.Store, null, addr, source); - - block.Append(storeOp); + outputs &= ~(1L << bit); } } @@ -340,15 +343,15 @@ namespace ARMeilleure.Translation { if (bit < RegsCount) { - return OperandHelper.Register(bit, baseType, GetOperandType(baseType, mode)); + return Register(bit, baseType, GetOperandType(baseType, mode)); } else if (baseType == RegisterType.Integer) { - return OperandHelper.Register(bit & RegsMask, RegisterType.Flag, OperandType.I32); + return Register(bit & RegsMask, RegisterType.Flag, OperandType.I32); } else if (baseType == RegisterType.Vector) { - return OperandHelper.Register(bit & RegsMask, RegisterType.FpFlag, OperandType.I32); + return Register(bit & RegsMask, RegisterType.FpFlag, OperandType.I32); } else { @@ -371,12 +374,9 @@ namespace ARMeilleure.Translation private static bool EndsWithReturn(BasicBlock block) { - if (!(block.GetLastOp() is Operation operation)) - { - return false; - } + Operation last = block.Operations.Last; - return operation.Instruction == Instruction.Return; + return last != default && last.Instruction == Instruction.Return; } } } \ No newline at end of file diff --git a/ARMeilleure/Translation/SsaConstruction.cs b/ARMeilleure/Translation/SsaConstruction.cs index 76cb9a4424..9ba7b8dfe1 100644 --- a/ARMeilleure/Translation/SsaConstruction.cs +++ b/ARMeilleure/Translation/SsaConstruction.cs @@ -4,7 +4,7 @@ using ARMeilleure.State; using System; using System.Collections.Generic; using System.Diagnostics; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Translation { @@ -18,7 +18,7 @@ namespace ARMeilleure.Translation public DefMap() { _map = new Dictionary(); - _phiMasks = new BitMap(RegisterConsts.TotalCount); + _phiMasks = new BitMap(Allocators.Default, RegisterConsts.TotalCount); } public bool TryAddOperand(int key, Operand operand) @@ -57,26 +57,26 @@ namespace ARMeilleure.Translation // First pass, get all defs and locals uses. for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { - if (node is not Operation operation) + for (int index = 0; index < node.SourcesCount; index++) { - continue; - } - - for (int index = 0; index < operation.SourcesCount; index++) - { - Operand src = operation.GetSource(index); + Operand src = node.GetSource(index); if (TryGetId(src, out int srcKey)) { - Operand local = localDefs[srcKey] ?? src; + Operand local = localDefs[srcKey]; - operation.SetSource(index, local); + if (local == default) + { + local = src; + } + + node.SetSource(index, local); } } - Operand dest = operation.Destination; + Operand dest = node.Destination; if (TryGetId(dest, out int destKey)) { @@ -84,7 +84,7 @@ namespace ARMeilleure.Translation localDefs[destKey] = local; - operation.Destination = local; + node.Destination = local; } } @@ -92,7 +92,7 @@ namespace ARMeilleure.Translation { Operand local = localDefs[key]; - if (local is null) + if (local == default) { continue; } @@ -119,28 +119,23 @@ namespace ARMeilleure.Translation // Second pass, rename variables with definitions on different blocks. for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { - for (Node node = block.Operations.First; node != null; node = node.ListNext) + for (Operation node = block.Operations.First; node != default; node = node.ListNext) { - if (node is not Operation operation) + for (int index = 0; index < node.SourcesCount; index++) { - continue; - } - - for (int index = 0; index < operation.SourcesCount; index++) - { - Operand src = operation.GetSource(index); + Operand src = node.GetSource(index); if (TryGetId(src, out int key)) { Operand local = localDefs[key]; - if (local is null) + if (local == default) { local = FindDef(globalDefs, block, src); localDefs[key] = local; } - operation.SetSource(index, local); + node.SetSource(index, local); } } } @@ -200,12 +195,14 @@ namespace ARMeilleure.Translation // then use the definition from that Phi. Operand local = Local(operand.Type); - PhiNode phi = new PhiNode(local, block.Predecessors.Count); + Operation operation = Operation.Factory.PhiOperation(local, block.Predecessors.Count); - AddPhi(block, phi); + AddPhi(block, operation); globalDefs[block.Index].TryAddOperand(GetId(operand), local); + PhiOperation phi = operation.AsPhi(); + for (int index = 0; index < block.Predecessors.Count; index++) { BasicBlock predecessor = block.Predecessors[index]; @@ -217,19 +214,19 @@ namespace ARMeilleure.Translation return local; } - private static void AddPhi(BasicBlock block, PhiNode phi) + private static void AddPhi(BasicBlock block, Operation phi) { - Node node = block.Operations.First; + Operation node = block.Operations.First; - if (node != null) + if (node != default) { - while (node.ListNext is PhiNode) + while (node.ListNext != default && node.ListNext.Instruction == Instruction.Phi) { node = node.ListNext; } } - if (node is PhiNode) + if (node != default && node.Instruction == Instruction.Phi) { block.Operations.AddAfter(node, phi); } @@ -241,34 +238,37 @@ namespace ARMeilleure.Translation private static bool TryGetId(Operand operand, out int result) { - if (operand is { Kind: OperandKind.Register }) + if (operand != default) { - Register reg = operand.GetRegister(); + if (operand.Kind == OperandKind.Register) + { + Register reg = operand.GetRegister(); - if (reg.Type == RegisterType.Integer) - { - result = reg.Index; - } - else if (reg.Type == RegisterType.Vector) - { - result = RegisterConsts.IntRegsCount + reg.Index; - } - else if (reg.Type == RegisterType.Flag) - { - result = RegisterConsts.IntAndVecRegsCount + reg.Index; - } - else /* if (reg.Type == RegisterType.FpFlag) */ - { - result = RegisterConsts.FpFlagsOffset + reg.Index; - } + if (reg.Type == RegisterType.Integer) + { + result = reg.Index; + } + else if (reg.Type == RegisterType.Vector) + { + result = RegisterConsts.IntRegsCount + reg.Index; + } + else if (reg.Type == RegisterType.Flag) + { + result = RegisterConsts.IntAndVecRegsCount + reg.Index; + } + else /* if (reg.Type == RegisterType.FpFlag) */ + { + result = RegisterConsts.FpFlagsOffset + reg.Index; + } - return true; - } - else if (operand is { Kind: OperandKind.LocalVariable } && operand.GetLocalNumber() > 0) - { - result = RegisterConsts.TotalCount + operand.GetLocalNumber() - 1; + return true; + } + else if (operand.Kind == OperandKind.LocalVariable && operand.GetLocalNumber() > 0) + { + result = RegisterConsts.TotalCount + operand.GetLocalNumber() - 1; - return true; + return true; + } } result = -1; diff --git a/ARMeilleure/Translation/SsaDeconstruction.cs b/ARMeilleure/Translation/SsaDeconstruction.cs index 2e9e3281a0..cd6bcca1fe 100644 --- a/ARMeilleure/Translation/SsaDeconstruction.cs +++ b/ARMeilleure/Translation/SsaDeconstruction.cs @@ -1,7 +1,7 @@ using ARMeilleure.IntermediateRepresentation; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; +using static ARMeilleure.IntermediateRepresentation.Operation.Factory; namespace ARMeilleure.Translation { @@ -11,34 +11,36 @@ namespace ARMeilleure.Translation { for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) { - Node node = block.Operations.First; + Operation operation = block.Operations.First; - while (node is PhiNode phi) + while (operation != default && operation.Instruction == Instruction.Phi) { - Node nextNode = node.ListNext; + Operation nextNode = operation.ListNext; - Operand local = Local(phi.Destination.Type); + Operand local = Local(operation.Destination.Type); + + PhiOperation phi = operation.AsPhi(); for (int index = 0; index < phi.SourcesCount; index++) { - BasicBlock predecessor = phi.GetBlock(index); + BasicBlock predecessor = phi.GetBlock(cfg, index); Operand source = phi.GetSource(index); predecessor.Append(Operation(Instruction.Copy, local, source)); - phi.SetSource(index, null); + phi.SetSource(index, default); } - Operation copyOp = Operation(Instruction.Copy, phi.Destination, local); + Operation copyOp = Operation(Instruction.Copy, operation.Destination, local); - block.Operations.AddBefore(node, copyOp); + block.Operations.AddBefore(operation, copyOp); - phi.Destination = null; + operation.Destination = default; - block.Operations.Remove(node); + block.Operations.Remove(operation); - node = nextNode; + operation = nextNode; } } } diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs index 2110a4e35f..03ed4c5eb1 100644 --- a/ARMeilleure/Translation/Translator.cs +++ b/ARMeilleure/Translation/Translator.cs @@ -13,12 +13,8 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; -using System.Runtime; using System.Threading; - -using static ARMeilleure.Common.BitMapPool; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; -using static ARMeilleure.IntermediateRepresentation.OperationHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Translation { @@ -193,13 +189,9 @@ namespace ARMeilleure.Translation ClearJitCache(); - DisposePools(); - Stubs.Dispose(); FunctionTable.Dispose(); CountTable.Dispose(); - - GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; } } @@ -266,8 +258,6 @@ namespace ARMeilleure.Translation Logger.EndPass(PassName.Decoding); - PreparePool(highCq ? 1 : 0); - Logger.StartPass(PassName.Translation); EmitSynchronization(context); @@ -281,7 +271,7 @@ namespace ARMeilleure.Translation ulong funcSize = funcRange.End - funcRange.Start; - Logger.EndPass(PassName.Translation); + Logger.EndPass(PassName.Translation, cfg); Logger.StartPass(PassName.RegisterUsage); @@ -298,8 +288,6 @@ namespace ARMeilleure.Translation if (!context.HasPtc) { func = Compiler.Compile(cfg, argTypes, OperandType.I64, options); - - ResetPool(highCq ? 1 : 0); } else { @@ -307,33 +295,16 @@ namespace ARMeilleure.Translation func = Compiler.Compile(cfg, argTypes, OperandType.I64, options, ptcInfo); - ResetPool(highCq ? 1 : 0); - Hash128 hash = Ptc.ComputeHash(Memory, address, funcSize); Ptc.WriteInfoCodeRelocUnwindInfo(address, funcSize, hash, highCq, ptcInfo); } - return new TranslatedFunction(func, counter, funcSize, highCq); - } + var result = new TranslatedFunction(func, counter, funcSize, highCq); - internal static void PreparePool(int groupId = 0) - { - PrepareOperandPool(groupId); - PrepareOperationPool(groupId); - } + Allocators.ResetAll(); - internal static void ResetPool(int groupId = 0) - { - ResetOperationPool(groupId); - ResetOperandPool(groupId); - } - - internal static void DisposePools() - { - DisposeOperandPools(); - DisposeOperationPools(); - DisposeBitMapPools(); + return result; } private struct Range @@ -408,7 +379,7 @@ namespace ARMeilleure.Translation EmitSynchronization(context); } - Operand lblPredicateSkip = null; + Operand lblPredicateSkip = default; if (opCode is OpCode32 op && op.Cond < Condition.Al) { @@ -426,7 +397,7 @@ namespace ARMeilleure.Translation throw new InvalidOperationException($"Invalid instruction \"{opCode.Instruction.Name}\"."); } - if (lblPredicateSkip != null) + if (lblPredicateSkip != default) { context.MarkLabel(lblPredicateSkip); } diff --git a/ARMeilleure/Translation/TranslatorStubs.cs b/ARMeilleure/Translation/TranslatorStubs.cs index aff2ac7e4c..48fa3a9478 100644 --- a/ARMeilleure/Translation/TranslatorStubs.cs +++ b/ARMeilleure/Translation/TranslatorStubs.cs @@ -5,7 +5,7 @@ using ARMeilleure.Translation.Cache; using System; using System.Reflection; using System.Runtime.InteropServices; -using static ARMeilleure.IntermediateRepresentation.OperandHelper; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; namespace ARMeilleure.Translation { @@ -145,7 +145,7 @@ namespace ARMeilleure.Translation Operand masked = context.BitwiseAnd(guestAddress, Const(~_translator.FunctionTable.Mask)); context.BranchIfTrue(lblFallback, masked); - Operand index = null; + Operand index = default; Operand page = Const((long)_translator.FunctionTable.Base); for (int i = 0; i < _translator.FunctionTable.Levels.Length; i++)