arunscape/Ryujinx
1
0
Fork 0
forked from Mirror/Ryujinx
Code Pull requests Activity

Optimize HybridAllocator (#2637)

Browse source 
* Store constant `Operand`s in the `LocalInfo`

Since the spill slot and register assigned is fixed, we can just store
the `Operand` reference in the `LocalInfo` struct. This allows skipping
hitting the intern-table for a look up.

* Skip `Uses`/`Assignments` management

Since the `HybridAllocator` is the last pass and we do not care about
uses/assignments we can skip managing that when setting destinations or
sources.

* Make `GetLocalInfo` inlineable

Also fix a possible issue where with numbered locals. See or-assignment
operator in `SetVisited(local)` before patch.

* Do not run `BlockPlacement` in LCQ

With the host mapped memory manager, there is a lot less cold code to
split from hot code. So disabling this in LCQ gives some extra
throughput - where we need it.

* Address Mou-Ikkai's feedback

* Apply suggestions from code review

Co-authored-by: VocalFan <45863583+Mou-Ikkai@users.noreply.github.com>

* Move check to an assert

Co-authored-by: VocalFan <45863583+Mou-Ikkai@users.noreply.github.com>
This commit is contained in:
FICTURE7 2021-09-29 03:38:37 +04:00 committed by GitHub
parent 1ae690ba2f
commit 312be74861
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 85 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

135
ARMeilleure/CodeGen/RegisterAllocators/HybridAllocator.cs
View file

@ -3,6 +3,7 @@ using ARMeilleure.Translation;
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using System.Numerics; using System.Numerics;
using System.Runtime.CompilerServices;
using static ARMeilleure.IntermediateRepresentation.Operand.Factory; using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
using static ARMeilleure.IntermediateRepresentation.Operation.Factory; using static ARMeilleure.IntermediateRepresentation.Operation.Factory;
@ -10,9 +11,6 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
{ {
class HybridAllocator : IRegisterAllocator class HybridAllocator : IRegisterAllocator
{ {
private const int RegistersCount = 16;
private const int MaxIROperands = 4;
private struct BlockInfo private struct BlockInfo
{ {
public bool HasCall { get; } public bool HasCall { get; }
@ -32,10 +30,10 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
{ {
public int Uses { get; set; } public int Uses { get; set; }
public int UsesAllocated { get; set; } public int UsesAllocated { get; set; }
public int Register { get; set; }
public int SpillOffset { get; set; }
public int Sequence { get; set; } public int Sequence { get; set; }
public Operand Temp { get; set; } public Operand Temp { get; set; }
public Operand Register { get; set; }
public Operand SpillOffset { get; set; }
public OperandType Type { get; } public OperandType Type { get; }
private int _first; private int _first;
@ -49,10 +47,10 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
Type = type; Type = type;
UsesAllocated = 0; UsesAllocated = 0;
Register = 0;
SpillOffset = 0;
Sequence = 0; Sequence = 0;
Temp = default; Temp = default;
Register = default;
SpillOffset = default;
_first = -1; _first = -1;
_last = -1; _last = -1;
@ -74,6 +72,38 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
} }
} }
private const int MaxIROperands = 4;
// The "visited" state is stored in the MSB of the local's value.
private const ulong VisitedMask = 1ul << 63;
private BlockInfo[] _blockInfo;
private LocalInfo[] _localInfo;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsVisited(Operand local)
{
Debug.Assert(local.Kind == OperandKind.LocalVariable);
return (local.GetValueUnsafe() & VisitedMask) != 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetVisited(Operand local)
{
Debug.Assert(local.Kind == OperandKind.LocalVariable);
local.GetValueUnsafe() |= VisitedMask;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private ref LocalInfo GetLocalInfo(Operand local)
{
Debug.Assert(local.Kind == OperandKind.LocalVariable);
Debug.Assert(IsVisited(local), "Local variable not visited. Used before defined?");
return ref _localInfo[(uint)local.GetValueUnsafe() - 1];
}
public AllocationResult RunPass(ControlFlowGraph cfg, StackAllocator stackAlloc, RegisterMasks regMasks) public AllocationResult RunPass(ControlFlowGraph cfg, StackAllocator stackAlloc, RegisterMasks regMasks)
{ {
int intUsedRegisters = 0; int intUsedRegisters = 0;
@ -82,35 +112,11 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
int intFreeRegisters = regMasks.IntAvailableRegisters; int intFreeRegisters = regMasks.IntAvailableRegisters;
int vecFreeRegisters = regMasks.VecAvailableRegisters; int vecFreeRegisters = regMasks.VecAvailableRegisters;
var blockInfo = new BlockInfo[cfg.Blocks.Count]; _blockInfo = new BlockInfo[cfg.Blocks.Count];
var localInfo = new LocalInfo[cfg.Blocks.Count * 3]; _localInfo = new LocalInfo[cfg.Blocks.Count * 3];
int localInfoCount = 0; int localInfoCount = 0;
// The "visited" state is stored in the MSB of the local's value.
const ulong VisitedMask = 1ul << 63;
bool IsVisited(Operand local)
{
return (local.GetValueUnsafe() & VisitedMask) != 0;
}
void SetVisited(Operand local)
{
local.GetValueUnsafe() |= VisitedMask | (uint)++localInfoCount;
}
ref LocalInfo GetLocalInfo(Operand local)
{
Debug.Assert(local.Kind == OperandKind.LocalVariable);
if (!IsVisited(local))
{
throw new InvalidOperationException("Local was not visisted yet. Used before defined?");
}
return ref localInfo[(uint)local.GetValueUnsafe() - 1];
}
for (int index = cfg.PostOrderBlocks.Length - 1; index >= 0; index--) for (int index = cfg.PostOrderBlocks.Length - 1; index >= 0; index--)
{ {
BasicBlock block = cfg.PostOrderBlocks[index]; BasicBlock block = cfg.PostOrderBlocks[index];
@ -127,10 +133,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
hasCall = true; hasCall = true;
} }
for (int i = 0; i < node.SourcesCount; i++) foreach (Operand source in node.SourcesUnsafe)
{ {
Operand source = node.GetSource(i);
if (source.Kind == OperandKind.LocalVariable) if (source.Kind == OperandKind.LocalVariable)
{ {
GetLocalInfo(source).SetBlockIndex(block.Index); GetLocalInfo(source).SetBlockIndex(block.Index);
@ -151,10 +155,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
} }
} }
for (int i = 0; i < node.DestinationsCount; i++) foreach (Operand dest in node.DestinationsUnsafe)
{ {
Operand dest = node.GetDestination(i);
if (dest.Kind == OperandKind.LocalVariable) if (dest.Kind == OperandKind.LocalVariable)
{ {
if (IsVisited(dest)) if (IsVisited(dest))
@ -163,13 +165,14 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
} }
else else
{ {
SetVisited(dest); dest.NumberLocal(++localInfoCount);
if (localInfoCount > localInfo.Length) if (localInfoCount > _localInfo.Length)
{ {
Array.Resize(ref localInfo, localInfoCount * 2); Array.Resize(ref _localInfo, localInfoCount * 2);
} }
SetVisited(dest);
GetLocalInfo(dest) = new LocalInfo(dest.Type, UsesCount(dest), block.Index); GetLocalInfo(dest) = new LocalInfo(dest.Type, UsesCount(dest), block.Index);
} }
} }
@ -187,7 +190,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
} }
} }
blockInfo[block.Index] = new BlockInfo(hasCall, intFixedRegisters, vecFixedRegisters); _blockInfo[block.Index] = new BlockInfo(hasCall, intFixedRegisters, vecFixedRegisters);
} }
int sequence = 0; int sequence = 0;
@ -196,7 +199,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
{ {
BasicBlock block = cfg.PostOrderBlocks[index]; BasicBlock block = cfg.PostOrderBlocks[index];
BlockInfo blkInfo = blockInfo[block.Index]; ref BlockInfo blkInfo = ref _blockInfo[block.Index];
int intLocalFreeRegisters = intFreeRegisters & ~blkInfo.IntFixedRegisters; int intLocalFreeRegisters = intFreeRegisters & ~blkInfo.IntFixedRegisters;
int vecLocalFreeRegisters = vecFreeRegisters & ~blkInfo.VecFixedRegisters; int vecLocalFreeRegisters = vecFreeRegisters & ~blkInfo.VecFixedRegisters;
@ -227,23 +230,23 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
Debug.Assert(info.UsesAllocated <= info.Uses); Debug.Assert(info.UsesAllocated <= info.Uses);
if (info.Register != -1) if (info.Register != default)
{ {
Operand reg = Register(info.Register, local.Type.ToRegisterType(), local.Type);
if (info.UsesAllocated == info.Uses) if (info.UsesAllocated == info.Uses)
{ {
Register reg = info.Register.GetRegister();
if (local.Type.IsInteger()) if (local.Type.IsInteger())
{ {
intLocalFreeRegisters |= 1 << info.Register; intLocalFreeRegisters |= 1 << reg.Index;
} }
else else
{ {
vecLocalFreeRegisters |= 1 << info.Register; vecLocalFreeRegisters |= 1 << reg.Index;
} }
} }
return reg; return info.Register;
} }
else else
{ {
@ -259,7 +262,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
info.Temp = temp; info.Temp = temp;
} }
Operation fillOp = Operation(Instruction.Fill, temp, Const(info.SpillOffset)); Operation fillOp = Operation(Instruction.Fill, temp, info.SpillOffset);
block.Operations.AddBefore(node, fillOp); block.Operations.AddBefore(node, fillOp);
@ -279,9 +282,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
{ {
ref LocalInfo info = ref GetLocalInfo(source); ref LocalInfo info = ref GetLocalInfo(source);
if (info.Register == -1) if (info.Register == default)
{ {
Operation fillOp = Operation(Instruction.Fill, node.Destination, Const(info.SpillOffset)); Operation fillOp = Operation(Instruction.Fill, node.Destination, info.SpillOffset);
block.Operations.AddBefore(node, fillOp); block.Operations.AddBefore(node, fillOp);
block.Operations.Remove(node); block.Operations.Remove(node);
@ -295,13 +298,11 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
if (!folded) if (!folded)
{ {
for (int i = 0; i < node.SourcesCount; i++) foreach (ref Operand source in node.SourcesUnsafe)
{ {
Operand source = node.GetSource(i);
if (source.Kind == OperandKind.LocalVariable) if (source.Kind == OperandKind.LocalVariable)
{ {
node.SetSource(i, AllocateRegister(source)); source = AllocateRegister(source);
} }
else if (source.Kind == OperandKind.Memory) else if (source.Kind == OperandKind.Memory)
{ {
@ -323,10 +324,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
int intLocalAsg = 0; int intLocalAsg = 0;
int vecLocalAsg = 0; int vecLocalAsg = 0;
for (int i = 0; i < node.DestinationsCount; i++) foreach (ref Operand dest in node.DestinationsUnsafe)
{ {
Operand dest = node.GetDestination(i);
if (dest.Kind != OperandKind.LocalVariable) if (dest.Kind != OperandKind.LocalVariable)
{ {
continue; continue;
@ -344,7 +343,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
{ {
int selectedReg = BitOperations.TrailingZeroCount(mask); int selectedReg = BitOperations.TrailingZeroCount(mask);
info.Register = selectedReg; info.Register = Register(selectedReg, info.Type.ToRegisterType(), info.Type);
if (dest.Type.IsInteger()) if (dest.Type.IsInteger())
{ {
@ -359,8 +358,8 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
} }
else else
{ {
info.Register = -1; info.Register = default;
info.SpillOffset = stackAlloc.Allocate(dest.Type.GetSizeInBytes()); info.SpillOffset = Const(stackAlloc.Allocate(dest.Type.GetSizeInBytes()));
} }
} }
@ -368,9 +367,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
Debug.Assert(info.UsesAllocated <= info.Uses); Debug.Assert(info.UsesAllocated <= info.Uses);
if (info.Register != -1) if (info.Register != default)
{ {
node.SetDestination(i, Register(info.Register, dest.Type.ToRegisterType(), dest.Type)); dest = info.Register;
} }
else else
{ {
@ -386,9 +385,9 @@ namespace ARMeilleure.CodeGen.RegisterAllocators
info.Temp = temp; info.Temp = temp;
} }
node.SetDestination(i, temp); dest = temp;
Operation spillOp = Operation(Instruction.Spill, default, Const(info.SpillOffset), temp); Operation spillOp = Operation(Instruction.Spill, default, info.SpillOffset, temp);
block.Operations.AddAfter(node, spillOp); block.Operations.AddAfter(node, spillOp);

14
ARMeilleure/CodeGen/X86/CodeGenerator.cs
View file

@ -97,15 +97,17 @@ namespace ARMeilleure.CodeGen.X86
Logger.StartPass(PassName.Optimization); Logger.StartPass(PassName.Optimization);
if ((cctx.Options & CompilerOptions.SsaForm) != 0 && if (cctx.Options.HasFlag(CompilerOptions.Optimize))
(cctx.Options & CompilerOptions.Optimize) != 0) {
if (cctx.Options.HasFlag(CompilerOptions.SsaForm))
{ {
Optimizer.RunPass(cfg); Optimizer.RunPass(cfg);
} }
X86Optimizer.RunPass(cfg);
BlockPlacement.RunPass(cfg); BlockPlacement.RunPass(cfg);
}
X86Optimizer.RunPass(cfg);
Logger.EndPass(PassName.Optimization, cfg); Logger.EndPass(PassName.Optimization, cfg);
@ -119,14 +121,14 @@ namespace ARMeilleure.CodeGen.X86
Logger.StartPass(PassName.RegisterAllocation); Logger.StartPass(PassName.RegisterAllocation);
if ((cctx.Options & CompilerOptions.SsaForm) != 0) if (cctx.Options.HasFlag(CompilerOptions.SsaForm))
{ {
Ssa.Deconstruct(cfg); Ssa.Deconstruct(cfg);
} }
IRegisterAllocator regAlloc; IRegisterAllocator regAlloc;
if ((cctx.Options & CompilerOptions.Lsra) != 0) if (cctx.Options.HasFlag(CompilerOptions.Lsra))
{ {
regAlloc = new LinearScanAllocator(); regAlloc = new LinearScanAllocator();
} }

1
ARMeilleure/IntermediateRepresentation/Operand.cs
View file

@ -146,6 +146,7 @@ namespace ARMeilleure.IntermediateRepresentation
return BitConverter.Int64BitsToDouble((long)Value); return BitConverter.Int64BitsToDouble((long)Value);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal ref ulong GetValueUnsafe() internal ref ulong GetValueUnsafe()
{ {
return ref _data->Value; return ref _data->Value;

16
ARMeilleure/IntermediateRepresentation/Operation.cs
View file

@ -53,8 +53,8 @@ namespace ARMeilleure.IntermediateRepresentation
public int DestinationsCount => _data->DestinationsCount; public int DestinationsCount => _data->DestinationsCount;
public int SourcesCount => _data->SourcesCount; public int SourcesCount => _data->SourcesCount;
private Span<Operand> Destinations => new(_data->Destinations, _data->DestinationsCount); internal Span<Operand> DestinationsUnsafe => new(_data->Destinations, _data->DestinationsCount);
private Span<Operand> Sources => new(_data->Sources, _data->SourcesCount); internal Span<Operand> SourcesUnsafe => new(_data->Sources, _data->SourcesCount);
public PhiOperation AsPhi() public PhiOperation AsPhi()
{ {
@ -65,17 +65,17 @@ namespace ARMeilleure.IntermediateRepresentation
public Operand GetDestination(int index) public Operand GetDestination(int index)
{ {
return Destinations[index]; return DestinationsUnsafe[index];
} }
public Operand GetSource(int index) public Operand GetSource(int index)
{ {
return Sources[index]; return SourcesUnsafe[index];
} }
public void SetDestination(int index, Operand dest) public void SetDestination(int index, Operand dest)
{ {
ref Operand curDest = ref Destinations[index]; ref Operand curDest = ref DestinationsUnsafe[index];
RemoveAssignment(curDest); RemoveAssignment(curDest);
AddAssignment(dest); AddAssignment(dest);
@ -85,7 +85,7 @@ namespace ARMeilleure.IntermediateRepresentation
public void SetSource(int index, Operand src) public void SetSource(int index, Operand src)
{ {
ref Operand curSrc = ref Sources[index]; ref Operand curSrc = ref SourcesUnsafe[index];
RemoveUse(curSrc); RemoveUse(curSrc);
AddUse(src); AddUse(src);
@ -274,8 +274,8 @@ namespace ARMeilleure.IntermediateRepresentation
EnsureCapacity(ref result._data->Destinations, ref result._data->DestinationsCount, destCount); EnsureCapacity(ref result._data->Destinations, ref result._data->DestinationsCount, destCount);
EnsureCapacity(ref result._data->Sources, ref result._data->SourcesCount, srcCount); EnsureCapacity(ref result._data->Sources, ref result._data->SourcesCount, srcCount);
result.Destinations.Clear(); result.DestinationsUnsafe.Clear();
result.Sources.Clear(); result.SourcesUnsafe.Clear();
return result; return result;
} }