[HLE/Kernel] Somewhat improved sync primitives

Ryujinx.Core/OsHle/AppletStateMgr.cs

@@ -33,14 +33,14 @@ namespace Ryujinx.Core.OsHle
         {
             Messages.Enqueue(Message);
 
-            MessageEvent.Handle.Set();
+            MessageEvent.WaitEvent.Set();
         }
 
         public bool TryDequeueMessage(out MessageInfo Message)
         {
             if (Messages.Count < 2)
             {
-                MessageEvent.Handle.Reset();
+                MessageEvent.WaitEvent.Reset();
             }
 
             return Messages.TryDequeue(out Message);

Ryujinx.Core/OsHle/Handles/KProcessScheduler.cs

@@ -7,6 +7,8 @@ namespace Ryujinx.Core.OsHle.Handles
 {
     class KProcessScheduler : IDisposable
     {
+        private const int LowestPriority = 0x40;
+
         private class SchedulerThread : IDisposable
         {
             public KThread Thread { get; private set; }
@@ -51,7 +53,7 @@ namespace Ryujinx.Core.OsHle.Handles
                 }
             }
 
-            public SchedulerThread Pop(int MinPriority = 0x40)
+            public SchedulerThread Pop(int MinPriority = LowestPriority)
             {
                 lock (Threads)
                 {
@@ -130,23 +132,42 @@ namespace Ryujinx.Core.OsHle.Handles
                     return;
                 }
 
-                if (!ActiveProcessors.Contains(Thread.ProcessorId))
+                if (ActiveProcessors.Add(Thread.ProcessorId))
                 {
-                    ActiveProcessors.Add(Thread.ProcessorId);
-
                     Thread.Thread.Execute();
 
-                    Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} running.");
+                    PrintDbgThreadInfo(Thread, "running.");
                 }
                 else
                 {
                     WaitingToRun[Thread.ProcessorId].Push(SchedThread);
 
-                    Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(SchedThread.Thread)} waiting to run.");
+                    PrintDbgThreadInfo(Thread, "waiting to run.");
                 }
             }
         }
 
+        public void RemoveThread(KThread Thread)
+        {
+            PrintDbgThreadInfo(Thread, "exited.");
+
+            lock (SchedLock)
+            {
+                SchedulerThread NewThread = WaitingToRun[Thread.ProcessorId].Pop();
+
+                if (NewThread == null)
+                {
+                    Logging.Debug(LogClass.KernelScheduler, $"Nothing to run on core {Thread.ProcessorId}!");
+
+                    ActiveProcessors.Remove(Thread.ProcessorId);
+
+                    return;
+                }
+
+                RunThread(NewThread);
+            }
+        }
+
         public void Suspend(int ProcessorId)
         {
             lock (SchedLock)
@@ -159,73 +180,44 @@ namespace Ryujinx.Core.OsHle.Handles
                 }
                 else
                 {
+                    Logging.Debug(LogClass.KernelScheduler, $"Nothing to run on core {ProcessorId}!");
+
                     ActiveProcessors.Remove(ProcessorId);
                 }
             }
         }
 
-        public void Resume(KThread CurrThread)
+        public void Yield(KThread Thread)
         {
-            SchedulerThread SchedThread;
+            PrintDbgThreadInfo(Thread, "yielded execution.");
-
-            Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(CurrThread)} entering ipc delay wait state.");
 
             lock (SchedLock)
             {
-                if (!AllThreads.TryGetValue(CurrThread, out SchedThread))
+                SchedulerThread SchedThread = WaitingToRun[Thread.ProcessorId].Pop(Thread.Priority);
+
+                if (SchedThread == null)
                 {
-                    Logging.Error(LogClass.KernelScheduler, $"{GetDbgThreadInfo(CurrThread)} was not found on the scheduler queue!");
+                    PrintDbgThreadInfo(Thread, "resumed because theres nothing better to run.");
 
                     return;
                 }
+
+                RunThread(SchedThread);
             }
 
-            TryResumingExecution(SchedThread);
+            Resume(Thread);
         }
 
-        public bool WaitForSignal(KThread Thread, int Timeout = -1)
+        public void Resume(KThread Thread)
         {
             SchedulerThread SchedThread;
 
-            Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} entering signal wait state.");
+            if (!AllThreads.TryGetValue(Thread, out SchedThread))
-
-            lock (SchedLock)
             {
-                SchedThread = WaitingToRun[Thread.ProcessorId].Pop();
+                throw new InvalidOperationException();
-
-                if (SchedThread != null)
-                {
-                    RunThread(SchedThread);
-                }
-                else
-                {
-                    ActiveProcessors.Remove(Thread.ProcessorId);
-                }
-
-                if (!AllThreads.TryGetValue(Thread, out SchedThread))
-                {
-                    Logging.Error(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} was not found on the scheduler queue!");
-
-                    return false;
-                }
-            }
-
-            bool Result;
-
-            if (Timeout >= 0)
-            {
-                Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} has wait timeout of {Timeout}ms.");
-
-                Result = SchedThread.WaitEvent.WaitOne(Timeout);
-            }
-            else
-            {
-                Result = SchedThread.WaitEvent.WaitOne();
             }
 
             TryResumingExecution(SchedThread);
-
-            return Result;
         }
 
         private void TryResumingExecution(SchedulerThread SchedThread)
@@ -236,51 +228,19 @@ namespace Ryujinx.Core.OsHle.Handles
             {
                 if (ActiveProcessors.Add(Thread.ProcessorId))
                 {
-                    Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} resuming execution...");
+                    PrintDbgThreadInfo(Thread, "resuming execution...");
 
                     return;
                 }
 
+                PrintDbgThreadInfo(Thread, "entering wait state...");
+
                 WaitingToRun[Thread.ProcessorId].Push(SchedThread);
             }
 
             SchedThread.WaitEvent.WaitOne();
 
-            Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} resuming execution...");
+            PrintDbgThreadInfo(Thread, "resuming execution...");
-        }
-
-        public void Yield(KThread Thread)
-        {
-            SchedulerThread SchedThread;
-
-            Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} yielded execution.");
-
-            lock (SchedLock)
-            {
-                SchedThread = WaitingToRun[Thread.ProcessorId].Pop(Thread.Priority);
-
-                if (SchedThread == null)
-                {
-                    Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} resumed because theres nothing better to run.");
-
-                    return;
-                }
-                
-                RunThread(SchedThread);
-
-                if (!AllThreads.TryGetValue(Thread, out SchedThread))
-                {
-                    Logging.Error(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} was not found on the scheduler queue!");
-
-                    return;
-                }
-
-                WaitingToRun[Thread.ProcessorId].Push(SchedThread);
-            }
-
-            SchedThread.WaitEvent.WaitOne();
-
-            Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} resuming execution...");
         }
 
         private void RunThread(SchedulerThread SchedThread)
@@ -291,32 +251,16 @@ namespace Ryujinx.Core.OsHle.Handles
             }
             else
             {
-                Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(SchedThread.Thread)} running.");
+                PrintDbgThreadInfo(SchedThread.Thread, "running.");
             }
         }
 
-        public void Signal(params KThread[] Threads)
+        private void PrintDbgThreadInfo(KThread Thread, string Message)
         {
-            lock (SchedLock)
+            Logging.Debug(LogClass.KernelScheduler, "(" +
-            {
+                "ThreadId: "    + Thread.ThreadId    + ", " +
-                foreach (KThread Thread in Threads)
+                "ProcessorId: " + Thread.ProcessorId + ", " +
-                {
+                "Priority: "    + Thread.Priority    + ") " + Message);
-                    if (AllThreads.TryGetValue(Thread, out SchedulerThread SchedThread))
-                    {
-                        if (!WaitingToRun[Thread.ProcessorId].HasThread(SchedThread))
-                        {
-                            Logging.Debug(LogClass.KernelScheduler, $"{GetDbgThreadInfo(Thread)} signaled.");
-
-                            SchedThread.WaitEvent.Set();
-                        }
-                    }
-                }
-            }
-        }
-
-        private string GetDbgThreadInfo(KThread Thread)
-        {
-            return $"Thread {Thread.ThreadId} (core {Thread.ProcessorId}) prio {Thread.Priority}";
         }
 
         public void Dispose()

Ryujinx.Core/OsHle/Handles/KSynchronizationObject.cs

@@ -5,11 +5,11 @@ namespace Ryujinx.Core.OsHle.Handles
 {
     class KSynchronizationObject : IDisposable
     {
-        public ManualResetEvent Handle { get; private set; }
+        public ManualResetEvent WaitEvent { get; private set; }
 
         public KSynchronizationObject()
         {
-            Handle = new ManualResetEvent(false);
+            WaitEvent = new ManualResetEvent(false);
         }
 
         public void Dispose()
@@ -21,7 +21,7 @@ namespace Ryujinx.Core.OsHle.Handles
         {
             if (Disposing)
             {
-                Handle.Dispose();
+                WaitEvent.Dispose();
             }
         }
     }

Ryujinx.Core/OsHle/Handles/KThread.cs

@@ -6,8 +6,10 @@ namespace Ryujinx.Core.OsHle.Handles
     {
         public AThread Thread { get; private set; }
 
-        public int ProcessorId { get; private set; }
+        public int ProcessorId  { get; private set; }
-        public int Priority    { get; set; }
+
+        public int  Priority { get; set; }
+        public int  Handle   { get; set; }
 
         public int ThreadId => Thread.ThreadId;
 

Ryujinx.Core/OsHle/Horizon.cs

@@ -11,8 +11,7 @@ namespace Ryujinx.Core.OsHle
         internal const int HidSize  = 0x40000;
         internal const int FontSize = 0x50;
 
-        internal ConcurrentDictionary<long, Mutex>   Mutexes  { get; private set; }
+        private KProcessScheduler Scheduler;
-        internal ConcurrentDictionary<long, CondVar> CondVars { get; private set; }
 
         private ConcurrentDictionary<int, Process> Processes;
 
@@ -27,8 +26,7 @@ namespace Ryujinx.Core.OsHle
         {
             this.Ns = Ns;
 
-            Mutexes  = new ConcurrentDictionary<long, Mutex>();
+            Scheduler = new KProcessScheduler();
-            CondVars = new ConcurrentDictionary<long, CondVar>();
 
             Processes = new ConcurrentDictionary<int, Process>();
 
@@ -95,7 +93,7 @@ namespace Ryujinx.Core.OsHle
             MainProcess.Run(IsNro);
         }
 
-        public void SignalVsync() => VsyncEvent.Handle.Set();
+        public void SignalVsync() => VsyncEvent.WaitEvent.Set();
 
         private Process MakeProcess()
         {
@@ -110,7 +108,7 @@ namespace Ryujinx.Core.OsHle
                     ProcessId++;
                 }
 
-                Process = new Process(Ns, ProcessId);
+                Process = new Process(Ns, Scheduler, ProcessId);
 
                 Processes.TryAdd(ProcessId, Process);
             }
@@ -144,11 +142,6 @@ namespace Ryujinx.Core.OsHle
 
                 if (File.Exists(NextNro))
                 {
-                    //TODO: Those dictionaries shouldn't even exist,
-                    //the Mutex and CondVar helper classes should be static.
-                    Mutexes.Clear();
-                    CondVars.Clear();
-
                     LoadProgram(NextNro);
                 }
             }

Ryujinx.Core/OsHle/Kernel/ConditionVariable.cs

@@ -2,53 +2,58 @@ using Ryujinx.Core.OsHle.Handles;
 using System.Collections.Generic;
 using System.Threading;
 
-namespace Ryujinx.Core.OsHle
+namespace Ryujinx.Core.OsHle.Kernel
 {
-    class CondVar
+    class ConditionVariable
     {
         private Process Process;
 
         private long CondVarAddress;
-        private long Timeout;
 
         private bool OwnsCondVarValue;
 
-        private List<KThread> WaitingThreads;
+        private List<(KThread Thread, AutoResetEvent WaitEvent)> WaitingThreads;
 
-        public CondVar(Process Process, long CondVarAddress, long Timeout)
+        public ConditionVariable(Process Process, long CondVarAddress)
         {
             this.Process        = Process;
             this.CondVarAddress = CondVarAddress;
-            this.Timeout        = Timeout;
 
-            WaitingThreads = new List<KThread>();
+            WaitingThreads = new List<(KThread, AutoResetEvent)>();
         }
 
-        public bool WaitForSignal(KThread Thread)
+        public bool WaitForSignal(KThread Thread, long Timeout)
         {
+            bool Result = true;
+
             int Count = Process.Memory.ReadInt32(CondVarAddress);
 
             if (Count <= 0)
             {
-                lock (WaitingThreads)
+                using (AutoResetEvent WaitEvent = new AutoResetEvent(false))
                 {
-                    WaitingThreads.Add(Thread);
-                }
-
-                if (Timeout == -1)
-                {
-                    Process.Scheduler.WaitForSignal(Thread);
-                }
-                else
-                {
-                    bool Result = Process.Scheduler.WaitForSignal(Thread, (int)(Timeout / 1000000));
-
                     lock (WaitingThreads)
                     {
-                        WaitingThreads.Remove(Thread);
+                        WaitingThreads.Add((Thread, WaitEvent));
                     }
 
-                    return Result;
+                    Process.Scheduler.Suspend(Thread.ProcessorId);
+
+                    if (Timeout < 0)
+                    {
+                        Result = WaitEvent.WaitOne();
+                    }
+                    else
+                    {
+                        Result = WaitEvent.WaitOne((int)(Timeout / 1000000));
+
+                        lock (WaitingThreads)
+                        {
+                            WaitingThreads.Remove((Thread, WaitEvent));
+                        }
+                    }
+
+                    Process.Scheduler.Resume(Thread);
                 }
             }
 
@@ -63,57 +68,49 @@ namespace Ryujinx.Core.OsHle
 
             ReleaseCondVarValue();
 
-            return true;
+            return Result;
         }
 
         public void SetSignal(KThread Thread, int Count)
         {
             lock (WaitingThreads)
             {
-                if (Count == -1)
+                if (Count < 0)
                 {
-                    Process.Scheduler.Signal(WaitingThreads.ToArray());
-
-                    AcquireCondVarValue();
-
                     Process.Memory.WriteInt32(CondVarAddress, WaitingThreads.Count);
 
-                    ReleaseCondVarValue();
+                    foreach ((_, AutoResetEvent WaitEvent) in WaitingThreads)
+                    {
+                        WaitEvent.Set();
+                    }
 
                     WaitingThreads.Clear();
                 }
                 else
                 {
-                    if (WaitingThreads.Count > 0)
+                    Process.Memory.WriteInt32(CondVarAddress, Count);
+
+                    while (WaitingThreads.Count > 0 && Count-- > 0)
                     {
-                        int HighestPriority  = WaitingThreads[0].Priority;
+                        int HighestPriority  = WaitingThreads[0].Thread.Priority;
                         int HighestPrioIndex = 0;
 
                         for (int Index = 1; Index < WaitingThreads.Count; Index++)
                         {
-                            if (HighestPriority > WaitingThreads[Index].Priority)
+                            if (HighestPriority > WaitingThreads[Index].Thread.Priority)
                             {
-                                HighestPriority = WaitingThreads[Index].Priority;
+                                HighestPriority = WaitingThreads[Index].Thread.Priority;
 
                                 HighestPrioIndex = Index;
                             }
                         }
 
-                        Process.Scheduler.Signal(WaitingThreads[HighestPrioIndex]);
+                        WaitingThreads[HighestPrioIndex].WaitEvent.Set();
 
                         WaitingThreads.RemoveAt(HighestPrioIndex);
                     }
-
-                    AcquireCondVarValue();
-
-                    Process.Memory.WriteInt32(CondVarAddress, Count);
-
-                    ReleaseCondVarValue();
                 }
             }
-
-            Process.Scheduler.Suspend(Thread.ProcessorId);
-            Process.Scheduler.Resume(Thread);
         }
 
         private void AcquireCondVarValue()

Ryujinx.Core/OsHle/Kernel/KernelErr.cs

@@ -1,4 +1,4 @@
-namespace Ryujinx.Core.OsHle
+namespace Ryujinx.Core.OsHle.Kernel
 {
     static class KernelErr
     {

Ryujinx.Core/OsHle/Kernel/MutualExclusion.cs

@@ -0,0 +1,91 @@
+using Ryujinx.Core.OsHle.Handles;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace Ryujinx.Core.OsHle.Kernel
+{
+    class MutualExclusion
+    {
+        private const int MutexHasListenersMask = 0x40000000;
+
+        private Process Process;
+
+        private long MutexAddress;
+
+        private List<(KThread Thread, AutoResetEvent WaitEvent)> WaitingThreads;
+
+        public MutualExclusion(Process Process, long MutexAddress)
+        {
+            this.Process      = Process;
+            this.MutexAddress = MutexAddress;
+
+            WaitingThreads = new List<(KThread, AutoResetEvent)>();
+        }
+
+        public void WaitForLock(KThread RequestingThread)
+        {
+            int OwnerThreadHandle = Process.Memory.ReadInt32(MutexAddress) & ~MutexHasListenersMask;
+
+            WaitForLock(RequestingThread, OwnerThreadHandle);
+        }
+
+        public void WaitForLock(KThread RequestingThread, int OwnerThreadHandle)
+        {
+            if (OwnerThreadHandle == RequestingThread.Handle ||
+                OwnerThreadHandle == 0)
+            {
+                return;
+            }
+
+            using (AutoResetEvent WaitEvent = new AutoResetEvent(false))
+            {
+                lock (WaitingThreads)
+                {
+                    WaitingThreads.Add((RequestingThread, WaitEvent));
+                }
+
+                Process.Scheduler.Suspend(RequestingThread.ProcessorId);
+
+                WaitEvent.WaitOne();
+
+                Process.Scheduler.Resume(RequestingThread);
+            }
+        }
+
+        public void Unlock()
+        {
+            lock (WaitingThreads)
+            {
+                int HasListeners = WaitingThreads.Count > 1 ? MutexHasListenersMask : 0;
+
+                if (WaitingThreads.Count > 0)
+                {
+                    int HighestPriority  = WaitingThreads[0].Thread.Priority;
+                    int HighestPrioIndex = 0;
+
+                    for (int Index = 1; Index < WaitingThreads.Count; Index++)
+                    {
+                        if (HighestPriority > WaitingThreads[Index].Thread.Priority)
+                        {
+                            HighestPriority = WaitingThreads[Index].Thread.Priority;
+
+                            HighestPrioIndex = Index;
+                        }
+                    }
+
+                    int Handle = WaitingThreads[HighestPrioIndex].Thread.Handle;
+
+                    WaitingThreads[HighestPrioIndex].WaitEvent.Set();
+
+                    WaitingThreads.RemoveAt(HighestPrioIndex);
+
+                    Process.Memory.WriteInt32(MutexAddress, HasListeners | Handle);
+                }
+                else
+                {
+                    Process.Memory.WriteInt32(MutexAddress, 0);
+                }
+            }
+        }
+    }
+}

Ryujinx.Core/OsHle/Kernel/SvcHandler.cs

@@ -3,9 +3,10 @@ using ChocolArm64.Memory;
 using ChocolArm64.State;
 using Ryujinx.Core.OsHle.Handles;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 
-namespace Ryujinx.Core.OsHle.Svc
+namespace Ryujinx.Core.OsHle.Kernel
 {
     partial class SvcHandler : IDisposable
     {
@@ -17,6 +18,9 @@ namespace Ryujinx.Core.OsHle.Svc
         private Process Process;
         private AMemory Memory;
 
+        private ConcurrentDictionary<long, MutualExclusion>     Mutexes;
+        private ConcurrentDictionary<long, ConditionVariable> CondVars;
+
         private HashSet<(HSharedMem, long)> MappedSharedMems;
 
         private ulong CurrentHeapSize;
@@ -66,6 +70,9 @@ namespace Ryujinx.Core.OsHle.Svc
             this.Process = Process;
             this.Memory  = Process.Memory;
 
+            Mutexes  = new ConcurrentDictionary<long, MutualExclusion>();
+            CondVars = new ConcurrentDictionary<long, ConditionVariable>();
+
             MappedSharedMems = new HashSet<(HSharedMem, long)>();
         }
 

Ryujinx.Core/OsHle/Kernel/SvcMemory.cs

@@ -4,7 +4,7 @@ using Ryujinx.Core.OsHle.Handles;
 
 using static Ryujinx.Core.OsHle.ErrorCode;
 
-namespace Ryujinx.Core.OsHle.Svc
+namespace Ryujinx.Core.OsHle.Kernel
 {
     partial class SvcHandler
     {

Ryujinx.Core/OsHle/Kernel/SvcSystem.cs

@@ -9,7 +9,7 @@ using System.Threading;
 
 using static Ryujinx.Core.OsHle.ErrorCode;
 
-namespace Ryujinx.Core.OsHle.Svc
+namespace Ryujinx.Core.OsHle.Kernel
 {
     partial class SvcHandler
     {
@@ -69,7 +69,7 @@ namespace Ryujinx.Core.OsHle.Svc
 
             if (Event != null)
             {
-                Event.Handle.Reset();
+                Event.WaitEvent.Reset();
 
                 ThreadState.X0 = 0;
             }
@@ -106,7 +106,7 @@ namespace Ryujinx.Core.OsHle.Svc
                     return;
                 }
 
-                Handles[Index] = SyncObj.Handle;
+                Handles[Index] = SyncObj.WaitEvent;
             }
 
             Process.Scheduler.Suspend(CurrThread.ProcessorId);

Ryujinx.Core/OsHle/Kernel/SvcThread.cs

@@ -1,9 +1,10 @@
 using ChocolArm64.State;
 using Ryujinx.Core.OsHle.Handles;
+using System.Threading;
 
 using static Ryujinx.Core.OsHle.ErrorCode;
 
-namespace Ryujinx.Core.OsHle.Svc
+namespace Ryujinx.Core.OsHle.Kernel
 {
     partial class SvcHandler
     {
@@ -15,42 +16,43 @@ namespace Ryujinx.Core.OsHle.Svc
             int  Priority    =  (int)ThreadState.X4;
             int  ProcessorId =  (int)ThreadState.X5;
 
-            if (Ns.Os.TryGetProcess(ThreadState.ProcessId, out Process Process))
+            if (ProcessorId == -2)
             {
-                if (ProcessorId == -2)
+                //TODO: Get this value from the NPDM file.
-                {
+                ProcessorId = 0;
-                    //TODO: Get this value from the NPDM file.
-                    ProcessorId = 0;
-                }
-
-                int Handle = Process.MakeThread(
-                    EntryPoint,
-                    StackTop,
-                    ArgsPtr,
-                    Priority,
-                    ProcessorId);
-
-                ThreadState.X0 = 0;
-                ThreadState.X1 = (ulong)Handle;
             }
 
-            //TODO: Error codes.
+            int Handle = Process.MakeThread(
+                EntryPoint,
+                StackTop,
+                ArgsPtr,
+                Priority,
+                ProcessorId);
+
+            ThreadState.X0 = 0;
+            ThreadState.X1 = (ulong)Handle;
         }
 
         private void SvcStartThread(AThreadState ThreadState)
         {
             int Handle = (int)ThreadState.X0;
 
-            KThread Thread = Process.HandleTable.GetData<KThread>(Handle);
+            KThread CurrThread = Process.HandleTable.GetData<KThread>(Handle);
 
-            if (Thread != null)
+            if (CurrThread != null)
             {
-                Process.Scheduler.StartThread(Thread);
+                Process.Scheduler.StartThread(CurrThread);
+
+                Process.Scheduler.Yield(Process.GetThread(ThreadState.Tpidr));
 
                 ThreadState.X0 = 0;
             }
+            else
+            {
+                Logging.Warn(LogClass.KernelSvc, $"Invalid thread handle 0x{Handle:x8}!");
 
-            //TODO: Error codes.
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+            }
         }
 
         private void SvcExitThread(AThreadState ThreadState)
@@ -58,8 +60,6 @@ namespace Ryujinx.Core.OsHle.Svc
             KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
 
             CurrThread.Thread.StopExecution();
-
-            CurrThread.Handle.Set();
         }
 
         private void SvcSleepThread(AThreadState ThreadState)
@@ -74,7 +74,11 @@ namespace Ryujinx.Core.OsHle.Svc
             }
             else
             {
-                Process.Scheduler.WaitForSignal(CurrThread, (int)(NanoSecs / 1000000));
+                Process.Scheduler.Suspend(CurrThread.ProcessorId);
+
+                Thread.Sleep((int)(NanoSecs / 1000000));
+
+                Process.Scheduler.Resume(CurrThread);
             }
         }
 
@@ -82,15 +86,19 @@ namespace Ryujinx.Core.OsHle.Svc
         {
             int Handle = (int)ThreadState.X1;
 
-            KThread Thread = Process.HandleTable.GetData<KThread>(Handle);
+            KThread CurrThread = Process.HandleTable.GetData<KThread>(Handle);
 
-            if (Thread != null)
+            if (CurrThread != null)
             {
                 ThreadState.X0 = 0;
-                ThreadState.X1 = (ulong)Thread.Priority;
+                ThreadState.X1 = (ulong)CurrThread.Priority;
             }
+            else
+            {
+                Logging.Warn(LogClass.KernelSvc, $"Invalid thread handle 0x{Handle:x8}!");
 
-            //TODO: Error codes.
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+            }
         }
 
         private void SvcSetThreadPriority(AThreadState ThreadState)
@@ -98,16 +106,20 @@ namespace Ryujinx.Core.OsHle.Svc
             int Prio   = (int)ThreadState.X0;
             int Handle = (int)ThreadState.X1;
 
-            KThread Thread = Process.HandleTable.GetData<KThread>(Handle);
+            KThread CurrThread = Process.HandleTable.GetData<KThread>(Handle);
 
-            if (Thread != null)
+            if (CurrThread != null)
             {
-                Thread.Priority = Prio;
+                CurrThread.Priority = Prio;
 
                 ThreadState.X0 = 0;
             }
+            else
+            {
+                Logging.Warn(LogClass.KernelSvc, $"Invalid thread handle 0x{Handle:x8}!");
 
-            //TODO: Error codes.
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+            }
         }
 
         private void SvcSetThreadCoreMask(AThreadState ThreadState)
@@ -119,21 +131,19 @@ namespace Ryujinx.Core.OsHle.Svc
 
         private void SvcGetCurrentProcessorNumber(AThreadState ThreadState)
         {
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
+            ThreadState.X0 = (ulong)Process.GetThread(ThreadState.Tpidr).ProcessorId;
-
-            ThreadState.X0 = (ulong)CurrThread.ProcessorId;
         }
 
         private void SvcGetThreadId(AThreadState ThreadState)
         {
             int Handle = (int)ThreadState.X1;
 
-            KThread Thread = Process.HandleTable.GetData<KThread>(Handle);
+            KThread CurrThread = Process.HandleTable.GetData<KThread>(Handle);
 
-            if (Thread != null)
+            if (CurrThread != null)
             {
                 ThreadState.X0 = 0;
-                ThreadState.X1 = (ulong)Thread.ThreadId;
+                ThreadState.X1 = (ulong)CurrThread.ThreadId;
             }
             else
             {

Ryujinx.Core/OsHle/Kernel/SvcThreadSync.cs

@@ -0,0 +1,120 @@
+using ChocolArm64.State;
+using Ryujinx.Core.OsHle.Handles;
+
+using static Ryujinx.Core.OsHle.ErrorCode;
+
+namespace Ryujinx.Core.OsHle.Kernel
+{
+    partial class SvcHandler
+    {
+        private void SvcArbitrateLock(AThreadState ThreadState)
+        {
+            int  OwnerThreadHandle      =  (int)ThreadState.X0;
+            long MutexAddress           = (long)ThreadState.X1;
+            int  RequestingThreadHandle =  (int)ThreadState.X2;
+
+            KThread OwnerThread = Process.HandleTable.GetData<KThread>(OwnerThreadHandle);
+
+            if (OwnerThread == null)
+            {
+                Logging.Warn(LogClass.KernelSvc, $"Invalid owner thread handle 0x{OwnerThreadHandle:x8}!");
+
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                return;
+            }
+
+            KThread RequestingThread = Process.HandleTable.GetData<KThread>(RequestingThreadHandle);
+
+            if (RequestingThread == null)
+            {
+                Logging.Warn(LogClass.KernelSvc, $"Invalid requesting thread handle 0x{RequestingThreadHandle:x8}!");
+
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+
+                return;
+            }
+
+            MutualExclusion Mutex = GetMutex(MutexAddress);
+
+            Mutex.WaitForLock(RequestingThread, OwnerThreadHandle);
+
+            ThreadState.X0 = 0;
+        }
+
+        private void SvcArbitrateUnlock(AThreadState ThreadState)
+        {
+            long MutexAddress = (long)ThreadState.X0;
+
+            GetMutex(MutexAddress).Unlock();
+
+            Process.Scheduler.Yield(Process.GetThread(ThreadState.Tpidr));
+
+            ThreadState.X0 = 0;
+        }
+
+        private void SvcWaitProcessWideKeyAtomic(AThreadState ThreadState)
+        {
+            long MutexAddress   = (long)ThreadState.X0;
+            long CondVarAddress = (long)ThreadState.X1;
+            int  ThreadHandle   =  (int)ThreadState.X2;
+            long Timeout        = (long)ThreadState.X3;
+
+            KThread Thread = Process.HandleTable.GetData<KThread>(ThreadHandle);
+
+            if (Thread == null)
+            {
+                Logging.Warn(LogClass.KernelSvc, $"Invalid thread handle 0x{ThreadHandle:x8}!");
+
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.InvalidHandle);
+            }
+
+            MutualExclusion Mutex = GetMutex(MutexAddress);
+
+            Mutex.Unlock();
+
+            if (!GetCondVar(CondVarAddress).WaitForSignal(Thread, Timeout))
+            {
+                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.Timeout);
+
+                return;
+            }
+
+            Mutex.WaitForLock(Thread);
+
+            ThreadState.X0 = 0;
+        }
+
+        private void SvcSignalProcessWideKey(AThreadState ThreadState)
+        {
+            long CondVarAddress = (long)ThreadState.X0;
+            int  Count          =  (int)ThreadState.X1;
+
+            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
+
+            GetCondVar(CondVarAddress).SetSignal(CurrThread, Count);
+
+            ThreadState.X0 = 0;
+        }
+
+        private MutualExclusion GetMutex(long MutexAddress)
+        {
+            MutualExclusion MutexFactory(long Key)
+            {
+                return new MutualExclusion(Process, MutexAddress);
+            }
+
+            return Mutexes.GetOrAdd(MutexAddress, MutexFactory);
+        }
+
+        private ConditionVariable GetCondVar(long CondVarAddress)
+        {
+            ConditionVariable CondVarFactory(long Key)
+            {
+                return new ConditionVariable(Process, CondVarAddress);
+            }
+
+            return CondVars.GetOrAdd(CondVarAddress, CondVarFactory);
+        }
+    }
+}

Ryujinx.Core/OsHle/Mutex.cs

@@ -1,122 +0,0 @@
-using Ryujinx.Core.OsHle.Handles;
-using System.Collections.Concurrent;
-using System.Threading;
-
-namespace Ryujinx.Core.OsHle
-{
-    class Mutex
-    {
-        private const int MutexHasListenersMask = 0x40000000;
-
-        private Process Process;
-
-        private long MutexAddress;
-
-        private bool OwnsMutexValue;
-
-        private object EnterWaitLock;
-
-        private ConcurrentQueue<KThread> WaitingThreads;
-
-        public Mutex(Process Process, long MutexAddress, int OwnerThreadHandle)
-        {
-            this.Process      = Process;
-            this.MutexAddress = MutexAddress;
-
-            //Process.Memory.WriteInt32(MutexAddress, OwnerThreadHandle);
-
-            EnterWaitLock = new object();
-
-            WaitingThreads = new ConcurrentQueue<KThread>();
-        }
-
-        public void WaitForLock(KThread RequestingThread, int RequestingThreadHandle)
-        {
-            AcquireMutexValue();
-
-            lock (EnterWaitLock)
-            {
-                int CurrentThreadHandle = Process.Memory.ReadInt32(MutexAddress) & ~MutexHasListenersMask;
-
-                if (CurrentThreadHandle == RequestingThreadHandle ||
-                    CurrentThreadHandle == 0)
-                {
-                    return;
-                }
-
-                Process.Memory.WriteInt32(MutexAddress, CurrentThreadHandle | MutexHasListenersMask);
-
-                ReleaseMutexValue();
-
-                WaitingThreads.Enqueue(RequestingThread);
-            }
-
-            Process.Scheduler.WaitForSignal(RequestingThread);
-        }
-
-        public void GiveUpLock(int ThreadHandle)
-        {
-            AcquireMutexValue();
-
-            lock (EnterWaitLock)
-            {
-                int CurrentThread = Process.Memory.ReadInt32(MutexAddress) & ~MutexHasListenersMask;
-
-                if (CurrentThread == ThreadHandle)
-                {
-                    Unlock();
-                }
-            }
-
-            ReleaseMutexValue();
-        }
-
-        public void Unlock()
-        {
-            AcquireMutexValue();
-
-            lock (EnterWaitLock)
-            {
-                int HasListeners = WaitingThreads.Count > 1 ? MutexHasListenersMask : 0;
-
-                Process.Memory.WriteInt32(MutexAddress, HasListeners);
-
-                ReleaseMutexValue();
-
-                KThread[] UnlockedThreads = new KThread[WaitingThreads.Count];
-
-                int Index = 0;
-
-                while (WaitingThreads.TryDequeue(out KThread Thread))
-                {
-                    UnlockedThreads[Index++] = Thread;
-                }
-
-                Process.Scheduler.Signal(UnlockedThreads);
-            }
-        }
-
-        private void AcquireMutexValue()
-        {
-            if (!OwnsMutexValue)
-            {
-                while (!Process.Memory.AcquireAddress(MutexAddress))
-                {
-                    Thread.Yield();
-                }
-
-                OwnsMutexValue = true;
-            }
-        }
-
-        private void ReleaseMutexValue()
-        {
-            if (OwnsMutexValue)
-            {
-                OwnsMutexValue = false;
-
-                Process.Memory.ReleaseAddress(MutexAddress);
-            }
-        }
-    }
-}

Ryujinx.Core/OsHle/Process.cs

@@ -5,8 +5,8 @@ using Ryujinx.Core.Loaders;
 using Ryujinx.Core.Loaders.Executables;
 using Ryujinx.Core.OsHle.Exceptions;
 using Ryujinx.Core.OsHle.Handles;
+using Ryujinx.Core.OsHle.Kernel;
 using Ryujinx.Core.OsHle.Services.Nv;
-using Ryujinx.Core.OsHle.Svc;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
@@ -55,17 +55,16 @@ namespace Ryujinx.Core.OsHle
 
         private bool Disposed;
 
-        public Process(Switch Ns, int ProcessId)
+        public Process(Switch Ns, KProcessScheduler Scheduler, int ProcessId)
         {
             this.Ns        = Ns;
+            this.Scheduler = Scheduler;
             this.ProcessId = ProcessId;
 
             Memory = new AMemory();
 
             HandleTable = new KProcessHandleTable();
 
-            Scheduler = new KProcessScheduler();
-
             AppletState = new AppletStateMgr();
 
             SvcHandler = new SvcHandler(Ns, this);
@@ -127,7 +126,7 @@ namespace Ryujinx.Core.OsHle
 
             long StackTop = MemoryRegions.MainStackAddress + MemoryRegions.MainStackSize;
 
-            int Handle = MakeThread(Executables[0].ImageBase, StackTop, 0, 0, 0);
+            int Handle = MakeThread(Executables[0].ImageBase, StackTop, 0, 44, 0);
 
             if (Handle == -1)
             {
@@ -188,28 +187,32 @@ namespace Ryujinx.Core.OsHle
 
             AThread Thread = new AThread(GetTranslator(), Memory, EntryPoint);
 
-            KThread ThreadHnd = new KThread(Thread, ProcessorId, Priority);
+            KThread KernelThread = new KThread(Thread, ProcessorId, Priority);
 
-            int Handle = HandleTable.OpenHandle(ThreadHnd);
+            int Handle = HandleTable.OpenHandle(KernelThread);
+
+            KernelThread.Handle = Handle;
 
             int ThreadId = GetFreeTlsSlot(Thread);
 
             long Tpidr = MemoryRegions.TlsPagesAddress + ThreadId * TlsSize;
 
+            Thread.ThreadState.ProcessId = ProcessId;
+            Thread.ThreadState.ThreadId  = ThreadId;
+            Thread.ThreadState.CntfrqEl0 = TickFreq;
+            Thread.ThreadState.Tpidr     = Tpidr;
+
+            Thread.ThreadState.X0  = (ulong)ArgsPtr;
+            Thread.ThreadState.X1  = (ulong)Handle;
+            Thread.ThreadState.X31 = (ulong)StackTop;
+
             Thread.ThreadState.Break     += BreakHandler;
             Thread.ThreadState.SvcCall   += SvcHandler.SvcCall;
             Thread.ThreadState.Undefined += UndefinedHandler;
-            Thread.ThreadState.ProcessId  = ProcessId;
-            Thread.ThreadState.ThreadId   = ThreadId;
-            Thread.ThreadState.CntfrqEl0  = TickFreq;
-            Thread.ThreadState.Tpidr      = Tpidr;
-            Thread.ThreadState.X0         = (ulong)ArgsPtr;
-            Thread.ThreadState.X1         = (ulong)Handle;
-            Thread.ThreadState.X31        = (ulong)StackTop;
 
             Thread.WorkFinished += ThreadFinished;
 
-            ThreadsByTpidr.TryAdd(Thread.ThreadState.Tpidr, ThreadHnd);
+            ThreadsByTpidr.TryAdd(Thread.ThreadState.Tpidr, KernelThread);
 
             return Handle;
         }
@@ -293,6 +296,12 @@ namespace Ryujinx.Core.OsHle
                 Logging.Info(LogClass.KernelScheduler, $"Thread {Thread.ThreadId} exiting...");
 
                 TlsSlots.TryRemove(GetTlsSlot(Thread.ThreadState.Tpidr), out _);
+
+                KThread KernelThread = GetThread(Thread.ThreadState.Tpidr);
+
+                Scheduler.RemoveThread(KernelThread);
+
+                KernelThread.WaitEvent.Set();
             }
 
             if (TlsSlots.Count == 0)

Ryujinx.Core/OsHle/Services/Aud/IAudioOutManager.cs

@@ -76,7 +76,7 @@ namespace Ryujinx.Core.OsHle.Services.Aud
 
             ReleaseCallback Callback = () =>
             {
-                ReleaseEvent.Handle.Set();
+                ReleaseEvent.WaitEvent.Set();
             };
 
             int Track = AudioOut.OpenTrack(SampleRate, Channels, Callback, out AudioFormat Format);

Ryujinx.Core/OsHle/Services/Aud/IAudioRenderer.cs

@@ -47,7 +47,7 @@ namespace Ryujinx.Core.OsHle.Services.Aud
             }
 
             //TODO: We shouldn't be signaling this here.
-            UpdateEvent.Handle.Set();
+            UpdateEvent.WaitEvent.Set();
 
             return 0;
         }

Ryujinx.Core/OsHle/Services/Vi/NvFlinger.cs

@@ -391,7 +391,7 @@ namespace Ryujinx.Core.OsHle.Services.Android
         {
             BufferQueue[Slot].State = BufferState.Free;
 
-            ReleaseEvent.Handle.Set();
+            ReleaseEvent.WaitEvent.Set();
 
             lock (WaitBufferFree)
             {

Ryujinx.Core/OsHle/Svc/SvcThreadSync.cs

@@ -1,85 +0,0 @@
-using ChocolArm64.State;
-using Ryujinx.Core.OsHle.Handles;
-
-using static Ryujinx.Core.OsHle.ErrorCode;
-
-namespace Ryujinx.Core.OsHle.Svc
-{
-    partial class SvcHandler
-    {
-        private void SvcArbitrateLock(AThreadState ThreadState)
-        {
-            int  OwnerThreadHandle      =  (int)ThreadState.X0;
-            long MutexAddress           = (long)ThreadState.X1;
-            int  RequestingThreadHandle =  (int)ThreadState.X2;
-
-            KThread RequestingThread = Process.HandleTable.GetData<KThread>(RequestingThreadHandle);
-
-            Mutex M = new Mutex(Process, MutexAddress, OwnerThreadHandle);
-
-            M = Ns.Os.Mutexes.GetOrAdd(MutexAddress, M);
-
-            M.WaitForLock(RequestingThread, RequestingThreadHandle);
-
-            ThreadState.X0 = 0;
-        }
-
-        private void SvcArbitrateUnlock(AThreadState ThreadState)
-        {
-            long MutexAddress = (long)ThreadState.X0;
-
-            if (Ns.Os.Mutexes.TryGetValue(MutexAddress, out Mutex M))
-            {
-                M.Unlock();
-            }
-
-            ThreadState.X0 = 0;
-        }
-
-        private void SvcWaitProcessWideKeyAtomic(AThreadState ThreadState)
-        {
-            long MutexAddress   = (long)ThreadState.X0;
-            long CondVarAddress = (long)ThreadState.X1;
-            int  ThreadHandle   =  (int)ThreadState.X2;
-            long Timeout        = (long)ThreadState.X3;
-
-            KThread Thread = Process.HandleTable.GetData<KThread>(ThreadHandle);
-
-            Mutex M = new Mutex(Process, MutexAddress, ThreadHandle);
-
-            M = Ns.Os.Mutexes.GetOrAdd(MutexAddress, M);
-
-            M.GiveUpLock(ThreadHandle);
-
-            CondVar Cv = new CondVar(Process, CondVarAddress, Timeout);
-
-            Cv = Ns.Os.CondVars.GetOrAdd(CondVarAddress, Cv);
-
-            if (!Cv.WaitForSignal(Thread))
-            {
-                ThreadState.X0 = MakeError(ErrorModule.Kernel, KernelErr.Timeout);
-
-                return;
-            }
-
-            M.WaitForLock(Thread, ThreadHandle);
-
-            ThreadState.X0 = 0;
-        }
-
-        private void SvcSignalProcessWideKey(AThreadState ThreadState)
-        {
-            long CondVarAddress = (long)ThreadState.X0;
-            int  Count          =  (int)ThreadState.X1;
-
-            KThread CurrThread = Process.GetThread(ThreadState.Tpidr);
-
-            if (Ns.Os.CondVars.TryGetValue(CondVarAddress, out CondVar Cv))
-            {
-                Cv.SetSignal(CurrThread, Count);
-            }
-
-            ThreadState.X0 = 0;
-        }
-    }
-}

Ryujinx/Ui/GLScreen.cs

@@ -1,6 +1,5 @@
 using OpenTK;
 using OpenTK.Graphics;
-using OpenTK.Graphics.OpenGL;
 using OpenTK.Input;
 using Ryujinx.Core;
 using Ryujinx.Core.Input;