Implement GPU syncpoints (#980)

* Implement GPU syncpoints

This adds support for GPU syncpoints on the GPU backend & nvservices.

Everything that was implemented here is based on my researches,
hardware testing of the GM20B and reversing of nvservices (8.1.0).

Thanks to @fincs for the informations about some behaviours of the pusher
and for the initial informations about syncpoints.

* syncpoint: address gdkchan's comments

* Add some missing logic to handle SubmitGpfifo correctly

* Handle the NV event API correctly

* evnt => hostEvent

* Finish addressing gdkchan's comments

* nvservices: write the output buffer even when an error is returned

* dma pusher: Implemnet prefetch barrier

lso fix when the commands should be prefetch.

* Partially fix prefetch barrier

* Add a missing syncpoint check in QueryEvent of NvHostSyncPt

* Address Ac_K's comments and fix GetSyncpoint for ChannelResourcePolicy == Channel

* fix SyncptWait & SyncptWaitEx cmds logic

* Address ripinperi's comments

* Address gdkchan's comments

* Move user event management to the control channel

* Fix mm implementation, nvdec works again

* Address ripinperi's comments

* Address gdkchan's comments

* Implement nvhost-ctrl close accurately + make nvservices dispose channels when stopping the emulator

* Fix typo in MultiMediaOperationType

Ryujinx.Graphics.Gpu/ClassId.cs

@@ -9,6 +9,7 @@ namespace Ryujinx.Graphics.Gpu
         Engine3D            = 0xb197,
         EngineCompute       = 0xb1c0,
         EngineInline2Memory = 0xa140,
-        EngineDma           = 0xb0b5
+        EngineDma           = 0xb0b5,
+        EngineGpfifo        = 0xb06f
     }
 }

Ryujinx.Graphics.Gpu/DmaPusher.cs

@@ -1,4 +1,6 @@
+using System;
 using System.Collections.Concurrent;
+using System.Runtime.InteropServices;
 using System.Threading;
 
 namespace Ryujinx.Graphics.Gpu
@@ -8,10 +10,61 @@ namespace Ryujinx.Graphics.Gpu
     /// </summary>
     public class DmaPusher
     {
-        private ConcurrentQueue<ulong> _ibBuffer;
+        private ConcurrentQueue<CommandBuffer> _commandBufferQueue;
 
-        private ulong _dmaPut;
-        private ulong _dmaGet;
+        private enum CommandBufferType
+        {
+            Prefetch,
+            NoPrefetch,
+        }
+
+        private struct CommandBuffer
+        {
+            /// <summary>
+            /// The type of the command buffer.
+            /// </summary>
+            public CommandBufferType Type;
+
+            /// <summary>
+            /// Fetched data.
+            /// </summary>
+            public int[] Words;
+
+            /// <summary>
+            /// The GPFIFO entry address. (used in NoPrefetch mode)
+            /// </summary>
+            public ulong EntryAddress;
+
+            /// <summary>
+            /// The count of entries inside this GPFIFO entry.
+            /// </summary>
+            public uint EntryCount;
+
+            /// <summary>
+            /// Fetch the command buffer.
+            /// </summary>
+            public void Fetch(GpuContext context)
+            {
+                if (Words == null)
+                {
+                    Words = MemoryMarshal.Cast<byte, int>(context.MemoryAccessor.GetSpan(EntryAddress, EntryCount * 4)).ToArray();
+                }
+            }
+
+            /// <summary>
+            /// Read inside the command buffer.
+            /// </summary>
+            /// <param name="context">The GPU context</param>
+            /// <param name="index">The index inside the command buffer</param>
+            /// <returns>The value read</returns>
+            public int ReadAt(GpuContext context, int index)
+            {
+                return Words[index];
+            }
+        }
+
+        private CommandBuffer _currentCommandBuffer;
+        private int           _wordsPosition;
 
         /// <summary>
         /// Internal GPFIFO state.
@@ -32,9 +85,6 @@ namespace Ryujinx.Graphics.Gpu
         private bool _sliActive;
 
         private bool _ibEnable;
-        private bool _nonMain;
-
-        private ulong _dmaMGet;
 
         private GpuContext _context;
 
@@ -48,24 +98,91 @@ namespace Ryujinx.Graphics.Gpu
         {
             _context = context;
 
-            _ibBuffer = new ConcurrentQueue<ulong>();
-
             _ibEnable = true;
 
+            _commandBufferQueue = new ConcurrentQueue<CommandBuffer>();
+
             _event = new AutoResetEvent(false);
         }
 
         /// <summary>
-        /// Pushes a GPFIFO entry.
+        /// Signal the pusher that there are new entries to process.
         /// </summary>
-        /// <param name="entry">GPFIFO entry</param>
-        public void Push(ulong entry)
+        public void SignalNewEntries()
         {
-            _ibBuffer.Enqueue(entry);
-
             _event.Set();
         }
 
+        /// <summary>
+        /// Push a GPFIFO entry in the form of a prefetched command buffer.
+        /// It is intended to be used by nvservices to handle special cases.
+        /// </summary>
+        /// <param name="commandBuffer">The command buffer containing the prefetched commands</param>
+        public void PushHostCommandBuffer(int[] commandBuffer)
+        {
+            _commandBufferQueue.Enqueue(new CommandBuffer
+            {
+                Type         = CommandBufferType.Prefetch,
+                Words        = commandBuffer,
+                EntryAddress = ulong.MaxValue,
+                EntryCount   = (uint)commandBuffer.Length
+            });
+        }
+
+        /// <summary>
+        /// Create a CommandBuffer from a GPFIFO entry.
+        /// </summary>
+        /// <param name="entry">The GPFIFO entry</param>
+        /// <returns>A new CommandBuffer based on the GPFIFO entry</returns>
+        private CommandBuffer CreateCommandBuffer(ulong entry)
+        {
+            ulong length       = (entry >> 42) & 0x1fffff;
+            ulong startAddress = entry & 0xfffffffffc;
+
+            bool noPrefetch = (entry & (1UL << 63)) != 0;
+
+            CommandBufferType type = CommandBufferType.Prefetch;
+
+            if (noPrefetch)
+            {
+                type = CommandBufferType.NoPrefetch;
+            }
+
+            return new CommandBuffer
+            {
+                Type         = type,
+                Words        = null,
+                EntryAddress = startAddress,
+                EntryCount   = (uint)length
+            };
+        }
+
+        /// <summary>
+        /// Pushes GPFIFO entries.
+        /// </summary>
+        /// <param name="entries">GPFIFO entries</param>
+        public void PushEntries(ReadOnlySpan<ulong> entries)
+        {
+            bool beforeBarrier = true;
+
+            foreach (ulong entry in entries)
+            {
+                CommandBuffer commandBuffer = CreateCommandBuffer(entry);
+
+                if (beforeBarrier && commandBuffer.Type == CommandBufferType.Prefetch)
+                {
+                    commandBuffer.Fetch(_context);
+                }
+
+                if (commandBuffer.Type == CommandBufferType.NoPrefetch)
+                {
+                    beforeBarrier = false;
+                }
+
+                _commandBufferQueue.Enqueue(commandBuffer);
+            }
+        }
+
         /// <summary>
         /// Waits until commands are pushed to the FIFO.
         /// </summary>
@@ -89,16 +206,9 @@ namespace Ryujinx.Graphics.Gpu
         /// <returns>True if the FIFO still has commands to be processed, false otherwise</returns>
         private bool Step()
         {
-            if (_dmaGet != _dmaPut)
+            if (_wordsPosition != _currentCommandBuffer.EntryCount)
             {
-                int word = _context.MemoryAccessor.ReadInt32(_dmaGet);
-
-                _dmaGet += 4;
-
-                if (!_nonMain)
-                {
-                    _dmaMGet = _dmaGet;
-                }
+                int word = _currentCommandBuffer.ReadAt(_context, _wordsPosition++);
 
                 if (_state.LengthPending != 0)
                 {
@@ -170,14 +280,12 @@ namespace Ryujinx.Graphics.Gpu
                     }
                 }
             }
-            else if (_ibEnable && _ibBuffer.TryDequeue(out ulong entry))
+            else if (_ibEnable && _commandBufferQueue.TryDequeue(out CommandBuffer entry))
             {
-                ulong length = (entry >> 42) & 0x1fffff;
+                _currentCommandBuffer = entry;
+                _wordsPosition        = 0;
 
-                _dmaGet = entry & 0xfffffffffc;
-                _dmaPut = _dmaGet + length * 4;
-
-                _nonMain = (entry & (1UL << 41)) != 0;
+                _currentCommandBuffer.Fetch(_context);
             }
             else
             {

Ryujinx.Graphics.Gpu/Engine/MethodFifo.cs

@@ -0,0 +1,77 @@
+using Ryujinx.Graphics.Gpu.State;
+using System;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Engine
+{
+    partial class Methods
+    {
+        /// <summary>
+        /// Waits for the GPU to be idle.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void WaitForIdle(GpuState state, int argument)
+        {
+            PerformDeferredDraws();
+
+            _context.Renderer.Pipeline.Barrier();
+        }
+
+        /// <summary>
+        /// Send macro code/data to the MME.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void SendMacroCodeData(GpuState state, int argument)
+        {
+            int macroUploadAddress = state.Get<int>(MethodOffset.MacroUploadAddress);
+
+            _context.Fifo.SendMacroCodeData(macroUploadAddress++, argument);
+
+            state.Write((int)MethodOffset.MacroUploadAddress, macroUploadAddress);
+        }
+
+        /// <summary>
+        /// Bind a macro index to a position for the MME.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void BindMacro(GpuState state, int argument)
+        {
+            int macroBindingIndex = state.Get<int>(MethodOffset.MacroBindingIndex);
+
+            _context.Fifo.BindMacro(macroBindingIndex++, argument);
+
+            state.Write((int)MethodOffset.MacroBindingIndex, macroBindingIndex);
+        }
+
+        public void SetMmeShadowRamControl(GpuState state, int argument)
+        {
+            _context.Fifo.SetMmeShadowRamControl((ShadowRamControl)argument);
+        }
+
+        /// <summary>
+        /// Apply a fence operation on a syncpoint.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void FenceAction(GpuState state, int argument)
+        {
+            uint threshold = state.Get<uint>(MethodOffset.FenceValue);
+
+            FenceActionOperation operation = (FenceActionOperation)(argument & 1);
+
+            uint syncpointId = (uint)(argument >> 8) & 0xFF;
+
+            if (operation == FenceActionOperation.Acquire)
+            {
+                _context.Synchronization.WaitOnSyncpoint(syncpointId, threshold, Timeout.InfiniteTimeSpan);
+            }
+            else if (operation == FenceActionOperation.Increment)
+            {
+                _context.Synchronization.IncrementSyncpoint(syncpointId);
+            }
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Engine/MethodIncrementSyncpoint.cs

@@ -0,0 +1,19 @@
+using Ryujinx.Graphics.Gpu.State;
+
+namespace Ryujinx.Graphics.Gpu.Engine
+{
+    partial class Methods
+    {
+        /// <summary>
+        /// Performs an incrementation on a syncpoint.
+        /// </summary>
+        /// <param name="state">Current GPU state</param>
+        /// <param name="argument">Method call argument</param>
+        public void IncrementSyncpoint(GpuState state, int argument)
+        {
+            uint syncpointId = (uint)(argument) & 0xFFFF;
+
+            _context.Synchronization.IncrementSyncpoint(syncpointId);
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Engine/MethodReport.cs

@@ -26,16 +26,16 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
             switch (mode)
             {
-                case ReportMode.Semaphore: ReportSemaphore(state);     break;
+                case ReportMode.Release: ReleaseSemaphore(state);    break;
                 case ReportMode.Counter: ReportCounter(state, type); break;
             }
         }
 
         /// <summary>
-        /// Writes a GPU semaphore value to guest memory.
+        /// Writes (or Releases) a GPU semaphore value to guest memory.
         /// </summary>
         /// <param name="state">Current GPU state</param>
-        private void ReportSemaphore(GpuState state)
+        private void ReleaseSemaphore(GpuState state)
         {
             var rs = state.Get<ReportState>(MethodOffset.ReportState);
 

Ryujinx.Graphics.Gpu/Engine/Methods.cs

@@ -65,6 +65,8 @@ namespace Ryujinx.Graphics.Gpu.Engine
 
             state.RegisterCallback(MethodOffset.Dispatch, Dispatch);
 
+            state.RegisterCallback(MethodOffset.SyncpointAction, IncrementSyncpoint);
+
             state.RegisterCallback(MethodOffset.CopyBuffer,  CopyBuffer);
             state.RegisterCallback(MethodOffset.CopyTexture, CopyTexture);
 
@@ -94,6 +96,19 @@ namespace Ryujinx.Graphics.Gpu.Engine
             state.RegisterCallback(MethodOffset.UniformBufferBindFragment,       UniformBufferBindFragment);
         }
 
+        /// <summary>
+        /// Register callback for Fifo method calls that triggers an action on the GPFIFO.
+        /// </summary>
+        /// <param name="state">GPU state where the triggers will be registered</param>
+        public void RegisterCallbacksForFifo(GpuState state)
+        {
+            state.RegisterCallback(MethodOffset.FenceAction,            FenceAction);
+            state.RegisterCallback(MethodOffset.WaitForIdle,            WaitForIdle);
+            state.RegisterCallback(MethodOffset.SendMacroCodeData,      SendMacroCodeData);
+            state.RegisterCallback(MethodOffset.BindMacro,              BindMacro);
+            state.RegisterCallback(MethodOffset.SetMmeShadowRamControl, SetMmeShadowRamControl);
+        }
+
         /// <summary>
         /// Updates host state based on the current guest GPU state.
         /// </summary>

Ryujinx.Graphics.Gpu/GpuContext.cs

@@ -1,6 +1,7 @@
 using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu.Engine;
 using Ryujinx.Graphics.Gpu.Memory;
+using Ryujinx.Graphics.Gpu.Synchronization;
 using System;
 
 namespace Ryujinx.Graphics.Gpu
@@ -45,6 +46,11 @@ namespace Ryujinx.Graphics.Gpu
         /// </summary>
         public DmaPusher DmaPusher { get; }
 
+        /// <summary>
+        /// GPU synchronization manager.
+        /// </summary>
+        public SynchronizationManager Synchronization { get; }
+
         /// <summary>
         /// Presentation window.
         /// </summary>
@@ -81,6 +87,8 @@ namespace Ryujinx.Graphics.Gpu
 
             DmaPusher = new DmaPusher(this);
 
+            Synchronization = new SynchronizationManager();
+
             Window = new Window(this);
 
             _caps = new Lazy<Capabilities>(Renderer.GetCapabilities);

Ryujinx.Graphics.Gpu/NvGpuFifo.cs

@@ -123,6 +123,8 @@ namespace Ryujinx.Graphics.Gpu
 
         private SubChannel[] _subChannels;
 
+        private SubChannel _fifoChannel;
+
         /// <summary>
         /// Creates a new instance of the GPU commands FIFO.
         /// </summary>
@@ -135,76 +137,68 @@ namespace Ryujinx.Graphics.Gpu
 
             _mme = new int[MmeWords];
 
+            _fifoChannel = new SubChannel();
+
+            _context.Methods.RegisterCallbacksForFifo(_fifoChannel.State);
+
             _subChannels = new SubChannel[8];
 
             for (int index = 0; index < _subChannels.Length; index++)
             {
                 _subChannels[index] = new SubChannel();
 
-                context.Methods.RegisterCallbacks(_subChannels[index].State);
+                _context.Methods.RegisterCallbacks(_subChannels[index].State);
             }
         }
 
+        /// <summary>
+        /// Send macro code/data to the MME
+        /// </summary>
+        /// <param name="index">The index in the MME</param>
+        /// <param name="data">The data to use</param>
+        public void SendMacroCodeData(int index, int data)
+        {
+            _mme[index] = data;
+        }
+
+        /// <summary>
+        /// Bind a macro index to a position for the MME
+        /// </summary>
+        /// <param name="index">The macro index</param>
+        /// <param name="position">The position of the macro</param>
+        public void BindMacro(int index, int position)
+        {
+            _macros[index] = new CachedMacro(position);
+        }
+
+        /// <summary>
+        /// Change the shadow RAM setting
+        /// </summary>
+        /// <param name="shadowCtrl">The new Shadow RAM setting</param>
+        public void SetMmeShadowRamControl(ShadowRamControl shadowCtrl)
+        {
+            _shadowCtrl = shadowCtrl;
+        }
+
         /// <summary>
         /// Calls a GPU method.
         /// </summary>
         /// <param name="meth">GPU method call parameters</param>
         public void CallMethod(MethodParams meth)
         {
-            if ((NvGpuFifoMeth)meth.Method == NvGpuFifoMeth.BindChannel)
+            if ((MethodOffset)meth.Method == MethodOffset.BindChannel)
             {
-                _subChannels[meth.SubChannel].Class = (ClassId)meth.Argument;
+                _subChannels[meth.SubChannel] = new SubChannel
+                {
+                    Class = (ClassId)meth.Argument
+                };
+
+                _context.Methods.RegisterCallbacks(_subChannels[meth.SubChannel].State);
             }
             else if (meth.Method < 0x60)
             {
-                switch ((NvGpuFifoMeth)meth.Method)
-                {
-                    case NvGpuFifoMeth.WaitForIdle:
-                    {
-                        _context.Methods.PerformDeferredDraws();
-
-                        _context.Renderer.Pipeline.Barrier();
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SetMacroUploadAddress:
-                    {
-                        _currMacroPosition = meth.Argument;
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SendMacroCodeData:
-                    {
-                        _mme[_currMacroPosition++] = meth.Argument;
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SetMacroBindingIndex:
-                    {
-                        _currMacroBindIndex = meth.Argument;
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.BindMacro:
-                    {
-                        int position = meth.Argument;
-
-                        _macros[_currMacroBindIndex++] = new CachedMacro(position);
-
-                        break;
-                    }
-
-                    case NvGpuFifoMeth.SetMmeShadowRamControl:
-                    {
-                        _shadowCtrl = (ShadowRamControl)meth.Argument;
-
-                        break;
-                    }
-                }
+                // TODO: check if macros are shared between subchannels or not. For now let's assume they are.
+                _fifoChannel.State.CallMethod(meth);
             }
             else if (meth.Method < 0xe00)
             {

Ryujinx.Graphics.Gpu/NvGpuFifoMeth.cs

@@ -1,16 +0,0 @@
-namespace Ryujinx.Graphics.Gpu
-{
-    /// <summary>
-    /// GPU commands FIFO processor commands.
-    /// </summary>
-    enum NvGpuFifoMeth
-    {
-        BindChannel            = 0,
-        WaitForIdle            = 0x44,
-        SetMacroUploadAddress  = 0x45,
-        SendMacroCodeData      = 0x46,
-        SetMacroBindingIndex   = 0x47,
-        BindMacro              = 0x48,
-        SetMmeShadowRamControl = 0x49
-    }
-}

Ryujinx.Graphics.Gpu/State/FenceActionOperation.cs

@@ -0,0 +1,11 @@
+namespace Ryujinx.Graphics.Gpu.State
+{
+    /// <summary>
+    /// Fence action operations.
+    /// </summary>
+    enum FenceActionOperation
+    {
+        Acquire   = 0,
+        Increment = 1
+    }
+}

Ryujinx.Graphics.Gpu/State/MethodOffset.cs

@@ -8,6 +8,15 @@ namespace Ryujinx.Graphics.Gpu.State
     /// </remarks>
     enum MethodOffset
     {
+        BindChannel                     = 0x00,
+        FenceValue                      = 0x1c,
+        FenceAction                     = 0x1d,
+        WaitForIdle                     = 0x44,
+        MacroUploadAddress              = 0x45,
+        SendMacroCodeData               = 0x46,
+        MacroBindingIndex               = 0x47,
+        BindMacro                       = 0x48,
+        SetMmeShadowRamControl          = 0x49,
         I2mParams                       = 0x60,
         LaunchDma                       = 0x6c,
         LoadInlineData                  = 0x6d,
@@ -15,6 +24,7 @@ namespace Ryujinx.Graphics.Gpu.State
         CopySrcTexture                  = 0x8c,
         DispatchParamsAddress           = 0xad,
         Dispatch                        = 0xaf,
+        SyncpointAction                 = 0xb2,
         CopyBuffer                      = 0xc0,
         RasterizeEnable                 = 0xdf,
         CopyBufferParams                = 0x100,

Ryujinx.Graphics.Gpu/State/ReportMode.cs

@@ -5,7 +5,8 @@ namespace Ryujinx.Graphics.Gpu.State
     /// </summary>
     enum ReportMode
     {
-        Semaphore = 0,
+        Release = 0,
+        Acquire = 1,
         Counter = 2
     }
 }

Ryujinx.Graphics.Gpu/Synchronization/SynchronizationManager.cs

@@ -0,0 +1,134 @@
+using System;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Synchronization
+{
+    /// <summary>
+    /// GPU synchronization manager.
+    /// </summary>
+    public class SynchronizationManager
+    {
+        /// <summary>
+        /// The maximum number of syncpoints supported by the GM20B.
+        /// </summary>
+        public const int MaxHardwareSyncpoints = 192;
+
+        /// <summary>
+        /// Array containing all hardware syncpoints.
+        /// </summary>
+        private Syncpoint[] _syncpoints;
+
+        public SynchronizationManager()
+        {
+            _syncpoints = new Syncpoint[MaxHardwareSyncpoints];
+
+            for (uint i = 0; i < _syncpoints.Length; i++)
+            {
+                _syncpoints[i] = new Syncpoint(i);
+            }
+        }
+
+        /// <summary>
+        /// Increment the value of a syncpoint with a given id.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>The incremented value of the syncpoint</returns>
+        public uint IncrementSyncpoint(uint id)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            return _syncpoints[id].Increment();
+        }
+
+        /// <summary>
+        /// Get the value of a syncpoint with a given id.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>The value of the syncpoint</returns>
+        public uint GetSyncpointValue(uint id)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            return _syncpoints[id].Value;
+        }
+
+        /// <summary>
+        /// Register a new callback on a syncpoint with a given id at a target threshold.
+        /// The callback will be called once the threshold is reached and will automatically be unregistered.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <param name="threshold">The target threshold</param>
+        /// <param name="callback">The callback to call when the threshold is reached</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>The created SyncpointWaiterHandle object or null if already past threshold</returns>
+        public SyncpointWaiterHandle RegisterCallbackOnSyncpoint(uint id, uint threshold, Action callback)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            return _syncpoints[id].RegisterCallback(threshold, callback);
+        }
+
+        /// <summary>
+        /// Unregister a callback on a given syncpoint.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <param name="waiterInformation">The waiter information to unregister</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        public void UnregisterCallback(uint id, SyncpointWaiterHandle waiterInformation)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            _syncpoints[id].UnregisterCallback(waiterInformation);
+        }
+
+        /// <summary>
+        /// Wait on a syncpoint with a given id at a target threshold.
+        /// The callback will be called once the threshold is reached and will automatically be unregistered.
+        /// </summary>
+        /// <param name="id">The id of the syncpoint</param>
+        /// <param name="threshold">The target threshold</param>
+        /// <param name="timeout">The timeout</param>
+        /// <exception cref="System.ArgumentOutOfRangeException">Thrown when id >= MaxHardwareSyncpoints</exception>
+        /// <returns>True if timed out</returns>
+        public bool WaitOnSyncpoint(uint id, uint threshold, TimeSpan timeout)
+        {
+            if (id >= MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            using (ManualResetEvent waitEvent = new ManualResetEvent(false))
+            {
+                var info = _syncpoints[id].RegisterCallback(threshold, () => waitEvent.Set());
+
+                if (info == null)
+                {
+                    return false;
+                }
+
+                bool signaled = waitEvent.WaitOne(timeout);
+
+                if (!signaled && info != null)
+                {
+                    _syncpoints[id].UnregisterCallback(info);
+                }
+
+                return !signaled;
+            }
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Synchronization/Syncpoint.cs

@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Synchronization
+{
+    /// <summary>
+    /// Represents GPU hardware syncpoint.
+    /// </summary>
+    class Syncpoint
+    {
+        private int _storedValue;
+
+        public readonly uint Id;
+
+        // TODO: get rid of this lock
+        private object _listLock = new object();
+
+        /// <summary>
+        /// The value of the syncpoint.
+        /// </summary>
+        public uint Value => (uint)_storedValue;
+
+        // TODO: switch to something handling concurrency?
+        private List<SyncpointWaiterHandle> _waiters;
+
+        public Syncpoint(uint id)
+        {
+            Id       = id;
+            _waiters = new List<SyncpointWaiterHandle>();
+        }
+
+        /// <summary>
+        /// Register a new callback for a target threshold.
+        /// The callback will be called once the threshold is reached and will automatically be unregistered.
+        /// </summary>
+        /// <param name="threshold">The target threshold</param>
+        /// <param name="callback">The callback to call when the threshold is reached</param>
+        /// <returns>The created SyncpointWaiterHandle object or null if already past threshold</returns>
+        public SyncpointWaiterHandle RegisterCallback(uint threshold, Action callback)
+        {
+            lock (_listLock)
+            {
+                if (Value >= threshold)
+                {
+                    callback();
+
+                    return null;
+                }
+                else
+                {
+                    SyncpointWaiterHandle waiterInformation = new SyncpointWaiterHandle
+                    {
+                        Threshold = threshold,
+                        Callback  = callback
+                    };
+
+                    _waiters.Add(waiterInformation);
+
+                    return waiterInformation;
+                }
+            }
+        }
+
+        public void UnregisterCallback(SyncpointWaiterHandle waiterInformation)
+        {
+            lock (_listLock)
+            {
+                _waiters.Remove(waiterInformation);
+            }
+        }
+
+        /// <summary>
+        /// Increment the syncpoint
+        /// </summary>
+        /// <returns>The incremented value of the syncpoint</returns>
+        public uint Increment()
+        {
+            uint currentValue = (uint)Interlocked.Increment(ref _storedValue);
+
+            lock (_listLock)
+            {
+                _waiters.RemoveAll(item =>
+                {
+                    bool isPastThreshold = currentValue >= item.Threshold;
+
+                    if (isPastThreshold)
+                    {
+                        item.Callback();
+                    }
+
+                    return isPastThreshold;
+                });
+            }
+
+            return currentValue;
+        }
+    }
+}

Ryujinx.Graphics.Gpu/Synchronization/SyncpointWaiterHandle.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace Ryujinx.Graphics.Gpu.Synchronization
+{
+    public class SyncpointWaiterHandle
+    {
+        internal uint   Threshold;
+        internal Action Callback;
+    }
+}

Ryujinx.Graphics.Gpu/Window.cs

@@ -30,12 +30,17 @@ namespace Ryujinx.Graphics.Gpu
             public ImageCrop Crop { get; }
 
             /// <summary>
-            /// Texture release callback.
+            /// Texture acquire callback.
             /// </summary>
-            public Action<object> Callback { get; }
+            public Action<GpuContext, object> AcquireCallback { get; }
 
             /// <summary>
-            /// User defined object, passed to the release callback.
+            /// Texture release callback.
+            /// </summary>
+            public Action<object> ReleaseCallback { get; }
+
+            /// <summary>
+            /// User defined object, passed to the various callbacks.
             /// </summary>
             public object UserObj { get; }
 
@@ -44,17 +49,20 @@ namespace Ryujinx.Graphics.Gpu
             /// </summary>
             /// <param name="info">Information of the texture to be presented</param>
             /// <param name="crop">Texture crop region</param>
-            /// <param name="callback">Texture release callback</param>
+            /// <param name="acquireCallback">Texture acquire callback</param>
+            /// <param name="releaseCallback">Texture release callback</param>
             /// <param name="userObj">User defined object passed to the release callback, can be used to identify the texture</param>
             public PresentationTexture(
                 TextureInfo                info,
                 ImageCrop                  crop,
-                Action<object> callback,
+                Action<GpuContext, object> acquireCallback,
+                Action<object>             releaseCallback,
                 object                     userObj)
             {
                 Info            = info;
                 Crop            = crop;
-                Callback = callback;
+                AcquireCallback = acquireCallback;
+                ReleaseCallback = releaseCallback;
                 UserObj         = userObj;
             }
         }
@@ -87,7 +95,8 @@ namespace Ryujinx.Graphics.Gpu
         /// <param name="format">Texture format</param>
         /// <param name="bytesPerPixel">Texture format bytes per pixel (must match the format)</param>
         /// <param name="crop">Texture crop region</param>
-        /// <param name="callback">Texture release callback</param>
+        /// <param name="acquireCallback">Texture acquire callback</param>
+        /// <param name="releaseCallback">Texture release callback</param>
         /// <param name="userObj">User defined object passed to the release callback</param>
         public void EnqueueFrameThreadSafe(
             ulong                      address,
@@ -99,7 +108,8 @@ namespace Ryujinx.Graphics.Gpu
             Format                     format,
             int                        bytesPerPixel,
             ImageCrop                  crop,
-            Action<object> callback,
+            Action<GpuContext, object> acquireCallback,
+            Action<object>             releaseCallback,
             object                     userObj)
         {
             FormatInfo formatInfo = new FormatInfo(format, 1, 1, bytesPerPixel);
@@ -120,7 +130,7 @@ namespace Ryujinx.Graphics.Gpu
                 Target.Texture2D,
                 formatInfo);
 
-            _frameQueue.Enqueue(new PresentationTexture(info, crop, callback, userObj));
+            _frameQueue.Enqueue(new PresentationTexture(info, crop, acquireCallback, releaseCallback, userObj));
         }
 
         /// <summary>
@@ -134,6 +144,8 @@ namespace Ryujinx.Graphics.Gpu
 
             if (_frameQueue.TryDequeue(out PresentationTexture pt))
             {
+                pt.AcquireCallback(_context, pt.UserObj);
+
                 Texture texture = _context.Methods.TextureManager.FindOrCreateTexture(pt.Info);
 
                 texture.SynchronizeMemory();
@@ -142,7 +154,7 @@ namespace Ryujinx.Graphics.Gpu
 
                 swapBuffersCallback();
 
-                pt.Callback(pt.UserObj);
+                pt.ReleaseCallback(pt.UserObj);
             }
         }
     }

Ryujinx.HLE/HOS/Horizon.cs

@@ -17,9 +17,11 @@ using Ryujinx.HLE.HOS.Kernel.Memory;
 using Ryujinx.HLE.HOS.Kernel.Process;
 using Ryujinx.HLE.HOS.Kernel.Threading;
 using Ryujinx.HLE.HOS.Services.Mii;
+using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl;
 using Ryujinx.HLE.HOS.Services.Pcv.Bpc;
 using Ryujinx.HLE.HOS.Services.Settings;
 using Ryujinx.HLE.HOS.Services.Sm;
+using Ryujinx.HLE.HOS.Services.SurfaceFlinger;
 using Ryujinx.HLE.HOS.Services.Time.Clock;
 using Ryujinx.HLE.HOS.SystemState;
 using Ryujinx.HLE.Loaders.Executables;
@@ -39,6 +41,7 @@ using TimeServiceManager = Ryujinx.HLE.HOS.Services.Time.TimeManager;
 using NsoExecutable      = Ryujinx.HLE.Loaders.Executables.NsoExecutable;
 
 using static LibHac.Fs.ApplicationSaveDataManagement;
+using Ryujinx.HLE.HOS.Services.Nv;
 
 namespace Ryujinx.HLE.HOS
 {
@@ -131,6 +134,8 @@ namespace Ryujinx.HLE.HOS
 
         internal long HidBaseAddress { get; private set; }
 
+        internal NvHostSyncpt HostSyncpoint { get; private set; }
+
         public Horizon(Switch device, ContentManager contentManager)
         {
             ControlData = new BlitStruct<ApplicationControlProperty>(1);
@@ -259,6 +264,8 @@ namespace Ryujinx.HLE.HOS
             TimeServiceManager.Instance.SetupEphemeralNetworkSystemClock();
 
             DatabaseImpl.Instance.InitializeDatabase(device);
+
+            HostSyncpoint = new NvHostSyncpt(device);
         }
 
         public void LoadCart(string exeFsDir, string romFsFile = null)
@@ -870,6 +877,10 @@ namespace Ryujinx.HLE.HOS
                     Device.VsyncEvent.Set();
                 }
 
+                // Destroy nvservices channels as KThread could be waiting on some user events.
+                // This is safe as KThread that are likely to call ioctls are going to be terminated by the post handler hook on the SVC facade.
+                INvDrvServices.Destroy();
+
                 // This is needed as the IPC Dummy KThread is also counted in the ThreadCounter.
                 ThreadCounter.Signal();
 

Ryujinx.HLE/HOS/IdDictionary.cs

@@ -8,6 +8,8 @@ namespace Ryujinx.HLE.HOS
     {
         private ConcurrentDictionary<int, object> _objs;
 
+        public ICollection<object> Values => _objs.Values;
+
         public IdDictionary()
         {
             _objs = new ConcurrentDictionary<int, object>();

Ryujinx.HLE/HOS/Services/Mm/IRequest.cs

@@ -1,21 +1,30 @@
 using Ryujinx.Common.Logging;
+using Ryujinx.HLE.HOS.Services.Mm.Types;
+using System.Collections.Generic;
 
 namespace Ryujinx.HLE.HOS.Services.Mm
 {
     [Service("mm:u")]
     class IRequest : IpcService
     {
-        public IRequest(ServiceCtx context) { }
+        private static object                  _sessionListLock = new object();
+        private static List<MultiMediaSession> _sessionList     = new List<MultiMediaSession>();
+
+        private static uint _uniqueId = 1;
+
+        public IRequest(ServiceCtx context) {}
 
         [Command(0)]
         // InitializeOld(u32, u32, u32)
         public ResultCode InitializeOld(ServiceCtx context)
         {
-            int unknown0 = context.RequestData.ReadInt32();
-            int unknown1 = context.RequestData.ReadInt32();
-            int unknown2 = context.RequestData.ReadInt32();
+            MultiMediaOperationType operationType    = (MultiMediaOperationType)context.RequestData.ReadUInt32();
+            int                     fgmId            = context.RequestData.ReadInt32();
+            bool                    isAutoClearEvent = context.RequestData.ReadInt32() != 0;
 
-            Logger.PrintStub(LogClass.ServiceMm, new { unknown0, unknown1, unknown2 });
+            Logger.PrintStub(LogClass.ServiceMm, new { operationType, fgmId, isAutoClearEvent });
+
+            Register(operationType, fgmId, isAutoClearEvent);
 
             return ResultCode.Success;
         }
@@ -24,7 +33,14 @@ namespace Ryujinx.HLE.HOS.Services.Mm
         // FinalizeOld(u32)
         public ResultCode FinalizeOld(ServiceCtx context)
         {
-            Logger.PrintStub(LogClass.ServiceMm);
+            MultiMediaOperationType operationType = (MultiMediaOperationType)context.RequestData.ReadUInt32();
+
+            Logger.PrintStub(LogClass.ServiceMm, new { operationType });
+
+            lock (_sessionListLock)
+            {
+                _sessionList.Remove(GetSessionByType(operationType));
+            }
 
             return ResultCode.Success;
         }
@@ -33,11 +49,17 @@ namespace Ryujinx.HLE.HOS.Services.Mm
         // SetAndWaitOld(u32, u32, u32)
         public ResultCode SetAndWaitOld(ServiceCtx context)
         {
-            int unknown0 = context.RequestData.ReadInt32();
-            int unknown1 = context.RequestData.ReadInt32();
-            int unknown2 = context.RequestData.ReadInt32();
+            MultiMediaOperationType operationType = (MultiMediaOperationType)context.RequestData.ReadUInt32();
+            uint                    value         = context.RequestData.ReadUInt32();
+            int                     timeout       = context.RequestData.ReadInt32();
+
+            Logger.PrintStub(LogClass.ServiceMm, new { operationType, value, timeout });
+
+            lock (_sessionListLock)
+            {
+                GetSessionByType(operationType)?.SetAndWait(value, timeout);
+            }
 
-            Logger.PrintStub(LogClass.ServiceMm, new { unknown0, unknown1, unknown2 });
             return ResultCode.Success;
         }
 
@@ -45,20 +67,35 @@ namespace Ryujinx.HLE.HOS.Services.Mm
         // GetOld(u32) -> u32
         public ResultCode GetOld(ServiceCtx context)
         {
-            int unknown0 = context.RequestData.ReadInt32();
+            MultiMediaOperationType operationType = (MultiMediaOperationType)context.RequestData.ReadUInt32();
 
-            Logger.PrintStub(LogClass.ServiceMm, new { unknown0 });
+            Logger.PrintStub(LogClass.ServiceMm, new { operationType });
 
-            context.ResponseData.Write(0);
+            lock (_sessionListLock)
+            {
+                MultiMediaSession session = GetSessionByType(operationType);
+
+                uint currentValue = session == null ? 0 : session.CurrentValue;
+
+                context.ResponseData.Write(currentValue);
+            }
 
             return ResultCode.Success;
         }
 
         [Command(4)]
-        // Initialize()
+        // Initialize(u32, u32, u32) -> u32
         public ResultCode Initialize(ServiceCtx context)
         {
-            Logger.PrintStub(LogClass.ServiceMm);
+            MultiMediaOperationType operationType    = (MultiMediaOperationType)context.RequestData.ReadUInt32();
+            int                     fgmId            = context.RequestData.ReadInt32();
+            bool                    isAutoClearEvent = context.RequestData.ReadInt32() != 0;
+
+            Logger.PrintStub(LogClass.ServiceMm, new { operationType, fgmId, isAutoClearEvent });
+
+            uint id = Register(operationType, fgmId, isAutoClearEvent);
+
+            context.ResponseData.Write(id);
 
             return ResultCode.Success;
         }
@@ -67,7 +104,14 @@ namespace Ryujinx.HLE.HOS.Services.Mm
         // Finalize(u32)
         public ResultCode Finalize(ServiceCtx context)
         {
-            Logger.PrintStub(LogClass.ServiceMm);
+            uint id = context.RequestData.ReadUInt32();
+
+            Logger.PrintStub(LogClass.ServiceMm, new { id });
+
+            lock (_sessionListLock)
+            {
+                _sessionList.Remove(GetSessionById(id));
+            }
 
             return ResultCode.Success;
         }
@@ -76,11 +120,16 @@ namespace Ryujinx.HLE.HOS.Services.Mm
         // SetAndWait(u32, u32, u32)
         public ResultCode SetAndWait(ServiceCtx context)
         {
-            int unknown0 = context.RequestData.ReadInt32();
-            int unknown1 = context.RequestData.ReadInt32();
-            int unknown2 = context.RequestData.ReadInt32();
+            uint id      = context.RequestData.ReadUInt32();
+            uint value   = context.RequestData.ReadUInt32();
+            int  timeout = context.RequestData.ReadInt32();
 
-            Logger.PrintStub(LogClass.ServiceMm, new { unknown0, unknown1, unknown2 });
+            Logger.PrintStub(LogClass.ServiceMm, new { id, value, timeout });
+
+            lock (_sessionListLock)
+            {
+                GetSessionById(id)?.SetAndWait(value, timeout);
+            }
 
             return ResultCode.Success;
         }
@@ -89,13 +138,59 @@ namespace Ryujinx.HLE.HOS.Services.Mm
         // Get(u32) -> u32
         public ResultCode Get(ServiceCtx context)
         {
-            int unknown0 = context.RequestData.ReadInt32();
+            uint id = context.RequestData.ReadUInt32();
 
-            Logger.PrintStub(LogClass.ServiceMm, new { unknown0 });
+            Logger.PrintStub(LogClass.ServiceMm, new { id });
 
-            context.ResponseData.Write(0);
+            lock (_sessionListLock)
+            {
+                MultiMediaSession session = GetSessionById(id);
+
+                uint currentValue = session == null ? 0 : session.CurrentValue;
+
+                context.ResponseData.Write(currentValue);
+            }
 
             return ResultCode.Success;
         }
+
+        private MultiMediaSession GetSessionById(uint id)
+        {
+            foreach (MultiMediaSession session in _sessionList)
+            {
+                if (session.Id == id)
+                {
+                    return session;
+                }
+            }
+
+            return null;
+        }
+
+        private MultiMediaSession GetSessionByType(MultiMediaOperationType type)
+        {
+            foreach (MultiMediaSession session in _sessionList)
+            {
+                if (session.Type == type)
+                {
+                    return session;
+                }
+            }
+
+            return null;
+        }
+
+        private uint Register(MultiMediaOperationType type, int fgmId, bool isAutoClearEvent)
+        {
+            lock (_sessionListLock)
+            {
+                // Nintendo ignore the fgm id as the other interfaces were deprecated.
+                MultiMediaSession session = new MultiMediaSession(_uniqueId++, type, isAutoClearEvent);
+
+                _sessionList.Add(session);
+
+                return session.Id;
+            }
+        }
     }
 }

Ryujinx.HLE/HOS/Services/Mm/Types/MultiMediaOperationType.cs

@@ -0,0 +1,11 @@
+namespace Ryujinx.HLE.HOS.Services.Mm.Types
+{
+    enum MultiMediaOperationType : uint
+    {
+        // TODO: figure out the unknown variants.
+        Unknown2    = 2,
+        VideoDecode = 5,
+        VideoEncode = 6,
+        Unknown7    = 7
+    }
+}

Ryujinx.HLE/HOS/Services/Mm/Types/MultiMediaSession.cs

@@ -0,0 +1,24 @@
+namespace Ryujinx.HLE.HOS.Services.Mm.Types
+{
+    class MultiMediaSession
+    {
+        public MultiMediaOperationType Type { get; }
+
+        public bool IsAutoClearEvent { get; }
+        public uint Id               { get; }
+        public uint CurrentValue     { get; private set; }
+
+        public MultiMediaSession(uint id, MultiMediaOperationType type, bool isAutoClearEvent)
+        {
+            Type             = type;
+            Id               = id;
+            IsAutoClearEvent = isAutoClearEvent;
+            CurrentValue     = 0;
+        }
+
+        public void SetAndWait(uint value, int timeout)
+        {
+            CurrentValue = value;
+        }
+    }
+}

Ryujinx.HLE/HOS/Services/Nv/INvDrvServices.cs

@@ -264,7 +264,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                         errorCode = ConvertInternalErrorCode(internalResult);
 
-                        if (errorCode == NvResult.Success && (ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
+                        if ((ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
                         {
                             context.Memory.WriteBytes(context.Request.GetBufferType0x22(0).Position, arguments.ToArray());
                         }
@@ -452,7 +452,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                         errorCode = ConvertInternalErrorCode(internalResult);
 
-                        if (errorCode == NvResult.Success && (ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
+                        if ((ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
                         {
                             context.Memory.WriteBytes(context.Request.GetBufferType0x22(0).Position, arguments.ToArray());
                         }
@@ -497,7 +497,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
                         errorCode = ConvertInternalErrorCode(internalResult);
 
-                        if (errorCode == NvResult.Success && (ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
+                        if ((ioctlCommand.DirectionValue & NvIoctl.Direction.Write) != 0)
                         {
                             context.Memory.WriteBytes(context.Request.GetBufferType0x22(0).Position, arguments.ToArray());
                             context.Memory.WriteBytes(inlineOutBufferPosition, inlineOutBuffer.ToArray());
@@ -519,5 +519,17 @@ namespace Ryujinx.HLE.HOS.Services.Nv
 
             return ResultCode.Success;
         }
+
+        public static void Destroy()
+        {
+            foreach (object entry in _deviceFileIdRegistry.Values)
+            {
+                NvDeviceFile deviceFile = (NvDeviceFile)entry;
+
+                deviceFile.Close();
+            }
+
+            _deviceFileIdRegistry.Clear();
+        }
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostChannel/NvHostChannelDeviceFile.cs

@@ -1,8 +1,9 @@
 using Ryujinx.Common.Logging;
-using Ryujinx.Graphics.Gpu;
 using Ryujinx.Graphics.Gpu.Memory;
+using Ryujinx.HLE.HOS.Services.Nv.Types;
 using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostAsGpu;
 using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel.Types;
+using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl;
 using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvMap;
 using System;
 using System.Runtime.CompilerServices;
@@ -12,21 +13,42 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel
 {
     class NvHostChannelDeviceFile : NvDeviceFile
     {
+        private const uint MaxModuleSyncpoint = 16;
+
         private uint _timeout;
         private uint _submitTimeout;
         private uint _timeslice;
 
-        private GpuContext _gpu;
+        private Switch _device;
 
         private ARMeilleure.Memory.MemoryManager _memory;
 
+        public enum ResourcePolicy
+        {
+            Device,
+            Channel
+        }
+
+        protected static uint[] DeviceSyncpoints = new uint[MaxModuleSyncpoint];
+
+        protected uint[] ChannelSyncpoints;
+
+        protected static ResourcePolicy ChannelResourcePolicy = ResourcePolicy.Device;
+
+        private NvFence _channelSyncpoint;
+
         public NvHostChannelDeviceFile(ServiceCtx context) : base(context)
         {
-            _gpu           = context.Device.Gpu;
+            _device        = context.Device;
             _memory        = context.Memory;
             _timeout       = 3000;
             _submitTimeout = 0;
             _timeslice     = 0;
+
+            ChannelSyncpoints = new uint[MaxModuleSyncpoint];
+
+            _channelSyncpoint.Id = _device.System.HostSyncpoint.AllocateSyncpoint(false);
+            _channelSyncpoint.UpdateValue(_device.System.HostSyncpoint);
         }
 
         public override NvInternalResult Ioctl(NvIoctl command, Span<byte> arguments)
@@ -132,9 +154,24 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel
 
         private NvInternalResult GetSyncpoint(ref GetParameterArguments arguments)
         {
-            arguments.Value = 0;
+            if (arguments.Parameter >= MaxModuleSyncpoint)
+            {
+                return NvInternalResult.InvalidInput;
+            }
 
-            Logger.PrintStub(LogClass.ServiceNv);
+            if (ChannelResourcePolicy == ResourcePolicy.Device)
+            {
+                arguments.Value = GetSyncpointDevice(_device.System.HostSyncpoint, arguments.Parameter, false);
+            }
+            else
+            {
+                arguments.Value = GetSyncpointChannel(arguments.Parameter, false);
+            }
+
+            if (arguments.Value == 0)
+            {
+                return NvInternalResult.TryAgain;
+            }
 
             return NvInternalResult.Success;
         }
@@ -293,6 +330,10 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel
 
         private NvInternalResult AllocGpfifoEx(ref AllocGpfifoExArguments arguments)
         {
+            _channelSyncpoint.UpdateValue(_device.System.HostSyncpoint);
+
+            arguments.Fence = _channelSyncpoint;
+
             Logger.PrintStub(LogClass.ServiceNv);
 
             return NvInternalResult.Success;
@@ -300,6 +341,10 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel
 
         private NvInternalResult AllocGpfifoEx2(ref AllocGpfifoExArguments arguments)
         {
+            _channelSyncpoint.UpdateValue(_device.System.HostSyncpoint);
+
+            arguments.Fence = _channelSyncpoint;
+
             Logger.PrintStub(LogClass.ServiceNv);
 
             return NvInternalResult.Success;
@@ -330,17 +375,125 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel
 
         protected NvInternalResult SubmitGpfifo(ref SubmitGpfifoArguments header, Span<ulong> entries)
         {
-            foreach (ulong entry in entries)
+            if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceWait) && header.Flags.HasFlag(SubmitGpfifoFlags.IncrementWithValue))
             {
-                _gpu.DmaPusher.Push(entry);
+                return NvInternalResult.InvalidInput;
             }
 
-            header.Fence.Id    = 0;
-            header.Fence.Value = 0;
+            if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceWait) && !_device.System.HostSyncpoint.IsSyncpointExpired(header.Fence.Id, header.Fence.Value))
+            {
+                _device.Gpu.DmaPusher.PushHostCommandBuffer(CreateWaitCommandBuffer(header.Fence));
+            }
+
+            _device.Gpu.DmaPusher.PushEntries(entries);
+
+            header.Fence.Id = _channelSyncpoint.Id;
+
+            if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceIncrement) || header.Flags.HasFlag(SubmitGpfifoFlags.IncrementWithValue))
+            {
+                uint incrementCount = header.Flags.HasFlag(SubmitGpfifoFlags.FenceIncrement) ? 2u : 0u;
+
+                if (header.Flags.HasFlag(SubmitGpfifoFlags.IncrementWithValue))
+                {
+                    incrementCount += header.Fence.Value;
+                }
+
+                header.Fence.Value = _device.System.HostSyncpoint.IncrementSyncpointMaxExt(header.Fence.Id, (int)incrementCount);
+            }
+            else
+            {
+                header.Fence.Value = _device.System.HostSyncpoint.ReadSyncpointMaxValue(header.Fence.Id);
+            }
+
+            if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceIncrement))
+            {
+                _device.Gpu.DmaPusher.PushHostCommandBuffer(CreateIncrementCommandBuffer(ref header.Fence, header.Flags));
+            }
+
+            header.Flags = SubmitGpfifoFlags.None;
+
+            _device.Gpu.DmaPusher.SignalNewEntries();
 
             return NvInternalResult.Success;
         }
 
+        public uint GetSyncpointChannel(uint index, bool isClientManaged)
+        {
+            if (ChannelSyncpoints[index] != 0)
+            {
+                return ChannelSyncpoints[index];
+            }
+
+            ChannelSyncpoints[index] = _device.System.HostSyncpoint.AllocateSyncpoint(isClientManaged);
+
+            return ChannelSyncpoints[index];
+        }
+
+        public static uint GetSyncpointDevice(NvHostSyncpt syncpointManager, uint index, bool isClientManaged)
+        {
+            if (DeviceSyncpoints[index] != 0)
+            {
+                return DeviceSyncpoints[index];
+            }
+
+            DeviceSyncpoints[index] = syncpointManager.AllocateSyncpoint(isClientManaged);
+
+            return DeviceSyncpoints[index];
+        }
+
+        private static int[] CreateWaitCommandBuffer(NvFence fence)
+        {
+            int[] commandBuffer = new int[4];
+
+            // SyncpointValue = fence.Value;
+            commandBuffer[0] = 0x2001001C;
+            commandBuffer[1] = (int)fence.Value;
+
+            // SyncpointAction(fence.id, increment: false, switch_en: true);
+            commandBuffer[2] = 0x2001001D;
+            commandBuffer[3] = (((int)fence.Id << 8) | (0 << 0) | (1 << 4));
+
+            return commandBuffer;
+        }
+
+        private int[] CreateIncrementCommandBuffer(ref NvFence fence, SubmitGpfifoFlags flags)
+        {
+            bool hasWfi = !flags.HasFlag(SubmitGpfifoFlags.SuppressWfi);
+
+            int[] commandBuffer;
+
+            int offset = 0;
+
+            if (hasWfi)
+            {
+                commandBuffer = new int[8];
+
+                // WaitForInterrupt(handle)
+                commandBuffer[offset++] = 0x2001001E;
+                commandBuffer[offset++] = 0x0;
+            }
+            else
+            {
+                commandBuffer = new int[6];
+            }
+
+            // SyncpointValue = 0x0;
+            commandBuffer[offset++] = 0x2001001C;
+            commandBuffer[offset++] = 0x0;
+
+            // Increment the syncpoint 2 times. (mitigate a hardware bug)
+
+            // SyncpointAction(fence.id, increment: true, switch_en: false);
+            commandBuffer[offset++] = 0x2001001D;
+            commandBuffer[offset++] = (((int)fence.Id << 8) | (1 << 0) | (0 << 4));
+
+            // SyncpointAction(fence.id, increment: true, switch_en: false);
+            commandBuffer[offset++] = 0x2001001D;
+            commandBuffer[offset++] = (((int)fence.Id << 8) | (1 << 0) | (0 << 4));
+
+            return commandBuffer;
+        }
+
         public override void Close() { }
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostChannel/Types/SubmitGpfifoArguments.cs

@@ -8,7 +8,7 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel.Types
     {
         public long              Address;
         public int               NumEntries;
-        public int     Flags;
+        public SubmitGpfifoFlags Flags;
         public NvFence           Fence;
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostChannel/Types/SubmitGpfifoFlags.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel.Types
+{
+    [Flags]
+    enum SubmitGpfifoFlags : uint
+    {
+        None,
+        FenceWait          = 1 << 0,
+        FenceIncrement     = 1 << 1,
+        HwFormat           = 1 << 2,
+        SuppressWfi        = 1 << 4,
+        IncrementWithValue = 1 << 8,
+    }
+}

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/NvHostCtrlDeviceFile.cs

@@ -1,4 +1,5 @@
 using Ryujinx.Common.Logging;
+using Ryujinx.Graphics.Gpu.Synchronization;
 using Ryujinx.HLE.HOS.Kernel.Common;
 using Ryujinx.HLE.HOS.Kernel.Threading;
 using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl.Types;
@@ -13,12 +14,11 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
 {
     internal class NvHostCtrlDeviceFile : NvDeviceFile
     {
-        private const int EventsCount = 64;
+        public const int EventsCount = 64;
 
         private bool          _isProductionMode;
-        private NvHostSyncpt  _syncpt;
+        private Switch        _device;
         private NvHostEvent[] _events;
-        private KEvent        _dummyEvent;
 
         public NvHostCtrlDeviceFile(ServiceCtx context) : base(context)
         {
@@ -31,9 +31,9 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
                 _isProductionMode = true;
             }
 
-            _syncpt     = new NvHostSyncpt();
+            _device = context.Device;
+
             _events = new NvHostEvent[EventsCount];
-            _dummyEvent = new KEvent(context.Device.System);
         }
 
         public override NvInternalResult Ioctl(NvIoctl command, Span<byte> arguments)
@@ -69,6 +69,9 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
                             configArgument.CopyTo(arguments);
                         }
                         break;
+                    case 0x1c:
+                        result = CallIoctlMethod<uint>(EventSignal, arguments);
+                        break;
                     case 0x1d:
                         result = CallIoctlMethod<EventWaitArguments>(EventWait, arguments);
                         break;
@@ -78,16 +81,45 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
                     case 0x1f:
                         result = CallIoctlMethod<uint>(EventRegister, arguments);
                         break;
+                    case 0x20:
+                        result = CallIoctlMethod<uint>(EventUnregister, arguments);
+                        break;
+                    case 0x21:
+                        result = CallIoctlMethod<ulong>(EventKill, arguments);
+                        break;
                 }
             }
 
             return result;
         }
 
+        private KEvent QueryEvent(uint eventId)
+        {
+            uint eventSlot;
+            uint syncpointId;
+
+            if ((eventId >> 28) == 1)
+            {
+                eventSlot   = eventId & 0xFFFF;
+                syncpointId = (eventId >> 16) & 0xFFF;
+            }
+            else
+            {
+                eventSlot   = eventId & 0xFF;
+                syncpointId = eventId >> 4;
+            }
+
+            if (eventSlot >= EventsCount || _events[eventSlot] == null || _events[eventSlot].Fence.Id != syncpointId)
+            {
+                return null;
+            }
+
+            return _events[eventSlot].Event;
+        }
+
         public override NvInternalResult QueryEvent(out int eventHandle, uint eventId)
         {
-            // TODO: implement SyncPts <=> KEvent logic accurately. For now we return a dummy event.
-            KEvent targetEvent = _dummyEvent;
+            KEvent targetEvent = QueryEvent(eventId);
 
             if (targetEvent != null)
             {
@@ -113,24 +145,26 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
 
         private NvInternalResult SyncptIncr(ref uint id)
         {
-            if (id >= NvHostSyncpt.SyncptsCount)
+            if (id >= SynchronizationManager.MaxHardwareSyncpoints)
             {
                 return NvInternalResult.InvalidInput;
             }
 
-            _syncpt.Increment((int)id);
+            _device.System.HostSyncpoint.Increment(id);
 
             return NvInternalResult.Success;
         }
 
         private NvInternalResult SyncptWait(ref SyncptWaitArguments arguments)
         {
-            return SyncptWait(ref arguments, out _);
+            uint dummyValue = 0;
+
+            return EventWait(ref arguments.Fence, ref dummyValue, arguments.Timeout, isWaitEventAsyncCmd: false, isWaitEventCmd: false);
         }
 
         private NvInternalResult SyncptWaitEx(ref SyncptWaitExArguments arguments)
         {
-            return SyncptWait(ref arguments.Input, out arguments.Value);
+            return EventWait(ref arguments.Input.Fence, ref arguments.Value, arguments.Input.Timeout, isWaitEventAsyncCmd: false, isWaitEventCmd: false);
         }
 
         private NvInternalResult SyncptReadMax(ref NvFence arguments)
@@ -182,194 +216,237 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
 
         private NvInternalResult EventWait(ref EventWaitArguments arguments)
         {
-            return EventWait(ref arguments, async: false);
+            return EventWait(ref arguments.Fence, ref arguments.Value, arguments.Timeout, isWaitEventAsyncCmd: false, isWaitEventCmd: true);
         }
 
         private NvInternalResult EventWaitAsync(ref EventWaitArguments arguments)
         {
-            return EventWait(ref arguments, async: true);
+            return EventWait(ref arguments.Fence, ref arguments.Value, arguments.Timeout, isWaitEventAsyncCmd: true, isWaitEventCmd: false);
         }
 
         private NvInternalResult EventRegister(ref uint userEventId)
         {
-            Logger.PrintStub(LogClass.ServiceNv);
+            NvInternalResult result = EventUnregister(ref userEventId);
+
+            if (result == NvInternalResult.Success)
+            {
+                _events[userEventId] = new NvHostEvent(_device.System.HostSyncpoint, userEventId, _device.System);
+            }
+
+            return result;
+        }
+
+        private NvInternalResult EventUnregister(ref uint userEventId)
+        {
+            if (userEventId >= EventsCount)
+            {
+                return NvInternalResult.InvalidInput;
+            }
+
+            NvHostEvent hostEvent = _events[userEventId];
+
+            if (hostEvent == null)
+            {
+                return NvInternalResult.Success;
+            }
+
+            if (hostEvent.State == NvHostEventState.Available ||
+                hostEvent.State == NvHostEventState.Cancelled ||
+                hostEvent.State == NvHostEventState.Signaled)
+            {
+                _events[userEventId].Dispose();
+                _events[userEventId] = null;
+
+                return NvInternalResult.Success;
+            }
+
+            return NvInternalResult.Busy;
+        }
+
+        private NvInternalResult EventKill(ref ulong eventMask)
+        {
+            NvInternalResult result = NvInternalResult.Success;
+
+            for (uint eventId = 0; eventId < EventsCount; eventId++)
+            {
+                if ((eventMask & (1UL << (int)eventId)) != 0)
+                {
+                    NvInternalResult tmp = EventUnregister(ref eventId);
+
+                    if (tmp != NvInternalResult.Success)
+                    {
+                        result = tmp;
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        private NvInternalResult EventSignal(ref uint userEventId)
+        {
+            uint eventId = userEventId & ushort.MaxValue;
+
+            if (eventId >= EventsCount)
+            {
+                return NvInternalResult.InvalidInput;
+            }
+
+            NvHostEvent hostEvent = _events[eventId];
+
+            if (hostEvent == null)
+            {
+                return NvInternalResult.InvalidInput;
+            }
+
+            NvHostEventState oldState = hostEvent.State;
+
+            if (oldState == NvHostEventState.Waiting)
+            {
+                hostEvent.State = NvHostEventState.Cancelling;
+
+                hostEvent.Cancel(_device.Gpu);
+            }
+
+            hostEvent.State = NvHostEventState.Cancelled;
 
             return NvInternalResult.Success;
         }
 
         private NvInternalResult SyncptReadMinOrMax(ref NvFence arguments, bool max)
         {
-            if (arguments.Id >= NvHostSyncpt.SyncptsCount)
+            if (arguments.Id >= SynchronizationManager.MaxHardwareSyncpoints)
             {
                 return NvInternalResult.InvalidInput;
             }
 
             if (max)
             {
-                arguments.Value = (uint)_syncpt.GetMax((int)arguments.Id);
+                arguments.Value = _device.System.HostSyncpoint.ReadSyncpointMaxValue(arguments.Id);
             }
             else
             {
-                arguments.Value = (uint)_syncpt.GetMin((int)arguments.Id);
+                arguments.Value = _device.System.HostSyncpoint.ReadSyncpointValue(arguments.Id);
             }
 
             return NvInternalResult.Success;
         }
 
-        private NvInternalResult SyncptWait(ref SyncptWaitArguments arguments, out int value)
+        private NvInternalResult EventWait(ref NvFence fence, ref uint value, int timeout, bool isWaitEventAsyncCmd, bool isWaitEventCmd)
         {
-            if (arguments.Id >= NvHostSyncpt.SyncptsCount)
-            {
-                value = 0;
-
-                return NvInternalResult.InvalidInput;
-            }
-
-            NvInternalResult result;
-
-            if (_syncpt.MinCompare((int)arguments.Id, arguments.Thresh))
-            {
-                result = NvInternalResult.Success;
-            }
-            else if (arguments.Timeout == 0)
-            {
-                result = NvInternalResult.TryAgain;
-            }
-            else
-            {
-                Logger.PrintDebug(LogClass.ServiceNv, $"Waiting syncpt with timeout of {arguments.Timeout}ms...");
-
-                using (ManualResetEvent waitEvent = new ManualResetEvent(false))
-                {
-                    _syncpt.AddWaiter(arguments.Thresh, waitEvent);
-
-                    // Note: Negative (> INT_MAX) timeouts aren't valid on .NET,
-                    // in this case we just use the maximum timeout possible.
-                    int timeout = arguments.Timeout;
-
-                    if (timeout < -1)
-                    {
-                        timeout = int.MaxValue;
-                    }
-
-                    if (timeout == -1)
-                    {
-                        waitEvent.WaitOne();
-
-                        result = NvInternalResult.Success;
-                    }
-                    else if (waitEvent.WaitOne(timeout))
-                    {
-                        result = NvInternalResult.Success;
-                    }
-                    else
-                    {
-                        result = NvInternalResult.TimedOut;
-                    }
-                }
-
-                Logger.PrintDebug(LogClass.ServiceNv, "Resuming...");
-            }
-
-            value = _syncpt.GetMin((int)arguments.Id);
-
-            return result;
-        }
-
-        private NvInternalResult EventWait(ref EventWaitArguments arguments, bool async)
-        {
-            if (arguments.Id >= NvHostSyncpt.SyncptsCount)
+            if (fence.Id >= SynchronizationManager.MaxHardwareSyncpoints)
             {
                 return NvInternalResult.InvalidInput;
             }
 
-            if (_syncpt.MinCompare(arguments.Id, arguments.Thresh))
+            // First try to check if the syncpoint is already expired on the CPU side
+            if (_device.System.HostSyncpoint.IsSyncpointExpired(fence.Id, fence.Value))
             {
-                arguments.Value = _syncpt.GetMin(arguments.Id);
+                value = _device.System.HostSyncpoint.ReadSyncpointMinValue(fence.Id);
 
                 return NvInternalResult.Success;
             }
 
-            if (!async)
+            // Try to invalidate the CPU cache and check for expiration again.
+            uint newCachedSyncpointValue = _device.System.HostSyncpoint.UpdateMin(fence.Id);
+
+            // Has the fence already expired?
+            if (_device.System.HostSyncpoint.IsSyncpointExpired(fence.Id, fence.Value))
             {
-                arguments.Value = 0;
+                value = newCachedSyncpointValue;
+
+                return NvInternalResult.Success;
             }
 
-            if (arguments.Timeout == 0)
+            // If the timeout is 0, directly return.
+            if (timeout == 0)
             {
                 return NvInternalResult.TryAgain;
             }
 
-            NvHostEvent Event;
+            // The syncpoint value isn't at the fence yet, we need to wait.
+
+            if (!isWaitEventAsyncCmd)
+            {
+                value = 0;
+            }
+
+            NvHostEvent hostEvent;
 
             NvInternalResult result;
 
-            int eventIndex;
+            uint eventIndex;
 
-            if (async)
+            if (isWaitEventAsyncCmd)
             {
-                eventIndex = arguments.Value;
+                eventIndex = value;
 
-                if ((uint)eventIndex >= EventsCount)
+                if (eventIndex >= EventsCount)
                 {
                     return NvInternalResult.InvalidInput;
                 }
 
-                Event = _events[eventIndex];
+                hostEvent = _events[eventIndex];
             }
             else
             {
-                Event = GetFreeEvent(arguments.Id, out eventIndex);
+                hostEvent = GetFreeEvent(fence.Id, out eventIndex);
             }
 
-            if (Event != null &&
-               (Event.State == NvHostEventState.Registered ||
-                Event.State == NvHostEventState.Free))
+            if (hostEvent != null &&
+               (hostEvent.State == NvHostEventState.Available ||
+                hostEvent.State == NvHostEventState.Signaled  ||
+                hostEvent.State == NvHostEventState.Cancelled))
             {
-                Event.Id     = arguments.Id;
-                Event.Thresh = arguments.Thresh;
+                hostEvent.Wait(_device.Gpu, fence);
 
-                Event.State = NvHostEventState.Waiting;
-
-                if (!async)
+                if (isWaitEventCmd)
                 {
-                    arguments.Value = ((arguments.Id & 0xfff) << 16) | 0x10000000;
+                    value = ((fence.Id & 0xfff) << 16) | 0x10000000;
                 }
                 else
                 {
-                    arguments.Value = arguments.Id << 4;
+                    value = fence.Id << 4;
                 }
 
-                arguments.Value |= eventIndex;
+                value |= eventIndex;
 
                 result = NvInternalResult.TryAgain;
             }
             else
             {
+                Logger.PrintError(LogClass.ServiceNv, $"Invalid Event at index {eventIndex} (isWaitEventAsyncCmd: {isWaitEventAsyncCmd}, isWaitEventCmd: {isWaitEventCmd})");
+
+                if (hostEvent != null)
+                {
+                    Logger.PrintError(LogClass.ServiceNv, hostEvent.DumpState(_device.Gpu));
+                }
+
                 result = NvInternalResult.InvalidInput;
             }
 
             return result;
         }
 
-        private NvHostEvent GetFreeEvent(int id, out int eventIndex)
+        public NvHostEvent GetFreeEvent(uint id, out uint eventIndex)
         {
             eventIndex = EventsCount;
 
-            int nullIndex = EventsCount;
+            uint nullIndex = EventsCount;
 
-            for (int index = 0; index < EventsCount; index++)
+            for (uint index = 0; index < EventsCount; index++)
             {
                 NvHostEvent Event = _events[index];
 
                 if (Event != null)
                 {
-                    if (Event.State == NvHostEventState.Registered ||
-                        Event.State == NvHostEventState.Free)
+                    if (Event.State == NvHostEventState.Available ||
+                        Event.State == NvHostEventState.Signaled   ||
+                        Event.State == NvHostEventState.Cancelled)
                     {
                         eventIndex = index;
 
-                        if (Event.Id == id)
+                        if (Event.Fence.Id == id)
                         {
                             return Event;
                         }
@@ -385,7 +462,9 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
             {
                 eventIndex = nullIndex;
 
-                return _events[nullIndex] = new NvHostEvent();
+                EventRegister(ref eventIndex);
+
+                return _events[nullIndex];
             }
 
             if (eventIndex < EventsCount)
@@ -396,6 +475,44 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
             return null;
         }
 
-        public override void Close() { }
+        public override void Close()
+        {
+            Logger.PrintWarning(LogClass.ServiceNv, "Closing channel");
+
+            // If the device file need to be closed, cancel all user events and dispose events.
+            for (int i = 0; i < _events.Length; i++)
+            {
+                NvHostEvent evnt = _events[i];
+
+                if (evnt != null)
+                {
+                    if (evnt.State == NvHostEventState.Waiting)
+                    {
+                        evnt.State = NvHostEventState.Cancelling;
+
+                        evnt.Cancel(_device.Gpu);
+                    }
+                    else if (evnt.State == NvHostEventState.Signaling)
+                    {
+                        // Wait at max 9ms if the guest app is trying to signal the event while closing it..
+                        int retryCount = 0;
+                        do
+                        {
+                            if (retryCount++ > 9)
+                            {
+                                break;
+                            }
+
+                            // TODO: This should be handled by the kernel (reschedule the current thread ect), waiting for Kernel decoupling work.
+                            Thread.Sleep(1);
+                        } while (evnt.State != NvHostEventState.Signaled);
+                    }
+
+                    evnt.Dispose();
+
+                   _events[i] = null;
+                }
+            }
+        }
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/Types/EventWaitArguments.cs

@@ -1,13 +1,13 @@
-using System.Runtime.InteropServices;
+using Ryujinx.HLE.HOS.Services.Nv.Types;
+using System.Runtime.InteropServices;
 
 namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl.Types
 {
     [StructLayout(LayoutKind.Sequential)]
     struct EventWaitArguments
     {
-        public int Id;
-        public int Thresh;
+        public NvFence Fence;
         public int     Timeout;
-        public int Value;
+        public uint    Value;
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/Types/NvHostEvent.cs

@@ -1,10 +1,101 @@
+using Ryujinx.Graphics.Gpu;
+using Ryujinx.Graphics.Gpu.Synchronization;
+using Ryujinx.HLE.HOS.Kernel.Threading;
+using Ryujinx.HLE.HOS.Services.Nv.Types;
+using System;
+
 namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
 {
-    class NvHostEvent
+    class NvHostEvent : IDisposable
     {
-        public int Id;
-        public int Thresh;
-
+        public NvFence          Fence;
         public NvHostEventState State;
+        public KEvent           Event;
+
+        private uint                  _eventId;
+        private NvHostSyncpt          _syncpointManager;
+        private SyncpointWaiterHandle _waiterInformation;
+
+        public NvHostEvent(NvHostSyncpt syncpointManager, uint eventId, Horizon system)
+        {
+            Fence.Id = 0;
+
+            State = NvHostEventState.Available;
+
+            Event = new KEvent(system);
+
+            _eventId = eventId;
+
+            _syncpointManager = syncpointManager;
+        }
+
+        public void Reset()
+        {
+            Fence.Id    = NvFence.InvalidSyncPointId;
+            Fence.Value = 0;
+            State       = NvHostEventState.Available;
+        }
+
+        private void Signal()
+        {
+            NvHostEventState oldState = State;
+
+            State = NvHostEventState.Signaling;
+
+            if (oldState == NvHostEventState.Waiting)
+            {
+                Event.WritableEvent.Signal();
+            }
+
+            State = NvHostEventState.Signaled;
+        }
+
+        private void GpuSignaled()
+        {
+            Signal();
+        }
+
+        public void Cancel(GpuContext gpuContext)
+        {
+            if (_waiterInformation != null)
+            {
+                gpuContext.Synchronization.UnregisterCallback(Fence.Id, _waiterInformation);
+
+                Signal();
+            }
+
+            Event.WritableEvent.Clear();
+        }
+
+        public void Wait(GpuContext gpuContext, NvFence fence)
+        {
+            Fence = fence;
+            State = NvHostEventState.Waiting;
+
+            _waiterInformation = gpuContext.Synchronization.RegisterCallbackOnSyncpoint(Fence.Id, Fence.Value, GpuSignaled);
+        }
+
+        public string DumpState(GpuContext gpuContext)
+        {
+            string res = $"\nNvHostEvent {_eventId}:\n";
+            res += $"\tState: {State}\n";
+
+            if (State == NvHostEventState.Waiting)
+            {
+                res += "\tFence:\n";
+                res += $"\t\tId            : {Fence.Id}\n";
+                res += $"\t\tThreshold     : {Fence.Value}\n";
+                res += $"\t\tCurrent Value : {gpuContext.Synchronization.GetSyncpointValue(Fence.Id)}\n";
+                res += $"\t\tWaiter Valid  : {_waiterInformation != null}\n";
+            }
+
+            return res;
+        }
+
+        public void Dispose()
+        {
+            Event.ReadableEvent.DecrementReferenceCount();
+            Event.WritableEvent.DecrementReferenceCount();
+        }
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/Types/NvHostEventState.cs

@@ -2,9 +2,11 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
 {
     enum NvHostEventState
     {
-        Registered = 0,
+        Available  = 0,
         Waiting    = 1,
-        Busy       = 2,
-        Free       = 5
+        Cancelling = 2,
+        Signaling  = 3,
+        Signaled   = 4,
+        Cancelled  = 5
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/Types/NvHostSyncPt.cs

@@ -1,100 +1,181 @@
+using Ryujinx.Common.Logging;
+using Ryujinx.Graphics.Gpu.Synchronization;
+using Ryujinx.HLE.HOS.Kernel.Threading;
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
 using System.Threading;
 
 namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl
 {
     class NvHostSyncpt
     {
-        public const int SyncptsCount = 192;
-
         private int[]  _counterMin;
         private int[]  _counterMax;
+        private bool[] _clientManaged;
+        private bool[] _assigned;
 
-        private long _eventMask;
+        private Switch _device;
 
-        private ConcurrentDictionary<EventWaitHandle, int> _waiters;
+        private object _syncpointAllocatorLock = new object();
 
-        public NvHostSyncpt()
+        public NvHostSyncpt(Switch device)
         {
-            _counterMin = new int[SyncptsCount];
-            _counterMax = new int[SyncptsCount];
-
-            _waiters = new ConcurrentDictionary<EventWaitHandle, int>();
+            _device        = device;
+            _counterMin    = new int[SynchronizationManager.MaxHardwareSyncpoints];
+            _counterMax    = new int[SynchronizationManager.MaxHardwareSyncpoints];
+            _clientManaged = new bool[SynchronizationManager.MaxHardwareSyncpoints];
+            _assigned      = new bool[SynchronizationManager.MaxHardwareSyncpoints];
         }
 
-        public int GetMin(int id)
+        private void ReserveSyncpointLocked(uint id, bool isClientManaged)
         {
-            return _counterMin[id];
+            if (id >= SynchronizationManager.MaxHardwareSyncpoints || _assigned[id])
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
             }
 
-        public int GetMax(int id)
-        {
-            return _counterMax[id];
+            _assigned[id]      = true;
+            _clientManaged[id] = isClientManaged;
         }
 
-        public int Increment(int id)
+        public uint AllocateSyncpoint(bool isClientManaged)
         {
-            if (((_eventMask >> id) & 1) != 0)
+            lock (_syncpointAllocatorLock)
             {
-                Interlocked.Increment(ref _counterMax[id]);
-            }
-
-            return IncrementMin(id);
-        }
-
-        public int IncrementMin(int id)
+                for (uint i = 1; i < SynchronizationManager.MaxHardwareSyncpoints; i++)
                 {
-            int value = Interlocked.Increment(ref _counterMin[id]);
-
-            WakeUpWaiters(id, value);
-
-            return value;
-        }
-
-        public int IncrementMax(int id)
+                    if (!_assigned[i])
                     {
-            return Interlocked.Increment(ref _counterMax[id]);
-        }
-
-        public void AddWaiter(int threshold, EventWaitHandle waitEvent)
-        {
-            if (!_waiters.TryAdd(waitEvent, threshold))
-            {
-                throw new InvalidOperationException();
-            }
-        }
-
-        public bool RemoveWaiter(EventWaitHandle waitEvent)
-        {
-            return _waiters.TryRemove(waitEvent, out _);
-        }
-
-        private void WakeUpWaiters(int id, int newValue)
-        {
-            foreach (KeyValuePair<EventWaitHandle, int> kv in _waiters)
-            {
-                if (MinCompare(id, newValue, _counterMax[id], kv.Value))
-                {
-                    kv.Key.Set();
-
-                    _waiters.TryRemove(kv.Key, out _);
+                        ReserveSyncpointLocked(i, isClientManaged);
+                        return i;
                     }
                 }
             }
 
-        public bool MinCompare(int id, int threshold)
-        {
-            return MinCompare(id, _counterMin[id], _counterMax[id], threshold);
+            Logger.PrintError(LogClass.ServiceNv, "Cannot allocate a new syncpoint!");
+
+            return 0;
         }
 
-        private bool MinCompare(int id, int min, int max, int threshold)
+        public void ReleaseSyncpoint(uint id)
+        {
+            if (id == 0)
+            {
+                return;
+            }
+
+            lock (_syncpointAllocatorLock)
+            {
+                if (id >= SynchronizationManager.MaxHardwareSyncpoints || !_assigned[id])
+                {
+                    throw new ArgumentOutOfRangeException(nameof(id));
+                }
+
+                _assigned[id]      = false;
+                _clientManaged[id] = false;
+
+                SetSyncpointMinEqualSyncpointMax(id);
+            }
+        }
+
+        public void SetSyncpointMinEqualSyncpointMax(uint id)
+        {
+            if (id >= SynchronizationManager.MaxHardwareSyncpoints)
+            {
+                throw new ArgumentOutOfRangeException(nameof(id));
+            }
+
+            int value = (int)ReadSyncpointValue(id);
+
+            Interlocked.Exchange(ref _counterMax[id], value);
+        }
+
+        public uint ReadSyncpointValue(uint id)
+        {
+            return UpdateMin(id);
+        }
+
+        public uint ReadSyncpointMinValue(uint id)
+        {
+            return (uint)_counterMin[id];
+        }
+
+        public uint ReadSyncpointMaxValue(uint id)
+        {
+            return (uint)_counterMax[id];
+        }
+
+        private bool IsClientManaged(uint id)
+        {
+            if (id >= SynchronizationManager.MaxHardwareSyncpoints)
+            {
+                return false;
+            }
+
+            return _clientManaged[id];
+        }
+
+        public void Increment(uint id)
+        {
+            if (IsClientManaged(id))
+            {
+                IncrementSyncpointMax(id);
+            }
+
+            IncrementSyncpointGPU(id);
+        }
+
+        public uint UpdateMin(uint id)
+        {
+            uint newValue = _device.Gpu.Synchronization.GetSyncpointValue(id);
+
+            Interlocked.Exchange(ref _counterMin[id], (int)newValue);
+
+            return newValue;
+        }
+
+        private void IncrementSyncpointGPU(uint id)
+        {
+            _device.Gpu.Synchronization.IncrementSyncpoint(id);
+        }
+
+        public void IncrementSyncpointMin(uint id)
+        {
+            Interlocked.Increment(ref _counterMin[id]);
+        }
+
+        public uint IncrementSyncpointMaxExt(uint id, int count)
+        {
+            if (count == 0)
+            {
+                return ReadSyncpointMaxValue(id);
+            }
+
+            uint result = 0;
+
+            for (int i = 0; i < count; i++)
+            {
+                result = IncrementSyncpointMax(id);
+            }
+
+            return result;
+        }
+
+        private uint IncrementSyncpointMax(uint id)
+        {
+            return (uint)Interlocked.Increment(ref _counterMax[id]);
+        }
+
+        public bool IsSyncpointExpired(uint id, uint threshold)
+        {
+            return MinCompare(id, _counterMin[id], _counterMax[id], (int)threshold);
+        }
+
+        private bool MinCompare(uint id, int min, int max, int threshold)
         {
             int minDiff = min - threshold;
             int maxDiff = max - threshold;
 
-            if (((_eventMask >> id) & 1) != 0)
+            if (IsClientManaged(id))
             {
                 return minDiff >= 0;
             }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/Types/SyncptWaitArguments.cs

@@ -1,12 +1,12 @@
-using System.Runtime.InteropServices;
+using Ryujinx.HLE.HOS.Services.Nv.Types;
+using System.Runtime.InteropServices;
 
 namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl.Types
 {
     [StructLayout(LayoutKind.Sequential)]
     struct SyncptWaitArguments
     {
-        public uint Id;
-        public int  Thresh;
+        public NvFence Fence;
         public int     Timeout;
     }
 }

Ryujinx.HLE/HOS/Services/Nv/NvDrvServices/NvHostCtrl/Types/SyncptWaitExArguments.cs

@@ -6,6 +6,6 @@ namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl.Types
     struct SyncptWaitExArguments
     {
         public SyncptWaitArguments Input;
-        public int                 Value;
+        public uint                Value;
     }
 }

Ryujinx.HLE/HOS/Services/Nv/Types/NvFence.cs

@@ -1,11 +1,36 @@
-using System.Runtime.InteropServices;
+using Ryujinx.Graphics.Gpu;
+using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl;
+using System;
+using System.Runtime.InteropServices;
 
 namespace Ryujinx.HLE.HOS.Services.Nv.Types
 {
     [StructLayout(LayoutKind.Sequential, Size = 0x8)]
     internal struct NvFence
     {
+        public const uint InvalidSyncPointId = uint.MaxValue;
+
         public uint Id;
         public uint Value;
+
+        public bool IsValid()
+        {
+            return Id != InvalidSyncPointId;
+        }
+
+        public void UpdateValue(NvHostSyncpt hostSyncpt)
+        {
+            Value = hostSyncpt.ReadSyncpointValue(Id);
+        }
+
+        public bool Wait(GpuContext gpuContext, TimeSpan timeout)
+        {
+            if (IsValid())
+            {
+                return gpuContext.Synchronization.WaitOnSyncpoint(Id, Value, timeout);
+            }
+
+            return false;
+        }
     }
 }

Ryujinx.HLE/HOS/Services/SurfaceFlinger/NvFlinger.cs

@@ -1,3 +1,4 @@
+using Ryujinx.Common;
 using Ryujinx.Common.Logging;
 using Ryujinx.Graphics.GAL;
 using Ryujinx.Graphics.Gpu;
@@ -5,7 +6,9 @@ using Ryujinx.HLE.HOS.Kernel.Threading;
 using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvMap;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
 using System.Threading;
@@ -117,15 +120,37 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
         private ResultCode GbpDequeueBuffer(ServiceCtx context, BinaryReader parcelReader)
         {
             // TODO: Errors.
-            int format        = parcelReader.ReadInt32();
+            int async  = parcelReader.ReadInt32();
             int width  = parcelReader.ReadInt32();
             int height = parcelReader.ReadInt32();
-            int getTimestamps = parcelReader.ReadInt32();
+            int format = parcelReader.ReadInt32();
             int usage  = parcelReader.ReadInt32();
 
             int slot = GetFreeSlotBlocking(width, height);
 
-            return MakeReplyParcel(context, slot, 1, 0x24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+            MultiFence multiFence = MultiFence.NoFence;
+
+            using (MemoryStream ms = new MemoryStream())
+            {
+                BinaryWriter writer = new BinaryWriter(ms);
+
+                // Allocated slot
+                writer.Write(slot);
+
+                // Has multi fence
+                writer.Write(1);
+
+                // Write the multi fnece
+                WriteFlattenedObject(writer, multiFence);
+
+                // Padding
+                writer.Write(0);
+
+                // Status
+                writer.Write(0);
+
+                return MakeReplyParcel(context, ms.ToArray());
+            }
         }
 
         private ResultCode GbpQueueBuffer(ServiceCtx context, BinaryReader parcelReader)
@@ -142,8 +167,8 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
             parcelReader.BaseStream.Position = Position;
 
             _bufferQueue[slot].Transform = queueBufferObject.Transform;
+            _bufferQueue[slot].Fence     = queueBufferObject.Fence;
             _bufferQueue[slot].Crop      = queueBufferObject.Crop;
-
             _bufferQueue[slot].State     = BufferState.Queued;
 
             SendFrameBuffer(context, slot);
@@ -219,14 +244,19 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
             return reader.ReadBytes((int)flattenedObjectSize);
         }
 
-        private unsafe T ReadFlattenedObject<T>(BinaryReader reader) where T: struct
+        private T ReadFlattenedObject<T>(BinaryReader reader) where T: struct
         {
-            byte[] data = ReadFlattenedObject(reader);
+            long flattenedObjectSize = reader.ReadInt64();
 
-            fixed (byte* ptr = data)
-            {
-                return Marshal.PtrToStructure<T>((IntPtr)ptr);
+            Debug.Assert(flattenedObjectSize == Unsafe.SizeOf<T>());
+
+            return reader.ReadStruct<T>();
         }
+
+        private unsafe void WriteFlattenedObject<T>(BinaryWriter writer, T value) where T : struct
+        {
+            writer.Write(Unsafe.SizeOf<T>());
+            writer.WriteStruct(value);
         }
 
         private ResultCode MakeReplyParcel(ServiceCtx context, params int[] ints)
@@ -328,10 +358,21 @@ namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
                 format,
                 bytesPerPixel,
                 crop,
+                AcquireBuffer,
                 ReleaseBuffer,
                 slot);
         }
 
+        private void AcquireBuffer(GpuContext context, object slot)
+        {
+            AcquireBuffer(context, (int)slot);
+        }
+
+        private void AcquireBuffer(GpuContext context, int slot)
+        {
+            _bufferQueue[slot].Fence.WaitForever(context);
+        }
+
         private void ReleaseBuffer(object slot)
         {
             ReleaseBuffer((int)slot);

Ryujinx.HLE/HOS/Services/SurfaceFlinger/Types/BufferEntry.cs

@@ -8,6 +8,8 @@
 
         public Rect Crop;
 
+        public MultiFence Fence;
+
         public GbpBuffer Data;
     }
 }

Ryujinx.HLE/HOS/Services/SurfaceFlinger/Types/MultiFence.cs

@@ -1,24 +1,49 @@
-using Ryujinx.HLE.HOS.Services.Nv.Types;
+using Ryujinx.Graphics.Gpu;
+using Ryujinx.HLE.HOS.Services.Nv.Types;
+using System;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Threading;
 
 namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 {
-    [StructLayout(LayoutKind.Explicit, Size = 0x24)]
+    [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 0x24)]
     struct MultiFence
     {
-        [FieldOffset(0x0)]
         public int FenceCount;
 
-        [FieldOffset(0x4)]
-        public NvFence Fence0;
+        private byte _fenceStorageStart;
 
-        [FieldOffset(0xC)]
-        public NvFence Fence1;
+        private Span<byte> _storage => MemoryMarshal.CreateSpan(ref _fenceStorageStart, Unsafe.SizeOf<NvFence>() * 4);
 
-        [FieldOffset(0x14)]
-        public NvFence Fence2;
+        private Span<NvFence> _nvFences => MemoryMarshal.Cast<byte, NvFence>(_storage);
 
-        [FieldOffset(0x1C)]
-        public NvFence Fence3;
+        public static MultiFence NoFence
+        {
+            get
+            {
+                MultiFence fence = new MultiFence
+                {
+                    FenceCount = 0
+                };
+
+                fence._nvFences[0].Id = NvFence.InvalidSyncPointId;
+
+                return fence;
+            }
+        }
+
+        public void WaitForever(GpuContext gpuContext)
+        {
+            Wait(gpuContext, Timeout.InfiniteTimeSpan);
+        }
+
+        public void Wait(GpuContext gpuContext, TimeSpan timeout)
+        {
+            for (int i = 0; i < FenceCount; i++)
+            {
+                _nvFences[i].Wait(gpuContext, timeout);
+            }
+        }
     }
 }

Ryujinx.HLE/HOS/Services/SurfaceFlinger/Types/QueueBufferObject.cs

@@ -2,7 +2,7 @@
 
 namespace Ryujinx.HLE.HOS.Services.SurfaceFlinger
 {
-    [StructLayout(LayoutKind.Explicit)]
+    [StructLayout(LayoutKind.Explicit, Pack = 1)]
     struct QueueBufferObject
     {
         [FieldOffset(0x0)]