using Ryujinx.Common.Logging; using Ryujinx.Graphics.Gpu; using Ryujinx.Graphics.Gpu.Memory; using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel.Types; using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostCtrl; using Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvMap; using Ryujinx.HLE.HOS.Services.Nv.Types; using Ryujinx.Memory; using System; using System.Collections.Concurrent; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Ryujinx.HLE.HOS.Services.Nv.NvDrvServices.NvHostChannel { class NvHostChannelDeviceFile : NvDeviceFile { private static readonly ConcurrentDictionary _host1xContextRegistry = new(); private const uint MaxModuleSyncpoint = 16; private uint _timeout; private uint _submitTimeout; private uint _timeslice; private readonly Switch _device; private readonly IVirtualMemoryManager _memory; private readonly Host1xContext _host1xContext; public GpuChannel Channel { get; } public enum ResourcePolicy { Device, Channel } protected static uint[] DeviceSyncpoints = new uint[MaxModuleSyncpoint]; protected uint[] ChannelSyncpoints; protected static ResourcePolicy ChannelResourcePolicy = ResourcePolicy.Device; private NvFence _channelSyncpoint; public NvHostChannelDeviceFile(ServiceCtx context, IVirtualMemoryManager memory, long owner) : base(context, owner) { _device = context.Device; _memory = memory; _timeout = 3000; _submitTimeout = 0; _timeslice = 0; _host1xContext = GetHost1XContext(context.Device.Gpu, owner); Channel = _device.Gpu.CreateChannel(); ChannelSyncpoints = new uint[MaxModuleSyncpoint]; _channelSyncpoint.Id = _device.System.HostSyncpoint.AllocateSyncpoint(false); _channelSyncpoint.UpdateValue(_device.System.HostSyncpoint); } public override NvInternalResult Ioctl(NvIoctl command, Span arguments) { NvInternalResult result = NvInternalResult.NotImplemented; if (command.Type == NvIoctl.NvHostCustomMagic) { switch (command.Number) { case 0x01: result = Submit(arguments); break; case 0x02: result = CallIoctlMethod(GetSyncpoint, arguments); break; case 0x03: result = CallIoctlMethod(GetWaitBase, arguments); break; case 0x07: result = CallIoctlMethod(SetSubmitTimeout, arguments); break; case 0x09: result = MapCommandBuffer(arguments); break; case 0x0a: result = UnmapCommandBuffer(arguments); break; } } else if (command.Type == NvIoctl.NvHostMagic) { switch (command.Number) { case 0x01: result = CallIoctlMethod(SetNvMapFd, arguments); break; case 0x03: result = CallIoctlMethod(SetTimeout, arguments); break; case 0x08: result = SubmitGpfifo(arguments); break; case 0x09: result = CallIoctlMethod(AllocObjCtx, arguments); break; case 0x0b: result = CallIoctlMethod(ZcullBind, arguments); break; case 0x0c: result = CallIoctlMethod(SetErrorNotifier, arguments); break; case 0x0d: result = CallIoctlMethod(SetPriority, arguments); break; case 0x18: result = CallIoctlMethod(AllocGpfifoEx, arguments); break; case 0x1a: result = CallIoctlMethod(AllocGpfifoEx2, arguments); break; case 0x1d: result = CallIoctlMethod(SetTimeslice, arguments); break; } } else if (command.Type == NvIoctl.NvGpuMagic) { switch (command.Number) { case 0x14: result = CallIoctlMethod(SetUserData, arguments); break; } } return result; } private NvInternalResult Submit(Span arguments) { SubmitArguments submitHeader = GetSpanAndSkip(ref arguments, 1)[0]; Span commandBuffers = GetSpanAndSkip(ref arguments, submitHeader.CmdBufsCount); Span relocs = GetSpanAndSkip(ref arguments, submitHeader.RelocsCount); Span relocShifts = GetSpanAndSkip(ref arguments, submitHeader.RelocsCount); Span syncptIncrs = GetSpanAndSkip(ref arguments, submitHeader.SyncptIncrsCount); Span waitChecks = GetSpanAndSkip(ref arguments, submitHeader.SyncptIncrsCount); // ? Span fences = GetSpanAndSkip(ref arguments, submitHeader.FencesCount); lock (_device) { for (int i = 0; i < syncptIncrs.Length; i++) { SyncptIncr syncptIncr = syncptIncrs[i]; uint id = syncptIncr.Id; fences[i].Id = id; fences[i].Thresh = Context.Device.System.HostSyncpoint.IncrementSyncpointMax(id, syncptIncr.Incrs); } foreach (CommandBuffer commandBuffer in commandBuffers) { NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBuffer.Mem); var data = _memory.GetSpan(map.Address + commandBuffer.Offset, commandBuffer.WordsCount * 4); _host1xContext.Host1x.Submit(MemoryMarshal.Cast(data)); } } fences[0].Thresh = Context.Device.System.HostSyncpoint.IncrementSyncpointMax(fences[0].Id, 1); Span tmpCmdBuff = stackalloc int[1]; tmpCmdBuff[0] = (4 << 28) | (int)fences[0].Id; _host1xContext.Host1x.Submit(tmpCmdBuff); return NvInternalResult.Success; } private Span GetSpanAndSkip(ref Span arguments, int count) where T : unmanaged { Span output = MemoryMarshal.Cast(arguments).Slice(0, count); arguments = arguments.Slice(Unsafe.SizeOf() * count); return output; } private NvInternalResult GetSyncpoint(ref GetParameterArguments arguments) { if (arguments.Parameter >= MaxModuleSyncpoint) { return NvInternalResult.InvalidInput; } if (ChannelResourcePolicy == ResourcePolicy.Device) { arguments.Value = GetSyncpointDevice(_device.System.HostSyncpoint, arguments.Parameter, false); } else { arguments.Value = GetSyncpointChannel(arguments.Parameter, false); } if (arguments.Value == 0) { return NvInternalResult.TryAgain; } return NvInternalResult.Success; } private NvInternalResult GetWaitBase(ref GetParameterArguments arguments) { arguments.Value = 0; Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult SetSubmitTimeout(ref uint submitTimeout) { _submitTimeout = submitTimeout; Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult MapCommandBuffer(Span arguments) { int headerSize = Unsafe.SizeOf(); MapCommandBufferArguments commandBufferHeader = MemoryMarshal.Cast(arguments)[0]; Span commandBufferEntries = MemoryMarshal.Cast(arguments.Slice(headerSize)).Slice(0, commandBufferHeader.NumEntries); foreach (ref CommandBufferHandle commandBufferEntry in commandBufferEntries) { NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBufferEntry.MapHandle); if (map == null) { Logger.Warning?.Print(LogClass.ServiceNv, $"Invalid handle 0x{commandBufferEntry.MapHandle:x8}!"); return NvInternalResult.InvalidInput; } lock (map) { if (map.DmaMapAddress == 0) { ulong va = _host1xContext.MemoryAllocator.GetFreeAddress((ulong)map.Size, out ulong freeAddressStartPosition, 1, MemoryManager.PageSize); if (va != NvMemoryAllocator.PteUnmapped && va <= uint.MaxValue && (va + (uint)map.Size) <= uint.MaxValue) { _host1xContext.MemoryAllocator.AllocateRange(va, (uint)map.Size, freeAddressStartPosition); _host1xContext.Smmu.Map(map.Address, va, (uint)map.Size); map.DmaMapAddress = va; } else { map.DmaMapAddress = NvMemoryAllocator.PteUnmapped; } } commandBufferEntry.MapAddress = (int)map.DmaMapAddress; } } return NvInternalResult.Success; } private NvInternalResult UnmapCommandBuffer(Span arguments) { int headerSize = Unsafe.SizeOf(); MapCommandBufferArguments commandBufferHeader = MemoryMarshal.Cast(arguments)[0]; Span commandBufferEntries = MemoryMarshal.Cast(arguments.Slice(headerSize)).Slice(0, commandBufferHeader.NumEntries); foreach (ref CommandBufferHandle commandBufferEntry in commandBufferEntries) { NvMapHandle map = NvMapDeviceFile.GetMapFromHandle(Owner, commandBufferEntry.MapHandle); if (map == null) { Logger.Warning?.Print(LogClass.ServiceNv, $"Invalid handle 0x{commandBufferEntry.MapHandle:x8}!"); return NvInternalResult.InvalidInput; } lock (map) { if (map.DmaMapAddress != 0) { // FIXME: // To make unmapping work, we need separate address space per channel. // Right now NVDEC and VIC share the GPU address space which is not correct at all. // _host1xContext.MemoryAllocator.Free((ulong)map.DmaMapAddress, (uint)map.Size); // map.DmaMapAddress = 0; } } } return NvInternalResult.Success; } private NvInternalResult SetNvMapFd(ref int nvMapFd) { Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult SetTimeout(ref uint timeout) { _timeout = timeout; Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult SubmitGpfifo(Span arguments) { int headerSize = Unsafe.SizeOf(); SubmitGpfifoArguments gpfifoSubmissionHeader = MemoryMarshal.Cast(arguments)[0]; Span gpfifoEntries = MemoryMarshal.Cast(arguments.Slice(headerSize)).Slice(0, gpfifoSubmissionHeader.NumEntries); return SubmitGpfifo(ref gpfifoSubmissionHeader, gpfifoEntries); } private NvInternalResult AllocObjCtx(ref AllocObjCtxArguments arguments) { Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult ZcullBind(ref ZcullBindArguments arguments) { Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult SetErrorNotifier(ref SetErrorNotifierArguments arguments) { Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult SetPriority(ref NvChannelPriority priority) { switch (priority) { case NvChannelPriority.Low: _timeslice = 1300; // Timeslice low priority in micro-seconds break; case NvChannelPriority.Medium: _timeslice = 2600; // Timeslice medium priority in micro-seconds break; case NvChannelPriority.High: _timeslice = 5200; // Timeslice high priority in micro-seconds break; default: return NvInternalResult.InvalidInput; } Logger.Stub?.PrintStub(LogClass.ServiceNv); // TODO: disable and preempt channel when GPU scheduler will be implemented. return NvInternalResult.Success; } private NvInternalResult AllocGpfifoEx(ref AllocGpfifoExArguments arguments) { _channelSyncpoint.UpdateValue(_device.System.HostSyncpoint); arguments.Fence = _channelSyncpoint; Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult AllocGpfifoEx2(ref AllocGpfifoExArguments arguments) { _channelSyncpoint.UpdateValue(_device.System.HostSyncpoint); arguments.Fence = _channelSyncpoint; Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } private NvInternalResult SetTimeslice(ref uint timeslice) { if (timeslice < 1000 || timeslice > 50000) { return NvInternalResult.InvalidInput; } _timeslice = timeslice; // in micro-seconds Logger.Stub?.PrintStub(LogClass.ServiceNv); // TODO: disable and preempt channel when GPU scheduler will be implemented. return NvInternalResult.Success; } private NvInternalResult SetUserData(ref ulong userData) { Logger.Stub?.PrintStub(LogClass.ServiceNv); return NvInternalResult.Success; } protected NvInternalResult SubmitGpfifo(ref SubmitGpfifoArguments header, Span entries) { if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceWait) && header.Flags.HasFlag(SubmitGpfifoFlags.IncrementWithValue)) { return NvInternalResult.InvalidInput; } if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceWait) && !_device.System.HostSyncpoint.IsSyncpointExpired(header.Fence.Id, header.Fence.Value)) { Channel.PushHostCommandBuffer(CreateWaitCommandBuffer(header.Fence)); } Channel.PushEntries(entries); header.Fence.Id = _channelSyncpoint.Id; if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceIncrement) || header.Flags.HasFlag(SubmitGpfifoFlags.IncrementWithValue)) { uint incrementCount = header.Flags.HasFlag(SubmitGpfifoFlags.FenceIncrement) ? 2u : 0u; if (header.Flags.HasFlag(SubmitGpfifoFlags.IncrementWithValue)) { incrementCount += header.Fence.Value; } header.Fence.Value = _device.System.HostSyncpoint.IncrementSyncpointMaxExt(header.Fence.Id, (int)incrementCount); } else { header.Fence.Value = _device.System.HostSyncpoint.ReadSyncpointMaxValue(header.Fence.Id); } if (header.Flags.HasFlag(SubmitGpfifoFlags.FenceIncrement)) { Channel.PushHostCommandBuffer(CreateIncrementCommandBuffer(ref header.Fence, header.Flags)); } header.Flags = SubmitGpfifoFlags.None; _device.Gpu.GPFifo.SignalNewEntries(); return NvInternalResult.Success; } public uint GetSyncpointChannel(uint index, bool isClientManaged) { if (ChannelSyncpoints[index] != 0) { return ChannelSyncpoints[index]; } ChannelSyncpoints[index] = _device.System.HostSyncpoint.AllocateSyncpoint(isClientManaged); return ChannelSyncpoints[index]; } public static uint GetSyncpointDevice(NvHostSyncpt syncpointManager, uint index, bool isClientManaged) { if (DeviceSyncpoints[index] != 0) { return DeviceSyncpoints[index]; } DeviceSyncpoints[index] = syncpointManager.AllocateSyncpoint(isClientManaged); return DeviceSyncpoints[index]; } private static int[] CreateWaitCommandBuffer(NvFence fence) { int[] commandBuffer = new int[4]; // SyncpointValue = fence.Value; commandBuffer[0] = 0x2001001C; commandBuffer[1] = (int)fence.Value; // SyncpointAction(fence.id, increment: false, switch_en: true); commandBuffer[2] = 0x2001001D; commandBuffer[3] = (((int)fence.Id << 8) | (0 << 0) | (1 << 4)); return commandBuffer; } private int[] CreateIncrementCommandBuffer(ref NvFence fence, SubmitGpfifoFlags flags) { bool hasWfi = !flags.HasFlag(SubmitGpfifoFlags.SuppressWfi); int[] commandBuffer; int offset = 0; if (hasWfi) { commandBuffer = new int[8]; // WaitForInterrupt(handle) commandBuffer[offset++] = 0x2001001E; commandBuffer[offset++] = 0x0; } else { commandBuffer = new int[6]; } // SyncpointValue = 0x0; commandBuffer[offset++] = 0x2001001C; commandBuffer[offset++] = 0x0; // Increment the syncpoint 2 times. (mitigate a hardware bug) // SyncpointAction(fence.id, increment: true, switch_en: false); commandBuffer[offset++] = 0x2001001D; commandBuffer[offset++] = (((int)fence.Id << 8) | (1 << 0) | (0 << 4)); // SyncpointAction(fence.id, increment: true, switch_en: false); commandBuffer[offset++] = 0x2001001D; commandBuffer[offset++] = (((int)fence.Id << 8) | (1 << 0) | (0 << 4)); return commandBuffer; } public override void Close() { Channel.Dispose(); } private static Host1xContext GetHost1XContext(GpuContext gpu, long pid) { return _host1xContextRegistry.GetOrAdd(pid, (long key) => new Host1xContext(gpu, key)); } public static void Destroy() { foreach (Host1xContext host1xContext in _host1xContextRegistry.Values) { host1xContext.Dispose(); } _host1xContextRegistry.Clear(); } } }