Ryujinx/Ryujinx.HLE/HOS/Kernel/Threading/KScheduler.cs
Mary e96ef6d532
kernel: Implement thread pinning support (#2840)
* kernel: Implement Thread pinning support

This commit adds support for 8.x thread pinning changes and implement SynchronizePreemptionState syscall.

Based on kernel 13.x reverse.

* Address gdkchan's comment

* kernel: fix missing critical section leave in SetActivity

Fix Unity games

* Implement missing bits on the interrupt handler and inline update pinning function as it cannot be generic

* Fix some bugs in SetActivity and SetCoreAndAffinityMask

* Address gdkchan's comments
2021-12-30 10:55:06 +01:00

653 lines
No EOL
23 KiB
C#

using Ryujinx.Common;
using Ryujinx.HLE.HOS.Kernel.Process;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Threading;
namespace Ryujinx.HLE.HOS.Kernel.Threading
{
partial class KScheduler : IDisposable
{
public const int PrioritiesCount = 64;
public const int CpuCoresCount = 4;
private const int RoundRobinTimeQuantumMs = 10;
private static readonly int[] PreemptionPriorities = new int[] { 59, 59, 59, 63 };
private readonly KernelContext _context;
private readonly int _coreId;
private struct SchedulingState
{
public volatile bool NeedsScheduling;
public volatile KThread SelectedThread;
}
private SchedulingState _state;
private AutoResetEvent _idleInterruptEvent;
private readonly object _idleInterruptEventLock;
private KThread _previousThread;
private KThread _currentThread;
private readonly KThread _idleThread;
public KThread PreviousThread => _previousThread;
public KThread CurrentThread => _currentThread;
public long LastContextSwitchTime { get; private set; }
public long TotalIdleTimeTicks => _idleThread.TotalTimeRunning;
public KScheduler(KernelContext context, int coreId)
{
_context = context;
_coreId = coreId;
_idleInterruptEvent = new AutoResetEvent(false);
_idleInterruptEventLock = new object();
KThread idleThread = CreateIdleThread(context, coreId);
_currentThread = idleThread;
_idleThread = idleThread;
idleThread.StartHostThread();
idleThread.SchedulerWaitEvent.Set();
}
private KThread CreateIdleThread(KernelContext context, int cpuCore)
{
KThread idleThread = new KThread(context);
idleThread.Initialize(0UL, 0UL, 0UL, PrioritiesCount, cpuCore, null, ThreadType.Dummy, IdleThreadLoop);
return idleThread;
}
public static ulong SelectThreads(KernelContext context)
{
if (context.ThreadReselectionRequested)
{
return SelectThreadsImpl(context);
}
else
{
return 0UL;
}
}
private static ulong SelectThreadsImpl(KernelContext context)
{
context.ThreadReselectionRequested = false;
ulong scheduledCoresMask = 0UL;
for (int core = 0; core < CpuCoresCount; core++)
{
KThread thread = context.PriorityQueue.ScheduledThreads(core).FirstOrDefault();
if (thread != null &&
thread.Owner != null &&
thread.Owner.PinnedThreads[core] != null &&
thread.Owner.PinnedThreads[core] != thread)
{
KThread candidate = thread.Owner.PinnedThreads[core];
if (candidate.KernelWaitersCount == 0 && !thread.Owner.IsExceptionUserThread(candidate))
{
if (candidate.SchedFlags == ThreadSchedState.Running)
{
thread = candidate;
}
else
{
thread = null;
}
}
}
scheduledCoresMask |= context.Schedulers[core].SelectThread(thread);
}
for (int core = 0; core < CpuCoresCount; core++)
{
// If the core is not idle (there's already a thread running on it),
// then we don't need to attempt load balancing.
if (context.PriorityQueue.ScheduledThreads(core).Any())
{
continue;
}
int[] srcCoresHighestPrioThreads = new int[CpuCoresCount];
int srcCoresHighestPrioThreadsCount = 0;
KThread dst = null;
// Select candidate threads that could run on this core.
// Give preference to threads that are not yet selected.
foreach (KThread suggested in context.PriorityQueue.SuggestedThreads(core))
{
if (suggested.ActiveCore < 0 || suggested != context.Schedulers[suggested.ActiveCore]._state.SelectedThread)
{
dst = suggested;
break;
}
srcCoresHighestPrioThreads[srcCoresHighestPrioThreadsCount++] = suggested.ActiveCore;
}
// Not yet selected candidate found.
if (dst != null)
{
// Priorities < 2 are used for the kernel message dispatching
// threads, we should skip load balancing entirely.
if (dst.DynamicPriority >= 2)
{
context.PriorityQueue.TransferToCore(dst.DynamicPriority, core, dst);
scheduledCoresMask |= context.Schedulers[core].SelectThread(dst);
}
continue;
}
// All candidates are already selected, choose the best one
// (the first one that doesn't make the source core idle if moved).
for (int index = 0; index < srcCoresHighestPrioThreadsCount; index++)
{
int srcCore = srcCoresHighestPrioThreads[index];
KThread src = context.PriorityQueue.ScheduledThreads(srcCore).ElementAtOrDefault(1);
if (src != null)
{
// Run the second thread on the queue on the source core,
// move the first one to the current core.
KThread origSelectedCoreSrc = context.Schedulers[srcCore]._state.SelectedThread;
scheduledCoresMask |= context.Schedulers[srcCore].SelectThread(src);
context.PriorityQueue.TransferToCore(origSelectedCoreSrc.DynamicPriority, core, origSelectedCoreSrc);
scheduledCoresMask |= context.Schedulers[core].SelectThread(origSelectedCoreSrc);
}
}
}
return scheduledCoresMask;
}
private ulong SelectThread(KThread nextThread)
{
KThread previousThread = _state.SelectedThread;
if (previousThread != nextThread)
{
if (previousThread != null)
{
previousThread.LastScheduledTime = PerformanceCounter.ElapsedTicks;
}
_state.SelectedThread = nextThread;
_state.NeedsScheduling = true;
return 1UL << _coreId;
}
else
{
return 0UL;
}
}
public static void EnableScheduling(KernelContext context, ulong scheduledCoresMask)
{
KScheduler currentScheduler = context.Schedulers[KernelStatic.GetCurrentThread().CurrentCore];
// Note that "RescheduleCurrentCore" will block, so "RescheduleOtherCores" must be done first.
currentScheduler.RescheduleOtherCores(scheduledCoresMask);
currentScheduler.RescheduleCurrentCore();
}
public static void EnableSchedulingFromForeignThread(KernelContext context, ulong scheduledCoresMask)
{
RescheduleOtherCores(context, scheduledCoresMask);
}
private void RescheduleCurrentCore()
{
if (_state.NeedsScheduling)
{
Schedule();
}
}
private void RescheduleOtherCores(ulong scheduledCoresMask)
{
RescheduleOtherCores(_context, scheduledCoresMask & ~(1UL << _coreId));
}
private static void RescheduleOtherCores(KernelContext context, ulong scheduledCoresMask)
{
while (scheduledCoresMask != 0)
{
int coreToSignal = BitOperations.TrailingZeroCount(scheduledCoresMask);
KThread threadToSignal = context.Schedulers[coreToSignal]._currentThread;
// Request the thread running on that core to stop and reschedule, if we have one.
if (threadToSignal != context.Schedulers[coreToSignal]._idleThread)
{
threadToSignal.Context.RequestInterrupt();
}
// If the core is idle, ensure that the idle thread is awaken.
context.Schedulers[coreToSignal]._idleInterruptEvent.Set();
scheduledCoresMask &= ~(1UL << coreToSignal);
}
}
private void IdleThreadLoop()
{
while (_context.Running)
{
_state.NeedsScheduling = false;
Thread.MemoryBarrier();
KThread nextThread = PickNextThread(_state.SelectedThread);
if (_idleThread != nextThread)
{
_idleThread.SchedulerWaitEvent.Reset();
WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, _idleThread.SchedulerWaitEvent);
}
_idleInterruptEvent.WaitOne();
}
lock (_idleInterruptEventLock)
{
_idleInterruptEvent.Dispose();
_idleInterruptEvent = null;
}
}
public void Schedule()
{
_state.NeedsScheduling = false;
Thread.MemoryBarrier();
KThread currentThread = KernelStatic.GetCurrentThread();
KThread selectedThread = _state.SelectedThread;
// If the thread is already scheduled and running on the core, we have nothing to do.
if (currentThread == selectedThread)
{
return;
}
currentThread.SchedulerWaitEvent.Reset();
currentThread.ThreadContext.Unlock();
// Wake all the threads that might be waiting until this thread context is unlocked.
for (int core = 0; core < CpuCoresCount; core++)
{
_context.Schedulers[core]._idleInterruptEvent.Set();
}
KThread nextThread = PickNextThread(selectedThread);
if (currentThread.Context.Running)
{
// Wait until this thread is scheduled again, and allow the next thread to run.
WaitHandle.SignalAndWait(nextThread.SchedulerWaitEvent, currentThread.SchedulerWaitEvent);
}
else
{
// Allow the next thread to run.
nextThread.SchedulerWaitEvent.Set();
// We don't need to wait since the thread is exiting, however we need to
// make sure this thread will never call the scheduler again, since it is
// no longer assigned to a core.
currentThread.MakeUnschedulable();
// Just to be sure, set the core to a invalid value.
// This will trigger a exception if it attempts to call schedule again,
// rather than leaving the scheduler in a invalid state.
currentThread.CurrentCore = -1;
}
}
private KThread PickNextThread(KThread selectedThread)
{
while (true)
{
if (selectedThread != null)
{
// Try to run the selected thread.
// We need to acquire the context lock to be sure the thread is not
// already running on another core. If it is, then we return here
// and the caller should try again once there is something available for scheduling.
// The thread currently running on the core should have been requested to
// interrupt so this is not expected to take long.
// The idle thread must also be paused if we are scheduling a thread
// on the core, as the scheduled thread will handle the next switch.
if (selectedThread.ThreadContext.Lock())
{
SwitchTo(selectedThread);
if (!_state.NeedsScheduling)
{
return selectedThread;
}
selectedThread.ThreadContext.Unlock();
}
else
{
return _idleThread;
}
}
else
{
// The core is idle now, make sure that the idle thread can run
// and switch the core when a thread is available.
SwitchTo(null);
return _idleThread;
}
_state.NeedsScheduling = false;
Thread.MemoryBarrier();
selectedThread = _state.SelectedThread;
}
}
private void SwitchTo(KThread nextThread)
{
KProcess currentProcess = KernelStatic.GetCurrentProcess();
KThread currentThread = KernelStatic.GetCurrentThread();
nextThread ??= _idleThread;
if (currentThread != nextThread)
{
long previousTicks = LastContextSwitchTime;
long currentTicks = PerformanceCounter.ElapsedTicks;
long ticksDelta = currentTicks - previousTicks;
currentThread.AddCpuTime(ticksDelta);
if (currentProcess != null)
{
currentProcess.AddCpuTime(ticksDelta);
}
LastContextSwitchTime = currentTicks;
if (currentProcess != null)
{
_previousThread = !currentThread.TerminationRequested && currentThread.ActiveCore == _coreId ? currentThread : null;
}
else if (currentThread == _idleThread)
{
_previousThread = null;
}
}
if (nextThread.CurrentCore != _coreId)
{
nextThread.CurrentCore = _coreId;
}
_currentThread = nextThread;
}
public static void PreemptionThreadLoop(KernelContext context)
{
while (context.Running)
{
context.CriticalSection.Enter();
for (int core = 0; core < CpuCoresCount; core++)
{
RotateScheduledQueue(context, core, PreemptionPriorities[core]);
}
context.CriticalSection.Leave();
Thread.Sleep(RoundRobinTimeQuantumMs);
}
}
private static void RotateScheduledQueue(KernelContext context, int core, int prio)
{
IEnumerable<KThread> scheduledThreads = context.PriorityQueue.ScheduledThreads(core);
KThread selectedThread = scheduledThreads.FirstOrDefault(x => x.DynamicPriority == prio);
KThread nextThread = null;
// Yield priority queue.
if (selectedThread != null)
{
nextThread = context.PriorityQueue.Reschedule(prio, core, selectedThread);
}
IEnumerable<KThread> SuitableCandidates()
{
foreach (KThread suggested in context.PriorityQueue.SuggestedThreads(core))
{
int suggestedCore = suggested.ActiveCore;
if (suggestedCore >= 0)
{
KThread selectedSuggestedCore = context.PriorityQueue.ScheduledThreads(suggestedCore).FirstOrDefault();
if (selectedSuggestedCore == suggested || (selectedSuggestedCore != null && selectedSuggestedCore.DynamicPriority < 2))
{
continue;
}
}
// If the candidate was scheduled after the current thread, then it's not worth it.
if (nextThread == selectedThread ||
nextThread == null ||
nextThread.LastScheduledTime >= suggested.LastScheduledTime)
{
yield return suggested;
}
}
}
// Select candidate threads that could run on this core.
// Only take into account threads that are not yet selected.
KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority == prio);
if (dst != null)
{
context.PriorityQueue.TransferToCore(prio, core, dst);
}
// If the priority of the currently selected thread is lower or same as the preemption priority,
// then try to migrate a thread with lower priority.
KThread bestCandidate = context.PriorityQueue.ScheduledThreads(core).FirstOrDefault();
if (bestCandidate != null && bestCandidate.DynamicPriority >= prio)
{
dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority < bestCandidate.DynamicPriority);
if (dst != null)
{
context.PriorityQueue.TransferToCore(dst.DynamicPriority, core, dst);
}
}
context.ThreadReselectionRequested = true;
}
public static void Yield(KernelContext context)
{
KThread currentThread = KernelStatic.GetCurrentThread();
if (!currentThread.IsSchedulable)
{
return;
}
context.CriticalSection.Enter();
if (currentThread.SchedFlags != ThreadSchedState.Running)
{
context.CriticalSection.Leave();
return;
}
KThread nextThread = context.PriorityQueue.Reschedule(currentThread.DynamicPriority, currentThread.ActiveCore, currentThread);
if (nextThread != currentThread)
{
context.ThreadReselectionRequested = true;
}
context.CriticalSection.Leave();
}
public static void YieldWithLoadBalancing(KernelContext context)
{
KThread currentThread = KernelStatic.GetCurrentThread();
if (!currentThread.IsSchedulable)
{
return;
}
context.CriticalSection.Enter();
if (currentThread.SchedFlags != ThreadSchedState.Running)
{
context.CriticalSection.Leave();
return;
}
int prio = currentThread.DynamicPriority;
int core = currentThread.ActiveCore;
// Move current thread to the end of the queue.
KThread nextThread = context.PriorityQueue.Reschedule(prio, core, currentThread);
IEnumerable<KThread> SuitableCandidates()
{
foreach (KThread suggested in context.PriorityQueue.SuggestedThreads(core))
{
int suggestedCore = suggested.ActiveCore;
if (suggestedCore >= 0)
{
KThread selectedSuggestedCore = context.Schedulers[suggestedCore]._state.SelectedThread;
if (selectedSuggestedCore == suggested || (selectedSuggestedCore != null && selectedSuggestedCore.DynamicPriority < 2))
{
continue;
}
}
// If the candidate was scheduled after the current thread, then it's not worth it,
// unless the priority is higher than the current one.
if (suggested.LastScheduledTime <= nextThread.LastScheduledTime ||
suggested.DynamicPriority < nextThread.DynamicPriority)
{
yield return suggested;
}
}
}
KThread dst = SuitableCandidates().FirstOrDefault(x => x.DynamicPriority <= prio);
if (dst != null)
{
context.PriorityQueue.TransferToCore(dst.DynamicPriority, core, dst);
context.ThreadReselectionRequested = true;
}
else if (currentThread != nextThread)
{
context.ThreadReselectionRequested = true;
}
context.CriticalSection.Leave();
}
public static void YieldToAnyThread(KernelContext context)
{
KThread currentThread = KernelStatic.GetCurrentThread();
if (!currentThread.IsSchedulable)
{
return;
}
context.CriticalSection.Enter();
if (currentThread.SchedFlags != ThreadSchedState.Running)
{
context.CriticalSection.Leave();
return;
}
int core = currentThread.ActiveCore;
context.PriorityQueue.TransferToCore(currentThread.DynamicPriority, -1, currentThread);
if (!context.PriorityQueue.ScheduledThreads(core).Any())
{
KThread selectedThread = null;
foreach (KThread suggested in context.PriorityQueue.SuggestedThreads(core))
{
int suggestedCore = suggested.ActiveCore;
if (suggestedCore < 0)
{
continue;
}
KThread firstCandidate = context.PriorityQueue.ScheduledThreads(suggestedCore).FirstOrDefault();
if (firstCandidate == suggested)
{
continue;
}
if (firstCandidate == null || firstCandidate.DynamicPriority >= 2)
{
context.PriorityQueue.TransferToCore(suggested.DynamicPriority, core, suggested);
}
selectedThread = suggested;
break;
}
if (currentThread != selectedThread)
{
context.ThreadReselectionRequested = true;
}
}
else
{
context.ThreadReselectionRequested = true;
}
context.CriticalSection.Leave();
}
public void Dispose()
{
// Ensure that the idle thread is not blocked and can exit.
lock (_idleInterruptEventLock)
{
if (_idleInterruptEvent != null)
{
_idleInterruptEvent.Set();
}
}
}
}
}