mirror of
https://git.suyu.dev/suyu/suyu.git
synced 2024-11-09 00:12:45 +01:00
Merge pull request #5275 from FernandoS27/fast-native-clock
X86/NativeClock: Improve performance of clock calculations on hot path.
This commit is contained in:
commit
a7fd61fcce
5 changed files with 174 additions and 104 deletions
|
@ -98,7 +98,6 @@ add_library(common STATIC
|
||||||
algorithm.h
|
algorithm.h
|
||||||
alignment.h
|
alignment.h
|
||||||
assert.h
|
assert.h
|
||||||
atomic_ops.cpp
|
|
||||||
atomic_ops.h
|
atomic_ops.h
|
||||||
detached_tasks.cpp
|
detached_tasks.cpp
|
||||||
detached_tasks.h
|
detached_tasks.h
|
||||||
|
|
|
@ -1,75 +0,0 @@
|
||||||
// Copyright 2020 yuzu Emulator Project
|
|
||||||
// Licensed under GPLv2 or any later version
|
|
||||||
// Refer to the license.txt file included.
|
|
||||||
|
|
||||||
#include <cstring>
|
|
||||||
|
|
||||||
#include "common/atomic_ops.h"
|
|
||||||
|
|
||||||
#if _MSC_VER
|
|
||||||
#include <intrin.h>
|
|
||||||
#endif
|
|
||||||
|
|
||||||
namespace Common {
|
|
||||||
|
|
||||||
#if _MSC_VER
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u8* pointer, u8 value, u8 expected) {
|
|
||||||
const u8 result =
|
|
||||||
_InterlockedCompareExchange8(reinterpret_cast<volatile char*>(pointer), value, expected);
|
|
||||||
return result == expected;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u16* pointer, u16 value, u16 expected) {
|
|
||||||
const u16 result =
|
|
||||||
_InterlockedCompareExchange16(reinterpret_cast<volatile short*>(pointer), value, expected);
|
|
||||||
return result == expected;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u32* pointer, u32 value, u32 expected) {
|
|
||||||
const u32 result =
|
|
||||||
_InterlockedCompareExchange(reinterpret_cast<volatile long*>(pointer), value, expected);
|
|
||||||
return result == expected;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u64* pointer, u64 value, u64 expected) {
|
|
||||||
const u64 result = _InterlockedCompareExchange64(reinterpret_cast<volatile __int64*>(pointer),
|
|
||||||
value, expected);
|
|
||||||
return result == expected;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u64* pointer, u128 value, u128 expected) {
|
|
||||||
return _InterlockedCompareExchange128(reinterpret_cast<volatile __int64*>(pointer), value[1],
|
|
||||||
value[0],
|
|
||||||
reinterpret_cast<__int64*>(expected.data())) != 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
#else
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u8* pointer, u8 value, u8 expected) {
|
|
||||||
return __sync_bool_compare_and_swap(pointer, expected, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u16* pointer, u16 value, u16 expected) {
|
|
||||||
return __sync_bool_compare_and_swap(pointer, expected, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u32* pointer, u32 value, u32 expected) {
|
|
||||||
return __sync_bool_compare_and_swap(pointer, expected, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u64* pointer, u64 value, u64 expected) {
|
|
||||||
return __sync_bool_compare_and_swap(pointer, expected, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool AtomicCompareAndSwap(volatile u64* pointer, u128 value, u128 expected) {
|
|
||||||
unsigned __int128 value_a;
|
|
||||||
unsigned __int128 expected_a;
|
|
||||||
std::memcpy(&value_a, value.data(), sizeof(u128));
|
|
||||||
std::memcpy(&expected_a, expected.data(), sizeof(u128));
|
|
||||||
return __sync_bool_compare_and_swap((unsigned __int128*)pointer, expected_a, value_a);
|
|
||||||
}
|
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|
||||||
} // namespace Common
|
|
|
@ -4,14 +4,75 @@
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include <cstring>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
#include "common/common_types.h"
|
#include "common/common_types.h"
|
||||||
|
|
||||||
|
#if _MSC_VER
|
||||||
|
#include <intrin.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
namespace Common {
|
namespace Common {
|
||||||
|
|
||||||
[[nodiscard]] bool AtomicCompareAndSwap(volatile u8* pointer, u8 value, u8 expected);
|
#if _MSC_VER
|
||||||
[[nodiscard]] bool AtomicCompareAndSwap(volatile u16* pointer, u16 value, u16 expected);
|
|
||||||
[[nodiscard]] bool AtomicCompareAndSwap(volatile u32* pointer, u32 value, u32 expected);
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u8* pointer, u8 value, u8 expected) {
|
||||||
[[nodiscard]] bool AtomicCompareAndSwap(volatile u64* pointer, u64 value, u64 expected);
|
const u8 result =
|
||||||
[[nodiscard]] bool AtomicCompareAndSwap(volatile u64* pointer, u128 value, u128 expected);
|
_InterlockedCompareExchange8(reinterpret_cast<volatile char*>(pointer), value, expected);
|
||||||
|
return result == expected;
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u16* pointer, u16 value, u16 expected) {
|
||||||
|
const u16 result =
|
||||||
|
_InterlockedCompareExchange16(reinterpret_cast<volatile short*>(pointer), value, expected);
|
||||||
|
return result == expected;
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u32* pointer, u32 value, u32 expected) {
|
||||||
|
const u32 result =
|
||||||
|
_InterlockedCompareExchange(reinterpret_cast<volatile long*>(pointer), value, expected);
|
||||||
|
return result == expected;
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u64* pointer, u64 value, u64 expected) {
|
||||||
|
const u64 result = _InterlockedCompareExchange64(reinterpret_cast<volatile __int64*>(pointer),
|
||||||
|
value, expected);
|
||||||
|
return result == expected;
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u64* pointer, u128 value, u128 expected) {
|
||||||
|
return _InterlockedCompareExchange128(reinterpret_cast<volatile __int64*>(pointer), value[1],
|
||||||
|
value[0],
|
||||||
|
reinterpret_cast<__int64*>(expected.data())) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u8* pointer, u8 value, u8 expected) {
|
||||||
|
return __sync_bool_compare_and_swap(pointer, expected, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u16* pointer, u16 value, u16 expected) {
|
||||||
|
return __sync_bool_compare_and_swap(pointer, expected, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u32* pointer, u32 value, u32 expected) {
|
||||||
|
return __sync_bool_compare_and_swap(pointer, expected, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u64* pointer, u64 value, u64 expected) {
|
||||||
|
return __sync_bool_compare_and_swap(pointer, expected, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] inline bool AtomicCompareAndSwap(volatile u64* pointer, u128 value, u128 expected) {
|
||||||
|
unsigned __int128 value_a;
|
||||||
|
unsigned __int128 expected_a;
|
||||||
|
std::memcpy(&value_a, value.data(), sizeof(u128));
|
||||||
|
std::memcpy(&expected_a, expected.data(), sizeof(u128));
|
||||||
|
return __sync_bool_compare_and_swap((unsigned __int128*)pointer, expected_a, value_a);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
} // namespace Common
|
} // namespace Common
|
||||||
|
|
|
@ -2,19 +2,74 @@
|
||||||
// Licensed under GPLv2 or any later version
|
// Licensed under GPLv2 or any later version
|
||||||
// Refer to the license.txt file included.
|
// Refer to the license.txt file included.
|
||||||
|
|
||||||
|
#include <array>
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
#include <limits>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
#ifdef _MSC_VER
|
#ifdef _MSC_VER
|
||||||
#include <intrin.h>
|
#include <intrin.h>
|
||||||
|
|
||||||
|
#pragma intrinsic(__umulh)
|
||||||
|
#pragma intrinsic(_udiv128)
|
||||||
#else
|
#else
|
||||||
#include <x86intrin.h>
|
#include <x86intrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#include "common/atomic_ops.h"
|
||||||
#include "common/uint128.h"
|
#include "common/uint128.h"
|
||||||
#include "common/x64/native_clock.h"
|
#include "common/x64/native_clock.h"
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
[[nodiscard]] u64 GetFixedPoint64Factor(u64 numerator, u64 divisor) {
|
||||||
|
#ifdef __SIZEOF_INT128__
|
||||||
|
const auto base = static_cast<unsigned __int128>(numerator) << 64ULL;
|
||||||
|
return static_cast<u64>(base / divisor);
|
||||||
|
#elif defined(_M_X64) || defined(_M_ARM64)
|
||||||
|
std::array<u64, 2> r = {0, numerator};
|
||||||
|
u64 remainder;
|
||||||
|
#if _MSC_VER < 1923
|
||||||
|
return udiv128(r[1], r[0], divisor, &remainder);
|
||||||
|
#else
|
||||||
|
return _udiv128(r[1], r[0], divisor, &remainder);
|
||||||
|
#endif
|
||||||
|
#else
|
||||||
|
// This one is bit more inaccurate.
|
||||||
|
return MultiplyAndDivide64(std::numeric_limits<u64>::max(), numerator, divisor);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
[[nodiscard]] u64 MultiplyHigh(u64 a, u64 b) {
|
||||||
|
#ifdef __SIZEOF_INT128__
|
||||||
|
return (static_cast<unsigned __int128>(a) * static_cast<unsigned __int128>(b)) >> 64;
|
||||||
|
#elif defined(_M_X64) || defined(_M_ARM64)
|
||||||
|
return __umulh(a, b); // MSVC
|
||||||
|
#else
|
||||||
|
// Generic fallback
|
||||||
|
const u64 a_lo = u32(a);
|
||||||
|
const u64 a_hi = a >> 32;
|
||||||
|
const u64 b_lo = u32(b);
|
||||||
|
const u64 b_hi = b >> 32;
|
||||||
|
|
||||||
|
const u64 a_x_b_hi = a_hi * b_hi;
|
||||||
|
const u64 a_x_b_mid = a_hi * b_lo;
|
||||||
|
const u64 b_x_a_mid = b_hi * a_lo;
|
||||||
|
const u64 a_x_b_lo = a_lo * b_lo;
|
||||||
|
|
||||||
|
const u64 carry_bit = (static_cast<u64>(static_cast<u32>(a_x_b_mid)) +
|
||||||
|
static_cast<u64>(static_cast<u32>(b_x_a_mid)) + (a_x_b_lo >> 32)) >>
|
||||||
|
32;
|
||||||
|
|
||||||
|
const u64 multhi = a_x_b_hi + (a_x_b_mid >> 32) + (b_x_a_mid >> 32) + carry_bit;
|
||||||
|
|
||||||
|
return multhi;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
namespace Common {
|
namespace Common {
|
||||||
|
|
||||||
u64 EstimateRDTSCFrequency() {
|
u64 EstimateRDTSCFrequency() {
|
||||||
|
@ -48,54 +103,71 @@ NativeClock::NativeClock(u64 emulated_cpu_frequency_, u64 emulated_clock_frequen
|
||||||
: WallClock(emulated_cpu_frequency_, emulated_clock_frequency_, true), rtsc_frequency{
|
: WallClock(emulated_cpu_frequency_, emulated_clock_frequency_, true), rtsc_frequency{
|
||||||
rtsc_frequency_} {
|
rtsc_frequency_} {
|
||||||
_mm_mfence();
|
_mm_mfence();
|
||||||
last_measure = __rdtsc();
|
time_point.inner.last_measure = __rdtsc();
|
||||||
accumulated_ticks = 0U;
|
time_point.inner.accumulated_ticks = 0U;
|
||||||
|
ns_rtsc_factor = GetFixedPoint64Factor(1000000000, rtsc_frequency);
|
||||||
|
us_rtsc_factor = GetFixedPoint64Factor(1000000, rtsc_frequency);
|
||||||
|
ms_rtsc_factor = GetFixedPoint64Factor(1000, rtsc_frequency);
|
||||||
|
clock_rtsc_factor = GetFixedPoint64Factor(emulated_clock_frequency, rtsc_frequency);
|
||||||
|
cpu_rtsc_factor = GetFixedPoint64Factor(emulated_cpu_frequency, rtsc_frequency);
|
||||||
}
|
}
|
||||||
|
|
||||||
u64 NativeClock::GetRTSC() {
|
u64 NativeClock::GetRTSC() {
|
||||||
std::scoped_lock scope{rtsc_serialize};
|
TimePoint new_time_point{};
|
||||||
|
TimePoint current_time_point{};
|
||||||
|
do {
|
||||||
|
current_time_point.pack = time_point.pack;
|
||||||
_mm_mfence();
|
_mm_mfence();
|
||||||
const u64 current_measure = __rdtsc();
|
const u64 current_measure = __rdtsc();
|
||||||
u64 diff = current_measure - last_measure;
|
u64 diff = current_measure - current_time_point.inner.last_measure;
|
||||||
diff = diff & ~static_cast<u64>(static_cast<s64>(diff) >> 63); // max(diff, 0)
|
diff = diff & ~static_cast<u64>(static_cast<s64>(diff) >> 63); // max(diff, 0)
|
||||||
if (current_measure > last_measure) {
|
new_time_point.inner.last_measure = current_measure > current_time_point.inner.last_measure
|
||||||
last_measure = current_measure;
|
? current_measure
|
||||||
}
|
: current_time_point.inner.last_measure;
|
||||||
accumulated_ticks += diff;
|
new_time_point.inner.accumulated_ticks = current_time_point.inner.accumulated_ticks + diff;
|
||||||
|
} while (!Common::AtomicCompareAndSwap(time_point.pack.data(), new_time_point.pack,
|
||||||
|
current_time_point.pack));
|
||||||
/// The clock cannot be more precise than the guest timer, remove the lower bits
|
/// The clock cannot be more precise than the guest timer, remove the lower bits
|
||||||
return accumulated_ticks & inaccuracy_mask;
|
return new_time_point.inner.accumulated_ticks & inaccuracy_mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
void NativeClock::Pause(bool is_paused) {
|
void NativeClock::Pause(bool is_paused) {
|
||||||
if (!is_paused) {
|
if (!is_paused) {
|
||||||
|
TimePoint current_time_point{};
|
||||||
|
TimePoint new_time_point{};
|
||||||
|
do {
|
||||||
|
current_time_point.pack = time_point.pack;
|
||||||
|
new_time_point.pack = current_time_point.pack;
|
||||||
_mm_mfence();
|
_mm_mfence();
|
||||||
last_measure = __rdtsc();
|
new_time_point.inner.last_measure = __rdtsc();
|
||||||
|
} while (!Common::AtomicCompareAndSwap(time_point.pack.data(), new_time_point.pack,
|
||||||
|
current_time_point.pack));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::chrono::nanoseconds NativeClock::GetTimeNS() {
|
std::chrono::nanoseconds NativeClock::GetTimeNS() {
|
||||||
const u64 rtsc_value = GetRTSC();
|
const u64 rtsc_value = GetRTSC();
|
||||||
return std::chrono::nanoseconds{MultiplyAndDivide64(rtsc_value, 1000000000, rtsc_frequency)};
|
return std::chrono::nanoseconds{MultiplyHigh(rtsc_value, ns_rtsc_factor)};
|
||||||
}
|
}
|
||||||
|
|
||||||
std::chrono::microseconds NativeClock::GetTimeUS() {
|
std::chrono::microseconds NativeClock::GetTimeUS() {
|
||||||
const u64 rtsc_value = GetRTSC();
|
const u64 rtsc_value = GetRTSC();
|
||||||
return std::chrono::microseconds{MultiplyAndDivide64(rtsc_value, 1000000, rtsc_frequency)};
|
return std::chrono::microseconds{MultiplyHigh(rtsc_value, us_rtsc_factor)};
|
||||||
}
|
}
|
||||||
|
|
||||||
std::chrono::milliseconds NativeClock::GetTimeMS() {
|
std::chrono::milliseconds NativeClock::GetTimeMS() {
|
||||||
const u64 rtsc_value = GetRTSC();
|
const u64 rtsc_value = GetRTSC();
|
||||||
return std::chrono::milliseconds{MultiplyAndDivide64(rtsc_value, 1000, rtsc_frequency)};
|
return std::chrono::milliseconds{MultiplyHigh(rtsc_value, ms_rtsc_factor)};
|
||||||
}
|
}
|
||||||
|
|
||||||
u64 NativeClock::GetClockCycles() {
|
u64 NativeClock::GetClockCycles() {
|
||||||
const u64 rtsc_value = GetRTSC();
|
const u64 rtsc_value = GetRTSC();
|
||||||
return MultiplyAndDivide64(rtsc_value, emulated_clock_frequency, rtsc_frequency);
|
return MultiplyHigh(rtsc_value, clock_rtsc_factor);
|
||||||
}
|
}
|
||||||
|
|
||||||
u64 NativeClock::GetCPUCycles() {
|
u64 NativeClock::GetCPUCycles() {
|
||||||
const u64 rtsc_value = GetRTSC();
|
const u64 rtsc_value = GetRTSC();
|
||||||
return MultiplyAndDivide64(rtsc_value, emulated_cpu_frequency, rtsc_frequency);
|
return MultiplyHigh(rtsc_value, cpu_rtsc_factor);
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace X64
|
} // namespace X64
|
||||||
|
|
|
@ -6,7 +6,6 @@
|
||||||
|
|
||||||
#include <optional>
|
#include <optional>
|
||||||
|
|
||||||
#include "common/spin_lock.h"
|
|
||||||
#include "common/wall_clock.h"
|
#include "common/wall_clock.h"
|
||||||
|
|
||||||
namespace Common {
|
namespace Common {
|
||||||
|
@ -32,14 +31,28 @@ public:
|
||||||
private:
|
private:
|
||||||
u64 GetRTSC();
|
u64 GetRTSC();
|
||||||
|
|
||||||
|
union alignas(16) TimePoint {
|
||||||
|
TimePoint() : pack{} {}
|
||||||
|
u128 pack{};
|
||||||
|
struct Inner {
|
||||||
|
u64 last_measure{};
|
||||||
|
u64 accumulated_ticks{};
|
||||||
|
} inner;
|
||||||
|
};
|
||||||
|
|
||||||
/// value used to reduce the native clocks accuracy as some apss rely on
|
/// value used to reduce the native clocks accuracy as some apss rely on
|
||||||
/// undefined behavior where the level of accuracy in the clock shouldn't
|
/// undefined behavior where the level of accuracy in the clock shouldn't
|
||||||
/// be higher.
|
/// be higher.
|
||||||
static constexpr u64 inaccuracy_mask = ~(UINT64_C(0x400) - 1);
|
static constexpr u64 inaccuracy_mask = ~(UINT64_C(0x400) - 1);
|
||||||
|
|
||||||
SpinLock rtsc_serialize{};
|
TimePoint time_point;
|
||||||
u64 last_measure{};
|
// factors
|
||||||
u64 accumulated_ticks{};
|
u64 clock_rtsc_factor{};
|
||||||
|
u64 cpu_rtsc_factor{};
|
||||||
|
u64 ns_rtsc_factor{};
|
||||||
|
u64 us_rtsc_factor{};
|
||||||
|
u64 ms_rtsc_factor{};
|
||||||
|
|
||||||
u64 rtsc_frequency;
|
u64 rtsc_frequency;
|
||||||
};
|
};
|
||||||
} // namespace X64
|
} // namespace X64
|
||||||
|
|
Loading…
Reference in a new issue