nold/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere synced 2024-12-22 12:21:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

util: better match true std::atomic semantics

Browse source
This commit is contained in:
Michael Scire 2021-10-20 11:02:17 -07:00
parent c6d7174dd3
commit aed9d3f535
12 changed files with 102 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
exosphere/program/source/secmon_exception_handler.cpp
View file

@ -22,7 +22,7 @@ namespace ams::secmon {
constexpr inline uintptr_t PMC = MemoryRegionVirtualDevicePmc.GetAddress();
constinit util::Atomic<bool> g_is_locked{false};
constinit util::Atomic<bool> g_is_locked = false;
}

2
exosphere/program/source/smc/secmon_smc_se_lock.cpp
View file

@ -21,7 +21,7 @@ namespace ams::secmon::smc {
namespace {
constinit util::Atomic<bool> g_is_locked{false};
constinit util::Atomic<bool> g_is_locked = false;
ALWAYS_INLINE bool TryLockSecurityEngineImpl() {
bool value = false;

8
libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp
View file

@ -66,7 +66,7 @@ namespace ams::kern {
KSlabHeapImpl::Free(allocated + i);
}
m_count.FetchAdd(sizeof(PageBuffer) / sizeof(T));
m_count += sizeof(PageBuffer) / sizeof(T);
}
}
@ -89,7 +89,7 @@ namespace ams::kern {
for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) {
KSlabHeapImpl::Free(allocated + i);
}
m_count.FetchAdd(sizeof(PageBuffer) / sizeof(T));
m_count += sizeof(PageBuffer) / sizeof(T);
}
}
}
@ -99,7 +99,7 @@ namespace ams::kern {
std::construct_at(allocated);
/* Update our tracking. */
size_t used = m_used.FetchAdd(1) + 1;
const size_t used = ++m_used;
size_t peak = m_peak.Load();
while (peak < used) {
if (m_peak.CompareExchangeWeak<std::memory_order_relaxed>(peak, used)) {
@ -113,7 +113,7 @@ namespace ams::kern {
ALWAYS_INLINE void Free(T *t) {
KSlabHeapImpl::Free(t);
m_used.FetchSub(1);
--m_used;
}
};

2
libraries/libmesosphere/include/mesosphere/kern_k_process.hpp
View file

@ -288,7 +288,7 @@ namespace ams::kern {
KThread *GetExceptionThread() const { return m_exception_thread; }
void AddCpuTime(s64 diff) { m_cpu_time.FetchAdd(diff); }
void AddCpuTime(s64 diff) { m_cpu_time += diff; }
s64 GetCpuTime() { return m_cpu_time.Load(); }
constexpr s64 GetScheduledCount() const { return m_schedule_count; }

8
libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp
View file

@ -176,8 +176,6 @@ namespace ams::kern {
};
static_assert(ams::util::HasRedBlackKeyType<ConditionVariableComparator>);
static_assert(std::same_as<ams::util::RedBlackKeyType<ConditionVariableComparator, void>, ConditionVariableComparator::RedBlackKeyType>);
private:
static constinit inline util::Atomic<u64> s_next_thread_id{0};
private:
util::IntrusiveListNode m_process_list_node{};
util::IntrusiveRedBlackTreeNode m_condvar_arbiter_tree_node{};
@ -348,11 +346,11 @@ namespace ams::kern {
#endif
ALWAYS_INLINE void RegisterDpc(DpcFlag flag) {
this->GetStackParameters().dpc_flags.FetchOr(flag);
this->GetStackParameters().dpc_flags |= flag;
}
ALWAYS_INLINE void ClearDpc(DpcFlag flag) {
this->GetStackParameters().dpc_flags.FetchAnd(~flag);
this->GetStackParameters().dpc_flags &= ~flag;
}
ALWAYS_INLINE u8 GetDpc() const {
@ -544,7 +542,7 @@ namespace ams::kern {
constexpr bool IsAttachedToDebugger() const { return m_debug_attached; }
void AddCpuTime(s32 core_id, s64 amount) {
m_cpu_time.FetchAdd(amount);
m_cpu_time += amount;
/* TODO: Debug kernels track per-core tick counts. Should we? */
MESOSPHERE_UNUSED(core_id);
}

2
libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp
View file

@ -287,7 +287,7 @@ namespace ams::kern::arch::arm64::cpu {
break;
}
m_target_cores.FetchAnd(~(1ul << GetCurrentCoreId()));
m_target_cores &= (~(1ul << GetCurrentCoreId()));
}
ALWAYS_INLINE void SetEventLocally() {

2
libraries/libmesosphere/source/kern_k_client_port.cpp
View file

@ -28,7 +28,7 @@ namespace ams::kern {
void KClientPort::OnSessionFinalized() {
KScopedSchedulerLock sl;
if (m_num_sessions.FetchSub(1) == m_max_sessions) {
if (const auto prev = m_num_sessions--; prev == m_max_sessions) {
this->NotifyAvailable();
}
}

12
libraries/libmesosphere/source/kern_k_process.cpp
View file

@ -25,8 +25,8 @@ namespace ams::kern {
constexpr u64 ProcessIdMin = InitialProcessIdMax + 1;
constexpr u64 ProcessIdMax = std::numeric_limits<u64>::max();
constinit util::Atomic<u64> g_initial_process_id{InitialProcessIdMin};
constinit util::Atomic<u64> g_process_id{ProcessIdMin};
constinit util::Atomic<u64> g_initial_process_id = InitialProcessIdMin;
constinit util::Atomic<u64> g_process_id = ProcessIdMin;
Result TerminateChildren(KProcess *process, const KThread *thread_to_not_terminate) {
/* Request that all children threads terminate. */
@ -299,7 +299,7 @@ namespace ams::kern {
R_TRY(m_capabilities.Initialize(caps, num_caps, std::addressof(m_page_table)));
/* Initialize the process id. */
m_process_id = g_initial_process_id.FetchAdd(1);
m_process_id = g_initial_process_id++;
MESOSPHERE_ABORT_UNLESS(InitialProcessIdMin <= m_process_id);
MESOSPHERE_ABORT_UNLESS(m_process_id <= InitialProcessIdMax);
@ -409,7 +409,7 @@ namespace ams::kern {
R_TRY(m_capabilities.Initialize(user_caps, num_caps, std::addressof(m_page_table)));
/* Initialize the process id. */
m_process_id = g_process_id.FetchAdd(1);
m_process_id = g_process_id++;
MESOSPHERE_ABORT_UNLESS(ProcessIdMin <= m_process_id);
MESOSPHERE_ABORT_UNLESS(m_process_id <= ProcessIdMax);
@ -791,13 +791,13 @@ namespace ams::kern {
void KProcess::IncrementRunningThreadCount() {
MESOSPHERE_ASSERT(m_num_running_threads.Load() >= 0);
m_num_running_threads.FetchAdd(1);
++m_num_running_threads;
}
void KProcess::DecrementRunningThreadCount() {
MESOSPHERE_ASSERT(m_num_running_threads.Load() > 0);
if (m_num_running_threads.FetchSub(1) == 1) {
if (const auto prev = m_num_running_threads--; prev == 1) {
this->Terminate();
}
}

4
libraries/libmesosphere/source/kern_k_thread.cpp
View file

@ -21,6 +21,8 @@ namespace ams::kern {
constexpr inline s32 TerminatingThreadPriority = ams::svc::SystemThreadPriorityHighest - 1;
constinit util::Atomic<u64> g_thread_id = 0;
constexpr ALWAYS_INLINE bool IsKernelAddressKey(KProcessAddress key) {
const uintptr_t key_uptr = GetInteger(key);
return KernelVirtualAddressSpaceBase <= key_uptr && key_uptr <= KernelVirtualAddressSpaceLast && (key_uptr & 1) == 0;
@ -219,7 +221,7 @@ namespace ams::kern {
this->SetInExceptionHandler();
/* Set thread ID. */
m_thread_id = s_next_thread_id.FetchAdd(1);
m_thread_id = g_thread_id++;
/* We initialized! */
m_initialized = true;

6
libraries/libmesosphere/source/kern_panic.cpp
View file

@ -42,15 +42,15 @@ namespace ams::kern {
return arr;
}();
constinit util::Atomic<s32> g_next_ticket{0};
constinit util::Atomic<s32> g_current_ticket{0};
constinit util::Atomic<s32> g_next_ticket = 0;
constinit util::Atomic<s32> g_current_ticket = 0;
constinit std::array<s32, cpu::NumCores> g_core_tickets = NegativeArray;
s32 GetCoreTicket() {
const s32 core_id = GetCurrentCoreId();
if (g_core_tickets[core_id] == -1) {
g_core_tickets[core_id] = 2 * g_next_ticket.FetchAdd(1);
g_core_tickets[core_id] = 2 * (g_next_ticket++);
}
return g_core_tickets[core_id];
}

50
libraries/libvapours/include/vapours/util/arch/arm64/util_atomic.hpp
View file

@ -110,6 +110,11 @@ namespace ams::util {
using StorageType = impl::AtomicStorage<T>;
static constexpr bool IsIntegral = std::integral<T>;
static constexpr bool IsPointer = std::is_pointer<T>::value;
static constexpr bool HasArithmeticFunctions = IsIntegral || IsPointer;
using DifferenceType = typename std::conditional<IsIntegral, T, typename std::conditional<IsPointer, std::ptrdiff_t, void>::type>::type;
static constexpr ALWAYS_INLINE T ConvertToType(StorageType s) {
if constexpr (std::integral<T>) {
@ -140,8 +145,8 @@ namespace ams::util {
ALWAYS_INLINE volatile StorageType *GetStoragePointer() { return reinterpret_cast< volatile StorageType *>(std::addressof(m_v)); }
ALWAYS_INLINE const volatile StorageType *GetStoragePointer() const { return reinterpret_cast<const volatile StorageType *>(std::addressof(m_v)); }
public:
ALWAYS_INLINE explicit Atomic() { /* ... */ }
constexpr ALWAYS_INLINE explicit Atomic(T v) : m_v(ConvertToStorage(v)) { /* ... */ }
ALWAYS_INLINE Atomic() { /* ... */ }
constexpr ALWAYS_INLINE Atomic(T v) : m_v(ConvertToStorage(v)) { /* ... */ }
constexpr ALWAYS_INLINE T operator=(T desired) {
if (std::is_constant_evaluated()) {
@ -296,27 +301,44 @@ namespace ams::util {
return true;
}
#define AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(_OPERATION_, _OPERATOR_) \
template<bool Enable = IsIntegral, typename = typename std::enable_if<Enable, void>::type> \
ALWAYS_INLINE T Fetch ## _OPERATION_(T arg) { \
static_assert(Enable); \
#define AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(_OPERATION_, _OPERATOR_, _POINTER_ALLOWED_) \
template<bool Enable = (IsIntegral || (_POINTER_ALLOWED_ && IsPointer)), typename = typename std::enable_if<Enable, void>::type> \
ALWAYS_INLINE T Fetch ## _OPERATION_(DifferenceType arg) { \
static_assert(Enable == (IsIntegral || (_POINTER_ALLOWED_ && IsPointer))); \
volatile StorageType * const p = this->GetStoragePointer(); \
const StorageType s = ConvertToStorage(arg); \
\
StorageType current; \
do { \
current = impl::LoadAcquireExclusiveForAtomic<StorageType>(p); \
} while (AMS_UNLIKELY(!impl::StoreReleaseExclusiveForAtomic<StorageType>(p, current _OPERATOR_ s))); \
return static_cast<T>(current); \
} while (AMS_UNLIKELY(!impl::StoreReleaseExclusiveForAtomic<StorageType>(p, ConvertToStorage(ConvertToType(current) _OPERATOR_ arg)))); \
return ConvertToType(current); \
} \
\
template<bool Enable = (IsIntegral || (_POINTER_ALLOWED_ && IsPointer)), typename = typename std::enable_if<Enable, void>::type> \
ALWAYS_INLINE T operator _OPERATOR_##=(DifferenceType arg) { \
static_assert(Enable == (IsIntegral || (_POINTER_ALLOWED_ && IsPointer))); \
return this->Fetch ## _OPERATION_(arg) _OPERATOR_ arg; \
}
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Add, +)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Sub, -)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(And, &)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Or, |)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Xor, ^)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Add, +, true)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Sub, -, true)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(And, &, false)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Or, |, false)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Xor, ^, false)
#undef AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator++() { static_assert(Enable == HasArithmeticFunctions); return this->FetchAdd(1) + 1; }
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator++(int) { static_assert(Enable == HasArithmeticFunctions); return this->FetchAdd(1); }
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator--() { static_assert(Enable == HasArithmeticFunctions); return this->FetchSub(1) - 1; }
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator--(int) { static_assert(Enable == HasArithmeticFunctions); return this->FetchSub(1); }
};

45
libraries/libvapours/include/vapours/util/arch/generic/util_atomic.hpp
View file

@ -55,13 +55,20 @@ namespace ams::util {
class Atomic {
NON_COPYABLE(Atomic);
NON_MOVEABLE(Atomic);
private:
static constexpr bool IsIntegral = std::integral<T>;
static constexpr bool IsPointer = std::is_pointer<T>::value;
static constexpr bool HasArithmeticFunctions = IsIntegral || IsPointer;
using DifferenceType = typename std::conditional<IsIntegral, T, typename std::conditional<IsPointer, std::ptrdiff_t, void>::type>::type;
private:
static_assert(std::atomic<T>::is_always_lock_free);
private:
std::atomic<T> m_v;
public:
ALWAYS_INLINE explicit Atomic() { /* ... */ }
constexpr ALWAYS_INLINE explicit Atomic(T v) : m_v(v) { /* ... */ }
ALWAYS_INLINE Atomic() { /* ... */ }
constexpr ALWAYS_INLINE Atomic(T v) : m_v(v) { /* ... */ }
ALWAYS_INLINE T operator=(T desired) {
return (m_v = desired);
@ -93,18 +100,38 @@ namespace ams::util {
}
#define AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(_OPERATION_, _OPERATION_LOWER_) \
ALWAYS_INLINE T Fetch ## _OPERATION_(T arg) { \
#define AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(_OPERATION_, _OPERATION_LOWER_, _OPERATOR_, _POINTER_ALLOWED_) \
template<bool Enable = (IsIntegral || (_POINTER_ALLOWED_ && IsPointer)), typename = typename std::enable_if<Enable, void>::type> \
ALWAYS_INLINE T Fetch ## _OPERATION_(DifferenceType arg) { \
static_assert(Enable == (IsIntegral || (_POINTER_ALLOWED_ && IsPointer))); \
return m_v.fetch_##_OPERATION_LOWER_(arg); \
} \
\
template<bool Enable = (IsIntegral || (_POINTER_ALLOWED_ && IsPointer)), typename = typename std::enable_if<Enable, void>::type> \
ALWAYS_INLINE T operator _OPERATOR_##=(DifferenceType arg) { \
static_assert(Enable == (IsIntegral || (_POINTER_ALLOWED_ && IsPointer))); \
return this->Fetch##_OPERATION_(arg) _OPERATOR_ arg; \
}
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Add, add)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Sub, sub)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(And, and)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Or, or)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Xor, xor)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Add, add, +, true)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Sub, sub, -, true)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(And, and, &, false)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Or, or, |, false)
AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Xor, xor, ^, false)
#undef AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator++() { static_assert(Enable == HasArithmeticFunctions); return this->FetchAdd(1) + 1; }
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator++(int) { static_assert(Enable == HasArithmeticFunctions); return this->FetchAdd(1); }
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator--() { static_assert(Enable == HasArithmeticFunctions); return this->FetchSub(1) - 1; }
template<bool Enable = HasArithmeticFunctions, typename = typename std::enable_if<Enable, void>::type>
ALWAYS_INLINE T operator--(int) { static_assert(Enable == HasArithmeticFunctions); return this->FetchSub(1); }
};