diff --git a/libraries/libmesosphere/Makefile b/libraries/libmesosphere/Makefile index 8c8c0720c..c60cc8650 100644 --- a/libraries/libmesosphere/Makefile +++ b/libraries/libmesosphere/Makefile @@ -142,8 +142,6 @@ $(OFILES_SRC) : $(HFILES_BIN) kern_libc_generic.o: CFLAGS += -fno-builtin -kern_k_auto_object.o kern_k_debug_base_process_holder.o: CXXFLAGS += -fno-lto - #--------------------------------------------------------------------------------- %_bin.h %.bin.o : %.bin #--------------------------------------------------------------------------------- diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp index f1dca1b7f..aa12f494f 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp @@ -41,8 +41,48 @@ namespace ams::kern { virtual const char *GetTypeName() { return GetStaticTypeName(); } \ private: - class KAutoObject { + public: + class ReferenceCount { + NON_COPYABLE(ReferenceCount); + NON_MOVEABLE(ReferenceCount); + private: + using Storage = u32; + private: + util::Atomic m_value; + public: + ALWAYS_INLINE explicit ReferenceCount() { /* ... */ } + constexpr ALWAYS_INLINE explicit ReferenceCount(Storage v) : m_value(v) { /* ... */ } + + ALWAYS_INLINE void operator=(Storage v) { m_value = v; } + + ALWAYS_INLINE Storage GetValue() const { return m_value.Load(); } + + ALWAYS_INLINE bool Open() { + /* Atomically increment the reference count, only if it's positive. */ + u32 cur = m_value.Load(); + do { + if (AMS_UNLIKELY(cur == 0)) { + MESOSPHERE_AUDIT(cur != 0); + return false; + } + MESOSPHERE_ABORT_UNLESS(cur < cur + 1); + } while (AMS_UNLIKELY(!m_value.CompareExchangeWeak(cur, cur + 1))); + + return true; + } + + ALWAYS_INLINE bool Close() { + /* Atomically decrement the reference count, not allowing it to become negative. */ + u32 cur = m_value.Load(); + do { + MESOSPHERE_ABORT_UNLESS(cur > 0); + } while (AMS_UNLIKELY(!m_value.CompareExchangeWeak(cur, cur - 1))); + + /* Return whether the object was closed. */ + return cur - 1 == 0; + } + }; protected: class TypeObj { private: @@ -74,7 +114,7 @@ namespace ams::kern { MESOSPHERE_AUTOOBJECT_TRAITS(KAutoObject, KAutoObject); private: KAutoObject *m_next_closed_object; - std::atomic m_ref_count; + ReferenceCount m_ref_count; #if defined(MESOSPHERE_ENABLE_DEVIRTUALIZED_DYNAMIC_CAST) ClassTokenType m_class_token; #endif @@ -98,7 +138,7 @@ namespace ams::kern { virtual KProcess *GetOwner() const { return nullptr; } u32 GetReferenceCount() const { - return m_ref_count.load(); + return m_ref_count.GetValue(); } ALWAYS_INLINE bool IsDerivedFrom(const TypeObj &rhs) const { @@ -141,8 +181,19 @@ namespace ams::kern { } } - bool Open(); - void Close(); + NOINLINE bool Open() { + MESOSPHERE_ASSERT_THIS(); + + return m_ref_count.Open(); + } + + NOINLINE void Close() { + MESOSPHERE_ASSERT_THIS(); + + if (m_ref_count.Close()) { + this->ScheduleDestruction(); + } + } private: /* NOTE: This has to be defined *after* KThread is defined. */ /* Nintendo seems to handle this by defining Open/Close() in a cpp, but we'd like them to remain in headers. */ @@ -256,5 +307,52 @@ namespace ams::kern { constexpr ALWAYS_INLINE bool IsNotNull() const { return m_obj != nullptr; } }; + template requires std::derived_from + class KSharedAutoObject { + private: + T *m_object; + KAutoObject::ReferenceCount m_ref_count; + public: + explicit KSharedAutoObject() : m_object(nullptr) { /* ... */ } + + void Attach(T *obj) { + MESOSPHERE_ASSERT(m_object == nullptr); + + /* Set our object. */ + m_object = obj; + + /* Open reference to our object. */ + m_object->Open(); + + /* Set our reference count. */ + m_ref_count = 1; + } + + bool Open() { + return m_ref_count.Open(); + } + + void Close() { + if (m_ref_count.Close()) { + this->Detach(); + } + } + + ALWAYS_INLINE T *Get() const { + return m_object; + } + private: + void Detach() { + /* Close our object, if we have one. */ + if (T * const object = m_object; AMS_LIKELY(object != nullptr)) { + /* Set our object to a debug sentinel value, which will cause crash if accessed. */ + m_object = reinterpret_cast(1); + + /* Close reference to our object. */ + object->Close(); + } + } + }; + } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp index 723452df3..f4cd50fe4 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_client_port.hpp @@ -28,12 +28,12 @@ namespace ams::kern { class KClientPort final : public KSynchronizationObject { MESOSPHERE_AUTOOBJECT_TRAITS(KClientPort, KSynchronizationObject); private: - std::atomic m_num_sessions; - std::atomic m_peak_sessions; + util::Atomic m_num_sessions; + util::Atomic m_peak_sessions; s32 m_max_sessions; KPort *m_parent; public: - constexpr KClientPort() : m_num_sessions(), m_peak_sessions(), m_max_sessions(), m_parent() { /* ... */ } + constexpr KClientPort() : m_num_sessions(0), m_peak_sessions(0), m_max_sessions(), m_parent() { /* ... */ } void Initialize(KPort *parent, s32 max_sessions); void OnSessionFinalized(); @@ -41,8 +41,8 @@ namespace ams::kern { constexpr const KPort *GetParent() const { return m_parent; } - ALWAYS_INLINE s32 GetNumSessions() const { return m_num_sessions; } - ALWAYS_INLINE s32 GetPeakSessions() const { return m_peak_sessions; } + ALWAYS_INLINE s32 GetNumSessions() const { return m_num_sessions.Load(); } + ALWAYS_INLINE s32 GetPeakSessions() const { return m_peak_sessions.Load(); } ALWAYS_INLINE s32 GetMaxSessions() const { return m_max_sessions; } bool IsLight() const; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp index ffe1544af..ff8797866 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp @@ -25,26 +25,10 @@ namespace ams::kern { class KDebugBase : public KSynchronizationObject { protected: using DebugEventList = util::IntrusiveListBaseTraits::ListType; - private: - class ProcessHolder { - private: - friend class KDebugBase; - private: - KProcess *m_process; - std::atomic m_ref_count; - private: - explicit ProcessHolder() : m_process(nullptr) { /* ... */ } - - void Attach(KProcess *process); - void Detach(); - - bool Open(); - void Close(); - }; private: DebugEventList m_event_info_list; u32 m_continue_flags; - ProcessHolder m_process_holder; + KSharedAutoObject m_process_holder; KLightLock m_lock; KProcess::State m_old_process_state; bool m_is_attached; @@ -89,7 +73,7 @@ namespace ams::kern { } ALWAYS_INLINE KProcess *GetProcessUnsafe() const { - return m_process_holder.m_process; + return m_process_holder.Get(); } private: void PushDebugEvent(ams::svc::DebugEvent event, uintptr_t param0 = 0, uintptr_t param1 = 0, uintptr_t param2 = 0, uintptr_t param3 = 0, uintptr_t param4 = 0); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp index 85995c402..2de558c96 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_dynamic_slab_heap.hpp @@ -29,9 +29,9 @@ namespace ams::kern { private: using PageBuffer = KDynamicPageManager::PageBuffer; private: - std::atomic m_used{}; - std::atomic m_peak{}; - std::atomic m_count{}; + util::Atomic m_used{0}; + util::Atomic m_peak{0}; + util::Atomic m_count{0}; KVirtualAddress m_address{}; size_t m_size{}; public: @@ -39,9 +39,9 @@ namespace ams::kern { constexpr ALWAYS_INLINE KVirtualAddress GetAddress() const { return m_address; } constexpr ALWAYS_INLINE size_t GetSize() const { return m_size; } - constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.load(); } - constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.load(); } - constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.load(); } + constexpr ALWAYS_INLINE size_t GetUsed() const { return m_used.Load(); } + constexpr ALWAYS_INLINE size_t GetPeak() const { return m_peak.Load(); } + constexpr ALWAYS_INLINE size_t GetCount() const { return m_count.Load(); } constexpr ALWAYS_INLINE bool IsInRange(KVirtualAddress addr) const { return this->GetAddress() <= addr && addr <= this->GetAddress() + this->GetSize() - 1; @@ -58,7 +58,7 @@ namespace ams::kern { KSlabHeapImpl::Initialize(); /* Allocate until we have the correct number of objects. */ - while (m_count.load() < num_objects) { + while (m_count.Load() < num_objects) { auto *allocated = reinterpret_cast(page_allocator->Allocate()); MESOSPHERE_ABORT_UNLESS(allocated != nullptr); @@ -66,7 +66,7 @@ namespace ams::kern { KSlabHeapImpl::Free(allocated + i); } - m_count.fetch_add(sizeof(PageBuffer) / sizeof(T)); + m_count.FetchAdd(sizeof(PageBuffer) / sizeof(T)); } } @@ -89,7 +89,7 @@ namespace ams::kern { for (size_t i = 1; i < sizeof(PageBuffer) / sizeof(T); i++) { KSlabHeapImpl::Free(allocated + i); } - m_count.fetch_add(sizeof(PageBuffer) / sizeof(T)); + m_count.FetchAdd(sizeof(PageBuffer) / sizeof(T)); } } } @@ -99,10 +99,10 @@ namespace ams::kern { std::construct_at(allocated); /* Update our tracking. */ - size_t used = m_used.fetch_add(1) + 1; - size_t peak = m_peak.load(); + size_t used = m_used.FetchAdd(1) + 1; + size_t peak = m_peak.Load(); while (peak < used) { - if (m_peak.compare_exchange_weak(peak, used, std::memory_order_relaxed)) { + if (m_peak.CompareExchangeWeak(peak, used)) { break; } } @@ -113,7 +113,7 @@ namespace ams::kern { ALWAYS_INLINE void Free(T *t) { KSlabHeapImpl::Free(t); - m_used.fetch_sub(1); + m_used.FetchSub(1); } }; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp index 36bbd1af8..bb5443823 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp @@ -23,7 +23,7 @@ namespace ams::kern { class KLightLock { private: - std::atomic m_tag; + util::Atomic m_tag; public: constexpr KLightLock() : m_tag(0) { /* ... */ } @@ -31,12 +31,11 @@ namespace ams::kern { MESOSPHERE_ASSERT_THIS(); const uintptr_t cur_thread = reinterpret_cast(GetCurrentThreadPointer()); - const uintptr_t cur_thread_tag = (cur_thread | 1); while (true) { - uintptr_t old_tag = m_tag.load(std::memory_order_relaxed); + uintptr_t old_tag = m_tag.Load(); - while (!m_tag.compare_exchange_weak(old_tag, (old_tag == 0) ? cur_thread : old_tag | 1, std::memory_order_acquire)) { + while (!m_tag.CompareExchangeWeak(old_tag, (old_tag == 0) ? cur_thread : (old_tag | 1))) { /* ... */ } @@ -52,7 +51,7 @@ namespace ams::kern { const uintptr_t cur_thread = reinterpret_cast(GetCurrentThreadPointer()); uintptr_t expected = cur_thread; - if (!m_tag.compare_exchange_strong(expected, 0, std::memory_order_release)) { + if (!m_tag.CompareExchangeStrong(expected, 0)) { this->UnlockSlowPath(cur_thread); } } @@ -60,8 +59,8 @@ namespace ams::kern { bool LockSlowPath(uintptr_t owner, uintptr_t cur_thread); void UnlockSlowPath(uintptr_t cur_thread); - ALWAYS_INLINE bool IsLocked() const { return m_tag.load() != 0; } - ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.load() | 0x1ul) == (reinterpret_cast(GetCurrentThreadPointer()) | 0x1ul); } + ALWAYS_INLINE bool IsLocked() const { return m_tag.Load() != 0; } + ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.Load() | 0x1ul) == (reinterpret_cast(GetCurrentThreadPointer()) | 0x1ul); } }; using KScopedLightLock = KScopedLock; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp index 4246f1680..e1a9ed684 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -58,7 +58,7 @@ namespace ams::kern { using TLPIterator = TLPTree::iterator; private: KProcessPageTable m_page_table{}; - std::atomic m_used_kernel_memory_size{}; + util::Atomic m_used_kernel_memory_size{0}; TLPTree m_fully_used_tlp_tree{}; TLPTree m_partially_used_tlp_tree{}; s32 m_ideal_core_id{}; @@ -77,7 +77,7 @@ namespace ams::kern { bool m_is_initialized{}; bool m_is_application{}; char m_name[13]{}; - std::atomic m_num_running_threads{}; + util::Atomic m_num_running_threads{0}; u32 m_flags{}; KMemoryManager::Pool m_memory_pool{}; s64 m_schedule_count{}; @@ -109,14 +109,14 @@ namespace ams::kern { KThread *m_running_threads[cpu::NumCores]{}; u64 m_running_thread_idle_counts[cpu::NumCores]{}; KThread *m_pinned_threads[cpu::NumCores]{}; - std::atomic m_cpu_time{}; - std::atomic m_num_process_switches{}; - std::atomic m_num_thread_switches{}; - std::atomic m_num_fpu_switches{}; - std::atomic m_num_supervisor_calls{}; - std::atomic m_num_ipc_messages{}; - std::atomic m_num_ipc_replies{}; - std::atomic m_num_ipc_receives{}; + util::Atomic m_cpu_time{0}; + util::Atomic m_num_process_switches{0}; + util::Atomic m_num_thread_switches{0}; + util::Atomic m_num_fpu_switches{0}; + util::Atomic m_num_supervisor_calls{0}; + util::Atomic m_num_ipc_messages{0}; + util::Atomic m_num_ipc_replies{0}; + util::Atomic m_num_ipc_receives{0}; KDynamicPageManager m_dynamic_page_manager{}; KMemoryBlockSlabManager m_memory_block_slab_manager{}; KBlockInfoManager m_block_info_manager{}; @@ -288,8 +288,8 @@ namespace ams::kern { KThread *GetExceptionThread() const { return m_exception_thread; } - void AddCpuTime(s64 diff) { m_cpu_time += diff; } - s64 GetCpuTime() { return m_cpu_time; } + void AddCpuTime(s64 diff) { m_cpu_time.FetchAdd(diff); } + s64 GetCpuTime() { return m_cpu_time.Load(); } constexpr s64 GetScheduledCount() const { return m_schedule_count; } void IncrementScheduledCount() { ++m_schedule_count; } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp index 3d953c532..208d6df13 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp @@ -39,14 +39,16 @@ namespace ams::kern { static_assert(ams::svc::HighestThreadPriority <= HighestCoreMigrationAllowedPriority); struct SchedulingState { - std::atomic needs_scheduling; - bool interrupt_task_runnable; - bool should_count_idle; - u64 idle_count; - KThread *highest_priority_thread; - void *idle_thread_stack; - KThread *prev_thread; - KInterruptTaskManager *interrupt_task_manager; + util::Atomic needs_scheduling{false}; + bool interrupt_task_runnable{false}; + bool should_count_idle{false}; + u64 idle_count{0}; + KThread *highest_priority_thread{nullptr}; + void *idle_thread_stack{nullptr}; + util::Atomic prev_thread{nullptr}; + KInterruptTaskManager *interrupt_task_manager{nullptr}; + + constexpr SchedulingState() = default; }; private: friend class KScopedSchedulerLock; @@ -58,7 +60,7 @@ namespace ams::kern { s32 m_core_id; s64 m_last_context_switch_time; KThread *m_idle_thread; - std::atomic m_current_thread; + util::Atomic m_current_thread; public: constexpr KScheduler() : m_state(), m_is_active(false), m_core_id(0), m_last_context_switch_time(0), m_idle_thread(nullptr), m_current_thread(nullptr) @@ -98,11 +100,11 @@ namespace ams::kern { } ALWAYS_INLINE KThread *GetPreviousThread() const { - return m_state.prev_thread; + return m_state.prev_thread.Load(); } ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const { - return m_current_thread; + return m_current_thread.Load(); } ALWAYS_INLINE s64 GetLastContextSwitchTime() const { @@ -182,7 +184,7 @@ namespace ams::kern { GetCurrentThread().EnableDispatch(); - if (m_state.needs_scheduling.load()) { + if (m_state.needs_scheduling.Load()) { /* Disable interrupts, and then check again if rescheduling is needed. */ KScopedInterruptDisable intr_disable; @@ -192,7 +194,7 @@ namespace ams::kern { ALWAYS_INLINE void RescheduleCurrentCoreImpl() { /* Check that scheduling is needed. */ - if (AMS_LIKELY(m_state.needs_scheduling.load())) { + if (AMS_LIKELY(m_state.needs_scheduling.Load())) { GetCurrentThread().DisableDispatch(); this->Schedule(); GetCurrentThread().EnableDispatch(); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_session.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_session.hpp index a1bc97def..a149e0a2d 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_session.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_session.hpp @@ -35,7 +35,7 @@ namespace ams::kern { ServerClosed = 3, }; private: - std::atomic::type> m_atomic_state; + util::Atomic::type> m_atomic_state; bool m_initialized; KServerSession m_server; KClientSession m_client; @@ -48,7 +48,7 @@ namespace ams::kern { } ALWAYS_INLINE State GetState() const { - return static_cast(m_atomic_state.load()); + return static_cast(m_atomic_state.Load()); } public: constexpr KSession() diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp index dda3ce48b..c63a5691f 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_slab_heap.hpp @@ -76,21 +76,18 @@ namespace ams::kern { NON_MOVEABLE(KSlabHeapBase); private: size_t m_obj_size{}; - uintptr_t m_peak{}; + util::Atomic m_peak{0}; uintptr_t m_start{}; uintptr_t m_end{}; private: ALWAYS_INLINE void UpdatePeakImpl(uintptr_t obj) { - static_assert(std::atomic_ref::is_always_lock_free); - std::atomic_ref peak_ref(m_peak); - const uintptr_t alloc_peak = obj + this->GetObjectSize(); - uintptr_t cur_peak = m_peak; + uintptr_t cur_peak = m_peak.Load(); do { if (alloc_peak <= cur_peak) { break; } - } while (!peak_ref.compare_exchange_strong(cur_peak, alloc_peak)); + } while (!m_peak.CompareExchangeStrong(cur_peak, alloc_peak)); } public: constexpr KSlabHeapBase() = default; @@ -112,8 +109,9 @@ namespace ams::kern { /* Set our tracking variables. */ const size_t num_obj = (memory_size / obj_size); m_start = reinterpret_cast(memory); - m_end = m_start + num_obj * obj_size; - m_peak = m_start; + m_end = m_start + num_obj * obj_size; + + m_peak.Store(m_start); /* Free the objects. */ u8 *cur = reinterpret_cast(m_end); @@ -177,7 +175,7 @@ namespace ams::kern { } ALWAYS_INLINE size_t GetPeakIndex() const { - return this->GetObjectIndex(reinterpret_cast(m_peak)); + return this->GetObjectIndex(reinterpret_cast(m_peak.Load())); } ALWAYS_INLINE uintptr_t GetSlabHeapAddress() const { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp index cb80cb211..6f1c880a2 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp @@ -37,7 +37,7 @@ namespace ams::kern { virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); } void NotifyAvailable(Result result); - void NotifyAvailable() { + ALWAYS_INLINE void NotifyAvailable() { return this->NotifyAvailable(ResultSuccess()); } public: diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index 99d1d1b4b..23fe71058 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -89,7 +89,7 @@ namespace ams::kern { KThreadContext *context; KThread *cur_thread; s16 disable_count; - std::atomic dpc_flags; + util::Atomic dpc_flags; u8 current_svc_id; bool is_calling_svc; bool is_in_exception_handler; @@ -177,7 +177,7 @@ namespace ams::kern { static_assert(ams::util::HasRedBlackKeyType); static_assert(std::same_as, ConditionVariableComparator::RedBlackKeyType>); private: - static constinit inline std::atomic s_next_thread_id = 0; + static constinit inline util::Atomic s_next_thread_id{0}; private: util::IntrusiveListNode m_process_list_node{}; util::IntrusiveRedBlackTreeNode m_condvar_arbiter_tree_node{}; @@ -192,7 +192,7 @@ namespace ams::kern { u64 m_virtual_affinity_mask{}; KAffinityMask m_physical_affinity_mask{}; u64 m_thread_id{}; - std::atomic m_cpu_time{}; + util::Atomic m_cpu_time{0}; KProcessAddress m_address_key{}; KProcess *m_parent{}; void *m_kernel_stack_top{}; @@ -227,7 +227,7 @@ namespace ams::kern { s32 m_original_physical_ideal_core_id{}; s32 m_num_core_migration_disables{}; ThreadState m_thread_state{}; - std::atomic m_termination_requested{}; + util::Atomic m_termination_requested{false}; bool m_wait_cancelled{}; bool m_cancellable{}; bool m_signaled{}; @@ -348,15 +348,15 @@ namespace ams::kern { #endif ALWAYS_INLINE void RegisterDpc(DpcFlag flag) { - this->GetStackParameters().dpc_flags.fetch_or(flag); + this->GetStackParameters().dpc_flags.FetchOr(flag); } ALWAYS_INLINE void ClearDpc(DpcFlag flag) { - this->GetStackParameters().dpc_flags.fetch_and(~flag); + this->GetStackParameters().dpc_flags.FetchAnd(~flag); } ALWAYS_INLINE u8 GetDpc() const { - return this->GetStackParameters().dpc_flags.load(); + return this->GetStackParameters().dpc_flags.Load(); } ALWAYS_INLINE bool HasDpc() const { @@ -516,7 +516,7 @@ namespace ams::kern { m_closed_object = object; /* Schedule destruction DPC. */ - if ((this->GetStackParameters().dpc_flags.load(std::memory_order_relaxed) & DpcFlag_PerformDestruction) == 0) { + if ((this->GetStackParameters().dpc_flags.Load() & DpcFlag_PerformDestruction) == 0) { this->RegisterDpc(DpcFlag_PerformDestruction); } } @@ -544,12 +544,12 @@ namespace ams::kern { constexpr bool IsAttachedToDebugger() const { return m_debug_attached; } void AddCpuTime(s32 core_id, s64 amount) { - m_cpu_time += amount; + m_cpu_time.FetchAdd(amount); /* TODO: Debug kernels track per-core tick counts. Should we? */ MESOSPHERE_UNUSED(core_id); } - s64 GetCpuTime() const { return m_cpu_time.load(); } + s64 GetCpuTime() const { return m_cpu_time.Load(); } s64 GetCpuTime(s32 core_id) const { MESOSPHERE_ABORT_UNLESS(0 <= core_id && core_id < static_cast(cpu::NumCores)); @@ -591,7 +591,7 @@ namespace ams::kern { ALWAYS_INLINE void *GetKernelStackTop() const { return m_kernel_stack_top; } ALWAYS_INLINE bool IsTerminationRequested() const { - return m_termination_requested.load() || this->GetRawState() == ThreadState_Terminated; + return m_termination_requested.Load() || this->GetRawState() == ThreadState_Terminated; } size_t GetKernelStackUsage() const; diff --git a/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp b/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp index 9ed79aee3..bfcad7a6b 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_cpu.cpp @@ -96,7 +96,7 @@ namespace ams::kern::arch::arm64::cpu { KLightLock m_lock; KLightLock m_cv_lock; KLightConditionVariable m_cv; - std::atomic m_target_cores; + util::Atomic m_target_cores; volatile Operation m_operation; private: static void ThreadFunction(uintptr_t _this) { @@ -109,7 +109,7 @@ namespace ams::kern::arch::arm64::cpu { /* Wait for a request to come in. */ { KScopedLightLock lk(m_cv_lock); - while ((m_target_cores.load() & (1ul << core_id)) == 0) { + while ((m_target_cores.Load() & (1ul << core_id)) == 0) { m_cv.Wait(std::addressof(m_cv_lock)); } } @@ -120,7 +120,7 @@ namespace ams::kern::arch::arm64::cpu { /* Broadcast, if there's nothing pending. */ { KScopedLightLock lk(m_cv_lock); - if (m_target_cores.load() == 0) { + if (m_target_cores.Load() == 0) { m_cv.Broadcast(); } } @@ -129,7 +129,7 @@ namespace ams::kern::arch::arm64::cpu { void ProcessOperation(); public: - constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(), m_target_cores(), m_operation(Operation::Idle) { /* ... */ } + constexpr KCacheHelperInterruptHandler() : KInterruptHandler(), m_lock(), m_cv_lock(), m_cv(), m_target_cores(0), m_operation(Operation::Idle) { /* ... */ } void Initialize(s32 core_id) { /* Reserve a thread from the system limit. */ @@ -163,7 +163,7 @@ namespace ams::kern::arch::arm64::cpu { if ((op == Operation::InstructionMemoryBarrier) || (Kernel::GetState() == Kernel::State::Initializing)) { /* Check that there's no on-going operation. */ MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle); - MESOSPHERE_ABORT_UNLESS(m_target_cores.load() == 0); + MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0); /* Set operation. */ m_operation = op; @@ -171,13 +171,13 @@ namespace ams::kern::arch::arm64::cpu { /* For certain operations, we want to send an interrupt. */ m_target_cores = other_cores_mask; - const u64 target_mask = m_target_cores.load(); + const u64 target_mask = m_target_cores.Load(); DataSynchronizationBarrier(); Kernel::GetInterruptManager().SendInterProcessorInterrupt(KInterruptName_CacheOperation, target_mask); this->ProcessOperation(); - while (m_target_cores.load() != 0) { + while (m_target_cores.Load() != 0) { cpu::Yield(); } @@ -189,7 +189,7 @@ namespace ams::kern::arch::arm64::cpu { /* Check that there's no on-going operation. */ MESOSPHERE_ABORT_UNLESS(m_operation == Operation::Idle); - MESOSPHERE_ABORT_UNLESS(m_target_cores.load() == 0); + MESOSPHERE_ABORT_UNLESS(m_target_cores.Load() == 0); /* Set operation. */ m_operation = op; @@ -199,7 +199,7 @@ namespace ams::kern::arch::arm64::cpu { /* Use the condvar. */ m_cv.Broadcast(); - while (m_target_cores.load() != 0) { + while (m_target_cores.Load() != 0) { m_cv.Wait(std::addressof(m_cv_lock)); } @@ -287,7 +287,7 @@ namespace ams::kern::arch::arm64::cpu { break; } - m_target_cores &= ~(1ul << GetCurrentCoreId()); + m_target_cores.FetchAnd(~(1ul << GetCurrentCoreId())); } ALWAYS_INLINE void SetEventLocally() { diff --git a/libraries/libmesosphere/source/kern_k_auto_object.cpp b/libraries/libmesosphere/source/kern_k_auto_object.cpp index e50616a04..57722a3c8 100644 --- a/libraries/libmesosphere/source/kern_k_auto_object.cpp +++ b/libraries/libmesosphere/source/kern_k_auto_object.cpp @@ -27,35 +27,4 @@ namespace ams::kern { return obj; } - NOINLINE bool KAutoObject::Open() { - MESOSPHERE_ASSERT_THIS(); - - /* Atomically increment the reference count, only if it's positive. */ - u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed); - do { - if (AMS_UNLIKELY(cur_ref_count == 0)) { - MESOSPHERE_AUDIT(cur_ref_count != 0); - return false; - } - MESOSPHERE_ABORT_UNLESS(cur_ref_count < cur_ref_count + 1); - } while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count + 1, std::memory_order_relaxed)); - - return true; - } - - NOINLINE void KAutoObject::Close() { - MESOSPHERE_ASSERT_THIS(); - - /* Atomically decrement the reference count, not allowing it to become negative. */ - u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed); - do { - MESOSPHERE_ABORT_UNLESS(cur_ref_count > 0); - } while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, std::memory_order_relaxed)); - - /* If ref count hits zero, schedule the object for destruction. */ - if (cur_ref_count - 1 == 0) { - this->ScheduleDestruction(); - } - } - } diff --git a/libraries/libmesosphere/source/kern_k_client_port.cpp b/libraries/libmesosphere/source/kern_k_client_port.cpp index 4333bd5a6..9b523a2b4 100644 --- a/libraries/libmesosphere/source/kern_k_client_port.cpp +++ b/libraries/libmesosphere/source/kern_k_client_port.cpp @@ -28,8 +28,7 @@ namespace ams::kern { void KClientPort::OnSessionFinalized() { KScopedSchedulerLock sl; - const auto prev = m_num_sessions--; - if (prev == m_max_sessions) { + if (m_num_sessions.FetchSub(1) == m_max_sessions) { this->NotifyAvailable(); } } @@ -56,7 +55,7 @@ namespace ams::kern { bool KClientPort::IsSignaled() const { MESOSPHERE_ASSERT_THIS(); - return m_num_sessions < m_max_sessions; + return m_num_sessions.Load() < m_max_sessions; } Result KClientPort::CreateSession(KClientSession **out) { @@ -99,7 +98,6 @@ namespace ams::kern { /* Check that we successfully created a session. */ R_UNLESS(session != nullptr, svc::ResultOutOfResource()); - /* Update the session counts. */ auto count_guard = SCOPE_GUARD { session->Close(); }; { @@ -107,22 +105,22 @@ namespace ams::kern { s32 new_sessions; { const auto max = m_max_sessions; - auto cur_sessions = m_num_sessions.load(std::memory_order_acquire); + auto cur_sessions = m_num_sessions.Load(); do { R_UNLESS(cur_sessions < max, svc::ResultOutOfSessions()); new_sessions = cur_sessions + 1; - } while (!m_num_sessions.compare_exchange_weak(cur_sessions, new_sessions, std::memory_order_relaxed)); + } while (!m_num_sessions.CompareExchangeWeak(cur_sessions, new_sessions)); } /* Atomically update the peak session tracking. */ { - auto peak = m_peak_sessions.load(std::memory_order_acquire); + auto peak = m_peak_sessions.Load(); do { if (peak >= new_sessions) { break; } - } while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed)); + } while (!m_peak_sessions.CompareExchangeWeak(peak, new_sessions)); } } count_guard.Cancel(); @@ -183,22 +181,22 @@ namespace ams::kern { s32 new_sessions; { const auto max = m_max_sessions; - auto cur_sessions = m_num_sessions.load(std::memory_order_acquire); + auto cur_sessions = m_num_sessions.Load(); do { R_UNLESS(cur_sessions < max, svc::ResultOutOfSessions()); new_sessions = cur_sessions + 1; - } while (!m_num_sessions.compare_exchange_weak(cur_sessions, new_sessions, std::memory_order_relaxed)); + } while (!m_num_sessions.CompareExchangeWeak(cur_sessions, new_sessions)); } /* Atomically update the peak session tracking. */ { - auto peak = m_peak_sessions.load(std::memory_order_acquire); + auto peak = m_peak_sessions.Load(); do { if (peak >= new_sessions) { break; } - } while (!m_peak_sessions.compare_exchange_weak(peak, new_sessions, std::memory_order_relaxed)); + } while (!m_peak_sessions.CompareExchangeWeak(peak, new_sessions)); } } count_guard.Cancel(); diff --git a/libraries/libmesosphere/source/kern_k_debug_base.cpp b/libraries/libmesosphere/source/kern_k_debug_base.cpp index 3e64d4ea8..225c6a44f 100644 --- a/libraries/libmesosphere/source/kern_k_debug_base.cpp +++ b/libraries/libmesosphere/source/kern_k_debug_base.cpp @@ -25,31 +25,6 @@ namespace ams::kern { } - void KDebugBase::ProcessHolder::Attach(KProcess *process) { - MESOSPHERE_ASSERT(m_process == nullptr); - - /* Set our process. */ - m_process = process; - - /* Open reference to our process. */ - m_process->Open(); - - /* Set our reference count. */ - m_ref_count = 1; - } - - void KDebugBase::ProcessHolder::Detach() { - /* Close our process, if we have one. */ - KProcess * const process = m_process; - if (AMS_LIKELY(process != nullptr)) { - /* Set our process to a debug sentinel value, which will cause crash if accessed. */ - m_process = reinterpret_cast(1); - - /* Close reference to our process. */ - process->Close(); - } - } - void KDebugBase::Initialize() { /* Clear the continue flags. */ m_continue_flags = 0; diff --git a/libraries/libmesosphere/source/kern_k_debug_base_process_holder.cpp b/libraries/libmesosphere/source/kern_k_debug_base_process_holder.cpp deleted file mode 100644 index 42e75180f..000000000 --- a/libraries/libmesosphere/source/kern_k_debug_base_process_holder.cpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#include - -namespace ams::kern { - - NOINLINE bool KDebugBase::ProcessHolder::Open() { - /* Atomically increment the reference count, only if it's positive. */ - u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed); - do { - if (AMS_UNLIKELY(cur_ref_count == 0)) { - MESOSPHERE_AUDIT(cur_ref_count != 0); - return false; - } - MESOSPHERE_ABORT_UNLESS(cur_ref_count < cur_ref_count + 1); - } while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count + 1, std::memory_order_relaxed)); - - return true; - } - - NOINLINE void KDebugBase::ProcessHolder::Close() { - /* Atomically decrement the reference count, not allowing it to become negative. */ - u32 cur_ref_count = m_ref_count.load(std::memory_order_relaxed); - do { - MESOSPHERE_ABORT_UNLESS(cur_ref_count > 0); - } while (!m_ref_count.compare_exchange_weak(cur_ref_count, cur_ref_count - 1, std::memory_order_relaxed)); - - /* If ref count hits zero, schedule the object for destruction. */ - if (cur_ref_count - 1 == 0) { - this->Detach(); - } - } - -} diff --git a/libraries/libmesosphere/source/kern_k_light_lock.cpp b/libraries/libmesosphere/source/kern_k_light_lock.cpp index f1949ce8d..3c488b370 100644 --- a/libraries/libmesosphere/source/kern_k_light_lock.cpp +++ b/libraries/libmesosphere/source/kern_k_light_lock.cpp @@ -40,7 +40,7 @@ namespace ams::kern { KScopedSchedulerLock sl; /* Ensure we actually have locking to do. */ - if (m_tag.load(std::memory_order_relaxed) != _owner) { + if (m_tag.Load() != _owner) { return false; } @@ -68,16 +68,13 @@ namespace ams::kern { KScopedSchedulerLock sl; /* Get the next owner. */ - s32 num_waiters = 0; + s32 num_waiters; KThread *next_owner = owner_thread->RemoveWaiterByKey(std::addressof(num_waiters), reinterpret_cast(std::addressof(m_tag))); /* Pass the lock to the next owner. */ uintptr_t next_tag = 0; if (next_owner != nullptr) { - next_tag = reinterpret_cast(next_owner); - if (num_waiters > 1) { - next_tag |= 0x1; - } + next_tag = reinterpret_cast(next_owner) | static_cast(num_waiters > 1); next_owner->EndWait(ResultSuccess()); @@ -92,7 +89,7 @@ namespace ams::kern { } /* Write the new tag value. */ - m_tag.store(next_tag); + m_tag.Store(next_tag); } } diff --git a/libraries/libmesosphere/source/kern_k_process.cpp b/libraries/libmesosphere/source/kern_k_process.cpp index d774b6ee8..f5253e986 100644 --- a/libraries/libmesosphere/source/kern_k_process.cpp +++ b/libraries/libmesosphere/source/kern_k_process.cpp @@ -25,8 +25,8 @@ namespace ams::kern { constexpr u64 ProcessIdMin = InitialProcessIdMax + 1; constexpr u64 ProcessIdMax = std::numeric_limits::max(); - std::atomic g_initial_process_id = InitialProcessIdMin; - std::atomic g_process_id = ProcessIdMin; + constinit util::Atomic g_initial_process_id{InitialProcessIdMin}; + constinit util::Atomic g_process_id{ProcessIdMin}; Result TerminateChildren(KProcess *process, const KThread *thread_to_not_terminate) { /* Request that all children threads terminate. */ @@ -299,7 +299,7 @@ namespace ams::kern { R_TRY(m_capabilities.Initialize(caps, num_caps, std::addressof(m_page_table))); /* Initialize the process id. */ - m_process_id = g_initial_process_id++; + m_process_id = g_initial_process_id.FetchAdd(1); MESOSPHERE_ABORT_UNLESS(InitialProcessIdMin <= m_process_id); MESOSPHERE_ABORT_UNLESS(m_process_id <= InitialProcessIdMax); @@ -409,7 +409,7 @@ namespace ams::kern { R_TRY(m_capabilities.Initialize(user_caps, num_caps, std::addressof(m_page_table))); /* Initialize the process id. */ - m_process_id = g_process_id++; + m_process_id = g_process_id.FetchAdd(1); MESOSPHERE_ABORT_UNLESS(ProcessIdMin <= m_process_id); MESOSPHERE_ABORT_UNLESS(m_process_id <= ProcessIdMax); @@ -789,15 +789,15 @@ namespace ams::kern { } void KProcess::IncrementRunningThreadCount() { - MESOSPHERE_ASSERT(m_num_running_threads.load() >= 0); + MESOSPHERE_ASSERT(m_num_running_threads.Load() >= 0); - m_num_running_threads.fetch_add(1); + m_num_running_threads.FetchAdd(1); } void KProcess::DecrementRunningThreadCount() { - MESOSPHERE_ASSERT(m_num_running_threads.load() > 0); + MESOSPHERE_ASSERT(m_num_running_threads.Load() > 0); - if (m_num_running_threads.fetch_sub(1) == 1) { + if (m_num_running_threads.FetchSub(1) == 1) { this->Terminate(); } } diff --git a/libraries/libmesosphere/source/kern_k_scheduler.cpp b/libraries/libmesosphere/source/kern_k_scheduler.cpp index 417eb324c..dcba85e38 100644 --- a/libraries/libmesosphere/source/kern_k_scheduler.cpp +++ b/libraries/libmesosphere/source/kern_k_scheduler.cpp @@ -246,9 +246,9 @@ namespace ams::kern { if (cur_process != nullptr) { /* NOTE: Combining this into AMS_LIKELY(!... && ...) triggers an internal compiler error: Segmentation fault in GCC 9.2.0. */ if (AMS_LIKELY(!cur_thread->IsTerminationRequested()) && AMS_LIKELY(cur_thread->GetActiveCore() == m_core_id)) { - m_state.prev_thread = cur_thread; + m_state.prev_thread.Store(cur_thread); } else { - m_state.prev_thread = nullptr; + m_state.prev_thread.Store(nullptr); } } @@ -270,13 +270,9 @@ namespace ams::kern { void KScheduler::ClearPreviousThread(KThread *thread) { MESOSPHERE_ASSERT(IsSchedulerLockedByCurrentThread()); for (size_t i = 0; i < cpu::NumCores; ++i) { - /* Get an atomic reference to the core scheduler's previous thread. */ - std::atomic_ref prev_thread(Kernel::GetScheduler(static_cast(i)).m_state.prev_thread); - static_assert(std::atomic_ref::is_always_lock_free); - /* Atomically clear the previous thread if it's our target. */ KThread *compare = thread; - prev_thread.compare_exchange_strong(compare, nullptr); + Kernel::GetScheduler(static_cast(i)).m_state.prev_thread.CompareExchangeStrong(compare, nullptr); } } diff --git a/libraries/libmesosphere/source/kern_k_thread.cpp b/libraries/libmesosphere/source/kern_k_thread.cpp index 2d88e9a74..e24ba6d53 100644 --- a/libraries/libmesosphere/source/kern_k_thread.cpp +++ b/libraries/libmesosphere/source/kern_k_thread.cpp @@ -219,7 +219,7 @@ namespace ams::kern { this->SetInExceptionHandler(); /* Set thread ID. */ - m_thread_id = s_next_thread_id++; + m_thread_id = s_next_thread_id.FetchAdd(1); /* We initialized! */ m_initialized = true; @@ -707,7 +707,7 @@ namespace ams::kern { KScopedSchedulerLock sl; /* Determine the priority value to use. */ - const s32 target_priority = m_termination_requested.load() && priority >= TerminatingThreadPriority ? TerminatingThreadPriority : priority; + const s32 target_priority = m_termination_requested.Load() && priority >= TerminatingThreadPriority ? TerminatingThreadPriority : priority; /* Change our base priority. */ if (this->GetStackParameters().is_pinned) { @@ -1183,7 +1183,7 @@ namespace ams::kern { const bool first_request = [&] ALWAYS_INLINE_LAMBDA () -> bool { /* Perform an atomic compare-and-swap from false to true. */ u8 expected = false; - return m_termination_requested.compare_exchange_strong(expected, true); + return m_termination_requested.CompareExchangeStrong(expected, true); }(); /* If this is the first request, start termination procedure. */ diff --git a/libraries/libmesosphere/source/kern_panic.cpp b/libraries/libmesosphere/source/kern_panic.cpp index d580400d4..f15afdfd8 100644 --- a/libraries/libmesosphere/source/kern_panic.cpp +++ b/libraries/libmesosphere/source/kern_panic.cpp @@ -42,15 +42,15 @@ namespace ams::kern { return arr; }(); - std::atomic g_next_ticket = 0; - std::atomic g_current_ticket = 0; + constinit util::Atomic g_next_ticket{0}; + constinit util::Atomic g_current_ticket{0}; - std::array g_core_tickets = NegativeArray; + constinit std::array g_core_tickets = NegativeArray; s32 GetCoreTicket() { const s32 core_id = GetCurrentCoreId(); if (g_core_tickets[core_id] == -1) { - g_core_tickets[core_id] = 2 * g_next_ticket.fetch_add(1); + g_core_tickets[core_id] = 2 * g_next_ticket.FetchAdd(1); } return g_core_tickets[core_id]; } @@ -58,24 +58,21 @@ namespace ams::kern { void WaitCoreTicket() { const s32 expected = GetCoreTicket(); const s32 desired = expected + 1; - s32 compare = g_current_ticket; + s32 compare = g_current_ticket.Load(); do { if (compare == desired) { break; } compare = expected; - } while (!g_current_ticket.compare_exchange_weak(compare, desired)); + } while (!g_current_ticket.CompareExchangeWeak(compare, desired)); } void ReleaseCoreTicket() { const s32 expected = GetCoreTicket() + 1; const s32 desired = expected + 1; - s32 compare = g_current_ticket; - do { - if (compare != expected) { - break; - } - } while (!g_current_ticket.compare_exchange_weak(compare, desired)); + + s32 compare = expected; + g_current_ticket.CompareExchangeStrong(compare, desired); } ALWAYS_INLINE KExceptionContext *GetPanicExceptionContext(int core_id) { diff --git a/libraries/libvapours/include/vapours/util.hpp b/libraries/libvapours/include/vapours/util.hpp index 7acef5d16..7075630d4 100644 --- a/libraries/libvapours/include/vapours/util.hpp +++ b/libraries/libvapours/include/vapours/util.hpp @@ -53,6 +53,8 @@ #include #include +#include + #ifdef ATMOSPHERE_IS_STRATOSPHERE #include #endif diff --git a/libraries/libvapours/include/vapours/util/arch/arm64/util_atomic.hpp b/libraries/libvapours/include/vapours/util/arch/arm64/util_atomic.hpp new file mode 100644 index 000000000..3d8d412d8 --- /dev/null +++ b/libraries/libvapours/include/vapours/util/arch/arm64/util_atomic.hpp @@ -0,0 +1,323 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::util { + + namespace impl { + + template + struct AtomicIntegerStorage; + + template requires (sizeof(T) == sizeof(u8)) + struct AtomicIntegerStorage { + using Type = u8; + }; + + template requires (sizeof(T) == sizeof(u16)) + struct AtomicIntegerStorage { + using Type = u16; + }; + + template requires (sizeof(T) == sizeof(u32)) + struct AtomicIntegerStorage { + using Type = u32; + }; + + template requires (sizeof(T) == sizeof(u64)) + struct AtomicIntegerStorage { + using Type = u64; + }; + + template + concept UsableAtomicType = (sizeof(T) <= sizeof(u64)) && !std::is_const::value && !std::is_volatile::value && (std::is_pointer::value || requires (const T &t) { + std::bit_cast::Type, T>(t); + }); + + template + using AtomicStorage = typename AtomicIntegerStorage::Type; + + static_assert(std::same_as, uintptr_t>); + static_assert(std::same_as, u8>); + static_assert(std::same_as, u8>); + static_assert(std::same_as, u16>); + static_assert(std::same_as, u16>); + static_assert(std::same_as, u32>); + static_assert(std::same_as, u32>); + static_assert(std::same_as, u64>); + static_assert(std::same_as, u64>); + + ALWAYS_INLINE void ClearExclusiveForAtomic() { + __asm__ __volatile__("clrex" ::: "memory"); + } + + #define AMS_UTIL_IMPL_DEFINE_ATOMIC_LOAD_FUNCTION(_FNAME_, _MNEMONIC_) \ + template T _FNAME_ ##ForAtomic(const volatile T *); \ + \ + template<> ALWAYS_INLINE u8 _FNAME_ ##ForAtomic(const volatile u8 *p) { u8 v; __asm__ __volatile__(_MNEMONIC_ "b %w[v], %[p]" : [v]"=r"(v) : [p]"Q"(*p) : "memory"); return v; } \ + template<> ALWAYS_INLINE u16 _FNAME_ ##ForAtomic(const volatile u16 *p) { u16 v; __asm__ __volatile__(_MNEMONIC_ "h %w[v], %[p]" : [v]"=r"(v) : [p]"Q"(*p) : "memory"); return v; } \ + template<> ALWAYS_INLINE u32 _FNAME_ ##ForAtomic(const volatile u32 *p) { u32 v; __asm__ __volatile__(_MNEMONIC_ " %w[v], %[p]" : [v]"=r"(v) : [p]"Q"(*p) : "memory"); return v; } \ + template<> ALWAYS_INLINE u64 _FNAME_ ##ForAtomic(const volatile u64 *p) { u64 v; __asm__ __volatile__(_MNEMONIC_ " %[v], %[p]" : [v]"=r"(v) : [p]"Q"(*p) : "memory"); return v; } + + AMS_UTIL_IMPL_DEFINE_ATOMIC_LOAD_FUNCTION(LoadAcquire, "ldar") + AMS_UTIL_IMPL_DEFINE_ATOMIC_LOAD_FUNCTION(LoadExclusive, "ldxr") + AMS_UTIL_IMPL_DEFINE_ATOMIC_LOAD_FUNCTION(LoadAcquireExclusive, "ldaxr") + + #undef AMS_UTIL_IMPL_DEFINE_ATOMIC_LOAD_FUNCTION + + template void StoreReleaseForAtomic(volatile T *, T); + + template<> ALWAYS_INLINE void StoreReleaseForAtomic(volatile u8 *p, u8 v) { __asm__ __volatile__("stlrb %w[v], %[p]" : : [v]"r"(v), [p]"Q"(*p) : "memory"); } + template<> ALWAYS_INLINE void StoreReleaseForAtomic(volatile u16 *p, u16 v) { __asm__ __volatile__("stlrh %w[v], %[p]" : : [v]"r"(v), [p]"Q"(*p) : "memory"); } + template<> ALWAYS_INLINE void StoreReleaseForAtomic(volatile u32 *p, u32 v) { __asm__ __volatile__("stlr %w[v], %[p]" : : [v]"r"(v), [p]"Q"(*p) : "memory"); } + template<> ALWAYS_INLINE void StoreReleaseForAtomic(volatile u64 *p, u64 v) { __asm__ __volatile__("stlr %[v], %[p]" : : [v]"r"(v), [p]"Q"(*p) : "memory"); } + + #define AMS_UTIL_IMPL_DEFINE_ATOMIC_STORE_EXCLUSIVE_FUNCTION(_FNAME_, _MNEMONIC_) \ + template bool _FNAME_ ##ForAtomic(volatile T *, T); \ + \ + template<> ALWAYS_INLINE bool _FNAME_ ##ForAtomic(volatile u8 *p, u8 v) { int result; __asm__ __volatile__(_MNEMONIC_ "b %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); return result == 0; } \ + template<> ALWAYS_INLINE bool _FNAME_ ##ForAtomic(volatile u16 *p, u16 v) { int result; __asm__ __volatile__(_MNEMONIC_ "h %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); return result == 0; } \ + template<> ALWAYS_INLINE bool _FNAME_ ##ForAtomic(volatile u32 *p, u32 v) { int result; __asm__ __volatile__(_MNEMONIC_ " %w[result], %w[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); return result == 0; } \ + template<> ALWAYS_INLINE bool _FNAME_ ##ForAtomic(volatile u64 *p, u64 v) { int result; __asm__ __volatile__(_MNEMONIC_ " %w[result], %[v], %[p]" : [result]"=&r"(result) : [v]"r"(v), [p]"Q"(*p) : "memory"); return result == 0; } + + AMS_UTIL_IMPL_DEFINE_ATOMIC_STORE_EXCLUSIVE_FUNCTION(StoreExclusive, "stxr") + AMS_UTIL_IMPL_DEFINE_ATOMIC_STORE_EXCLUSIVE_FUNCTION(StoreReleaseExclusive, "stlxr") + + #undef AMS_UTIL_IMPL_DEFINE_ATOMIC_STORE_EXCLUSIVE_FUNCTION + + } + + template + class Atomic { + NON_COPYABLE(Atomic); + NON_MOVEABLE(Atomic); + private: + using StorageType = impl::AtomicStorage; + + static constexpr bool IsIntegral = std::integral; + + static constexpr ALWAYS_INLINE T ConvertToType(StorageType s) { + if constexpr (std::integral) { + return static_cast(s); + } else if constexpr(std::is_pointer::value) { + return reinterpret_cast(s); + } else { + return std::bit_cast(s); + } + } + + static constexpr ALWAYS_INLINE StorageType ConvertToStorage(T arg) { + if constexpr (std::integral) { + return static_cast(arg); + } else if constexpr(std::is_pointer::value) { + if (std::is_constant_evaluated() && arg == nullptr) { + return 0; + } + + return reinterpret_cast(arg); + } else { + return std::bit_cast(arg); + } + } + private: + StorageType m_v; + private: + ALWAYS_INLINE volatile StorageType *GetStoragePointer() { return reinterpret_cast< volatile StorageType *>(std::addressof(m_v)); } + ALWAYS_INLINE const volatile StorageType *GetStoragePointer() const { return reinterpret_cast(std::addressof(m_v)); } + public: + ALWAYS_INLINE explicit Atomic() { /* ... */ } + constexpr ALWAYS_INLINE explicit Atomic(T v) : m_v(ConvertToStorage(v)) { /* ... */ } + + constexpr ALWAYS_INLINE T operator=(T desired) { + if (std::is_constant_evaluated()) { + m_v = ConvertToStorage(desired); + } else { + this->Store(desired); + } + return desired; + } + + template + ALWAYS_INLINE T Load() const { + if constexpr (Order != std::memory_order_relaxed) { + return ConvertToType(impl::LoadAcquireForAtomic(this->GetStoragePointer())); + } else { + return ConvertToType(*this->GetStoragePointer()); + } + } + + template + ALWAYS_INLINE void Store(T arg) { + if constexpr (Order != std::memory_order_relaxed) { + impl::StoreReleaseForAtomic(this->GetStoragePointer(), ConvertToStorage(arg)); + } else { + *this->GetStoragePointer() = ConvertToStorage(arg); + } + } + + template + ALWAYS_INLINE T Exchange(T arg) { + volatile StorageType * const p = this->GetStoragePointer(); + const StorageType s = ConvertToStorage(arg); + + StorageType current; + + if constexpr (Order == std::memory_order_relaxed) { + do { + current = impl::LoadExclusiveForAtomic(p); + } while (AMS_UNLIKELY(!impl::StoreExclusiveForAtomic(p, s))); + } else if constexpr (Order == std::memory_order_consume || Order == std::memory_order_acquire) { + do { + current = impl::LoadAcquireExclusiveForAtomic(p); + } while (AMS_UNLIKELY(!impl::StoreExclusiveForAtomic(p, s))); + } else if constexpr (Order == std::memory_order_release) { + do { + current = impl::LoadExclusiveForAtomic(p); + } while (AMS_UNLIKELY(!impl::StoreReleaseExclusiveForAtomic(p, s))); + } else if constexpr (Order == std::memory_order_acq_rel || Order == std::memory_order_seq_cst) { + do { + current = impl::LoadAcquireExclusiveForAtomic(p); + } while (AMS_UNLIKELY(!impl::StoreReleaseExclusiveForAtomic(p, s))); + } else { + static_assert(Order != Order, "Invalid memory order"); + } + + return current; + } + + template + ALWAYS_INLINE bool CompareExchangeWeak(T &expected, T desired) { + volatile StorageType * const p = this->GetStoragePointer(); + const StorageType e = ConvertToStorage(expected); + const StorageType d = ConvertToStorage(desired); + + if constexpr (Order == std::memory_order_relaxed) { + const StorageType current = impl::LoadExclusiveForAtomic(p); + if (AMS_UNLIKELY(current != e)) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + + return AMS_LIKELY(impl::StoreExclusiveForAtomic(p, d)); + } else if constexpr (Order == std::memory_order_consume || Order == std::memory_order_acquire) { + const StorageType current = impl::LoadAcquireExclusiveForAtomic(p); + if (AMS_UNLIKELY(current != e)) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + + return AMS_LIKELY(impl::StoreExclusiveForAtomic(p, d)); + } else if constexpr (Order == std::memory_order_release) { + const StorageType current = impl::LoadExclusiveForAtomic(p); + if (AMS_UNLIKELY(current != e)) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + + return AMS_LIKELY(impl::StoreReleaseExclusiveForAtomic(p, d)); + } else if constexpr (Order == std::memory_order_acq_rel || Order == std::memory_order_seq_cst) { + const StorageType current = impl::LoadAcquireExclusiveForAtomic(p); + if (AMS_UNLIKELY(current != e)) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + + return AMS_LIKELY(impl::StoreReleaseExclusiveForAtomic(p, d)); + } else { + static_assert(Order != Order, "Invalid memory order"); + } + } + + template + ALWAYS_INLINE bool CompareExchangeStrong(T &expected, T desired) { + volatile StorageType * const p = this->GetStoragePointer(); + const StorageType e = ConvertToStorage(expected); + const StorageType d = ConvertToStorage(desired); + + if constexpr (Order == std::memory_order_relaxed) { + StorageType current; + do { + if (current = impl::LoadExclusiveForAtomic(p); current != e) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + } while (!impl::StoreExclusiveForAtomic(p, d)); + } else if constexpr (Order == std::memory_order_consume || Order == std::memory_order_acquire) { + StorageType current; + do { + if (current = impl::LoadAcquireExclusiveForAtomic(p); current != e) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + } while (!impl::StoreExclusiveForAtomic(p, d)); + } else if constexpr (Order == std::memory_order_release) { + StorageType current; + do { + if (current = impl::LoadExclusiveForAtomic(p); current != e) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + } while (!impl::StoreReleaseExclusiveForAtomic(p, d)); + } else if constexpr (Order == std::memory_order_acq_rel || Order == std::memory_order_seq_cst) { + StorageType current; + do { + if (current = impl::LoadAcquireExclusiveForAtomic(p); current != e) { + impl::ClearExclusiveForAtomic(); + expected = ConvertToType(current); + return false; + } + } while (!impl::StoreReleaseExclusiveForAtomic(p, d)); + } else { + static_assert(Order != Order, "Invalid memory order"); + } + + return true; + } + + #define AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(_OPERATION_, _OPERATOR_) \ + template::type> \ + ALWAYS_INLINE T Fetch ## _OPERATION_(T arg) { \ + static_assert(Enable); \ + volatile StorageType * const p = this->GetStoragePointer(); \ + const StorageType s = ConvertToStorage(arg); \ + \ + StorageType current; \ + do { \ + current = impl::LoadAcquireExclusiveForAtomic(p); \ + } while (AMS_UNLIKELY(!impl::StoreReleaseExclusiveForAtomic(p, current _OPERATOR_ s))); \ + return static_cast(current); \ + } + + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Add, +) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Sub, -) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(And, &) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Or, |) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Xor, ^) + + #undef AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION + }; + + +} \ No newline at end of file diff --git a/libraries/libvapours/include/vapours/util/arch/generic/util_atomic.hpp b/libraries/libvapours/include/vapours/util/arch/generic/util_atomic.hpp new file mode 100644 index 000000000..5c57f0a45 --- /dev/null +++ b/libraries/libvapours/include/vapours/util/arch/generic/util_atomic.hpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::util { + + namespace impl { + + template + struct AtomicIntegerStorage; + + template requires (sizeof(T) == sizeof(u8)) + struct AtomicIntegerStorage { + using Type = u8; + }; + + template requires (sizeof(T) == sizeof(u16)) + struct AtomicIntegerStorage { + using Type = u16; + }; + + template requires (sizeof(T) == sizeof(u32)) + struct AtomicIntegerStorage { + using Type = u32; + }; + + template requires (sizeof(T) == sizeof(u64)) + struct AtomicIntegerStorage { + using Type = u64; + }; + + template + concept UsableAtomicType = (sizeof(T) <= sizeof(u64)) && !std::is_const::value && !std::is_volatile::value && (std::is_pointer::value || requires (const T &t) { + std::bit_cast::Type, T>(t); + }); + + } + + template + class Atomic { + NON_COPYABLE(Atomic); + NON_MOVEABLE(Atomic); + private: + static_assert(std::atomic::is_always_lock_free); + private: + std::atomic m_v; + public: + ALWAYS_INLINE explicit Atomic() { /* ... */ } + constexpr ALWAYS_INLINE explicit Atomic(T v) : m_v(v) { /* ... */ } + + ALWAYS_INLINE T operator=(T desired) { + return (m_v = desired); + } + + template + ALWAYS_INLINE T Load() const { + return m_v.load(Order); + } + + template + ALWAYS_INLINE void Store(T arg) { + return m_v.store(Order); + } + + template + ALWAYS_INLINE T Exchange(T arg) { + return m_v.exchange(arg, Order); + } + + template + ALWAYS_INLINE bool CompareExchangeWeak(T &expected, T desired) { + return m_v.compare_exchange_weak(expected, desired, Order); + } + + template + ALWAYS_INLINE bool CompareExchangeStrong(T &expected, T desired) { + return m_v.compare_exchange_strong(expected, desired, Order); + } + + + #define AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(_OPERATION_, _OPERATION_LOWER_) \ + ALWAYS_INLINE T Fetch ## _OPERATION_(T arg) { \ + return m_v.fetch_##_OPERATION_LOWER_(arg); \ + } + + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Add, add) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Sub, sub) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(And, and) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Or, or) + AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION(Xor, xor) + + #undef AMS_UTIL_IMPL_DEFINE_ATOMIC_FETCH_OPERATE_FUNCTION + }; + + +} \ No newline at end of file diff --git a/libraries/libvapours/include/vapours/util/util_atomic.hpp b/libraries/libvapours/include/vapours/util/util_atomic.hpp new file mode 100644 index 000000000..8b1526b87 --- /dev/null +++ b/libraries/libvapours/include/vapours/util/util_atomic.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +#if defined(ATMOSPHERE_ARCH_ARM64) + + #include + +#else + + #include + +#endif \ No newline at end of file