From 97cba5e8812d0f47b09a2dab418313022e80fe1a Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Mon, 13 Apr 2020 17:07:06 -0700 Subject: [PATCH] os: implement ReadWriteLock --- ...ernal_critical_section_impl.os.horizon.hpp | 10 +- .../include/stratosphere/os/os_rw_lock.hpp | 84 +++-- .../stratosphere/os/os_rw_lock_api.hpp | 40 ++ .../stratosphere/os/os_rw_lock_common.hpp | 24 ++ .../stratosphere/os/os_rw_lock_types.hpp | 70 ++++ .../include/stratosphere/os/os_tick.hpp | 1 + .../source/ams/ams_exosphere_api.cpp | 2 +- .../source/os/impl/os_rw_lock_impl.hpp | 185 ++++++++++ .../os_rw_lock_target_impl.os.horizon.cpp | 349 ++++++++++++++++++ .../os_rw_lock_target_impl.os.horizon.hpp | 52 +++ .../source/os/impl/os_tick_manager.hpp | 4 + .../source/os/impl/os_tick_manager_impl.hpp | 4 + .../impl/os_tick_manager_impl.os.horizon.hpp | 16 + .../libstratosphere/source/os/os_rw_lock.cpp | 102 +++++ .../libstratosphere/source/os/os_tick.cpp | 4 + .../source/util/util_uuid_api.cpp | 6 +- .../arch/arm64/svc_thread_local_region.hpp | 4 +- .../vapours/svc/ipc/svc_message_buffer.hpp | 30 +- .../include/vapours/util/util_bitpack.hpp | 12 +- .../loader/source/ldr_capabilities.cpp | 8 +- 20 files changed, 941 insertions(+), 66 deletions(-) create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_rw_lock_api.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_rw_lock_common.hpp create mode 100644 libraries/libstratosphere/include/stratosphere/os/os_rw_lock_types.hpp create mode 100644 libraries/libstratosphere/source/os/impl/os_rw_lock_impl.hpp create mode 100644 libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.cpp create mode 100644 libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.hpp create mode 100644 libraries/libstratosphere/source/os/os_rw_lock.cpp diff --git a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp index 1b8dabfd2..b4c611cc3 100644 --- a/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp +++ b/libraries/libstratosphere/include/stratosphere/os/impl/os_internal_critical_section_impl.os.horizon.hpp @@ -19,12 +19,18 @@ namespace ams::os::impl { - class ReadWriteLockImpl; + #if defined(ATMOSPHERE_OS_HORIZON) + class ReadWriteLockHorizonImpl; + #endif + class InternalConditionVariableImpl; class InternalCriticalSectionImpl { private: - friend class ReadWriteLockImpl; + #if defined(ATMOSPHERE_OS_HORIZON) + friend class ReadWriteLockHorizonImpl; + #endif + friend class InternalConditionVariableImpl; private: u32 thread_handle; diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_lock.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock.hpp index aa5919c31..c0272c52b 100644 --- a/libraries/libstratosphere/include/stratosphere/os/os_rw_lock.hpp +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock.hpp @@ -15,7 +15,9 @@ */ #pragma once -#include "os_common_types.hpp" +#include +#include +#include namespace ams::os { @@ -23,67 +25,83 @@ namespace ams::os { NON_COPYABLE(ReadWriteLock); NON_MOVEABLE(ReadWriteLock); private: - ::RwLock r; + ReadWriteLockType rw_lock; public: - ReadWriteLock() { - rwlockInit(&this->r); - } + constexpr explicit ReadWriteLock() : rw_lock{{}, 0, ::ams::os::ReadWriteLockType::State_Initialized, nullptr, 0, {}, {}} { /* ... */ } - bool IsWriteLockHeldByCurrentThread() const { - return rwlockIsWriteLockHeldByCurrentThread(const_cast<::RwLock *>(&this->r)); - } - - bool IsLockOwner() const { - return rwlockIsOwnedByCurrentThread(const_cast<::RwLock *>(&this->r)); - } + ~ReadWriteLock() { os::FinalizeReadWriteLock(std::addressof(this->rw_lock)); } void AcquireReadLock() { - rwlockReadLock(&this->r); - } - - void ReleaseReadLock() { - rwlockReadUnlock(&this->r); + return os::AcquireReadLock(std::addressof(this->rw_lock)); } bool TryAcquireReadLock() { - return rwlockTryReadLock(&this->r); + return os::TryAcquireReadLock(std::addressof(this->rw_lock)); + } + + void ReleaseReadLock() { + return os::ReleaseReadLock(std::addressof(this->rw_lock)); } void AcquireWriteLock() { - rwlockWriteLock(&this->r); - } - - void ReleaseWriteLock() { - rwlockWriteUnlock(&this->r); + return os::AcquireWriteLock(std::addressof(this->rw_lock)); } bool TryAcquireWriteLock() { - return rwlockTryWriteLock(&this->r); + return os::TryAcquireWriteLock(std::addressof(this->rw_lock)); + } + + void ReleaseWriteLock() { + return os::ReleaseWriteLock(std::addressof(this->rw_lock)); + } + + bool IsReadLockHeld() const { + return os::IsReadLockHeld(std::addressof(this->rw_lock)); + } + + bool IsWriteLockHeldByCurrentThread() const { + return os::IsWriteLockHeldByCurrentThread(std::addressof(this->rw_lock)); + } + + bool IsLockOwner() const { + return os::IsReadWriteLockOwnerThread(std::addressof(this->rw_lock)); } void lock_shared() { - this->AcquireReadLock(); - } - - void unlock_shared() { - this->ReleaseReadLock(); + return this->AcquireReadLock(); } bool try_lock_shared() { return this->TryAcquireReadLock(); } - void lock() { - this->AcquireWriteLock(); + void unlock_shared() { + return this->ReleaseReadLock(); } - void unlock() { - this->ReleaseWriteLock(); + void lock() { + return this->AcquireWriteLock(); } bool try_lock() { return this->TryAcquireWriteLock(); } + + void unlock() { + return this->ReleaseWriteLock(); + } + + operator ReadWriteLockType &() { + return this->rw_lock; + } + + operator const ReadWriteLockType &() const { + return this->rw_lock; + } + + ReadWriteLockType *GetBase() { + return std::addressof(this->rw_lock); + } }; } \ No newline at end of file diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_api.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_api.hpp new file mode 100644 index 000000000..3129d43fe --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_api.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::os { + + struct ReadWriteLockType; + + void InitalizeReadWriteLock(ReadWriteLockType *rw_lock); + void FinalizeReadWriteLock(ReadWriteLockType *rw_lock); + + void AcquireReadLock(ReadWriteLockType *rw_lock); + bool TryAcquireReadLock(ReadWriteLockType *rw_lock); + void ReleaseReadLock(ReadWriteLockType *rw_lock); + + void AcquireWriteLock(ReadWriteLockType *rw_lock); + bool TryAcquireWriteLock(ReadWriteLockType *rw_lock); + void ReleaseWriteLock(ReadWriteLockType *rw_lock); + + bool IsReadLockHeld(const ReadWriteLockType *rw_lock); + bool IsWriteLockHeldByCurrentThread(const ReadWriteLockType *rw_lock); + bool IsReadWriteLockOwnerThread(const ReadWriteLockType *rw_lock); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_common.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_common.hpp new file mode 100644 index 000000000..748fa237e --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_common.hpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os { + + constexpr inline s32 ReadWriteLockCountMax = (1 << (BITSIZEOF(u16) - 1)) - 1; + constexpr inline s32 ReadWriteLockWaiterCountMax = (1 << BITSIZEOF(u8)) - 1; + +} \ No newline at end of file diff --git a/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_types.hpp b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_types.hpp new file mode 100644 index 000000000..4d6acf17e --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_rw_lock_types.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include +#include + +namespace ams::os { + + struct ThreadType; + + struct ReadWriteLockType { + enum State { + State_NotInitialized = 0, + State_Initialized = 1, + }; + + struct LockCount { + union { + s32 _arr[sizeof(impl::InternalCriticalSectionStorage) / sizeof(s32)]; + impl::InternalCriticalSectionStorage cs_storage; + }; + util::BitPack32 counter; + }; + static_assert(std::is_pod::value); + static_assert(std::is_trivial::value); + + union { + struct { + LockCount c; + u32 write_lock_count; + } aligned; + struct { + u32 write_lock_count; + LockCount c; + } not_aligned; + } lock_count; + + u32 reserved_1; + + u8 state; + ThreadType *owner_thread; + u32 reserved_2; + + union { + s32 _arr[sizeof(impl::InternalConditionVariableStorage) / sizeof(s32)]; + impl::InternalConditionVariableStorage _storage; + } cv_read_lock; + union { + s32 _arr[sizeof(impl::InternalConditionVariableStorage) / sizeof(s32)]; + impl::InternalConditionVariableStorage _storage; + } cv_write_lock; + }; + static_assert(std::is_trivial::value); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_tick.hpp b/libraries/libstratosphere/include/stratosphere/os/os_tick.hpp index 67d42b585..05338e77d 100644 --- a/libraries/libstratosphere/include/stratosphere/os/os_tick.hpp +++ b/libraries/libstratosphere/include/stratosphere/os/os_tick.hpp @@ -22,6 +22,7 @@ namespace ams::os { /* Tick API. */ Tick GetSystemTick(); + Tick GetSystemTickOrdered(); s64 GetSystemTickFrequency(); TimeSpan ConvertToTimeSpan(Tick tick); Tick ConvertToTick(TimeSpan ts); diff --git a/libraries/libstratosphere/source/ams/ams_exosphere_api.cpp b/libraries/libstratosphere/source/ams/ams_exosphere_api.cpp index 18f6a69ae..915166445 100644 --- a/libraries/libstratosphere/source/ams/ams_exosphere_api.cpp +++ b/libraries/libstratosphere/source/ams/ams_exosphere_api.cpp @@ -26,7 +26,7 @@ namespace ams::exosphere { R_ABORT_UNLESS(ResultNotPresent()); } - return ApiInfo{ util::BitPack64(exosphere_cfg) }; + return ApiInfo{ util::BitPack64{exosphere_cfg} }; } void ForceRebootToRcm() { diff --git a/libraries/libstratosphere/source/os/impl/os_rw_lock_impl.hpp b/libraries/libstratosphere/source/os/impl/os_rw_lock_impl.hpp new file mode 100644 index 000000000..9c728871b --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_rw_lock_impl.hpp @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +#if defined(ATMOSPHERE_OS_HORIZON) + #include "os_rw_lock_target_impl.os.horizon.hpp" +#else + #error "Unknown OS for os::ReadWriteLockTargetImpl" +#endif + +namespace ams::os::impl { + + static_assert(alignof(ReadWriteLockType) == sizeof(u64) || alignof(ReadWriteLockType) == sizeof(u32)); + constexpr inline bool IsReadWriteLockGuaranteedAlignment = alignof(ReadWriteLockType) == sizeof(u64); + + struct ReadWriteLockCounter { + using ReadLockCount = util::BitPack32::Field< 0, BITSIZEOF(u16) - 1, u32>; + using WriteLocked = util::BitPack32::Field< ReadLockCount::Next, 1, u32>; + using ReadLockWaiterCount = util::BitPack32::Field< WriteLocked::Next, BITSIZEOF(u8), u32>; + using WriteLockWaiterCount = util::BitPack32::Field; + }; + static_assert(ReadWriteLockCounter::WriteLockWaiterCount::Next == BITSIZEOF(u32)); + + ALWAYS_INLINE void ClearReadLockCount(ReadWriteLockType::LockCount &lc) { + lc.counter.Set(0); + } + + ALWAYS_INLINE void ClearWriteLocked(ReadWriteLockType::LockCount &lc) { + lc.counter.Set(0); + } + + ALWAYS_INLINE void ClearReadLockWaiterCount(ReadWriteLockType::LockCount &lc) { + lc.counter.Set(0); + } + + ALWAYS_INLINE void ClearWriteLockWaiterCount(ReadWriteLockType::LockCount &lc) { + lc.counter.Set(0); + } + + ALWAYS_INLINE void ClearWriteLockCount(ReadWriteLockType *rw_lock) { + if constexpr (IsReadWriteLockGuaranteedAlignment) { + rw_lock->lock_count.aligned.write_lock_count = 0; + } else { + if (reinterpret_cast(std::addressof(rw_lock->lock_count)) & sizeof(u32)) { + rw_lock->lock_count.not_aligned.write_lock_count = 0; + } else { + rw_lock->lock_count.aligned.write_lock_count = 0; + } + } + } + + ALWAYS_INLINE ReadWriteLockType::LockCount &GetLockCount(ReadWriteLockType *rw_lock) { + if constexpr (IsReadWriteLockGuaranteedAlignment) { + return rw_lock->lock_count.aligned.c; + } else { + if (reinterpret_cast(std::addressof(rw_lock->lock_count)) & sizeof(u32)) { + return rw_lock->lock_count.not_aligned.c; + } else { + return rw_lock->lock_count.aligned.c; + } + } + } + + ALWAYS_INLINE const ReadWriteLockType::LockCount &GetLockCount(const ReadWriteLockType *rw_lock) { + if constexpr (IsReadWriteLockGuaranteedAlignment) { + return rw_lock->lock_count.aligned.c; + } else { + if (reinterpret_cast(std::addressof(rw_lock->lock_count)) & sizeof(u32)) { + return rw_lock->lock_count.not_aligned.c; + } else { + return rw_lock->lock_count.aligned.c; + } + } + } + + ALWAYS_INLINE u32 GetReadLockCount(const ReadWriteLockType::LockCount &lc) { + return lc.counter.Get(); + } + + ALWAYS_INLINE u32 GetWriteLocked(const ReadWriteLockType::LockCount &lc) { + return lc.counter.Get(); + } + + ALWAYS_INLINE u32 GetReadLockWaiterCount(const ReadWriteLockType::LockCount &lc) { + return lc.counter.Get(); + } + + ALWAYS_INLINE u32 GetWriteLockWaiterCount(const ReadWriteLockType::LockCount &lc) { + return lc.counter.Get(); + } + + ALWAYS_INLINE u32 &GetWriteLockCount(ReadWriteLockType &rw_lock) { + if constexpr (IsReadWriteLockGuaranteedAlignment) { + return rw_lock.lock_count.aligned.write_lock_count; + } else { + if (reinterpret_cast(std::addressof(rw_lock.lock_count)) & sizeof(u32)) { + return rw_lock.lock_count.not_aligned.write_lock_count; + } else { + return rw_lock.lock_count.aligned.write_lock_count; + } + } + } + + ALWAYS_INLINE const u32 &GetWriteLockCount(const ReadWriteLockType &rw_lock) { + if constexpr (IsReadWriteLockGuaranteedAlignment) { + return rw_lock.lock_count.aligned.write_lock_count; + } else { + if (reinterpret_cast(std::addressof(rw_lock.lock_count)) & sizeof(u32)) { + return rw_lock.lock_count.not_aligned.write_lock_count; + } else { + return rw_lock.lock_count.aligned.write_lock_count; + } + } + } + + ALWAYS_INLINE void IncReadLockCount(ReadWriteLockType::LockCount &lc) { + const u32 read_lock_count = lc.counter.Get(); + AMS_ASSERT(read_lock_count < ReadWriteLockCountMax); + lc.counter.Set(read_lock_count + 1); + } + + ALWAYS_INLINE void DecReadLockCount(ReadWriteLockType::LockCount &lc) { + const u32 read_lock_count = lc.counter.Get(); + AMS_ASSERT(read_lock_count > 0); + lc.counter.Set(read_lock_count - 1); + } + + ALWAYS_INLINE void IncReadLockWaiterCount(ReadWriteLockType::LockCount &lc) { + const u32 read_lock_waiter_count = lc.counter.Get(); + AMS_ASSERT(read_lock_waiter_count < ReadWriteLockWaiterCountMax); + lc.counter.Set(read_lock_waiter_count + 1); + } + + ALWAYS_INLINE void DecReadLockWaiterCount(ReadWriteLockType::LockCount &lc) { + const u32 read_lock_waiter_count = lc.counter.Get(); + AMS_ASSERT(read_lock_waiter_count > 0); + lc.counter.Set(read_lock_waiter_count - 1); + } + + ALWAYS_INLINE void IncWriteLockWaiterCount(ReadWriteLockType::LockCount &lc) { + const u32 write_lock_waiter_count = lc.counter.Get(); + AMS_ASSERT(write_lock_waiter_count < ReadWriteLockWaiterCountMax); + lc.counter.Set(write_lock_waiter_count + 1); + } + + ALWAYS_INLINE void DecWriteLockWaiterCount(ReadWriteLockType::LockCount &lc) { + const u32 write_lock_waiter_count = lc.counter.Get(); + AMS_ASSERT(write_lock_waiter_count > 0); + lc.counter.Set(write_lock_waiter_count - 1); + } + + + ALWAYS_INLINE void IncWriteLockCount(ReadWriteLockType &rw_lock) { + u32 &write_lock_count = GetWriteLockCount(rw_lock); + AMS_ASSERT(write_lock_count < ReadWriteLockCountMax); + ++write_lock_count; + } + + ALWAYS_INLINE void DecWriteLockCount(ReadWriteLockType &rw_lock) { + u32 &write_lock_count = GetWriteLockCount(rw_lock); + AMS_ASSERT(write_lock_count > 0); + --write_lock_count; + } + + ALWAYS_INLINE void SetWriteLocked(ReadWriteLockType::LockCount &lc) { + lc.counter.Set(1); + } + + using ReadWriteLockImpl = ReadWriteLockTargetImpl; + +} diff --git a/libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.cpp b/libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.cpp new file mode 100644 index 000000000..70354347e --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.cpp @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "os_rw_lock_impl.hpp" +#include "os_thread_manager.hpp" + +namespace ams::os::impl { + + #define ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(dst_ref, expected_ref, desired_ref, success, fail) \ + (__atomic_compare_exchange(reinterpret_cast(&dst_ref), reinterpret_cast(&expected_ref), reinterpret_cast(&desired_ref), true, success, fail)) + + + void ReadWriteLockHorizonImpl::AcquireReadLockWriteLocked(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(GetWriteLocked(GetLockCount(rw_lock)) == 1); + AMS_ASSERT((GetThreadHandle(GetLockCount(rw_lock)) | svc::HandleWaitMask) == (impl::GetCurrentThreadHandle() | svc::HandleWaitMask)); + alignas(alignof(u64)) LockCount expected = GetLockCount(rw_lock); + alignas(alignof(u64)) LockCount lock_count; + do { + lock_count = expected; + IncReadLockCount(lock_count); + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + } + + void ReadWriteLockHorizonImpl::ReleaseReadLockWriteLocked(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(GetWriteLocked(GetLockCount(rw_lock)) == 1); + AMS_ASSERT((GetThreadHandle(GetLockCount(rw_lock)) | svc::HandleWaitMask) == (impl::GetCurrentThreadHandle() | svc::HandleWaitMask)); + alignas(alignof(u64)) LockCount expected = GetLockCount(rw_lock); + alignas(alignof(u64)) LockCount lock_count; + do { + lock_count = expected; + DecReadLockCount(lock_count); + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + return ReleaseWriteLockImpl(rw_lock); + } + + void ReadWriteLockHorizonImpl::ReleaseWriteLockImpl(os::ReadWriteLockType *rw_lock) { + alignas(alignof(u64)) LockCount expected = GetLockCount(rw_lock); + alignas(alignof(u64)) LockCount lock_count = expected; + AMS_ASSERT(GetWriteLocked(lock_count) == 1); + if (GetReadLockCount(lock_count) == 0 && GetWriteLockCount(*rw_lock) == 0) { + rw_lock->owner_thread = nullptr; + + if (GetWriteLockWaiterCount(lock_count) > 0) { + GetReference(rw_lock->cv_write_lock._storage).Signal(); + } else if (GetReadLockWaiterCount(lock_count) > 0) { + GetReference(rw_lock->cv_read_lock._storage).Broadcast(); + } + + do { + ClearWriteLocked(lock_count); + SetThreadHandle(lock_count, (GetThreadHandle(lock_count) & svc::HandleWaitMask) ? GetThreadHandle(lock_count) : svc::InvalidHandle); + } while(!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELEASE, __ATOMIC_RELAXED)); + + if (GetThreadHandle(lock_count)) { + R_ABORT_UNLESS(svc::ArbitrateUnlock(GetThreadHandleAddress(GetLockCount(rw_lock)))); + } + } + } + + void ReadWriteLockHorizonImpl::AcquireReadLock(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(::ams::svc::GetThreadLocalRegion()->disable_count == 0); + + auto *cur_thread = impl::GetCurrentThread(); + if (rw_lock->owner_thread == cur_thread) { + return AcquireReadLockWriteLocked(rw_lock); + } else { + const u32 cur_handle = cur_thread->thread_impl->handle; + bool arbitrated = false; + bool got_read_lock, needs_arbitrate_lock; + + alignas(alignof(u64)) LockCount lock_count; + alignas(alignof(u64)) LockCount expected; + while (true) { + expected = GetLockCount(rw_lock); + do { + lock_count = expected; + if (GetWriteLocked(lock_count) != 0 || GetWriteLockWaiterCount(lock_count) != 0) { + if (!arbitrated) { + IncReadLockWaiterCount(lock_count); + } + const u32 h = GetThreadHandle(lock_count); + got_read_lock = false; + needs_arbitrate_lock = h != 0; + SetThreadHandle(lock_count, needs_arbitrate_lock ? (h | svc::HandleWaitMask) : cur_handle); + } else { + if (arbitrated) { + DecReadLockWaiterCount(lock_count); + } + IncReadLockCount(lock_count); + got_read_lock = true; + needs_arbitrate_lock = false; + } + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + + if (!needs_arbitrate_lock) { + break; + } + + R_ABORT_UNLESS(svc::ArbitrateLock(GetThreadHandle(lock_count) & ~svc::HandleWaitMask, GetThreadHandleAddress(GetLockCount(rw_lock)), cur_handle)); + + const u32 new_handle = GetThreadHandle(GetLockCount(rw_lock)); + if ((new_handle | svc::HandleWaitMask) == (cur_handle | svc::HandleWaitMask)) { + SetThreadHandle(lock_count, new_handle); + break; + } + + arbitrated = true; + } + + if (got_read_lock) { + return; + } + + expected = lock_count; + do { + while (GetWriteLockWaiterCount(lock_count)) { + GetReference(rw_lock->cv_read_lock._storage).Wait(GetPointer(GetLockCount(rw_lock).cs_storage)); + expected = GetLockCount(rw_lock); + lock_count = expected; + } + DecReadLockWaiterCount(lock_count); + IncReadLockCount(lock_count); + SetThreadHandle(lock_count, (GetThreadHandle(lock_count) & svc::HandleWaitMask) ? GetThreadHandle(lock_count) : svc::InvalidHandle); + } while(!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELEASE, __ATOMIC_RELAXED)); + if (GetThreadHandle(lock_count)) { + R_ABORT_UNLESS(svc::ArbitrateUnlock(GetThreadHandleAddress(GetLockCount(rw_lock)))); + } + } + } + + bool ReadWriteLockHorizonImpl::TryAcquireReadLock(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(::ams::svc::GetThreadLocalRegion()->disable_count == 0); + + auto *cur_thread = impl::GetCurrentThread(); + if (rw_lock->owner_thread == cur_thread) { + AcquireReadLockWriteLocked(rw_lock); + return true; + } else { + alignas(alignof(u64)) LockCount expected = GetLockCount(rw_lock); + alignas(alignof(u64)) LockCount lock_count; + while (true) { + lock_count = expected; + AMS_ASSERT(GetWriteLocked(lock_count) == 0 || GetThreadHandle(lock_count) != 0); + if (GetWriteLocked(lock_count) == 0 && GetWriteLockWaiterCount(lock_count) == 0) { + IncReadLockCount(lock_count); + if (ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { + return true; + } + } else { + return false; + } + } + } + } + + void ReadWriteLockHorizonImpl::ReleaseReadLock(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(::ams::svc::GetThreadLocalRegion()->disable_count == 0); + + AMS_ASSERT(GetReadLockCount(GetLockCount(rw_lock)) > 0); + auto *cur_thread = impl::GetCurrentThread(); + if (rw_lock->owner_thread == cur_thread) { + return ReleaseReadLockWriteLocked(rw_lock); + } else { + alignas(alignof(u64)) LockCount expected = GetLockCount(rw_lock); + alignas(alignof(u64)) LockCount lock_count; + while (true) { + lock_count = expected; + DecReadLockCount(lock_count); + if (GetReadLockCount(lock_count) == 0 && GetWriteLockWaiterCount(lock_count) != 0) { + break; + } + if (ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) { + return; + } + } + + AMS_ASSERT(GetWriteLocked(lock_count) == 0); + + const u32 cur_handle = cur_thread->thread_impl->handle; + do { + do { + const u32 h = GetThreadHandle(lock_count); + SetThreadHandle(lock_count, h != 0 ? (h | svc::HandleWaitMask) : cur_handle); + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + + if (GetThreadHandle(lock_count) == cur_handle) { + break; + } + + R_ABORT_UNLESS(svc::ArbitrateLock(GetThreadHandle(lock_count) & ~svc::HandleWaitMask, GetThreadHandleAddress(GetLockCount(rw_lock)), cur_handle)); + expected = GetLockCount(rw_lock); + } while ((GetThreadHandle(GetLockCount(rw_lock)) | svc::HandleWaitMask) != (GetThreadHandle(lock_count) | svc::HandleWaitMask)); + + do { + lock_count = expected; + AMS_ASSERT(GetReadLockCount(lock_count) == 1 && GetWriteLockWaiterCount(lock_count) > 0); + DecReadLockCount(lock_count); + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + GetReference(rw_lock->cv_write_lock._storage).Signal(); + + do { + lock_count = expected; + AMS_ASSERT(GetReadLockCount(lock_count) == 0 && GetWriteLockWaiterCount(lock_count) > 0); + SetThreadHandle(lock_count, (GetThreadHandle(lock_count) & svc::HandleWaitMask) ? GetThreadHandle(lock_count) : svc::InvalidHandle); + } while(!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_RELEASE, __ATOMIC_RELAXED)); + + if (GetThreadHandle(lock_count) & svc::HandleWaitMask) { + R_ABORT_UNLESS(svc::ArbitrateUnlock(GetThreadHandleAddress(GetLockCount(rw_lock)))); + } + } + } + + void ReadWriteLockHorizonImpl::AcquireWriteLock(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(::ams::svc::GetThreadLocalRegion()->disable_count == 0); + + auto *cur_thread = impl::GetCurrentThread(); + if (rw_lock->owner_thread == cur_thread) { + AMS_ASSERT(GetWriteLocked(GetLockCount(rw_lock)) == 1); + IncWriteLockCount(*rw_lock); + } else { + const u32 cur_handle = cur_thread->thread_impl->handle; + alignas(alignof(u64)) LockCount expected; + alignas(alignof(u64)) LockCount lock_count; + bool arbitrated = false; + bool got_write_lock, needs_arbitrate_lock; + + while (true) { + expected = GetLockCount(rw_lock); + do { + lock_count = expected; + AMS_ASSERT(GetWriteLocked(lock_count) == 0 || GetThreadHandle(lock_count) != svc::InvalidHandle); + if (GetReadLockCount(lock_count) > 0 || GetThreadHandle(lock_count) != svc::InvalidHandle) { + if (!arbitrated) { + IncWriteLockWaiterCount(lock_count); + } + const u32 h = GetThreadHandle(lock_count); + got_write_lock = false; + needs_arbitrate_lock = h != 0; + SetThreadHandle(lock_count, needs_arbitrate_lock ? (h | svc::HandleWaitMask) : cur_handle); + } else { + if (arbitrated) { + DecWriteLockWaiterCount(lock_count); + } + SetWriteLocked(lock_count); + got_write_lock = true; + needs_arbitrate_lock = false; + } + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + + if (got_write_lock) { + break; + } + + if (needs_arbitrate_lock) { + R_ABORT_UNLESS(svc::ArbitrateLock(GetThreadHandle(lock_count) & ~svc::HandleWaitMask, GetThreadHandleAddress(GetLockCount(rw_lock)), cur_handle)); + arbitrated = true; + expected = GetLockCount(rw_lock); + if ((GetThreadHandle(lock_count) | svc::HandleWaitMask) != (cur_handle | svc::HandleWaitMask)) { + continue; + } + } + + do { + lock_count = expected; + + AMS_ASSERT(GetWriteLocked(lock_count) == 0); + while (GetReadLockCount(lock_count) > 0) { + GetReference(rw_lock->cv_write_lock._storage).Wait(GetPointer(GetLockCount(rw_lock).cs_storage)); + expected = GetLockCount(rw_lock); + lock_count = expected; + if (GetWriteLocked(lock_count) == 1) { + break; + } + } + if (GetWriteLocked(lock_count) == 1) { + continue; + } + + DecWriteLockWaiterCount(lock_count); + SetWriteLocked(lock_count); + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + + break; + } + + AMS_ASSERT(GetWriteLockCount(*rw_lock) == 0); + IncWriteLockCount(*rw_lock); + rw_lock->owner_thread = cur_thread; + } + } + + bool ReadWriteLockHorizonImpl::TryAcquireWriteLock(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(::ams::svc::GetThreadLocalRegion()->disable_count == 0); + + auto *cur_thread = impl::GetCurrentThread(); + if (rw_lock->owner_thread == cur_thread) { + AMS_ASSERT(GetWriteLocked(GetLockCount(rw_lock)) == 1); + IncWriteLockCount(*rw_lock); + return true; + } else { + const u32 cur_handle = cur_thread->thread_impl->handle; + alignas(alignof(u64)) LockCount expected = GetLockCount(rw_lock); + alignas(alignof(u64)) LockCount lock_count; + + do { + lock_count = expected; + AMS_ASSERT(GetWriteLocked(lock_count) == 0 || GetThreadHandle(lock_count) != svc::InvalidHandle); + + if (GetReadLockCount(lock_count) > 0 || GetThreadHandle(lock_count) != svc::InvalidHandle) { + return false; + } + + SetWriteLocked(lock_count); + SetThreadHandle(lock_count, cur_handle); + } while (!ATOMIC_COMPARE_EXCHANGE_LOCK_COUNT(GetLockCount(rw_lock), expected, lock_count, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)); + + + AMS_ASSERT(GetWriteLockCount(*rw_lock) == 0); + IncWriteLockCount(*rw_lock); + rw_lock->owner_thread = cur_thread; + return true; + } + } + + void ReadWriteLockHorizonImpl::ReleaseWriteLock(os::ReadWriteLockType *rw_lock) { + AMS_ASSERT(::ams::svc::GetThreadLocalRegion()->disable_count == 0); + + AMS_ASSERT(GetWriteLockCount(*rw_lock) > 0); + AMS_ASSERT(GetWriteLocked(GetLockCount(rw_lock)) != 0); + + DecWriteLockCount(*rw_lock); + return ReleaseWriteLockImpl(rw_lock); + } + +} diff --git a/libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.hpp b/libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.hpp new file mode 100644 index 000000000..457cf7c2b --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_rw_lock_target_impl.os.horizon.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os::impl { + + class ReadWriteLockHorizonImpl { + private: + using LockCount = os::ReadWriteLockType::LockCount; + private: + static ALWAYS_INLINE u32 GetThreadHandle(const LockCount &lc) { + return GetReference(lc.cs_storage).Get()->thread_handle; + } + + static ALWAYS_INLINE uintptr_t GetThreadHandleAddress(LockCount &lc) { + return reinterpret_cast(std::addressof(GetReference(lc.cs_storage).Get()->thread_handle)); + } + + static ALWAYS_INLINE void SetThreadHandle(LockCount &lc, u32 handle) { + GetReference(lc.cs_storage).Get()->thread_handle = handle; + } + + static void AcquireReadLockWriteLocked(os::ReadWriteLockType *rw_lock); + static void ReleaseReadLockWriteLocked(os::ReadWriteLockType *rw_lock); + static void ReleaseWriteLockImpl(os::ReadWriteLockType *rw_lock); + public: + static void AcquireReadLock(os::ReadWriteLockType *rw_lock); + static bool TryAcquireReadLock(os::ReadWriteLockType *rw_lock); + static void ReleaseReadLock(os::ReadWriteLockType *rw_lock); + + static void AcquireWriteLock(os::ReadWriteLockType *rw_lock); + static bool TryAcquireWriteLock(os::ReadWriteLockType *rw_lock); + static void ReleaseWriteLock(os::ReadWriteLockType *rw_lock); + }; + + using ReadWriteLockTargetImpl = ReadWriteLockHorizonImpl; + +} diff --git a/libraries/libstratosphere/source/os/impl/os_tick_manager.hpp b/libraries/libstratosphere/source/os/impl/os_tick_manager.hpp index 69a8a5ee7..302a394d1 100644 --- a/libraries/libstratosphere/source/os/impl/os_tick_manager.hpp +++ b/libraries/libstratosphere/source/os/impl/os_tick_manager.hpp @@ -27,4 +27,8 @@ namespace ams::os::impl { return GetTickManager().GetTick(); } + ALWAYS_INLINE Tick GetCurrentTickOrdered() { + return GetTickManager().GetSystemTickOrdered(); + } + } diff --git a/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.hpp b/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.hpp index eda9cb7cc..68996b6ac 100644 --- a/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.hpp +++ b/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.hpp @@ -37,6 +37,10 @@ namespace ams::os::impl { return this->impl.GetTick(); } + ALWAYS_INLINE Tick GetSystemTickOrdered() const { + return this->impl.GetSystemTickOrdered(); + } + ALWAYS_INLINE s64 GetTickFrequency() const { return this->impl.GetTickFrequency(); } diff --git a/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.os.horizon.hpp b/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.os.horizon.hpp index 3ab32219b..7e965b7b7 100644 --- a/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.os.horizon.hpp +++ b/libraries/libstratosphere/source/os/impl/os_tick_manager_impl.os.horizon.hpp @@ -32,6 +32,22 @@ namespace ams::os::impl { return Tick(tick); } + ALWAYS_INLINE Tick GetSystemTickOrdered() const { + s64 tick; + #if defined(ATMOSPHERE_ARCH_ARM64) + __asm__ __volatile__("dsb ish\n" + "isb\n" + "mrs %[tick], cntpct_el0\n" + "isb" + : [tick]"=&r"(tick) + : + : "memory"); + #else + #error "Unknown Architecture for TickManagerImpl::GetSystemTickOrdered" + #endif + return Tick(tick); + } + static constexpr ALWAYS_INLINE s64 GetTickFrequency() { return static_cast(::ams::svc::TicksPerSecond); } diff --git a/libraries/libstratosphere/source/os/os_rw_lock.cpp b/libraries/libstratosphere/source/os/os_rw_lock.cpp new file mode 100644 index 000000000..fc2738c96 --- /dev/null +++ b/libraries/libstratosphere/source/os/os_rw_lock.cpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/os_thread_manager.hpp" +#include "impl/os_rw_lock_impl.hpp" + +namespace ams::os { + + void InitalizeReadWriteLock(ReadWriteLockType *rw_lock) { + /* Create objects. */ + new (GetPointer(impl::GetLockCount(rw_lock).cs_storage)) impl::InternalCriticalSection; + new (GetPointer(rw_lock->cv_read_lock._storage)) impl::InternalConditionVariable; + new (GetPointer(rw_lock->cv_write_lock._storage)) impl::InternalConditionVariable; + + /* Set member variables. */ + impl::ClearReadLockCount(impl::GetLockCount(rw_lock)); + impl::ClearWriteLocked(impl::GetLockCount(rw_lock)); + impl::ClearReadLockWaiterCount(impl::GetLockCount(rw_lock)); + impl::ClearWriteLockWaiterCount(impl::GetLockCount(rw_lock)); + impl::ClearWriteLockCount(rw_lock); + rw_lock->owner_thread = nullptr; + + /* Mark initialized. */ + rw_lock->state = ReadWriteLockType::State_Initialized; + } + + void FinalizeReadWriteLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + + /* Don't allow finalizing a locked lock. */ + AMS_ASSERT(impl::GetReadLockCount(impl::GetLockCount(rw_lock)) == 0); + AMS_ASSERT(impl::GetWriteLocked(impl::GetLockCount(rw_lock)) == 0); + + /* Mark not initialized. */ + rw_lock->state = ReadWriteLockType::State_NotInitialized; + + /* Destroy objects. */ + GetReference(rw_lock->cv_write_lock._storage).~InternalConditionVariable(); + GetReference(rw_lock->cv_read_lock._storage).~InternalConditionVariable(); + GetReference(impl::GetLockCount(rw_lock).cs_storage).~InternalCriticalSection(); + } + + void AcquireReadLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return impl::ReadWriteLockImpl::AcquireReadLock(rw_lock); + } + + bool TryAcquireReadLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return impl::ReadWriteLockImpl::TryAcquireReadLock(rw_lock); + } + + void ReleaseReadLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return impl::ReadWriteLockImpl::ReleaseReadLock(rw_lock); + } + + void AcquireWriteLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return impl::ReadWriteLockImpl::AcquireWriteLock(rw_lock); + } + + bool TryAcquireWriteLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return impl::ReadWriteLockImpl::TryAcquireWriteLock(rw_lock); + } + + void ReleaseWriteLock(ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + AMS_ABORT_UNLESS(rw_lock->owner_thread == impl::GetCurrentThread()); + return impl::ReadWriteLockImpl::ReleaseWriteLock(rw_lock); + } + + bool IsReadLockHeld(const ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return impl::GetReadLockCount(impl::GetLockCount(rw_lock)) != 0; + } + + bool IsWriteLockHeldByCurrentThread(const ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return rw_lock->owner_thread == impl::GetCurrentThread() && impl::GetWriteLockCount(*rw_lock) != 0; + } + + bool IsReadWriteLockOwnerThread(const ReadWriteLockType *rw_lock) { + AMS_ASSERT(rw_lock->state == ReadWriteLockType::State_Initialized); + return rw_lock->owner_thread == impl::GetCurrentThread(); + } + +} diff --git a/libraries/libstratosphere/source/os/os_tick.cpp b/libraries/libstratosphere/source/os/os_tick.cpp index 71804c35e..2873bffe4 100644 --- a/libraries/libstratosphere/source/os/os_tick.cpp +++ b/libraries/libstratosphere/source/os/os_tick.cpp @@ -22,6 +22,10 @@ namespace ams::os { return impl::GetTickManager().GetTick(); } + Tick GetSystemTickOrdered() { + return impl::GetTickManager().GetSystemTickOrdered(); + } + s64 GetSystemTickFrequency() { return impl::GetTickManager().GetTickFrequency(); } diff --git a/libraries/libstratosphere/source/util/util_uuid_api.cpp b/libraries/libstratosphere/source/util/util_uuid_api.cpp index b7f031959..7f2910c7f 100644 --- a/libraries/libstratosphere/source/util/util_uuid_api.cpp +++ b/libraries/libstratosphere/source/util/util_uuid_api.cpp @@ -42,7 +42,7 @@ namespace ams::util { inline Uuid Convert() const { /* Convert the fields from native endian to big endian. */ - util::BitPack32 converted[4] = {util::BitPack32(0), util::BitPack32(0), util::BitPack32(0), util::BitPack32(0)}; + util::BitPack32 converted[4] = {util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}}; converted[0].Set(util::ConvertToBigEndian(this->data[0].Get())); @@ -75,7 +75,7 @@ namespace ams::util { constexpr u8 Reserved = 0x1; /* Generate a random uuid. */ - UuidImpl uuid = {util::BitPack32(0), util::BitPack32(0), util::BitPack32(0), util::BitPack32(0)}; + UuidImpl uuid = {util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}}; os::GenerateRandomBytes(uuid.data, sizeof(uuid.data)); /* Set version and reserved. */ @@ -97,7 +97,7 @@ namespace ams::util { constexpr u8 Reserved = 0x1; /* Generate a uuid from a SHA1 hash. */ - UuidImpl uuid = {util::BitPack32(0), util::BitPack32(0), util::BitPack32(0), util::BitPack32(0)}; + UuidImpl uuid = {util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}}; std::memcpy(uuid.data, sha1_hash, sizeof(uuid.data)); /* Set version and reserved. */ diff --git a/libraries/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp b/libraries/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp index c871ff4c1..b2732e858 100644 --- a/libraries/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp +++ b/libraries/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp @@ -23,8 +23,8 @@ namespace ams::svc::arch::arm64 { struct ThreadLocalRegion { u32 message_buffer[MessageBufferSize / sizeof(u32)]; - u16 disable_count; - u16 preemption_state; + volatile u16 disable_count; + volatile u16 interrupt_flag; /* TODO: How should we handle libnx vs Nintendo user thread local space? */ uintptr_t TODO[(0x200 - 0x108) / sizeof(uintptr_t)]; }; diff --git a/libraries/libvapours/include/vapours/svc/ipc/svc_message_buffer.hpp b/libraries/libvapours/include/vapours/svc/ipc/svc_message_buffer.hpp index 6afa9d97f..9a979e874 100644 --- a/libraries/libvapours/include/vapours/svc/ipc/svc_message_buffer.hpp +++ b/libraries/libvapours/include/vapours/svc/ipc/svc_message_buffer.hpp @@ -58,11 +58,11 @@ namespace ams::svc::ipc { private: util::BitPack32 header[2]; public: - constexpr ALWAYS_INLINE MessageHeader() : header({util::BitPack32(0), util::BitPack32(0)}) { + constexpr ALWAYS_INLINE MessageHeader() : header({util::BitPack32{0}, util::BitPack32{0}}) { this->header[0].Set(NullTag); } - constexpr ALWAYS_INLINE MessageHeader(u16 tag, bool special, s32 ptr, s32 send, s32 recv, s32 exch, s32 raw, s32 recv_list) : header({util::BitPack32(0), util::BitPack32(0)}) { + constexpr ALWAYS_INLINE MessageHeader(u16 tag, bool special, s32 ptr, s32 send, s32 recv, s32 exch, s32 raw, s32 recv_list) : header({util::BitPack32{0}, util::BitPack32{0}}) { this->header[0].Set(tag); this->header[0].Set(ptr); this->header[0].Set(send); @@ -74,11 +74,11 @@ namespace ams::svc::ipc { this->header[1].Set(special); } - ALWAYS_INLINE explicit MessageHeader(const MessageBuffer &buf) : header({util::BitPack32(0), util::BitPack32(0)}) { + ALWAYS_INLINE explicit MessageHeader(const MessageBuffer &buf) : header({util::BitPack32{0}, util::BitPack32{0}}) { buf.Get(0, this->header, util::size(this->header)); } - ALWAYS_INLINE explicit MessageHeader(const u32 *msg) : header({util::BitPack32(msg[0]), util::BitPack32(msg[1])}) { /* ... */ } + ALWAYS_INLINE explicit MessageHeader(const u32 *msg) : header({util::BitPack32{msg[0]}, util::BitPack32{msg[1]}}) { /* ... */ } constexpr ALWAYS_INLINE u16 GetTag() const { return this->header[0].Get(); @@ -143,13 +143,13 @@ namespace ams::svc::ipc { util::BitPack32 header; bool has_header; public: - constexpr ALWAYS_INLINE explicit SpecialHeader(bool pid, s32 copy, s32 move) : header(0), has_header(true) { + constexpr ALWAYS_INLINE explicit SpecialHeader(bool pid, s32 copy, s32 move) : header{0}, has_header(true) { this->header.Set(pid); this->header.Set(copy); this->header.Set(move); } - ALWAYS_INLINE explicit SpecialHeader(const MessageBuffer &buf, const MessageHeader &hdr) : header(0), has_header(hdr.GetHasSpecialHeader()) { + ALWAYS_INLINE explicit SpecialHeader(const MessageBuffer &buf, const MessageHeader &hdr) : header{0}, has_header(hdr.GetHasSpecialHeader()) { if (this->has_header) { buf.Get(MessageHeader::GetDataSize() / sizeof(util::BitPack32), std::addressof(this->header), sizeof(this->header) / sizeof(util::BitPack32)); } @@ -219,9 +219,9 @@ namespace ams::svc::ipc { private: util::BitPack32 data[3]; public: - constexpr ALWAYS_INLINE MapAliasDescriptor() : data({util::BitPack32(0), util::BitPack32(0), util::BitPack32(0)}) { /* ... */ } + constexpr ALWAYS_INLINE MapAliasDescriptor() : data({util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}}) { /* ... */ } - ALWAYS_INLINE MapAliasDescriptor(const void *buffer, size_t _size, Attribute attr = Attribute_Ipc) : data({util::BitPack32(0), util::BitPack32(0), util::BitPack32(0)}) { + ALWAYS_INLINE MapAliasDescriptor(const void *buffer, size_t _size, Attribute attr = Attribute_Ipc) : data({util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}}) { const u64 address = reinterpret_cast(buffer); const u64 size = static_cast(_size); this->data[0] = { static_cast(size) }; @@ -233,7 +233,7 @@ namespace ams::svc::ipc { this->data[2].Set(GetAddressHigh(address)); } - ALWAYS_INLINE MapAliasDescriptor(const MessageBuffer &buf, s32 index) : data({util::BitPack32(0), util::BitPack32(0), util::BitPack32(0)}) { + ALWAYS_INLINE MapAliasDescriptor(const MessageBuffer &buf, s32 index) : data({util::BitPack32{0}, util::BitPack32{0}, util::BitPack32{0}}) { buf.Get(index, this->data, util::size(this->data)); } @@ -283,9 +283,9 @@ namespace ams::svc::ipc { private: util::BitPack32 data[2]; public: - constexpr ALWAYS_INLINE PointerDescriptor() : data({util::BitPack32(0), util::BitPack32(0)}) { /* ... */ } + constexpr ALWAYS_INLINE PointerDescriptor() : data({util::BitPack32{0}, util::BitPack32{0}}) { /* ... */ } - ALWAYS_INLINE PointerDescriptor(const void *buffer, size_t size, s32 index) : data({util::BitPack32(0), util::BitPack32(0)}) { + ALWAYS_INLINE PointerDescriptor(const void *buffer, size_t size, s32 index) : data({util::BitPack32{0}, util::BitPack32{0}}) { const u64 address = reinterpret_cast(buffer); this->data[0].Set(index); @@ -296,7 +296,7 @@ namespace ams::svc::ipc { this->data[1] = { static_cast(address) }; } - ALWAYS_INLINE PointerDescriptor(const MessageBuffer &buf, s32 index) : data({util::BitPack32(0), util::BitPack32(0)}) { + ALWAYS_INLINE PointerDescriptor(const MessageBuffer &buf, s32 index) : data({util::BitPack32{0}, util::BitPack32{0}}) { buf.Get(index, this->data, util::size(this->data)); } @@ -338,9 +338,9 @@ namespace ams::svc::ipc { private: util::BitPack32 data[2]; public: - constexpr ALWAYS_INLINE ReceiveListEntry() : data({util::BitPack32(0), util::BitPack32(0)}) { /* ... */ } + constexpr ALWAYS_INLINE ReceiveListEntry() : data({util::BitPack32{0}, util::BitPack32{0}}) { /* ... */ } - ALWAYS_INLINE ReceiveListEntry(const void *buffer, size_t size) : data({util::BitPack32(0), util::BitPack32(0)}) { + ALWAYS_INLINE ReceiveListEntry(const void *buffer, size_t size) : data({util::BitPack32{0}, util::BitPack32{0}}) { const u64 address = reinterpret_cast(buffer); this->data[0] = { static_cast(address) }; @@ -349,7 +349,7 @@ namespace ams::svc::ipc { this->data[1].Set(size); } - ALWAYS_INLINE ReceiveListEntry(u32 a, u32 b) : data({util::BitPack32(a), util::BitPack32(b)}) { /* ... */ } + ALWAYS_INLINE ReceiveListEntry(u32 a, u32 b) : data({util::BitPack32{a}, util::BitPack32{b}}) { /* ... */ } constexpr ALWAYS_INLINE uintptr_t GetAddress() { const u64 address = (static_cast(this->data[1].Get()) << AddressLow::Count) | this->data[0].Get(); diff --git a/libraries/libvapours/include/vapours/util/util_bitpack.hpp b/libraries/libvapours/include/vapours/util/util_bitpack.hpp index 8c992c3f2..fac3c681a 100644 --- a/libraries/libvapours/include/vapours/util/util_bitpack.hpp +++ b/libraries/libvapours/include/vapours/util/util_bitpack.hpp @@ -23,7 +23,8 @@ namespace ams::util { namespace impl { template - class BitPack { + struct BitPack { + IntegralStorageType value; private: static_assert(std::is_integral::value); static_assert(std::is_unsigned::value); @@ -49,16 +50,13 @@ namespace ams::util { static constexpr size_t Next = Index + Count; using BitPackType = BitPack; + static_assert(std::is_pod::value); static_assert(Mask != 0); static_assert(std::is_integral::value || std::is_enum::value); static_assert(!std::is_same::value || Count == 1); }; - private: - IntegralStorageType value; public: - constexpr ALWAYS_INLINE BitPack(IntegralStorageType v) : value(v) { /* ... */ } - constexpr ALWAYS_INLINE void Clear() { constexpr IntegralStorageType Zero = IntegralStorageType(0); this->value = Zero; @@ -86,6 +84,10 @@ namespace ams::util { using BitPack32 = impl::BitPack; using BitPack64 = impl::BitPack; + static_assert(std::is_pod::value); + static_assert(std::is_pod::value); + static_assert(std::is_pod::value); + static_assert(std::is_pod::value); static_assert(std::is_trivially_destructible::value); static_assert(std::is_trivially_destructible::value); static_assert(std::is_trivially_destructible::value); diff --git a/stratosphere/loader/source/ldr_capabilities.cpp b/stratosphere/loader/source/ldr_capabilities.cpp index 1274c762d..f61064b83 100644 --- a/stratosphere/loader/source/ldr_capabilities.cpp +++ b/stratosphere/loader/source/ldr_capabilities.cpp @@ -42,8 +42,7 @@ namespace ams::ldr::caps { constexpr ALWAYS_INLINE typename name::Type Get##name() const { return this->Get(); } constexpr ALWAYS_INLINE CapabilityId GetCapabilityId(util::BitPack32 cap) { - using RawValue = CapabilityField<0, BITSIZEOF(u32)>; - return static_cast(__builtin_ctz(~cap.Get())); + return static_cast(__builtin_ctz(~cap.value)); } #define CAPABILITY_CLASS_NAME(id) Capability##id @@ -53,7 +52,6 @@ namespace ams::ldr::caps { public: \ static constexpr CapabilityId Id = CapabilityId::id; \ using IdBits = CapabilityField<0, static_cast(Id) + 1>; \ - using RawValue = CapabilityField; \ static constexpr u32 IdBitsValue = (static_cast(1) << static_cast(Id)) - 1; \ private: \ util::BitPack32 value; \ @@ -62,9 +60,9 @@ namespace ams::ldr::caps { constexpr ALWAYS_INLINE typename FieldType::Type Get() const { return this->value.Get(); } \ template \ constexpr ALWAYS_INLINE void Set(typename FieldType::Type fv) { this->value.Set(fv); } \ - constexpr ALWAYS_INLINE u32 GetValue() const { return this->Get(); } \ + constexpr ALWAYS_INLINE u32 GetValue() const { return this->value.value; } \ public: \ - constexpr ALWAYS_INLINE CAPABILITY_CLASS_NAME(id)(util::BitPack32 v) : value(v) { /* ... */ } \ + constexpr ALWAYS_INLINE CAPABILITY_CLASS_NAME(id)(util::BitPack32 v) : value{v} { /* ... */ } \ \ static constexpr CAPABILITY_CLASS_NAME(id) Decode(util::BitPack32 v) { return CAPABILITY_CLASS_NAME(id)(v); } \ \