nold/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere synced 2024-09-19 22:59:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

kern: optimize KHandleTable to use indices instead of pointers

Browse source
This commit is contained in:
Michael Scire 2021-04-07 14:07:10 -07:00 committed by SciresM
parent 4407237f5b
commit b4498734e4
2 changed files with 83 additions and 106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp
View file

@ -53,46 +53,29 @@ namespace ams::kern {
return pack.Get<HandleEncoded>(); return pack.Get<HandleEncoded>();
} }
class Entry { union EntryInfo {
private:
union {
struct { struct {
u16 linear_id; u16 linear_id;
u16 type; u16 type;
} info; } info;
Entry *next_free_entry; s32 next_free_index;
} m_meta;
KAutoObject *m_object;
public:
constexpr Entry() : m_meta(), m_object(nullptr) { /* ... */ }
constexpr ALWAYS_INLINE void SetFree(Entry *next) { constexpr ALWAYS_INLINE u16 GetLinearId() const { return info.linear_id; }
m_object = nullptr; constexpr ALWAYS_INLINE u16 GetType() const { return info.type; }
m_meta.next_free_entry = next; constexpr ALWAYS_INLINE s32 GetNextFreeIndex() const { return next_free_index; }
}
constexpr ALWAYS_INLINE void SetUsed(KAutoObject *obj, u16 linear_id, u16 type) {
m_object = obj;
m_meta.info = { linear_id, type };
}
constexpr ALWAYS_INLINE KAutoObject *GetObject() const { return m_object; }
constexpr ALWAYS_INLINE Entry *GetNextFreeEntry() const { return m_meta.next_free_entry; }
constexpr ALWAYS_INLINE u16 GetLinearId() const { return m_meta.info.linear_id; }
constexpr ALWAYS_INLINE u16 GetType() const { return m_meta.info.type; }
}; };
private: private:
mutable KSpinLock m_lock; EntryInfo m_entry_infos[MaxTableSize];
Entry *m_table; KAutoObject *m_objects[MaxTableSize];
Entry *m_free_head; s32 m_free_head_index;
Entry m_entries[MaxTableSize];
u16 m_table_size; u16 m_table_size;
u16 m_max_count; u16 m_max_count;
u16 m_next_linear_id; u16 m_next_linear_id;
u16 m_count; u16 m_count;
mutable KSpinLock m_lock;
public: public:
constexpr KHandleTable() : constexpr KHandleTable() :
m_lock(), m_table(nullptr), m_free_head(nullptr), m_entries(), m_table_size(0), m_max_count(0), m_next_linear_id(MinLinearId), m_count(0) m_entry_infos(), m_objects(), m_free_head_index(-1), m_table_size(0), m_max_count(0), m_next_linear_id(MinLinearId), m_count(0), m_lock()
{ MESOSPHERE_ASSERT_THIS(); } { MESOSPHERE_ASSERT_THIS(); }
constexpr NOINLINE Result Initialize(s32 size) { constexpr NOINLINE Result Initialize(s32 size) {
@ -101,19 +84,18 @@ namespace ams::kern {
R_UNLESS(size <= static_cast<s32>(MaxTableSize), svc::ResultOutOfMemory()); R_UNLESS(size <= static_cast<s32>(MaxTableSize), svc::ResultOutOfMemory());
/* Initialize all fields. */ /* Initialize all fields. */
m_table = m_entries; m_max_count = 0;
m_table_size = (size <= 0) ? MaxTableSize : size; m_table_size = (size <= 0) ? MaxTableSize : size;
m_next_linear_id = MinLinearId; m_next_linear_id = MinLinearId;
m_count = 0; m_count = 0;
m_max_count = 0; m_free_head_index = -1;
/* Free all entries. */ /* Free all entries. */
for (size_t i = 0; i < static_cast<size_t>(m_table_size - 1); i++) { for (s32 i = 0; i < static_cast<s32>(m_table_size); ++i) {
m_entries[i].SetFree(std::addressof(m_entries[i + 1])); m_objects[i] = nullptr;
m_entry_infos[i].next_free_index = i - 1;
m_free_head_index = i;
} }
m_entries[m_table_size - 1].SetFree(nullptr);
m_free_head = std::addressof(m_entries[0]);
return ResultSuccess(); return ResultSuccess();
} }
@ -134,7 +116,7 @@ namespace ams::kern {
if constexpr (std::is_same<T, KAutoObject>::value) { if constexpr (std::is_same<T, KAutoObject>::value) {
return this->GetObjectImpl(handle); return this->GetObjectImpl(handle);
} else { } else {
if (auto *obj = this->GetObjectImpl(handle); obj != nullptr) { if (auto *obj = this->GetObjectImpl(handle); AMS_LIKELY(obj != nullptr)) {
return obj->DynamicCast<T*>(); return obj->DynamicCast<T*>();
} else { } else {
return nullptr; return nullptr;
@ -256,27 +238,29 @@ namespace ams::kern {
NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj, u16 type); NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj, u16 type);
NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj, u16 type); NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj, u16 type);
constexpr ALWAYS_INLINE Entry *AllocateEntry() { constexpr ALWAYS_INLINE s32 AllocateEntry() {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(m_count < m_table_size); MESOSPHERE_ASSERT(m_count < m_table_size);
Entry *entry = m_free_head; const auto index = m_free_head_index;
m_free_head = entry->GetNextFreeEntry();
m_count++; m_free_head_index = m_entry_infos[index].GetNextFreeIndex();
m_max_count = std::max(m_max_count, m_count);
return entry; m_max_count = std::max(m_max_count, ++m_count);
return index;
} }
constexpr ALWAYS_INLINE void FreeEntry(Entry *entry) { constexpr ALWAYS_INLINE void FreeEntry(s32 index) {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
MESOSPHERE_ASSERT(m_count > 0); MESOSPHERE_ASSERT(m_count > 0);
entry->SetFree(m_free_head); m_objects[index] = nullptr;
m_free_head = entry; m_entry_infos[index].next_free_index = m_free_head_index;
m_count--; m_free_head_index = index;
--m_count;
} }
constexpr ALWAYS_INLINE u16 AllocateLinearId() { constexpr ALWAYS_INLINE u16 AllocateLinearId() {
@ -287,13 +271,7 @@ namespace ams::kern {
return id; return id;
} }
constexpr ALWAYS_INLINE size_t GetEntryIndex(Entry *entry) { constexpr ALWAYS_INLINE bool IsValidHandle(ams::svc::Handle handle) const {
const size_t index = entry - m_table;
MESOSPHERE_ASSERT(index < m_table_size);
return index;
}
constexpr ALWAYS_INLINE Entry *FindEntry(ams::svc::Handle handle) const {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Unpack the handle. */ /* Unpack the handle. */
@ -306,38 +284,38 @@ namespace ams::kern {
MESOSPHERE_UNUSED(reserved); MESOSPHERE_UNUSED(reserved);
/* Validate our indexing information. */ /* Validate our indexing information. */
if (raw_value == 0) { if (AMS_UNLIKELY(raw_value == 0)) {
return nullptr; return false;
} }
if (linear_id == 0) { if (AMS_UNLIKELY(linear_id == 0)) {
return nullptr; return false;
} }
if (index >= m_table_size) { if (AMS_UNLIKELY(index >= m_table_size)) {
return nullptr; return false;
} }
/* Get the entry, and ensure our serial id is correct. */ /* Check that there's an object, and our serial id is correct. */
Entry *entry = std::addressof(m_table[index]); if (AMS_UNLIKELY(m_objects[index] == nullptr)) {
if (entry->GetObject() == nullptr) { return false;
return nullptr;
} }
if (entry->GetLinearId() != linear_id) { if (AMS_UNLIKELY(m_entry_infos[index].GetLinearId() != linear_id)) {
return nullptr; return false;
} }
return entry; return true;
} }
constexpr ALWAYS_INLINE KAutoObject *GetObjectImpl(ams::svc::Handle handle) const { constexpr ALWAYS_INLINE KAutoObject *GetObjectImpl(ams::svc::Handle handle) const {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Handles must not have reserved bits set. */ /* Handles must not have reserved bits set. */
if (GetHandleBitPack(handle).Get<HandleReserved>() != 0) { const auto handle_pack = GetHandleBitPack(handle);
if (AMS_UNLIKELY(handle_pack.Get<HandleReserved>() != 0)) {
return nullptr; return nullptr;
} }
if (Entry *entry = this->FindEntry(handle); entry != nullptr) { if (AMS_LIKELY(this->IsValidHandle(handle))) {
return entry->GetObject(); return m_objects[handle_pack.Get<HandleIndex>()];
} else { } else {
return nullptr; return nullptr;
} }
@ -347,18 +325,17 @@ namespace ams::kern {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Index must be in bounds. */ /* Index must be in bounds. */
if (index >= m_table_size || m_table == nullptr) { if (AMS_UNLIKELY(index >= m_table_size)) {
return nullptr; return nullptr;
} }
/* Ensure entry has an object. */ /* Ensure entry has an object. */
Entry *entry = std::addressof(m_table[index]); if (KAutoObject *obj = m_objects[index]; obj != nullptr) {
if (entry->GetObject() == nullptr) { *out_handle = EncodeHandle(index, m_entry_infos[index].GetLinearId());
return obj;
} else {
return nullptr; return nullptr;
} }
*out_handle = EncodeHandle(index, entry->GetLinearId());
return entry->GetObject();
} }
}; };

50
libraries/libmesosphere/source/kern_k_handle_table.cpp
View file

@ -21,23 +21,18 @@ namespace ams::kern {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Get the table and clear our record of it. */ /* Get the table and clear our record of it. */
Entry *saved_table = nullptr;
u16 saved_table_size = 0; u16 saved_table_size = 0;
{ {
KScopedDisableDispatch dd; KScopedDisableDispatch dd;
KScopedSpinLock lk(m_lock); KScopedSpinLock lk(m_lock);
std::swap(m_table, saved_table);
std::swap(m_table_size, saved_table_size); std::swap(m_table_size, saved_table_size);
} }
/* Close and free all entries. */ /* Close and free all entries. */
for (size_t i = 0; i < saved_table_size; i++) { for (size_t i = 0; i < saved_table_size; i++) {
Entry *entry = std::addressof(saved_table[i]); if (KAutoObject *obj = m_objects[i]; obj != nullptr) {
if (KAutoObject *obj = entry->GetObject(); obj != nullptr) {
obj->Close(); obj->Close();
this->FreeEntry(entry);
} }
} }
@ -48,12 +43,13 @@ namespace ams::kern {
MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT_THIS();
/* Don't allow removal of a pseudo-handle. */ /* Don't allow removal of a pseudo-handle. */
if (ams::svc::IsPseudoHandle(handle)) { if (AMS_UNLIKELY(ams::svc::IsPseudoHandle(handle))) {
return false; return false;
} }
/* Handles must not have reserved bits set. */ /* Handles must not have reserved bits set. */
if (GetHandleBitPack(handle).Get<HandleReserved>() != 0) { const auto handle_pack = GetHandleBitPack(handle);
if (AMS_UNLIKELY(handle_pack.Get<HandleReserved>() != 0)) {
return false; return false;
} }
@ -63,9 +59,11 @@ namespace ams::kern {
KScopedDisableDispatch dd; KScopedDisableDispatch dd;
KScopedSpinLock lk(m_lock); KScopedSpinLock lk(m_lock);
if (Entry *entry = this->FindEntry(handle); entry != nullptr) { if (AMS_LIKELY(this->IsValidHandle(handle))) {
obj = entry->GetObject(); const auto index = handle_pack.Get<HandleIndex>();
this->FreeEntry(entry);
obj = m_objects[index];
this->FreeEntry(index);
} else { } else {
return false; return false;
} }
@ -87,10 +85,14 @@ namespace ams::kern {
/* Allocate entry, set output handle. */ /* Allocate entry, set output handle. */
{ {
const auto linear_id = this->AllocateLinearId(); const auto linear_id = this->AllocateLinearId();
Entry *entry = this->AllocateEntry(); const auto index = this->AllocateEntry();
entry->SetUsed(obj, linear_id, type);
m_entry_infos[index].info = { .linear_id = linear_id, .type = type };
m_objects[index] = obj;
obj->Open(); obj->Open();
*out_handle = EncodeHandle(this->GetEntryIndex(entry), linear_id);
*out_handle = EncodeHandle(index, linear_id);
} }
return ResultSuccess(); return ResultSuccess();
@ -104,7 +106,7 @@ namespace ams::kern {
/* Never exceed our capacity. */ /* Never exceed our capacity. */
R_UNLESS(m_count < m_table_size, svc::ResultOutOfHandles()); R_UNLESS(m_count < m_table_size, svc::ResultOutOfHandles());
*out_handle = EncodeHandle(this->GetEntryIndex(this->AllocateEntry()), this->AllocateLinearId()); *out_handle = EncodeHandle(this->AllocateEntry(), this->AllocateLinearId());
return ResultSuccess(); return ResultSuccess();
} }
@ -122,13 +124,10 @@ namespace ams::kern {
MESOSPHERE_ASSERT(linear_id != 0); MESOSPHERE_ASSERT(linear_id != 0);
MESOSPHERE_UNUSED(linear_id, reserved); MESOSPHERE_UNUSED(linear_id, reserved);
if (index < m_table_size) { if (AMS_LIKELY(index < m_table_size)) {
/* Free the entry. */
/* NOTE: This code does not check the linear id. */ /* NOTE: This code does not check the linear id. */
Entry *entry = std::addressof(m_table[index]); MESOSPHERE_ASSERT(m_objects[index] == nullptr);
MESOSPHERE_ASSERT(entry->GetObject() == nullptr); this->FreeEntry(index);
this->FreeEntry(entry);
} }
} }
@ -146,12 +145,13 @@ namespace ams::kern {
MESOSPHERE_ASSERT(linear_id != 0); MESOSPHERE_ASSERT(linear_id != 0);
MESOSPHERE_UNUSED(reserved); MESOSPHERE_UNUSED(reserved);
if (index < m_table_size) { if (AMS_LIKELY(index < m_table_size)) {
/* Set the entry. */ /* Set the entry. */
Entry *entry = std::addressof(m_table[index]); MESOSPHERE_ASSERT(m_objects[index] == nullptr);
MESOSPHERE_ASSERT(entry->GetObject() == nullptr);
m_entry_infos[index].info = { .linear_id = linear_id, .type = type };
m_objects[index] = obj;
entry->SetUsed(obj, linear_id, type);
obj->Open(); obj->Open();
} }
} }