kern: devirtualize most things that are free to devirtualize (see #1672)

libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp

@@ -34,19 +34,22 @@ namespace ams::kern::arch::arm64::init {
         }
     }
 
-    class KInitialPageTable {
+    /* NOTE: Nintendo uses virtual functions, rather than a concept + template. */
+    template<typename T>
+    concept IsInitialPageAllocator = requires (T &t, KPhysicalAddress phys_addr, size_t size) {
+        { t.Allocate(size) }        -> std::same_as<KPhysicalAddress>;
+        { t.Free(phys_addr, size) } -> std::same_as<void>;
+    };
+
+    template<IsInitialPageAllocator _PageAllocator>
+    class KInitialPageTableTemplate {
         public:
-            class IPageAllocator {
+            using PageAllocator = _PageAllocator;
-                public:
-                    virtual KPhysicalAddress Allocate(size_t size) = 0;
-                    virtual void Free(KPhysicalAddress phys_addr, size_t size) = 0;
-            };
         private:
             KPhysicalAddress m_l1_tables[2];
             u32 m_num_entries[2];
-
         public:
-            KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, IPageAllocator &allocator) {
+            KInitialPageTableTemplate(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) {
                 /* Set tables. */
                 m_l1_tables[0] = AllocateNewPageTable(allocator);
                 m_l1_tables[1] = AllocateNewPageTable(allocator);
@@ -56,7 +59,7 @@ namespace ams::kern::arch::arm64::init {
                 m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1;
             }
 
-            KInitialPageTable() {
+            KInitialPageTableTemplate() {
                 /* Set tables. */
                 m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize);
                 m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize);
@@ -95,7 +98,7 @@ namespace ams::kern::arch::arm64::init {
                 return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1));
             }
 
-            static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(IPageAllocator &allocator) {
+            static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator) {
                 auto address = allocator.Allocate(PageSize);
                 ClearNewPageTable(address);
                 return address;
@@ -323,7 +326,7 @@ namespace ams::kern::arch::arm64::init {
                 }
             }
         public:
-            void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, IPageAllocator &allocator) {
+            void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator) {
                 /* Ensure that addresses and sizes are page aligned. */
                 MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr),    PageSize));
                 MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr),    PageSize));
@@ -700,10 +703,9 @@ namespace ams::kern::arch::arm64::init {
                 this->PhysicallyRandomize(virt_addr, size, L3BlockSize, do_copy);
                 cpu::StoreEntireCacheForInit();
             }
-
     };
 
-    class KInitialPageAllocator final : public KInitialPageTable::IPageAllocator {
+    class KInitialPageAllocator final {
         private:
             static constexpr inline size_t FreeUnitSize = BITSIZEOF(u64) * PageSize;
             struct FreeListEntry {
@@ -807,11 +809,11 @@ namespace ams::kern::arch::arm64::init {
                 }
             }
 
-            virtual KPhysicalAddress Allocate(size_t size) override {
+            KPhysicalAddress Allocate(size_t size) {
                 return this->Allocate(size, size);
             }
 
-            virtual void Free(KPhysicalAddress phys_addr, size_t size) override {
+            void Free(KPhysicalAddress phys_addr, size_t size) {
                 auto **prev_next = std::addressof(m_state.free_head);
                 auto *new_chunk  = reinterpret_cast<FreeListEntry *>(GetInteger(phys_addr));
                 if (auto *cur = m_state.free_head; cur != nullptr) {
@@ -863,5 +865,8 @@ namespace ams::kern::arch::arm64::init {
                 *prev_next = new_chunk;
             }
     };
+    static_assert(IsInitialPageAllocator<KInitialPageAllocator>);
+
+    using KInitialPageTable = KInitialPageTableTemplate<KInitialPageAllocator>;
 
 }

libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_debug.hpp

@@ -34,8 +34,9 @@ namespace ams::kern::arch::arm64 {
 
             static void PostDestroy(uintptr_t arg) { MESOSPHERE_UNUSED(arg); /* ... */ }
         public:
-            virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) override;
+            /* NOTE: These are virtual in Nintendo's kernel. */
-            virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) override;
+            Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
+            Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);
         private:
             Result GetFpuContext(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags);
             Result SetFpuContext(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags);

libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp

@@ -38,11 +38,11 @@ namespace ams::kern {
             static constexpr inline ClassTokenType ClassToken() { return ::ams::kern::ClassToken<CLASS>; }      \
         public:                                                                                                 \
             using BaseClass = BASE_CLASS;                                                                       \
-            static constexpr ALWAYS_INLINE TypeObj GetStaticTypeObj() {                                         \
+            static consteval ALWAYS_INLINE TypeObj GetStaticTypeObj() {                                         \
                 constexpr ClassTokenType Token = ClassToken();                                                  \
                 return TypeObj(TypeName, Token);                                                                \
             }                                                                                                   \
-            static constexpr ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; }                 \
+            static consteval ALWAYS_INLINE const char *GetStaticTypeName() { return TypeName; }                 \
             virtual TypeObj GetTypeObj() const { return GetStaticTypeObj(); }                                   \
             virtual const char *GetTypeName() { return GetStaticTypeName(); }                                   \
         private:
@@ -143,7 +143,8 @@ namespace ams::kern {
             /* is already using CRTP for slab heap, we have devirtualized it for performance gain. */
             /* virtual void Finalize() { MESOSPHERE_ASSERT_THIS(); } */
 
-            virtual KProcess *GetOwner() const { return nullptr; }
+            /* NOTE: This is a virtual function which is unused-except-for-debug in Nintendo's kernel. */
+            /* virtual KProcess *GetOwner() const { return nullptr; } */
 
             u32 GetReferenceCount() const {
                 return m_ref_count.GetValue();

libraries/libmesosphere/include/mesosphere/kern_k_auto_object_container.hpp

@@ -60,9 +60,38 @@ namespace ams::kern {
             void Initialize() { MESOSPHERE_ASSERT_THIS(); }
             void Finalize() { MESOSPHERE_ASSERT_THIS(); }
 
-            void Register(KAutoObjectWithList *obj);
+            void Register(KAutoObjectWithList *obj) {
-            void Unregister(KAutoObjectWithList *obj);
+                MESOSPHERE_ASSERT_THIS();
-            size_t GetOwnedCount(KProcess *owner);
+
+                KScopedLightLock lk(m_lock);
+
+                m_object_list.insert(*obj);
+            }
+
+            void Unregister(KAutoObjectWithList *obj) {
+                MESOSPHERE_ASSERT_THIS();
+
+                KScopedLightLock lk(m_lock);
+
+                m_object_list.erase(m_object_list.iterator_to(*obj));
+            }
+
+            template<typename T> requires (std::derived_from<T, KAutoObjectWithList> && requires (const T &t) { { t.GetOwner() } -> std::convertible_to<const KProcess *>; })
+            size_t GetOwnedCount(const KProcess *owner) {
+                MESOSPHERE_ASSERT_THIS();
+
+                KScopedLightLock lk(m_lock);
+
+                size_t count = 0;
+
+                for (const auto &obj : m_object_list) {
+                    if (const T * const derived = obj.DynamicCast<T *>(); derived != nullptr && derived->GetOwner() == owner) {
+                        ++count;
+                    }
+                }
+
+                return count;
+            }
     };
 
 

libraries/libmesosphere/include/mesosphere/kern_k_debug_base.hpp

@@ -38,6 +38,7 @@ namespace ams::kern {
             bool Is64Bit() const;
         public:
             void Initialize();
+            void Finalize();
 
             Result Attach(KProcess *process);
             Result BreakProcess();
@@ -52,9 +53,6 @@ namespace ams::kern {
             Result GetThreadContext(ams::svc::ThreadContext *out, u64 thread_id, u32 context_flags);
             Result SetThreadContext(const ams::svc::ThreadContext &ctx, u64 thread_id, u32 context_flags);
 
-            virtual Result GetThreadContextImpl(ams::svc::ThreadContext *out, KThread *thread, u32 context_flags) = 0;
-            virtual Result SetThreadContextImpl(const ams::svc::ThreadContext &ctx, KThread *thread, u32 context_flags) = 0;
-
             Result GetRunningThreadInfo(ams::svc::LastThreadContext *out_context, u64 *out_thread_id);
 
             Result GetDebugEventInfo(ams::svc::lp64::DebugEventInfo *out);
@@ -82,8 +80,10 @@ namespace ams::kern {
             template<typename T> requires (std::same_as<T, ams::svc::lp64::DebugEventInfo> || std::same_as<T, ams::svc::ilp32::DebugEventInfo>)
             Result GetDebugEventInfoImpl(T *out);
         public:
-            virtual void OnFinalizeSynchronizationObject() override;
             virtual bool IsSignaled() const override;
+        private:
+            /* NOTE: This is public/virtual override in Nintendo's kernel. */
+            void OnFinalizeSynchronizationObject();
         private:
             static Result ProcessDebugEvent(ams::svc::DebugEvent event, uintptr_t param0, uintptr_t param1, uintptr_t param2, uintptr_t param3, uintptr_t param4);
         public:

libraries/libmesosphere/include/mesosphere/kern_k_event.hpp

@@ -46,7 +46,7 @@ namespace ams::kern {
 
             static void PostDestroy(uintptr_t arg);
 
-            virtual KProcess *GetOwner() const override { return m_owner; }
+            KProcess *GetOwner() const { return m_owner; }
 
             KReadableEvent &GetReadableEvent() { return m_readable_event; }
 

libraries/libmesosphere/include/mesosphere/kern_k_synchronization_object.hpp

@@ -35,7 +35,8 @@ namespace ams::kern {
             constexpr ALWAYS_INLINE explicit KSynchronizationObject(util::ConstantInitializeTag) : KAutoObjectWithList(util::ConstantInitialize), m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
             ALWAYS_INLINE explicit KSynchronizationObject() : m_thread_list_head(), m_thread_list_tail() { MESOSPHERE_ASSERT_THIS(); }
 
-            virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); }
+            /* NOTE: This is a virtual function which is overridden only by KDebugBase in Nintendo's kernel. */
+            /* virtual void OnFinalizeSynchronizationObject() { MESOSPHERE_ASSERT_THIS(); } */
 
             void NotifyAvailable(Result result);
             ALWAYS_INLINE void NotifyAvailable() {

libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp

@@ -620,7 +620,7 @@ namespace ams::kern {
             void Finalize();
 
             virtual bool IsSignaled() const override;
-            virtual void OnTimer() override;
+            void OnTimer();
             virtual void DoWorkerTask() override;
         public:
             static constexpr bool IsConditionVariableThreadTreeValid() {
@@ -674,4 +674,8 @@ namespace ams::kern {
         GetCurrentThread().SetClosedObject(this);
     }
 
+    ALWAYS_INLINE void KTimerTask::OnTimer() {
+        static_cast<KThread *>(this)->OnTimer();
+    }
+
 }

libraries/libmesosphere/include/mesosphere/kern_k_timer_task.hpp

@@ -41,8 +41,9 @@ namespace ams::kern {
                 return m_time;
             }
 
-            virtual void OnTimer() = 0;
+            /* NOTE: This is virtual in Nintendo's kernel. Prior to 13.0.0, KWaitObject was also a TimerTask; this is no longer the case. */
-
+            /* Since this is now KThread exclusive, we have devirtualized (see inline declaration for this inside kern_kthread.hpp). */
+            void OnTimer();
     };
 
 }

libraries/libmesosphere/source/kern_k_auto_object_container.cpp

@@ -1,53 +0,0 @@
-/*
- * Copyright (c) Atmosphère-NX
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-#include <mesosphere.hpp>
-
-namespace ams::kern {
-
-
-    void KAutoObjectWithListContainer::Register(KAutoObjectWithList *obj) {
-        MESOSPHERE_ASSERT_THIS();
-
-        KScopedLightLock lk(m_lock);
-
-        m_object_list.insert(*obj);
-    }
-
-    void KAutoObjectWithListContainer::Unregister(KAutoObjectWithList *obj) {
-        MESOSPHERE_ASSERT_THIS();
-
-        KScopedLightLock lk(m_lock);
-
-        m_object_list.erase(m_object_list.iterator_to(*obj));
-    }
-
-    size_t KAutoObjectWithListContainer::GetOwnedCount(KProcess *owner) {
-        MESOSPHERE_ASSERT_THIS();
-
-        KScopedLightLock lk(m_lock);
-
-        size_t count = 0;
-
-        for (auto &obj : m_object_list) {
-            if (obj.GetOwner() == owner) {
-                count++;
-            }
-        }
-
-        return count;
-    }
-
-}

libraries/libmesosphere/source/kern_k_debug_base.cpp

@@ -528,7 +528,8 @@ namespace ams::kern {
             }
 
             /* Get the thread context. */
-            return this->GetThreadContextImpl(out, thread, context_flags);
+            static_assert(std::derived_from<KDebug, KDebugBase>);
+            return static_cast<KDebug *>(this)->GetThreadContextImpl(out, thread, context_flags);
         }
     }
 
@@ -619,7 +620,8 @@ namespace ams::kern {
             }
 
             /* Set the thread context. */
-            return this->SetThreadContextImpl(ctx, thread, context_flags);
+            static_assert(std::derived_from<KDebug, KDebugBase>);
+            return static_cast<KDebug *>(this)->SetThreadContextImpl(ctx, thread, context_flags);
         }
     }
 
@@ -984,6 +986,14 @@ namespace ams::kern {
         return this->GetDebugEventInfoImpl(out);
     }
 
+    void KDebugBase::Finalize() {
+        /* Perform base finalization. */
+        KSynchronizationObject::Finalize();
+
+        /* Perform post-synchronization finalization. */
+        this->OnFinalizeSynchronizationObject();
+    }
+
     void KDebugBase::OnFinalizeSynchronizationObject() {
         /* Detach from our process, if we have one. */
         if (this->IsAttached() && this->OpenProcess()) {

libraries/libmesosphere/source/kern_k_synchronization_object.cpp

@@ -83,7 +83,8 @@ namespace ams::kern {
         }
         #endif
 
-        this->OnFinalizeSynchronizationObject();
+        /* NOTE: In Nintendo's kernel, the following is virtual and called here. */
+        /* this->OnFinalizeSynchronizationObject(); */
     }
 
     Result KSynchronizationObject::Wait(s32 *out_index, KSynchronizationObject **objects, const s32 num_objects, s64 timeout) {

mesosphere/kernel_ldr/source/kern_init_loader.cpp

@@ -62,7 +62,7 @@ namespace ams::kern::init::loader {
             }
         }
 
-        void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::IPageAllocator &allocator) {
+        void SetupInitialIdentityMapping(KInitialPageTable &init_pt, uintptr_t base_address, uintptr_t kernel_size, uintptr_t page_table_region, size_t page_table_region_size, KInitialPageTable::PageAllocator &allocator) {
             /* Map in an RWX identity mapping for the kernel. */
             constexpr PageTableEntry KernelRWXIdentityAttribute(PageTableEntry::Permission_KernelRWX, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped);
             init_pt.Map(base_address, kernel_size, base_address, KernelRWXIdentityAttribute, allocator);