diff --git a/libraries/libmesosphere/include/mesosphere/kern_build_config.hpp b/libraries/libmesosphere/include/mesosphere/kern_build_config.hpp index f3ae3cf0c..8dcee9a88 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_build_config.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_build_config.hpp @@ -29,6 +29,14 @@ #define MESOSPHERE_ENABLE_KERNEL_STACK_USAGE #endif +#if defined(MESOSPHERE_BUILD_FOR_DEBUGGING) +#define MESOSPHERE_NOINLINE_IF_DEBUG NOINLINE +#define MESOSPHERE_ALWAYS_INLINE_IF_RELEASE NOINLINE +#else +#define MESOSPHERE_NOINLINE_IF_DEBUG +#define MESOSPHERE_ALWAYS_INLINE_IF_RELEASE ALWAYS_INLINE +#endif + //#define MESOSPHERE_BUILD_FOR_TRACING //#define MESOSPHERE_ENABLE_PERFORMANCE_COUNTER #define MESOSPHERE_ENABLE_PANIC_REGISTER_DUMP diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp index 72f8ec06b..223123537 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object.hpp @@ -190,13 +190,13 @@ namespace ams::kern { } } - NOINLINE bool Open() { + MESOSPHERE_ALWAYS_INLINE_IF_RELEASE bool Open() { MESOSPHERE_ASSERT_THIS(); return m_ref_count.Open(); } - NOINLINE void Close() { + MESOSPHERE_ALWAYS_INLINE_IF_RELEASE void Close() { MESOSPHERE_ASSERT_THIS(); if (m_ref_count.Close()) { diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp index 58d60da89..8000f9997 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_auto_object_impls.hpp @@ -100,7 +100,7 @@ namespace ams::kern { } - ALWAYS_INLINE void KAutoObject::ScheduleDestruction() { + inline NOINLINE void KAutoObject::ScheduleDestruction() { MESOSPHERE_ASSERT_THIS(); /* Set our object to destroy. */ diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp index 5600aacfd..2e1fb7d51 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_handle_table.hpp @@ -74,7 +74,7 @@ namespace ams::kern { explicit KHandleTable() : m_lock(), m_free_head_index(-1), m_count() { MESOSPHERE_ASSERT_THIS(); } - constexpr NOINLINE Result Initialize(s32 size) { + constexpr MESOSPHERE_NOINLINE_IF_DEBUG Result Initialize(s32 size) { MESOSPHERE_ASSERT_THIS(); R_UNLESS(size <= static_cast(MaxTableSize), svc::ResultOutOfMemory()); @@ -100,8 +100,8 @@ namespace ams::kern { constexpr ALWAYS_INLINE size_t GetCount() const { return m_count; } constexpr ALWAYS_INLINE size_t GetMaxCount() const { return m_max_count; } - NOINLINE Result Finalize(); - NOINLINE bool Remove(ams::svc::Handle handle); + MESOSPHERE_NOINLINE_IF_DEBUG Result Finalize(); + MESOSPHERE_NOINLINE_IF_DEBUG bool Remove(ams::svc::Handle handle); template ALWAYS_INLINE KScopedAutoObject GetObjectWithoutPseudoHandle(ams::svc::Handle handle) const { @@ -180,11 +180,11 @@ namespace ams::kern { return this->GetObjectByIndexImpl(out_handle, index); } - NOINLINE Result Reserve(ams::svc::Handle *out_handle); - NOINLINE void Unreserve(ams::svc::Handle handle); + MESOSPHERE_NOINLINE_IF_DEBUG Result Reserve(ams::svc::Handle *out_handle); + MESOSPHERE_NOINLINE_IF_DEBUG void Unreserve(ams::svc::Handle handle); - NOINLINE Result Add(ams::svc::Handle *out_handle, KAutoObject *obj); - NOINLINE void Register(ams::svc::Handle handle, KAutoObject *obj); + MESOSPHERE_NOINLINE_IF_DEBUG Result Add(ams::svc::Handle *out_handle, KAutoObject *obj); + MESOSPHERE_NOINLINE_IF_DEBUG void Register(ams::svc::Handle handle, KAutoObject *obj); template ALWAYS_INLINE bool GetMultipleObjects(T **out, const ams::svc::Handle *handles, size_t num_handles) const { @@ -297,7 +297,7 @@ namespace ams::kern { return true; } - constexpr NOINLINE KAutoObject *GetObjectImpl(ams::svc::Handle handle) const { + constexpr MESOSPHERE_NOINLINE_IF_DEBUG KAutoObject *GetObjectImpl(ams::svc::Handle handle) const { MESOSPHERE_ASSERT_THIS(); /* Handles must not have reserved bits set. */ diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp index 63c6a2f94..c96f56362 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_light_lock.hpp @@ -56,8 +56,8 @@ namespace ams::kern { } } - bool LockSlowPath(uintptr_t owner, uintptr_t cur_thread); - void UnlockSlowPath(uintptr_t cur_thread); + NOINLINE bool LockSlowPath(uintptr_t owner, uintptr_t cur_thread); + NOINLINE void UnlockSlowPath(uintptr_t cur_thread); ALWAYS_INLINE bool IsLocked() const { return m_tag.Load() != 0; } ALWAYS_INLINE bool IsLockedByCurrentThread() const { return (m_tag.Load() | 0x1ul) == (reinterpret_cast(GetCurrentThreadPointer()) | 0x1ul); } diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index e451b9622..2c4970727 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -126,47 +126,47 @@ namespace ams::kern { static ALWAYS_INLINE KVirtualAddress GetLinearVirtualAddress(KPhysicalAddress address) { return GetInteger(address) + s_linear_phys_to_virt_diff; } static ALWAYS_INLINE KPhysicalAddress GetLinearPhysicalAddress(KVirtualAddress address) { return GetInteger(address) + s_linear_virt_to_phys_diff; } - static NOINLINE const KMemoryRegion *Find(KVirtualAddress address) { return Find(address, GetVirtualMemoryRegionTree()); } - static NOINLINE const KMemoryRegion *Find(KPhysicalAddress address) { return Find(address, GetPhysicalMemoryRegionTree()); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *Find(KVirtualAddress address) { return Find(address, GetVirtualMemoryRegionTree()); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *Find(KPhysicalAddress address) { return Find(address, GetPhysicalMemoryRegionTree()); } - static NOINLINE const KMemoryRegion *FindLinear(KVirtualAddress address) { return Find(address, GetVirtualLinearMemoryRegionTree()); } - static NOINLINE const KMemoryRegion *FindLinear(KPhysicalAddress address) { return Find(address, GetPhysicalLinearMemoryRegionTree()); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *FindLinear(KVirtualAddress address) { return Find(address, GetVirtualLinearMemoryRegionTree()); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *FindLinear(KPhysicalAddress address) { return Find(address, GetPhysicalLinearMemoryRegionTree()); } - static NOINLINE KVirtualAddress GetMainStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscMainStack); } - static NOINLINE KVirtualAddress GetIdleStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscIdleStack); } - static NOINLINE KVirtualAddress GetExceptionStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscExceptionStack); } + static MESOSPHERE_NOINLINE_IF_DEBUG KVirtualAddress GetMainStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscMainStack); } + static MESOSPHERE_NOINLINE_IF_DEBUG KVirtualAddress GetIdleStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscIdleStack); } + static MESOSPHERE_NOINLINE_IF_DEBUG KVirtualAddress GetExceptionStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscExceptionStack); } - static NOINLINE KVirtualAddress GetSlabRegionAddress() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)).GetAddress(); } + static MESOSPHERE_NOINLINE_IF_DEBUG KVirtualAddress GetSlabRegionAddress() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)).GetAddress(); } - static NOINLINE const KMemoryRegion &GetDeviceRegion(KMemoryRegionType type) { return Dereference(GetPhysicalMemoryRegionTree().FindFirstDerived(type)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetDeviceRegion(KMemoryRegionType type) { return Dereference(GetPhysicalMemoryRegionTree().FindFirstDerived(type)); } static KPhysicalAddress GetDevicePhysicalAddress(KMemoryRegionType type) { return GetDeviceRegion(type).GetAddress(); } static KVirtualAddress GetDeviceVirtualAddress(KMemoryRegionType type) { return GetDeviceRegion(type).GetPairAddress(); } - static NOINLINE const KMemoryRegion &GetPoolManagementRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramPoolManagement)); } - static NOINLINE const KMemoryRegion &GetPageTableHeapRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); } - static NOINLINE const KMemoryRegion &GetKernelStackRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); } - static NOINLINE const KMemoryRegion &GetTempRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); } - static NOINLINE const KMemoryRegion &GetSlabRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetPoolManagementRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramPoolManagement)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetPageTableHeapRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetKernelStackRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetTempRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetSlabRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)); } - static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); } - static NOINLINE const KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { return Dereference(FindLinear(address)); } - static NOINLINE const KMemoryRegion &GetPhysicalLinearRegion(KPhysicalAddress address) { return Dereference(FindLinear(address)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) { return Dereference(FindLinear(address)); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion &GetPhysicalLinearRegion(KPhysicalAddress address) { return Dereference(FindLinear(address)); } - static NOINLINE const KMemoryRegion *GetPhysicalKernelTraceBufferRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_KernelTraceBuffer); } - static NOINLINE const KMemoryRegion *GetPhysicalOnMemoryBootImageRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_OnMemoryBootImage); } - static NOINLINE const KMemoryRegion *GetPhysicalDTBRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DTB); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *GetPhysicalKernelTraceBufferRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_KernelTraceBuffer); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *GetPhysicalOnMemoryBootImageRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_OnMemoryBootImage); } + static MESOSPHERE_NOINLINE_IF_DEBUG const KMemoryRegion *GetPhysicalDTBRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DTB); } - static NOINLINE bool IsHeapPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address) { return IsTypedAddress(region, address, GetPhysicalLinearMemoryRegionTree(), KMemoryRegionType_DramUserPool); } - static NOINLINE bool IsHeapVirtualAddress(const KMemoryRegion *®ion, KVirtualAddress address) { return IsTypedAddress(region, address, GetVirtualLinearMemoryRegionTree(), KMemoryRegionType_VirtualDramUserPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG bool IsHeapPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address) { return IsTypedAddress(region, address, GetPhysicalLinearMemoryRegionTree(), KMemoryRegionType_DramUserPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG bool IsHeapVirtualAddress(const KMemoryRegion *®ion, KVirtualAddress address) { return IsTypedAddress(region, address, GetVirtualLinearMemoryRegionTree(), KMemoryRegionType_VirtualDramUserPool); } - static NOINLINE bool IsHeapPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address, size_t size) { return IsTypedAddress(region, address, size, GetPhysicalLinearMemoryRegionTree(), KMemoryRegionType_DramUserPool); } - static NOINLINE bool IsHeapVirtualAddress(const KMemoryRegion *®ion, KVirtualAddress address, size_t size) { return IsTypedAddress(region, address, size, GetVirtualLinearMemoryRegionTree(), KMemoryRegionType_VirtualDramUserPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG bool IsHeapPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address, size_t size) { return IsTypedAddress(region, address, size, GetPhysicalLinearMemoryRegionTree(), KMemoryRegionType_DramUserPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG bool IsHeapVirtualAddress(const KMemoryRegion *®ion, KVirtualAddress address, size_t size) { return IsTypedAddress(region, address, size, GetVirtualLinearMemoryRegionTree(), KMemoryRegionType_VirtualDramUserPool); } - static NOINLINE bool IsLinearMappedPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address) { return IsTypedAddress(region, address, GetPhysicalLinearMemoryRegionTree(), static_cast(KMemoryRegionAttr_LinearMapped)); } - static NOINLINE bool IsLinearMappedPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address, size_t size) { return IsTypedAddress(region, address, size, GetPhysicalLinearMemoryRegionTree(), static_cast(KMemoryRegionAttr_LinearMapped)); } + static MESOSPHERE_NOINLINE_IF_DEBUG bool IsLinearMappedPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address) { return IsTypedAddress(region, address, GetPhysicalLinearMemoryRegionTree(), static_cast(KMemoryRegionAttr_LinearMapped)); } + static MESOSPHERE_NOINLINE_IF_DEBUG bool IsLinearMappedPhysicalAddress(const KMemoryRegion *®ion, KPhysicalAddress address, size_t size) { return IsTypedAddress(region, address, size, GetPhysicalLinearMemoryRegionTree(), static_cast(KMemoryRegionAttr_LinearMapped)); } - static NOINLINE std::tuple GetTotalAndKernelMemorySizes() { + static std::tuple GetTotalAndKernelMemorySizes() { size_t total_size = 0, kernel_size = 0; for (const auto ®ion : GetPhysicalMemoryRegionTree()) { if (region.IsDerivedFrom(KMemoryRegionType_Dram)) { @@ -189,44 +189,43 @@ namespace ams::kern { static size_t GetResourceRegionSizeForInit(); - static NOINLINE auto GetKernelRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Kernel); } - static NOINLINE auto GetKernelCodeRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelCode); } - static NOINLINE auto GetKernelStackRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelStack); } - static NOINLINE auto GetKernelMiscRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelMisc); } - static NOINLINE auto GetKernelSlabRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelSlab); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Kernel); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelCodeRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelCode); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelStackRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelStack); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelMiscRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelMisc); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelSlabRegionExtents() { return GetVirtualMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelSlab); } - static NOINLINE auto GetLinearRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_LinearMapped); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetLinearRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_LinearMapped); } - static NOINLINE auto GetLinearRegionVirtualExtents() { + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetLinearRegionVirtualExtents() { const auto physical = GetLinearRegionPhysicalExtents(); return KMemoryRegion(GetInteger(GetLinearVirtualAddress(physical.GetAddress())), GetInteger(GetLinearVirtualAddress(physical.GetLastAddress())), 0, KMemoryRegionType_None); } - static NOINLINE auto GetMainMemoryPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram); } - static NOINLINE auto GetCarveoutRegionExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetMainMemoryPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetCarveoutRegionExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected); } - static NOINLINE auto GetKernelRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelBase); } - static NOINLINE auto GetKernelCodeRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelCode); } - static NOINLINE auto GetKernelSlabRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelSlab); } - static NOINLINE auto GetKernelPageTableHeapRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelPtHeap); } - static NOINLINE auto GetKernelInitPageTableRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelInitPt); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelBase); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelCodeRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelCode); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelSlabRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelSlab); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelPageTableHeapRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelPtHeap); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelInitPageTableRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramKernelInitPt); } - static NOINLINE auto GetKernelPoolPartitionRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolPartition); } - static NOINLINE auto GetKernelPoolManagementRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolManagement); } - static NOINLINE auto GetKernelSystemPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemPool); } - static NOINLINE auto GetKernelSystemNonSecurePoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemNonSecurePool); } - static NOINLINE auto GetKernelAppletPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramAppletPool); } - static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelPoolPartitionRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolPartition); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelPoolManagementRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramPoolManagement); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelSystemPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelSystemNonSecurePoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramSystemNonSecurePool); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelAppletPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramAppletPool); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelApplicationPoolRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool); } - static NOINLINE bool HasKernelSystemNonSecurePoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramSystemNonSecurePool) != nullptr; } - static NOINLINE bool HasKernelAppletPoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramAppletPool) != nullptr; } - static NOINLINE bool HasKernelApplicationPoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramApplicationPool) != nullptr; } + static MESOSPHERE_NOINLINE_IF_DEBUG bool HasKernelSystemNonSecurePoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramSystemNonSecurePool) != nullptr; } + static MESOSPHERE_NOINLINE_IF_DEBUG bool HasKernelAppletPoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramAppletPool) != nullptr; } + static MESOSPHERE_NOINLINE_IF_DEBUG bool HasKernelApplicationPoolRegion() { return GetPhysicalMemoryRegionTree().FindFirstDerived(KMemoryRegionType_DramApplicationPool) != nullptr; } - static NOINLINE auto GetKernelTraceBufferRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer); } + static MESOSPHERE_NOINLINE_IF_DEBUG auto GetKernelTraceBufferRegionPhysicalExtents() { return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer); } }; - namespace init { /* These should be generic, regardless of board. */ diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp index b26e1053b..79957c4dc 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp @@ -126,7 +126,7 @@ namespace ams::kern { GetCurrentThread().DisableDispatch(); } - static ALWAYS_INLINE void EnableScheduling(u64 cores_needing_scheduling) { + static NOINLINE void EnableScheduling(u64 cores_needing_scheduling) { MESOSPHERE_ASSERT(GetCurrentThread().GetDisableDispatchCount() >= 1); GetCurrentScheduler().RescheduleOtherCores(cores_needing_scheduling); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp index e12ac1184..14ff730b7 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler_lock.hpp @@ -45,7 +45,7 @@ namespace ams::kern { return m_owner_thread == GetCurrentThreadPointer(); } - NOINLINE void Lock() { + MESOSPHERE_ALWAYS_INLINE_IF_RELEASE void Lock() { MESOSPHERE_ASSERT_THIS(); if (this->IsLockedByCurrentThread()) { @@ -67,7 +67,7 @@ namespace ams::kern { } } - NOINLINE void Unlock() { + MESOSPHERE_ALWAYS_INLINE_IF_RELEASE void Unlock() { MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); MESOSPHERE_ASSERT(m_lock_count > 0); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index a3f19b0ae..b2664d4f0 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -687,7 +687,7 @@ namespace ams::kern { GetCurrentThread().DisableDispatch(); } - ~KScopedDisableDispatch(); + NOINLINE ~KScopedDisableDispatch(); }; ALWAYS_INLINE KExceptionContext *GetExceptionContext(KThread *thread) { diff --git a/mesosphere/kernel/kernel.ld b/mesosphere/kernel/kernel.ld index 2235dae10..11bc534e3 100644 --- a/mesosphere/kernel/kernel.ld +++ b/mesosphere/kernel/kernel.ld @@ -12,7 +12,7 @@ PHDRS SECTIONS { /* =========== CODE section =========== */ - PROVIDE(__start__ = 0x0); + PROVIDE(__start__ = 0x80060000); . = __start__; __code_start = . ;