diff --git a/libraries/config/templates/stratosphere.mk b/libraries/config/templates/stratosphere.mk index f1eebe79c..f8a86bb54 100644 --- a/libraries/config/templates/stratosphere.mk +++ b/libraries/config/templates/stratosphere.mk @@ -15,7 +15,7 @@ include $(DEVKITPRO)/libnx/switch_rules #--------------------------------------------------------------------------------- # options for code generation #--------------------------------------------------------------------------------- -export DEFINES = $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE +export DEFINES = $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE -D_GNU_SOURCE export SETTINGS = $(ATMOSPHERE_SETTINGS) -O2 export CFLAGS = $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE) export CXXFLAGS = $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) diff --git a/libraries/libstratosphere/Makefile b/libraries/libstratosphere/Makefile index 16328c048..15aad5a6c 100644 --- a/libraries/libstratosphere/Makefile +++ b/libraries/libstratosphere/Makefile @@ -15,7 +15,7 @@ include $(DEVKITPRO)/libnx/switch_rules #--------------------------------------------------------------------------------- # options for code generation #--------------------------------------------------------------------------------- -DEFINES := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE +DEFINES := $(ATMOSPHERE_DEFINES) -DATMOSPHERE_IS_STRATOSPHERE -D_GNU_SOURCE SETTINGS := $(ATMOSPHERE_SETTINGS) -O2 CFLAGS := $(ATMOSPHERE_CFLAGS) $(SETTINGS) $(DEFINES) $(INCLUDE) CXXFLAGS := $(CFLAGS) $(ATMOSPHERE_CXXFLAGS) -flto diff --git a/libraries/libstratosphere/include/stratosphere.hpp b/libraries/libstratosphere/include/stratosphere.hpp index bc308430d..eafde1b2b 100644 --- a/libraries/libstratosphere/include/stratosphere.hpp +++ b/libraries/libstratosphere/include/stratosphere.hpp @@ -22,11 +22,15 @@ /* Libstratosphere-only utility. */ #include "stratosphere/util.hpp" +/* Sadly required shims. */ +#include "stratosphere/svc/svc_stratosphere_shims.hpp" + /* Critical modules with no dependencies. */ #include "stratosphere/ams.hpp" #include "stratosphere/os.hpp" #include "stratosphere/dd.hpp" #include "stratosphere/lmem.hpp" +#include "stratosphere/mem.hpp" /* Pull in all ID definitions from NCM. */ #include "stratosphere/ncm/ncm_ids.hpp" @@ -54,7 +58,8 @@ #include "stratosphere/spl.hpp" #include "stratosphere/updater.hpp" + /* Include FS last. */ #include "stratosphere/fs.hpp" #include "stratosphere/fssrv.hpp" -#include "stratosphere/fssystem.hpp" +#include "stratosphere/fssystem.hpp" \ No newline at end of file diff --git a/libraries/libstratosphere/include/stratosphere/mem.hpp b/libraries/libstratosphere/include/stratosphere/mem.hpp new file mode 100644 index 000000000..c583d6e8c --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem.hpp @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include +#include diff --git a/libraries/libstratosphere/include/stratosphere/mem/impl/heap/mem_impl_heap_cached_heap.hpp b/libraries/libstratosphere/include/stratosphere/mem/impl/heap/mem_impl_heap_cached_heap.hpp new file mode 100644 index 000000000..8f0c5197e --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem/impl/heap/mem_impl_heap_cached_heap.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::mem::impl::heap { + + class TlsHeapCache; + + class CachedHeap final { + NON_COPYABLE(CachedHeap); + private: + TlsHeapCache *tls_heap_cache; + public: + constexpr CachedHeap() : tls_heap_cache() { /* ... */ } + ~CachedHeap() { this->Finalize(); } + + ALWAYS_INLINE CachedHeap(CachedHeap &&rhs) : tls_heap_cache(rhs.tls_heap_cache) { + rhs.tls_heap_cache = nullptr; + } + ALWAYS_INLINE CachedHeap &operator=(CachedHeap &&rhs) { + this->Reset(); + this->tls_heap_cache = rhs.tls_heap_cache; + rhs.tls_heap_cache = nullptr; + return *this; + } + + void *Allocate(size_t n); + void *Allocate(size_t n, size_t align); + size_t GetAllocationSize(const void *ptr); + errno_t Free(void *p); + errno_t FreeWithSize(void *p, size_t size); + errno_t Reallocate(void *ptr, size_t size, void **p); + errno_t Shrink(void *ptr, size_t size); + + void ReleaseAllCache(); + void Finalize(); + bool CheckCache(); + errno_t QueryV(int query, std::va_list vl); + errno_t Query(int query, ...); + + void Reset() { this->Finalize(); } + void Reset(TlsHeapCache *thc); + TlsHeapCache *Release(); + + constexpr explicit ALWAYS_INLINE operator bool() const { return this->tls_heap_cache != nullptr; } + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/mem/impl/heap/mem_impl_heap_central_heap.hpp b/libraries/libstratosphere/include/stratosphere/mem/impl/heap/mem_impl_heap_central_heap.hpp new file mode 100644 index 000000000..8b2e7d73d --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem/impl/heap/mem_impl_heap_central_heap.hpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::mem::impl::heap { + + class CachedHeap; + class TlsHeapCentral; + + using HeapWalkCallback = int (*)(void *ptr, size_t size, void *user_data); + + class CentralHeap final { + NON_COPYABLE(CentralHeap); + NON_MOVEABLE(CentralHeap); + public: + static constexpr size_t PageSize = 4_KB; + static constexpr size_t MinimumAlignment = alignof(u64); + using DestructorHandler = void (*)(void *start, void *end); + private: + TlsHeapCentral *tls_heap_central; + bool use_virtual_memory; + u32 option; + u8 *start; + u8 *end; + public: + constexpr CentralHeap() : tls_heap_central(), use_virtual_memory(), option(), start(), end() { /* ... */ } + ~CentralHeap() { this->Finalize(); } + + errno_t Initialize(void *start, size_t size, u32 option); + void Finalize(); + + ALWAYS_INLINE void *Allocate(size_t n) { return this->Allocate(n, MinimumAlignment); } + void *Allocate(size_t n, size_t align); + size_t GetAllocationSize(const void *ptr); + errno_t Free(void *p); + errno_t FreeWithSize(void *p, size_t size); + errno_t Reallocate(void *ptr, size_t size, void **p); + errno_t Shrink(void *ptr, size_t size); + + bool MakeCache(CachedHeap *cached_heap); + errno_t WalkAllocatedPointers(HeapWalkCallback callback, void *user_data); + errno_t QueryV(int query, std::va_list vl); + errno_t Query(int query, ...); + private: + errno_t QueryVImpl(int query, std::va_list *vl_ptr); + }; + + static_assert(sizeof(CentralHeap) <= sizeof(::ams::mem::impl::InternalCentralHeapStorage)); + static_assert(alignof(CentralHeap) <= alignof(void *)); + +} diff --git a/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_common.hpp b/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_common.hpp new file mode 100644 index 000000000..ff0bbaf56 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_common.hpp @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::mem::impl { + + constexpr inline size_t MaxSize = static_cast(std::numeric_limits::max()); + + using errno_t = int; + + enum DumpMode { + DumpMode_Basic = (0 << 0), + DumpMode_Spans = (1 << 0), + DumpMode_Pointers = (1 << 1), + DumpMode_Pages = (1 << 2), + DumpMode_All = (DumpMode_Pages | DumpMode_Pointers | DumpMode_Spans | DumpMode_Basic), + }; + + enum AllocQuery { + AllocQuery_Dump = 0, + AllocQuery_PageSize = 1, + AllocQuery_AllocatedSize = 2, + AllocQuery_FreeSize = 3, + AllocQuery_SystemSize = 4, + AllocQuery_MaxAllocatableSize = 5, + AllocQuery_IsClean = 6, + AllocQuery_HeapHash = 7, + AllocQuery_UnifyFreeList = 8, + AllocQuery_SetColor = 9, + AllocQuery_GetColor = 10, + AllocQuery_SetName = 11, + AllocQuery_GetName = 12, + /* AllocQuery_Thirteen = 13, */ + AllocQuery_CheckCache = 14, + AllocQuery_ClearCache = 15, + AllocQuery_FinalizeCache = 16, + AllocQuery_FreeSizeMapped = 17, + AllocQuery_MaxAllocatableSizeMapped = 18, + AllocQuery_DumpJson = 19, + }; + + enum HeapOption { + HeapOption_UseEnvironment = (1 << 0), + HeapOption_DisableCache = (1 << 2), + }; + + struct HeapHash { + size_t alloc_count; + size_t alloc_size; + size_t hash; + }; + static_assert(std::is_pod::value); + +} diff --git a/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_declarations.hpp b/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_declarations.hpp new file mode 100644 index 000000000..1784e05b2 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_declarations.hpp @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +namespace ams::mem::impl { + + namespace heap { + + class CentralHeap; + + } + + using InternalCentralHeapStorage = ::ams::util::TypedStorage<::ams::mem::impl::heap::CentralHeap, sizeof(void *) * 6, alignof(void *)>; + +} \ No newline at end of file diff --git a/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_heap.hpp b/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_heap.hpp new file mode 100644 index 000000000..0a918081a --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem/impl/mem_impl_heap.hpp @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include +#include +#include diff --git a/libraries/libstratosphere/include/stratosphere/mem/mem_standard_allocator.hpp b/libraries/libstratosphere/include/stratosphere/mem/mem_standard_allocator.hpp new file mode 100644 index 000000000..4a265c4ca --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/mem/mem_standard_allocator.hpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019-2020 Adubbz, Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include + +namespace ams::mem { + + class StandardAllocator { + NON_COPYABLE(StandardAllocator); + NON_MOVEABLE(StandardAllocator); + public: + using WalkCallback = int (*)(void *ptr, size_t size, void *user_data); + + struct AllocatorHash { + size_t allocated_count; + size_t allocated_size; + size_t hash; + }; + private: + bool initialized; + bool enable_thread_cache; + uintptr_t unused; + os::TlsSlot tls_slot; + impl::InternalCentralHeapStorage central_heap_storage; + public: + StandardAllocator(); + StandardAllocator(void *mem, size_t size); + StandardAllocator(void *mem, size_t size, bool enable_cache); + + ~StandardAllocator() { + if (this->initialized) { + this->Finalize(); + } + } + + void Initialize(void *mem, size_t size); + void Initialize(void *mem, size_t size, bool enable_cache); + void Finalize(); + + void *Allocate(size_t size); + void *Allocate(size_t size, size_t alignment); + void Free(void *ptr); + void *Reallocate(void *ptr, size_t new_size); + size_t Shrink(void *ptr, size_t new_size); + + void ClearThreadCache() const; + void CleanUpManagementArea() const; + + size_t GetSizeOf(const void *ptr) const; + size_t GetTotalFreeSize() const; + size_t GetAllocatableSize() const; + + void WalkAllocatedBlocks(WalkCallback callback, void *user_data) const; + + void Dump() const; + AllocatorHash Hash() const; + }; + +} diff --git a/libraries/libstratosphere/include/stratosphere/os.hpp b/libraries/libstratosphere/include/stratosphere/os.hpp index abd5e77d2..061da55e1 100644 --- a/libraries/libstratosphere/include/stratosphere/os.hpp +++ b/libraries/libstratosphere/include/stratosphere/os.hpp @@ -16,21 +16,25 @@ #pragma once -#include "os/os_common_types.hpp" -#include "os/os_memory_common.hpp" -#include "os/os_tick.hpp" -#include "os/os_managed_handle.hpp" -#include "os/os_process_handle.hpp" -#include "os/os_random.hpp" -#include "os/os_mutex.hpp" -#include "os/os_condvar.hpp" -#include "os/os_rw_lock.hpp" -#include "os/os_semaphore.hpp" -#include "os/os_timeout_helper.hpp" -#include "os/os_event.hpp" -#include "os/os_system_event.hpp" -#include "os/os_interrupt_event.hpp" -#include "os/os_thread.hpp" -#include "os/os_message_queue.hpp" -#include "os/os_waitable_holder.hpp" -#include "os/os_waitable_manager.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/libraries/libstratosphere/include/stratosphere/os/os_memory_common.hpp b/libraries/libstratosphere/include/stratosphere/os/os_memory_common.hpp index dcf1ad944..070e3a8ab 100644 --- a/libraries/libstratosphere/include/stratosphere/os/os_memory_common.hpp +++ b/libraries/libstratosphere/include/stratosphere/os/os_memory_common.hpp @@ -22,4 +22,12 @@ namespace ams::os { constexpr inline size_t MemoryBlockUnitSize = 0x200000; + enum MemoryPermission { + MemoryPermission_None = (0 << 0), + MemoryPermission_ReadOnly = (1 << 0), + MemoryPermission_WriteOnly = (1 << 1), + + MemoryPermission_ReadWrite = MemoryPermission_ReadOnly | MemoryPermission_WriteOnly, + }; + } diff --git a/libraries/libstratosphere/include/stratosphere/os/os_memory_heap_api.hpp b/libraries/libstratosphere/include/stratosphere/os/os_memory_heap_api.hpp new file mode 100644 index 000000000..fc051cd01 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_memory_heap_api.hpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::os { + + Result AllocateMemoryBlock(uintptr_t *out_address, size_t size); + void FreeMemoryBlock(uintptr_t address, size_t size); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_memory_permission.hpp b/libraries/libstratosphere/include/stratosphere/os/os_memory_permission.hpp new file mode 100644 index 000000000..588bba655 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_memory_permission.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::os { + + void SetMemoryPermission(uintptr_t address, size_t size, MemoryPermission perm); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_memory_virtual_address_api.hpp b/libraries/libstratosphere/include/stratosphere/os/os_memory_virtual_address_api.hpp new file mode 100644 index 000000000..be91e072f --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_memory_virtual_address_api.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include +#include + +namespace ams::os { + + bool IsVirtualAddressMemoryEnabled(); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_thread.hpp b/libraries/libstratosphere/include/stratosphere/os/os_thread.hpp index 4dab3d8eb..2e24d57be 100644 --- a/libraries/libstratosphere/include/stratosphere/os/os_thread.hpp +++ b/libraries/libstratosphere/include/stratosphere/os/os_thread.hpp @@ -101,10 +101,19 @@ namespace ams::os { } }; - NX_INLINE u32 GetCurrentThreadPriority() { - u32 prio; + ALWAYS_INLINE s32 GetCurrentThreadPriority() { + s32 prio; R_ABORT_UNLESS(svcGetThreadPriority(&prio, CUR_THREAD_HANDLE)); return prio; } + /* TODO: ThreadManager? */ + ALWAYS_INLINE s32 GetCurrentProcessorNumber() { + return svcGetCurrentProcessorNumber(); + } + + ALWAYS_INLINE s32 GetCurrentCoreNumber() { + return GetCurrentProcessorNumber(); + } + } diff --git a/libraries/libstratosphere/include/stratosphere/os/os_thread_local_storage_api.hpp b/libraries/libstratosphere/include/stratosphere/os/os_thread_local_storage_api.hpp new file mode 100644 index 000000000..0d6cd2a2a --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_thread_local_storage_api.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include +#include + +namespace ams::os { + + Result AllocateTlsSlot(TlsSlot *out, TlsDestructor destructor); + + void FreeTlsSlot(TlsSlot slot); + + uintptr_t GetTlsValue(TlsSlot slot); + void SetTlsValue(TlsSlot slot, uintptr_t value); + +} diff --git a/libraries/libstratosphere/include/stratosphere/os/os_thread_local_storage_common.hpp b/libraries/libstratosphere/include/stratosphere/os/os_thread_local_storage_common.hpp new file mode 100644 index 000000000..055ce4bc0 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/os/os_thread_local_storage_common.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include + +namespace ams::os { + + struct TlsSlot { + u32 _value; + }; + + using TlsDestructor = void (*)(uintptr_t arg); + + constexpr inline size_t TlsSlotCountMax = 16; + constexpr inline size_t SdkTlsSlotCountMax = 16; + +} diff --git a/libraries/libstratosphere/include/stratosphere/svc/svc_stratosphere_shims.hpp b/libraries/libstratosphere/include/stratosphere/svc/svc_stratosphere_shims.hpp new file mode 100644 index 000000000..e519c4754 --- /dev/null +++ b/libraries/libstratosphere/include/stratosphere/svc/svc_stratosphere_shims.hpp @@ -0,0 +1,491 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#if defined(ATMOSPHERE_BOARD_NINTENDO_NX) && defined(ATMOSPHERE_ARCH_ARM64) + +namespace ams::svc::aarch64::lp64 { + + ALWAYS_INLINE Result SetHeapSize(::ams::svc::Address *out_address, ::ams::svc::Size size) { + return ::svcSetHeapSize(reinterpret_cast(out_address), size); + } + + ALWAYS_INLINE Result SetMemoryPermission(::ams::svc::Address address, ::ams::svc::Size size, ::ams::svc::MemoryPermission perm) { + return ::svcSetMemoryPermission(reinterpret_cast(static_cast(address)), size, static_cast(perm)); + } + + ALWAYS_INLINE Result SetMemoryAttribute(::ams::svc::Address address, ::ams::svc::Size size, uint32_t mask, uint32_t attr) { + return ::svcSetMemoryAttribute(reinterpret_cast(static_cast(address)), size, mask, attr); + } + + ALWAYS_INLINE Result MapMemory(::ams::svc::Address dst_address, ::ams::svc::Address src_address, ::ams::svc::Size size) { + return ::svcMapMemory(reinterpret_cast(static_cast(dst_address)), reinterpret_cast(static_cast(src_address)), size); + } + + ALWAYS_INLINE Result UnmapMemory(::ams::svc::Address dst_address, ::ams::svc::Address src_address, ::ams::svc::Size size) { + return ::svcUnmapMemory(reinterpret_cast(static_cast(dst_address)), reinterpret_cast(static_cast(src_address)), size); + } + + ALWAYS_INLINE Result QueryMemory(::ams::svc::UserPointer< ::ams::svc::lp64::MemoryInfo *> out_memory_info, ::ams::svc::PageInfo *out_page_info, ::ams::svc::Address address) { + return ::svcQueryMemory(reinterpret_cast<::MemoryInfo *>(out_memory_info.GetPointerUnsafe()), reinterpret_cast(out_page_info), address); + } + + ALWAYS_INLINE void ExitProcess() { + return ::svcExitProcess(); + } + + ALWAYS_INLINE Result CreateThread(::ams::svc::Handle *out_handle, ::ams::svc::ThreadFunc func, ::ams::svc::Address arg, ::ams::svc::Address stack_bottom, int32_t priority, int32_t core_id) { + return ::svcCreateThread(out_handle, reinterpret_cast(static_cast(func)), reinterpret_cast(static_cast(arg)), reinterpret_cast(static_cast(stack_bottom)), priority, core_id); + } + + ALWAYS_INLINE Result StartThread(::ams::svc::Handle thread_handle) { + return ::svcStartThread(thread_handle); + } + + ALWAYS_INLINE void ExitThread() { + return ::svcExitThread(); + } + + ALWAYS_INLINE void SleepThread(int64_t ns) { + return ::svcSleepThread(ns); + } + + ALWAYS_INLINE Result GetThreadPriority(int32_t *out_priority, ::ams::svc::Handle thread_handle) { + return ::svcGetThreadPriority(out_priority, thread_handle); + } + + ALWAYS_INLINE Result SetThreadPriority(::ams::svc::Handle thread_handle, int32_t priority) { + return ::svcSetThreadPriority(thread_handle, priority); + } + + ALWAYS_INLINE Result GetThreadCoreMask(int32_t *out_core_id, uint64_t *out_affinity_mask, ::ams::svc::Handle thread_handle) { + return ::svcGetThreadCoreMask(out_core_id, out_affinity_mask, thread_handle); + } + + ALWAYS_INLINE Result SetThreadCoreMask(::ams::svc::Handle thread_handle, int32_t core_id, uint64_t affinity_mask) { + return ::svcSetThreadCoreMask(thread_handle, core_id, affinity_mask); + } + + ALWAYS_INLINE int32_t GetCurrentProcessorNumber() { + return ::svcGetCurrentProcessorNumber(); + } + + ALWAYS_INLINE Result SignalEvent(::ams::svc::Handle event_handle) { + return ::svcSignalEvent(event_handle); + } + + ALWAYS_INLINE Result ClearEvent(::ams::svc::Handle event_handle) { + return ::svcClearEvent(event_handle); + } + + ALWAYS_INLINE Result MapSharedMemory(::ams::svc::Handle shmem_handle, ::ams::svc::Address address, ::ams::svc::Size size, ::ams::svc::MemoryPermission map_perm) { + return ::svcMapSharedMemory(shmem_handle, reinterpret_cast(static_cast(address)), size, static_cast(map_perm)); + } + + ALWAYS_INLINE Result UnmapSharedMemory(::ams::svc::Handle shmem_handle, ::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcUnmapSharedMemory(shmem_handle, reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result CreateTransferMemory(::ams::svc::Handle *out_handle, ::ams::svc::Address address, ::ams::svc::Size size, ::ams::svc::MemoryPermission map_perm) { + return ::svcCreateTransferMemory(out_handle, reinterpret_cast(static_cast(address)), size, static_cast(map_perm)); + } + + ALWAYS_INLINE Result CloseHandle(::ams::svc::Handle handle) { + return ::svcCloseHandle(handle); + } + + ALWAYS_INLINE Result ResetSignal(::ams::svc::Handle handle) { + return ::svcResetSignal(handle); + } + + ALWAYS_INLINE Result WaitSynchronization(int32_t *out_index, ::ams::svc::UserPointer handles, int32_t numHandles, int64_t timeout_ns) { + return ::svcWaitSynchronization(out_index, handles.GetPointerUnsafe(), numHandles, timeout_ns); + } + + ALWAYS_INLINE Result CancelSynchronization(::ams::svc::Handle handle) { + return ::svcCancelSynchronization(handle); + } + + ALWAYS_INLINE Result ArbitrateLock(::ams::svc::Handle thread_handle, ::ams::svc::Address address, uint32_t tag) { + return ::svcArbitrateLock(thread_handle, reinterpret_cast(static_cast(address)), tag); + } + + ALWAYS_INLINE Result ArbitrateUnlock(::ams::svc::Address address) { + return ::svcArbitrateUnlock(reinterpret_cast(static_cast(address))); + } + + ALWAYS_INLINE Result WaitProcessWideKeyAtomic(::ams::svc::Address address, ::ams::svc::Address cv_key, uint32_t tag, int64_t timeout_ns) { + return ::svcWaitProcessWideKeyAtomic(reinterpret_cast(static_cast(address)), reinterpret_cast(static_cast(cv_key)), tag, timeout_ns); + } + + ALWAYS_INLINE void SignalProcessWideKey(::ams::svc::Address cv_key, int32_t count) { + return ::svcSignalProcessWideKey(reinterpret_cast(static_cast(cv_key)), count); + } + + ALWAYS_INLINE int64_t GetSystemTick() { + return ::svcGetSystemTick(); + } + + ALWAYS_INLINE Result ConnectToNamedPort(::ams::svc::Handle *out_handle, ::ams::svc::UserPointer name) { + return ::svcConnectToNamedPort(out_handle, name.GetPointerUnsafe()); + } + + ALWAYS_INLINE Result SendSyncRequestLight(::ams::svc::Handle session_handle) { + return ::svcSendSyncRequestLight(session_handle); + } + + ALWAYS_INLINE Result SendSyncRequest(::ams::svc::Handle session_handle) { + return ::svcSendSyncRequest(session_handle); + } + + ALWAYS_INLINE Result SendSyncRequestWithUserBuffer(::ams::svc::Address message_buffer, ::ams::svc::Size message_buffer_size, ::ams::svc::Handle session_handle) { + return ::svcSendSyncRequestWithUserBuffer(reinterpret_cast(static_cast(message_buffer)), message_buffer_size, session_handle); + } + + ALWAYS_INLINE Result SendAsyncRequestWithUserBuffer(::ams::svc::Handle *out_event_handle, ::ams::svc::Address message_buffer, ::ams::svc::Size message_buffer_size, ::ams::svc::Handle session_handle) { + return ::svcSendAsyncRequestWithUserBuffer(out_event_handle, reinterpret_cast(static_cast(message_buffer)), message_buffer_size, session_handle); + } + + ALWAYS_INLINE Result GetProcessId(uint64_t *out_process_id, ::ams::svc::Handle process_handle) { + return ::svcGetProcessId(out_process_id, process_handle); + } + + ALWAYS_INLINE Result GetThreadId(uint64_t *out_thread_id, ::ams::svc::Handle thread_handle) { + return ::svcGetThreadId(out_thread_id, thread_handle); + } + + ALWAYS_INLINE void Break(::ams::svc::BreakReason break_reason, ::ams::svc::Address arg, ::ams::svc::Size size) { + ::svcBreak(break_reason, arg, size); + } + + ALWAYS_INLINE Result OutputDebugString(::ams::svc::UserPointer debug_str, ::ams::svc::Size len) { + return ::svcOutputDebugString(debug_str.GetPointerUnsafe(), len); + } + + ALWAYS_INLINE void ReturnFromException(::ams::Result result) { + return ::svcReturnFromException(result.GetValue()); + } + + ALWAYS_INLINE Result GetInfo(uint64_t *out, ::ams::svc::InfoType info_type, ::ams::svc::Handle handle, uint64_t info_subtype) { + return ::svcGetInfo(out, static_cast(info_type), handle, info_subtype); + } + + ALWAYS_INLINE void FlushEntireDataCache() { + return ::svcFlushEntireDataCache(); + } + + ALWAYS_INLINE Result FlushDataCache(::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcFlushDataCache(reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result MapPhysicalMemory(::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcMapPhysicalMemory(reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result UnmapPhysicalMemory(::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcUnmapPhysicalMemory(reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result GetDebugFutureThreadInfo(::ams::svc::lp64::LastThreadContext *out_context, uint64_t *thread_id, ::ams::svc::Handle debug_handle, int64_t ns) { + return ::svcGetDebugFutureThreadInfo(reinterpret_cast<::LastThreadContext *>(out_context), thread_id, debug_handle, ns); + } + + ALWAYS_INLINE Result GetLastThreadInfo(::ams::svc::lp64::LastThreadContext *out_context, ::ams::svc::Address *out_tls_address, uint32_t *out_flags) { + return ::svcGetLastThreadInfo(reinterpret_cast<::LastThreadContext *>(out_context), reinterpret_cast(out_tls_address), out_flags); + } + + ALWAYS_INLINE Result GetResourceLimitLimitValue(int64_t *out_limit_value, ::ams::svc::Handle resource_limit_handle, ::ams::svc::LimitableResource which) { + return ::svcGetResourceLimitLimitValue(out_limit_value, resource_limit_handle, static_cast<::LimitableResource>(which)); + } + + ALWAYS_INLINE Result GetResourceLimitCurrentValue(int64_t *out_current_value, ::ams::svc::Handle resource_limit_handle, ::ams::svc::LimitableResource which) { + return ::svcGetResourceLimitCurrentValue(out_current_value, resource_limit_handle, static_cast<::LimitableResource>(which)); + } + + ALWAYS_INLINE Result SetThreadActivity(::ams::svc::Handle thread_handle, ::ams::svc::ThreadActivity thread_activity) { + return ::svcSetThreadActivity(thread_handle, static_cast<::ThreadActivity>(thread_activity)); + } + + ALWAYS_INLINE Result GetThreadContext3(::ams::svc::UserPointer< ::ams::svc::ThreadContext *> out_context, ::ams::svc::Handle thread_handle) { + return ::svcGetThreadContext3(reinterpret_cast<::ThreadContext *>(out_context.GetPointerUnsafe()), thread_handle); + } + + ALWAYS_INLINE Result WaitForAddress(::ams::svc::Address address, ::ams::svc::ArbitrationType arb_type, int32_t value, int64_t timeout_ns) { + return ::svcWaitForAddress(reinterpret_cast(static_cast(address)), arb_type, value, timeout_ns); + } + + ALWAYS_INLINE Result SignalToAddress(::ams::svc::Address address, ::ams::svc::SignalType signal_type, int32_t value, int32_t count) { + return ::svcSignalToAddress(reinterpret_cast(static_cast(address)), signal_type, value, count); + } + + ALWAYS_INLINE void SynchronizePreemptionState() { + return ::svcSynchronizePreemptionState(); + } + + ALWAYS_INLINE void KernelDebug(::ams::svc::KernelDebugType kern_debug_type, uint64_t arg0, uint64_t arg1, uint64_t arg2) { + return ::svcKernelDebug(kern_debug_type, arg0, arg1, arg2); + } + + ALWAYS_INLINE void ChangeKernelTraceState(::ams::svc::KernelTraceState kern_trace_state) { + return ::svcChangeKernelTraceState(kern_trace_state); + } + + ALWAYS_INLINE Result CreateSession(::ams::svc::Handle *out_server_session_handle, ::ams::svc::Handle *out_client_session_handle, bool is_light, ::ams::svc::Address name) { + return ::svcCreateSession(out_server_session_handle, out_client_session_handle, is_light, name); + } + + ALWAYS_INLINE Result AcceptSession(::ams::svc::Handle *out_handle, ::ams::svc::Handle port) { + return ::svcAcceptSession(out_handle, port); + } + + ALWAYS_INLINE Result ReplyAndReceiveLight(::ams::svc::Handle handle) { + return ::svcReplyAndReceiveLight(handle); + } + + ALWAYS_INLINE Result ReplyAndReceive(int32_t *out_index, ::ams::svc::UserPointer handles, int32_t num_handles, ::ams::svc::Handle reply_target, int64_t timeout_ns) { + return ::svcReplyAndReceive(out_index, handles.GetPointerUnsafe(), num_handles, reply_target, timeout_ns); + } + + ALWAYS_INLINE Result ReplyAndReceiveWithUserBuffer(int32_t *out_index, ::ams::svc::Address message_buffer, ::ams::svc::Size message_buffer_size, ::ams::svc::UserPointer handles, int32_t num_handles, ::ams::svc::Handle reply_target, int64_t timeout_ns) { + return ::svcReplyAndReceiveWithUserBuffer(out_index, reinterpret_cast(static_cast(message_buffer)), message_buffer_size, handles.GetPointerUnsafe(), num_handles, reply_target, timeout_ns); + } + + ALWAYS_INLINE Result CreateEvent(::ams::svc::Handle *out_write_handle, ::ams::svc::Handle *out_read_handle) { + return ::svcCreateEvent(out_write_handle, out_read_handle); + } + + ALWAYS_INLINE Result MapPhysicalMemoryUnsafe(::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcMapPhysicalMemoryUnsafe(reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result UnmapPhysicalMemoryUnsafe(::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcUnmapPhysicalMemoryUnsafe(reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result SetUnsafeLimit(::ams::svc::Size limit) { + return ::svcSetUnsafeLimit(limit); + } + + ALWAYS_INLINE Result CreateCodeMemory(::ams::svc::Handle *out_handle, ::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcCreateCodeMemory(out_handle, reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result ControlCodeMemory(::ams::svc::Handle code_memory_handle, ::ams::svc::CodeMemoryOperation operation, uint64_t address, uint64_t size, ::ams::svc::MemoryPermission perm) { + return ::svcControlCodeMemory(code_memory_handle, static_cast<::CodeMapOperation>(operation), reinterpret_cast(address), size, static_cast(perm)); + } + + ALWAYS_INLINE void SleepSystem() { + return ::svcSleepSystem(); + } + + ALWAYS_INLINE Result ReadWriteRegister(uint32_t *out_value, ::ams::svc::PhysicalAddress address, uint32_t mask, uint32_t value) { + return ::svcReadWriteRegister(out_value, address, mask, value); + } + + ALWAYS_INLINE Result SetProcessActivity(::ams::svc::Handle process_handle, ::ams::svc::ProcessActivity process_activity) { + return ::svcSetProcessActivity(process_handle, static_cast<::ProcessActivity>(process_activity)); + } + + ALWAYS_INLINE Result CreateSharedMemory(::ams::svc::Handle *out_handle, ::ams::svc::Size size, ::ams::svc::MemoryPermission owner_perm, ::ams::svc::MemoryPermission remote_perm) { + return ::svcCreateSharedMemory(out_handle, size, static_cast(owner_perm), static_cast(remote_perm)); + } + + ALWAYS_INLINE Result MapTransferMemory(::ams::svc::Handle trmem_handle, ::ams::svc::Address address, ::ams::svc::Size size, ::ams::svc::MemoryPermission owner_perm) { + return ::svcMapTransferMemory(trmem_handle, reinterpret_cast(static_cast(address)), size, static_cast(owner_perm)); + } + + ALWAYS_INLINE Result UnmapTransferMemory(::ams::svc::Handle trmem_handle, ::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcUnmapTransferMemory(trmem_handle, reinterpret_cast(static_cast(address)), size); + } + + ALWAYS_INLINE Result CreateInterruptEvent(::ams::svc::Handle *out_read_handle, int32_t interrupt_id, ::ams::svc::InterruptType interrupt_type) { + return ::svcCreateInterruptEvent(out_read_handle, interrupt_id, static_cast(interrupt_type)); + } + + ALWAYS_INLINE Result QueryPhysicalAddress(::ams::svc::lp64::PhysicalMemoryInfo *out_info, ::ams::svc::Address address) { + return ::svcQueryPhysicalAddress(reinterpret_cast<::PhysicalMemoryInfo *>(out_info), address); + } + + ALWAYS_INLINE Result QueryIoMapping(::ams::svc::Address *out_address, ::ams::svc::PhysicalAddress physical_address, ::ams::svc::Size size) { + return ::svcQueryIoMapping(reinterpret_cast(out_address), physical_address, size); + } + + ALWAYS_INLINE Result CreateDeviceAddressSpace(::ams::svc::Handle *out_handle, uint64_t das_address, uint64_t das_size) { + return ::svcCreateDeviceAddressSpace(out_handle, das_address, das_size); + } + + ALWAYS_INLINE Result AttachDeviceAddressSpace(::ams::svc::DeviceName device_name, ::ams::svc::Handle das_handle) { + return ::svcAttachDeviceAddressSpace(static_cast(device_name), das_handle); + } + + ALWAYS_INLINE Result DetachDeviceAddressSpace(::ams::svc::DeviceName device_name, ::ams::svc::Handle das_handle) { + return ::svcDetachDeviceAddressSpace(static_cast(device_name), das_handle); + } + + ALWAYS_INLINE Result MapDeviceAddressSpaceByForce(::ams::svc::Handle das_handle, ::ams::svc::Handle process_handle, uint64_t process_address, ::ams::svc::Size size, uint64_t device_address, ::ams::svc::MemoryPermission device_perm) { + return ::svcMapDeviceAddressSpaceByForce(das_handle, process_handle, process_address, size, device_address, static_cast(device_perm)); + } + + ALWAYS_INLINE Result MapDeviceAddressSpaceAligned(::ams::svc::Handle das_handle, ::ams::svc::Handle process_handle, uint64_t process_address, ::ams::svc::Size size, uint64_t device_address, ::ams::svc::MemoryPermission device_perm) { + return ::svcMapDeviceAddressSpaceAligned(das_handle, process_handle, process_address, size, device_address, static_cast(device_perm)); + } + + ALWAYS_INLINE Result MapDeviceAddressSpace(::ams::svc::Size *out_mapped_size, ::ams::svc::Handle das_handle, ::ams::svc::Handle process_handle, uint64_t process_address, ::ams::svc::Size size, uint64_t device_address, ::ams::svc::MemoryPermission device_perm) { + return ::svcMapDeviceAddressSpace(reinterpret_cast(out_mapped_size), das_handle, process_handle, process_address, size, device_address, static_cast(device_perm)); + } + + ALWAYS_INLINE Result UnmapDeviceAddressSpace(::ams::svc::Handle das_handle, ::ams::svc::Handle process_handle, uint64_t process_address, ::ams::svc::Size size, uint64_t device_address) { + return ::svcUnmapDeviceAddressSpace(das_handle, process_handle, process_address, size, device_address); + } + + ALWAYS_INLINE Result InvalidateProcessDataCache(::ams::svc::Handle process_handle, uint64_t address, uint64_t size) { + return ::svcInvalidateProcessDataCache(process_handle, address, size); + } + + ALWAYS_INLINE Result StoreProcessDataCache(::ams::svc::Handle process_handle, uint64_t address, uint64_t size) { + return ::svcStoreProcessDataCache(process_handle, address, size); + } + + ALWAYS_INLINE Result FlushProcessDataCache(::ams::svc::Handle process_handle, uint64_t address, uint64_t size) { + return ::svcFlushProcessDataCache(process_handle, address, size); + } + + ALWAYS_INLINE Result DebugActiveProcess(::ams::svc::Handle *out_handle, uint64_t process_id) { + return ::svcDebugActiveProcess(out_handle, process_id); + } + + ALWAYS_INLINE Result BreakDebugProcess(::ams::svc::Handle debug_handle) { + return ::svcBreakDebugProcess(debug_handle); + } + + ALWAYS_INLINE Result TerminateDebugProcess(::ams::svc::Handle debug_handle) { + return ::svcTerminateDebugProcess(debug_handle); + } + + ALWAYS_INLINE Result GetDebugEvent(::ams::svc::UserPointer< ::ams::svc::lp64::DebugEventInfo *> out_info, ::ams::svc::Handle debug_handle) { + return ::svcGetDebugEvent(out_info.GetPointerUnsafe(), debug_handle); + } + + ALWAYS_INLINE Result ContinueDebugEvent(::ams::svc::Handle debug_handle, uint32_t flags, ::ams::svc::UserPointer thread_ids, int32_t num_thread_ids) { + return ::svcContinueDebugEvent(debug_handle, flags, const_cast(thread_ids.GetPointerUnsafe()), num_thread_ids); + } + + ALWAYS_INLINE Result GetProcessList(int32_t *out_num_processes, ::ams::svc::UserPointer out_process_ids, int32_t max_out_count) { + return ::svcGetProcessList(out_num_processes, out_process_ids.GetPointerUnsafe(), max_out_count); + } + + ALWAYS_INLINE Result GetThreadList(int32_t *out_num_threads, ::ams::svc::UserPointer out_thread_ids, int32_t max_out_count, ::ams::svc::Handle debug_handle) { + return ::svcGetThreadList(out_num_threads, out_thread_ids.GetPointerUnsafe(), max_out_count, debug_handle); + } + + ALWAYS_INLINE Result GetDebugThreadContext(::ams::svc::UserPointer< ::ams::svc::ThreadContext *> out_context, ::ams::svc::Handle debug_handle, uint64_t thread_id, uint32_t context_flags) { + return ::svcGetDebugThreadContext(reinterpret_cast<::ThreadContext *>(out_context.GetPointerUnsafe()), debug_handle, thread_id, context_flags); + } + + ALWAYS_INLINE Result SetDebugThreadContext(::ams::svc::Handle debug_handle, uint64_t thread_id, ::ams::svc::UserPointer context, uint32_t context_flags) { + return ::svcSetDebugThreadContext(debug_handle, thread_id, reinterpret_cast(context.GetPointerUnsafe()), context_flags); + } + + ALWAYS_INLINE Result QueryDebugProcessMemory(::ams::svc::UserPointer< ::ams::svc::lp64::MemoryInfo *> out_memory_info, ::ams::svc::PageInfo *out_page_info, ::ams::svc::Handle process_handle, ::ams::svc::Address address) { + return ::svcQueryDebugProcessMemory(reinterpret_cast<::MemoryInfo *>(out_memory_info.GetPointerUnsafe()), reinterpret_cast(out_page_info), process_handle, address); + } + + ALWAYS_INLINE Result ReadDebugProcessMemory(::ams::svc::Address buffer, ::ams::svc::Handle debug_handle, ::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcReadDebugProcessMemory(reinterpret_cast(static_cast(buffer)), debug_handle, address, size); + } + + ALWAYS_INLINE Result WriteDebugProcessMemory(::ams::svc::Handle debug_handle, ::ams::svc::Address buffer, ::ams::svc::Address address, ::ams::svc::Size size) { + return ::svcWriteDebugProcessMemory(debug_handle, reinterpret_cast(static_cast(buffer)), address, size); + } + + ALWAYS_INLINE Result SetHardwareBreakPoint(::ams::svc::HardwareBreakPointRegisterName name, uint64_t flags, uint64_t value) { + return ::svcSetHardwareBreakPoint(static_cast(name), flags, value); + } + + ALWAYS_INLINE Result GetDebugThreadParam(uint64_t *out_64, uint32_t *out_32, ::ams::svc::Handle debug_handle, uint64_t thread_id, ::ams::svc::DebugThreadParam param) { + return ::svcGetDebugThreadParam(out_64, out_32, debug_handle, thread_id, static_cast<::DebugThreadParam>(param)); + } + + ALWAYS_INLINE Result GetSystemInfo(uint64_t *out, ::ams::svc::SystemInfoType info_type, ::ams::svc::Handle handle, uint64_t info_subtype) { + return ::svcGetSystemInfo(out, static_cast(info_type), handle, info_subtype); + } + + ALWAYS_INLINE Result CreatePort(::ams::svc::Handle *out_server_handle, ::ams::svc::Handle *out_client_handle, int32_t max_sessions, bool is_light, ::ams::svc::Address name) { + return ::svcCreatePort(out_server_handle, out_client_handle, max_sessions, is_light, reinterpret_cast(static_cast(name))); + } + + ALWAYS_INLINE Result ManageNamedPort(::ams::svc::Handle *out_server_handle, ::ams::svc::UserPointer name, int32_t max_sessions) { + return ::svcManageNamedPort(out_server_handle, name.GetPointerUnsafe(), max_sessions); + } + + ALWAYS_INLINE Result ConnectToPort(::ams::svc::Handle *out_handle, ::ams::svc::Handle port) { + return ::svcConnectToPort(out_handle, port); + } + + ALWAYS_INLINE Result SetProcessMemoryPermission(::ams::svc::Handle process_handle, uint64_t address, uint64_t size, ::ams::svc::MemoryPermission perm) { + return ::svcSetProcessMemoryPermission(process_handle, address, size, static_cast(perm)); + } + + ALWAYS_INLINE Result MapProcessMemory(::ams::svc::Address dst_address, ::ams::svc::Handle process_handle, uint64_t src_address, ::ams::svc::Size size) { + return ::svcMapProcessMemory(reinterpret_cast(static_cast(dst_address)), process_handle, src_address, size); + } + + ALWAYS_INLINE Result UnmapProcessMemory(::ams::svc::Address dst_address, ::ams::svc::Handle process_handle, uint64_t src_address, ::ams::svc::Size size) { + return ::svcUnmapProcessMemory(reinterpret_cast(static_cast(dst_address)), process_handle, src_address, size); + } + + ALWAYS_INLINE Result QueryProcessMemory(::ams::svc::UserPointer< ::ams::svc::lp64::MemoryInfo *> out_memory_info, ::ams::svc::PageInfo *out_page_info, ::ams::svc::Handle process_handle, uint64_t address) { + return ::svcQueryProcessMemory(reinterpret_cast<::MemoryInfo *>(out_memory_info.GetPointerUnsafe()), reinterpret_cast(out_page_info), process_handle, address); + } + + ALWAYS_INLINE Result MapProcessCodeMemory(::ams::svc::Handle process_handle, uint64_t dst_address, uint64_t src_address, uint64_t size) { + return ::svcMapProcessCodeMemory(process_handle, dst_address, src_address, size); + } + + ALWAYS_INLINE Result UnmapProcessCodeMemory(::ams::svc::Handle process_handle, uint64_t dst_address, uint64_t src_address, uint64_t size) { + return ::svcUnmapProcessCodeMemory(process_handle, dst_address, src_address, size); + } + + ALWAYS_INLINE Result CreateProcess(::ams::svc::Handle *out_handle, ::ams::svc::UserPointer parameters, ::ams::svc::UserPointer caps, int32_t num_caps) { + return ::svcCreateProcess(out_handle, parameters.GetPointerUnsafe(), caps.GetPointerUnsafe(), num_caps); + } + + ALWAYS_INLINE Result StartProcess(::ams::svc::Handle process_handle, int32_t priority, int32_t core_id, uint64_t main_thread_stack_size) { + return ::svcStartProcess(process_handle, priority, core_id, main_thread_stack_size); + } + + ALWAYS_INLINE Result TerminateProcess(::ams::svc::Handle process_handle) { + return ::svcTerminateProcess(process_handle); + } + + ALWAYS_INLINE Result GetProcessInfo(int64_t *out_info, ::ams::svc::Handle process_handle, ::ams::svc::ProcessInfoType info_type) { + return ::svcGetProcessInfo(out_info, process_handle, static_cast<::ProcessInfoType>(info_type)); + } + + ALWAYS_INLINE Result CreateResourceLimit(::ams::svc::Handle *out_handle) { + return ::svcCreateResourceLimit(out_handle); + } + + ALWAYS_INLINE Result SetResourceLimitLimitValue(::ams::svc::Handle resource_limit_handle, ::ams::svc::LimitableResource which, int64_t limit_value) { + return ::svcSetResourceLimitLimitValue(resource_limit_handle, static_cast<::LimitableResource>(which), limit_value); + } + + ALWAYS_INLINE void CallSecureMonitor(::ams::svc::lp64::SecureMonitorArguments *args) { + ::svcCallSecureMonitor(reinterpret_cast<::SecmonArgs *>(args)); + } + +} + +#endif \ No newline at end of file diff --git a/libraries/libstratosphere/source/diag/diag_assertion_impl.cpp b/libraries/libstratosphere/source/diag/diag_assertion_impl.cpp index 33fb125c8..ca66e9a1a 100644 --- a/libraries/libstratosphere/source/diag/diag_assertion_impl.cpp +++ b/libraries/libstratosphere/source/diag/diag_assertion_impl.cpp @@ -41,7 +41,7 @@ namespace ams::diag { __builtin_unreachable(); } - ALWAYS_INLINE void DebugLog(const char *format, ...) __attribute__((format(printf, 1, 2))); + inline void DebugLog(const char *format, ...) __attribute__((format(printf, 1, 2))); #ifdef AMS_ENABLE_DEBUG_PRINT os::Mutex g_debug_log_lock; @@ -55,7 +55,7 @@ namespace ams::diag { svc::OutputDebugString(g_debug_buffer, strlen(g_debug_buffer)); } - void DebugLog(const char *format, ...) __attribute__((format(printf, 1, 2))) { + void DebugLog(const char *format, ...) { ::std::va_list vl; va_start(vl, format); DebugLogImpl(format, vl); diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_cached_heap.cpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_cached_heap.cpp new file mode 100644 index 000000000..7f70a68ba --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_cached_heap.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "mem_impl_heap_platform.hpp" +#include "mem_impl_heap_tls_heap_static.hpp" +#include "mem_impl_heap_tls_heap_central.hpp" + +namespace ams::mem::impl::heap { + + void *CachedHeap::Allocate(size_t n) { + return this->tls_heap_cache->Allocate(n); + } + + void *CachedHeap::Allocate(size_t n, size_t align) { + return this->tls_heap_cache->Allocate(n, align); + } + + size_t CachedHeap::GetAllocationSize(const void *ptr) { + return this->tls_heap_cache->GetAllocationSize(ptr); + } + + errno_t CachedHeap::Free(void *p) { + return this->tls_heap_cache->Free(p); + } + + errno_t CachedHeap::FreeWithSize(void *p, size_t size) { + return this->tls_heap_cache->FreeWithSize(p, size); + } + + errno_t CachedHeap::Reallocate(void *ptr, size_t size, void **p) { + return this->tls_heap_cache->Reallocate(ptr, size, p); + } + + errno_t CachedHeap::Shrink(void *ptr, size_t size) { + return this->tls_heap_cache->Shrink(ptr, size); + } + + void CachedHeap::ReleaseAllCache() { + if (this->tls_heap_cache) { + this->tls_heap_cache->ReleaseAllCache(); + } + } + + void CachedHeap::Finalize() { + if (this->tls_heap_cache) { + this->tls_heap_cache->Finalize(); + this->tls_heap_cache = nullptr; + } + } + + bool CachedHeap::CheckCache() { + bool cache = false; + auto err = this->Query(AllocQuery_CheckCache, std::addressof(cache)); + AMS_ASSERT(err != 0); + return cache; + } + + errno_t CachedHeap::QueryV(int _query, std::va_list vl) { + const AllocQuery query = static_cast(_query); + switch (query) { + case AllocQuery_CheckCache: + { + bool *out = va_arg(vl, bool *); + if (out) { + *out = (this->tls_heap_cache == nullptr) || this->tls_heap_cache->CheckCache(); + } + return 0; + } + case AllocQuery_ClearCache: + { + this->ReleaseAllCache(); + return 0; + } + case AllocQuery_FinalizeCache: + { + this->Finalize(); + return 0; + } + default: + return EINVAL; + } + } + + errno_t CachedHeap::Query(int query, ...) { + std::va_list vl; + va_start(vl, query); + auto err = this->QueryV(query, vl); + va_end(vl); + return err; + } + + void CachedHeap::Reset(TlsHeapCache *thc) { + this->Finalize(); + this->tls_heap_cache = thc; + } + + TlsHeapCache *CachedHeap::Release() { + TlsHeapCache *ret = this->tls_heap_cache; + this->tls_heap_cache = nullptr; + return ret; + } + +} diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_central_heap.cpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_central_heap.cpp new file mode 100644 index 000000000..4685dbf4a --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_central_heap.cpp @@ -0,0 +1,409 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "mem_impl_heap_platform.hpp" +#include "mem_impl_heap_tls_heap_static.hpp" +#include "mem_impl_heap_tls_heap_central.hpp" + +namespace ams::mem::impl::heap { + + errno_t CentralHeap::Initialize(void *start, size_t size, u32 option) { + /* Validate size. */ + if (size == 0 || !util::IsAligned(size, PageSize)) { + return EINVAL; + } + + /* Don't allow initializing twice. */ + if (this->start) { + return EEXIST; + } + + if (start) { + /* We were provided with a region to use as backing memory. */ + u8 *aligned_start = reinterpret_cast(util::AlignUp(reinterpret_cast(start), PageSize)); + u8 *aligned_end = reinterpret_cast(util::AlignDown(reinterpret_cast(start) + size, PageSize)); + if (aligned_start >= aligned_end) { + return EINVAL; + } + + this->start = aligned_start; + this->end = aligned_end; + this->option = option; + this->tls_heap_central = new (this->start) TlsHeapCentral; + if (auto err = this->tls_heap_central->Initialize(this->start, this->end - this->start, false); err != 0) { + this->tls_heap_central->~TlsHeapCentral(); + this->tls_heap_central = nullptr; + AMS_ASSERT(err == 0); + return err; + } + this->use_virtual_memory = false; + } else { + /* We were not provided with a region to use as backing. */ + void *mem; + if (auto err = AllocateVirtualMemory(std::addressof(mem), size); err != 0) { + return err; + } + if (!util::IsAligned(reinterpret_cast(mem), PageSize)) { + FreeVirtualMemory(mem, size); + size += PageSize; + if (auto err = AllocateVirtualMemory(std::addressof(mem), size); err != 0) { + return err; + } + } + this->start = static_cast(mem); + this->end = this->start + size; + this->option = option; + void *central = reinterpret_cast(util::AlignUp(reinterpret_cast(mem), PageSize)); + if (auto err = AllocatePhysicalMemory(central, sizeof(TlsHeapCentral)); err != 0) { + return err; + } + this->tls_heap_central = new (central) TlsHeapCentral; + if (auto err = this->tls_heap_central->Initialize(central, size, true); err != 0) { + this->tls_heap_central->~TlsHeapCentral(); + this->tls_heap_central = nullptr; + AMS_ASSERT(err == 0); + return err; + } + this->use_virtual_memory = true; + } + + return 0; + } + + void CentralHeap::Finalize() { + if (this->tls_heap_central) { + this->tls_heap_central->~TlsHeapCentral(); + } + if (this->use_virtual_memory) { + mem::impl::physical_free(util::AlignUp(static_cast(this->start), PageSize), this->end - this->start); + mem::impl::virtual_free(this->start, this->end - this->start); + } + this->tls_heap_central = nullptr; + this->use_virtual_memory = false; + this->option = 0; + this->start = nullptr; + this->end = nullptr; + } + + void *CentralHeap::Allocate(size_t n, size_t align) { + if (!util::IsPowerOfTwo(align)) { + return nullptr; + } + if (n > MaxSize) { + return nullptr; + } + if (align > PageSize) { + return this->tls_heap_central->CacheLargeMemoryWithBigAlign(util::AlignUp(n, PageSize), align); + } + + const size_t real_size = TlsHeapStatic::GetRealSizeFromSizeAndAlignment(util::AlignUp(n, align), align); + const auto cls = TlsHeapStatic::GetClassFromSize(real_size); + if (!cls) { + return this->tls_heap_central->CacheLargeMemory(real_size); + } + if (real_size == 0) { + return nullptr; + } + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + return this->tls_heap_central->CacheSmallMemory(cls, align); + } + + size_t CentralHeap::GetAllocationSize(const void *ptr) { + const auto cls = this->tls_heap_central->GetClassFromPointer(ptr); + if (cls > 0) { + /* Check that the pointer has alignment from out allocator. */ + if (!util::IsAligned(reinterpret_cast(ptr), MinimumAlignment)) { + return 0; + } + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + return TlsHeapStatic::GetChunkSize(cls); + } else if (ptr != nullptr) { + return this->tls_heap_central->GetAllocationSize(ptr); + } else { + return 0; + } + } + + errno_t CentralHeap::Free(void *ptr) { + /* Allow Free(nullptr) */ + if (ptr == nullptr) { + return 0; + } + + /* Check that the pointer has alignment from out allocator. */ + if(!util::IsAligned(reinterpret_cast(ptr), MinimumAlignment)) { + AMS_ASSERT(util::IsAligned(reinterpret_cast(ptr), MinimumAlignment)); + return EFAULT; + } + + const auto cls = this->tls_heap_central->GetClassFromPointer(ptr); + if (cls >= 0) { + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + if (cls) { + return this->tls_heap_central->UncacheSmallMemory(ptr); + } else { + return this->tls_heap_central->UncacheLargeMemory(ptr); + } + } else { + AMS_ASSERT(cls >= 0); + return EFAULT; + } + } + + errno_t CentralHeap::FreeWithSize(void *ptr, size_t size) { + if (TlsHeapStatic::GetClassFromSize(size)) { + return this->tls_heap_central->UncacheSmallMemory(ptr); + } else { + return this->tls_heap_central->UncacheLargeMemory(ptr); + } + } + + errno_t CentralHeap::Reallocate(void *ptr, size_t size, void **p) { + AMS_ASSERT(ptr != nullptr && size != 0); + if (!size) { + return EINVAL; + } + if (size > MaxSize) { + return ENOMEM; + } + + const auto cls_from_size = TlsHeapStatic::GetClassFromSize(size); + const auto cls_from_ptr = this->tls_heap_central->GetClassFromPointer(ptr); + if (cls_from_ptr) { + if (cls_from_ptr <= 0) { + return EFAULT; + } else if (cls_from_size && cls_from_size <= cls_from_ptr) { + *p = ptr; + return 0; + } else { + const size_t new_chunk_size = TlsHeapStatic::GetChunkSize(cls_from_ptr); + *p = this->Allocate(new_chunk_size); + if (*p) { + std::memcpy(*p, ptr, size); + return this->tls_heap_central->UncacheSmallMemory(ptr); + } else { + return ENOMEM; + } + } + } else if (cls_from_size) { + *p = this->Allocate(size); + if (*p) { + std::memcpy(*p, ptr, size); + return this->tls_heap_central->UncacheLargeMemory(ptr); + } else { + return ENOMEM; + } + } else { + return this->tls_heap_central->ReallocateLargeMemory(ptr, size, p); + } + } + + errno_t CentralHeap::Shrink(void *ptr, size_t size) { + AMS_ASSERT(ptr != nullptr && size != 0); + if (!size) { + return EINVAL; + } + if (size > MaxSize) { + return ENOMEM; + } + + const auto cls_from_size = TlsHeapStatic::GetClassFromSize(size); + const auto cls_from_ptr = this->tls_heap_central->GetClassFromPointer(ptr); + if (cls_from_ptr) { + if (cls_from_ptr <= 0) { + return EFAULT; + } else if (cls_from_size && cls_from_size <= cls_from_ptr) { + return 0; + } else { + return EINVAL; + } + } else if (cls_from_size) { + return this->tls_heap_central->ShrinkLargeMemory(ptr, PageSize); + } else { + return this->tls_heap_central->ShrinkLargeMemory(ptr, size); + } + } + + bool CentralHeap::MakeCache(CachedHeap *cached_heap) { + if (cached_heap == nullptr) { + return false; + } + + AMS_ASSERT(this->tls_heap_central != nullptr); + const auto cls = TlsHeapStatic::GetClassFromSize(sizeof(*cached_heap)); + void *tls_heap_cache = this->tls_heap_central->CacheSmallMemoryForSystem(cls); + if (tls_heap_cache == nullptr) { + return false; + } + + new (tls_heap_cache) TlsHeapCache(this->tls_heap_central, this->option); + if (this->tls_heap_central->AddThreadCache(reinterpret_cast(tls_heap_cache)) != 0) { + this->tls_heap_central->UncacheSmallMemory(tls_heap_cache); + return false; + } + + cached_heap->Reset(reinterpret_cast(tls_heap_cache)); + return true; + } + + errno_t CentralHeap::WalkAllocatedPointers(HeapWalkCallback callback, void *user_data) { + if (!callback || !this->tls_heap_central) { + return EINVAL; + } + return this->tls_heap_central->WalkAllocatedPointers(callback, user_data); + } + + errno_t CentralHeap::QueryV(int query, std::va_list vl) { + return this->QueryVImpl(query, std::addressof(vl)); + } + + errno_t CentralHeap::Query(int query, ...) { + std::va_list vl; + va_start(vl, query); + auto err = this->QueryVImpl(query, std::addressof(vl)); + va_end(vl); + return err; + } + + errno_t CentralHeap::QueryVImpl(int _query, std::va_list *vl_ptr) { + const AllocQuery query = static_cast(_query); + switch (query) { + case AllocQuery_Dump: + case AllocQuery_DumpJson: + { + auto dump_mode = static_cast(va_arg(*vl_ptr, int)); + auto fd = va_arg(*vl_ptr, int); + if (this->tls_heap_central) { + this->tls_heap_central->Dump(dump_mode, fd, query == AllocQuery_DumpJson); + } + return 0; + } + case AllocQuery_PageSize: + { + size_t *out = va_arg(*vl_ptr, size_t *); + if (out) { + *out = PageSize; + } + return 0; + } + case AllocQuery_AllocatedSize: + case AllocQuery_FreeSize: + case AllocQuery_SystemSize: + case AllocQuery_MaxAllocatableSize: + { + size_t *out = va_arg(*vl_ptr, size_t *); + if (!out) { + return 0; + } + if (!this->tls_heap_central) { + *out = 0; + return 0; + } + TlsHeapMemStats stats; + this->tls_heap_central->GetMemStats(std::addressof(stats)); + switch (query) { + case AllocQuery_AllocatedSize: + default: + *out = stats.allocated_size; + break; + case AllocQuery_FreeSize: + *out = stats.free_size; + break; + case AllocQuery_SystemSize: + *out = stats.system_size; + break; + case AllocQuery_MaxAllocatableSize: + *out = stats.max_allocatable_size; + break; + } + return 0; + } + case AllocQuery_IsClean: + { + int *out = va_arg(*vl_ptr, int *); + if (out) { + *out = !this->tls_heap_central || this->tls_heap_central->IsClean(); + } + return 0; + } + case AllocQuery_HeapHash: + { + HeapHash *out = va_arg(*vl_ptr, HeapHash *); + if (out) { + if (this->tls_heap_central) { + this->tls_heap_central->CalculateHeapHash(out); + } else { + *out = {}; + } + } + return 0; + } + case AllocQuery_UnifyFreeList: + /* NOTE: Nintendo does not check that the ptr is not null for this query, even though they do for other queries. */ + this->tls_heap_central->IsClean(); + return 0; + case AllocQuery_SetColor: + { + /* NOTE: Nintendo does not check that the ptr is not null for this query, even though they do for other queries. */ + void *ptr = va_arg(*vl_ptr, void *); + int color = va_arg(*vl_ptr, int); + return this->tls_heap_central->SetColor(ptr, color); + } + case AllocQuery_GetColor: + { + /* NOTE: Nintendo does not check that the ptr is not null for this query, even though they do for other queries. */ + void *ptr = va_arg(*vl_ptr, void *); + int *out = va_arg(*vl_ptr, int *); + return this->tls_heap_central->GetColor(ptr, out); + } + case AllocQuery_SetName: + { + /* NOTE: Nintendo does not check that the ptr is not null for this query, even though they do for other queries. */ + void *ptr = va_arg(*vl_ptr, void *); + const char *name = va_arg(*vl_ptr, const char *); + return this->tls_heap_central->SetName(ptr, name); + } + case AllocQuery_GetName: + { + /* NOTE: Nintendo does not check that the ptr is not null for this query, even though they do for other queries. */ + void *ptr = va_arg(*vl_ptr, void *); + char *dst = va_arg(*vl_ptr, char *); + size_t dst_size = va_arg(*vl_ptr, size_t); + return this->tls_heap_central->GetName(ptr, dst, dst_size); + } + case AllocQuery_FreeSizeMapped: + case AllocQuery_MaxAllocatableSizeMapped: + { + /* NOTE: Nintendo does not check that the ptr is not null for this query, even though they do for other queries. */ + size_t *out = va_arg(*vl_ptr, size_t *); + size_t free_size; + size_t max_allocatable_size; + auto err = this->tls_heap_central->GetMappedMemStats(std::addressof(free_size), std::addressof(max_allocatable_size)); + if (err == 0) { + if (query == AllocQuery_FreeSizeMapped) { + *out = free_size; + } else { + *out = max_allocatable_size; + } + } + return err; + } + default: + return EINVAL; + } + } + +} diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_platform.hpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_platform.hpp new file mode 100644 index 000000000..06088c0be --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_platform.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include "../mem_impl_platform.hpp" + +namespace ams::mem::impl::heap { + + using Prot = mem::impl::Prot; + + inline errno_t AllocateVirtualMemory(void **ptr, size_t size) { + return ::ams::mem::impl::virtual_alloc(ptr, size); + } + + inline errno_t FreeVirtualMemory(void *ptr, size_t size) { + return ::ams::mem::impl::virtual_free(ptr, size); + } + + inline errno_t AllocatePhysicalMemory(void *ptr, size_t size) { + return ::ams::mem::impl::physical_alloc(ptr, size, static_cast(Prot_read | Prot_write)); + } + + inline errno_t FreePhysicalMemory(void *ptr, size_t size) { + return ::ams::mem::impl::physical_free(ptr, size); + } + +} \ No newline at end of file diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_cache.cpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_cache.cpp new file mode 100644 index 000000000..2f5a453ad --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_cache.cpp @@ -0,0 +1,557 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "mem_impl_heap_platform.hpp" +#include "mem_impl_heap_tls_heap_static.hpp" +#include "mem_impl_heap_tls_heap_cache.hpp" +#include "mem_impl_heap_tls_heap_central.hpp" + +namespace ams::mem::impl::heap { + + TlsHeapCache::TlsHeapCache(TlsHeapCentral *central, u32 option) { + /* Choose function impls based on option. */ + if ((option & HeapOption_DisableCache) != 0) { + this->allocate = AllocateImpl; + this->allocate_aligned = AllocateAlignedImpl; + this->free = FreeImpl; + this->free_with_size = FreeWithSizeImpl; + this->get_allocation_size = GetAllocationSizeImpl; + this->reallocate = ReallocateImpl; + this->shrink = ShrinkImpl; + } else { + this->allocate = AllocateImpl; + this->allocate_aligned = AllocateAlignedImpl; + this->free = FreeImpl; + this->free_with_size = FreeWithSizeImpl; + this->get_allocation_size = GetAllocationSizeImpl; + this->reallocate = ReallocateImpl; + this->shrink = ShrinkImpl; + } + + /* Generate random bytes to mangle pointers. */ + if (auto err = gen_random(std::addressof(this->mangle_val), sizeof(this->mangle_val)); err != 0) { + s64 epoch_time; + epochtime(std::addressof(epoch_time)); + this->mangle_val = reinterpret_cast(std::addressof(epoch_time)) ^ static_cast(epoch_time); + } + + /* Set member variables. */ + this->central = central; + this->total_heap_size = central->GetTotalHeapSize(); + this->heap_option = option; + this->total_cached_size = 0; + this->largest_class = 0; + + /* Setup chunks. */ + for (size_t i = 0; i < TlsHeapStatic::NumClassInfo; i++) { + this->small_mem_lists[i] = nullptr; + this->cached_size[i] = 0; + this->chunk_count[i] = 1; + } + + /* Set fixed chunk counts for particularly small chunks. */ + this->chunk_count[1] = MaxChunkCount; + this->chunk_count[2] = MaxChunkCount; + this->chunk_count[3] = MaxChunkCount; + this->chunk_count[4] = MaxChunkCount / 2; + this->chunk_count[5] = MaxChunkCount / 2; + this->chunk_count[6] = MaxChunkCount / 2; + this->chunk_count[7] = MaxChunkCount / 4; + this->chunk_count[8] = MaxChunkCount / 4; + this->chunk_count[9] = MaxChunkCount / 4; + } + + void TlsHeapCache::Finalize() { + /* Free all small mem lists. */ + this->ReleaseAllCache(); + + /* Remove this cache from the owner central heap. */ + this->central->RemoveThreadCache(this); + this->central->UncacheSmallMemory(this); + } + + bool TlsHeapCache::CheckCache() const { + for (size_t i = 0; i < util::size(this->small_mem_lists); i++) { + void *ptr = this->small_mem_lists[i]; + if (ptr) { + s64 depth = -static_cast(this->cached_size[i] / TlsHeapStatic::GetChunkSize(i)); + while (ptr) { + ptr = *reinterpret_cast(this->ManglePointer(ptr)); + if ((++depth) == 0) { + AMS_ASSERT(ptr == nullptr); + break; + } + } + } + } + + return true; + } + + void TlsHeapCache::ReleaseAllCache() { + for (size_t i = 0; i < util::size(this->small_mem_lists); i++) { + if (this->small_mem_lists[i]) { + this->central->UncacheSmallMemoryList(this, this->small_mem_lists[i]); + this->small_mem_lists[i] = nullptr; + this->cached_size[i] = 0; + } + } + + this->total_cached_size = 0; + this->largest_class = 0; + } + + template<> + void *TlsHeapCache::AllocateImpl(TlsHeapCache *_this, size_t size) { + /* Validate allocation size. */ + if (size == 0 || size > MaxSize) { + return nullptr; + } + + if (const size_t cls = TlsHeapStatic::GetClassFromSize(size); cls != 0) { + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + return _this->central->CacheSmallMemory(cls); + } else { + /* If allocating a huge size, release our cache. */ + if (size >= _this->total_heap_size / 4) { + _this->ReleaseAllCache(); + } + return _this->central->CacheLargeMemory(size); + } + } + + template<> + void *TlsHeapCache::AllocateImpl(TlsHeapCache *_this, size_t size) { + /* Validate allocation size. */ + if (size == 0 || size > MaxSize) { + return nullptr; + } + + if (size_t cls = TlsHeapStatic::GetClassFromSize(size); cls != 0) { + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + /* Allocate a chunk. */ + void *ptr = _this->small_mem_lists[cls]; + if (ptr == nullptr) { + const size_t prev_cls = cls; + size_t count = _this->chunk_count[cls]; + + size_t n = _this->central->CacheSmallMemoryList(_this, std::addressof(cls), count, std::addressof(ptr)); + if (n == 0) { + return nullptr; + } + + if (cls == prev_cls) { + if (count < MaxChunkCount) { + count++; + } + _this->chunk_count[cls] = std::max(count, n); + } else { + AMS_ASSERT(n == 1); + } + + const size_t csize = TlsHeapStatic::GetChunkSize(cls) * (n - 1); + _this->cached_size[cls] += csize; + if (_this->cached_size[cls] > _this->cached_size[_this->largest_class]) { + _this->largest_class = cls; + } + _this->total_cached_size += csize; + } + + /* Demangle our pointer, update free list. */ + ptr = _this->ManglePointer(ptr); + _this->small_mem_lists[cls] = *reinterpret_cast(ptr); + + return ptr; + } else { + /* If allocating a huge size, release our cache. */ + if (size >= _this->total_heap_size / 4) { + _this->ReleaseAllCache(); + } + return _this->central->CacheLargeMemory(size); + } + } + + template<> + void *TlsHeapCache::AllocateAlignedImpl(TlsHeapCache *_this, size_t size, size_t align) { + /* Ensure valid alignment. */ + if (!util::IsPowerOfTwo(align)) { + return nullptr; + } + + /* NOTE: Nintendo does not check size == 0 here, despite doing so in Alloc */ + if (size > MaxSize) { + return nullptr; + } + + /* Handle big alignment. */ + if (align > TlsHeapStatic::PageSize) { + return _this->central->CacheLargeMemoryWithBigAlign(util::AlignUp(size, TlsHeapStatic::PageSize), align); + } + + const size_t real_size = TlsHeapStatic::GetRealSizeFromSizeAndAlignment(util::AlignUp(size, align), align); + + if (const size_t cls = TlsHeapStatic::GetClassFromSize(real_size); cls != 0) { + if (real_size == 0) { + return nullptr; + } + + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + return _this->central->CacheSmallMemory(cls, align); + } else { + /* If allocating a huge size, release our cache. */ + if (real_size >= _this->total_heap_size / 4) { + _this->ReleaseAllCache(); + } + return _this->central->CacheLargeMemory(real_size); + } + } + + template<> + void *TlsHeapCache::AllocateAlignedImpl(TlsHeapCache *_this, size_t size, size_t align) { + /* Ensure valid alignment. */ + if (!util::IsPowerOfTwo(align)) { + return nullptr; + } + + /* NOTE: Nintendo does not check size == 0 here, despite doing so in Alloc */ + if (size > MaxSize) { + return nullptr; + } + + /* Handle big alignment. */ + if (align > TlsHeapStatic::PageSize) { + return _this->central->CacheLargeMemoryWithBigAlign(util::AlignUp(size, TlsHeapStatic::PageSize), align); + } + + const size_t real_size = TlsHeapStatic::GetRealSizeFromSizeAndAlignment(util::AlignUp(size, align), align); + + if (size_t cls = TlsHeapStatic::GetClassFromSize(real_size); cls != 0) { + if (real_size == 0) { + return nullptr; + } + + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + + /* Allocate a chunk. */ + void *ptr = _this->small_mem_lists[cls]; + if (ptr == nullptr) { + const size_t prev_cls = cls; + size_t count = _this->chunk_count[cls]; + + size_t n = _this->central->CacheSmallMemoryList(_this, std::addressof(cls), count, std::addressof(ptr), align); + if (n == 0) { + return nullptr; + } + + if (cls == prev_cls) { + if (count < MaxChunkCount) { + count++; + } + _this->chunk_count[cls] = std::max(count, n); + } else { + AMS_ASSERT(n == 1); + } + + const s32 csize = TlsHeapStatic::GetChunkSize(cls) * (n - 1); + _this->total_cached_size += csize; + _this->cached_size[cls] += csize; + if (_this->cached_size[cls] > _this->cached_size[_this->largest_class]) { + _this->largest_class = cls; + } + } + + /* Demangle our pointer, update free list. */ + ptr = _this->ManglePointer(ptr); + _this->small_mem_lists[cls] = *reinterpret_cast(ptr); + + return ptr; + } else { + /* If allocating a huge size, release our cache. */ + if (size >= _this->total_heap_size / 4) { + _this->ReleaseAllCache(); + } + return _this->central->CacheLargeMemory(size); + } + } + + template<> + errno_t TlsHeapCache::FreeImpl(TlsHeapCache *_this, void *ptr) { + const size_t cls = _this->central->GetClassFromPointer(ptr); + if (cls == 0) { + return _this->central->UncacheLargeMemory(ptr); + } + + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + + if (static_cast(cls) >= 0) { + return _this->central->UncacheSmallMemory(ptr); + } else if (ptr == nullptr) { + return 0; + } else { + return EFAULT; + } + } + + template<> + errno_t TlsHeapCache::FreeImpl(TlsHeapCache *_this, void *ptr) { + const size_t cls = _this->central->GetClassFromPointer(ptr); + if (cls == 0) { + return _this->central->UncacheLargeMemory(ptr); + } + + AMS_ASSERT(cls < TlsHeapStatic::NumClassInfo); + + if (static_cast(cls) >= 0) { + *reinterpret_cast(ptr) = _this->small_mem_lists[cls]; + _this->small_mem_lists[cls] = _this->ManglePointer(ptr); + + const s32 csize = TlsHeapStatic::GetChunkSize(cls); + _this->total_cached_size += csize; + _this->cached_size[cls] += csize; + if (_this->cached_size[cls] > _this->cached_size[_this->largest_class]) { + _this->largest_class = cls; + } + + errno_t err = 0; + if (!_this->central->CheckCachedSize(_this->total_cached_size)) { + _this->central->UncacheSmallMemoryList(_this, _this->small_mem_lists[_this->largest_class]); + _this->small_mem_lists[_this->largest_class] = nullptr; + _this->total_cached_size -= _this->cached_size[_this->largest_class]; + _this->cached_size[_this->largest_class] = 0; + + s32 largest_class = 0; + s32 biggest_size = -1; + for (size_t i = 0; i < TlsHeapStatic::NumClassInfo; i++) { + if (biggest_size < _this->cached_size[i]) { + biggest_size = _this->cached_size[i]; + largest_class = static_cast(i); + } + } + _this->largest_class = largest_class; + } + return err; + } else if (ptr == nullptr) { + return 0; + } else { + return EFAULT; + } + } + + template<> + errno_t TlsHeapCache::FreeWithSizeImpl(TlsHeapCache *_this, void *ptr, size_t size) { + if (ptr == nullptr) { + return 0; + } + + const size_t cls = TlsHeapStatic::GetClassFromSize(size); + if (cls == 0) { + return _this->central->UncacheLargeMemory(ptr); + } else { + return _this->central->UncacheSmallMemory(ptr); + } + } + + template<> + errno_t TlsHeapCache::FreeWithSizeImpl(TlsHeapCache *_this, void *ptr, size_t size) { + if (ptr == nullptr) { + return 0; + } + + const size_t cls = TlsHeapStatic::GetClassFromSize(size); + if (cls == 0) { + return _this->central->UncacheLargeMemory(ptr); + } else { + *reinterpret_cast(ptr) = _this->small_mem_lists[cls]; + _this->small_mem_lists[cls] = _this->ManglePointer(ptr); + + const s32 csize = TlsHeapStatic::GetChunkSize(cls); + _this->total_cached_size += csize; + _this->cached_size[cls] += csize; + if (_this->cached_size[cls] > _this->cached_size[_this->largest_class]) { + _this->largest_class = cls; + } + + errno_t err = 0; + if (!_this->central->CheckCachedSize(_this->total_cached_size)) { + _this->central->UncacheSmallMemoryList(_this, _this->small_mem_lists[_this->largest_class]); + _this->small_mem_lists[_this->largest_class] = nullptr; + _this->total_cached_size -= _this->cached_size[_this->largest_class]; + _this->cached_size[_this->largest_class] = 0; + + s32 largest_class = 0; + s32 biggest_size = -1; + for (size_t i = 0; i < TlsHeapStatic::NumClassInfo; i++) { + if (biggest_size < _this->cached_size[i]) { + biggest_size = _this->cached_size[i]; + largest_class = static_cast(i); + } + } + _this->largest_class = largest_class; + } + return err; + } + } + + template<> + size_t TlsHeapCache::GetAllocationSizeImpl(TlsHeapCache *_this, const void *ptr) { + return _this->GetAllocationSizeCommonImpl(ptr); + } + + template<> + size_t TlsHeapCache::GetAllocationSizeImpl(TlsHeapCache *_this, const void *ptr) { + return _this->GetAllocationSizeCommonImpl(ptr); + } + + size_t TlsHeapCache::GetAllocationSizeCommonImpl(const void *ptr) const { + const s32 cls = this->central->GetClassFromPointer(ptr); + if (cls > 0) { + if (!util::IsAligned(ptr, alignof(u64))) { + /* All pointers we allocate have alignment at least 8. */ + return 0; + } + + /* Validate class. */ + AMS_ASSERT(cls < static_cast(TlsHeapStatic::NumClassInfo)); + if (cls < 0) { + return 0; + } + + return TlsHeapStatic::GetChunkSize(cls); + } else if (ptr != nullptr) { + return this->central->GetAllocationSize(ptr); + } else { + return 0; + } + } + + template<> + errno_t TlsHeapCache::ReallocateImpl(TlsHeapCache *_this, void *ptr, size_t size, void **p) { + AMS_ASSERT(ptr != nullptr && size != 0); + if (size > MaxSize) { + return ENOMEM; + } + + size_t alloc_size, copy_size; + + const s32 cls_from_size = TlsHeapStatic::GetClassFromSize(size); + const s32 cls_from_ptr = _this->central->GetClassFromPointer(ptr); + if (cls_from_ptr < 0) { + /* error case. */ + return EFAULT; + } else if (cls_from_size) { + if (cls_from_ptr > 0) { + if (cls_from_size <= cls_from_ptr) { + *p = ptr; + return 0; + } else { + alloc_size = size; + copy_size = TlsHeapStatic::GetChunkSize(cls_from_ptr); + } + } else /* if (cls_from_ptr == 0) */ { + alloc_size = size; + copy_size = size; + } + } else if (cls_from_ptr == 0) { + return _this->central->ReallocateLargeMemory(ptr, size, p); + } else /* if (cls_from_ptr > 0) */ { + alloc_size = size; + copy_size = TlsHeapStatic::GetChunkSize(cls_from_ptr); + } + + *p = AllocateImpl(_this, alloc_size); + if (*p == nullptr) { + return ENOMEM; + } + std::memcpy(*p, ptr, copy_size); + return FreeImpl(_this, ptr); + } + + template<> + errno_t TlsHeapCache::ReallocateImpl(TlsHeapCache *_this, void *ptr, size_t size, void **p) { + AMS_ASSERT(ptr != nullptr && size != 0); + if (size > MaxSize) { + return ENOMEM; + } + + size_t alloc_size, copy_size; + + const s32 cls_from_size = TlsHeapStatic::GetClassFromSize(size); + const s32 cls_from_ptr = _this->central->GetClassFromPointer(ptr); + if (cls_from_ptr < 0) { + /* error case. */ + return EFAULT; + } else if (cls_from_size) { + if (cls_from_ptr > 0) { + if (cls_from_size <= cls_from_ptr) { + *p = ptr; + return 0; + } else { + alloc_size = size; + copy_size = TlsHeapStatic::GetChunkSize(cls_from_ptr); + } + } else /* if (cls_from_ptr == 0) */ { + alloc_size = size; + copy_size = size; + } + } else if (cls_from_ptr == 0) { + return _this->central->ReallocateLargeMemory(ptr, size, p); + } else /* if (cls_from_ptr > 0) */ { + alloc_size = size; + copy_size = TlsHeapStatic::GetChunkSize(cls_from_ptr); + } + + *p = AllocateImpl(_this, alloc_size); + if (*p == nullptr) { + return ENOMEM; + } + std::memcpy(*p, ptr, copy_size); + return FreeImpl(_this, ptr); + } + + template<> + errno_t TlsHeapCache::ShrinkImpl(TlsHeapCache *_this, void *ptr, size_t size) { + return _this->ShrinkCommonImpl(ptr, size); + } + + template<> + errno_t TlsHeapCache::ShrinkImpl(TlsHeapCache *_this, void *ptr, size_t size) { + return _this->ShrinkCommonImpl(ptr, size); + } + + errno_t TlsHeapCache::ShrinkCommonImpl(void *ptr, size_t size) const { + AMS_ASSERT(ptr != nullptr && size != 0); + if (size > MaxSize) { + return ENOMEM; + } + + const s32 cls_from_size = TlsHeapStatic::GetClassFromSize(size); + const s32 cls_from_ptr = this->central->GetClassFromPointer(ptr); + if (cls_from_ptr) { + if (cls_from_ptr <= 0) { + return EFAULT; + } else if (cls_from_size && cls_from_size <= cls_from_ptr) { + return 0; + } else { + return EINVAL; + } + } else if (cls_from_size) { + return this->central->ShrinkLargeMemory(ptr, TlsHeapStatic::PageSize); + } else { + return this->central->ShrinkLargeMemory(ptr, size); + } + } + +} diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_cache.hpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_cache.hpp new file mode 100644 index 000000000..694d5902a --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_cache.hpp @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include "mem_impl_heap_platform.hpp" +#include "mem_impl_heap_tls_heap_static.hpp" + +namespace ams::mem::impl::heap { + + class TlsHeapCentral; + + #define FOREACH_TLS_HEAP_CACHE_FUNC(HANDLER) \ + HANDLER(void *, Allocate, allocate, size_t size); \ + HANDLER(void *, AllocateAligned, allocate_aligned, size_t size, size_t align); \ + HANDLER(errno_t, Free, free, void *ptr); \ + HANDLER(errno_t, FreeWithSize, free_with_size, void *ptr, size_t size); \ + HANDLER(size_t, GetAllocationSize, get_allocation_size, const void *ptr); \ + HANDLER(errno_t, Reallocate, reallocate, void *ptr, size_t size, void **p); \ + HANDLER(errno_t, Shrink, shrink, void *ptr, size_t size); + + class TlsHeapCache { + public: + static constexpr size_t MaxChunkCount = BITSIZEOF(u64); + public: + #define TLS_HEAP_CACHE_DECLARE_TYPEDEF(RETURN, NAME, MEMBER_NAME, ...) \ + using NAME##Func = RETURN (*)(TlsHeapCache *, ## __VA_ARGS__) + + FOREACH_TLS_HEAP_CACHE_FUNC(TLS_HEAP_CACHE_DECLARE_TYPEDEF) + + #undef TLS_HEAP_CACHE_DECLARE_TYPEDEF + private: + #define TLS_HEAP_CACHE_DECLARE_MEMBER(RETURN, NAME, MEMBER_NAME, ...) \ + NAME##Func MEMBER_NAME; + + FOREACH_TLS_HEAP_CACHE_FUNC(TLS_HEAP_CACHE_DECLARE_MEMBER) + + #undef TLS_HEAP_CACHE_DECLARE_MEMBER + + uintptr_t mangle_val; + TlsHeapCentral *central; + size_t total_heap_size; + u32 heap_option; + s32 total_cached_size; + s32 largest_class; + void *small_mem_lists[TlsHeapStatic::NumClassInfo]; + s32 cached_size[TlsHeapStatic::NumClassInfo]; + u8 chunk_count[TlsHeapStatic::NumClassInfo]; + public: + TlsHeapCache(TlsHeapCentral *central, u32 option); + void Finalize(); + + void *ManglePointer(void *ptr) const { + return reinterpret_cast(reinterpret_cast(ptr) ^ this->mangle_val); + } + + bool CheckCache() const; + void ReleaseAllCache(); + + public: + /* TODO: Better handler with type info to macro this? */ + ALWAYS_INLINE void *Allocate(size_t size) { return this->allocate(this, size); } + ALWAYS_INLINE void *Allocate(size_t size, size_t align) { return this->allocate_aligned(this, size, align); } + ALWAYS_INLINE errno_t Free(void *ptr) { return this->free(this, ptr); } + ALWAYS_INLINE errno_t FreeWithSize(void *ptr, size_t size) { return this->free_with_size(this, ptr, size); } + ALWAYS_INLINE size_t GetAllocationSize(const void *ptr) { return this->get_allocation_size(this, ptr); } + ALWAYS_INLINE errno_t Reallocate(void *ptr, size_t size, void **p) { return this->reallocate(this, ptr, size, p); } + ALWAYS_INLINE errno_t Shrink(void *ptr, size_t size) { return this->shrink(this, ptr, size); } + private: + #define TLS_HEAP_CACHE_DECLARE_TEMPLATE(RETURN, NAME, MEMBER_NAME, ...) \ + template static RETURN NAME##Impl(TlsHeapCache *_this, ## __VA_ARGS__ ) + + FOREACH_TLS_HEAP_CACHE_FUNC(TLS_HEAP_CACHE_DECLARE_TEMPLATE) + + #undef TLS_HEAP_CACHE_DECLARE_TEMPLATE + + size_t GetAllocationSizeCommonImpl(const void *ptr) const; + errno_t ShrinkCommonImpl(void *ptr, size_t size) const; + }; + + #define TLS_HEAP_CACHE_DECLARE_INSTANTIATION(RETURN, NAME, MEMBER_NAME, ...) \ + template<> RETURN TlsHeapCache::NAME##Impl(TlsHeapCache *_this, ##__VA_ARGS__); \ + template<> RETURN TlsHeapCache::NAME##Impl(TlsHeapCache *_this, ##__VA_ARGS__) + + FOREACH_TLS_HEAP_CACHE_FUNC(TLS_HEAP_CACHE_DECLARE_INSTANTIATION) + + #undef FOREACH_TLS_HEAP_CACHE_FUNC + + +} \ No newline at end of file diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.cpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.cpp new file mode 100644 index 000000000..fd2554985 --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.cpp @@ -0,0 +1,1492 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "mem_impl_heap_platform.hpp" +#include "mem_impl_heap_tls_heap_static.hpp" +#include "mem_impl_heap_tls_heap_central.hpp" + +namespace ams::mem::impl::heap { + + namespace { + + char g_log_buf[4_KB]; + + void InitializeSpanPage(SpanPage *sp) { + static_assert(SpanPage::MaxSpanCount <= BITSIZEOF(u64)); + constexpr size_t NumUnusedBits = BITSIZEOF(u64) - SpanPage::MaxSpanCount; + + sp->info.free_count = SpanPage::MaxSpanCount; + sp->info.is_sticky = 0; + sp->info.alloc_bitmap = (static_cast(1) << NumUnusedBits) - 1; + + ListClearLink(sp); + + sp->info.span_of_spanpage.start.u = 0; + sp->info.span_of_spanpage.num_pages = 0; + sp->info.span_of_spanpage.aux.small.objects = nullptr; + sp->info.span_of_spanpage.object_count = 0; + sp->info.span_of_spanpage.page_class = 0; + sp->info.span_of_spanpage.status = Span::Status_NotUsed; + sp->info.span_of_spanpage.id = 0; + + ListClearLink(std::addressof(sp->info.span_of_spanpage)); + } + + void RegisterSpan(SpanTable *span_table, Span *span) { + const size_t idx = TlsHeapStatic::GetPageIndex(span->start.u - reinterpret_cast(span_table)); + span->page_class = 0; + span_table->page_to_span[idx] = span; + span_table->page_to_span[idx + span->num_pages - 1] = span; + } + + void UnregisterSpan(SpanTable *span_table, Span *span) { + AMS_ASSERT(span->page_class == 0); + const size_t idx = TlsHeapStatic::GetPageIndex(span->start.u - reinterpret_cast(span_table)); + span_table->page_to_span[idx] = nullptr; + span_table->page_to_span[idx + span->num_pages - 1] = nullptr; + } + + void ChangeRangeOfSpan(SpanTable *span_table, Span *span, uintptr_t start, size_t new_pages) { + const size_t idx = TlsHeapStatic::GetPageIndex(span->start.u - reinterpret_cast(span_table)); + if (span->start.u == start) { + if (span->num_pages != 1) { + span_table->page_to_span[idx + span->num_pages - 1] = nullptr; + } + span_table->page_to_span[idx + new_pages - 1] = span; + span->num_pages = new_pages; + } else { + span_table->page_to_span[idx] = nullptr; + span_table->page_to_span[idx + span->num_pages - 1] = nullptr; + + const size_t new_idx = TlsHeapStatic::GetPageIndex(start - reinterpret_cast(span_table)); + span_table->page_to_span[new_idx] = span; + span_table->page_to_span[new_idx + new_pages - 1] = span; + + span->start.u = start; + span->num_pages = new_pages; + } + } + + void MigrateSpan(SpanTable *span_table, Span *from, Span *to) { + AMS_ASSERT(from != to); + + std::memcpy(to, from, sizeof(*from)); + + from->status = Span::Status_NotUsed; + + if (from->list_next) { + to->list_next = from->list_next; + from->list_next->list_prev = to; + from->list_next = nullptr; + } else { + to->list_next = nullptr; + } + + if (from->list_prev) { + to->list_prev = from->list_prev; + from->list_prev->list_next = to; + from->list_prev = nullptr; + } else { + to->list_prev = nullptr; + } + + const size_t idx = TlsHeapStatic::GetPageIndex(to->start.u - reinterpret_cast(span_table)); + if (from->page_class) { + for (size_t i = 0; i < to->num_pages; i++) { + span_table->page_to_span[idx + i] = to; + } + } else { + span_table->page_to_span[idx] = to; + span_table->page_to_span[idx + to->num_pages - 1] = to; + } + } + + bool IsNthSmallMemoryMarked(const Span *span, size_t n) { + return (span->aux.small.is_allocated[n / BITSIZEOF(u64)] & (1ull << (n % BITSIZEOF(u64)))) != 0; + } + + void MarkNthSmallMemory(Span *span, size_t n) { + span->aux.small.is_allocated[n / BITSIZEOF(u64)] |= (1ull << (n % BITSIZEOF(u64))); + } + + void UnmarkNthSmallMemory(Span *span, size_t n) { + span->aux.small.is_allocated[n / BITSIZEOF(u64)] &= ~(1ull << (n % BITSIZEOF(u64))); + } + + void *AllocateSmallMemory(Span *span) { + Span::SmallMemory *sm = span->aux.small.objects; + const size_t chunk_size = TlsHeapStatic::GetChunkSize(span->page_class); + const uintptr_t span_end = span->start.u + (span->num_pages * TlsHeapStatic::PageSize); + if (span->start.u <= reinterpret_cast(sm) && reinterpret_cast(sm) < span_end) { + const size_t idx = (reinterpret_cast(sm) - span->start.u) / chunk_size; + if (reinterpret_cast(sm) == (span->start.u + idx * chunk_size) && !IsNthSmallMemoryMarked(span, idx)) { + MarkNthSmallMemory(span, idx); + span->aux.small.objects = sm->next; + sm->next = nullptr; + span->object_count++; + return sm; + } + } else if (sm == nullptr) { + return nullptr; + } + + /* Data corruption error. */ + AMS_ASSERT(false); + span->aux.small.objects = nullptr; + return nullptr; + } + + struct MangledSmallMemory { + Span::SmallMemory *from; + Span::SmallMemory *to; + }; + + size_t AllocateSmallMemory(Span *span, TlsHeapCache *cache, size_t n, MangledSmallMemory *memlist) { + auto ManglePointer = [cache](void *ptr) ALWAYS_INLINE_LAMBDA { + return static_cast(cache->ManglePointer(ptr)); + }; + + Span::SmallMemory *sm = span->aux.small.objects; + if (sm) { + size_t count = 0; + memlist->from = ManglePointer(sm); + + const size_t chunk_size = TlsHeapStatic::GetChunkSize(span->page_class); + const uintptr_t span_end = span->start.u + (span->num_pages * TlsHeapStatic::PageSize); + while (span->start.u <= reinterpret_cast(sm) && reinterpret_cast(sm) < span_end) { + const size_t idx = (reinterpret_cast(sm) - span->start.u) / chunk_size; + if (span->start.u + idx * chunk_size != reinterpret_cast(sm)) { + break; + } + if (IsNthSmallMemoryMarked(span, idx)) { + break; + } + MarkNthSmallMemory(span, idx); + count++; + + Span::SmallMemory *next = sm->next; + sm->next = ManglePointer(next); + if (count >= n || next == nullptr) { + memlist->to = sm; + memlist->to->next = nullptr; + span->aux.small.objects = next; + span->object_count += count; + return count; + } + + sm = next; + } + + /* Data corruption error. */ + Span::SmallMemory *prev = span->aux.small.objects; + Span::SmallMemory *cur = span->aux.small.objects; + while (cur != sm) { + prev = cur; + cur = ManglePointer(cur->next); + } + + memlist->to = ManglePointer(prev); + memlist->to->next = nullptr; + span->aux.small.objects = nullptr; + span->object_count += count; + return count; + } else { + memlist->from = nullptr; + memlist->to = nullptr; + return 0; + } + } + + void ReleaseSmallMemory(Span *span, void *ptr) { + AMS_ASSERT(span->object_count > 0); + + const size_t span_ofs = reinterpret_cast(ptr) - span->start.u; + const size_t chunk_size = TlsHeapStatic::GetChunkSize(span->page_class); + const size_t span_idx = span_ofs / chunk_size; + if (span_ofs != (span_idx * chunk_size)) { + /* Invalid pointer. Do the best we can. */ + ptr = reinterpret_cast(span->start.u + span_idx * chunk_size); + } + if (IsNthSmallMemoryMarked(span, span_idx)) { + UnmarkNthSmallMemory(span, span_idx); + + Span::SmallMemory *sm = reinterpret_cast(ptr); + sm->next = span->aux.small.objects; + span->aux.small.objects = sm; + span->object_count--; + } else { + /* Double free error. */ + /* TODO: Anything? */ + } + } + + void SpanToSmallMemorySpan(SpanTable *span_table, Span *span, size_t cls) { + AMS_ASSERT(cls != 0 && span->page_class == 0); + + const size_t span_idx = TlsHeapStatic::GetPageIndex(span->start.u - reinterpret_cast(span_table)); + Span **table_entry = std::addressof(span_table->page_to_span[span_idx]); + + span->page_class = cls; + for (size_t i = 0; i < span->num_pages; i++) { + table_entry[i] = span; + } + + std::atomic_thread_fence(std::memory_order_release); + + std::memset(std::addressof(span_table->pageclass_cache[span_idx]), cls, span->num_pages); + } + + void SmallMemorySpanToSpan(SpanTable *span_table, Span *span) { + AMS_ASSERT(span->page_class != 0); + + const size_t span_idx = TlsHeapStatic::GetPageIndex(span->start.u - reinterpret_cast(span_table)); + Span **table_entry = std::addressof(span_table->page_to_span[span_idx]); + + for (size_t i = 0; i < span->num_pages; i++) { + table_entry[i] = nullptr; + } + span->page_class = 0; + + table_entry[0] = span; + table_entry[span->num_pages - 1] = span; + + std::atomic_thread_fence(std::memory_order_release); + + std::memset(std::addressof(span_table->pageclass_cache[span_idx]), 0, span->num_pages); + } + + void InitSmallMemorySpan(Span *span, size_t cls, bool for_system, int id) { + AMS_ASSERT(cls != 0); + span->page_class = cls; + + AMS_ASSERT(span->status == Span::Status_InUse); + if (for_system) { + span->status = Span::Status_InUseSystem; + } + + span->aux.small.objects = span->start.sm; + span->object_count = 0; + span->id = id; + + const size_t chunk_size = TlsHeapStatic::GetChunkSize(cls); + const size_t num_chunks = (span->num_pages * TlsHeapStatic::PageSize) / chunk_size; + AMS_ASSERT(num_chunks <= sizeof(span->aux.small.is_allocated) * BITSIZEOF(u8)); + + Span::SmallMemory *last = reinterpret_cast(span->start.u + num_chunks * chunk_size); + Span::SmallMemory *prev = reinterpret_cast(span->start.u); + for (Span::SmallMemory *cur = reinterpret_cast(span->start.u + chunk_size); cur != last; cur = reinterpret_cast(reinterpret_cast(cur) + chunk_size)) { + prev->next = cur; + prev = cur; + } + prev->next = nullptr; + std::memset(span->aux.small.is_allocated, 0, sizeof(span->aux.small.is_allocated)); + } + + } + + errno_t TlsHeapCentral::Initialize(void *start, size_t size, bool use_virtual_memory) { + AMS_ASSERT(size > 0); + AMS_ASSERT(TlsHeapStatic::IsPageAligned(start)); + AMS_ASSERT(TlsHeapStatic::IsPageAligned(size)); + + /* Clear lists. */ + ListClearLink(std::addressof(this->spanpage_list)); + ListClearLink(std::addressof(this->full_spanpage_list)); + for (size_t i = 0; i < util::size(this->freelists); i++) { + ListClearLink(std::addressof(this->freelists[i])); + } + for (size_t i = 0; i < util::size(this->freelists_bitmap); i++) { + this->freelists_bitmap[i] = 0; + } + for (size_t i = 0; i < util::size(this->smallmem_lists); i++) { + ListClearLink(std::addressof(this->smallmem_lists[i])); + } + + /* Setup span table. */ + const size_t total_pages = TlsHeapStatic::GetPageIndex(size); + const size_t n = total_pages * sizeof(Span *); + this->span_table.total_pages = total_pages; + this->span_table.page_to_span = reinterpret_cast(static_cast(start) + sizeof(*this)); + this->span_table.pageclass_cache = static_cast(start) + sizeof(*this) + n; + + u8 *meta_end = this->span_table.pageclass_cache + total_pages; + size_t num_physical_page_flags; + if (use_virtual_memory) { + this->physical_page_flags = meta_end; + const uintptr_t phys_start = TlsHeapStatic::AlignDownPhysicalPage(reinterpret_cast(start)); + const uintptr_t phys_end = TlsHeapStatic::AlignUpPhysicalPage(reinterpret_cast(start) + size); + num_physical_page_flags = TlsHeapStatic::GetPhysicalPageIndex(phys_end - phys_start); + meta_end = TlsHeapStatic::AlignUpPage(meta_end + num_physical_page_flags); + } else { + this->physical_page_flags = nullptr; + num_physical_page_flags = 0; + meta_end = TlsHeapStatic::AlignUpPage(meta_end); + } + AMS_ASSERT(TlsHeapStatic::IsPageAligned(meta_end)); + + if (use_virtual_memory) { + const uintptr_t phys_end = TlsHeapStatic::AlignUpPhysicalPage(reinterpret_cast(meta_end) + TlsHeapStatic::PageSize); + size_t phys_size = phys_end - reinterpret_cast(start); + phys_size = std::min(phys_size, size); + if (auto err = AllocatePhysicalMemory(start, phys_size); err != 0) { + this->span_table.total_pages = 0; + return err; + } + std::memset(this->physical_page_flags, 0, num_physical_page_flags); + std::memset(this->physical_page_flags, 1, TlsHeapStatic::GetPhysicalPageIndex(phys_end) - TlsHeapStatic::GetPhysicalPageIndex(reinterpret_cast(start))); + } + + std::memset(this->span_table.page_to_span, 0, n); + std::memset(this->span_table.pageclass_cache, 0, total_pages); + + SpanPage *span_page = reinterpret_cast(meta_end); + InitializeSpanPage(span_page); + ListInsertAfter(std::addressof(this->spanpage_list), span_page); + + meta_end += TlsHeapStatic::PageSize; + AMS_ASSERT(TlsHeapStatic::IsPageAligned(meta_end)); + + /* Setup the spans. */ + Span *span = this->AllocateSpanFromSpanPage(span_page); + AMS_ASSERT(span != nullptr); + + Span *span_admin = this->AllocateSpanFromSpanPage(span_page); + AMS_ASSERT(span_admin != nullptr); + + span_page->info.is_sticky = 1; + + span->start.u = reinterpret_cast(meta_end); + span->num_pages = TlsHeapStatic::GetPageIndex(reinterpret_cast(start) + size - reinterpret_cast(meta_end)); + span->aux.small.objects = nullptr; + span->object_count = 0; + span->status = Span::Status_InFreeList; + span->id = 0; + + span_page->info.span_of_spanpage.start.u = reinterpret_cast(span_page); + span_page->info.span_of_spanpage.num_pages = 1; + span_page->info.span_of_spanpage.aux.small.objects = nullptr; + span_page->info.span_of_spanpage.object_count = 0; + span_page->info.span_of_spanpage.status = Span::Status_InUseSystem; + span_page->info.span_of_spanpage.id = 0; + + span_admin->start.u = reinterpret_cast(start); + span_admin->num_pages = TlsHeapStatic::GetPageIndex(reinterpret_cast(span_page) - reinterpret_cast(start)); + span_admin->aux.small.objects = nullptr; + span_admin->object_count = 0; + span_admin->status = Span::Status_InUseSystem; + span_admin->id = 0; + + RegisterSpan(std::addressof(this->span_table), span_admin); + RegisterSpan(std::addressof(this->span_table), std::addressof(span_page->info.span_of_spanpage)); + RegisterSpan(std::addressof(this->span_table), span); + + this->AddToFreeBlockList(span); + + this->num_threads = 1; + this->static_thread_quota = std::min((this->span_table.total_pages * TlsHeapStatic::PageSize) / sizeof(void *), 2_MB); + this->dynamic_thread_quota = this->static_thread_quota; + this->use_virtual_memory = use_virtual_memory; + + return 0; + } + + bool TlsHeapCentral::IsClean() { + std::scoped_lock lk(this->lock); + + this->MakeFreeSpan(std::numeric_limits::max()); + + Span *span = this->GetFirstSpan(); + Span *next = GetNextSpan(std::addressof(this->span_table), span); + if (next && next->status == Span::Status_InFreeList && GetNextSpan(std::addressof(this->span_table), next) == nullptr) { + return true; + } else { + return false; + } + } + + errno_t TlsHeapCentral::ReallocateLargeMemory(void *ptr, size_t size, void **p) { + if (!TlsHeapStatic::IsPageAligned(ptr)) { + return EFAULT; + } + + AMS_ASSERT(size <= MaxSize); + + /* NOTE: This function uses locks unsafely (unscoped) */ + + this->lock.Lock(); + + Span *ptr_span = GetSpanFromPointer(std::addressof(this->span_table), ptr); + if (!ptr_span) { + AMS_ASSERT(ptr_span != nullptr); + this->lock.Unlock(); + return EFAULT; + } + + const size_t num_pages = TlsHeapStatic::GetPageIndex(size + TlsHeapStatic::PageSize - 1); + if (ptr_span->num_pages != num_pages) { + if (ptr_span->num_pages >= num_pages) { + Span *span = this->AllocateSpanStruct(); + if (span != nullptr) { + span->start.u = ptr_span->start.u + (num_pages * TlsHeapStatic::PageSize); + span->num_pages = ptr_span->num_pages - num_pages; + span->id = 0; + span->status = Span::Status_InUse; + ChangeRangeOfSpan(std::addressof(this->span_table), ptr_span, ptr_span->start.u, num_pages); + RegisterSpan(std::addressof(this->span_table), span); + this->FreePagesImpl(span); + } + } else { + Span *next_span = GetNextSpan(std::addressof(this->span_table), ptr_span); + if (!next_span || next_span->status != Span::Status_InFreeList || next_span->num_pages < num_pages - ptr_span->num_pages) { + this->lock.Unlock(); + + this->lock.Lock(); + + Span *span = this->AllocatePagesImpl(num_pages); + if (span) { + *p = span->start.p; + std::memcpy(std::addressof(span->aux.large), std::addressof(ptr_span->aux.large), sizeof(ptr_span->aux.large)); + } else { + *p = nullptr; + } + + this->lock.Unlock(); + if (*p == nullptr) { + return ENOMEM; + } + + std::memcpy(*p, ptr, num_pages * TlsHeapStatic::PageSize); + + this->lock.Lock(); + this->FreePagesImpl(ptr_span); + this->lock.Unlock(); + + return 0; + } + + if (this->use_virtual_memory && this->AllocatePhysical(next_span->start.p, (num_pages - ptr_span->num_pages) * TlsHeapStatic::PageSize)) { + this->lock.Unlock(); + return ENOMEM; + } + + this->RemoveFromFreeBlockList(next_span); + if (next_span->num_pages == num_pages - ptr_span->num_pages) { + UnregisterSpan(std::addressof(this->span_table), next_span); + ChangeRangeOfSpan(std::addressof(this->span_table), ptr_span, ptr_span->start.u, num_pages); + SpanPage *sp = GetSpanPage(next_span); + this->FreeSpanToSpanPage(next_span, sp); + this->DestroySpanPageIfEmpty(sp, false); + } else { + const uintptr_t new_end = ptr_span->start.u + num_pages * TlsHeapStatic::PageSize; + const size_t new_num_pages = next_span->num_pages - (num_pages - ptr_span->num_pages); + ChangeRangeOfSpan(std::addressof(this->span_table), next_span, new_end, new_num_pages); + ChangeRangeOfSpan(std::addressof(this->span_table), ptr_span, ptr_span->start.u, num_pages); + this->MergeIntoFreeList(next_span); + } + } + } + + *p = ptr; + this->lock.Unlock(); + return 0; + } + + errno_t TlsHeapCentral::ShrinkLargeMemory(void *ptr, size_t size) { + if (!TlsHeapStatic::IsPageAligned(ptr)) { + return EFAULT; + } + + AMS_ASSERT(size <= MaxSize); + + std::scoped_lock lk(this->lock); + + Span *ptr_span = GetSpanFromPointer(std::addressof(this->span_table), ptr); + if (!ptr_span) { + AMS_ASSERT(ptr_span != nullptr); + return EFAULT; + } + + const size_t num_pages = TlsHeapStatic::GetPageIndex(size + TlsHeapStatic::PageSize - 1); + if (ptr_span->num_pages != num_pages) { + if (ptr_span->num_pages < num_pages) { + return EINVAL; + } + Span *span = this->AllocateSpanStruct(); + if (span != nullptr) { + span->start.u = ptr_span->start.u + (num_pages * TlsHeapStatic::PageSize); + span->num_pages = ptr_span->num_pages - num_pages; + span->id = 0; + span->status = Span::Status_InUse; + ChangeRangeOfSpan(std::addressof(this->span_table), ptr_span, ptr_span->start.u, num_pages); + RegisterSpan(std::addressof(this->span_table), span); + this->FreePagesImpl(span); + } + } + + return 0; + } + + void TlsHeapCentral::CalculateHeapHash(HeapHash *out) { + size_t alloc_count = 0; + size_t alloc_size = 0; + size_t hash = 0; + + { + std::scoped_lock lk(this->lock); + + for (Span *span = GetSpanFromPointer(std::addressof(this->span_table), this); span != nullptr; span = GetNextSpan(std::addressof(this->span_table), span)) { + if (span->status != Span::Status_InUse) { + continue; + } + + const size_t size = span->num_pages * TlsHeapStatic::PageSize; + + if (span->page_class == 0) { + alloc_count++; + alloc_size += size; + hash += std::hash{}(size) + std::hash{}(span->start.u); + } else { + const size_t chunk_size = TlsHeapStatic::GetChunkSize(span->page_class); + + const size_t n = size / chunk_size; + AMS_ASSERT(n <= TlsHeapStatic::PageSize); + + static_assert(util::IsAligned(TlsHeapStatic::PageSize, BITSIZEOF(FreeListAvailableWord))); + FreeListAvailableWord flags[TlsHeapStatic::PageSize / BITSIZEOF(FreeListAvailableWord)]; + std::memset(flags, 0, sizeof(flags)); + + for (Span::SmallMemory *sm = span->aux.small.objects; sm != nullptr; sm = sm->next) { + const size_t idx = (reinterpret_cast(sm) - span->start.u) / chunk_size; + flags[FreeListAvailableIndex(idx)] |= FreeListAvailableMask(idx); + } + + for (size_t i = 0; i < n; i++) { + if (!(flags[FreeListAvailableIndex(i)] & FreeListAvailableMask(i))) { + alloc_count++; + alloc_size += chunk_size; + hash += std::hash{}(chunk_size) + std::hash{}(span->start.u + chunk_size * n); + } + } + } + } + } + + out->alloc_count = alloc_count; + out->alloc_size = alloc_size; + out->hash = hash; + } + + SpanPage *TlsHeapCentral::AllocateSpanPage() { + Span *span = this->SearchFreeSpan(1); + if (span == nullptr) { + return nullptr; + } + AMS_ASSERT(span->page_class == 0); + + if (this->use_virtual_memory && this->AllocatePhysical(span->start.p, TlsHeapStatic::PageSize) != 0) { + return nullptr; + } + + SpanPage *sp = static_cast(span->start.p); + + InitializeSpanPage(sp); + Span *new_span = GetSpanPageSpan(sp); + if (span->num_pages == 1) { + this->RemoveFromFreeBlockList(span); + MigrateSpan(std::addressof(this->span_table), span, new_span); + AMS_ASSERT(new_span->num_pages == 1); + new_span->status = Span::Status_InUseSystem; + + SpanPage *sp_of_span = GetSpanPage(span); + this->FreeSpanToSpanPage(span, sp_of_span); + this->DestroySpanPageIfEmpty(sp_of_span, false); + } else { + new_span->start.u = span->start.u; + new_span->num_pages = 1; + new_span->status = Span::Status_InUseSystem; + new_span->id = 0; + + if (span->num_pages - 1 < FreeListCount) { + this->RemoveFromFreeBlockList(span); + } + ChangeRangeOfSpan(std::addressof(this->span_table), span, span->start.u + TlsHeapStatic::PageSize, span->num_pages - 1); + RegisterSpan(std::addressof(this->span_table), new_span); + + if (span->num_pages < FreeListCount) { + this->AddToFreeBlockList(span); + } + } + + ListInsertAfter(std::addressof(this->spanpage_list), sp); + return sp; + } + + Span *TlsHeapCentral::AllocateSpanFromSpanPage(SpanPage *sp) { + const size_t span_idx = __builtin_clzll(~sp->info.alloc_bitmap); + AMS_ASSERT(span_idx < SpanPage::MaxSpanCount); + + constexpr u64 TopBit = static_cast(1) << (BITSIZEOF(u64) - 1); + sp->info.alloc_bitmap |= (TopBit >> span_idx); + sp->info.free_count--; + + Span *span = std::addressof(sp->spans[span_idx]); + ListClearLink(span); + span->status = Span::Status_NotUsed; + span->page_class = 0; + span->object_count = 0; + + if (sp->info.free_count == 0) { + ListRemoveSelf(sp); + ListInsertAfter(std::addressof(this->full_spanpage_list), sp); + } + + return span; + } + + Span *TlsHeapCentral::SplitSpan(Span *span, size_t num_pages, Span *new_span) { + AMS_ASSERT(span->status == Span::Status_InFreeList); + AMS_ASSERT(span->num_pages > num_pages); + + const size_t remaining_pages = span->num_pages - num_pages; + + uintptr_t new_start, old_start; + if (num_pages < FreeListCount) { + new_start = span->start.u; + old_start = span->start.u + (num_pages * TlsHeapStatic::PageSize); + } else { + new_start = span->start.u + (remaining_pages * TlsHeapStatic::PageSize); + old_start = span->start.u; + } + + + if (remaining_pages >= FreeListCount) { + ChangeRangeOfSpan(std::addressof(this->span_table), span, old_start, remaining_pages); + } else { + this->RemoveFromFreeBlockList(span); + ChangeRangeOfSpan(std::addressof(this->span_table), span, old_start, remaining_pages); + this->AddToFreeBlockList(span); + } + + new_span->start.u = new_start; + new_span->num_pages = num_pages; + new_span->page_class = 0; + new_span->status = Span::Status_InUse; + new_span->id = 0; + new_span->aux.large_clear.zero = 0; + span->aux.large_clear.zero = 0; + + if (this->use_virtual_memory && this->AllocatePhysical(new_span->start.p, new_span->num_pages * TlsHeapStatic::PageSize) != 0) { + new_span->status = Span::Status_InFreeList; + this->MergeIntoFreeList(new_span); + return nullptr; + } + + RegisterSpan(std::addressof(this->span_table), new_span); + return new_span; + } + + void TlsHeapCentral::MergeFreeSpans(Span *span, Span *span_to_merge, uintptr_t start) { + const size_t total_pages = span->num_pages + span_to_merge->num_pages; + UnregisterSpan(std::addressof(this->span_table), span_to_merge); + SpanPage *span_page = GetSpanPage(span_to_merge); + this->FreeSpanToSpanPage(span_to_merge, span_page); + ChangeRangeOfSpan(std::addressof(this->span_table), span, start, total_pages); + } + + bool TlsHeapCentral::DestroySpanPageIfEmpty(SpanPage *sp, bool full) { + if (sp->info.is_sticky) { + if (!(full || sp->info.free_count > 0x10)) { + return false; + } + if (!sp->info.free_count) { + return false; + } + + size_t first = this->FreeListFirstNonEmpty(0); + + while (first < FreeListCount) { + for (Span *target = ListGetNext(std::addressof(this->freelists[first])); target; target = ListGetNext(target)) { + AMS_ASSERT(target->status == Span::Status_InFreeList); + + SpanPage *target_sp = GetSpanPage(target); + if (target_sp != sp) { + Span *new_span = this->AllocateSpanFromSpanPage(sp); + AMS_ASSERT(new_span != nullptr); + + MigrateSpan(std::addressof(this->span_table), target, new_span); + this->FreeSpanToSpanPage(target, target_sp); + this->DestroySpanPageIfEmpty(target_sp, full); + } + } + first = this->FreeListFirstNonEmpty(first + 1); + } + + return false; + } else if (sp->info.free_count == SpanPage::MaxSpanCount) { + if (!ListGetNext(sp) && !ListGetPrev(sp)) { + return false; + } + SpanPage *other_sp = GetSpanPage(this->GetFirstSpan()); + + SpanPage *target; + if (other_sp->info.free_count > 0x10) { + target = other_sp; + } else { + for (target = ListGetNext(std::addressof(this->spanpage_list)); target && (target == sp || !target->info.free_count); target = ListGetNext(target)) { + /* ... */ + } + if (!target) { + if (!other_sp->info.free_count) { + return false; + } + target = other_sp; + } + } + + Span *new_span = this->AllocateSpanFromSpanPage(target); + AMS_ASSERT(new_span != nullptr); + + MigrateSpan(std::addressof(this->span_table), GetSpanPageSpan(sp), new_span); + + ListRemoveSelf(sp); + this->FreePagesImpl(new_span); + return true; + } else { + return false; + } + } + + Span *TlsHeapCentral::GetFirstSpan() const { + Span *span = GetSpanFromPointer(std::addressof(this->span_table), reinterpret_cast(this)); + AMS_ASSERT(span != nullptr); + return GetNextSpan(std::addressof(this->span_table), span); + } + + Span *TlsHeapCentral::MakeFreeSpan(size_t num_pages) { + while (true) { + SpanPage *sp; + for (sp = ListGetNext(std::addressof(this->spanpage_list)); sp && !this->DestroySpanPageIfEmpty(sp, true); sp = ListGetNext(sp)) { + /* ... */ + } + if (!sp) { + break; + } + if (Span *span = this->SearchFreeSpan(num_pages); span != nullptr) { + return span; + } + } + return nullptr; + } + + Span *TlsHeapCentral::SearchFreeSpan(size_t num_pages) const { + size_t start = FreeListCount - 1; + if (num_pages < FreeListCount) { + start = this->FreeListFirstNonEmpty(num_pages - 1); + if (start == FreeListCount) { + return nullptr; + } + } + + Span *cur = ListGetNext(std::addressof(this->freelists[start])); + Span *best = cur; + if (start == FreeListCount - 1) { + if (num_pages >= FreeListCount) { + best = nullptr; + while (cur) { + if (num_pages <= cur->num_pages) { + if (best) { + if (cur->num_pages >= best->num_pages) { + if (cur->num_pages == best->num_pages && cur->start.u < best->start.u) { + best = cur; + } + } else { + best = cur; + } + } else { + best = cur; + } + } + cur = ListGetNext(cur); + } + } else { + while (cur) { + if (cur->num_pages >= best->num_pages) { + if (cur->num_pages == best->num_pages && cur->start.u < best->start.u) { + best = cur; + } + } else { + best = cur; + } + cur = ListGetNext(cur); + } + } + } + + if (best != nullptr) { + AMS_ASSERT(best->status == Span::Status_InFreeList); + } + + return best; + } + + void TlsHeapCentral::FreeSpanToSpanPage(Span *span, SpanPage *sp) { + span->status = Span::Status_NotUsed; + + const size_t span_idx = span - sp->spans; + + constexpr u64 TopBit = static_cast(1) << (BITSIZEOF(u64) - 1); + sp->info.alloc_bitmap &= ~(TopBit >> span_idx); + if ((++(sp->info.free_count)) == 1) { + ListRemoveSelf(sp); + ListInsertAfter(std::addressof(this->spanpage_list), sp); + } + } + + void TlsHeapCentral::FreeSpanToSpanPage(Span *span) { + return this->FreeSpanToSpanPage(span, GetSpanPage(span)); + } + + void TlsHeapCentral::MergeIntoFreeList(Span *&span) { + AMS_ASSERT(!span->list_prev && !span->list_next); + AMS_ASSERT(span->status != Span::Status_InUse); + + Span *prev_span = GetPrevSpan(std::addressof(this->span_table), span); + Span *next_span = GetNextSpan(std::addressof(this->span_table), span); + const bool prev_free = prev_span && prev_span->status == Span::Status_InFreeList; + const bool prev_small = prev_span && prev_span->num_pages < FreeListCount; + const bool next_free = next_span && next_span->status == Span::Status_InFreeList; + const bool next_small = next_span && next_span->num_pages < FreeListCount; + + if (prev_free) { + if (next_free) { + if (prev_small) { + if (next_small) { + this->RemoveFromFreeBlockList(prev_span); + this->RemoveFromFreeBlockList(next_span); + this->MergeFreeSpans(prev_span, span, prev_span->start.u); + this->MergeFreeSpans(prev_span, next_span, prev_span->start.u); + this->AddToFreeBlockList(prev_span); + span = prev_span; + } else { + this->RemoveFromFreeBlockList(prev_span); + this->MergeFreeSpans(prev_span, span, prev_span->start.u); + this->MergeFreeSpans(next_span, prev_span, prev_span->start.u); + span = next_span; + } + } else { + this->RemoveFromFreeBlockList(next_span); + this->MergeFreeSpans(prev_span, span, prev_span->start.u); + this->MergeFreeSpans(prev_span, next_span, prev_span->start.u); + span = prev_span; + } + } else { + if (prev_small) { + this->RemoveFromFreeBlockList(prev_span); + this->MergeFreeSpans(prev_span, span, prev_span->start.u); + this->AddToFreeBlockList(prev_span); + } else { + this->MergeFreeSpans(prev_span, span, prev_span->start.u); + } + span = prev_span; + } + } else if (next_free) { + if (next_small) { + this->RemoveFromFreeBlockList(next_span); + this->MergeFreeSpans(span, next_span, span->start.u); + this->AddToFreeBlockList(span); + } else { + this->MergeFreeSpans(next_span, span, span->start.u); + span = next_span; + } + } else { + this->AddToFreeBlockList(span); + } + + AMS_ASSERT(GetSpanPageSpan(GetSpanPage(span)) != span); + AMS_ASSERT(span->status == Span::Status_InFreeList); + } + + errno_t TlsHeapCentral::AllocatePhysical(void *start, size_t size) { + return 0; + } + + errno_t TlsHeapCentral::FreePhysical(void *start, size_t size) { + const uintptr_t start_alignup = TlsHeapStatic::AlignUpPhysicalPage(reinterpret_cast(start)); + const uintptr_t end_aligndown = TlsHeapStatic::AlignDownPhysicalPage(reinterpret_cast(start) + size); + if (end_aligndown <= start_alignup) { + return 0; + } + + const uintptr_t phys_heap_start = TlsHeapStatic::AlignDownPhysicalPage(reinterpret_cast(this)); + const uintptr_t i = TlsHeapStatic::GetPhysicalPageIndex(start_alignup - phys_heap_start); + const uintptr_t idx_end = TlsHeapStatic::GetPhysicalPageIndex(end_aligndown - phys_heap_start); + AMS_ASSERT(i < idx_end); + + if (i + 1 == idx_end) { + if (this->physical_page_flags[i]) { + this->physical_page_flags[i] = 2; + } + } else { + const void *set_flag = ::memchr(std::addressof(this->physical_page_flags[i]), 1, idx_end - i); + if (set_flag) { + const uintptr_t set_idx = reinterpret_cast(set_flag) - this->physical_page_flags; + const void *lst_flag = ::memrchr(std::addressof(this->physical_page_flags[set_idx]), 1, idx_end - set_idx); + const uintptr_t lst_idx = (lst_flag) ? (reinterpret_cast(lst_flag) - this->physical_page_flags + 1) : idx_end; + std::memset(std::addressof(this->physical_page_flags[set_idx]), 2, lst_idx - set_idx); + } + } + + return 0; + } + + Span *TlsHeapCentral::AllocatePagesImpl(size_t num_pages) { + if (num_pages >= this->span_table.total_pages / 4) { + this->MakeFreeSpan(std::numeric_limits::max()); + } + + Span *span = this->SearchFreeSpan(num_pages); + if (span == nullptr) { + span = this->MakeFreeSpan(num_pages); + if (span == nullptr) { + return nullptr; + } + } + + AMS_ASSERT(span->status == Span::Status_InFreeList); + + if (num_pages == span->num_pages) { + if (this->use_virtual_memory && this->AllocatePhysical(span->start.p, span->num_pages * TlsHeapStatic::PageSize) != 0) { + return nullptr; + } else { + this->RemoveFromFreeBlockList(span); + span->status = Span::Status_InUse; + span->id = 0; + span->aux.large_clear.zero = 0; + return span; + } + } else { + Span *new_span = this->AllocateSpanStruct(); + if (new_span == nullptr) { + return nullptr; + } + auto new_span_guard = SCOPE_GUARD { this->FreeSpanToSpanPage(new_span); }; + + const size_t prev_pages = span->num_pages; + span = GetSpanFromPointer(std::addressof(this->span_table), span->start.p); + const size_t found_pages = span->num_pages; + + + if (found_pages != prev_pages) { + span = this->SearchFreeSpan(num_pages); + if (span == nullptr) { + return nullptr; + } + } + + if (found_pages == prev_pages || num_pages != span->num_pages) { + /* We're going to use the new span. */ + new_span_guard.Cancel(); + + return this->SplitSpan(span, num_pages, new_span); + } else if (this->use_virtual_memory && this->AllocatePhysical(span->start.p, span->num_pages * TlsHeapStatic::PageSize) != 0) { + return nullptr; + } else { + this->RemoveFromFreeBlockList(span); + span->status = Span::Status_InUse; + span->id = 0; + span->aux.large_clear.zero = 0; + return span; + } + } + } + + Span *TlsHeapCentral::AllocatePagesWithBigAlignImpl(size_t num_pages, size_t align) { + if (num_pages >= this->span_table.total_pages / 4) { + this->MakeFreeSpan(std::numeric_limits::max()); + } + + Span *before_span = this->AllocateSpanStruct(); + if (!before_span) { + return nullptr; + } + + Span *after_span = this->AllocateSpanStruct(); + if (!after_span) { + this->FreeSpanToSpanPage(before_span); + return nullptr; + } + + AMS_ASSERT(align <= TlsHeapStatic::PageSize); + AMS_ASSERT(util::IsPowerOfTwo(align)); + + Span *span = this->SearchFreeSpan(num_pages); + if (span == nullptr) { + span = this->MakeFreeSpan(num_pages); + if (span == nullptr) { + this->FreeSpanToSpanPage(before_span); + this->FreeSpanToSpanPage(after_span); + return nullptr; + } + } + + AMS_ASSERT(span->status == Span::Status_InFreeList); + + const uintptr_t aligned_start = util::AlignUp(span->start.u, align); + if (this->use_virtual_memory && this->AllocatePhysical(reinterpret_cast(aligned_start), num_pages * TlsHeapStatic::PageSize) != 0) { + this->FreeSpanToSpanPage(before_span); + this->FreeSpanToSpanPage(after_span); + return nullptr; + } else if (aligned_start == span->start.u) { + /* We don't need the before span, but we do need the after span. */ + this->FreeSpanToSpanPage(before_span); + + after_span->start.u = aligned_start + num_pages * TlsHeapStatic::PageSize; + after_span->num_pages = span->num_pages - num_pages; + after_span->page_class = 0; + after_span->status = Span::Status_InFreeList; + after_span->id = 0; + + this->RemoveFromFreeBlockList(span); + + span->status = Span::Status_InUse; + span->aux.large_clear.zero = 0; + ChangeRangeOfSpan(std::addressof(this->span_table), span, aligned_start, num_pages); + + RegisterSpan(std::addressof(this->span_table), after_span); + this->MergeIntoFreeList(after_span); + + return span; + } else { + /* We need the before span. */ + before_span->start.u = span->start.u; + before_span->num_pages = TlsHeapStatic::GetPageIndex(aligned_start - span->start.u); + before_span->page_class = 0; + before_span->status = Span::Status_InFreeList; + before_span->id = 0; + + const size_t after_pages = span->num_pages - before_span->num_pages - num_pages; + if (after_pages) { + /* We need the after span. */ + after_span->start.u = aligned_start + num_pages * TlsHeapStatic::PageSize; + after_span->num_pages = after_pages; + after_span->page_class = 0; + after_span->status = Span::Status_InFreeList; + after_span->id = 0; + + this->RemoveFromFreeBlockList(span); + + span->status = Span::Status_InUse; + span->aux.large_clear.zero = 0; + + ChangeRangeOfSpan(std::addressof(this->span_table), span, aligned_start, num_pages); + + RegisterSpan(std::addressof(this->span_table), before_span); + RegisterSpan(std::addressof(this->span_table), after_span); + this->MergeIntoFreeList(before_span); + this->MergeIntoFreeList(after_span); + + return span; + } else { + /* We don't need the after span. */ + this->FreeSpanToSpanPage(after_span); + + this->RemoveFromFreeBlockList(span); + + span->status = Span::Status_InUse; + span->aux.large_clear.zero = 0; + + ChangeRangeOfSpan(std::addressof(this->span_table), span, aligned_start, num_pages); + + RegisterSpan(std::addressof(this->span_table), before_span); + this->MergeIntoFreeList(before_span); + + return span; + } + } + } + + void TlsHeapCentral::FreePagesImpl(Span *span) { + AMS_ASSERT(span && span->page_class == 0); + + if (span->status == Span::Status_InFreeList) { + /* Double free error. */ + } else { + span->status = Span::Status_InFreeList; + if (this->use_virtual_memory) { + const uintptr_t start = span->start.u; + const uintptr_t end = span->start.u + (span->num_pages * TlsHeapStatic::PageSize); + uintptr_t start_alignup = TlsHeapStatic::AlignUpPhysicalPage(start); + uintptr_t end_aligndown = TlsHeapStatic::AlignDownPhysicalPage(end); + + this->MergeIntoFreeList(span); + + const uintptr_t new_start = span->start.u; + const uintptr_t new_end = span->start.u + (span->num_pages * TlsHeapStatic::PageSize); + + if (start != start_alignup && new_start + TlsHeapStatic::PhysicalPageSize <= start_alignup) { + start_alignup -= TlsHeapStatic::PhysicalPageSize; + } + if (end != end_aligndown && end_aligndown + TlsHeapStatic::PhysicalPageSize <= new_end) { + end_aligndown += TlsHeapStatic::PhysicalPageSize; + } + + if (start_alignup < end_aligndown) { + auto err = this->FreePhysical(reinterpret_cast(start_alignup), end_aligndown - start_alignup); + AMS_ASSERT(err == 0); + } + } else { + this->MergeIntoFreeList(span); + } + } + } + + void *TlsHeapCentral::CacheSmallMemoryImpl(size_t cls, size_t align, bool for_system) { + AMS_ASSERT(cls != 0 && cls < TlsHeapStatic::NumClassInfo); + + Span *span = ListGetNext(std::addressof(this->smallmem_lists[cls])); + while (true) { + if (for_system) { + while (span && span->status != Span::Status_InUseSystem) { + span = ListGetNext(span); + } + } else { + while (span && (span->status == Span::Status_InUseSystem || span->id)) { + span = ListGetNext(span); + } + } + + if (!span) { + break; + } + + void *mem = AllocateSmallMemory(span); + if (mem) { + if (!span->aux.small.objects) { + ListRemoveSelf(span); + } + return mem; + } + + AMS_ASSERT(!span->aux.small.objects); + + Span *old = span; + span = ListGetNext(span); + ListRemoveSelf(old); + } + + + Span *new_span = this->AllocatePagesImpl(TlsHeapStatic::GetNumPages(cls)); + if (new_span) { + SpanToSmallMemorySpan(std::addressof(this->span_table), new_span, cls); + ListInsertAfter(std::addressof(this->smallmem_lists[cls]), new_span); + InitSmallMemorySpan(new_span, cls, for_system, 0); + + void *mem = AllocateSmallMemory(new_span); + if (!new_span->aux.small.objects) { + ListRemoveSelf(new_span); + } + return mem; + } else { + for (size_t cur_cls = cls; cur_cls < TlsHeapStatic::NumClassInfo; cur_cls++) { + if (align == 0 || util::IsAligned(TlsHeapStatic::GetChunkSize(cur_cls), align)) { + span = ListGetNext(std::addressof(this->smallmem_lists[cur_cls])); + if (for_system) { + while (span && span->status != Span::Status_InUseSystem) { + span = ListGetNext(span); + } + } else { + while (span && (span->status == Span::Status_InUseSystem)) { + span = ListGetNext(span); + } + } + if (!span) { + continue; + } + + void *mem = AllocateSmallMemory(span); + if (!mem) { + continue; + } + + if (!span->aux.small.objects) { + ListRemoveSelf(span); + } + return mem; + } + } + return nullptr; + } + } + + errno_t TlsHeapCentral::UncacheSmallMemoryImpl(void *ptr) { + Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); + if (span && span->page_class) { + if (!span->aux.small.objects) { + ListInsertAfter(std::addressof(this->smallmem_lists[span->page_class]), span); + } + + ReleaseSmallMemory(span, ptr); + + if (!span->object_count) { + span->aux.small.objects = nullptr; + ListRemoveSelf(span); + AMS_ASSERT(span->page_class != 0); + SmallMemorySpanToSpan(std::addressof(this->span_table), span); + this->FreePagesImpl(span); + } + + return 0; + } else { + AMS_ASSERT(span); + AMS_ASSERT(span->page_class == 0); + return EFAULT; + } + } + + size_t TlsHeapCentral::CacheSmallMemoryListImpl(TlsHeapCache *cache, size_t *cls, size_t count, void **p, s32 cpu_id, size_t align) { + AMS_ASSERT(*cls != 0 && *cls < TlsHeapStatic::NumClassInfo); + + Span::SmallMemory head = {}; + Span::SmallMemory *hptr = std::addressof(head); + + Span *span = ListGetNext(std::addressof(this->smallmem_lists[*cls])); + size_t n = 0; + + while (span) { + if (span->status != Span::Status_InUseSystem && span->id == cpu_id) { + MangledSmallMemory memlist; + if (size_t num = AllocateSmallMemory(span, cache, count - n, std::addressof(memlist)); num != 0) { + hptr->next = memlist.from; + hptr = memlist.to; + memlist.to->next = nullptr; + n += num; + if (n >= count) { + if (!span->aux.small.objects) { + ListRemoveSelf(span); + } + *p = head.next; + return n; + } + } + + AMS_ASSERT(span->aux.small.objects); + + Span *old = span; + span = ListGetNext(span); + ListRemoveSelf(old); + } else { + span = ListGetNext(span); + } + } + + Span *new_span = this->AllocatePagesImpl(TlsHeapStatic::GetNumPages(*cls)); + if (new_span) { + SpanToSmallMemorySpan(std::addressof(this->span_table), new_span, *cls); + ListInsertAfter(std::addressof(this->smallmem_lists[*cls]), new_span); + InitSmallMemorySpan(new_span, *cls, false, cpu_id); + + MangledSmallMemory memlist; + size_t num = AllocateSmallMemory(new_span, cache, count - n, std::addressof(memlist)); + AMS_ASSERT(num > 0); + + hptr->next = memlist.from; + hptr = memlist.to; + memlist.to->next = nullptr; + n += num; + + if (!new_span->aux.small.objects) { + ListRemoveSelf(new_span); + } + + *p = head.next; + return n; + } else if (head.next) { + *p = head.next; + return n; + } else { + for (size_t cur_cls = *cls; cur_cls < TlsHeapStatic::NumClassInfo; cur_cls++) { + if (align == 0 || util::IsAligned(TlsHeapStatic::GetChunkSize(cur_cls), align)) { + span = ListGetNext(std::addressof(this->smallmem_lists[cur_cls])); + + while (span && (span->status == Span::Status_InUseSystem)) { + span = ListGetNext(span); + } + + if (!span) { + continue; + } + + void *mem = AllocateSmallMemory(span); + if (!mem) { + continue; + } + + if (!span->aux.small.objects) { + ListRemoveSelf(span); + } + + reinterpret_cast(mem)->next = nullptr; + *p = cache->ManglePointer(mem); + *cls = cur_cls; + return 1; + } + } + + *p = nullptr; + return 0; + } + } + + errno_t TlsHeapCentral::WalkAllocatedPointersImpl(HeapWalkCallback callback, void *user_data) { + errno_t err = ENOENT; + + for (Span *span = GetSpanFromPointer(std::addressof(this->span_table), this); span != nullptr; span = GetNextSpan(std::addressof(this->span_table), span)) { + if (span->status != Span::Status_InUse) { + continue; + } + + const size_t size = span->num_pages * TlsHeapStatic::PageSize; + + if (span->page_class != 0) { + static_assert(util::IsAligned(TlsHeapStatic::PageSize, BITSIZEOF(FreeListAvailableWord))); + FreeListAvailableWord flags[TlsHeapStatic::PageSize / BITSIZEOF(FreeListAvailableWord)]; + std::memset(flags, 0, sizeof(flags)); + + const size_t chunk_size = TlsHeapStatic::GetChunkSize(span->page_class); + const size_t n = size / chunk_size; + + AMS_ASSERT(n <= TlsHeapStatic::PageSize); + if (n <= TlsHeapStatic::PageSize) { + for (Span::SmallMemory *sm = span->aux.small.objects; sm != nullptr; sm = sm->next) { + const size_t idx = (reinterpret_cast(sm) - span->start.u) / chunk_size; + flags[FreeListAvailableIndex(idx)] |= FreeListAvailableMask(idx); + } + + for (size_t i = 0; i < n; i++) { + if (!(flags[FreeListAvailableIndex(i)] & FreeListAvailableMask(i))) { + if (s32 res = this->CallWalkCallback(callback, reinterpret_cast(span->start.u + chunk_size * i), chunk_size, user_data); res != 0) { + if (res >= 0) { + err = res; + break; + } else { + err = 0; + } + } + } + } + } else { + return EIO; + } + } else { + if (s32 res = this->CallWalkCallback(callback, span->start.p, size, user_data); res != 0) { + if (res >= 0) { + err = res; + break; + } else { + err = 0; + } + } + } + } + + return err; + } + + errno_t TlsHeapCentral::GetMappedMemStatsImpl(size_t *out_free_size, size_t *out_max_allocatable_size) { + if (!this->use_virtual_memory) { + return EOPNOTSUPP; + } + + /* TODO: Is this worth supporting? */ + return EOPNOTSUPP; + } + + errno_t TlsHeapCentral::GetMemStatsImpl(TlsHeapMemStats *out) { + size_t max_allocatable_size = 0; + size_t free_size = 0; + size_t system_size = 0; + size_t allocated_size = 0; + size_t num_free_regions = 0; + size_t num_free_spans = 0; + size_t wip_allocatable_size = 0; + + Span *span = GetSpanFromPointer(std::addressof(this->span_table), this); + while (span) { + const size_t size = span->num_pages * TlsHeapStatic::PageSize; + + if (span->status == Span::Status_InUse || span->status == Span::Status_InUseSystem) { + /* Found a usable span, so end our contiguous run. */ + max_allocatable_size = std::max(max_allocatable_size, wip_allocatable_size); + wip_allocatable_size = 0; + + if (span->status == Span::Status_InUseSystem) { + system_size += size; + } else if (span->page_class && span->aux.small.objects) { + const size_t chunk_size = TlsHeapStatic::GetChunkSize(span->page_class); + const size_t used_size = chunk_size * span->object_count; + allocated_size += used_size; + free_size += size - used_size; + } else { + allocated_size += size; + } + } else { + /* Free span. */ + free_size += size; + num_free_spans++; + if (wip_allocatable_size == 0) { + num_free_regions++; + } + wip_allocatable_size += size; + } + + span = GetNextSpan(std::addressof(this->span_table), span); + } + + max_allocatable_size = std::max(max_allocatable_size, wip_allocatable_size); + + bool sp_full = true; + for (SpanPage *sp = ListGetNext(std::addressof(this->spanpage_list)); sp != nullptr; sp = ListGetNext(sp)) { + if (sp->info.is_sticky == 0 && CanAllocateSpan(sp)) { + sp_full = false; + break; + } + } + + if (num_free_spans == 1 && allocated_size == 0) { + sp_full = false; + } + + if (sp_full && num_free_regions < 2 && max_allocatable_size >= TlsHeapStatic::PageSize) { + max_allocatable_size -= TlsHeapStatic::PageSize; + } + + if (max_allocatable_size < 3 * TlsHeapStatic::PageSize) { + max_allocatable_size = 0; + } + + out->max_allocatable_size = max_allocatable_size; + out->free_size = free_size; + out->allocated_size = allocated_size; + out->system_size = system_size; + + return 0; + } + + void TlsHeapCentral::DumpImpl(DumpMode dump_mode, int fd, bool json) { + AMS_ABORT("Not yet implemented"); + } + +} diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.hpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.hpp new file mode 100644 index 000000000..c67bf839a --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_central.hpp @@ -0,0 +1,547 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include "mem_impl_heap_platform.hpp" +#include "mem_impl_heap_tls_heap_static.hpp" +#include "mem_impl_heap_tls_heap_cache.hpp" + +namespace ams::mem::impl::heap { + + /* Simple intrusive list. */ + template + struct ListHeader { + T *list_next; + }; + + template + struct ListElement : public ListHeader { + T *list_prev; + }; + + template + constexpr inline void ListClearLink(ListHeader *l) { + l->list_next = nullptr; + } + + template + constexpr inline void ListClearLink(ListElement *l) { + l->list_next = nullptr; + l->list_prev = nullptr; + } + + template + constexpr inline T *ListGetNext(const ListHeader *l) { + return l->list_next; + } + + template + constexpr inline T *ListGetNext(const ListElement *l) { + return l->list_next; + } + + template + constexpr inline T *ListGetPrev(const ListElement *l) { + return l->list_prev; + } + + template + constexpr inline void ListInsertAfter(ListHeader *hdr, T *e) { + e->list_next = hdr->list_next; + e->list_prev = static_cast(hdr); + + if (hdr->list_next != nullptr) { + hdr->list_next->list_prev = e; + } + hdr->list_next = e; + } + + template + constexpr inline void ListRemoveSelf(T *e) { + if (e->list_next != nullptr) { + e->list_next->list_prev = e->list_prev; + } + if (e->list_prev != nullptr) { + e->list_prev->list_next = e->list_next; + } + e->list_next = nullptr; + e->list_prev = nullptr; + } + + struct Span : public ListElement { + struct SmallMemory { + SmallMemory *next; + }; + + enum Status : u8 { + Status_NotUsed = 0, + Status_InUse = 1, + Status_InFreeList = 2, + Status_InUseSystem = 3, + }; + + u16 object_count; + u8 page_class; + u8 status; + s32 id; + union { + uintptr_t u; + void *p; + SmallMemory *sm; + char *cp; + } start; + uintptr_t num_pages; + union { + struct { + SmallMemory *objects; + u64 is_allocated[8]; + } small; + struct { + u8 color[3]; + char name[0x10]; + } large; + struct { + u32 zero; + } large_clear; + } aux; + }; + + struct SpanPage : public ListElement { + struct Info { + u64 alloc_bitmap; + u16 free_count; + u8 is_sticky; + Span span_of_spanpage; + } info; + Span spans[(TlsHeapStatic::PageSize - sizeof(Info) - sizeof(ListElement)) / sizeof(Span)]; + + static constexpr size_t MaxSpanCount = sizeof(spans) / sizeof(spans[0]); + }; + static_assert(sizeof(SpanPage) <= TlsHeapStatic::PageSize); + + static constexpr ALWAYS_INLINE bool CanAllocateSpan(const SpanPage *span_page) { + return span_page->info.alloc_bitmap != ~(decltype(span_page->info.alloc_bitmap){}); + } + + struct SpanTable { + uintptr_t total_pages; + Span **page_to_span; + u8 *pageclass_cache; + }; + + struct TlsHeapMemStats { + size_t allocated_size; + size_t free_size; + size_t system_size; + size_t max_allocatable_size; + }; + + ALWAYS_INLINE Span *GetSpanFromPointer(const SpanTable *table, const void *ptr) { + const size_t idx = TlsHeapStatic::GetPageIndex(reinterpret_cast(ptr) - reinterpret_cast(table)); + if (idx < table->total_pages) { + return table->page_to_span[idx]; + } else { + return nullptr; + } + } + + ALWAYS_INLINE SpanPage *GetSpanPage(Span *span) { + return reinterpret_cast(TlsHeapStatic::AlignDownPage(reinterpret_cast(span))); + } + + ALWAYS_INLINE Span *GetSpanPageSpan(SpanPage *span_page) { + return std::addressof(span_page->info.span_of_spanpage); + } + + ALWAYS_INLINE Span *GetPrevSpan(const SpanTable *span_table, const Span *span) { + return GetSpanFromPointer(span_table, reinterpret_cast(span->start.u - 1)); + } + + ALWAYS_INLINE Span *GetNextSpan(const SpanTable *span_table, const Span *span) { + return GetSpanFromPointer(span_table, reinterpret_cast(span->start.u + span->num_pages * TlsHeapStatic::PageSize)); + } + + class TlsHeapCentral { + private: + using FreeListAvailableWord = u64; + + static constexpr size_t FreeListCount = 0x100; + static constexpr size_t NumFreeListBitmaps = FreeListCount / BITSIZEOF(FreeListAvailableWord); + + static constexpr ALWAYS_INLINE size_t FreeListAvailableIndex(size_t which) { + return which / BITSIZEOF(FreeListAvailableWord); + } + + static constexpr ALWAYS_INLINE size_t FreeListAvailableBit(size_t which) { + return which % BITSIZEOF(FreeListAvailableWord); + } + + static constexpr ALWAYS_INLINE FreeListAvailableWord FreeListAvailableMask(size_t which) { + return static_cast(1) << FreeListAvailableBit(which); + } + + static_assert(NumFreeListBitmaps * BITSIZEOF(FreeListAvailableWord) == FreeListCount); + private: + SpanTable span_table; + u8 *physical_page_flags; + s32 num_threads; + s32 static_thread_quota; + s32 dynamic_thread_quota; + bool use_virtual_memory; + os::RecursiveMutex lock; + ListHeader spanpage_list; + ListHeader full_spanpage_list; + ListHeader freelists[FreeListCount]; + FreeListAvailableWord freelists_bitmap[NumFreeListBitmaps]; + ListHeader smallmem_lists[TlsHeapStatic::NumClassInfo]; + public: + TlsHeapCentral() { + this->span_table.total_pages = 0; + } + + errno_t Initialize(void *start, size_t size, bool use_virtual_memory); + bool IsClean(); + + errno_t ReallocateLargeMemory(void *ptr, size_t size, void **p); + errno_t ShrinkLargeMemory(void *ptr, size_t size); + + void CalculateHeapHash(HeapHash *out); + + errno_t AddThreadCache(TlsHeapCache *cache) { + std::scoped_lock lk(this->lock); + + /* Add thread and recalculate. */ + this->num_threads++; + this->dynamic_thread_quota = this->GetTotalHeapSize() / (2 * this->num_threads); + + return 0; + } + + errno_t RemoveThreadCache(TlsHeapCache *cache) { + std::scoped_lock lk(this->lock); + + /* Remove thread and recalculate. */ + this->num_threads--; + this->dynamic_thread_quota = this->GetTotalHeapSize() / (2 * this->num_threads); + + return 0; + } + + void *CacheLargeMemory(size_t size) { + std::scoped_lock lk(this->lock); + + const size_t num_pages = util::AlignUp(size, TlsHeapStatic::PageSize) / TlsHeapStatic::PageSize; + if (Span *span = this->AllocatePagesImpl(num_pages); span != nullptr) { + return span->start.p; + } else { + return nullptr; + } + } + + void *CacheLargeMemoryWithBigAlign(size_t size, size_t align) { + std::scoped_lock lk(this->lock); + + const size_t num_pages = util::AlignUp(size, TlsHeapStatic::PageSize) / TlsHeapStatic::PageSize; + + Span *span = nullptr; + if (align > TlsHeapStatic::PageSize) { + span = this->AllocatePagesWithBigAlignImpl(num_pages, align); + } else { + span = this->AllocatePagesImpl(num_pages); + } + + if (span != nullptr) { + return span->start.p; + } else { + return nullptr; + } + } + + void *CacheSmallMemory(size_t cls, size_t align = 0) { + std::scoped_lock lk(this->lock); + + return this->CacheSmallMemoryImpl(cls, align, false); + } + + void *CacheSmallMemoryForSystem(size_t cls) { + std::scoped_lock lk(this->lock); + + return this->CacheSmallMemoryImpl(cls, 0, true); + } + + size_t CacheSmallMemoryList(TlsHeapCache *cache, size_t *cls, size_t count, void **p, size_t align = 0) { + std::scoped_lock lk(this->lock); + + s32 cpu_id = 0; + if (*cls < 8) { + getcpu(std::addressof(cpu_id)); + } + + return this->CacheSmallMemoryListImpl(cache, cls, count, p, cpu_id, 0); + } + + bool CheckCachedSize(s32 size) const { + return size < this->dynamic_thread_quota && size < this->static_thread_quota; + } + + void Dump(DumpMode dump_mode, int fd, bool json) { + std::scoped_lock lk(this->lock); + return this->DumpImpl(dump_mode, fd, json); + } + + size_t GetAllocationSize(const void *ptr) { + if (TlsHeapStatic::IsPageAligned(ptr)) { + Span *span = nullptr; + { + std::scoped_lock lk(this->lock); + span = GetSpanFromPointer(std::addressof(this->span_table), ptr); + } + if (span != nullptr) { + return span->num_pages * TlsHeapStatic::PageSize; + } else { + AMS_ASSERT(span != nullptr); + return 0; + } + } else { + /* TODO: Handle error? */ + return 0; + } + } + + size_t GetClassFromPointer(const void *ptr) { + std::atomic_thread_fence(std::memory_order_acquire); + + const size_t idx = (reinterpret_cast(ptr) - reinterpret_cast(this)) / TlsHeapStatic::PageSize; + if (idx < this->span_table.total_pages) { + if (ptr != nullptr) { + std::scoped_lock lk(this->lock); + Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); + if (span != nullptr) { + AMS_ASSERT(span->page_class == this->span_table.pageclass_cache[idx]); + } else { + AMS_ASSERT(span != nullptr); + } + } + return this->span_table.pageclass_cache[idx]; + } else { + /* TODO: Handle error? */ + return 0xFFFFFFFF; + } + } + + errno_t GetColor(const void *ptr, int *out) { + if (out == nullptr) { + return EINVAL; + } + + std::scoped_lock lk(this->lock); + if (Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); span != nullptr && !span->page_class) { + *out = (span->aux.large.color[0] << 0) | (span->aux.large.color[1] << 0) | (span->aux.large.color[2] << 16); + return 0; + } else { + return EINVAL; + } + } + + errno_t SetColor(const void *ptr, int color) { + std::scoped_lock lk(this->lock); + if (Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); span != nullptr && !span->page_class) { + span->aux.large.color[0] = (color >> 0) & 0xFF; + span->aux.large.color[1] = (color >> 8) & 0xFF; + span->aux.large.color[2] = (color >> 16) & 0xFF; + return 0; + } else { + return EINVAL; + } + } + + errno_t GetMappedMemStats(size_t *out_free_size, size_t *out_max_allocatable_size) { + std::scoped_lock lk(this->lock); + + return this->GetMappedMemStatsImpl(out_free_size, out_max_allocatable_size); + } + + errno_t GetMemStats(TlsHeapMemStats *out) { + std::scoped_lock lk(this->lock); + + return this->GetMemStatsImpl(out); + } + + errno_t GetName(const void *ptr, char *dst, size_t dst_size) { + std::scoped_lock lk(this->lock); + if (Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); span != nullptr && !span->page_class) { + strlcpy(dst, span->aux.large.name, dst_size); + return 0; + } else { + return EINVAL; + } + } + + errno_t SetName(const void *ptr, const char *name) { + std::scoped_lock lk(this->lock); + if (Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); span != nullptr && !span->page_class) { + strlcpy(span->aux.large.name, name, sizeof(span->aux.large.name)); + return 0; + } else { + return EINVAL; + } + } + + size_t GetTotalHeapSize() const { + return this->span_table.total_pages * TlsHeapStatic::PageSize; + } + + errno_t UncacheLargeMemory(void *ptr) { + if (TlsHeapStatic::IsPageAligned(ptr)) { + std::scoped_lock lk(this->lock); + if (Span *span = GetSpanFromPointer(std::addressof(this->span_table), ptr); span != nullptr) { + this->FreePagesImpl(span); + return 0; + } else { + return EFAULT; + } + } else { + return EFAULT; + } + } + + errno_t UncacheSmallMemory(void *ptr) { + std::scoped_lock lk(this->lock); + return this->UncacheSmallMemoryImpl(ptr); + } + + errno_t UncacheSmallMemoryList(TlsHeapCache *cache, void *ptr) { + std::scoped_lock lk(this->lock); + + while (true) { + if (ptr == nullptr) { + return 0; + } + ptr = cache->ManglePointer(ptr); + void *next = *reinterpret_cast(ptr); + if (auto err = this->UncacheSmallMemoryImpl(ptr); err != 0) { + return err; + } + ptr = next; + } + } + + errno_t WalkAllocatedPointers(HeapWalkCallback callback, void *user_data) { + /* Explicitly handle locking, as we will release the lock during callback. */ + this->lock.lock(); + ON_SCOPE_EXIT { this->lock.unlock(); }; + + return this->WalkAllocatedPointersImpl(callback, user_data); + } + private: + SpanPage *AllocateSpanPage(); + Span *AllocateSpanFromSpanPage(SpanPage *sp); + + Span *SplitSpan(Span *span, size_t num_pages, Span *new_span); + void MergeFreeSpans(Span *span, Span *span_to_merge, uintptr_t start); + + bool DestroySpanPageIfEmpty(SpanPage *sp, bool full); + Span *GetFirstSpan() const; + Span *MakeFreeSpan(size_t num_pages); + Span *SearchFreeSpan(size_t num_pages) const; + + void FreeSpanToSpanPage(Span *span, SpanPage *sp); + void FreeSpanToSpanPage(Span *span); + + void MergeIntoFreeList(Span *&span); + + errno_t AllocatePhysical(void *start, size_t size); + errno_t FreePhysical(void *start, size_t size); + private: + Span *AllocatePagesImpl(size_t num_pages); + Span *AllocatePagesWithBigAlignImpl(size_t num_pages, size_t align); + void FreePagesImpl(Span *span); + + void *CacheSmallMemoryImpl(size_t cls, size_t align, bool for_system); + errno_t UncacheSmallMemoryImpl(void *ptr); + + size_t CacheSmallMemoryListImpl(TlsHeapCache *cache, size_t *cls, size_t count, void **p, s32 cpu_id, size_t align); + + errno_t WalkAllocatedPointersImpl(HeapWalkCallback callback, void *user_data); + + errno_t GetMappedMemStatsImpl(size_t *out_free_size, size_t *out_max_allocatable_size); + errno_t GetMemStatsImpl(TlsHeapMemStats *out); + + void DumpImpl(DumpMode dump_mode, int fd, bool json); + private: + size_t FreeListFirstNonEmpty(size_t start) const { + if (start < FreeListCount) { + for (size_t i = FreeListAvailableIndex(start); i < util::size(this->freelists_bitmap); i++) { + const FreeListAvailableWord masked = this->freelists_bitmap[i] & ~(FreeListAvailableMask(start) - 1); + if (masked) { + const size_t b = __builtin_ctzll(masked); + const size_t res = i * BITSIZEOF(FreeListAvailableWord) + b; + AMS_ASSERT(res < FreeListCount); + return res; + } + start = (i + 1) * BITSIZEOF(FreeListAvailableWord); + } + } + return FreeListCount; + } + + ALWAYS_INLINE void AddToFreeBlockList(Span *span) { + AMS_ASSERT(GetSpanPageSpan(GetSpanPage(span)) != span); + AMS_ASSERT(span->status == Span::Status_InFreeList); + const size_t which = std::min(span->num_pages, FreeListCount) - 1; + ListInsertAfter(std::addressof(this->freelists[which]), span); + this->freelists_bitmap[FreeListAvailableIndex(which)] |= FreeListAvailableMask(which); + } + + ALWAYS_INLINE void RemoveFromFreeBlockList(Span *span) { + const size_t which = std::min(span->num_pages, FreeListCount) - 1; + ListRemoveSelf(span); + if (!ListGetNext(std::addressof(this->freelists[which]))) { + this->freelists_bitmap[FreeListAvailableIndex(which)] &= ~FreeListAvailableMask(which); + } + } + + Span *AllocateSpanStruct() { + SpanPage *sp = ListGetNext(std::addressof(this->spanpage_list)); + while (sp && (sp->info.is_sticky || !CanAllocateSpan(sp))) { + sp = ListGetNext(sp); + } + + if (sp == nullptr) { + sp = this->AllocateSpanPage(); + } + + if (sp != nullptr) { + return this->AllocateSpanFromSpanPage(sp); + } else { + return nullptr; + } + } + + s32 CallWalkCallback(HeapWalkCallback callback, void *ptr, size_t size, void *user_data) { + this->lock.unlock(); + int res = callback(ptr, size, user_data); + this->lock.lock(); + if (res) { + return 0; + } else { + return -1; + } + } + }; + + +} \ No newline at end of file diff --git a/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_static.hpp b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_static.hpp new file mode 100644 index 000000000..eace75938 --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/heap/mem_impl_heap_tls_heap_static.hpp @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include +#include "mem_impl_heap_platform.hpp" + +namespace ams::mem::impl::heap { + + class TlsHeapStatic { + public: + struct ClassInfo { + u16 num_pages; + u16 chunk_size; + }; + + static constexpr size_t NumClassInfo = 57; + + static constexpr size_t MaxSizeWithClass = 0xC00; + static constexpr size_t ChunkGranularity = 0x10; + static constexpr size_t PageSize = 4_KB; + static constexpr size_t PhysicalPageSize = 256_KB; + public: + static constexpr inline std::array ClassInfos = { + ClassInfo{ .num_pages = 0, .chunk_size = 0x000, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x010, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x020, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x030, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x040, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x050, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x060, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x070, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x080, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x090, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x0A0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x0B0, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x0C0, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x0D0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x0E0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x0F0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x100, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x110, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x120, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x130, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x140, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x150, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x160, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x170, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x180, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x190, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x1A0, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x1B0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x1C0, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x1D0, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x1E0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x200, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x210, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x220, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x240, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x260, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x270, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x280, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x2A0, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x2D0, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x2E0, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x300, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x330, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x360, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x380, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x3B0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x400, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x450, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x490, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x4C0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x550, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x600, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0x660, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x6D0, }, + ClassInfo{ .num_pages = 1, .chunk_size = 0x800, }, + ClassInfo{ .num_pages = 3, .chunk_size = 0x990, }, + ClassInfo{ .num_pages = 2, .chunk_size = 0xAA0, }, + }; + + static constexpr inline std::array SizeToClass = [] { + std::array arr = {}; + arr[0] = 1; + for (size_t i = 1; i < arr.size(); i++) { + const size_t cur_size = i * ChunkGranularity; + for (size_t j = 0; j < ClassInfos.size(); j++) { + if (ClassInfos[j].chunk_size >= cur_size) { + arr[i] = j; + break; + } + } + } + return arr; + }(); + public: + static constexpr ALWAYS_INLINE size_t GetClassFromSize(size_t size) { + AMS_ASSERT(size <= MaxSize); + const size_t idx = util::AlignUp(size, ChunkGranularity) / ChunkGranularity; + if (idx < MaxSizeWithClass / ChunkGranularity) { + return SizeToClass[idx]; + } else { + return 0; + } + } + + static constexpr ALWAYS_INLINE size_t GetRealSizeFromSizeAndAlignment(size_t size, size_t align) { + AMS_ASSERT(size <= MaxSize); + const size_t idx = util::AlignUp(size, ChunkGranularity) / ChunkGranularity; + if (size == 0 || idx >= MaxSizeWithClass / ChunkGranularity) { + return size; + } + const auto cls = SizeToClass[idx]; + if (!cls) { + return PageSize; + } + AMS_ASSERT(align != 0); + const size_t mask = align - 1; + for (auto i = cls; i < ClassInfos.size(); i++) { + if ((ClassInfos[i].chunk_size & mask) == 0) { + return ClassInfos[i].chunk_size; + } + } + return PageSize; + } + + static constexpr ALWAYS_INLINE bool IsPageAligned(uintptr_t ptr) { + return util::IsAligned(ptr, PageSize); + } + + static ALWAYS_INLINE bool IsPageAligned(const void *ptr) { + return IsPageAligned(reinterpret_cast(ptr)); + } + + static constexpr ALWAYS_INLINE size_t GetPageIndex(uintptr_t ptr) { + return ptr / PageSize; + } + + static constexpr ALWAYS_INLINE size_t GetPhysicalPageIndex(uintptr_t ptr) { + return ptr / PhysicalPageSize; + } + + static constexpr ALWAYS_INLINE uintptr_t AlignUpPage(uintptr_t ptr) { + return util::AlignUp(ptr, PageSize); + } + + template + static ALWAYS_INLINE T *AlignUpPage(T *ptr) { + static_assert(std::is_pod::value); + static_assert(util::IsAligned(PageSize, alignof(T))); + return reinterpret_cast(AlignUpPage(reinterpret_cast(ptr))); + } + + static constexpr ALWAYS_INLINE uintptr_t AlignDownPage(uintptr_t ptr) { + return util::AlignDown(ptr, PageSize); + } + + template + static ALWAYS_INLINE T *AlignDownPage(T *ptr) { + static_assert(std::is_pod::value); + static_assert(util::IsAligned(PageSize, alignof(T))); + return reinterpret_cast(AlignDownPage(reinterpret_cast(ptr))); + } + + static constexpr ALWAYS_INLINE uintptr_t AlignUpPhysicalPage(uintptr_t ptr) { + return util::AlignUp(ptr, PhysicalPageSize); + } + + template + static ALWAYS_INLINE T *AlignUpPhysicalPage(T *ptr) { + static_assert(std::is_pod::value); + static_assert(util::IsAligned(PhysicalPageSize, alignof(T))); + return reinterpret_cast(AlignUpPhysicalPage(reinterpret_cast(ptr))); + } + + static constexpr ALWAYS_INLINE uintptr_t AlignDownPhysicalPage(uintptr_t ptr) { + return util::AlignDown(ptr, PhysicalPageSize); + } + + template + static ALWAYS_INLINE T *AlignDownPhysicalPage(T *ptr) { + static_assert(std::is_pod::value); + static_assert(util::IsAligned(PhysicalPageSize, alignof(T))); + return reinterpret_cast(AlignDownPhysicalPage(reinterpret_cast(ptr))); + } + + static constexpr ALWAYS_INLINE size_t GetChunkSize(size_t cls) { + return ClassInfos[cls].chunk_size; + } + + static constexpr ALWAYS_INLINE size_t GetNumPages(size_t cls) { + return ClassInfos[cls].num_pages; + } + }; + +} \ No newline at end of file diff --git a/libraries/libstratosphere/source/mem/impl/mem_impl_platform.hpp b/libraries/libstratosphere/source/mem/impl/mem_impl_platform.hpp new file mode 100644 index 000000000..18a53b1a1 --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/mem_impl_platform.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::mem::impl { + + enum Prot { + Prot_none = (0 << 0), + Prot_read = (1 << 0), + Prot_write = (1 << 1), + Prot_exec = (1 << 2), + }; + + errno_t virtual_alloc(void **ptr, size_t size); + errno_t virtual_free(void *ptr, size_t size); + errno_t physical_alloc(void *ptr, size_t size, Prot prot); + errno_t physical_free(void *ptr, size_t size); + + size_t strlcpy(char *dst, const char *src, size_t size); + + errno_t gen_random(void *dst, size_t dst_size); + + errno_t epochtime(s64 *dst); + + errno_t getcpu(s32 *out); + +} diff --git a/libraries/libstratosphere/source/mem/impl/mem_impl_platform.os.horizon.cpp b/libraries/libstratosphere/source/mem/impl/mem_impl_platform.os.horizon.cpp new file mode 100644 index 000000000..d1f7e6634 --- /dev/null +++ b/libraries/libstratosphere/source/mem/impl/mem_impl_platform.os.horizon.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "mem_impl_platform.hpp" + +namespace ams::mem::impl { + + namespace { + + os::Mutex g_virt_mem_enabled_lock; + bool g_virt_mem_enabled_detected; + bool g_virt_mem_enabled; + + void EnsureVirtualAddressMemoryDetected() { + std::scoped_lock lk(g_virt_mem_enabled_lock); + if (AMS_LIKELY(g_virt_mem_enabled_detected)) { + return; + } + g_virt_mem_enabled = os::IsVirtualAddressMemoryEnabled(); + } + + ALWAYS_INLINE bool IsVirtualAddressMemoryEnabled() { + EnsureVirtualAddressMemoryDetected(); + return g_virt_mem_enabled; + } + + ALWAYS_INLINE errno_t ConvertResult(Result result) { + /* TODO: Actually implement this in a meaningful way. */ + if (R_FAILED(result)) { + return EINVAL; + } + return 0; + } + + ALWAYS_INLINE os::MemoryPermission ConvertToOsPermission(Prot prot) { + static_assert(static_cast(Prot_read) == static_cast(os::MemoryPermission_ReadOnly)); + static_assert(static_cast(Prot_write) == static_cast(os::MemoryPermission_WriteOnly)); + return static_cast(prot & os::MemoryPermission_ReadWrite); + } + + } + + errno_t virtual_alloc(void **ptr, size_t size) { + /* Ensure size isn't too large. */ + if (size > mem::impl::MaxSize) { + return EINVAL; + } + + /* Allocate virtual memory. */ + uintptr_t addr; + if (IsVirtualAddressMemoryEnabled()) { + /* TODO: Support virtual address memory. */ + AMS_ABORT("Virtual address memory not supported yet"); + } else { + if (auto err = ConvertResult(os::AllocateMemoryBlock(std::addressof(addr), util::AlignUp(size, os::MemoryBlockUnitSize))); err != 0) { + return err; + } + os::SetMemoryPermission(addr, size, os::MemoryPermission_None); + } + + return 0; + } + + errno_t virtual_free(void *ptr, size_t size) { + /* Ensure size isn't zero. */ + if (size == 0) { + return EINVAL; + } + + if (IsVirtualAddressMemoryEnabled()) { + /* TODO: Support virtual address memory. */ + AMS_ABORT("Virtual address memory not supported yet"); + } else { + os::FreeMemoryBlock(reinterpret_cast(ptr), util::AlignUp(size, os::MemoryBlockUnitSize)); + } + + return 0; + } + + errno_t physical_alloc(void *ptr, size_t size, Prot prot) { + /* Detect empty allocation. */ + const uintptr_t aligned_start = util::AlignDown(reinterpret_cast(ptr), os::MemoryPageSize); + const uintptr_t aligned_end = util::AlignUp(reinterpret_cast(ptr) + size, os::MemoryPageSize); + const size_t aligned_size = aligned_end - aligned_start; + if (aligned_end <= aligned_start) { + return 0; + } + + if (IsVirtualAddressMemoryEnabled()) { + /* TODO: Support virtual address memory. */ + AMS_ABORT("Virtual address memory not supported yet"); + } else { + os::SetMemoryPermission(aligned_start, aligned_size, ConvertToOsPermission(prot)); + } + + return 0; + } + + errno_t physical_free(void *ptr, size_t size) { + /* Detect empty allocation. */ + const uintptr_t aligned_start = util::AlignDown(reinterpret_cast(ptr), os::MemoryPageSize); + const uintptr_t aligned_end = util::AlignUp(reinterpret_cast(ptr) + size, os::MemoryPageSize); + const size_t aligned_size = aligned_end - aligned_start; + if (aligned_end <= aligned_start) { + return 0; + } + + if (IsVirtualAddressMemoryEnabled()) { + /* TODO: Support virtual address memory. */ + AMS_ABORT("Virtual address memory not supported yet"); + } else { + os::SetMemoryPermission(aligned_start, aligned_size, os::MemoryPermission_None); + } + + return 0; + } + + size_t strlcpy(char *dst, const char *src, size_t size) { + const size_t src_size = std::strlen(src); + if (src_size >= size) { + if (size) { + std::memcpy(dst, src, size - 1); + dst[size - 1] = 0; + } + } else { + std::memcpy(dst, src, src_size + 1); + } + return src_size; + } + + errno_t gen_random(void *dst, size_t dst_size) { + os::GenerateRandomBytes(dst, dst_size); + return 0; + } + + errno_t epochtime(s64 *dst) { + /* TODO: What is this calc? */ + auto ts = os::ConvertToTimeSpan(os::GetSystemTick()); + *dst = (ts.GetNanoSeconds() / INT64_C(100)) + INT64_C(0x8A09F909AE60000); + return 0; + } + + errno_t getcpu(s32 *out) { + *out = os::GetCurrentCoreNumber(); + return 0; + } + +} diff --git a/libraries/libstratosphere/source/mem/mem_standard_allocator.cpp b/libraries/libstratosphere/source/mem/mem_standard_allocator.cpp new file mode 100644 index 000000000..cc7125d40 --- /dev/null +++ b/libraries/libstratosphere/source/mem/mem_standard_allocator.cpp @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/mem_impl_platform.hpp" +#include "impl/heap/mem_impl_heap_tls_heap_static.hpp" +#include "impl/heap/mem_impl_heap_tls_heap_cache.hpp" +#include "impl/heap/mem_impl_heap_tls_heap_central.hpp" + +namespace ams::mem { + + constexpr inline size_t DefaultAlignment = alignof(std::max_align_t); + constexpr inline size_t MinimumAllocatorSize = 16_KB; + + namespace { + + void ThreadDestroy(uintptr_t arg) { + if (arg) { + reinterpret_cast(arg)->Finalize(); + } + } + + ALWAYS_INLINE impl::heap::CentralHeap *GetCentral(const impl::InternalCentralHeapStorage *storage) { + return reinterpret_cast(const_cast(storage)); + } + + ALWAYS_INLINE impl::heap::CentralHeap *GetCentral(const impl::InternalCentralHeapStorage &storage) { + return GetCentral(std::addressof(storage)); + } + + ALWAYS_INLINE void GetCache(impl::heap::CentralHeap *central, os::TlsSlot slot) { + impl::heap::CachedHeap tmp_cache; + + if (central->MakeCache(std::addressof(tmp_cache))) { + impl::heap::TlsHeapCache *cache = tmp_cache.Release(); + os::SetTlsValue(slot, reinterpret_cast(cache)); + } + } + + struct InternalHash { + size_t allocated_count; + size_t allocated_size; + crypto::Sha1Generator sha1; + }; + + int InternalHashCallback(void *ptr, size_t size, void *user_data) { + InternalHash *hash = reinterpret_cast(user_data); + hash->sha1.Update(reinterpret_cast(std::addressof(ptr)), sizeof(ptr)); + hash->sha1.Update(reinterpret_cast(std::addressof(size)), sizeof(size)); + hash->allocated_count++; + hash->allocated_size += size; + return 1; + } + + } + + StandardAllocator::StandardAllocator() : initialized(false), enable_thread_cache(false), unused(0) { + static_assert(sizeof(impl::heap::CentralHeap) <= sizeof(this->central_heap_storage)); + new (std::addressof(this->central_heap_storage)) impl::heap::CentralHeap; + } + + StandardAllocator::StandardAllocator(void *mem, size_t size) : StandardAllocator() { + this->Initialize(mem, size); + } + + StandardAllocator::StandardAllocator(void *mem, size_t size, bool enable_cache) : StandardAllocator() { + this->Initialize(mem, size, enable_cache); + } + + void StandardAllocator::Initialize(void *mem, size_t size) { + this->Initialize(mem, size, false); + } + + void StandardAllocator::Initialize(void *mem, size_t size, bool enable_cache) { + AMS_ABORT_UNLESS(!this->initialized); + + const uintptr_t aligned_start = util::AlignUp(reinterpret_cast(mem), impl::heap::TlsHeapStatic::PageSize); + const uintptr_t aligned_end = util::AlignDown(reinterpret_cast(mem) + size, impl::heap::TlsHeapStatic::PageSize); + const size_t aligned_size = aligned_end - aligned_start; + + if (mem == nullptr) { + AMS_ABORT_UNLESS(os::IsVirtualAddressMemoryEnabled()); + AMS_ABORT_UNLESS(GetCentral(this->central_heap_storage)->Initialize(nullptr, size, 0) == 0); + } else { + AMS_ABORT_UNLESS(aligned_start < aligned_end); + AMS_ABORT_UNLESS(aligned_size >= MinimumAllocatorSize); + AMS_ABORT_UNLESS(GetCentral(this->central_heap_storage)->Initialize(reinterpret_cast(aligned_start), aligned_size, 0) == 0); + } + + this->enable_thread_cache = enable_cache; + if (this->enable_thread_cache) { + R_ABORT_UNLESS(os::AllocateTlsSlot(std::addressof(this->tls_slot), ThreadDestroy)); + } + + this->initialized = true; + } + + void StandardAllocator::Finalize() { + AMS_ABORT_UNLESS(this->initialized); + + if (this->enable_thread_cache) { + os::FreeTlsSlot(this->tls_slot); + } + + GetCentral(this->central_heap_storage)->Finalize(); + + this->initialized = false; + } + + void *StandardAllocator::Allocate(size_t size) { + AMS_ASSERT(this->initialized); + return this->Allocate(size, DefaultAlignment); + } + + void *StandardAllocator::Allocate(size_t size, size_t alignment) { + AMS_ASSERT(this->initialized); + + impl::heap::TlsHeapCache *heap_cache = nullptr; + if (this->enable_thread_cache) { + heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + if (!heap_cache) { + GetCache(GetCentral(this->central_heap_storage), this->tls_slot); + heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + } + } + + void *ptr = nullptr; + if (heap_cache) { + ptr = heap_cache->Allocate(size, alignment); + if (ptr) { + return ptr; + } + + impl::heap::CachedHeap cache; + cache.Reset(heap_cache); + cache.Query(impl::AllocQuery_FinalizeCache); + os::SetTlsValue(this->tls_slot, 0); + } + + return GetCentral(this->central_heap_storage)->Allocate(size, alignment); + } + + void StandardAllocator::Free(void *ptr) { + AMS_ASSERT(this->initialized); + + if (ptr == nullptr) { + return; + } + + if (this->enable_thread_cache) { + impl::heap::TlsHeapCache *heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + if (heap_cache) { + heap_cache->Free(ptr); + return; + } + } + + auto err = GetCentral(this->central_heap_storage)->Free(ptr); + AMS_ASSERT(err == 0); + } + + void *StandardAllocator::Reallocate(void *ptr, size_t new_size) { + AMS_ASSERT(this->initialized); + + if (new_size > impl::MaxSize) { + return nullptr; + } + + if (ptr == nullptr) { + return this->Allocate(new_size); + } + + if (new_size == 0) { + this->Free(ptr); + return nullptr; + } + + size_t aligned_new_size = util::AlignUp(new_size, DefaultAlignment); + + + impl::heap::TlsHeapCache *heap_cache = nullptr; + if (this->enable_thread_cache) { + heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + if (!heap_cache) { + GetCache(GetCentral(this->central_heap_storage), this->tls_slot); + heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + } + } + + void *p = nullptr; + impl::errno_t err; + if (heap_cache) { + err = heap_cache->Reallocate(ptr, aligned_new_size, std::addressof(p)); + } else { + err = GetCentral(this->central_heap_storage)->Reallocate(ptr, aligned_new_size, std::addressof(p)); + } + + if (err == 0) { + return p; + } else { + return nullptr; + } + } + + size_t StandardAllocator::Shrink(void *ptr, size_t new_size) { + AMS_ASSERT(this->initialized); + + if (this->enable_thread_cache) { + impl::heap::TlsHeapCache *heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + if (heap_cache) { + if (heap_cache->Shrink(ptr, new_size) == 0) { + return heap_cache->GetAllocationSize(ptr); + } else { + return 0; + } + } + } + + if (GetCentral(this->central_heap_storage)->Shrink(ptr, new_size) == 0) { + return GetCentral(this->central_heap_storage)->GetAllocationSize(ptr); + } else { + return 0; + } + } + + void StandardAllocator::ClearThreadCache() const { + if (this->enable_thread_cache) { + impl::heap::TlsHeapCache *heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + impl::heap::CachedHeap cache; + cache.Reset(heap_cache); + cache.Query(impl::AllocQuery_ClearCache); + cache.Release(); + } + } + + void StandardAllocator::CleanUpManagementArea() const { + AMS_ASSERT(this->initialized); + + auto err = GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_UnifyFreeList); + AMS_ASSERT(err == 0); + } + + size_t StandardAllocator::GetSizeOf(const void *ptr) const { + AMS_ASSERT(this->initialized); + + if (!util::IsAligned(reinterpret_cast(ptr), DefaultAlignment)) { + return 0; + } + + impl::heap::TlsHeapCache *heap_cache = nullptr; + if (this->enable_thread_cache) { + heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + if (!heap_cache) { + GetCache(GetCentral(this->central_heap_storage), this->tls_slot); + heap_cache = reinterpret_cast(os::GetTlsValue(this->tls_slot)); + } + } + + if (heap_cache) { + return heap_cache->GetAllocationSize(ptr); + } else { + return GetCentral(this->central_heap_storage)->GetAllocationSize(ptr); + } + } + + size_t StandardAllocator::GetTotalFreeSize() const { + size_t size = 0; + + auto err = GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_FreeSizeMapped, std::addressof(size)); + if (err != 0) { + err = GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_FreeSize, std::addressof(size)); + } + AMS_ASSERT(err == 0); + + return size; + } + + size_t StandardAllocator::GetAllocatableSize() const { + size_t size = 0; + + auto err = GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_MaxAllocatableSizeMapped, std::addressof(size)); + if (err != 0) { + err = GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_MaxAllocatableSize, std::addressof(size)); + } + AMS_ASSERT(err == 0); + + return size; + } + + void StandardAllocator::WalkAllocatedBlocks(WalkCallback callback, void *user_data) const { + AMS_ASSERT(this->initialized); + this->ClearThreadCache(); + GetCentral(this->central_heap_storage)->WalkAllocatedPointers(callback, user_data); + } + + void StandardAllocator::Dump() const { + AMS_ASSERT(this->initialized); + + size_t tmp; + auto err = GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_MaxAllocatableSizeMapped, std::addressof(tmp)); + + if (err == 0) { + GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_Dump, impl::DumpMode_Spans | impl::DumpMode_Pointers, 1); + } else { + GetCentral(this->central_heap_storage)->Query(impl::AllocQuery_Dump, impl::DumpMode_All, 1); + } + } + + StandardAllocator::AllocatorHash StandardAllocator::Hash() const { + AMS_ASSERT(this->initialized); + + AllocatorHash alloc_hash; + { + char temp_hash[crypto::Sha1Generator::HashSize]; + InternalHash internal_hash; + internal_hash.allocated_count = 0; + internal_hash.allocated_size = 0; + internal_hash.sha1.Initialize(); + + this->WalkAllocatedBlocks(InternalHashCallback, reinterpret_cast(std::addressof(internal_hash))); + + alloc_hash.allocated_count = internal_hash.allocated_count; + alloc_hash.allocated_size = internal_hash.allocated_size; + + internal_hash.sha1.GetHash(temp_hash, sizeof(temp_hash)); + std::memcpy(std::addressof(alloc_hash.hash), temp_hash, sizeof(alloc_hash.hash)); + } + return alloc_hash; + } + + +} diff --git a/libraries/libstratosphere/source/os/impl/os_memory_permission_impl.hpp b/libraries/libstratosphere/source/os/impl/os_memory_permission_impl.hpp new file mode 100644 index 000000000..c9ae3b5f1 --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_memory_permission_impl.hpp @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +namespace ams::os::impl { + + void SetMemoryPermissionImpl(uintptr_t address, size_t size, MemoryPermission perm); + +} diff --git a/libraries/libstratosphere/source/os/impl/os_memory_permission_impl.os.horizon.cpp b/libraries/libstratosphere/source/os/impl/os_memory_permission_impl.os.horizon.cpp new file mode 100644 index 000000000..481c0c2bf --- /dev/null +++ b/libraries/libstratosphere/source/os/impl/os_memory_permission_impl.os.horizon.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::os::impl { + + namespace { + + void SetMemoryPermissionBySvc(uintptr_t start, size_t size, svc::MemoryPermission perm) { + uintptr_t cur_address = start; + size_t remaining_size = size; + while (remaining_size > 0) { + svc::MemoryInfo mem_info; + svc::PageInfo page_info; + R_ABORT_UNLESS(svc::QueryMemory(std::addressof(mem_info), std::addressof(page_info), cur_address)); + + size_t cur_size = std::min(mem_info.addr + mem_info.size - cur_address, remaining_size); + + if (mem_info.perm != perm) { + R_ABORT_UNLESS(svc::SetMemoryPermission(cur_address, cur_size, perm)); + } + + cur_address += cur_size; + remaining_size -= cur_size; + } + } + + } + + void SetMemoryPermissionImpl(uintptr_t address, size_t size, MemoryPermission perm) { + switch (perm) { + case MemoryPermission_None: + return SetMemoryPermissionBySvc(address, size, svc::MemoryPermission_None); + case MemoryPermission_ReadOnly: + return SetMemoryPermissionBySvc(address, size, svc::MemoryPermission_Read); + case MemoryPermission_ReadWrite: + return SetMemoryPermissionBySvc(address, size, svc::MemoryPermission_ReadWrite); + AMS_UNREACHABLE_DEFAULT_CASE(); + } + } + +} diff --git a/libraries/libstratosphere/source/os/os_memory_heap.cpp b/libraries/libstratosphere/source/os/os_memory_heap.cpp new file mode 100644 index 000000000..2d24b8877 --- /dev/null +++ b/libraries/libstratosphere/source/os/os_memory_heap.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::os { + + Result AllocateMemoryBlock(uintptr_t *out_address, size_t size) { + AMS_ABORT("Not implemented yet"); + } + + void FreeMemoryBlock(uintptr_t address, size_t size) { + AMS_ABORT("Not implemented yet"); + } + +} diff --git a/libraries/libstratosphere/source/os/os_memory_permission.cpp b/libraries/libstratosphere/source/os/os_memory_permission.cpp new file mode 100644 index 000000000..b3ea8209b --- /dev/null +++ b/libraries/libstratosphere/source/os/os_memory_permission.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include "impl/os_memory_permission_impl.hpp" + +namespace ams::os { + + void SetMemoryPermission(uintptr_t address, size_t size, MemoryPermission perm) { + return impl::SetMemoryPermissionImpl(address, size, perm); + } + +} diff --git a/libraries/libstratosphere/source/os/os_thread_local_storage_api.cpp b/libraries/libstratosphere/source/os/os_thread_local_storage_api.cpp new file mode 100644 index 000000000..b1099ad2e --- /dev/null +++ b/libraries/libstratosphere/source/os/os_thread_local_storage_api.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::os { + + /* TODO: How will this work without libnx? */ + + namespace { + + using LibnxTlsDestructor = void (*)(void *); + + } + + Result AllocateTlsSlot(TlsSlot *out, TlsDestructor destructor) { + s32 slot = ::threadTlsAlloc(reinterpret_cast(destructor)); + R_UNLESS(slot >= 0, os::ResultOutOfResource()); + + *out = { static_cast(slot) }; + return ResultSuccess(); + } + + void FreeTlsSlot(TlsSlot slot) { + ::threadTlsFree(static_cast(slot._value)); + } + + uintptr_t GetTlsValue(TlsSlot slot) { + return reinterpret_cast(::threadTlsGet(static_cast(slot._value))); + } + + void SetTlsValue(TlsSlot slot, uintptr_t value) { + ::threadTlsSet(static_cast(slot._value), reinterpret_cast(value)); + } + +} diff --git a/libraries/libstratosphere/source/os/os_virtual_address_memory.cpp b/libraries/libstratosphere/source/os/os_virtual_address_memory.cpp new file mode 100644 index 000000000..15d494c78 --- /dev/null +++ b/libraries/libstratosphere/source/os/os_virtual_address_memory.cpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::os { + + namespace { + + /* TODO: Remove, add VammManager */ + size_t GetSystemResourceSize() { + u64 v; + if (R_SUCCEEDED(svcGetInfo(std::addressof(v), InfoType_SystemResourceSizeTotal, CUR_PROCESS_HANDLE, 0))) { + return v; + } else { + return 0; + } + } + + } + + bool IsVirtualAddressMemoryEnabled() { + return GetSystemResourceSize() > 0; + } + +} diff --git a/libraries/libvapours/include/vapours/assert.hpp b/libraries/libvapours/include/vapours/assert.hpp index 74a5f0590..8402cbc52 100644 --- a/libraries/libvapours/include/vapours/assert.hpp +++ b/libraries/libvapours/include/vapours/assert.hpp @@ -46,11 +46,11 @@ namespace ams::diag { #endif #ifdef AMS_ENABLE_ASSERTIONS -#define AMS_ASSERT_IMPL(expr, ...) \ - ({ \ - if (const bool __tmp_ams_assert_val = (expr); AMS_UNLIKELY(!__tmp_ams_assert_val)) { \ - AMS_CALL_ASSERT_FAIL_IMPL(#expr, ## __VA_ARGS__); \ - } \ +#define AMS_ASSERT_IMPL(expr, ...) \ + ({ \ + if (const bool __tmp_ams_assert_val = static_cast(expr); AMS_UNLIKELY(!__tmp_ams_assert_val)) { \ + AMS_CALL_ASSERT_FAIL_IMPL(#expr, ## __VA_ARGS__); \ + } \ }) #else #define AMS_ASSERT_IMPL(expr, ...) AMS_UNUSED(expr, ## __VA_ARGS__) @@ -68,9 +68,9 @@ namespace ams::diag { #define AMS_ABORT(...) AMS_CALL_ABORT_IMPL("", ## __VA_ARGS__) -#define AMS_ABORT_UNLESS(expr, ...) \ - ({ \ - if (const bool __tmp_ams_assert_val = (expr); AMS_UNLIKELY(!__tmp_ams_assert_val)) { \ - AMS_CALL_ABORT_IMPL(#expr, ##__VA_ARGS__); \ - } \ +#define AMS_ABORT_UNLESS(expr, ...) \ + ({ \ + if (const bool __tmp_ams_assert_val = static_cast(expr); AMS_UNLIKELY(!__tmp_ams_assert_val)) { \ + AMS_CALL_ABORT_IMPL(#expr, ##__VA_ARGS__); \ + } \ }) diff --git a/libraries/libvapours/include/vapours/crypto.hpp b/libraries/libvapours/include/vapours/crypto.hpp index 8dc5ecb36..45e0df55d 100644 --- a/libraries/libvapours/include/vapours/crypto.hpp +++ b/libraries/libvapours/include/vapours/crypto.hpp @@ -19,6 +19,7 @@ #include #include +#include #include #include #include diff --git a/libraries/libvapours/include/vapours/crypto/crypto_sha1_generator.hpp b/libraries/libvapours/include/vapours/crypto/crypto_sha1_generator.hpp new file mode 100644 index 000000000..14b565f03 --- /dev/null +++ b/libraries/libvapours/include/vapours/crypto/crypto_sha1_generator.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include +#include +#include + +namespace ams::crypto { + + class Sha1Generator { + NON_COPYABLE(Sha1Generator); + NON_MOVEABLE(Sha1Generator); + private: + using Impl = impl::Sha1Impl; + public: + static constexpr size_t HashSize = Impl::HashSize; + static constexpr size_t BlockSize = Impl::BlockSize; + + static constexpr inline u8 Asn1Identifier[] = { + 0x30, 0x21, /* Sequence, size 0x21 */ + 0x30, 0x09, /* Sequence, size 0x09 */ + 0x06, 0x05, /* Object Identifier */ + 0x2B, 0x0E, 0x03, 0x02, 0x1A, /* SHA-1 */ + 0x05, 0x00, /* Null */ + 0x04, 0x14, /* Octet string, size 0x14 */ + }; + static constexpr size_t Asn1IdentifierSize = util::size(Asn1Identifier); + private: + Impl impl; + public: + Sha1Generator() { /* ... */ } + + void Initialize() { + this->impl.Initialize(); + } + + void Update(const void *data, size_t size) { + this->impl.Update(data, size); + } + + void GetHash(void *dst, size_t size) { + this->impl.GetHash(dst, size); + } + }; + + void GenerateSha1Hash(void *dst, size_t dst_size, const void *src, size_t src_size); + +} diff --git a/libraries/libvapours/include/vapours/crypto/crypto_sha256_generator.hpp b/libraries/libvapours/include/vapours/crypto/crypto_sha256_generator.hpp index 2585a35da..101ed2503 100644 --- a/libraries/libvapours/include/vapours/crypto/crypto_sha256_generator.hpp +++ b/libraries/libvapours/include/vapours/crypto/crypto_sha256_generator.hpp @@ -28,6 +28,8 @@ namespace ams::crypto { }; class Sha256Generator { + NON_COPYABLE(Sha256Generator); + NON_MOVEABLE(Sha256Generator); private: using Impl = impl::Sha256Impl; public: @@ -39,7 +41,7 @@ namespace ams::crypto { 0x30, 0x0D, /* Sequence, size 0x0D */ 0x06, 0x09, /* Object Identifier */ 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, /* SHA-256 */ - 0x00, /* Null */ + 0x05, 0x00, /* Null */ 0x04, 0x20, /* Octet string, size 0x20 */ }; static constexpr size_t Asn1IdentifierSize = util::size(Asn1Identifier); diff --git a/libraries/libvapours/include/vapours/crypto/impl/crypto_sha1_impl.hpp b/libraries/libvapours/include/vapours/crypto/impl/crypto_sha1_impl.hpp new file mode 100644 index 000000000..a16b9d5b8 --- /dev/null +++ b/libraries/libvapours/include/vapours/crypto/impl/crypto_sha1_impl.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once +#include +#include +#include +#include +#include + + +namespace ams::crypto::impl { + + class Sha1Impl { + public: + static constexpr size_t HashSize = 0x14; + static constexpr size_t BlockSize = 0x40; + private: + struct State { + u32 intermediate_hash[HashSize / sizeof(u32)]; + u8 buffer[BlockSize]; + u64 bits_consumed; + size_t num_buffered; + bool finalized; + }; + private: + State state; + public: + Sha1Impl() { /* ... */ } + ~Sha1Impl() { + static_assert(std::is_trivially_destructible::value); + ClearMemory(std::addressof(this->state), sizeof(this->state)); + } + + void Initialize(); + void Update(const void *data, size_t size); + void GetHash(void *dst, size_t size); + }; + + /* static_assert(HashFunction); */ + +} diff --git a/libraries/libvapours/include/vapours/svc/svc_types_common.hpp b/libraries/libvapours/include/vapours/svc/svc_types_common.hpp index df74ea0f1..1fe271945 100644 --- a/libraries/libvapours/include/vapours/svc/svc_types_common.hpp +++ b/libraries/libvapours/include/vapours/svc/svc_types_common.hpp @@ -60,6 +60,8 @@ namespace ams::svc { T pointer; public: constexpr ALWAYS_INLINE UserPointer(T p) : pointer(p) { /* ... */ } + + constexpr ALWAYS_INLINE T GetPointerUnsafe() { return this->pointer; } }; template @@ -168,11 +170,11 @@ namespace ams::svc { InitialProcessIdRangeInfo_Maximum = 1, }; - enum PhysicalMemoryInfo : u64 { - PhysicalMemoryInfo_Application = 0, - PhysicalMemoryInfo_Applet = 1, - PhysicalMemoryInfo_System = 2, - PhysicalMemoryInfo_SystemUnsafe = 3, + enum PhysicalMemorySystemInfo : u64 { + PhysicalMemorySystemInfo_Application = 0, + PhysicalMemorySystemInfo_Applet = 1, + PhysicalMemorySystemInfo_System = 2, + PhysicalMemorySystemInfo_SystemUnsafe = 3, }; enum LastThreadInfoFlag : u32 { diff --git a/libraries/libvapours/source/crypto/crypto_sha1_generator.cpp b/libraries/libvapours/source/crypto/crypto_sha1_generator.cpp new file mode 100644 index 000000000..035b782b6 --- /dev/null +++ b/libraries/libvapours/source/crypto/crypto_sha1_generator.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::crypto { + + void GenerateSha1Hash(void *dst, size_t dst_size, const void *src, size_t src_size) { + Sha1Generator gen; + + gen.Initialize(); + gen.Update(src, src_size); + gen.GetHash(dst, dst_size); + } + +} diff --git a/libraries/libvapours/source/crypto/impl/crypto_sha1_impl.arch.arm64.cpp b/libraries/libvapours/source/crypto/impl/crypto_sha1_impl.arch.arm64.cpp new file mode 100644 index 000000000..946f064b8 --- /dev/null +++ b/libraries/libvapours/source/crypto/impl/crypto_sha1_impl.arch.arm64.cpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::crypto::impl { + +#ifdef ATMOSPHERE_IS_STRATOSPHERE + + void Sha1Impl::Initialize() { + static_assert(sizeof(this->state) == sizeof(::Sha1Context)); + ::sha1ContextCreate(reinterpret_cast<::Sha1Context *>(std::addressof(this->state))); + } + + void Sha1Impl::Update(const void *data, size_t size) { + static_assert(sizeof(this->state) == sizeof(::Sha1Context)); + ::sha1ContextUpdate(reinterpret_cast<::Sha1Context *>(std::addressof(this->state)), data, size); + } + + void Sha1Impl::GetHash(void *dst, size_t size) { + static_assert(sizeof(this->state) == sizeof(::Sha1Context)); + AMS_ASSERT(size >= HashSize); + ::sha1ContextGetHash(reinterpret_cast<::Sha1Context *>(std::addressof(this->state)), dst); + } + +#else + + /* TODO: Non-EL0 implementation. */ + +#endif + +} \ No newline at end of file diff --git a/stratosphere/ams_mitm/source/fs_mitm/fsmitm_module.cpp b/stratosphere/ams_mitm/source/fs_mitm/fsmitm_module.cpp index 78f6e9d35..9602ae3e5 100644 --- a/stratosphere/ams_mitm/source/fs_mitm/fsmitm_module.cpp +++ b/stratosphere/ams_mitm/source/fs_mitm/fsmitm_module.cpp @@ -49,7 +49,7 @@ namespace ams::mitm::fs { void ProcessForServerOnAllThreads() { /* Initialize threads. */ if constexpr (NumExtraThreads > 0) { - const u32 priority = os::GetCurrentThreadPriority(); + const s32 priority = os::GetCurrentThreadPriority(); for (size_t i = 0; i < NumExtraThreads; i++) { R_ABORT_UNLESS(g_extra_threads[i].Initialize(LoopServerThread, nullptr, g_extra_thread_stacks[i], ThreadStackSize, priority)); } diff --git a/stratosphere/dmnt/source/dmnt_main.cpp b/stratosphere/dmnt/source/dmnt_main.cpp index 4d999dfbf..47f6b4a92 100644 --- a/stratosphere/dmnt/source/dmnt_main.cpp +++ b/stratosphere/dmnt/source/dmnt_main.cpp @@ -139,7 +139,7 @@ int main(int argc, char **argv) /* Initialize threads. */ if constexpr (NumExtraThreads > 0) { - const u32 priority = os::GetCurrentThreadPriority(); + const s32 priority = os::GetCurrentThreadPriority(); for (size_t i = 0; i < NumExtraThreads; i++) { R_ABORT_UNLESS(g_extra_threads[i].Initialize(LoopServerThread, nullptr, g_extra_thread_stacks[i], ThreadStackSize, priority)); }