kern: SvcChangeKernelTraceState

libraries/libmesosphere/include/mesosphere.hpp

@@ -27,6 +27,9 @@
 #include <mesosphere/kern_initial_process.hpp>
 #include <mesosphere/kern_k_exception_context.hpp>
 
+/* Tracing functionality. */
+#include <mesosphere/kern_k_trace.hpp>
+
 /* Core pre-initialization includes. */
 #include <mesosphere/kern_select_cpu.hpp>
 #include <mesosphere/kern_select_system_control.hpp>

libraries/libmesosphere/include/mesosphere/kern_common.hpp

@@ -37,4 +37,6 @@ namespace ams::kern {
 #define MESOSPHERE_ENABLE_DEBUG_PRINT
 #endif
 
+#define MESOSPHERE_BUILD_FOR_TRACING
+
 #include <mesosphere/svc/kern_svc_results.hpp>

libraries/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp

@@ -100,8 +100,6 @@ namespace ams::kern {
         KMemoryRegionType_DramSystemNonSecurePool = 0xDA6  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped,
         KMemoryRegionType_DramSystemPool          = 0x13A6 | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_LinearMapped | KMemoryRegionAttr_CarveoutProtected,
 
-
-
         KMemoryRegionType_DramKernel        = 0xE   | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
         KMemoryRegionType_DramKernelCode    = 0xCE  | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
         KMemoryRegionType_DramKernelSlab    = 0x14E | KMemoryRegionAttr_NoUserMap | KMemoryRegionAttr_CarveoutProtected,
@@ -526,6 +524,10 @@ namespace ams::kern {
                 return *GetVirtualMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_KernelTemp);
             }
 
+            static NOINLINE KMemoryRegion &GetKernelTraceBufferRegion() {
+                return *GetVirtualLinearMemoryRegionTree().FindFirstRegionByType(KMemoryRegionType_VirtualKernelTraceBuffer);
+            }
+
             static NOINLINE KMemoryRegion &GetVirtualLinearRegion(KVirtualAddress address) {
                 return *GetVirtualLinearMemoryRegionTree().FindContainingRegion(GetInteger(address));
             }
@@ -730,6 +732,10 @@ namespace ams::kern {
                 return GetVirtualLinearExtents(GetLinearRegionPhysicalExtents());
             }
 
+            static NOINLINE auto GetMainMemoryPhysicalExtents() {
+                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram);
+            }
+
             static NOINLINE auto GetCarveoutRegionExtents() {
                 return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionAttr_CarveoutProtected);
             }
@@ -777,6 +783,10 @@ namespace ams::kern {
             static NOINLINE auto GetKernelApplicationPoolRegionPhysicalExtents() {
                 return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_DramApplicationPool);
             }
+
+            static NOINLINE auto GetKernelTraceBufferRegionPhysicalExtents() {
+                return GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_KernelTraceBuffer);
+            }
     };
 
 

libraries/libmesosphere/include/mesosphere/kern_k_trace.hpp

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+#include <mesosphere/kern_common.hpp>
+#include <mesosphere/kern_k_spin_lock.hpp>
+
+namespace ams::kern {
+
+    #if defined(MESOSPHERE_BUILD_FOR_TRACING)
+        constexpr inline bool IsKTraceEnabled = true;
+    #else
+        constexpr inline bool IsKTraceEnabled = false;
+    #endif
+
+    constexpr inline size_t KTraceBufferSize = IsKTraceEnabled ? 16_MB : 0;
+
+    static_assert(IsKTraceEnabled || !IsKTraceEnabled);
+
+    class KTrace {
+        private:
+            static bool s_is_active;
+        public:
+            static void Initialize(KVirtualAddress address, size_t size);
+            static void Start();
+            static void Stop();
+
+            static ALWAYS_INLINE bool IsActive() { return s_is_active; }
+    };
+
+}
+
+#define MESOSPHERE_KTRACE_RESUME()                    \
+    ({                                                \
+        if constexpr (::ams::kern::IsKTraceEnabled) { \
+            ::ams::kern::KTrace::Start();             \
+        }                                             \
+    })
+
+#define MESOSPHERE_KTRACE_PAUSE()                     \
+    ({                                                \
+        if constexpr (::ams::kern::IsKTraceEnabled) { \
+            ::ams::kern::KTrace::Stop();              \
+        }                                             \
+    })

libraries/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp

@@ -338,39 +338,52 @@ namespace ams::kern::board::nintendo::nx {
     }
 
     size_t KSystemControl::Init::GetApplicationPoolSize() {
-        switch (GetMemoryArrangeForInit()) {
+        /* Get the base pool size. */
-            case smc::MemoryArrangement_4GB:
+        const size_t base_pool_size = [] ALWAYS_INLINE_LAMBDA () -> size_t {
-            default:
+            switch (GetMemoryArrangeForInit()) {
-                return 3285_MB;
+                case smc::MemoryArrangement_4GB:
-            case smc::MemoryArrangement_4GBForAppletDev:
+                default:
-                return 2048_MB;
+                    return 3285_MB;
-            case smc::MemoryArrangement_4GBForSystemDev:
+                case smc::MemoryArrangement_4GBForAppletDev:
-                return 3285_MB;
+                    return 2048_MB;
-            case smc::MemoryArrangement_6GB:
+                case smc::MemoryArrangement_4GBForSystemDev:
-                return 4916_MB;
+                    return 3285_MB;
-            case smc::MemoryArrangement_6GBForAppletDev:
+                case smc::MemoryArrangement_6GB:
-                return 3285_MB;
+                    return 4916_MB;
-            case smc::MemoryArrangement_8GB:
+                case smc::MemoryArrangement_6GBForAppletDev:
-                return 4916_MB;
+                    return 3285_MB;
-        }
+                case smc::MemoryArrangement_8GB:
+                    return 4916_MB;
+            }
+        }();
+
+        /* Return (possibly) adjusted size. */
+        return base_pool_size;
     }
 
     size_t KSystemControl::Init::GetAppletPoolSize() {
-        switch (GetMemoryArrangeForInit()) {
+        /* Get the base pool size. */
-            case smc::MemoryArrangement_4GB:
+        const size_t base_pool_size = [] ALWAYS_INLINE_LAMBDA () -> size_t {
-            default:
+            switch (GetMemoryArrangeForInit()) {
-                return 507_MB;
+                case smc::MemoryArrangement_4GB:
-            case smc::MemoryArrangement_4GBForAppletDev:
+                default:
-                return 1554_MB;
+                    return 507_MB;
-            case smc::MemoryArrangement_4GBForSystemDev:
+                case smc::MemoryArrangement_4GBForAppletDev:
-                return 448_MB;
+                    return 1554_MB;
-            case smc::MemoryArrangement_6GB:
+                case smc::MemoryArrangement_4GBForSystemDev:
-                return 562_MB;
+                    return 448_MB;
-            case smc::MemoryArrangement_6GBForAppletDev:
+                case smc::MemoryArrangement_6GB:
-                return 2193_MB;
+                    return 562_MB;
-            case smc::MemoryArrangement_8GB:
+                case smc::MemoryArrangement_6GBForAppletDev:
-                return 2193_MB;
+                    return 2193_MB;
-        }
+                case smc::MemoryArrangement_8GB:
+                    return 2193_MB;
+            }
+        }();
+
+        /* Return (possibly) adjusted size. */
+        constexpr size_t ExtraSystemMemoryForAtmosphere = 33_MB;
+        return base_pool_size - ExtraSystemMemoryForAtmosphere - KTraceBufferSize;
     }
 
     size_t KSystemControl::Init::GetMinimumNonSecureSystemPoolSize() {
@@ -461,6 +474,12 @@ namespace ams::kern::board::nintendo::nx {
             g_secure_applet_memory_address = Kernel::GetMemoryManager().AllocateContinuous(SecureAppletMemorySize / PageSize, 1, SecureAppletAllocateOption);
             MESOSPHERE_ABORT_UNLESS(g_secure_applet_memory_address != Null<KVirtualAddress>);
         }
+
+        /* Initialize KTrace. */
+        if constexpr (IsKTraceEnabled) {
+            const auto &ktrace = KMemoryLayout::GetKernelTraceBufferRegion();
+            KTrace::Initialize(ktrace.GetAddress(), ktrace.GetSize());
+        }
     }
 
     u32 KSystemControl::GetInitialProcessBinaryPool() {

libraries/libmesosphere/source/kern_k_memory_layout.cpp

@@ -227,7 +227,9 @@ namespace ams::kern {
 
         void SetupPoolPartitionMemoryRegions() {
             /* Start by identifying the extents of the DRAM memory region. */
-            const auto dram_extents = KMemoryLayout::GetPhysicalMemoryRegionTree().GetDerivedRegionExtents(KMemoryRegionType_Dram);
+            const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents();
+
+            const uintptr_t pool_end = dram_extents.GetEndAddress() - KTraceBufferSize;
 
             /* Get Application and Applet pool sizes. */
             const size_t application_pool_size       = KSystemControl::Init::GetApplicationPoolSize();
@@ -242,7 +244,7 @@ namespace ams::kern {
             const uintptr_t pool_partitions_start = KMemoryLayout::GetPhysicalMemoryRegionTree().FindFirstRegionByTypeAttr(KMemoryRegionType_DramPoolPartition)->GetAddress();
 
             /* Decide on starting addresses for our pools. */
-            const uintptr_t application_pool_start   = dram_extents.GetEndAddress() - application_pool_size;
+            const uintptr_t application_pool_start   = pool_end - application_pool_size;
             const uintptr_t applet_pool_start        = application_pool_start - applet_pool_size;
             const uintptr_t unsafe_system_pool_start = std::min(kernel_dram_start + CarveoutSizeMax, util::AlignDown(applet_pool_start - unsafe_system_pool_min_size, CarveoutAlignment));
             const size_t    unsafe_system_pool_size  = applet_pool_start - unsafe_system_pool_start;

libraries/libmesosphere/source/kern_k_trace.cpp

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2018-2020 Atmosphère-NX
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <mesosphere.hpp>
+
+namespace ams::kern {
+
+    /* Static initializations. */
+    constinit bool KTrace::s_is_active = false;
+
+    namespace {
+
+        constinit KSpinLock g_ktrace_lock;
+        constinit KVirtualAddress g_ktrace_buffer_address = Null<KVirtualAddress>;
+        constinit size_t g_ktrace_buffer_size = 0;
+
+        struct KTraceHeader {
+            u32 magic;
+            u32 offset;
+            u32 index;
+            u32 count;
+
+            static constexpr u32 Magic = util::FourCC<'K','T','R','0'>::Code;
+        };
+        static_assert(util::is_pod<KTraceHeader>::value);
+
+        struct KTraceRecord {
+            u8 core_id;
+            u8 type;
+            u16 process_id;
+            u32 thread_id;
+            u64 tick;
+            u64 data[6];
+        };
+        static_assert(util::is_pod<KTraceRecord>::value);
+        static_assert(sizeof(KTraceRecord) == 0x40);
+
+    }
+
+    void KTrace::Initialize(KVirtualAddress address, size_t size) {
+        /* Only perform tracing when on development hardware. */
+        if (KTargetSystem::IsDebugMode()) {
+            const size_t offset = util::AlignUp(sizeof(KTraceHeader), sizeof(KTraceRecord));
+            if (offset < size) {
+                /* Clear the trace buffer. */
+                std::memset(GetVoidPointer(address), 0, size);
+
+                /* Initialize the KTrace header. */
+                KTraceHeader *header = GetPointer<KTraceHeader>(address);
+                header->magic  = KTraceHeader::Magic;
+                header->offset = offset;
+                header->index  = 0;
+                header->count  = (size - offset) / sizeof(KTraceRecord);
+
+                /* Set the global data. */
+                g_ktrace_buffer_address = address;
+                g_ktrace_buffer_size    = size;
+            }
+        }
+    }
+
+    void KTrace::Start() {
+        if (g_ktrace_buffer_address != Null<KVirtualAddress>) {
+            /* Get exclusive access to the trace buffer. */
+            KScopedInterruptDisable di;
+            KScopedSpinLock lk(g_ktrace_lock);
+
+            /* Reset the header. */
+            KTraceHeader *header = GetPointer<KTraceHeader>(g_ktrace_buffer_address);
+            header->index = 0;
+
+            /* Reset the records. */
+            KTraceRecord *records = GetPointer<KTraceRecord>(g_ktrace_buffer_address + header->offset);
+            std::memset(records, 0, sizeof(*records) * header->count);
+
+            /* Note that we're active. */
+            s_is_active = true;
+        }
+    }
+
+    void KTrace::Stop() {
+        if (g_ktrace_buffer_address != Null<KVirtualAddress>) {
+            /* Get exclusive access to the trace buffer. */
+            KScopedInterruptDisable di;
+            KScopedSpinLock lk(g_ktrace_lock);
+
+            /* Note that we're paused. */
+            s_is_active = false;
+        }
+    }
+
+}

libraries/libmesosphere/source/kern_kernel.cpp

@@ -157,6 +157,10 @@ namespace ams::kern {
         PrintMemoryRegion("        SystemUnsafe",   KMemoryLayout::GetKernelSystemNonSecurePoolRegionPhysicalExtents());
         PrintMemoryRegion("        Applet",         KMemoryLayout::GetKernelAppletPoolRegionPhysicalExtents());
         PrintMemoryRegion("        Application",    KMemoryLayout::GetKernelApplicationPoolRegionPhysicalExtents());
+        if constexpr (IsKTraceEnabled) {
+            MESOSPHERE_LOG("    Debug\n");
+            PrintMemoryRegion("        Trace Buffer",   KMemoryLayout::GetKernelTraceBufferRegionPhysicalExtents());
+        }
         MESOSPHERE_LOG("\n");
     }
 

libraries/libmesosphere/source/svc/kern_svc_kernel_debug.cpp

@@ -39,12 +39,12 @@ namespace ams::kern::svc {
                 switch (kern_trace_state) {
                     case ams::svc::KernelTraceState_Enabled:
                         {
-                            /* TODO: MESOSPHERE_KTRACE_RESUME(); */
+                            MESOSPHERE_KTRACE_RESUME();
                         }
                         break;
                     case ams::svc::KernelTraceState_Disabled:
                         {
-                            /* TODO: MESOSPHERE_KTRACE_PAUSE(); */
+                            MESOSPHERE_KTRACE_PAUSE();
                         }
                         break;
                     default:

mesosphere/kernel/source/arch/arm64/init/kern_init_core.cpp

@@ -183,6 +183,11 @@ namespace ams::kern::init {
             MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_unk_debug_virt_addr), MiscUnknownDebugRegionSize, KMemoryRegionType_KernelMiscUnknownDebug));
             ttbr1_table.Map(misc_unk_debug_virt_addr, MiscUnknownDebugRegionSize, misc_unk_debug_phys_addr, KernelRoDataAttribute, g_initial_page_allocator);
         }
+            constexpr size_t MiscUnknownDebugRegionSize  = PageSize;
+            constexpr size_t MiscUnknownDebugRegionAlign = PageSize;
+            const KVirtualAddress misc_unk_debug_virt_addr = KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegionWithGuard(MiscUnknownDebugRegionSize, MiscUnknownDebugRegionAlign, KMemoryRegionType_KernelMisc, PageSize);
+            MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(misc_unk_debug_virt_addr), MiscUnknownDebugRegionSize, KMemoryRegionType_KernelMiscUnknownDebug));
+            ttbr1_table.Map(misc_unk_debug_virt_addr, MiscUnknownDebugRegionSize, misc_unk_debug_phys_addr, KernelRoDataAttribute, g_initial_page_allocator);
 
         /* Setup board-specific device physical regions. */
         SetupDevicePhysicalMemoryRegions();
@@ -241,6 +246,14 @@ namespace ams::kern::init {
         /* Insert a physical region for the kernel page table heap region */
         MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(slab_end_phys_addr), page_table_heap_size, KMemoryRegionType_DramKernelPtHeap));
 
+        /* Insert a physical region for the kernel trace buffer. */
+        static_assert(!IsKTraceEnabled || KTraceBufferSize > 0);
+        if constexpr (IsKTraceEnabled) {
+            const auto dram_extents = KMemoryLayout::GetMainMemoryPhysicalExtents();
+            const KPhysicalAddress ktrace_buffer_phys_addr = dram_extents.GetEndAddress() - KTraceBufferSize;
+            MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetPhysicalMemoryRegionTree().Insert(GetInteger(ktrace_buffer_phys_addr), KTraceBufferSize, KMemoryRegionType_KernelTraceBuffer));
+        }
+
         /* All DRAM regions that we haven't tagged by this point will be mapped under the linear mapping. Tag them. */
         for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
             if (region.GetType() == KMemoryRegionType_Dram) {
@@ -258,17 +271,37 @@ namespace ams::kern::init {
         const uintptr_t linear_region_phys_to_virt_diff = GetInteger(linear_region_start) - GetInteger(aligned_linear_phys_start);
 
         /* Map and create regions for all the linearly-mapped data. */
-        for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
+        {
-            if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
+            uintptr_t cur_phys_addr = 0;
-                continue;
+            uintptr_t cur_size = 0;
+            for (auto &region : KMemoryLayout::GetPhysicalMemoryRegionTree()) {
+                if (!region.HasTypeAttribute(KMemoryRegionAttr_LinearMapped)) {
+                    continue;
+                }
+
+                if (cur_phys_addr == 0) {
+                    cur_phys_addr = region.GetAddress();
+                    cur_size      = region.GetSize();
+                } else if (cur_phys_addr + cur_size == region.GetAddress()) {
+                    cur_size     += region.GetSize();
+                } else {
+                    const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff;
+                    ttbr1_table.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator);
+                    cur_phys_addr = region.GetAddress();
+                    cur_size      = region.GetSize();
+                }
+
+                const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff;
+                MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType())));
+                region.SetPairAddress(region_virt_addr);
+                KMemoryLayout::GetVirtualMemoryRegionTree().FindContainingRegion(region_virt_addr)->SetPairAddress(region.GetAddress());
             }
 
-            const uintptr_t region_virt_addr = region.GetAddress() + linear_region_phys_to_virt_diff;
+            /* Map the last block, which we may have skipped. */
-            ttbr1_table.Map(region_virt_addr, region.GetSize(), region.GetAddress(), KernelRwDataAttribute, g_initial_page_allocator);
+            if (cur_phys_addr != 0) {
-
+                const uintptr_t cur_virt_addr = cur_phys_addr + linear_region_phys_to_virt_diff;
-            MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(region_virt_addr, region.GetSize(), GetTypeForVirtualLinearMapping(region.GetType())));
+                ttbr1_table.Map(cur_virt_addr, cur_size, cur_phys_addr, KernelRwDataAttribute, g_initial_page_allocator);
-            region.SetPairAddress(region_virt_addr);
+            }
-            KMemoryLayout::GetVirtualMemoryRegionTree().FindContainingRegion(region_virt_addr)->SetPairAddress(region.GetAddress());
         }
 
         /* Create regions for and map all core-specific stacks. */