From 96937a611dffa9a04e4d41f4d972ecd9bcc8aa70 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Wed, 7 Apr 2021 17:07:01 -0700 Subject: [PATCH] kern: fuck the KPolice^H^H^H^H^H^HPageGroups --- .../arch/arm64/kern_k_process_page_table.hpp | 32 +- .../nintendo/nx/kern_k_device_page_table.hpp | 9 +- .../mesosphere/kern_k_page_table_base.hpp | 27 +- .../nintendo/nx/kern_k_device_page_table.cpp | 280 +++++++++------- .../source/kern_k_device_address_space.cpp | 22 +- .../source/kern_k_page_table_base.cpp | 305 ++++++++++++++---- .../source/svc/kern_svc_cache.cpp | 32 +- .../source/svc/kern_svc_process_memory.cpp | 17 +- 8 files changed, 493 insertions(+), 231 deletions(-) diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp index cf603b14f..2277a5bff 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_process_page_table.hpp @@ -36,6 +36,10 @@ namespace ams::kern::arch::arm64 { void Finalize() { m_page_table.Finalize(); } + ALWAYS_INLINE KScopedLightLock AcquireDeviceMapLock() { + return m_page_table.AcquireDeviceMapLock(); + } + Result SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission perm) { return m_page_table.SetMemoryPermission(addr, size, perm); } @@ -148,22 +152,30 @@ namespace ams::kern::arch::arm64 { return m_page_table.WriteDebugIoMemory(address, buffer, size); } - Result LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) { - return m_page_table.LockForDeviceAddressSpace(out, address, size, perm, is_aligned); + Result LockForMapDeviceAddressSpace(KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) { + return m_page_table.LockForMapDeviceAddressSpace(address, size, perm, is_aligned); + } + + Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size) { + return m_page_table.LockForUnmapDeviceAddressSpace(address, size); } Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size) { return m_page_table.UnlockForDeviceAddressSpace(address, size); } - Result MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size) { - return m_page_table.MakePageGroupForUnmapDeviceAddressSpace(out, address, size); - } - Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) { return m_page_table.UnlockForDeviceAddressSpacePartialMap(address, size, mapped_size); } + Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) { + return m_page_table.OpenMemoryRangeForMapDeviceAddressSpace(out, address, size, perm, is_aligned); + } + + Result OpenMemoryRangeForUnmapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size) { + return m_page_table.OpenMemoryRangeForUnmapDeviceAddressSpace(out, address, size); + } + Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size) { return m_page_table.LockForIpcUserBuffer(out, address, size); } @@ -188,6 +200,10 @@ namespace ams::kern::arch::arm64 { return m_page_table.UnlockForCodeMemory(address, size, pg); } + Result OpenMemoryRangeForProcessCacheOperation(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size) { + return m_page_table.OpenMemoryRangeForProcessCacheOperation(out, address, size); + } + Result CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) { return m_page_table.CopyMemoryFromLinearToUser(dst_addr, size, src_addr, src_state_mask, src_state, src_test_perm, src_attr_mask, src_attr); } @@ -240,6 +256,10 @@ namespace ams::kern::arch::arm64 { return m_page_table.UnmapPhysicalMemoryUnsafe(address, size); } + Result UnmapProcessMemory(KProcessAddress dst_address, size_t size, KProcessPageTable &src_page_table, KProcessAddress src_address) { + return m_page_table.UnmapProcessMemory(dst_address, size, src_page_table.m_page_table, src_address); + } + void DumpMemoryBlocks() const { return m_page_table.DumpMemoryBlocks(); } diff --git a/libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp b/libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp index 21814b198..6a60af86d 100644 --- a/libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_device_page_table.hpp @@ -69,8 +69,8 @@ namespace ams::kern::board::nintendo::nx { Result Attach(ams::svc::DeviceName device_name, u64 space_address, u64 space_size); Result Detach(ams::svc::DeviceName device_name); - Result Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings); - Result Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address); + Result Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings); + Result Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address); void Unmap(KDeviceVirtualAddress device_address, size_t size) { return this->UnmapImpl(device_address, size, false); @@ -78,12 +78,11 @@ namespace ams::kern::board::nintendo::nx { private: Result MapDevicePage(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KPhysicalAddress phys_addr, u64 size, KDeviceVirtualAddress address, ams::svc::MemoryPermission device_perm); - Result MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm); + Result MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned); void UnmapImpl(KDeviceVirtualAddress address, u64 size, bool force); bool IsFree(KDeviceVirtualAddress address, u64 size) const; - Result MakePageGroup(KPageGroup *out, KDeviceVirtualAddress address, u64 size) const; - bool Compare(const KPageGroup &pg, KDeviceVirtualAddress device_address) const; + bool Compare(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) const; public: static void Initialize(); diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp index 1a7afe4ef..6ec7498d3 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_page_table_base.hpp @@ -55,6 +55,13 @@ namespace ams::kern { public: using TraversalEntry = KPageTableImpl::TraversalEntry; using TraversalContext = KPageTableImpl::TraversalContext; + + struct MemoryRange { + KVirtualAddress address; + size_t size; + + void Close(); + }; protected: enum MemoryFillValue { MemoryFillValue_Zero = 0, @@ -155,6 +162,7 @@ namespace ams::kern { size_t m_mapped_ipc_server_memory{}; mutable KLightLock m_general_lock{}; mutable KLightLock m_map_physical_memory_lock{}; + KLightLock m_device_map_lock{}; KPageTableImpl m_impl{}; KMemoryBlockManager m_memory_block_manager{}; u32 m_allocate_option{}; @@ -199,6 +207,10 @@ namespace ams::kern { return this->CanContain(addr, size, KMemoryState_AliasCode); } + ALWAYS_INLINE KScopedLightLock AcquireDeviceMapLock() { + return KScopedLightLock(m_device_map_lock); + } + KProcessAddress GetRegionAddress(KMemoryState state) const; size_t GetRegionSize(KMemoryState state) const; bool CanContain(KProcessAddress addr, size_t size, KMemoryState state) const; @@ -290,6 +302,8 @@ namespace ams::kern { Result MakePageGroup(KPageGroup &pg, KProcessAddress addr, size_t num_pages); bool IsValidPageGroup(const KPageGroup &pg, KProcessAddress addr, size_t num_pages); + Result GetContiguousMemoryRangeWithState(MemoryRange *out, KProcessAddress address, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr); + NOINLINE Result MapPages(KProcessAddress *out_addr, size_t num_pages, size_t alignment, KPhysicalAddress phys_addr, bool is_pa_valid, KProcessAddress region_start, size_t region_num_pages, KMemoryState state, KMemoryPermission perm); Result MapIoImpl(KProcessAddress *out, PageLinkedList *page_list, KPhysicalAddress phys_addr, size_t size, KMemoryPermission perm); @@ -367,12 +381,15 @@ namespace ams::kern { Result WriteDebugMemory(KProcessAddress address, const void *buffer, size_t size); Result WriteDebugIoMemory(KProcessAddress address, const void *buffer, size_t size); - Result LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned); - Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size); + Result LockForMapDeviceAddressSpace(KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned); + Result LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size); - Result MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size); + Result UnlockForDeviceAddressSpace(KProcessAddress address, size_t size); Result UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size); + Result OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned); + Result OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange *out, KProcessAddress address, size_t size); + Result LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size); Result UnlockForIpcUserBuffer(KProcessAddress address, size_t size); @@ -381,6 +398,8 @@ namespace ams::kern { Result LockForCodeMemory(KPageGroup *out, KProcessAddress address, size_t size); Result UnlockForCodeMemory(KProcessAddress address, size_t size, const KPageGroup &pg); + Result OpenMemoryRangeForProcessCacheOperation(MemoryRange *out, KProcessAddress address, size_t size); + Result CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr); Result CopyMemoryFromLinearToKernel(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr); Result CopyMemoryFromUserToLinear(KProcessAddress dst_addr, size_t size, u32 dst_state_mask, u32 dst_state, KMemoryPermission dst_test_perm, u32 dst_attr_mask, u32 dst_attr, KProcessAddress src_addr); @@ -398,6 +417,8 @@ namespace ams::kern { Result MapPhysicalMemoryUnsafe(KProcessAddress address, size_t size); Result UnmapPhysicalMemoryUnsafe(KProcessAddress address, size_t size); + Result UnmapProcessMemory(KProcessAddress dst_address, size_t size, KPageTableBase &src_pt, KProcessAddress src_address); + void DumpMemoryBlocksLocked() const { MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); m_memory_block_manager.DumpBlocks(); diff --git a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp index 582df3f7f..b0fc3bcb0 100644 --- a/libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp +++ b/libraries/libmesosphere/source/board/nintendo/nx/kern_k_device_page_table.cpp @@ -216,6 +216,12 @@ namespace ams::kern::board::nintendo::nx { return (m_value & (1u << n)); } + template + constexpr ALWAYS_INLINE u32 SelectBits() const { + constexpr u32 Mask = ((1u << Bits) | ...); + return m_value & Mask; + } + constexpr ALWAYS_INLINE bool GetBit(Bit n) const { return this->SelectBit(n) != 0; } @@ -242,12 +248,14 @@ namespace ams::kern::board::nintendo::nx { constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBit(Bit_NonSecure); } constexpr ALWAYS_INLINE bool IsWriteable() const { return this->GetBit(Bit_Writeable); } constexpr ALWAYS_INLINE bool IsReadable() const { return this->GetBit(Bit_Readable); } - constexpr ALWAYS_INLINE bool IsValid() const { return this->IsWriteable() || this->IsReadable(); } + constexpr ALWAYS_INLINE bool IsValid() const { return this->SelectBits(); } - constexpr ALWAYS_INLINE u32 GetAttributes() const { return this->SelectBit(Bit_NonSecure) | this->SelectBit(Bit_Writeable) | this->SelectBit(Bit_Readable); } + constexpr ALWAYS_INLINE u32 GetAttributes() const { return this->SelectBits(); } constexpr ALWAYS_INLINE KPhysicalAddress GetPhysicalAddress() const { return (static_cast(m_value) << DevicePageBits) & PhysicalAddressMask; } + + ALWAYS_INLINE void InvalidateAttributes() { this->SetValue(m_value & ~(0xCu << 28)); } ALWAYS_INLINE void Invalidate() { this->SetValue(0); } }; @@ -847,7 +855,7 @@ namespace ams::kern::board::nintendo::nx { } /* Forcibly unmap all pages. */ - this->UnmapImpl(0, (1ul << DeviceVirtualAddressBits), true); + this->UnmapImpl(0, (1ul << DeviceVirtualAddressBits), false); /* Release all asids. */ for (size_t i = 0; i < TableCount; ++i) { @@ -1117,12 +1125,11 @@ namespace ams::kern::board::nintendo::nx { return ResultSuccess(); } - Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm) { + Result KDevicePageTable::MapImpl(size_t *out_mapped_size, s32 &num_pt, s32 max_pt, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool is_aligned) { /* Clear the output size. */ *out_mapped_size = 0; /* Get the size, and validate the address. */ - const u64 size = pg.GetNumPages() * PageSize; MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0); MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0); @@ -1130,28 +1137,33 @@ namespace ams::kern::board::nintendo::nx { R_UNLESS(this->IsFree(device_address, size), svc::ResultInvalidCurrentMemory()); /* Ensure that if we fail, we unmap anything we mapped. */ - auto unmap_guard = SCOPE_GUARD { this->UnmapImpl(device_address, size, true); }; + auto unmap_guard = SCOPE_GUARD { this->UnmapImpl(device_address, size, false); }; /* Iterate, mapping device pages. */ KDeviceVirtualAddress cur_addr = device_address; - for (auto it = pg.begin(); it != pg.end(); ++it) { - /* Require that we be able to map the device page. */ - R_UNLESS(IsHeapVirtualAddress(it->GetAddress()), svc::ResultInvalidCurrentMemory()); + while (true) { + /* Get the current contiguous range. */ + KPageTableBase::MemoryRange contig_range = {}; + R_TRY(page_table->OpenMemoryRangeForMapDeviceAddressSpace(std::addressof(contig_range), process_address + *out_mapped_size, size - *out_mapped_size, ConvertToKMemoryPermission(device_perm), is_aligned)); - /* Get the physical address for the page. */ - const KPhysicalAddress phys_addr = GetHeapPhysicalAddress(it->GetAddress()); + /* Ensure we close the range when we're done. */ + ON_SCOPE_EXIT { contig_range.Close(); }; /* Map the device page. */ - const u64 block_size = it->GetSize(); size_t mapped_size = 0; - R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, phys_addr, block_size, cur_addr, device_perm)); + R_TRY(this->MapDevicePage(std::addressof(mapped_size), num_pt, max_pt, GetHeapPhysicalAddress(contig_range.address), contig_range.size, cur_addr, device_perm)); /* Advance. */ - cur_addr += block_size; + cur_addr += contig_range.size; *out_mapped_size += mapped_size; /* If we didn't map as much as we wanted, break. */ - if (mapped_size < block_size) { + if (mapped_size < contig_range.size) { + break; + } + + /* Similarly, if we're done, break. */ + if (*out_mapped_size >= size) { break; } } @@ -1186,8 +1198,6 @@ namespace ams::kern::board::nintendo::nx { /* Check if there's nothing mapped at l1. */ if (l1 == nullptr || !l1[l1_index].IsValid()) { - MESOSPHERE_ASSERT(force); - const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index; const size_t map_count = std::min(remaining_in_entry, remaining / DevicePageSize); @@ -1201,30 +1211,12 @@ namespace ams::kern::board::nintendo::nx { const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index; const size_t map_count = std::min(remaining_in_entry, remaining / DevicePageSize); size_t num_closed = 0; - bool invalidated_tlb = false; + /* Invalidate the attributes of all entries. */ for (size_t i = 0; i < map_count; ++i) { if (l2[l2_index + i].IsValid()) { - /* Get the physical address. */ - const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress(); - MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr)); - - /* Invalidate the entry. */ - l2[l2_index + i].Invalidate(); + l2[l2_index + i].InvalidateAttributes(); ++num_closed; - - /* Try to add the page to the group. */ - if (R_FAILED(pg.AddBlock(GetHeapVirtualAddress(phys_addr), DevicePageSize / PageSize))) { - /* If we can't add it for deferred close, close it now. */ - cpu::StoreDataCache(std::addressof(l2[l2_index + i]), sizeof(PageTableEntry)); - InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l2[l2_index + i])))); - SmmuSynchronizationBarrier(); - - /* Close the page's reference. */ - mm.Close(GetHeapVirtualAddress(phys_addr), 1); - } - } else { - MESOSPHERE_ASSERT(force); } } cpu::StoreDataCache(std::addressof(l2[l2_index]), map_count * sizeof(PageTableEntry)); @@ -1235,6 +1227,38 @@ namespace ams::kern::board::nintendo::nx { } SmmuSynchronizationBarrier(); + /* Close the memory manager's references to the pages. */ + { + KPhysicalAddress contig_phys_addr = Null; + size_t contig_count = 0; + for (size_t i = 0; i < map_count; ++i) { + /* Get the physical address. */ + const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress(); + MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr)); + + /* Fully invalidate the entry. */ + l2[l2_index + i].Invalidate(); + + if (contig_count == 0) { + /* Ensure that our address/count is valid. */ + contig_phys_addr = phys_addr; + contig_count = contig_phys_addr != Null ? 1 : 0; + } else if (phys_addr == Null || phys_addr != (contig_phys_addr + (contig_count * DevicePageSize))) { + /* If we're no longer contiguous, close the range we've been building. */ + mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize); + + contig_phys_addr = phys_addr; + contig_count = contig_phys_addr != Null ? 1 : 0; + } else { + ++contig_count; + } + } + + if (contig_count > 0) { + mm.Close(GetHeapVirtualAddress(contig_phys_addr), (contig_count * DevicePageSize) / PageSize); + } + } + /* Close the pages. */ if (ptm.Close(KVirtualAddress(l2), num_closed)) { /* Invalidate the l1 entry. */ @@ -1243,22 +1267,12 @@ namespace ams::kern::board::nintendo::nx { /* Synchronize. */ InvalidatePtc(GetPageTablePhysicalAddress(KVirtualAddress(std::addressof(l1[l1_index])))); - InvalidateTlbSection(m_table_asids[l0_index], address); SmmuSynchronizationBarrier(); - /* We invalidated the tlb. */ - invalidated_tlb = true; - /* Free the l2 page. */ ptm.Free(KVirtualAddress(l2)); } - /* Invalidate the tlb if we haven't already. */ - if (!invalidated_tlb) { - InvalidateTlbSection(m_table_asids[l0_index], address); - SmmuSynchronizationBarrier(); - } - /* Advance. */ address += map_count * DevicePageSize; remaining -= map_count * DevicePageSize; @@ -1287,114 +1301,158 @@ namespace ams::kern::board::nintendo::nx { remaining -= DeviceLargePageSize; } } - - /* Close references to the pages in the group. */ - pg.Close(); } - Result KDevicePageTable::MakePageGroup(KPageGroup *out, KDeviceVirtualAddress address, u64 size) const { - MESOSPHERE_ASSERT((address & ~DeviceVirtualAddressMask) == 0); - MESOSPHERE_ASSERT(((address + size - 1) & ~DeviceVirtualAddressMask) == 0); + bool KDevicePageTable::Compare(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) const { + MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0); + MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0); + + /* We need to traverse the ranges that make up our mapping, to make sure they're all good. Start by getting a contiguous range. */ + KPageTableBase::MemoryRange contig_range = {}; + if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), process_address, size))) { + return false; + } + + /* Ensure that we close the range when we're done. */ + bool range_open = true; + ON_SCOPE_EXIT { if (range_open) { contig_range.Close(); } }; /* Walk the directory. */ - u64 remaining = size; - bool first = true; - u32 attr = 0; - while (remaining > 0) { - const size_t l0_index = (address / DeviceRegionSize); - const size_t l1_index = (address % DeviceRegionSize) / DeviceLargePageSize; - const size_t l2_index = (address % DeviceLargePageSize) / DevicePageSize; + KProcessAddress cur_process_address = process_address; + size_t remaining_size = size; + KPhysicalAddress cur_phys_address = GetHeapPhysicalAddress(contig_range.address); + size_t remaining_in_range = contig_range.size; + bool first = true; + u32 first_attr = 0; + while (remaining_size > 0) { + /* Convert the device address to a series of indices. */ + const size_t l0_index = (device_address / DeviceRegionSize); + const size_t l1_index = (device_address % DeviceRegionSize) / DeviceLargePageSize; + const size_t l2_index = (device_address % DeviceLargePageSize) / DevicePageSize; /* Get and validate l1. */ const PageDirectoryEntry *l1 = GetPointer(m_tables[l0_index]); - R_UNLESS(l1 != nullptr, svc::ResultInvalidCurrentMemory()); - R_UNLESS(l1[l1_index].IsValid(), svc::ResultInvalidCurrentMemory()); + if (!(l1 != nullptr && l1[l1_index].IsValid())) { + return false; + } if (l1[l1_index].IsTable()) { /* We're acting on an l2 entry. */ const PageTableEntry *l2 = GetPointer(GetPageTableVirtualAddress(l1[l1_index].GetPhysicalAddress())); + /* Determine the number of pages to check. */ const size_t remaining_in_entry = (PageTableSize / sizeof(PageTableEntry)) - l2_index; - const size_t map_count = std::min(remaining_in_entry, remaining / DevicePageSize); + const size_t map_count = std::min(remaining_in_entry, remaining_size / DevicePageSize); + /* Check each page. */ for (size_t i = 0; i < map_count; ++i) { /* Ensure the l2 entry is valid. */ - R_UNLESS(l2[l2_index + i].IsValid(), svc::ResultInvalidCurrentMemory()); - - /* Get the physical address. */ - const KPhysicalAddress phys_addr = l2[l2_index + i].GetPhysicalAddress(); - MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr)); - - /* Add to the group. */ - R_TRY(out->AddBlock(GetHeapVirtualAddress(phys_addr), DevicePageSize / PageSize)); - - /* If this is our first entry, get the attribute. */ - if (first) { - attr = l2[l2_index + i].GetAttributes(); - first = false; - } else { - /* Validate the attributes match the first entry. */ - R_UNLESS(l2[l2_index + i].GetAttributes() == attr, svc::ResultInvalidCurrentMemory()); + if (!l2[l2_index + i].IsValid()) { + return false; } + + /* Check that the attributes match the first attributes we encountered. */ + const u32 cur_attr = l2[l2_index + i].GetAttributes(); + if (!first && cur_attr != first_attr) { + return false; + } + + /* If there's nothing remaining in the range, refresh the range. */ + if (remaining_in_range == 0) { + contig_range.Close(); + + range_open = false; + if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), cur_process_address, remaining_size))) { + return false; + } + range_open = true; + + cur_phys_address = GetHeapPhysicalAddress(contig_range.address); + remaining_in_range = contig_range.size; + } + + /* Check that the physical address is expected. */ + if (l2[l2_index + i].GetPhysicalAddress() != cur_phys_address) { + return false; + } + + /* Advance. */ + cur_phys_address += DevicePageSize; + cur_process_address += DevicePageSize; + remaining_size -= DevicePageSize; + remaining_in_range -= DevicePageSize; + + first = false; + first_attr = cur_attr; } - /* Advance. */ - address += DevicePageSize * map_count; - remaining -= DevicePageSize * map_count; + /* Advance the device address. */ + device_address += map_count * DevicePageSize; } else { /* We're acting on an l1 entry. */ - R_UNLESS(l2_index == 0, svc::ResultInvalidCurrentMemory()); - R_UNLESS(remaining >= DeviceLargePageSize, svc::ResultInvalidCurrentMemory()); + if (!(l2_index == 0 && remaining_size >= DeviceLargePageSize)) { + return false; + } - /* Get the physical address. */ - const KPhysicalAddress phys_addr = l1[l1_index].GetPhysicalAddress(); - MESOSPHERE_ASSERT(IsHeapPhysicalAddress(phys_addr)); + /* Check that the attributes match the first attributes we encountered. */ + const u32 cur_attr = l1[l1_index].GetAttributes(); + if (!first && cur_attr != first_attr) { + return false; + } - /* Add to the group. */ - R_TRY(out->AddBlock(GetHeapVirtualAddress(phys_addr), DeviceLargePageSize / PageSize)); + /* If there's nothing remaining in the range, refresh the range. */ + if (remaining_in_range == 0) { + contig_range.Close(); - /* If this is our first entry, get the attribute. */ - if (first) { - attr = l1[l1_index].GetAttributes(); - first = false; - } else { - /* Validate the attributes match the first entry. */ - R_UNLESS(l1[l1_index].GetAttributes() == attr, svc::ResultInvalidCurrentMemory()); + range_open = false; + if (R_FAILED(page_table->OpenMemoryRangeForUnmapDeviceAddressSpace(std::addressof(contig_range), cur_process_address, remaining_size))) { + return false; + } + range_open = true; + + cur_phys_address = GetHeapPhysicalAddress(contig_range.address); + remaining_in_range = contig_range.size; + } + + /* Check that the physical address is expected, and there's enough in the range. */ + if (remaining_in_range < DeviceLargePageSize || l1[l1_index].GetPhysicalAddress() != cur_phys_address) { + return false; } /* Advance. */ - address += DeviceLargePageSize; - remaining -= DeviceLargePageSize; + cur_phys_address += DeviceLargePageSize; + cur_process_address += DeviceLargePageSize; + remaining_size -= DeviceLargePageSize; + remaining_in_range -= DeviceLargePageSize; + + first = false; + first_attr = cur_attr; + + /* Advance the device address. */ + device_address += DeviceLargePageSize; } } - return ResultSuccess(); + /* The range is valid! */ + return true; } - bool KDevicePageTable::Compare(const KPageGroup &compare_pg, KDeviceVirtualAddress device_address) const { - /* Check whether the page group we expect for the virtual address matches the page group we're validating. */ - KPageGroup calc_pg(std::addressof(Kernel::GetBlockInfoManager())); - return (R_SUCCEEDED(this->MakePageGroup(std::addressof(calc_pg), device_address, compare_pg.GetNumPages() * PageSize))) && - calc_pg.IsEquivalentTo(compare_pg); - } - - Result KDevicePageTable::Map(size_t *out_mapped_size, const KPageGroup &pg, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) { + Result KDevicePageTable::Map(size_t *out_mapped_size, KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address, ams::svc::MemoryPermission device_perm, bool refresh_mappings) { /* Clear the output size. */ *out_mapped_size = 0; /* Map the pages. */ s32 num_pt = 0; - return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits::max(), pg, device_address, device_perm); + return this->MapImpl(out_mapped_size, num_pt, refresh_mappings ? 1 : std::numeric_limits::max(), page_table, process_address, size, device_address, device_perm, refresh_mappings); } - Result KDevicePageTable::Unmap(const KPageGroup &pg, KDeviceVirtualAddress device_address) { + Result KDevicePageTable::Unmap(KProcessPageTable *page_table, KProcessAddress process_address, size_t size, KDeviceVirtualAddress device_address) { /* Validate address/size. */ - const size_t size = pg.GetNumPages() * PageSize; MESOSPHERE_ASSERT((device_address & ~DeviceVirtualAddressMask) == 0); MESOSPHERE_ASSERT(((device_address + size - 1) & ~DeviceVirtualAddressMask) == 0); /* Ensure the page group is correct. */ - R_UNLESS(this->Compare(pg, device_address), svc::ResultInvalidCurrentMemory()); + R_UNLESS(this->Compare(page_table, process_address, size, device_address), svc::ResultInvalidCurrentMemory()); /* Unmap the pages. */ this->UnmapImpl(device_address, size, false); diff --git a/libraries/libmesosphere/source/kern_k_device_address_space.cpp b/libraries/libmesosphere/source/kern_k_device_address_space.cpp index f9d0d23cd..28bfb6934 100644 --- a/libraries/libmesosphere/source/kern_k_device_address_space.cpp +++ b/libraries/libmesosphere/source/kern_k_device_address_space.cpp @@ -71,12 +71,11 @@ namespace ams::kern { /* Lock the address space. */ KScopedLightLock lk(m_lock); - /* Lock the pages. */ - KPageGroup pg(page_table->GetBlockInfoManager()); - R_TRY(page_table->LockForDeviceAddressSpace(std::addressof(pg), process_address, size, ConvertToKMemoryPermission(device_perm), is_aligned)); + /* Lock the page table to prevent concurrent device mapping operations. */ + KScopedLightLock pt_lk = page_table->AcquireDeviceMapLock(); - /* Close the pages we opened when we're done with them. */ - ON_SCOPE_EXIT { pg.Close(); }; + /* Lock the pages. */ + R_TRY(page_table->LockForMapDeviceAddressSpace(process_address, size, ConvertToKMemoryPermission(device_perm), is_aligned)); /* Ensure that if we fail, we don't keep unmapped pages locked. */ auto unlock_guard = SCOPE_GUARD { MESOSPHERE_R_ABORT_UNLESS(page_table->UnlockForDeviceAddressSpace(process_address, size)); }; @@ -87,7 +86,7 @@ namespace ams::kern { auto mapped_size_guard = SCOPE_GUARD { *out_mapped_size = 0; }; /* Perform the mapping. */ - R_TRY(m_table.Map(out_mapped_size, pg, device_address, device_perm, refresh_mappings)); + R_TRY(m_table.Map(out_mapped_size, page_table, process_address, size, device_address, device_perm, refresh_mappings)); /* Ensure that we unmap the pages if we fail to update the protections. */ /* NOTE: Nintendo does not check the result of this unmap call. */ @@ -113,19 +112,18 @@ namespace ams::kern { /* Lock the address space. */ KScopedLightLock lk(m_lock); - /* Make and open a page group for the unmapped region. */ - KPageGroup pg(page_table->GetBlockInfoManager()); - R_TRY(page_table->MakePageGroupForUnmapDeviceAddressSpace(std::addressof(pg), process_address, size)); + /* Lock the page table to prevent concurrent device mapping operations. */ + KScopedLightLock pt_lk = page_table->AcquireDeviceMapLock(); - /* Ensure the page group is closed on scope exit. */ - ON_SCOPE_EXIT { pg.Close(); }; + /* Lock the pages. */ + R_TRY(page_table->LockForUnmapDeviceAddressSpace(process_address, size)); /* If we fail to unmap, we want to do a partial unlock. */ { auto unlock_guard = SCOPE_GUARD { page_table->UnlockForDeviceAddressSpacePartialMap(process_address, size, size); }; /* Unmap. */ - R_TRY(m_table.Unmap(pg, device_address)); + R_TRY(m_table.Unmap(page_table, process_address, size, device_address)); unlock_guard.Cancel(); } diff --git a/libraries/libmesosphere/source/kern_k_page_table_base.cpp b/libraries/libmesosphere/source/kern_k_page_table_base.cpp index 3f2aff676..d47fad006 100644 --- a/libraries/libmesosphere/source/kern_k_page_table_base.cpp +++ b/libraries/libmesosphere/source/kern_k_page_table_base.cpp @@ -74,6 +74,10 @@ namespace ams::kern { } + void KPageTableBase::MemoryRange::Close() { + Kernel::GetMemoryManager().Close(address, size / PageSize); + } + Result KPageTableBase::InitializeForKernel(bool is_64_bit, void *table, KVirtualAddress start, KVirtualAddress end) { /* Initialize our members. */ m_address_space_width = (is_64_bit) ? BITSIZEOF(u64) : BITSIZEOF(u32); @@ -1391,6 +1395,49 @@ namespace ams::kern { return cur_block_address == GetHeapVirtualAddress(cur_addr) && cur_block_pages == (cur_size / PageSize); } + Result KPageTableBase::GetContiguousMemoryRangeWithState(MemoryRange *out, KProcessAddress address, size_t size, u32 state_mask, u32 state, u32 perm_mask, u32 perm, u32 attr_mask, u32 attr) { + MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); + + auto &impl = this->GetImpl(); + + /* Begin a traversal. */ + TraversalContext context; + TraversalEntry cur_entry = {}; + R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address), svc::ResultInvalidCurrentMemory()); + + /* The region we're traversing has to be heap. */ + const KPhysicalAddress phys_address = cur_entry.phys_addr; + R_UNLESS(this->IsHeapPhysicalAddress(phys_address), svc::ResultInvalidCurrentMemory()); + + /* Traverse until we have enough size or we aren't contiguous any more. */ + size_t contig_size; + for (contig_size = cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1)); contig_size < size; contig_size += cur_entry.block_size) { + if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) { + break; + } + if (cur_entry.phys_addr != phys_address + contig_size) { + break; + } + } + + /* Take the minimum size for our region. */ + size = std::min(size, contig_size); + + /* Check that the memory is contiguous. */ + R_TRY(this->CheckMemoryStateContiguous(address, size, + state_mask | KMemoryState_FlagReferenceCounted, state | KMemoryState_FlagReferenceCounted, + perm_mask, perm, + attr_mask, attr)); + + /* The memory is contiguous, so set the output range. */ + *out = { + .address = GetLinearMappedVirtualAddress(phys_address), + .size = size, + }; + + return ResultSuccess(); + } + Result KPageTableBase::SetMemoryPermission(KProcessAddress addr, size_t size, ams::svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; @@ -2578,7 +2625,7 @@ namespace ams::kern { return ResultSuccess(); } - Result KPageTableBase::LockForDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) { + Result KPageTableBase::LockForMapDeviceAddressSpace(KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) { /* Lightly validate the range before doing anything else. */ const size_t num_pages = size / PageSize; R_UNLESS(this->Contains(address, size), svc::ResultInvalidCurrentMemory()); @@ -2591,11 +2638,6 @@ namespace ams::kern { size_t num_allocator_blocks; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, test_state, test_state, perm, perm, KMemoryAttribute_IpcLocked | KMemoryAttribute_Locked, KMemoryAttribute_None, KMemoryAttribute_DeviceShared)); - /* Make the page group, if we should. */ - if (out != nullptr) { - R_TRY(this->MakePageGroup(*out, address, num_pages)); - } - /* Create an update allocator. */ Result allocator_result; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); @@ -2604,10 +2646,33 @@ namespace ams::kern { /* Update the memory blocks. */ m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, &KMemoryBlock::ShareToDevice, KMemoryPermission_None); - /* Open the page group. */ - if (out != nullptr) { - out->Open(); - } + return ResultSuccess(); + } + + Result KPageTableBase::LockForUnmapDeviceAddressSpace(KProcessAddress address, size_t size) { + /* Lightly validate the range before doing anything else. */ + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), svc::ResultInvalidCurrentMemory()); + + /* Lock the table. */ + KScopedLightLock lk(m_general_lock); + + /* Check the memory state. */ + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryStateContiguous(std::addressof(num_allocator_blocks), + address, size, + KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap, KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap, + KMemoryPermission_None, KMemoryPermission_None, + KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); + + /* Create an update allocator. */ + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + /* Update the memory blocks. */ + const KMemoryBlockManager::MemoryBlockLockFunction lock_func = m_enable_device_address_space_merge ? &KMemoryBlock::UpdateDeviceDisableMergeStateForShare : &KMemoryBlock::UpdateDeviceDisableMergeStateForShareRight; + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, KMemoryPermission_None); return ResultSuccess(); } @@ -2639,40 +2704,6 @@ namespace ams::kern { return ResultSuccess(); } - Result KPageTableBase::MakePageGroupForUnmapDeviceAddressSpace(KPageGroup *out, KProcessAddress address, size_t size) { - /* Lightly validate the range before doing anything else. */ - const size_t num_pages = size / PageSize; - R_UNLESS(this->Contains(address, size), svc::ResultInvalidCurrentMemory()); - - /* Lock the table. */ - KScopedLightLock lk(m_general_lock); - - /* Check the memory state. */ - size_t num_allocator_blocks; - R_TRY(this->CheckMemoryStateContiguous(std::addressof(num_allocator_blocks), - address, size, - KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap, KMemoryState_FlagReferenceCounted | KMemoryState_FlagCanDeviceMap, - KMemoryPermission_None, KMemoryPermission_None, - KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); - - /* Create an update allocator. */ - Result allocator_result; - KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); - R_TRY(allocator_result); - - /* Make the page group. */ - R_TRY(this->MakePageGroup(*out, address, num_pages)); - - /* Update the memory blocks. */ - const KMemoryBlockManager::MemoryBlockLockFunction lock_func = m_enable_device_address_space_merge ? &KMemoryBlock::UpdateDeviceDisableMergeStateForShare : &KMemoryBlock::UpdateDeviceDisableMergeStateForShareRight; - m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, KMemoryPermission_None); - - /* Open a reference to the pages in the page group. */ - out->Open(); - - return ResultSuccess(); - } - Result KPageTableBase::UnlockForDeviceAddressSpacePartialMap(KProcessAddress address, size_t size, size_t mapped_size) { /* Lightly validate the range before doing anything else. */ const size_t num_pages = size / PageSize; @@ -2689,23 +2720,23 @@ namespace ams::kern { size_t allocator_num_blocks = 0, unmapped_allocator_num_blocks = 0; if (unmapped_size) { if (m_enable_device_address_space_merge) { - R_TRY(this->CheckMemoryState(std::addressof(allocator_num_blocks), - address, size, - KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, - KMemoryPermission_None, KMemoryPermission_None, - KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); + R_TRY(this->CheckMemoryStateContiguous(std::addressof(allocator_num_blocks), + address, size, + KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, + KMemoryPermission_None, KMemoryPermission_None, + KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); } - R_TRY(this->CheckMemoryState(std::addressof(unmapped_allocator_num_blocks), - mapped_end_address, unmapped_size, - KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, - KMemoryPermission_None, KMemoryPermission_None, - KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); + R_TRY(this->CheckMemoryStateContiguous(std::addressof(unmapped_allocator_num_blocks), + mapped_end_address, unmapped_size, + KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, + KMemoryPermission_None, KMemoryPermission_None, + KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); } else { - R_TRY(this->CheckMemoryState(std::addressof(allocator_num_blocks), - address, size, - KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, - KMemoryPermission_None, KMemoryPermission_None, - KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); + R_TRY(this->CheckMemoryStateContiguous(std::addressof(allocator_num_blocks), + address, size, + KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, + KMemoryPermission_None, KMemoryPermission_None, + KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); } /* Create an update allocator for the region. */ @@ -2750,6 +2781,41 @@ namespace ams::kern { return ResultSuccess(); } + Result KPageTableBase::OpenMemoryRangeForMapDeviceAddressSpace(KPageTableBase::MemoryRange *out, KProcessAddress address, size_t size, KMemoryPermission perm, bool is_aligned) { + /* Lock the table. */ + KScopedLightLock lk(m_general_lock); + + /* Get the range. */ + const u32 test_state = KMemoryState_FlagReferenceCounted | (is_aligned ? KMemoryState_FlagCanAlignedDeviceMap : KMemoryState_FlagCanDeviceMap); + R_TRY(this->GetContiguousMemoryRangeWithState(out, + address, size, + test_state, test_state, + perm, perm, + KMemoryAttribute_IpcLocked | KMemoryAttribute_Locked, KMemoryAttribute_None)); + + /* We got the range, so open it. */ + Kernel::GetMemoryManager().Open(out->address, out->size / PageSize); + + return ResultSuccess(); + } + + Result KPageTableBase::OpenMemoryRangeForUnmapDeviceAddressSpace(MemoryRange *out, KProcessAddress address, size_t size) { + /* Lock the table. */ + KScopedLightLock lk(m_general_lock); + + /* Get the range. */ + R_TRY(this->GetContiguousMemoryRangeWithState(out, + address, size, + KMemoryState_FlagCanDeviceMap, KMemoryState_FlagCanDeviceMap, + KMemoryPermission_None, KMemoryPermission_None, + KMemoryAttribute_DeviceShared | KMemoryAttribute_Locked, KMemoryAttribute_DeviceShared)); + + /* We got the range, so open it. */ + Kernel::GetMemoryManager().Open(out->address, out->size / PageSize); + + return ResultSuccess(); + } + Result KPageTableBase::LockForIpcUserBuffer(KPhysicalAddress *out, KProcessAddress address, size_t size) { return this->LockMemoryAndOpen(nullptr, out, address, size, KMemoryState_FlagCanIpcUserBuffer, KMemoryState_FlagCanIpcUserBuffer, @@ -2804,6 +2870,23 @@ namespace ams::kern { KMemoryAttribute_Locked, std::addressof(pg)); } + Result KPageTableBase::OpenMemoryRangeForProcessCacheOperation(MemoryRange *out, KProcessAddress address, size_t size) { + /* Lock the table. */ + KScopedLightLock lk(m_general_lock); + + /* Get the range. */ + R_TRY(this->GetContiguousMemoryRangeWithState(out, + address, size, + KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted, + KMemoryPermission_UserRead, KMemoryPermission_UserRead, + KMemoryAttribute_Uncached, KMemoryAttribute_None)); + + /* We got the range, so open it. */ + Kernel::GetMemoryManager().Open(out->address, out->size / PageSize); + + return ResultSuccess(); + } + Result KPageTableBase::CopyMemoryFromLinearToUser(KProcessAddress dst_addr, size_t size, KProcessAddress src_addr, u32 src_state_mask, u32 src_state, KMemoryPermission src_test_perm, u32 src_attr_mask, u32 src_attr) { /* Lightly validate the range before doing anything else. */ R_UNLESS(this->Contains(src_addr, size), svc::ResultInvalidCurrentMemory()); @@ -4553,4 +4636,108 @@ namespace ams::kern { return ResultSuccess(); } + Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t size, KPageTableBase &src_page_table, KProcessAddress src_address) { + /* We need to lock both this table, and the current process's table, so set up an alias. */ + KPageTableBase &dst_page_table = *this; + + /* Acquire the table locks. */ + KScopedLightLockPair lk(src_page_table.m_general_lock, dst_page_table.m_general_lock); + + /* Check that the memory is mapped in the destination process. */ + size_t num_allocator_blocks; + R_TRY(dst_page_table.CheckMemoryState(std::addressof(num_allocator_blocks), dst_address, size, KMemoryState_All, KMemoryState_SharedCode, KMemoryPermission_UserReadWrite, KMemoryPermission_UserReadWrite, KMemoryAttribute_All, KMemoryAttribute_None)); + + /* Check that the memory is mapped in the source process. */ + R_TRY(src_page_table.CheckMemoryState(src_address, size, KMemoryState_FlagCanMapProcess, KMemoryState_FlagCanMapProcess, KMemoryPermission_None, KMemoryPermission_None, KMemoryAttribute_All, KMemoryAttribute_None)); + + /* Validate that the memory ranges are compatible. */ + { + /* Define a helper type. */ + struct ContiguousRangeInfo { + public: + KPageTableBase &m_pt; + TraversalContext m_context; + TraversalEntry m_entry; + KPhysicalAddress m_phys_addr; + size_t m_cur_size; + size_t m_remaining_size; + public: + ContiguousRangeInfo(KPageTableBase &pt, KProcessAddress address, size_t size) : m_pt(pt), m_remaining_size(size) { + /* Begin a traversal. */ + MESOSPHERE_ABORT_UNLESS(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry), std::addressof(m_context), address)); + + /* Setup tracking fields. */ + m_phys_addr = m_entry.phys_addr; + m_cur_size = std::min(m_remaining_size, m_entry.block_size - (GetInteger(m_phys_addr) & (m_entry.block_size - 1))); + + /* Consume the whole contiguous block. */ + this->DetermineContiguousBlockExtents(); + } + + void ContinueTraversal() { + /* Update our remaining size. */ + m_remaining_size = m_remaining_size - m_cur_size; + + /* Update our tracking fields. */ + if (m_remaining_size > 0) { + m_phys_addr = m_entry.phys_addr; + m_cur_size = std::min(m_remaining_size, m_entry.block_size); + + /* Consume the whole contiguous block. */ + this->DetermineContiguousBlockExtents(); + } + } + private: + void DetermineContiguousBlockExtents() { + /* Continue traversing until we're not contiguous, or we have enough. */ + while (m_cur_size < m_remaining_size) { + MESOSPHERE_ABORT_UNLESS(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry), std::addressof(m_context))); + + /* If we're not contiguous, we're done. */ + if (m_entry.phys_addr != m_phys_addr + m_cur_size) { + break; + } + + /* Update our current size. */ + m_cur_size = std::min(m_remaining_size, m_cur_size + m_entry.block_size); + } + } + }; + + /* Create ranges for both tables. */ + ContiguousRangeInfo src_range(src_page_table, src_address, size); + ContiguousRangeInfo dst_range(dst_page_table, dst_address, size); + + /* Validate the ranges. */ + while (src_range.m_remaining_size > 0 && dst_range.m_remaining_size > 0) { + R_UNLESS(src_range.m_phys_addr == dst_range.m_phys_addr, svc::ResultInvalidMemoryRegion()); + R_UNLESS(src_range.m_cur_size == dst_range.m_cur_size, svc::ResultInvalidMemoryRegion()); + + src_range.ContinueTraversal(); + dst_range.ContinueTraversal(); + } + } + + /* We no longer need to hold our lock on the source page table. */ + lk.TryUnlockHalf(src_page_table.m_general_lock); + + /* Create an update allocator. */ + Result allocator_result; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + /* We're going to perform an update, so create a helper. */ + KScopedPageTableUpdater updater(this); + + /* Unmap the memory. */ + const size_t num_pages = size / PageSize; + const KPageProperties unmap_properties = { KMemoryPermission_None, false, false, DisableMergeAttribute_None }; + R_TRY(this->Operate(updater.GetPageList(), dst_address, num_pages, Null, false, unmap_properties, OperationType_Unmap, false)); + + /* Apply the memory block update. */ + m_memory_block_manager.Update(std::addressof(allocator), dst_address, num_pages, KMemoryState_Free, KMemoryPermission_None, KMemoryAttribute_None, KMemoryBlockDisableMergeAttribute_None, KMemoryBlockDisableMergeAttribute_Normal); + + return ResultSuccess(); + } + } diff --git a/libraries/libmesosphere/source/svc/kern_svc_cache.cpp b/libraries/libmesosphere/source/svc/kern_svc_cache.cpp index 8deb3f595..e5b6e3ae4 100644 --- a/libraries/libmesosphere/source/svc/kern_svc_cache.cpp +++ b/libraries/libmesosphere/source/svc/kern_svc_cache.cpp @@ -30,32 +30,24 @@ namespace ams::kern::svc { /* Determine aligned extents. */ const uintptr_t aligned_start = util::AlignDown(address, PageSize); const uintptr_t aligned_end = util::AlignUp(address + size, PageSize); - const size_t num_pages = (aligned_end - aligned_start) / PageSize; - /* Create a page group for the process's memory. */ - KPageGroup pg(page_table.GetBlockInfoManager()); - - /* Make and open the page group. */ - R_TRY(page_table.MakeAndOpenPageGroup(std::addressof(pg), - aligned_start, num_pages, - KMemoryState_FlagReferenceCounted, KMemoryState_FlagReferenceCounted, - KMemoryPermission_UserRead, KMemoryPermission_UserRead, - KMemoryAttribute_Uncached, KMemoryAttribute_None)); - - /* Ensure we don't leak references to the pages we're operating on. */ - ON_SCOPE_EXIT { pg.Close(); }; - - /* Operate on all the blocks. */ + /* Iterate over and operate on contiguous ranges. */ uintptr_t cur_address = aligned_start; size_t remaining = size; - for (const auto &block : pg) { - /* Get the block extents. */ - KVirtualAddress operate_address = block.GetAddress(); - size_t operate_size = block.GetSize(); + while (remaining > 0) { + /* Get a contiguous range to operate on. */ + KPageTableBase::MemoryRange contig_range = {}; + R_TRY(page_table.OpenMemoryRangeForProcessCacheOperation(std::addressof(contig_range), cur_address, aligned_end - cur_address)); + + /* Close the range when we're done operating on it. */ + ON_SCOPE_EXIT { contig_range.Close(); }; /* Adjust to remain within range. */ + KVirtualAddress operate_address = contig_range.address; + size_t operate_size = contig_range.size; if (cur_address < address) { operate_address += (address - cur_address); + operate_size -= (address - cur_address); } if (operate_size > remaining) { operate_size = remaining; @@ -65,7 +57,7 @@ namespace ams::kern::svc { operation.Operate(GetVoidPointer(operate_address), operate_size); /* Advance. */ - cur_address += block.GetSize(); + cur_address += contig_range.size; remaining -= operate_size; } MESOSPHERE_ASSERT(remaining == 0); diff --git a/libraries/libmesosphere/source/svc/kern_svc_process_memory.cpp b/libraries/libmesosphere/source/svc/kern_svc_process_memory.cpp index bd8f8cb0c..7df608e67 100644 --- a/libraries/libmesosphere/source/svc/kern_svc_process_memory.cpp +++ b/libraries/libmesosphere/source/svc/kern_svc_process_memory.cpp @@ -122,21 +122,8 @@ namespace ams::kern::svc { R_UNLESS(src_pt.Contains(src_address, size), svc::ResultInvalidCurrentMemory()); R_UNLESS(dst_pt.CanContain(dst_address, size, KMemoryState_SharedCode), svc::ResultInvalidMemoryRegion()); - /* Create a new page group. */ - KPageGroup pg(dst_pt.GetBlockInfoManager()); - - /* Make the page group. */ - R_TRY(src_pt.MakeAndOpenPageGroup(std::addressof(pg), - src_address, size / PageSize, - KMemoryState_FlagCanMapProcess, KMemoryState_FlagCanMapProcess, - KMemoryPermission_None, KMemoryPermission_None, - KMemoryAttribute_All, KMemoryAttribute_None)); - - /* Close the page group when we're done. */ - ON_SCOPE_EXIT { pg.Close(); }; - - /* Unmap the group. */ - R_TRY(dst_pt.UnmapPageGroup(dst_address, pg, KMemoryState_SharedCode)); + /* Unmap the memory. */ + R_TRY(dst_pt.UnmapProcessMemory(dst_address, size, src_pt, src_address)); return ResultSuccess(); }