fs.mitm: lazily initialize sd-romfs metadata

stratosphere/ams_mitm/source/fs_mitm/fs_mitm_service.cpp

@@ -28,6 +28,7 @@ namespace ams::mitm::fs {
 
         constexpr const char AtmosphereHblWebContentDir[] = "/atmosphere/hbl_html/";
 
+        os::Mutex g_data_storage_lock;
         os::Mutex g_storage_cache_lock;
         std::unordered_map<u64, std::weak_ptr<IStorageInterface>> g_storage_cache;
 
@@ -236,6 +237,9 @@ namespace ams::mitm::fs {
         R_TRY(fsOpenDataStorageByCurrentProcessFwd(this->forward_service.get(), &data_storage));
         const sf::cmif::DomainObjectId target_object_id{serviceGetObjectId(&data_storage.s)};
 
+        /* Get a scoped lock. */
+        std::scoped_lock lk(g_data_storage_lock);
+
         /* Try to get a storage from the cache. */
         {
             std::shared_ptr<IStorageInterface> cached_storage = GetStorageCacheEntry(this->client_info.program_id);
@@ -252,10 +256,12 @@ namespace ams::mitm::fs {
             /* Create the layered storage. */
             FsFile data_file;
             if (R_SUCCEEDED(OpenAtmosphereSdFile(&data_file, this->client_info.program_id, "romfs.bin", OpenMode_Read))) {
-                auto *layered_storage = new LayeredRomfsStorage(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), std::make_unique<ReadOnlyStorageAdapter>(new FileStorage(new RemoteFile(data_file))), this->client_info.program_id);
+                auto layered_storage = std::make_shared<LayeredRomfsStorage>(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), std::make_unique<ReadOnlyStorageAdapter>(new FileStorage(new RemoteFile(data_file))), this->client_info.program_id);
+                layered_storage->BeginInitialize();
                 new_storage_intf = std::make_shared<IStorageInterface>(layered_storage);
             } else {
-                auto *layered_storage = new LayeredRomfsStorage(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), nullptr, this->client_info.program_id);
+                auto layered_storage = std::make_shared<LayeredRomfsStorage>(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), nullptr, this->client_info.program_id);
+                layered_storage->BeginInitialize();
                 new_storage_intf = std::make_shared<IStorageInterface>(layered_storage);
             }
 
@@ -278,6 +284,9 @@ namespace ams::mitm::fs {
         R_TRY(fsOpenDataStorageByDataIdFwd(this->forward_service.get(), &data_storage, static_cast<u64>(data_id), static_cast<NcmStorageId>(storage_id)));
         const sf::cmif::DomainObjectId target_object_id{serviceGetObjectId(&data_storage.s)};
 
+        /* Get a scoped lock. */
+        std::scoped_lock lk(g_data_storage_lock);
+
         /* Try to get a storage from the cache. */
         {
             std::shared_ptr<IStorageInterface> cached_storage = GetStorageCacheEntry(data_id);
@@ -294,10 +303,12 @@ namespace ams::mitm::fs {
             /* Create the layered storage. */
             FsFile data_file;
             if (R_SUCCEEDED(OpenAtmosphereSdFile(&data_file, data_id, "romfs.bin", OpenMode_Read))) {
-                auto *layered_storage = new LayeredRomfsStorage(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), std::make_unique<ReadOnlyStorageAdapter>(new FileStorage(new RemoteFile(data_file))), data_id);
+                auto layered_storage = std::make_shared<LayeredRomfsStorage>(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), std::make_unique<ReadOnlyStorageAdapter>(new FileStorage(new RemoteFile(data_file))), data_id);
+                layered_storage->BeginInitialize();
                 new_storage_intf = std::make_shared<IStorageInterface>(layered_storage);
             } else {
-                auto *layered_storage = new LayeredRomfsStorage(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), nullptr, data_id);
+                auto layered_storage = std::make_shared<LayeredRomfsStorage>(std::make_unique<ReadOnlyStorageAdapter>(new RemoteStorage(data_storage)), nullptr, data_id);
+                layered_storage->BeginInitialize();
                 new_storage_intf = std::make_shared<IStorageInterface>(layered_storage);
             }
 

stratosphere/ams_mitm/source/fs_mitm/fsmitm_layered_romfs_storage.cpp

@@ -20,9 +20,62 @@
 
 namespace ams::mitm::fs {
 
+    namespace {
+
+        os::Mutex g_mq_lock;
+        bool g_started_req_thread;
+        os::MessageQueue g_req_mq(1);
+        os::MessageQueue g_ack_mq(1);
+
+        void RomfsInitializerThreadFunction(void *arg) {
+            while (true) {
+                uintptr_t storage_uptr = 0;
+                g_req_mq.Receive(&storage_uptr);
+                std::shared_ptr<LayeredRomfsStorage> layered_storage = reinterpret_cast<LayeredRomfsStorage *>(storage_uptr)->GetShared();
+                g_ack_mq.Send(storage_uptr);
+                layered_storage->InitializeImpl();
+            }
+        }
+
+        constexpr size_t RomfsInitializerThreadStackSize = 0x8000;
+        constexpr int    RomfsInitializerThreadPriority  = 44;
+        os::StaticThread<RomfsInitializerThreadStackSize> g_romfs_initializer_thread(&RomfsInitializerThreadFunction, nullptr, RomfsInitializerThreadPriority);
+
+        void RequestInitializeStorage(uintptr_t storage_uptr) {
+            std::scoped_lock lk(g_mq_lock);
+
+            if (!g_started_req_thread) {
+                R_ASSERT(g_romfs_initializer_thread.Start());
+                g_started_req_thread = true;
+            }
+
+            g_req_mq.Send(storage_uptr);
+            uintptr_t ack = 0;
+            g_ack_mq.Receive(&ack);
+            AMS_ASSERT(ack == storage_uptr);
+        }
+
+    }
+
     using namespace ams::fs;
 
-    LayeredRomfsStorage::LayeredRomfsStorage(std::unique_ptr<IStorage> s_r, std::unique_ptr<IStorage> f_r, ncm::ProgramId pr_id) : storage_romfs(std::move(s_r)), file_romfs(std::move(f_r)), program_id(std::move(pr_id)) {
+    LayeredRomfsStorage::LayeredRomfsStorage(std::unique_ptr<IStorage> s_r, std::unique_ptr<IStorage> f_r, ncm::ProgramId pr_id) : storage_romfs(std::move(s_r)), file_romfs(std::move(f_r)), initialize_event(false, false), program_id(std::move(pr_id)), is_initialized(false), started_initialize(false) {
+        /* ... */
+    }
+
+    LayeredRomfsStorage::~LayeredRomfsStorage() {
+        for (size_t i = 0; i < this->source_infos.size(); i++) {
+            this->source_infos[i].Cleanup();
+        }
+    }
+
+    void LayeredRomfsStorage::BeginInitialize() {
+        AMS_ASSERT(!this->started_initialize);
+        RequestInitializeStorage(reinterpret_cast<uintptr_t>(this));
+        this->started_initialize = true;
+    }
+
+    void LayeredRomfsStorage::InitializeImpl() {
         /* Build new virtual romfs. */
         romfs::Builder builder(this->program_id);
 
@@ -37,12 +90,9 @@ namespace ams::mitm::fs {
         }
 
         builder.Build(&this->source_infos);
-    }
 
-    LayeredRomfsStorage::~LayeredRomfsStorage() {
+        this->is_initialized = true;
-        for (size_t i = 0; i < this->source_infos.size(); i++) {
+        this->initialize_event.Signal();
-            this->source_infos[i].Cleanup();
-        }
     }
 
     Result LayeredRomfsStorage::Read(s64 offset, void *buffer, size_t size) {
@@ -50,6 +100,10 @@ namespace ams::mitm::fs {
         R_UNLESS(size >= 0, fs::ResultInvalidSize());
         R_UNLESS(size > 0,  ResultSuccess());
 
+        /* Ensure we're initialized. */
+        if (!this->is_initialized) {
+            this->initialize_event.Wait();
+        }
 
         /* Validate offset/size. */
         const s64 virt_size = this->GetSize();
@@ -123,6 +177,11 @@ namespace ams::mitm::fs {
     }
 
     Result LayeredRomfsStorage::GetSize(s64 *out_size) {
+        /* Ensure we're initialized. */
+        if (!this->is_initialized) {
+            this->initialize_event.Wait();
+        }
+
         *out_size = this->GetSize();
         return ResultSuccess();
     }

stratosphere/ams_mitm/source/fs_mitm/fsmitm_layered_romfs_storage.hpp

@@ -20,12 +20,15 @@
 
 namespace ams::mitm::fs {
 
-    class LayeredRomfsStorage : public ams::fs::IStorage {
+    class LayeredRomfsStorage : public std::enable_shared_from_this<LayeredRomfsStorage>, public ams::fs::IStorage {
         private:
             std::vector<romfs::SourceInfo> source_infos;
             std::unique_ptr<ams::fs::IStorage> storage_romfs;
             std::unique_ptr<ams::fs::IStorage> file_romfs;
+            os::Event initialize_event;
             ncm::ProgramId program_id;
+            bool is_initialized;
+            bool started_initialize;
         protected:
             inline s64 GetSize() const {
                 const auto &back = this->source_infos.back();
@@ -35,6 +38,13 @@ namespace ams::mitm::fs {
             LayeredRomfsStorage(std::unique_ptr<ams::fs::IStorage> s_r, std::unique_ptr<ams::fs::IStorage> f_r, ncm::ProgramId pr_id);
             virtual ~LayeredRomfsStorage();
 
+            void BeginInitialize();
+            void InitializeImpl();
+
+            std::shared_ptr<LayeredRomfsStorage> GetShared() {
+                return this->shared_from_this();
+            }
+
             virtual Result Read(s64 offset, void *buffer, size_t size) override;
             virtual Result GetSize(s64 *out_size) override;
             virtual Result Flush() override;