nold/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere synced 2024-12-22 20:31:14 +00:00
Code Issues Projects Releases Packages Wiki Activity

fs.mitm: conserve memory when building romfs paths

Browse source
This commit is contained in:
Michael Scire 2019-12-30 02:51:32 -08:00
parent a2d2b1b346
commit 2ae298de24
2 changed files with 108 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
stratosphere/ams_mitm/source/fs_mitm/fsmitm_romfs.cpp
View file

@ -111,44 +111,57 @@ namespace ams::mitm::fs {
} }
} }
os::Mutex g_fs_romfs_path_lock;
char g_fs_romfs_path_buffer[fs::EntryNameLengthMax + 1];
__attribute__((noinline)) void OpenFileSystemRomfsDirectory(FsDir *out, ncm::ProgramId program_id, BuildDirectoryContext *parent, fs::OpenDirectoryMode mode, FsFileSystem *fs) {
std::scoped_lock lk(g_fs_romfs_path_lock);
parent->GetPath(g_fs_romfs_path_buffer);
R_ASSERT(mitm::fs::OpenAtmosphereRomfsDirectory(out, program_id, g_fs_romfs_path_buffer, mode, fs));
}
} }
Builder::Builder(ncm::ProgramId pr_id) : program_id(pr_id), num_dirs(0), num_files(0), dir_table_size(0), file_table_size(0), dir_hash_table_size(0), file_hash_table_size(0), file_partition_size(0) { Builder::Builder(ncm::ProgramId pr_id) : program_id(pr_id), num_dirs(0), num_files(0), dir_table_size(0), file_table_size(0), dir_hash_table_size(0), file_hash_table_size(0), file_partition_size(0) {
auto res = this->directories.emplace("", std::make_unique<BuildDirectoryContext>(BuildDirectoryContext::RootTag{})); auto res = this->directories.emplace(std::make_unique<BuildDirectoryContext>(BuildDirectoryContext::RootTag{}));
AMS_ASSERT(res.second); AMS_ASSERT(res.second);
this->root = res.first->second.get(); this->root = res.first->get();
this->num_dirs = 1; this->num_dirs = 1;
this->dir_table_size = 0x18; this->dir_table_size = 0x18;
} }
void Builder::AddDirectory(BuildDirectoryContext **out, BuildDirectoryContext *parent_ctx, std::unique_ptr<BuildDirectoryContext> child_ctx) { void Builder::AddDirectory(BuildDirectoryContext **out, BuildDirectoryContext *parent_ctx, std::unique_ptr<BuildDirectoryContext> child_ctx) {
/* Set parent context member. */
child_ctx->parent = parent_ctx;
/* Check if the directory already exists. */ /* Check if the directory already exists. */
auto existing = this->directories.find(child_ctx->path.get()); auto existing = this->directories.find(child_ctx);
if (existing != this->directories.end()) { if (existing != this->directories.end()) {
*out = existing->second.get(); *out = existing->get();
return; return;
} }
/* Add a new directory. */ /* Add a new directory. */
this->num_dirs++; this->num_dirs++;
this->dir_table_size += sizeof(DirectoryEntry) + util::AlignUp(child_ctx->path_len - child_ctx->cur_path_ofs, 4); this->dir_table_size += sizeof(DirectoryEntry) + util::AlignUp(child_ctx->path_len, 4);
child_ctx->parent = parent_ctx;
*out = child_ctx.get(); *out = child_ctx.get();
this->directories.emplace(child_ctx->path.get(), std::move(child_ctx)); this->directories.emplace(std::move(child_ctx));
} }
void Builder::AddFile(BuildDirectoryContext *parent_ctx, std::unique_ptr<BuildFileContext> file_ctx) { void Builder::AddFile(BuildDirectoryContext *parent_ctx, std::unique_ptr<BuildFileContext> file_ctx) {
/* Set parent context member. */
file_ctx->parent = parent_ctx;
/* Check if the file already exists. */ /* Check if the file already exists. */
if (this->files.find(file_ctx->path.get()) != this->files.end()) { if (this->files.find(file_ctx) != this->files.end()) {
return; return;
} }
/* Add a new file. */ /* Add a new file. */
this->num_files++; this->num_files++;
this->file_table_size += sizeof(FileEntry) + util::AlignUp(file_ctx->path_len - file_ctx->cur_path_ofs, 4); this->file_table_size += sizeof(FileEntry) + util::AlignUp(file_ctx->path_len, 4);
file_ctx->parent = parent_ctx; this->files.emplace(std::move(file_ctx));
this->files.emplace(file_ctx->path.get(), std::move(file_ctx));
} }
void Builder::VisitDirectory(FsFileSystem *fs, BuildDirectoryContext *parent) { void Builder::VisitDirectory(FsFileSystem *fs, BuildDirectoryContext *parent) {
@ -157,7 +170,7 @@ namespace ams::mitm::fs {
/* Get number of child directories. */ /* Get number of child directories. */
s64 num_child_dirs = 0; s64 num_child_dirs = 0;
{ {
R_ASSERT(mitm::fs::OpenAtmosphereRomfsDirectory(&dir, this->program_id, parent->path.get(), OpenDirectoryMode_Directory, fs)); OpenFileSystemRomfsDirectory(&dir, this->program_id, parent, OpenDirectoryMode_Directory, fs);
ON_SCOPE_EXIT { fsDirClose(&dir); }; ON_SCOPE_EXIT { fsDirClose(&dir); };
R_ASSERT(fsDirGetEntryCount(&dir, &num_child_dirs)); R_ASSERT(fsDirGetEntryCount(&dir, &num_child_dirs));
} }
@ -169,8 +182,8 @@ namespace ams::mitm::fs {
AMS_ASSERT(child_dirs != nullptr); AMS_ASSERT(child_dirs != nullptr);
s64 cur_child_dir_ind = 0; s64 cur_child_dir_ind = 0;
R_ASSERT(mitm::fs::OpenAtmosphereRomfsDirectory(&dir, this->program_id, parent->path.get(), OpenDirectoryMode_All, fs));
{ {
OpenFileSystemRomfsDirectory(&dir, this->program_id, parent, OpenDirectoryMode_All, fs);
ON_SCOPE_EXIT { fsDirClose(&dir); }; ON_SCOPE_EXIT { fsDirClose(&dir); };
s64 read_entries = 0; s64 read_entries = 0;
@ -183,12 +196,12 @@ namespace ams::mitm::fs {
AMS_ASSERT(this->dir_entry.type == FsDirEntryType_Dir || this->dir_entry.type == FsDirEntryType_File); AMS_ASSERT(this->dir_entry.type == FsDirEntryType_Dir || this->dir_entry.type == FsDirEntryType_File);
if (this->dir_entry.type == FsDirEntryType_Dir) { if (this->dir_entry.type == FsDirEntryType_Dir) {
BuildDirectoryContext *real_child = nullptr; BuildDirectoryContext *real_child = nullptr;
this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(parent->path.get(), parent->path_len, this->dir_entry.name, strlen(this->dir_entry.name))); this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(this->dir_entry.name, strlen(this->dir_entry.name)));
AMS_ASSERT(real_child != nullptr); AMS_ASSERT(real_child != nullptr);
child_dirs[cur_child_dir_ind++] = real_child; child_dirs[cur_child_dir_ind++] = real_child;
AMS_ASSERT(cur_child_dir_ind <= num_child_dirs); AMS_ASSERT(cur_child_dir_ind <= num_child_dirs);
} else /* if (this->dir_entry.type == FsDirEntryType_File) */ { } else /* if (this->dir_entry.type == FsDirEntryType_File) */ {
this->AddFile(parent, std::make_unique<BuildFileContext>(parent->path.get(), parent->path_len, this->dir_entry.name, strlen(this->dir_entry.name), this->dir_entry.file_size, 0, this->cur_source_type)); this->AddFile(parent, std::make_unique<BuildFileContext>(this->dir_entry.name, strlen(this->dir_entry.name), this->dir_entry.file_size, 0, this->cur_source_type));
} }
} }
} }
@ -206,7 +219,7 @@ namespace ams::mitm::fs {
if (parent_entry->file != EmptyEntry) { if (parent_entry->file != EmptyEntry) {
const FileEntry *cur_file = GetFileEntry(file_table, parent_entry->file); const FileEntry *cur_file = GetFileEntry(file_table, parent_entry->file);
while (true) { while (true) {
this->AddFile(parent, std::make_unique<BuildFileContext>(parent->path.get(), parent->path_len, cur_file->name, cur_file->name_size, cur_file->size, cur_file->offset, this->cur_source_type)); this->AddFile(parent, std::make_unique<BuildFileContext>(cur_file->name, cur_file->name_size, cur_file->size, cur_file->offset, this->cur_source_type));
if (cur_file->sibling == EmptyEntry) { if (cur_file->sibling == EmptyEntry) {
break; break;
} }
@ -218,7 +231,7 @@ namespace ams::mitm::fs {
u32 cur_child_offset = parent_entry->child; u32 cur_child_offset = parent_entry->child;
while (true) { while (true) {
BuildDirectoryContext *real_child = nullptr; BuildDirectoryContext *real_child = nullptr;
this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(parent->path.get(), parent->path_len, cur_child->name, cur_child->name_size)); this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(cur_child->name, cur_child->name_size));
AMS_ASSERT(real_child != nullptr); AMS_ASSERT(real_child != nullptr);
this->VisitDirectory(real_child, cur_child_offset, dir_table, dir_table_size, file_table, file_table_size); this->VisitDirectory(real_child, cur_child_offset, dir_table, dir_table_size, file_table, file_table_size);
@ -308,7 +321,7 @@ namespace ams::mitm::fs {
BuildFileContext *cur_file = nullptr; BuildFileContext *cur_file = nullptr;
BuildFileContext *prev_file = nullptr; BuildFileContext *prev_file = nullptr;
for (const auto &it : this->files) { for (const auto &it : this->files) {
cur_file = it.second.get(); cur_file = it.get();
/* By default, pad to 0x10 alignment. */ /* By default, pad to 0x10 alignment. */
this->file_partition_size = util::AlignUp(this->file_partition_size, 0x10); this->file_partition_size = util::AlignUp(this->file_partition_size, 0x10);
@ -327,14 +340,14 @@ namespace ams::mitm::fs {
cur_file->offset = this->file_partition_size; cur_file->offset = this->file_partition_size;
this->file_partition_size += cur_file->size; this->file_partition_size += cur_file->size;
cur_file->entry_offset = entry_offset; cur_file->entry_offset = entry_offset;
entry_offset += sizeof(FileEntry) + util::AlignUp(cur_file->path_len - cur_file->cur_path_ofs, 4); entry_offset += sizeof(FileEntry) + util::AlignUp(cur_file->path_len, 4);
/* Save current file as prev for next iteration. */ /* Save current file as prev for next iteration. */
prev_file = cur_file; prev_file = cur_file;
} }
/* Assign deferred parent/sibling ownership. */ /* Assign deferred parent/sibling ownership. */
for (auto it = this->files.rbegin(); it != this->files.rend(); it++) { for (auto it = this->files.rbegin(); it != this->files.rend(); it++) {
cur_file = it->second.get(); cur_file = it->get();
cur_file->sibling = cur_file->parent->file; cur_file->sibling = cur_file->parent->file;
cur_file->parent->file = cur_file; cur_file->parent->file = cur_file;
} }
@ -345,13 +358,13 @@ namespace ams::mitm::fs {
u32 entry_offset = 0; u32 entry_offset = 0;
BuildDirectoryContext *cur_dir = nullptr; BuildDirectoryContext *cur_dir = nullptr;
for (const auto &it : this->directories) { for (const auto &it : this->directories) {
cur_dir = it.second.get(); cur_dir = it.get();
cur_dir->entry_offset = entry_offset; cur_dir->entry_offset = entry_offset;
entry_offset += sizeof(DirectoryEntry) + util::AlignUp(cur_dir->path_len - cur_dir->cur_path_ofs, 4); entry_offset += sizeof(DirectoryEntry) + util::AlignUp(cur_dir->path_len, 4);
} }
/* Assign deferred parent/sibling ownership. */ /* Assign deferred parent/sibling ownership. */
for (auto it = this->directories.rbegin(); it != this->directories.rend(); it++) { for (auto it = this->directories.rbegin(); it != this->directories.rend(); it++) {
cur_dir = it->second.get(); cur_dir = it->get();
if (cur_dir == this->root) { if (cur_dir == this->root) {
continue; continue;
} }
@ -362,7 +375,7 @@ namespace ams::mitm::fs {
/* Populate file tables. */ /* Populate file tables. */
for (const auto &it : this->files) { for (const auto &it : this->files) {
BuildFileContext *cur_file = it.second.get(); BuildFileContext *cur_file = it.get();
FileEntry *cur_entry = GetFileEntry(file_table, cur_file->entry_offset); FileEntry *cur_entry = GetFileEntry(file_table, cur_file->entry_offset);
/* Set entry fields. */ /* Set entry fields. */
@ -372,15 +385,15 @@ namespace ams::mitm::fs {
cur_entry->size = cur_file->size; cur_entry->size = cur_file->size;
/* Insert into hash table. */ /* Insert into hash table. */
const u32 name_size = cur_file->path_len - cur_file->cur_path_ofs; const u32 name_size = cur_file->path_len;
const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_file->path.get() + cur_file->cur_path_ofs, 0, name_size) % num_file_hash_table_entries; const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_file->path.get(), 0, name_size) % num_file_hash_table_entries;
cur_entry->hash = file_hash_table[hash_ind]; cur_entry->hash = file_hash_table[hash_ind];
file_hash_table[hash_ind] = cur_file->entry_offset; file_hash_table[hash_ind] = cur_file->entry_offset;
/* Set name. */ /* Set name. */
cur_entry->name_size = name_size; cur_entry->name_size = name_size;
if (name_size) { if (name_size) {
std::memcpy(cur_entry->name, cur_file->path.get() + cur_file->cur_path_ofs, name_size); std::memcpy(cur_entry->name, cur_file->path.get(), name_size);
for (size_t i = name_size; i < util::AlignUp(name_size, 4); i++) { for (size_t i = name_size; i < util::AlignUp(name_size, 4); i++) {
cur_entry->name[i] = 0; cur_entry->name[i] = 0;
} }
@ -402,7 +415,9 @@ namespace ams::mitm::fs {
break; break;
case DataSourceType::LooseSdFile: case DataSourceType::LooseSdFile:
{ {
out_infos->emplace_back(cur_file->offset + FilePartitionOffset, cur_file->size, cur_file->source_type, cur_file->path.release()); char *new_path = new char[cur_file->GetPathLength() + 1];
cur_file->GetPath(new_path);
out_infos->emplace_back(cur_file->offset + FilePartitionOffset, cur_file->size, cur_file->source_type, new_path);
} }
break; break;
AMS_UNREACHABLE_DEFAULT_CASE(); AMS_UNREACHABLE_DEFAULT_CASE();
@ -411,7 +426,7 @@ namespace ams::mitm::fs {
/* Populate directory tables. */ /* Populate directory tables. */
for (const auto &it : this->directories) { for (const auto &it : this->directories) {
BuildDirectoryContext *cur_dir = it.second.get(); BuildDirectoryContext *cur_dir = it.get();
DirectoryEntry *cur_entry = GetDirectoryEntry(dir_table, cur_dir->entry_offset); DirectoryEntry *cur_entry = GetDirectoryEntry(dir_table, cur_dir->entry_offset);
/* Set entry fields. */ /* Set entry fields. */
@ -421,15 +436,15 @@ namespace ams::mitm::fs {
cur_entry->file = (cur_dir->file == nullptr) ? EmptyEntry : cur_dir->file->entry_offset; cur_entry->file = (cur_dir->file == nullptr) ? EmptyEntry : cur_dir->file->entry_offset;
/* Insert into hash table. */ /* Insert into hash table. */
const u32 name_size = cur_dir->path_len - cur_dir->cur_path_ofs; const u32 name_size = cur_dir->path_len;
const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_dir->path.get() + cur_dir->cur_path_ofs, 0, name_size) % num_dir_hash_table_entries; const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_dir->path.get(), 0, name_size) % num_dir_hash_table_entries;
cur_entry->hash = dir_hash_table[hash_ind]; cur_entry->hash = dir_hash_table[hash_ind];
dir_hash_table[hash_ind] = cur_dir->entry_offset; dir_hash_table[hash_ind] = cur_dir->entry_offset;
/* Set name. */ /* Set name. */
cur_entry->name_size = name_size; cur_entry->name_size = name_size;
if (name_size) { if (name_size) {
std::memcpy(cur_entry->name, cur_dir->path.get() + cur_dir->cur_path_ofs, name_size); std::memcpy(cur_entry->name, cur_dir->path.get(), name_size);
for (size_t i = name_size; i < util::AlignUp(name_size, 4); i++) { for (size_t i = name_size; i < util::AlignUp(name_size, 4); i++) {
cur_entry->name[i] = 0; cur_entry->name[i] = 0;
} }

83
stratosphere/ams_mitm/source/fs_mitm/fsmitm_romfs.hpp
View file

@ -118,24 +118,41 @@ namespace ams::mitm::fs::romfs {
BuildDirectoryContext *child; BuildDirectoryContext *child;
BuildDirectoryContext *sibling; BuildDirectoryContext *sibling;
BuildFileContext *file; BuildFileContext *file;
u32 cur_path_ofs;
u32 path_len; u32 path_len;
u32 entry_offset; u32 entry_offset;
struct RootTag{}; struct RootTag{};
BuildDirectoryContext(RootTag) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), cur_path_ofs(0), path_len(0), entry_offset(0) { BuildDirectoryContext(RootTag) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), path_len(0), entry_offset(0) {
this->path = std::make_unique<char[]>(1); this->path = std::make_unique<char[]>(1);
} }
BuildDirectoryContext(const char *parent_path, size_t parent_path_len, const char *entry_name, size_t entry_name_len) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), entry_offset(0) { BuildDirectoryContext(const char *entry_name, size_t entry_name_len) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), entry_offset(0) {
this->cur_path_ofs = parent_path_len + 1; this->path_len = entry_name_len;
this->path_len = this->cur_path_ofs + entry_name_len;
this->path = std::unique_ptr<char[]>(new char[this->path_len + 1]); this->path = std::unique_ptr<char[]>(new char[this->path_len + 1]);
std::memcpy(this->path.get(), parent_path, parent_path_len); std::memcpy(this->path.get(), entry_name, entry_name_len);
this->path[parent_path_len] = '/'; this->path[this->path_len] = '\x00';
std::memcpy(this->path.get() + parent_path_len + 1, entry_name, entry_name_len); }
this->path[this->path_len] = 0;
size_t GetPathLength() const {
if (this->parent == nullptr) {
return 0;
}
return this->parent->GetPathLength() + 1 + this->path_len;
}
size_t GetPath(char *dst) const {
if (this->parent == nullptr) {
dst[0] = '\x00';
return 0;
}
const size_t parent_len = this->parent->GetPath(dst);
dst[parent_len] = '/';
std::memcpy(dst + parent_len + 1, this->path.get(), this->path_len);
dst[parent_len + 1 + this->path_len] = '\x00';
return parent_len + 1 + this->path_len;
} }
}; };
@ -149,27 +166,45 @@ namespace ams::mitm::fs::romfs {
s64 offset; s64 offset;
s64 size; s64 size;
s64 orig_offset; s64 orig_offset;
u32 cur_path_ofs;
u32 path_len; u32 path_len;
u32 entry_offset; u32 entry_offset;
DataSourceType source_type; DataSourceType source_type;
BuildFileContext(const char *parent_path, size_t parent_path_len, const char *entry_name, size_t entry_name_len, s64 sz, s64 o_o, DataSourceType type) : parent(nullptr), sibling(nullptr), offset(0), size(sz), orig_offset(o_o), entry_offset(0), source_type(type) { BuildFileContext(const char *entry_name, size_t entry_name_len, s64 sz, s64 o_o, DataSourceType type) : parent(nullptr), sibling(nullptr), offset(0), size(sz), orig_offset(o_o), entry_offset(0), source_type(type) {
this->cur_path_ofs = parent_path_len + 1; this->path_len = entry_name_len;
this->path_len = this->cur_path_ofs + entry_name_len;
this->path = std::unique_ptr<char[]>(new char[this->path_len + 1]); this->path = std::unique_ptr<char[]>(new char[this->path_len + 1]);
std::memcpy(this->path.get(), parent_path, parent_path_len); std::memcpy(this->path.get(), entry_name, entry_name_len);
this->path[parent_path_len] = '/';
std::memcpy(this->path.get() + parent_path_len + 1, entry_name, entry_name_len);
this->path[this->path_len] = 0; this->path[this->path_len] = 0;
} }
size_t GetPathLength() const {
if (this->parent == nullptr) {
return 0;
}
return this->parent->GetPathLength() + 1 + this->path_len;
}
size_t GetPath(char *dst) const {
if (this->parent == nullptr) {
dst[0] = '\x00';
return 0;
}
const size_t parent_len = this->parent->GetPath(dst);
dst[parent_len] = '/';
std::memcpy(dst + parent_len + 1, this->path.get(), this->path_len);
dst[parent_len + 1 + this->path_len] = '\x00';
return parent_len + 1 + this->path_len;
}
}; };
struct Builder { struct Builder {
NON_COPYABLE(Builder); NON_COPYABLE(Builder);
NON_MOVEABLE(Builder); NON_MOVEABLE(Builder);
private: private:
struct PathCompare { template<typename T>
struct Comparator {
static constexpr inline int Compare(const char *a, const char *b) { static constexpr inline int Compare(const char *a, const char *b) {
unsigned char c1{}, c2{}; unsigned char c1{}, c2{};
while ((c1 = *a++) == (c2 = *b++)) { while ((c1 = *a++) == (c2 = *b++)) {
@ -180,18 +215,22 @@ namespace ams::mitm::fs::romfs {
return (c1 - c2); return (c1 - c2);
} }
constexpr bool operator()(const char *a, const char *b) const { constexpr bool operator()(const std::unique_ptr<T> &lhs, const std::unique_ptr<T> &rhs) const {
return PathCompare::Compare(a, b) < 0; char lhs_path[ams::fs::EntryNameLengthMax + 1];
char rhs_path[ams::fs::EntryNameLengthMax + 1];
lhs->GetPath(lhs_path);
rhs->GetPath(rhs_path);
return Comparator::Compare(lhs_path, rhs_path) < 0;
} }
}; };
template<typename T> template<typename T>
using PathMap = std::map<const char *, std::unique_ptr<T>, PathCompare>; using ContextSet = std::set<std::unique_ptr<T>, Comparator<T>>;
private: private:
ncm::ProgramId program_id; ncm::ProgramId program_id;
BuildDirectoryContext *root; BuildDirectoryContext *root;
PathMap<BuildDirectoryContext> directories; ContextSet<BuildDirectoryContext> directories;
PathMap<BuildFileContext> files; ContextSet<BuildFileContext> files;
size_t num_dirs; size_t num_dirs;
size_t num_files; size_t num_files;
size_t dir_table_size; size_t dir_table_size;