nold/Atmosphere
1
0
Fork 0
mirror of https://github.com/Atmosphere-NX/Atmosphere synced 2024-12-31 17:31:15 +00:00
Code Issues Projects Releases Packages Wiki Activity

fs.mitm: conserve memory when building romfs paths

Browse source
This commit is contained in:
Michael Scire 2019-12-30 02:51:32 -08:00
parent a2d2b1b346
commit 2ae298de24
2 changed files with 108 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
stratosphere/ams_mitm/source/fs_mitm/fsmitm_romfs.cpp
View file

@ -111,44 +111,57 @@ namespace ams::mitm::fs {
}
}
os::Mutex g_fs_romfs_path_lock;
char g_fs_romfs_path_buffer[fs::EntryNameLengthMax + 1];
__attribute__((noinline)) void OpenFileSystemRomfsDirectory(FsDir *out, ncm::ProgramId program_id, BuildDirectoryContext *parent, fs::OpenDirectoryMode mode, FsFileSystem *fs) {
std::scoped_lock lk(g_fs_romfs_path_lock);
parent->GetPath(g_fs_romfs_path_buffer);
R_ASSERT(mitm::fs::OpenAtmosphereRomfsDirectory(out, program_id, g_fs_romfs_path_buffer, mode, fs));
}
}
Builder::Builder(ncm::ProgramId pr_id) : program_id(pr_id), num_dirs(0), num_files(0), dir_table_size(0), file_table_size(0), dir_hash_table_size(0), file_hash_table_size(0), file_partition_size(0) {
auto res = this->directories.emplace("", std::make_unique<BuildDirectoryContext>(BuildDirectoryContext::RootTag{}));
auto res = this->directories.emplace(std::make_unique<BuildDirectoryContext>(BuildDirectoryContext::RootTag{}));
AMS_ASSERT(res.second);
this->root = res.first->second.get();
this->root = res.first->get();
this->num_dirs = 1;
this->dir_table_size = 0x18;
}
void Builder::AddDirectory(BuildDirectoryContext **out, BuildDirectoryContext *parent_ctx, std::unique_ptr<BuildDirectoryContext> child_ctx) {
/* Set parent context member. */
child_ctx->parent = parent_ctx;
/* Check if the directory already exists. */
auto existing = this->directories.find(child_ctx->path.get());
auto existing = this->directories.find(child_ctx);
if (existing != this->directories.end()) {
*out = existing->second.get();
*out = existing->get();
return;
}
/* Add a new directory. */
this->num_dirs++;
this->dir_table_size += sizeof(DirectoryEntry) + util::AlignUp(child_ctx->path_len - child_ctx->cur_path_ofs, 4);
child_ctx->parent = parent_ctx;
this->dir_table_size += sizeof(DirectoryEntry) + util::AlignUp(child_ctx->path_len, 4);
*out = child_ctx.get();
this->directories.emplace(child_ctx->path.get(), std::move(child_ctx));
this->directories.emplace(std::move(child_ctx));
}
void Builder::AddFile(BuildDirectoryContext *parent_ctx, std::unique_ptr<BuildFileContext> file_ctx) {
/* Set parent context member. */
file_ctx->parent = parent_ctx;
/* Check if the file already exists. */
if (this->files.find(file_ctx->path.get()) != this->files.end()) {
if (this->files.find(file_ctx) != this->files.end()) {
return;
}
/* Add a new file. */
this->num_files++;
this->file_table_size += sizeof(FileEntry) + util::AlignUp(file_ctx->path_len - file_ctx->cur_path_ofs, 4);
file_ctx->parent = parent_ctx;
this->files.emplace(file_ctx->path.get(), std::move(file_ctx));
this->file_table_size += sizeof(FileEntry) + util::AlignUp(file_ctx->path_len, 4);
this->files.emplace(std::move(file_ctx));
}
void Builder::VisitDirectory(FsFileSystem *fs, BuildDirectoryContext *parent) {
@ -157,7 +170,7 @@ namespace ams::mitm::fs {
/* Get number of child directories. */
s64 num_child_dirs = 0;
{
R_ASSERT(mitm::fs::OpenAtmosphereRomfsDirectory(&dir, this->program_id, parent->path.get(), OpenDirectoryMode_Directory, fs));
OpenFileSystemRomfsDirectory(&dir, this->program_id, parent, OpenDirectoryMode_Directory, fs);
ON_SCOPE_EXIT { fsDirClose(&dir); };
R_ASSERT(fsDirGetEntryCount(&dir, &num_child_dirs));
}
@ -169,8 +182,8 @@ namespace ams::mitm::fs {
AMS_ASSERT(child_dirs != nullptr);
s64 cur_child_dir_ind = 0;
R_ASSERT(mitm::fs::OpenAtmosphereRomfsDirectory(&dir, this->program_id, parent->path.get(), OpenDirectoryMode_All, fs));
{
OpenFileSystemRomfsDirectory(&dir, this->program_id, parent, OpenDirectoryMode_All, fs);
ON_SCOPE_EXIT { fsDirClose(&dir); };
s64 read_entries = 0;
@ -183,12 +196,12 @@ namespace ams::mitm::fs {
AMS_ASSERT(this->dir_entry.type == FsDirEntryType_Dir || this->dir_entry.type == FsDirEntryType_File);
if (this->dir_entry.type == FsDirEntryType_Dir) {
BuildDirectoryContext *real_child = nullptr;
this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(parent->path.get(), parent->path_len, this->dir_entry.name, strlen(this->dir_entry.name)));
this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(this->dir_entry.name, strlen(this->dir_entry.name)));
AMS_ASSERT(real_child != nullptr);
child_dirs[cur_child_dir_ind++] = real_child;
AMS_ASSERT(cur_child_dir_ind <= num_child_dirs);
} else /* if (this->dir_entry.type == FsDirEntryType_File) */ {
this->AddFile(parent, std::make_unique<BuildFileContext>(parent->path.get(), parent->path_len, this->dir_entry.name, strlen(this->dir_entry.name), this->dir_entry.file_size, 0, this->cur_source_type));
this->AddFile(parent, std::make_unique<BuildFileContext>(this->dir_entry.name, strlen(this->dir_entry.name), this->dir_entry.file_size, 0, this->cur_source_type));
}
}
}
@ -206,7 +219,7 @@ namespace ams::mitm::fs {
if (parent_entry->file != EmptyEntry) {
const FileEntry *cur_file = GetFileEntry(file_table, parent_entry->file);
while (true) {
this->AddFile(parent, std::make_unique<BuildFileContext>(parent->path.get(), parent->path_len, cur_file->name, cur_file->name_size, cur_file->size, cur_file->offset, this->cur_source_type));
this->AddFile(parent, std::make_unique<BuildFileContext>(cur_file->name, cur_file->name_size, cur_file->size, cur_file->offset, this->cur_source_type));
if (cur_file->sibling == EmptyEntry) {
break;
}
@ -218,7 +231,7 @@ namespace ams::mitm::fs {
u32 cur_child_offset = parent_entry->child;
while (true) {
BuildDirectoryContext *real_child = nullptr;
this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(parent->path.get(), parent->path_len, cur_child->name, cur_child->name_size));
this->AddDirectory(&real_child, parent, std::make_unique<BuildDirectoryContext>(cur_child->name, cur_child->name_size));
AMS_ASSERT(real_child != nullptr);
this->VisitDirectory(real_child, cur_child_offset, dir_table, dir_table_size, file_table, file_table_size);
@ -308,7 +321,7 @@ namespace ams::mitm::fs {
BuildFileContext *cur_file = nullptr;
BuildFileContext *prev_file = nullptr;
for (const auto &it : this->files) {
cur_file = it.second.get();
cur_file = it.get();
/* By default, pad to 0x10 alignment. */
this->file_partition_size = util::AlignUp(this->file_partition_size, 0x10);
@ -327,14 +340,14 @@ namespace ams::mitm::fs {
cur_file->offset = this->file_partition_size;
this->file_partition_size += cur_file->size;
cur_file->entry_offset = entry_offset;
entry_offset += sizeof(FileEntry) + util::AlignUp(cur_file->path_len - cur_file->cur_path_ofs, 4);
entry_offset += sizeof(FileEntry) + util::AlignUp(cur_file->path_len, 4);
/* Save current file as prev for next iteration. */
prev_file = cur_file;
}
/* Assign deferred parent/sibling ownership. */
for (auto it = this->files.rbegin(); it != this->files.rend(); it++) {
cur_file = it->second.get();
cur_file = it->get();
cur_file->sibling = cur_file->parent->file;
cur_file->parent->file = cur_file;
}
@ -345,13 +358,13 @@ namespace ams::mitm::fs {
u32 entry_offset = 0;
BuildDirectoryContext *cur_dir = nullptr;
for (const auto &it : this->directories) {
cur_dir = it.second.get();
cur_dir = it.get();
cur_dir->entry_offset = entry_offset;
entry_offset += sizeof(DirectoryEntry) + util::AlignUp(cur_dir->path_len - cur_dir->cur_path_ofs, 4);
entry_offset += sizeof(DirectoryEntry) + util::AlignUp(cur_dir->path_len, 4);
}
/* Assign deferred parent/sibling ownership. */
for (auto it = this->directories.rbegin(); it != this->directories.rend(); it++) {
cur_dir = it->second.get();
cur_dir = it->get();
if (cur_dir == this->root) {
continue;
}
@ -362,7 +375,7 @@ namespace ams::mitm::fs {
/* Populate file tables. */
for (const auto &it : this->files) {
BuildFileContext *cur_file = it.second.get();
BuildFileContext *cur_file = it.get();
FileEntry *cur_entry = GetFileEntry(file_table, cur_file->entry_offset);
/* Set entry fields. */
@ -372,15 +385,15 @@ namespace ams::mitm::fs {
cur_entry->size = cur_file->size;
/* Insert into hash table. */
const u32 name_size = cur_file->path_len - cur_file->cur_path_ofs;
const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_file->path.get() + cur_file->cur_path_ofs, 0, name_size) % num_file_hash_table_entries;
const u32 name_size = cur_file->path_len;
const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_file->path.get(), 0, name_size) % num_file_hash_table_entries;
cur_entry->hash = file_hash_table[hash_ind];
file_hash_table[hash_ind] = cur_file->entry_offset;
/* Set name. */
cur_entry->name_size = name_size;
if (name_size) {
std::memcpy(cur_entry->name, cur_file->path.get() + cur_file->cur_path_ofs, name_size);
std::memcpy(cur_entry->name, cur_file->path.get(), name_size);
for (size_t i = name_size; i < util::AlignUp(name_size, 4); i++) {
cur_entry->name[i] = 0;
}
@ -402,7 +415,9 @@ namespace ams::mitm::fs {
break;
case DataSourceType::LooseSdFile:
{
out_infos->emplace_back(cur_file->offset + FilePartitionOffset, cur_file->size, cur_file->source_type, cur_file->path.release());
char *new_path = new char[cur_file->GetPathLength() + 1];
cur_file->GetPath(new_path);
out_infos->emplace_back(cur_file->offset + FilePartitionOffset, cur_file->size, cur_file->source_type, new_path);
}
break;
AMS_UNREACHABLE_DEFAULT_CASE();
@ -411,7 +426,7 @@ namespace ams::mitm::fs {
/* Populate directory tables. */
for (const auto &it : this->directories) {
BuildDirectoryContext *cur_dir = it.second.get();
BuildDirectoryContext *cur_dir = it.get();
DirectoryEntry *cur_entry = GetDirectoryEntry(dir_table, cur_dir->entry_offset);
/* Set entry fields. */
@ -421,15 +436,15 @@ namespace ams::mitm::fs {
cur_entry->file = (cur_dir->file == nullptr) ? EmptyEntry : cur_dir->file->entry_offset;
/* Insert into hash table. */
const u32 name_size = cur_dir->path_len - cur_dir->cur_path_ofs;
const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_dir->path.get() + cur_dir->cur_path_ofs, 0, name_size) % num_dir_hash_table_entries;
const u32 name_size = cur_dir->path_len;
const size_t hash_ind = CalculatePathHash(cur_entry->parent, cur_dir->path.get(), 0, name_size) % num_dir_hash_table_entries;
cur_entry->hash = dir_hash_table[hash_ind];
dir_hash_table[hash_ind] = cur_dir->entry_offset;
/* Set name. */
cur_entry->name_size = name_size;
if (name_size) {
std::memcpy(cur_entry->name, cur_dir->path.get() + cur_dir->cur_path_ofs, name_size);
std::memcpy(cur_entry->name, cur_dir->path.get(), name_size);
for (size_t i = name_size; i < util::AlignUp(name_size, 4); i++) {
cur_entry->name[i] = 0;
}

83
stratosphere/ams_mitm/source/fs_mitm/fsmitm_romfs.hpp
View file

@ -118,24 +118,41 @@ namespace ams::mitm::fs::romfs {
BuildDirectoryContext *child;
BuildDirectoryContext *sibling;
BuildFileContext *file;
u32 cur_path_ofs;
u32 path_len;
u32 entry_offset;
struct RootTag{};
BuildDirectoryContext(RootTag) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), cur_path_ofs(0), path_len(0), entry_offset(0) {
BuildDirectoryContext(RootTag) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), path_len(0), entry_offset(0) {
this->path = std::make_unique<char[]>(1);
}
BuildDirectoryContext(const char *parent_path, size_t parent_path_len, const char *entry_name, size_t entry_name_len) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), entry_offset(0) {
this->cur_path_ofs = parent_path_len + 1;
this->path_len = this->cur_path_ofs + entry_name_len;
BuildDirectoryContext(const char *entry_name, size_t entry_name_len) : parent(nullptr), child(nullptr), sibling(nullptr), file(nullptr), entry_offset(0) {
this->path_len = entry_name_len;
this->path = std::unique_ptr<char[]>(new char[this->path_len + 1]);
std::memcpy(this->path.get(), parent_path, parent_path_len);
this->path[parent_path_len] = '/';
std::memcpy(this->path.get() + parent_path_len + 1, entry_name, entry_name_len);
this->path[this->path_len] = 0;
std::memcpy(this->path.get(), entry_name, entry_name_len);
this->path[this->path_len] = '\x00';
}
size_t GetPathLength() const {
if (this->parent == nullptr) {
return 0;
}
return this->parent->GetPathLength() + 1 + this->path_len;
}
size_t GetPath(char *dst) const {
if (this->parent == nullptr) {
dst[0] = '\x00';
return 0;
}
const size_t parent_len = this->parent->GetPath(dst);
dst[parent_len] = '/';
std::memcpy(dst + parent_len + 1, this->path.get(), this->path_len);
dst[parent_len + 1 + this->path_len] = '\x00';
return parent_len + 1 + this->path_len;
}
};
@ -149,27 +166,45 @@ namespace ams::mitm::fs::romfs {
s64 offset;
s64 size;
s64 orig_offset;
u32 cur_path_ofs;
u32 path_len;
u32 entry_offset;
DataSourceType source_type;
BuildFileContext(const char *parent_path, size_t parent_path_len, const char *entry_name, size_t entry_name_len, s64 sz, s64 o_o, DataSourceType type) : parent(nullptr), sibling(nullptr), offset(0), size(sz), orig_offset(o_o), entry_offset(0), source_type(type) {
this->cur_path_ofs = parent_path_len + 1;
this->path_len = this->cur_path_ofs + entry_name_len;
BuildFileContext(const char *entry_name, size_t entry_name_len, s64 sz, s64 o_o, DataSourceType type) : parent(nullptr), sibling(nullptr), offset(0), size(sz), orig_offset(o_o), entry_offset(0), source_type(type) {
this->path_len = entry_name_len;
this->path = std::unique_ptr<char[]>(new char[this->path_len + 1]);
std::memcpy(this->path.get(), parent_path, parent_path_len);
this->path[parent_path_len] = '/';
std::memcpy(this->path.get() + parent_path_len + 1, entry_name, entry_name_len);
std::memcpy(this->path.get(), entry_name, entry_name_len);
this->path[this->path_len] = 0;
}
size_t GetPathLength() const {
if (this->parent == nullptr) {
return 0;
}
return this->parent->GetPathLength() + 1 + this->path_len;
}
size_t GetPath(char *dst) const {
if (this->parent == nullptr) {
dst[0] = '\x00';
return 0;
}
const size_t parent_len = this->parent->GetPath(dst);
dst[parent_len] = '/';
std::memcpy(dst + parent_len + 1, this->path.get(), this->path_len);
dst[parent_len + 1 + this->path_len] = '\x00';
return parent_len + 1 + this->path_len;
}
};
struct Builder {
NON_COPYABLE(Builder);
NON_MOVEABLE(Builder);
private:
struct PathCompare {
template<typename T>
struct Comparator {
static constexpr inline int Compare(const char *a, const char *b) {
unsigned char c1{}, c2{};
while ((c1 = *a++) == (c2 = *b++)) {
@ -180,18 +215,22 @@ namespace ams::mitm::fs::romfs {
return (c1 - c2);
}
constexpr bool operator()(const char *a, const char *b) const {
return PathCompare::Compare(a, b) < 0;
constexpr bool operator()(const std::unique_ptr<T> &lhs, const std::unique_ptr<T> &rhs) const {
char lhs_path[ams::fs::EntryNameLengthMax + 1];
char rhs_path[ams::fs::EntryNameLengthMax + 1];
lhs->GetPath(lhs_path);
rhs->GetPath(rhs_path);
return Comparator::Compare(lhs_path, rhs_path) < 0;
}
};
template<typename T>
using PathMap = std::map<const char *, std::unique_ptr<T>, PathCompare>;
using ContextSet = std::set<std::unique_ptr<T>, Comparator<T>>;
private:
ncm::ProgramId program_id;
BuildDirectoryContext *root;
PathMap<BuildDirectoryContext> directories;
PathMap<BuildFileContext> files;
ContextSet<BuildDirectoryContext> directories;
ContextSet<BuildFileContext> files;
size_t num_dirs;
size_t num_files;
size_t dir_table_size;