From 9088f285d898118d5a9f4bb83ee165af06f5d9eb Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 14 Aug 2020 21:36:15 +0800 Subject: [PATCH] Add support for CompressedRomFs --- deps/libnintendo-hac | 2 +- src/CompressedArchiveIFile.cpp | 193 +++++++++++++++++++++++++++++++++ src/CompressedArchiveIFile.h | 51 +++++++++ src/RomfsProcess.cpp | 47 ++++++++ 4 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 src/CompressedArchiveIFile.cpp create mode 100644 src/CompressedArchiveIFile.h diff --git a/deps/libnintendo-hac b/deps/libnintendo-hac index b1b57ad..5dd0961 160000 --- a/deps/libnintendo-hac +++ b/deps/libnintendo-hac @@ -1 +1 @@ -Subproject commit b1b57ad02653c08638dcacbf6566a47ae366b4e1 +Subproject commit 5dd09615784de624ee8c14032869d30170b58fec diff --git a/src/CompressedArchiveIFile.cpp b/src/CompressedArchiveIFile.cpp new file mode 100644 index 0000000..02f8982 --- /dev/null +++ b/src/CompressedArchiveIFile.cpp @@ -0,0 +1,193 @@ +#include "CompressedArchiveIFile.h" +#include + +#include + +CompressedArchiveIFile::CompressedArchiveIFile(const fnd::SharedPtr& base_file, size_t compression_meta_offset) : + mFile(base_file), + mCompEntries(), + mLogicalFileSize(0), + mCacheCapacity(nn::hac::compression::kRomfsBlockSize), + mCurrentCacheDataSize(0), + mCache(std::shared_ptr(new byte_t[mCacheCapacity])), + mScratch(std::shared_ptr(new byte_t[mCacheCapacity])) +{ + // determine and check the compression metadata size + size_t compression_meta_size = (*mFile)->size() - compression_meta_offset; + if (compression_meta_size % sizeof(nn::hac::sCompressionEntry)) + { + fnd::Exception(kModuleName, "Invalid compression meta size"); + } + + // import raw metadata + std::shared_ptr entries_raw = std::shared_ptr(new byte_t[compression_meta_size]); + (*mFile)->read(entries_raw.get(), compression_meta_offset, compression_meta_size); + + // process metadata entries + nn::hac::sCompressionEntry* entries = (nn::hac::sCompressionEntry*)entries_raw.get(); + for (size_t idx = 0, num = compression_meta_size / sizeof(nn::hac::sCompressionEntry); idx < num; idx++) + { + if (idx == 0) + { + if (entries[idx].physical_offset.get() != 0x0) + throw fnd::Exception(kModuleName, "Entry 0 had a non-zero physical offset"); + if (entries[idx].virtual_offset.get() != 0x0) + throw fnd::Exception(kModuleName, "Entry 0 had a non-zero virtual offset"); + } + else + { + if (entries[idx].physical_offset.get() != align(entries[idx - 1].physical_offset.get() + entries[idx - 1].physical_size.get(), nn::hac::compression::kRomfsBlockAlign)) + throw fnd::Exception(kModuleName, "Entry was not physically aligned with previous entry"); + if (entries[idx].virtual_offset.get() <= entries[idx - 1].virtual_offset.get()) + throw fnd::Exception(kModuleName, "Entry was not virtually aligned with previous entry"); + + // set previous entry virtual_size = this->virtual_offset - prev->virtual_offset; + mCompEntries[mCompEntries.size() - 1].virtual_size = entries[idx].virtual_offset.get() - mCompEntries[mCompEntries.size() - 1].virtual_offset; + } + + if (entries[idx].physical_size.get() > nn::hac::compression::kRomfsBlockSize) + throw fnd::Exception(kModuleName, "Entry physical size was too large"); + + switch ((nn::hac::compression::CompressionType)entries[idx].compression_type) + { + case (nn::hac::compression::CompressionType::None): + case (nn::hac::compression::CompressionType::Lz4): + break; + default: + throw fnd::Exception(kModuleName, "Unsupported CompressionType"); + } + + mCompEntries.push_back({(nn::hac::compression::CompressionType)entries[idx].compression_type, entries[idx].virtual_offset.get(), 0, entries[idx].physical_offset.get(), entries[idx].physical_size.get()}); + } + + // determine logical file size and final entry size + importEntryDataToCache(mCompEntries.size() - 1); + mCompEntries[mCurrentEntryIndex].virtual_size = mCurrentCacheDataSize; + mLogicalFileSize = mCompEntries[mCurrentEntryIndex].virtual_offset + mCompEntries[mCurrentEntryIndex].virtual_size; + + /* + for (auto itr = mCompEntries.begin(); itr != mCompEntries.end(); itr++) + { + std::cout << "entry " << std::endl; + std::cout << " type: " << (uint32_t)itr->compression_type << std::endl; + std::cout << " phys_addr: 0x" << std::hex << itr->physical_offset << std::endl; + std::cout << " phys_size: 0x" << std::hex << itr->physical_size << std::endl; + std::cout << " virt_addr: 0x" << std::hex << itr->virtual_offset << std::endl; + std::cout << " virt_size: 0x" << std::hex << itr->virtual_size << std::endl; + } + + std::cout << "logical size: 0x" << std::hex << mLogicalFileSize << std::endl; + */ +} + +size_t CompressedArchiveIFile::size() +{ + return mLogicalFileSize; +} + +void CompressedArchiveIFile::seek(size_t offset) +{ + mLogicalOffset = std::min(offset, mLogicalFileSize); +} + +void CompressedArchiveIFile::read(byte_t* out, size_t len) +{ + // limit len to the end of the logical file + len = std::min(len, mLogicalFileSize - mLogicalOffset); + + for (size_t pos = 0, entry_index = getEntryIndexForLogicalOffset(mLogicalOffset); pos < len; entry_index++) + { + importEntryDataToCache(entry_index); + + // write padding if required + if (mCompEntries[entry_index].virtual_size > mCurrentCacheDataSize) + { + memset(mCache.get() + mCurrentCacheDataSize, 0, mCompEntries[entry_index].virtual_size - mCurrentCacheDataSize); + } + + // determine + size_t read_offset = mLogicalOffset - (size_t)mCompEntries[entry_index].virtual_offset; + size_t read_size = std::min(len, (size_t)mCompEntries[entry_index].virtual_size - read_offset); + + memcpy(out + pos, mCache.get() + read_offset, read_size); + + pos += read_size; + mLogicalOffset += read_size; + } +} + +void CompressedArchiveIFile::read(byte_t* out, size_t offset, size_t len) +{ + seek(offset); + read(out, len); +} + +void CompressedArchiveIFile::write(const byte_t* out, size_t len) +{ + throw fnd::Exception(kModuleName, "write() not supported"); +} + +void CompressedArchiveIFile::write(const byte_t* out, size_t offset, size_t len) +{ + throw fnd::Exception(kModuleName, "write() not supported"); +} + +void CompressedArchiveIFile::importEntryDataToCache(size_t entry_index) +{ + // return if entry already imported + if (mCurrentEntryIndex == entry_index && mCurrentCacheDataSize != 0) + return; + + // save index + mCurrentEntryIndex = entry_index; + + // reference entry + CompressionEntry& entry = mCompEntries[mCurrentEntryIndex]; + + if (entry.compression_type == nn::hac::compression::CompressionType::None) + { + (*mFile)->read(mCache.get(), entry.physical_offset, entry.physical_size); + mCurrentCacheDataSize = entry.physical_size; + } + else if (entry.compression_type == nn::hac::compression::CompressionType::Lz4) + { + (*mFile)->read(mScratch.get(), entry.physical_offset, entry.physical_size); + + mCurrentCacheDataSize = 0; + fnd::lz4::decompressData(mScratch.get(), entry.physical_size, mCache.get(), mCacheCapacity, mCurrentCacheDataSize); + + if (mCurrentCacheDataSize == 0) + { + throw fnd::Exception(kModuleName, "Decompression of final block failed"); + } + } +} + +size_t CompressedArchiveIFile::getEntryIndexForLogicalOffset(size_t logical_offset) +{ + // rule out bad offset + if (logical_offset > mLogicalFileSize) + throw fnd::Exception(kModuleName, "illegal logical offset"); + + size_t entry_index = 0; + + // try the current comp entry + if (mCompEntries[mCurrentEntryIndex].virtual_offset <= logical_offset && \ + mCompEntries[mCurrentEntryIndex].virtual_offset + mCompEntries[mCurrentEntryIndex].virtual_size >= logical_offset) + { + entry_index = mCurrentEntryIndex; + } + else + { + for (size_t index = 0; index < mCompEntries.size(); index++) + { + if (mCompEntries[index].virtual_offset <= logical_offset && \ + mCompEntries[index].virtual_offset + mCompEntries[index].virtual_size >= logical_offset) + { + entry_index = index; + } + } + } + + return entry_index; +} \ No newline at end of file diff --git a/src/CompressedArchiveIFile.h b/src/CompressedArchiveIFile.h new file mode 100644 index 0000000..795bb49 --- /dev/null +++ b/src/CompressedArchiveIFile.h @@ -0,0 +1,51 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +class CompressedArchiveIFile : public fnd::IFile +{ +public: + CompressedArchiveIFile(const fnd::SharedPtr& file, size_t compression_meta_offset); + + size_t size(); + void seek(size_t offset); + void read(byte_t* out, size_t len); + void read(byte_t* out, size_t offset, size_t len); + void write(const byte_t* out, size_t len); + void write(const byte_t* out, size_t offset, size_t len); +private: + const std::string kModuleName = "CompressedArchiveIFile"; + std::stringstream mErrorSs; + + struct CompressionEntry + { + nn::hac::compression::CompressionType compression_type; + uint64_t virtual_offset; + uint32_t virtual_size; + uint64_t physical_offset; + uint32_t physical_size; + }; + + // raw data + fnd::SharedPtr mFile; + + // compression metadata + std::vector mCompEntries; + size_t mLogicalFileSize; + size_t mLogicalOffset; + + // cached decompressed entry + size_t mCacheCapacity; // capacity + size_t mCurrentEntryIndex; // index of entry currently associated with the cache + uint32_t mCurrentCacheDataSize; // size of data currently in cache + std::shared_ptr mCache; // where decompressed data resides + std::shared_ptr mScratch; // same size as cache, but is used for storing data pre-compression + + // this will import entry to cache + void importEntryDataToCache(size_t entry_index); + size_t getEntryIndexForLogicalOffset(size_t logical_offset); +}; \ No newline at end of file diff --git a/src/RomfsProcess.cpp b/src/RomfsProcess.cpp index aba23fe..c945725 100644 --- a/src/RomfsProcess.cpp +++ b/src/RomfsProcess.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "CompressedArchiveIFile.h" #include "RomfsProcess.h" RomfsProcess::RomfsProcess() : @@ -262,6 +263,52 @@ void RomfsProcess::resolveRomfs() throw fnd::Exception(kModuleName, "Invalid ROMFS Header"); } + // check for romfs compression + size_t physical_size = (*mFile)->size(); + size_t logical_size = mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].offset.get() + mHdr.sections[nn::hac::romfs::FILE_NODE_TABLE].size.get(); + + // if logical size is greater than the physical size, check for compression meta footer + if (logical_size > physical_size) + { + // initial and final entries + nn::hac::sCompressionEntry entry[2]; + + (*mFile)->read((byte_t*)&entry[1], physical_size - sizeof(nn::hac::sCompressionEntry), sizeof(nn::hac::sCompressionEntry)); + + // the final compression entry should be for the romfs footer, for which the logical offset is detailed in the romfs header + // the compression is always enabled for non-header compression entries + if (entry[1].virtual_offset.get() != mHdr.sections[nn::hac::romfs::DIR_HASHMAP_TABLE].offset.get() || \ + entry[1].compression_type != (byte_t)nn::hac::compression::CompressionType::Lz4) + { + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)"); + } + + // the first compression entry follows the physical placement of the final data chunk (specified in the final compression entry) + size_t first_entry_offset = align(entry[1].physical_offset.get() + entry[1].physical_size.get(), nn::hac::compression::kRomfsBlockAlign); + + // quick check to make sure the offset at least before the last entry offset + if (first_entry_offset >= (physical_size - sizeof(nn::hac::sCompressionEntry))) + { + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad final compression entry)"); + } + + // read the first compression entry + (*mFile)->read((byte_t*)&entry[0], first_entry_offset, sizeof(nn::hac::sCompressionEntry)); + + // validate first compression entry + // this should be the same for all compressed romfs + if (entry[0].virtual_offset.get() != 0x0 || \ + entry[0].physical_offset.get() != 0x0 || \ + entry[0].physical_size.get() != 0x200 || \ + entry[0].compression_type != (byte_t)nn::hac::compression::CompressionType::None) + { + throw fnd::Exception(kModuleName, "RomFs appears corrupted (bad first compression entry)"); + } + + // wrap mFile in a class to transparantly decompress the image. + mFile = new CompressedArchiveIFile(mFile, first_entry_offset); + } + // read directory nodes mDirNodes.alloc(mHdr.sections[nn::hac::romfs::DIR_NODE_TABLE].size.get()); (*mFile)->read(mDirNodes.data(), mHdr.sections[nn::hac::romfs::DIR_NODE_TABLE].offset.get(), mDirNodes.size());