diff --git a/programs/nstool/source/HashTreeWrappedIFile.cpp b/programs/nstool/source/HashTreeWrappedIFile.cpp index dd54449..2b3543a 100644 --- a/programs/nstool/source/HashTreeWrappedIFile.cpp +++ b/programs/nstool/source/HashTreeWrappedIFile.cpp @@ -164,7 +164,8 @@ void HashTreeWrappedIFile::initialiseDataLayer(const HashTreeMeta& hdr) mDataBlockSize = hdr.getDataLayer().block_size; // allocate scratchpad - mScratch.alloc(mDataBlockSize * 0x10); + //mScratch.alloc(mDataBlockSize * 0x10); + mScratch.alloc(align(kFileExportBlockSize, mDataBlockSize)); } void HashTreeWrappedIFile::readData(size_t block_offset, size_t block_num) diff --git a/programs/nstool/source/HashTreeWrappedIFile.h b/programs/nstool/source/HashTreeWrappedIFile.h index 5e92014..456985b 100644 --- a/programs/nstool/source/HashTreeWrappedIFile.h +++ b/programs/nstool/source/HashTreeWrappedIFile.h @@ -21,6 +21,7 @@ public: void write(const byte_t* out, size_t offset, size_t len); private: const std::string kModuleName = "HashTreeWrappedIFile"; + static const size_t kFileExportBlockSize = 0x1000000; std::stringstream mErrorSs; bool mOwnIFile; diff --git a/programs/nstool/source/PfsProcess.cpp b/programs/nstool/source/PfsProcess.cpp index c82d3c5..d7ffe3d 100644 --- a/programs/nstool/source/PfsProcess.cpp +++ b/programs/nstool/source/PfsProcess.cpp @@ -3,104 +3,6 @@ #include "OffsetAdjustedIFile.h" #include "PfsProcess.h" -void PfsProcess::displayHeader() -{ - printf("[PartitionFS]\n"); - printf(" Type: %s\n", mPfs.getFsType() == mPfs.TYPE_PFS0? "PFS0" : "HFS0"); - printf(" FileNum: %" PRId64 "\n", mPfs.getFileList().getSize()); - if (mMountName.empty() == false) - printf(" MountPoint: %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : ""); -} - -void PfsProcess::displayFs() -{ - for (size_t i = 0; i < mPfs.getFileList().getSize(); i++) - { - printf(" %s", mPfs.getFileList()[i].name.c_str()); - if (mCliOutputType >= OUTPUT_VERBOSE) - { - if (mPfs.getFsType() == mPfs.TYPE_PFS0) - printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size); - else - printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ", hash_protected_size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size, mPfs.getFileList()[i].hash_protected_size); - } - else - { - printf("\n"); - } - - } -} - -size_t PfsProcess::determineHeaderSize(const nx::sPfsHeader* hdr) -{ - size_t fileEntrySize = 0; - if (std::string(hdr->signature, 4) == nx::pfs::kPfsSig) - fileEntrySize = sizeof(nx::sPfsFile); - else - fileEntrySize = sizeof(nx::sHashedPfsFile); - - return sizeof(nx::sPfsHeader) + hdr->file_num.get() * fileEntrySize + hdr->name_table_size.get(); -} - -bool PfsProcess::validateHeaderMagic(const nx::sPfsHeader* hdr) -{ - return std::string(hdr->signature, 4) == nx::pfs::kPfsSig || std::string(hdr->signature, 4) == nx::pfs::kHashedPfsSig; -} - -void PfsProcess::validateHfs() -{ - fnd::MemoryBlob scratch; - crypto::sha::sSha256Hash hash; - const fnd::List& file = mPfs.getFileList(); - for (size_t i = 0; i < file.getSize(); i++) - { - scratch.alloc(file[i].hash_protected_size); - mReader->read(scratch.getBytes(), file[i].offset, file[i].hash_protected_size); - crypto::sha::Sha256(scratch.getBytes(), scratch.getSize(), hash.bytes); - if (hash != file[i].hash) - { - if (mCliOutputType >= OUTPUT_MINIMAL) - printf("[WARNING] HFS0 %s%s%s: FAIL (bad hash)\n", !mMountName.empty()? mMountName.c_str() : "", !mMountName.empty()? "/" : "", file[i].name.c_str()); - - } - } -} - -void PfsProcess::extractFs() -{ - // allocate scratch memory - fnd::MemoryBlob scratch; - scratch.alloc(kFileExportBlockSize); - - // make extract dir - fnd::io::makeDirectory(mExtractPath); - - fnd::SimpleFile outFile; - const fnd::List& file = mPfs.getFileList(); - - std::string file_path; - for (size_t i = 0; i < file.getSize(); i++) - { - file_path.clear(); - fnd::io::appendToPath(file_path, mExtractPath); - fnd::io::appendToPath(file_path, file[i].name); - outFile.open(file_path, outFile.Create); - mReader->seek(file[i].offset); - for (size_t j = 0; j < (file[i].size / kFileExportBlockSize); j++) - { - mReader->read(scratch.getBytes(), kFileExportBlockSize); - outFile.write(scratch.getBytes(), kFileExportBlockSize); - } - if (file[i].size % kFileExportBlockSize) - { - mReader->read(scratch.getBytes(), file[i].size % kFileExportBlockSize); - outFile.write(scratch.getBytes(), file[i].size % kFileExportBlockSize); - } - outFile.close(); - } -} - PfsProcess::PfsProcess() : mReader(nullptr), mCliOutputType(OUTPUT_NORMAL), @@ -189,3 +91,106 @@ const nx::PfsHeader& PfsProcess::getPfsHeader() const { return mPfs; } + +void PfsProcess::displayHeader() +{ + printf("[PartitionFS]\n"); + printf(" Type: %s\n", mPfs.getFsType() == mPfs.TYPE_PFS0? "PFS0" : "HFS0"); + printf(" FileNum: %" PRId64 "\n", mPfs.getFileList().getSize()); + if (mMountName.empty() == false) + printf(" MountPoint: %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : ""); +} + +void PfsProcess::displayFs() +{ + for (size_t i = 0; i < mPfs.getFileList().getSize(); i++) + { + printf(" %s", mPfs.getFileList()[i].name.c_str()); + if (mCliOutputType >= OUTPUT_VERBOSE) + { + if (mPfs.getFsType() == mPfs.TYPE_PFS0) + printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size); + else + printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ", hash_protected_size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size, mPfs.getFileList()[i].hash_protected_size); + } + else + { + printf("\n"); + } + + } +} + +size_t PfsProcess::determineHeaderSize(const nx::sPfsHeader* hdr) +{ + size_t fileEntrySize = 0; + if (std::string(hdr->signature, 4) == nx::pfs::kPfsSig) + fileEntrySize = sizeof(nx::sPfsFile); + else + fileEntrySize = sizeof(nx::sHashedPfsFile); + + return sizeof(nx::sPfsHeader) + hdr->file_num.get() * fileEntrySize + hdr->name_table_size.get(); +} + +bool PfsProcess::validateHeaderMagic(const nx::sPfsHeader* hdr) +{ + return std::string(hdr->signature, 4) == nx::pfs::kPfsSig || std::string(hdr->signature, 4) == nx::pfs::kHashedPfsSig; +} + +void PfsProcess::validateHfs() +{ + // allocate when validate is invoked + mFileExtractBlock.alloc(kFileExportBlockSize); + + crypto::sha::sSha256Hash hash; + const fnd::List& file = mPfs.getFileList(); + for (size_t i = 0; i < file.getSize(); i++) + { + mFileExtractBlock.alloc(file[i].hash_protected_size); + mReader->read(mFileExtractBlock.getBytes(), file[i].offset, file[i].hash_protected_size); + crypto::sha::Sha256(mFileExtractBlock.getBytes(), mFileExtractBlock.getSize(), hash.bytes); + if (hash != file[i].hash) + { + if (mCliOutputType >= OUTPUT_MINIMAL) + printf("[WARNING] HFS0 %s%s%s: FAIL (bad hash)\n", !mMountName.empty()? mMountName.c_str() : "", !mMountName.empty()? "/" : "", file[i].name.c_str()); + + } + } +} + +void PfsProcess::extractFs() +{ + // allocate only when extractDir is invoked + mFileExtractBlock.alloc(kFileExportBlockSize); + + // make extract dir + fnd::io::makeDirectory(mExtractPath); + + fnd::SimpleFile outFile; + const fnd::List& file = mPfs.getFileList(); + + std::string file_path; + for (size_t i = 0; i < file.getSize(); i++) + { + file_path.clear(); + fnd::io::appendToPath(file_path, mExtractPath); + fnd::io::appendToPath(file_path, file[i].name); + + if (mCliOutputType >= OUTPUT_VERBOSE) + printf("extract=[%s]\n", file_path.c_str()); + + outFile.open(file_path, outFile.Create); + mReader->seek(file[i].offset); + for (size_t j = 0; j < (file[i].size / kFileExportBlockSize); j++) + { + mReader->read(mFileExtractBlock.getBytes(), kFileExportBlockSize); + outFile.write(mFileExtractBlock.getBytes(), kFileExportBlockSize); + } + if (file[i].size % kFileExportBlockSize) + { + mReader->read(mFileExtractBlock.getBytes(), file[i].size % kFileExportBlockSize); + outFile.write(mFileExtractBlock.getBytes(), file[i].size % kFileExportBlockSize); + } + outFile.close(); + } +} diff --git a/programs/nstool/source/PfsProcess.h b/programs/nstool/source/PfsProcess.h index 18ad507..c2be9da 100644 --- a/programs/nstool/source/PfsProcess.h +++ b/programs/nstool/source/PfsProcess.h @@ -39,6 +39,8 @@ private: std::string mMountName; bool mListFs; + fnd::MemoryBlob mFileExtractBlock; + nx::PfsHeader mPfs; void displayHeader(); diff --git a/programs/nstool/source/RomfsProcess.cpp b/programs/nstool/source/RomfsProcess.cpp index ac647f9..ef224ee 100644 --- a/programs/nstool/source/RomfsProcess.cpp +++ b/programs/nstool/source/RomfsProcess.cpp @@ -4,6 +4,83 @@ #include "OffsetAdjustedIFile.h" #include "RomfsProcess.h" +RomfsProcess::RomfsProcess() : + mReader(nullptr), + mCliOutputType(OUTPUT_NORMAL), + mVerify(false), + mExtractPath(), + mExtract(false), + mMountName(), + mListFs(false), + mDirNum(0), + mFileNum(0) +{ + mRootDir.name.clear(); + mRootDir.dir_list.clear(); + mRootDir.file_list.clear(); +} + +RomfsProcess::~RomfsProcess() +{ + if (mReader != nullptr) + { + delete mReader; + } +} + +void RomfsProcess::process() +{ + if (mReader == nullptr) + { + throw fnd::Exception(kModuleName, "No file reader set."); + } + + resolveRomfs(); + + if (mCliOutputType >= OUTPUT_NORMAL) + displayHeader(); + if (mListFs || mCliOutputType >= OUTPUT_VERBOSE) + displayFs(); + if (mExtract) + extractFs(); +} + +void RomfsProcess::setInputFile(fnd::IFile* file, size_t offset, size_t size) +{ + mReader = new OffsetAdjustedIFile(file, offset, size); +} + +void RomfsProcess::setCliOutputMode(CliOutputType type) +{ + mCliOutputType = type; +} + +void RomfsProcess::setVerifyMode(bool verify) +{ + mVerify = verify; +} + +void RomfsProcess::setMountPointName(const std::string& mount_name) +{ + mMountName = mount_name; +} + +void RomfsProcess::setExtractPath(const std::string& path) +{ + mExtract = true; + mExtractPath = path; +} + +void RomfsProcess::setListFs(bool list_fs) +{ + mListFs = list_fs; +} + +const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const +{ + return mRootDir; +} + void RomfsProcess::printTab(size_t tab) const { for (size_t i = 0; i < tab; i++) @@ -72,8 +149,10 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir) // allocate memory for file extraction +#ifdef NSTOOL_ALLOC_UNIQUE_SCRATCH fnd::MemoryBlob scratch; scratch.alloc(kFileExportBlockSize); +#endif // extract files fnd::SimpleFile outFile; @@ -91,13 +170,13 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir) mReader->seek(dir.file_list[i].offset); for (size_t j = 0; j < (dir.file_list[i].size / kFileExportBlockSize); j++) { - mReader->read(scratch.getBytes(), kFileExportBlockSize); - outFile.write(scratch.getBytes(), kFileExportBlockSize); + mReader->read(mFileExtractBlock.getBytes(), kFileExportBlockSize); + outFile.write(mFileExtractBlock.getBytes(), kFileExportBlockSize); } if (dir.file_list[i].size % kFileExportBlockSize) { - mReader->read(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize); - outFile.write(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize); + mReader->read(mFileExtractBlock.getBytes(), dir.file_list[i].size % kFileExportBlockSize); + outFile.write(mFileExtractBlock.getBytes(), dir.file_list[i].size % kFileExportBlockSize); } outFile.close(); } @@ -111,6 +190,8 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir) void RomfsProcess::extractFs() { + // allocate only when extractDir is invoked + mFileExtractBlock.alloc(kFileExportBlockSize); extractDir(mExtractPath, mRootDir); } @@ -220,81 +301,4 @@ void RomfsProcess::resolveRomfs() mDirNum = 0; mFileNum = 0; importDirectory(0, mRootDir); -} - -RomfsProcess::RomfsProcess() : - mReader(nullptr), - mCliOutputType(OUTPUT_NORMAL), - mVerify(false), - mExtractPath(), - mExtract(false), - mMountName(), - mListFs(false), - mDirNum(0), - mFileNum(0) -{ - mRootDir.name.clear(); - mRootDir.dir_list.clear(); - mRootDir.file_list.clear(); -} - -RomfsProcess::~RomfsProcess() -{ - if (mReader != nullptr) - { - delete mReader; - } -} - -void RomfsProcess::process() -{ - if (mReader == nullptr) - { - throw fnd::Exception(kModuleName, "No file reader set."); - } - - resolveRomfs(); - - if (mCliOutputType >= OUTPUT_NORMAL) - displayHeader(); - if (mListFs || mCliOutputType >= OUTPUT_VERBOSE) - displayFs(); - if (mExtract) - extractFs(); -} - -void RomfsProcess::setInputFile(fnd::IFile* file, size_t offset, size_t size) -{ - mReader = new OffsetAdjustedIFile(file, offset, size); -} - -void RomfsProcess::setCliOutputMode(CliOutputType type) -{ - mCliOutputType = type; -} - -void RomfsProcess::setVerifyMode(bool verify) -{ - mVerify = verify; -} - -void RomfsProcess::setMountPointName(const std::string& mount_name) -{ - mMountName = mount_name; -} - -void RomfsProcess::setExtractPath(const std::string& path) -{ - mExtract = true; - mExtractPath = path; -} - -void RomfsProcess::setListFs(bool list_fs) -{ - mListFs = list_fs; -} - -const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const -{ - return mRootDir; } \ No newline at end of file diff --git a/programs/nstool/source/RomfsProcess.h b/programs/nstool/source/RomfsProcess.h index 5c2a2e0..00f43f5 100644 --- a/programs/nstool/source/RomfsProcess.h +++ b/programs/nstool/source/RomfsProcess.h @@ -107,6 +107,7 @@ public: private: const std::string kModuleName = "RomfsProcess"; static const size_t kFileExportBlockSize = 0x1000000; + //static const size_t kFileExportBlockSize = 0x1000000; fnd::IFile* mReader; CliOutputType mCliOutputType; @@ -117,6 +118,8 @@ private: std::string mMountName; bool mListFs; + fnd::MemoryBlob mFileExtractBlock; + size_t mDirNum; size_t mFileNum; nx::sRomfsHeader mHdr;