nold/nstool
1
0
Fork 0
mirror of https://github.com/jakcron/nstool synced 2024-12-25 14:11:14 +00:00
Code Issues Projects Releases Packages Wiki Activity

[nstool] Aligned file extraction block sizes

Browse source 
For optimisation, IFile wrappers and processes that all used a cache have similar or the same cache sizes.
This commit is contained in:
jakcron 2018-05-26 21:13:21 +08:00
parent 76ce22dc69
commit 634b202b5a
6 changed files with 196 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
programs/nstool/source/HashTreeWrappedIFile.cpp
View file

@ -164,7 +164,8 @@ void HashTreeWrappedIFile::initialiseDataLayer(const HashTreeMeta& hdr)
mDataBlockSize = hdr.getDataLayer().block_size;
// allocate scratchpad
mScratch.alloc(mDataBlockSize * 0x10);
//mScratch.alloc(mDataBlockSize * 0x10);
mScratch.alloc(align(kFileExportBlockSize, mDataBlockSize));
}
void HashTreeWrappedIFile::readData(size_t block_offset, size_t block_num)

1
programs/nstool/source/HashTreeWrappedIFile.h
View file

@ -21,6 +21,7 @@ public:
void write(const byte_t* out, size_t offset, size_t len);
private:
const std::string kModuleName = "HashTreeWrappedIFile";
static const size_t kFileExportBlockSize = 0x1000000;
std::stringstream mErrorSs;
bool mOwnIFile;

201
programs/nstool/source/PfsProcess.cpp
View file

@ -3,104 +3,6 @@
#include "OffsetAdjustedIFile.h"
#include "PfsProcess.h"
void PfsProcess::displayHeader()
{
printf("[PartitionFS]\n");
printf(" Type: %s\n", mPfs.getFsType() == mPfs.TYPE_PFS0? "PFS0" : "HFS0");
printf(" FileNum: %" PRId64 "\n", mPfs.getFileList().getSize());
if (mMountName.empty() == false)
printf(" MountPoint: %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : "");
}
void PfsProcess::displayFs()
{
for (size_t i = 0; i < mPfs.getFileList().getSize(); i++)
{
printf(" %s", mPfs.getFileList()[i].name.c_str());
if (mCliOutputType >= OUTPUT_VERBOSE)
{
if (mPfs.getFsType() == mPfs.TYPE_PFS0)
printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size);
else
printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ", hash_protected_size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size, mPfs.getFileList()[i].hash_protected_size);
}
else
{
printf("\n");
}
}
}
size_t PfsProcess::determineHeaderSize(const nx::sPfsHeader* hdr)
{
size_t fileEntrySize = 0;
if (std::string(hdr->signature, 4) == nx::pfs::kPfsSig)
fileEntrySize = sizeof(nx::sPfsFile);
else
fileEntrySize = sizeof(nx::sHashedPfsFile);
return sizeof(nx::sPfsHeader) + hdr->file_num.get() * fileEntrySize + hdr->name_table_size.get();
}
bool PfsProcess::validateHeaderMagic(const nx::sPfsHeader* hdr)
{
return std::string(hdr->signature, 4) == nx::pfs::kPfsSig || std::string(hdr->signature, 4) == nx::pfs::kHashedPfsSig;
}
void PfsProcess::validateHfs()
{
fnd::MemoryBlob scratch;
crypto::sha::sSha256Hash hash;
const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
for (size_t i = 0; i < file.getSize(); i++)
{
scratch.alloc(file[i].hash_protected_size);
mReader->read(scratch.getBytes(), file[i].offset, file[i].hash_protected_size);
crypto::sha::Sha256(scratch.getBytes(), scratch.getSize(), hash.bytes);
if (hash != file[i].hash)
{
if (mCliOutputType >= OUTPUT_MINIMAL)
printf("[WARNING] HFS0 %s%s%s: FAIL (bad hash)\n", !mMountName.empty()? mMountName.c_str() : "", !mMountName.empty()? "/" : "", file[i].name.c_str());
}
}
}
void PfsProcess::extractFs()
{
// allocate scratch memory
fnd::MemoryBlob scratch;
scratch.alloc(kFileExportBlockSize);
// make extract dir
fnd::io::makeDirectory(mExtractPath);
fnd::SimpleFile outFile;
const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
std::string file_path;
for (size_t i = 0; i < file.getSize(); i++)
{
file_path.clear();
fnd::io::appendToPath(file_path, mExtractPath);
fnd::io::appendToPath(file_path, file[i].name);
outFile.open(file_path, outFile.Create);
mReader->seek(file[i].offset);
for (size_t j = 0; j < (file[i].size / kFileExportBlockSize); j++)
{
mReader->read(scratch.getBytes(), kFileExportBlockSize);
outFile.write(scratch.getBytes(), kFileExportBlockSize);
}
if (file[i].size % kFileExportBlockSize)
{
mReader->read(scratch.getBytes(), file[i].size % kFileExportBlockSize);
outFile.write(scratch.getBytes(), file[i].size % kFileExportBlockSize);
}
outFile.close();
}
}
PfsProcess::PfsProcess() :
mReader(nullptr),
mCliOutputType(OUTPUT_NORMAL),
@ -189,3 +91,106 @@ const nx::PfsHeader& PfsProcess::getPfsHeader() const
{
return mPfs;
}
void PfsProcess::displayHeader()
{
printf("[PartitionFS]\n");
printf(" Type: %s\n", mPfs.getFsType() == mPfs.TYPE_PFS0? "PFS0" : "HFS0");
printf(" FileNum: %" PRId64 "\n", mPfs.getFileList().getSize());
if (mMountName.empty() == false)
printf(" MountPoint: %s%s\n", mMountName.c_str(), mMountName.at(mMountName.length()-1) != '/' ? "/" : "");
}
void PfsProcess::displayFs()
{
for (size_t i = 0; i < mPfs.getFileList().getSize(); i++)
{
printf(" %s", mPfs.getFileList()[i].name.c_str());
if (mCliOutputType >= OUTPUT_VERBOSE)
{
if (mPfs.getFsType() == mPfs.TYPE_PFS0)
printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size);
else
printf(" (offset=0x%" PRIx64 ", size=0x%" PRIx64 ", hash_protected_size=0x%" PRIx64 ")\n", mPfs.getFileList()[i].offset, mPfs.getFileList()[i].size, mPfs.getFileList()[i].hash_protected_size);
}
else
{
printf("\n");
}
}
}
size_t PfsProcess::determineHeaderSize(const nx::sPfsHeader* hdr)
{
size_t fileEntrySize = 0;
if (std::string(hdr->signature, 4) == nx::pfs::kPfsSig)
fileEntrySize = sizeof(nx::sPfsFile);
else
fileEntrySize = sizeof(nx::sHashedPfsFile);
return sizeof(nx::sPfsHeader) + hdr->file_num.get() * fileEntrySize + hdr->name_table_size.get();
}
bool PfsProcess::validateHeaderMagic(const nx::sPfsHeader* hdr)
{
return std::string(hdr->signature, 4) == nx::pfs::kPfsSig || std::string(hdr->signature, 4) == nx::pfs::kHashedPfsSig;
}
void PfsProcess::validateHfs()
{
// allocate when validate is invoked
mFileExtractBlock.alloc(kFileExportBlockSize);
crypto::sha::sSha256Hash hash;
const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
for (size_t i = 0; i < file.getSize(); i++)
{
mFileExtractBlock.alloc(file[i].hash_protected_size);
mReader->read(mFileExtractBlock.getBytes(), file[i].offset, file[i].hash_protected_size);
crypto::sha::Sha256(mFileExtractBlock.getBytes(), mFileExtractBlock.getSize(), hash.bytes);
if (hash != file[i].hash)
{
if (mCliOutputType >= OUTPUT_MINIMAL)
printf("[WARNING] HFS0 %s%s%s: FAIL (bad hash)\n", !mMountName.empty()? mMountName.c_str() : "", !mMountName.empty()? "/" : "", file[i].name.c_str());
}
}
}
void PfsProcess::extractFs()
{
// allocate only when extractDir is invoked
mFileExtractBlock.alloc(kFileExportBlockSize);
// make extract dir
fnd::io::makeDirectory(mExtractPath);
fnd::SimpleFile outFile;
const fnd::List<nx::PfsHeader::sFile>& file = mPfs.getFileList();
std::string file_path;
for (size_t i = 0; i < file.getSize(); i++)
{
file_path.clear();
fnd::io::appendToPath(file_path, mExtractPath);
fnd::io::appendToPath(file_path, file[i].name);
if (mCliOutputType >= OUTPUT_VERBOSE)
printf("extract=[%s]\n", file_path.c_str());
outFile.open(file_path, outFile.Create);
mReader->seek(file[i].offset);
for (size_t j = 0; j < (file[i].size / kFileExportBlockSize); j++)
{
mReader->read(mFileExtractBlock.getBytes(), kFileExportBlockSize);
outFile.write(mFileExtractBlock.getBytes(), kFileExportBlockSize);
}
if (file[i].size % kFileExportBlockSize)
{
mReader->read(mFileExtractBlock.getBytes(), file[i].size % kFileExportBlockSize);
outFile.write(mFileExtractBlock.getBytes(), file[i].size % kFileExportBlockSize);
}
outFile.close();
}
}

2
programs/nstool/source/PfsProcess.h
View file

@ -39,6 +39,8 @@ private:
std::string mMountName;
bool mListFs;
fnd::MemoryBlob mFileExtractBlock;
nx::PfsHeader mPfs;
void displayHeader();

166
programs/nstool/source/RomfsProcess.cpp
View file

@ -4,6 +4,83 @@
#include "OffsetAdjustedIFile.h"
#include "RomfsProcess.h"
RomfsProcess::RomfsProcess() :
mReader(nullptr),
mCliOutputType(OUTPUT_NORMAL),
mVerify(false),
mExtractPath(),
mExtract(false),
mMountName(),
mListFs(false),
mDirNum(0),
mFileNum(0)
{
mRootDir.name.clear();
mRootDir.dir_list.clear();
mRootDir.file_list.clear();
}
RomfsProcess::~RomfsProcess()
{
if (mReader != nullptr)
{
delete mReader;
}
}
void RomfsProcess::process()
{
if (mReader == nullptr)
{
throw fnd::Exception(kModuleName, "No file reader set.");
}
resolveRomfs();
if (mCliOutputType >= OUTPUT_NORMAL)
displayHeader();
if (mListFs || mCliOutputType >= OUTPUT_VERBOSE)
displayFs();
if (mExtract)
extractFs();
}
void RomfsProcess::setInputFile(fnd::IFile* file, size_t offset, size_t size)
{
mReader = new OffsetAdjustedIFile(file, offset, size);
}
void RomfsProcess::setCliOutputMode(CliOutputType type)
{
mCliOutputType = type;
}
void RomfsProcess::setVerifyMode(bool verify)
{
mVerify = verify;
}
void RomfsProcess::setMountPointName(const std::string& mount_name)
{
mMountName = mount_name;
}
void RomfsProcess::setExtractPath(const std::string& path)
{
mExtract = true;
mExtractPath = path;
}
void RomfsProcess::setListFs(bool list_fs)
{
mListFs = list_fs;
}
const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const
{
return mRootDir;
}
void RomfsProcess::printTab(size_t tab) const
{
for (size_t i = 0; i < tab; i++)
@ -72,8 +149,10 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
// allocate memory for file extraction
#ifdef NSTOOL_ALLOC_UNIQUE_SCRATCH
fnd::MemoryBlob scratch;
scratch.alloc(kFileExportBlockSize);
#endif
// extract files
fnd::SimpleFile outFile;
@ -91,13 +170,13 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
mReader->seek(dir.file_list[i].offset);
for (size_t j = 0; j < (dir.file_list[i].size / kFileExportBlockSize); j++)
{
mReader->read(scratch.getBytes(), kFileExportBlockSize);
outFile.write(scratch.getBytes(), kFileExportBlockSize);
mReader->read(mFileExtractBlock.getBytes(), kFileExportBlockSize);
outFile.write(mFileExtractBlock.getBytes(), kFileExportBlockSize);
}
if (dir.file_list[i].size % kFileExportBlockSize)
{
mReader->read(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
outFile.write(scratch.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
mReader->read(mFileExtractBlock.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
outFile.write(mFileExtractBlock.getBytes(), dir.file_list[i].size % kFileExportBlockSize);
}
outFile.close();
}
@ -111,6 +190,8 @@ void RomfsProcess::extractDir(const std::string& path, const sDirectory& dir)
void RomfsProcess::extractFs()
{
// allocate only when extractDir is invoked
mFileExtractBlock.alloc(kFileExportBlockSize);
extractDir(mExtractPath, mRootDir);
}
@ -220,81 +301,4 @@ void RomfsProcess::resolveRomfs()
mDirNum = 0;
mFileNum = 0;
importDirectory(0, mRootDir);
}
RomfsProcess::RomfsProcess() :
mReader(nullptr),
mCliOutputType(OUTPUT_NORMAL),
mVerify(false),
mExtractPath(),
mExtract(false),
mMountName(),
mListFs(false),
mDirNum(0),
mFileNum(0)
{
mRootDir.name.clear();
mRootDir.dir_list.clear();
mRootDir.file_list.clear();
}
RomfsProcess::~RomfsProcess()
{
if (mReader != nullptr)
{
delete mReader;
}
}
void RomfsProcess::process()
{
if (mReader == nullptr)
{
throw fnd::Exception(kModuleName, "No file reader set.");
}
resolveRomfs();
if (mCliOutputType >= OUTPUT_NORMAL)
displayHeader();
if (mListFs || mCliOutputType >= OUTPUT_VERBOSE)
displayFs();
if (mExtract)
extractFs();
}
void RomfsProcess::setInputFile(fnd::IFile* file, size_t offset, size_t size)
{
mReader = new OffsetAdjustedIFile(file, offset, size);
}
void RomfsProcess::setCliOutputMode(CliOutputType type)
{
mCliOutputType = type;
}
void RomfsProcess::setVerifyMode(bool verify)
{
mVerify = verify;
}
void RomfsProcess::setMountPointName(const std::string& mount_name)
{
mMountName = mount_name;
}
void RomfsProcess::setExtractPath(const std::string& path)
{
mExtract = true;
mExtractPath = path;
}
void RomfsProcess::setListFs(bool list_fs)
{
mListFs = list_fs;
}
const RomfsProcess::sDirectory& RomfsProcess::getRootDir() const
{
return mRootDir;
}

3
programs/nstool/source/RomfsProcess.h
View file

@ -107,6 +107,7 @@ public:
private:
const std::string kModuleName = "RomfsProcess";
static const size_t kFileExportBlockSize = 0x1000000;
//static const size_t kFileExportBlockSize = 0x1000000;
fnd::IFile* mReader;
CliOutputType mCliOutputType;
@ -117,6 +118,8 @@ private:
std::string mMountName;
bool mListFs;
fnd::MemoryBlob mFileExtractBlock;
size_t mDirNum;
size_t mFileNum;
nx::sRomfsHeader mHdr;