From d617d66a87fb58524f6a330b2c9e73559af192a0 Mon Sep 17 00:00:00 2001 From: CedricMocquillon Date: Wed, 7 Apr 2021 18:58:46 +0200 Subject: [PATCH 1/2] Add file to BSA --- apps/bsatool/bsatool.cpp | 43 +++++++- components/bsa/bsa_file.cpp | 156 ++++++++++++++++++++++++++- components/bsa/bsa_file.hpp | 41 +++++-- components/bsa/compressedbsafile.cpp | 19 ++-- components/bsa/compressedbsafile.hpp | 2 +- components/vfs/bsaarchive.cpp | 4 +- 6 files changed, 238 insertions(+), 27 deletions(-) diff --git a/apps/bsatool/bsatool.cpp b/apps/bsatool/bsatool.cpp index 3afbd777f..8e8cf8918 100644 --- a/apps/bsatool/bsatool.cpp +++ b/apps/bsatool/bsatool.cpp @@ -20,6 +20,7 @@ struct Arguments std::string mode; std::string filename; std::string extractfile; + std::string addfile; std::string outdir; bool longformat; @@ -36,6 +37,10 @@ bool parseOptions (int argc, char** argv, Arguments &info) " Extract a file from the input archive.\n\n" " bsatool extractall archivefile [output_directory]\n" " Extract all files from the input archive.\n\n" + " bsatool add [-a] archivefile file_to_add\n" + " Add a file to the input archive.\n\n" + " bsatool create [-c] archivefile\n" + " Create an archive.\n\n" "Allowed options"); desc.add_options() @@ -95,7 +100,7 @@ bool parseOptions (int argc, char** argv, Arguments &info) } info.mode = variables["mode"].as(); - if (!(info.mode == "list" || info.mode == "extract" || info.mode == "extractall")) + if (!(info.mode == "list" || info.mode == "extract" || info.mode == "extractall" || info.mode == "add" || info.mode == "create")) { std::cout << std::endl << "ERROR: invalid mode \"" << info.mode << "\"\n\n" << desc << std::endl; @@ -126,6 +131,17 @@ bool parseOptions (int argc, char** argv, Arguments &info) if (variables["input-file"].as< std::vector >().size() > 2) info.outdir = variables["input-file"].as< std::vector >()[2]; } + else if (info.mode == "add") + { + if (variables["input-file"].as< std::vector >().size() < 1) + { + std::cout << "\nERROR: file to add unspecified\n\n" + << desc << std::endl; + return false; + } + if (variables["input-file"].as< std::vector >().size() > 1) + info.addfile = variables["input-file"].as< std::vector >()[1]; + } else if (variables["input-file"].as< std::vector >().size() > 1) info.outdir = variables["input-file"].as< std::vector >()[1]; @@ -138,6 +154,7 @@ bool parseOptions (int argc, char** argv, Arguments &info) int list(std::unique_ptr& bsa, Arguments& info); int extract(std::unique_ptr& bsa, Arguments& info); int extractAll(std::unique_ptr& bsa, Arguments& info); +int add(std::unique_ptr& bsa, Arguments& info); int main(int argc, char** argv) { @@ -157,6 +174,12 @@ int main(int argc, char** argv) else bsa = std::make_unique(Bsa::BSAFile()); + if (info.mode == "create") + { + bsa->open(info.filename); + return 0; + } + bsa->open(info.filename); if (info.mode == "list") @@ -165,6 +188,8 @@ int main(int argc, char** argv) return extract(bsa, info); else if (info.mode == "extractall") return extractAll(bsa, info); + else if (info.mode == "add") + return add(bsa, info); else { std::cout << "Unsupported mode. That is not supposed to happen." << std::endl; @@ -188,13 +213,13 @@ int list(std::unique_ptr& bsa, Arguments& info) { // Long format std::ios::fmtflags f(std::cout.flags()); - std::cout << std::setw(50) << std::left << file.name; + std::cout << std::setw(50) << std::left << file.name(); std::cout << std::setw(8) << std::left << std::dec << file.fileSize; std::cout << "@ 0x" << std::hex << file.offset << std::endl; std::cout.flags(f); } else - std::cout << file.name << std::endl; + std::cout << file.name() << std::endl; } return 0; @@ -253,7 +278,7 @@ int extractAll(std::unique_ptr& bsa, Arguments& info) { for (const auto &file : bsa->getList()) { - std::string extractPath(file.name); + std::string extractPath(file.name()); Misc::StringUtils::replaceAll(extractPath, "\\", "/"); // Get the target path (the path the file will be extracted to) @@ -272,7 +297,7 @@ int extractAll(std::unique_ptr& bsa, Arguments& info) // Get a stream for the file to extract // (inefficient because getFile iter on the list again) - Files::IStreamPtr data = bsa->getFile(file.name); + Files::IStreamPtr data = bsa->getFile(file.name()); bfs::ofstream out(target, std::ios::binary); // Write the file to disk @@ -283,3 +308,11 @@ int extractAll(std::unique_ptr& bsa, Arguments& info) return 0; } + +int add(std::unique_ptr& bsa, Arguments& info) +{ + boost::filesystem::fstream stream(info.addfile, std::ios_base::binary | std::ios_base::out | std::ios_base::in); + bsa->addFile(info.addfile, stream); + + return 0; +} diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index f6220b7ce..ef49a60d2 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -27,6 +27,7 @@ #include #include +#include using namespace Bsa; @@ -37,6 +38,31 @@ void BSAFile::fail(const std::string &msg) throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename); } +//the getHash code is from bsapack from ghostwheel +//the code is also the same as in https://github.com/arviceblot/bsatool_rs/commit/67cb59ec3aaeedc0849222ea387f031c33e48c81 +BSAFile::Hash getHash(const std::string& name) +{ + BSAFile::Hash hash; + unsigned l = (name.size() >> 1); + unsigned sum, off, temp, i, n; + + for (sum = off = i = 0; i < l; i++) { + sum ^= (((unsigned)(name[i])) << (off & 0x1F)); + off += 8; + } + hash.low = sum; + + for (sum = off = 0; i < name.size(); i++) { + temp = (((unsigned)(name[i])) << (off & 0x1F)); + sum ^= temp; + n = temp & 0x1F; + sum = (sum << (32 - n)) | (sum >> n); // binary "rotate right" + off += 8; + } + hash.high = sum; + return hash; +} + /// Read header information from the input source void BSAFile::readHeader() { @@ -113,14 +139,17 @@ void BSAFile::readHeader() // Read the offset info into a temporary buffer std::vector offsets(3*filenum); - input.read(reinterpret_cast(&offsets[0]), 12*filenum); + input.read(reinterpret_cast(offsets.data()), 12*filenum); // Read the string table mStringBuf.resize(dirsize-12*filenum); - input.read(&mStringBuf[0], mStringBuf.size()); + input.read(mStringBuf.data(), mStringBuf.size()); // Check our position assert(input.tellg() == std::streampos(12+dirsize)); + std::vector hashes(filenum); + static_assert(sizeof(Hash) == 8); + input.read(reinterpret_cast(hashes.data()), 8*filenum); // Calculate the offset of the data buffer. All file offsets are // relative to this. 12 header bytes + directory + hash table @@ -129,23 +158,72 @@ void BSAFile::readHeader() // Set up the the FileStruct table mFiles.resize(filenum); + size_t endOfNameBuffer = 0; for(size_t i=0;i fsize) fail("Archive contains offsets outside itself"); // Add the file name to the lookup - mLookup[fs.name] = i; + mLookup[fs.name()] = i; } + mStringBuf.resize(endOfNameBuffer); + + std::sort(mFiles.begin(), mFiles.end(), [](const FileStruct& left, const FileStruct& right) { + return left.offset < right.offset; + }); mIsLoaded = true; } +/// Write header information to the output sink +void Bsa::BSAFile::writeHeader() +{ + namespace bfs = boost::filesystem; + bfs::fstream output(mFilename, std::ios::binary | std::ios::in | std::ios::out); + + uint32_t head[3]; + head[0] = 0x100; + auto fileDataOffset = mFiles.empty() ? 12 : mFiles.front().offset; + head[1] = fileDataOffset - 12 - 8*mFiles.size(); + + output.seekp(0, std::ios_base::end); + + head[2] = mFiles.size(); + output.seekp(0); + output.write(reinterpret_cast(head), 12); + + std::sort(mFiles.begin(), mFiles.end(), [](const FileStruct& left, const FileStruct& right) { + return std::make_pair(left.hash.low, left.hash.high) < std::make_pair(right.hash.low, right.hash.high); + }); + + size_t filenum = mFiles.size(); + std::vector offsets(3* filenum); + std::vector hashes(filenum); + for(size_t i=0;i(offsets.data()), sizeof(uint32_t)*offsets.size()); + output.write(reinterpret_cast(mStringBuf.data()), mStringBuf.size()); + output.seekp(fileDataOffset - 8*mFiles.size(), std::ios_base::beg); + output.write(reinterpret_cast(hashes.data()), sizeof(Hash)*hashes.size()); +} + /// Get the index of a given file name, or -1 if not found int BSAFile::getIndex(const char *str) const { @@ -162,7 +240,22 @@ int BSAFile::getIndex(const char *str) const void BSAFile::open(const std::string &file) { mFilename = file; - readHeader(); + if(boost::filesystem::exists(file)) + readHeader(); + else + { + { boost::filesystem::fstream(mFilename, std::ios::binary | std::ios::out); } + writeHeader(); + } +} + +/// Close the archive, write the updated headers to the file +void Bsa::BSAFile::close() +{ + if (!mHasChanged) + return; + + writeHeader(); } Files::IStreamPtr BSAFile::getFile(const char *file) @@ -181,3 +274,56 @@ Files::IStreamPtr BSAFile::getFile(const FileStruct *file) { return Files::openConstrainedFileStream (mFilename.c_str (), file->offset, file->fileSize); } + +void Bsa::BSAFile::addFile(const std::string& filename, std::istream& file) +{ + namespace bfs = boost::filesystem; + + auto newStartOfDataBuffer = 12 + (12 + 8) * (mFiles.size() + 1) + mStringBuf.size() + filename.size() + 1; + if (mFiles.empty()) + bfs::resize_file(mFilename, newStartOfDataBuffer); + + bfs::fstream stream(mFilename, std::ios::binary | std::ios::in | std::ios::out); + + FileStruct newFile; + file.seekg(0, std::ios::end); + newFile.fileSize = file.tellg(); + newFile.setNameInfos(mStringBuf.size(), &mStringBuf); + newFile.hash = getHash(filename); + + if(mFiles.empty()) + newFile.offset = newStartOfDataBuffer; + else + { + std::vector buffer; + while (mFiles.front().offset < newStartOfDataBuffer) { + FileStruct& firstFile = mFiles.front(); + buffer.resize(firstFile.fileSize); + + stream.seekg(firstFile.offset, std::ios::beg); + stream.read(buffer.data(), firstFile.fileSize); + + stream.seekp(0, std::ios::end); + firstFile.offset = stream.tellp(); + + stream.write(buffer.data(), firstFile.fileSize); + + //ensure sort order is preserved + std::rotate(mFiles.begin(), mFiles.begin() + 1, mFiles.end()); + } + stream.seekp(0, std::ios::end); + newFile.offset = stream.tellp(); + } + + mStringBuf.insert(mStringBuf.end(), filename.begin(), filename.end()); + mStringBuf.push_back('\0'); + mFiles.push_back(newFile); + + mHasChanged = true; + + mLookup[filename.c_str()] = mFiles.size() - 1; + + stream.seekp(0, std::ios::end); + file.seekg(0, std::ios::beg); + stream << file.rdbuf(); +} diff --git a/components/bsa/bsa_file.hpp b/components/bsa/bsa_file.hpp index 3e7538401..fa6e5fc1c 100644 --- a/components/bsa/bsa_file.hpp +++ b/components/bsa/bsa_file.hpp @@ -43,20 +43,42 @@ namespace Bsa class BSAFile { public: + + #pragma pack(push) + #pragma pack(1) + struct Hash + { + uint32_t low, high; + }; + #pragma pack(pop) + /// Represents one file entry in the archive struct FileStruct { + void setNameInfos(size_t index, + std::vector* stringBuf + ) { + namesOffset = index; + namesBuffer = stringBuf; + } + // File size and offset in file. We store the offset from the // beginning of the file, not the offset into the data buffer // (which is what is stored in the archive.) uint32_t fileSize, offset; + Hash hash; // Zero-terminated file name - const char *name; + const char* name() const { return &(*namesBuffer)[namesOffset]; }; + + uint32_t namesOffset = 0; + std::vector* namesBuffer = nullptr; }; typedef std::vector FileList; protected: + bool mHasChanged = false; + /// Table of files in this archive FileList mFiles; @@ -72,7 +94,7 @@ protected: /// Case insensitive string comparison struct iltstr { - bool operator()(const char *s1, const char *s2) const + bool operator()(const std::string& s1, const std::string& s2) const { return Misc::StringUtils::ciLess(s1, s2); } }; @@ -80,7 +102,7 @@ protected: the files[] vector above. The iltstr ensures that file name checks are case insensitive. */ - typedef std::map Lookup; + typedef std::map Lookup; Lookup mLookup; /// Error handling @@ -88,9 +110,7 @@ protected: /// Read header information from the input source virtual void readHeader(); - - /// Read header information from the input source - + virtual void writeHeader(); /// Get the index of a given file name, or -1 if not found /// @note Thread safe. @@ -106,11 +126,16 @@ public: : mIsLoaded(false) { } - virtual ~BSAFile() = default; + virtual ~BSAFile() + { + close(); + } /// Open an archive file. void open(const std::string &file); + void close(); + /* ----------------------------------- * Archive file routines * ----------------------------------- @@ -131,6 +156,8 @@ public: */ virtual Files::IStreamPtr getFile(const FileStruct* file); + virtual void addFile(const std::string& filename, std::istream& file); + /// Get a list of all files /// @note Thread safe. const FileList &getList() const diff --git a/components/bsa/compressedbsafile.cpp b/components/bsa/compressedbsafile.cpp index 77e477ac5..aaeb5bffa 100644 --- a/components/bsa/compressedbsafile.cpp +++ b/components/bsa/compressedbsafile.cpp @@ -226,7 +226,6 @@ void CompressedBSAFile::readHeader() FileStruct fileStruct{}; fileStruct.fileSize = file.getSizeWithoutCompressionFlag(); fileStruct.offset = file.offset; - fileStruct.name = nullptr; mFiles.push_back(fileStruct); fullPaths.push_back(folder); @@ -249,7 +248,7 @@ void CompressedBSAFile::readHeader() } //The vector guarantees that its elements occupy contiguous memory - mFiles[fileIndex].name = reinterpret_cast(mStringBuf.data() + mStringBuffOffset); + mFiles[fileIndex].setNameInfos(mStringBuffOffset, &mStringBuf); fullPaths.at(fileIndex) += "\\" + std::string(mStringBuf.data() + mStringBuffOffset); @@ -276,7 +275,7 @@ void CompressedBSAFile::readHeader() fullPaths.at(fileIndex).c_str() + stringLength + 1u, mStringBuf.data() + mStringBuffOffset); - mFiles[fileIndex].name = reinterpret_cast(mStringBuf.data() + mStringBuffOffset); + mFiles[fileIndex].setNameInfos(mStringBuffOffset, &mStringBuf); mLookup[reinterpret_cast(mStringBuf.data() + mStringBuffOffset)] = fileIndex; mStringBuffOffset += stringLength + 1u; @@ -320,13 +319,19 @@ CompressedBSAFile::FileRecord CompressedBSAFile::getFileRecord(const std::string Files::IStreamPtr CompressedBSAFile::getFile(const FileStruct* file) { - FileRecord fileRec = getFileRecord(file->name); + FileRecord fileRec = getFileRecord(file->name()); if (!fileRec.isValid()) { - fail("File not found: " + std::string(file->name)); + fail("File not found: " + std::string(file->name())); } return getFile(fileRec); } +void CompressedBSAFile::addFile(const std::string& filename, std::istream& file) +{ + assert(false); //not implemented yet + fail("Add file is not implemented for compressed BSA: " + filename); +} + Files::IStreamPtr CompressedBSAFile::getFile(const char* file) { FileRecord fileRec = getFileRecord(file); @@ -430,10 +435,10 @@ void CompressedBSAFile::convertCompressedSizesToUncompressed() { for (auto & mFile : mFiles) { - const FileRecord& fileRecord = getFileRecord(mFile.name); + const FileRecord& fileRecord = getFileRecord(mFile.name()); if (!fileRecord.isValid()) { - fail("Could not find file " + std::string(mFile.name) + " in BSA"); + fail("Could not find file " + std::string(mFile.name()) + " in BSA"); } if (!fileRecord.isCompressed(mCompressedByDefault)) diff --git a/components/bsa/compressedbsafile.hpp b/components/bsa/compressedbsafile.hpp index deddfae38..215a1fc49 100644 --- a/components/bsa/compressedbsafile.hpp +++ b/components/bsa/compressedbsafile.hpp @@ -94,7 +94,7 @@ namespace Bsa Files::IStreamPtr getFile(const char* filePath) override; Files::IStreamPtr getFile(const FileStruct* fileStruct) override; - + void addFile(const std::string& filename, std::istream& file) override; }; } diff --git a/components/vfs/bsaarchive.cpp b/components/vfs/bsaarchive.cpp index e6d779aab..90899ac61 100644 --- a/components/vfs/bsaarchive.cpp +++ b/components/vfs/bsaarchive.cpp @@ -32,7 +32,7 @@ void BsaArchive::listResources(std::map &out, char (*normal { for (std::vector::iterator it = mResources.begin(); it != mResources.end(); ++it) { - std::string ent = it->mInfo->name; + std::string ent = it->mInfo->name(); std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); out[ent] = &*it; @@ -43,7 +43,7 @@ bool BsaArchive::contains(const std::string& file, char (*normalize_function)(ch { for (const auto& it : mResources) { - std::string ent = it.mInfo->name; + std::string ent = it.mInfo->name(); std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function); if(file == ent) return true; From 9a6f0691b6fec2ca575b9f1f4629e9d890d358c8 Mon Sep 17 00:00:00 2001 From: CedricMocquillon Date: Thu, 8 Apr 2021 20:13:50 +0200 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8cb55d36..81ee0d9b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -148,6 +148,7 @@ Feature #5730: Add graphic herbalism option to the launcher and documents Feature #5771: ori command should report where a mesh is loaded from and whether the x version is used. Feature #5813: Instanced groundcover support + Feature #5814: Bsatool should be able to create BSA archives, not only to extract it Feature #5910: Fall back to delta time when physics can't keep up Task #5480: Drop Qt4 support Task #5520: Improve cell name autocompleter implementation