Merge branch 'CreateBsaAddFileToBsa' into 'master'

Create BSA, add file to existing BSA

See merge request OpenMW/openmw!718

CHANGELOG.md

@@ -148,6 +148,7 @@
     Feature #5730: Add graphic herbalism option to the launcher and documents
     Feature #5771: ori command should report where a mesh is loaded from and whether the x version is used.
     Feature #5813: Instanced groundcover support
+    Feature #5814: Bsatool should be able to create BSA archives, not only to extract it
     Feature #5910: Fall back to delta time when physics can't keep up
     Task #5480: Drop Qt4 support
     Task #5520: Improve cell name autocompleter implementation

apps/bsatool/bsatool.cpp

@@ -20,6 +20,7 @@ struct Arguments
     std::string mode;
     std::string filename;
     std::string extractfile;
+    std::string addfile;
     std::string outdir;
 
     bool longformat;
@@ -36,6 +37,10 @@ bool parseOptions (int argc, char** argv, Arguments &info)
             "      Extract a file from the input archive.\n\n"
             "  bsatool extractall archivefile [output_directory]\n"
             "      Extract all files from the input archive.\n\n"
+            "  bsatool add [-a] archivefile file_to_add\n"
+            "      Add a file to the input archive.\n\n"
+            "  bsatool create [-c] archivefile\n"
+            "      Create an archive.\n\n"
             "Allowed options");
 
     desc.add_options()
@@ -95,7 +100,7 @@ bool parseOptions (int argc, char** argv, Arguments &info)
     }
 
     info.mode = variables["mode"].as<std::string>();
-    if (!(info.mode == "list" || info.mode == "extract" || info.mode == "extractall"))
+    if (!(info.mode == "list" || info.mode == "extract" || info.mode == "extractall" || info.mode == "add" || info.mode == "create"))
     {
         std::cout << std::endl << "ERROR: invalid mode \"" << info.mode << "\"\n\n"
             << desc << std::endl;
@@ -126,6 +131,17 @@ bool parseOptions (int argc, char** argv, Arguments &info)
         if (variables["input-file"].as< std::vector<std::string> >().size() > 2)
             info.outdir = variables["input-file"].as< std::vector<std::string> >()[2];
     }
+    else if (info.mode == "add")
+    {
+        if (variables["input-file"].as< std::vector<std::string> >().size() < 1)
+        {
+            std::cout << "\nERROR: file to add unspecified\n\n"
+                << desc << std::endl;
+            return false;
+        }
+        if (variables["input-file"].as< std::vector<std::string> >().size() > 1)
+            info.addfile = variables["input-file"].as< std::vector<std::string> >()[1];
+    }
     else if (variables["input-file"].as< std::vector<std::string> >().size() > 1)
         info.outdir = variables["input-file"].as< std::vector<std::string> >()[1];
 
@@ -138,6 +154,7 @@ bool parseOptions (int argc, char** argv, Arguments &info)
 int list(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info);
 int extract(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info);
 int extractAll(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info);
+int add(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info);
 
 int main(int argc, char** argv)
 {
@@ -157,6 +174,12 @@ int main(int argc, char** argv)
         else
             bsa = std::make_unique<Bsa::BSAFile>(Bsa::BSAFile());
 
+        if (info.mode == "create")
+        {
+            bsa->open(info.filename);
+            return 0;
+        }
+
         bsa->open(info.filename);
 
         if (info.mode == "list")
@@ -165,6 +188,8 @@ int main(int argc, char** argv)
             return extract(bsa, info);
         else if (info.mode == "extractall")
             return extractAll(bsa, info);
+        else if (info.mode == "add")
+            return add(bsa, info);
         else
         {
             std::cout << "Unsupported mode. That is not supposed to happen." << std::endl;
@@ -188,13 +213,13 @@ int list(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info)
         {
             // Long format
             std::ios::fmtflags f(std::cout.flags());
-            std::cout << std::setw(50) << std::left << file.name;
+            std::cout << std::setw(50) << std::left << file.name();
             std::cout << std::setw(8) << std::left << std::dec << file.fileSize;
             std::cout << "@ 0x" << std::hex << file.offset << std::endl;
             std::cout.flags(f);
         }
         else
-            std::cout << file.name << std::endl;
+            std::cout << file.name() << std::endl;
     }
 
     return 0;
@@ -253,7 +278,7 @@ int extractAll(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info)
 {
     for (const auto &file : bsa->getList())
     {
-        std::string extractPath(file.name);
+        std::string extractPath(file.name());
         Misc::StringUtils::replaceAll(extractPath, "\\", "/");
 
         // Get the target path (the path the file will be extracted to)
@@ -272,7 +297,7 @@ int extractAll(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info)
 
         // Get a stream for the file to extract
         // (inefficient because getFile iter on the list again)
-        Files::IStreamPtr data = bsa->getFile(file.name);
+        Files::IStreamPtr data = bsa->getFile(file.name());
         bfs::ofstream out(target, std::ios::binary);
 
         // Write the file to disk
@@ -283,3 +308,11 @@ int extractAll(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info)
 
     return 0;
 }
+
+int add(std::unique_ptr<Bsa::BSAFile>& bsa, Arguments& info)
+{
+    boost::filesystem::fstream stream(info.addfile, std::ios_base::binary | std::ios_base::out | std::ios_base::in);
+    bsa->addFile(info.addfile, stream);
+
+    return 0;
+}

components/bsa/bsa_file.cpp

@@ -27,6 +27,7 @@
 
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/fstream.hpp>
+#include <boost/filesystem/operations.hpp>
 
 using namespace Bsa;
 
@@ -37,6 +38,31 @@ void BSAFile::fail(const std::string &msg)
     throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename);
 }
 
+//the getHash code is from bsapack from ghostwheel
+//the code is also the same as in https://github.com/arviceblot/bsatool_rs/commit/67cb59ec3aaeedc0849222ea387f031c33e48c81
+BSAFile::Hash getHash(const std::string& name)
+{
+    BSAFile::Hash hash;
+    unsigned l = (name.size() >> 1);
+    unsigned sum, off, temp, i, n;
+
+    for (sum = off = i = 0; i < l; i++) {
+        sum ^= (((unsigned)(name[i])) << (off & 0x1F));
+        off += 8;
+    }
+    hash.low = sum;
+
+    for (sum = off = 0; i < name.size(); i++) {
+        temp = (((unsigned)(name[i])) << (off & 0x1F));
+        sum ^= temp;
+        n = temp & 0x1F;
+        sum = (sum << (32 - n)) | (sum >> n);  // binary "rotate right"
+        off += 8;
+    }
+    hash.high = sum;
+    return hash;
+}
+
 /// Read header information from the input source
 void BSAFile::readHeader()
 {
@@ -113,14 +139,17 @@ void BSAFile::readHeader()
 
     // Read the offset info into a temporary buffer
     std::vector<uint32_t> offsets(3*filenum);
-    input.read(reinterpret_cast<char*>(&offsets[0]), 12*filenum);
+    input.read(reinterpret_cast<char*>(offsets.data()), 12*filenum);
 
     // Read the string table
     mStringBuf.resize(dirsize-12*filenum);
-    input.read(&mStringBuf[0], mStringBuf.size());
+    input.read(mStringBuf.data(), mStringBuf.size());
 
     // Check our position
     assert(input.tellg() == std::streampos(12+dirsize));
+    std::vector<Hash> hashes(filenum);
+    static_assert(sizeof(Hash) == 8);
+    input.read(reinterpret_cast<char*>(hashes.data()), 8*filenum);
 
     // Calculate the offset of the data buffer. All file offsets are
     // relative to this. 12 header bytes + directory + hash table
@@ -129,23 +158,72 @@ void BSAFile::readHeader()
 
     // Set up the the FileStruct table
     mFiles.resize(filenum);
+    size_t endOfNameBuffer = 0;
     for(size_t i=0;i<filenum;i++)
     {
         FileStruct &fs = mFiles[i];
         fs.fileSize = offsets[i*2];
         fs.offset = offsets[i*2+1] + fileDataOffset;
-        fs.name = &mStringBuf[offsets[2*filenum+i]];
+        auto namesOffset = offsets[2*filenum+i];
+        fs.setNameInfos(namesOffset, &mStringBuf);
+        fs.hash = hashes[i];
+
+        endOfNameBuffer = std::max(endOfNameBuffer, namesOffset + std::strlen(fs.name())+1);
+        assert(endOfNameBuffer <= mStringBuf.size());
 
         if(fs.offset + fs.fileSize > fsize)
             fail("Archive contains offsets outside itself");
 
         // Add the file name to the lookup
-        mLookup[fs.name] = i;
+        mLookup[fs.name()] = i;
     }
+    mStringBuf.resize(endOfNameBuffer);
+
+    std::sort(mFiles.begin(), mFiles.end(), [](const FileStruct& left, const FileStruct& right) {
+        return left.offset < right.offset;
+    });
 
     mIsLoaded = true;
 }
 
+/// Write header information to the output sink
+void Bsa::BSAFile::writeHeader()
+{
+    namespace bfs = boost::filesystem;
+    bfs::fstream output(mFilename, std::ios::binary | std::ios::in | std::ios::out);
+
+    uint32_t head[3];
+    head[0] = 0x100;
+    auto fileDataOffset = mFiles.empty() ? 12 : mFiles.front().offset;
+    head[1] = fileDataOffset - 12 - 8*mFiles.size();
+
+    output.seekp(0, std::ios_base::end);
+
+    head[2] = mFiles.size();
+    output.seekp(0);
+    output.write(reinterpret_cast<char*>(head), 12);
+
+    std::sort(mFiles.begin(), mFiles.end(), [](const FileStruct& left, const FileStruct& right) {
+        return std::make_pair(left.hash.low, left.hash.high) < std::make_pair(right.hash.low, right.hash.high);
+    });
+
+    size_t filenum = mFiles.size();
+    std::vector<uint32_t> offsets(3* filenum);
+    std::vector<Hash> hashes(filenum);
+    for(size_t i=0;i<filenum;i++)
+    {
+        auto& f = mFiles[i];
+        offsets[i*2] = f.fileSize;
+        offsets[i*2+1] = f.offset - fileDataOffset;
+        offsets[2*filenum+i] = f.namesOffset;
+        hashes[i] = f.hash;
+    }
+    output.write(reinterpret_cast<char*>(offsets.data()), sizeof(uint32_t)*offsets.size());
+    output.write(reinterpret_cast<char*>(mStringBuf.data()), mStringBuf.size());
+    output.seekp(fileDataOffset - 8*mFiles.size(), std::ios_base::beg);
+    output.write(reinterpret_cast<char*>(hashes.data()), sizeof(Hash)*hashes.size());
+}
+
 /// Get the index of a given file name, or -1 if not found
 int BSAFile::getIndex(const char *str) const
 {
@@ -162,7 +240,22 @@ int BSAFile::getIndex(const char *str) const
 void BSAFile::open(const std::string &file)
 {
     mFilename = file;
-    readHeader();
+    if(boost::filesystem::exists(file))
+        readHeader();
+    else
+    {
+        { boost::filesystem::fstream(mFilename, std::ios::binary | std::ios::out); }
+        writeHeader();
+    }
+}
+
+/// Close the archive, write the updated headers to the file
+void Bsa::BSAFile::close()
+{
+    if (!mHasChanged)
+        return;
+
+    writeHeader();
 }
 
 Files::IStreamPtr BSAFile::getFile(const char *file)
@@ -181,3 +274,56 @@ Files::IStreamPtr BSAFile::getFile(const FileStruct *file)
 {
     return Files::openConstrainedFileStream (mFilename.c_str (), file->offset, file->fileSize);
 }
+
+void Bsa::BSAFile::addFile(const std::string& filename, std::istream& file)
+{
+    namespace bfs = boost::filesystem;
+
+    auto newStartOfDataBuffer = 12 + (12 + 8) * (mFiles.size() + 1) + mStringBuf.size() + filename.size() + 1;
+    if (mFiles.empty())
+        bfs::resize_file(mFilename, newStartOfDataBuffer);
+
+    bfs::fstream stream(mFilename, std::ios::binary | std::ios::in | std::ios::out);
+
+    FileStruct newFile;
+    file.seekg(0, std::ios::end);
+    newFile.fileSize = file.tellg();
+    newFile.setNameInfos(mStringBuf.size(), &mStringBuf);
+    newFile.hash = getHash(filename);
+
+    if(mFiles.empty())
+        newFile.offset = newStartOfDataBuffer;
+    else
+    {
+        std::vector<char> buffer;
+        while (mFiles.front().offset < newStartOfDataBuffer) {
+            FileStruct& firstFile = mFiles.front();
+            buffer.resize(firstFile.fileSize);
+
+            stream.seekg(firstFile.offset, std::ios::beg);
+            stream.read(buffer.data(), firstFile.fileSize);
+
+            stream.seekp(0, std::ios::end);
+            firstFile.offset = stream.tellp();
+
+            stream.write(buffer.data(), firstFile.fileSize);
+
+            //ensure sort order is preserved
+            std::rotate(mFiles.begin(), mFiles.begin() + 1, mFiles.end());
+        }
+        stream.seekp(0, std::ios::end);
+        newFile.offset = stream.tellp();
+    }
+
+    mStringBuf.insert(mStringBuf.end(), filename.begin(), filename.end());
+    mStringBuf.push_back('\0');
+    mFiles.push_back(newFile);
+
+    mHasChanged = true;
+
+    mLookup[filename.c_str()] = mFiles.size() - 1;
+
+    stream.seekp(0, std::ios::end);
+    file.seekg(0, std::ios::beg);
+    stream << file.rdbuf();
+}

components/bsa/bsa_file.hpp

@@ -43,20 +43,42 @@ namespace Bsa
 class BSAFile
 {
 public:
+
+    #pragma pack(push)
+    #pragma pack(1)
+    struct Hash
+    {
+        uint32_t low, high;
+    };
+    #pragma pack(pop)
+
     /// Represents one file entry in the archive
     struct FileStruct
     {
+        void setNameInfos(size_t index,
+            std::vector<char>* stringBuf
+        ) {
+            namesOffset = index;
+            namesBuffer = stringBuf;
+        }
+
         // File size and offset in file. We store the offset from the
         // beginning of the file, not the offset into the data buffer
         // (which is what is stored in the archive.)
         uint32_t fileSize, offset;
+        Hash hash;
 
         // Zero-terminated file name
-        const char *name;
+        const char* name() const { return &(*namesBuffer)[namesOffset]; };
+
+        uint32_t namesOffset = 0;
+        std::vector<char>* namesBuffer = nullptr;
     };
     typedef std::vector<FileStruct> FileList;
 
 protected:
+    bool mHasChanged = false;
+
     /// Table of files in this archive
     FileList mFiles;
 
@@ -72,7 +94,7 @@ protected:
     /// Case insensitive string comparison
     struct iltstr
     {
-        bool operator()(const char *s1, const char *s2) const
+        bool operator()(const std::string& s1, const std::string& s2) const
         { return Misc::StringUtils::ciLess(s1, s2); }
     };
 
@@ -80,7 +102,7 @@ protected:
         the files[] vector above. The iltstr ensures that file name
         checks are case insensitive.
     */
-    typedef std::map<const char*, int, iltstr> Lookup;
+    typedef std::map<std::string, int, iltstr> Lookup;
     Lookup mLookup;
 
     /// Error handling
@@ -88,9 +110,7 @@ protected:
 
     /// Read header information from the input source
     virtual void readHeader();
-
-    /// Read header information from the input source
-
+    virtual void writeHeader();
 
     /// Get the index of a given file name, or -1 if not found
     /// @note Thread safe.
@@ -106,11 +126,16 @@ public:
       : mIsLoaded(false)
     { }
 
-    virtual ~BSAFile() = default;
+    virtual ~BSAFile()
+    {
+        close();
+    }
 
     /// Open an archive file.
     void open(const std::string &file);
 
+    void close();
+
     /* -----------------------------------
      * Archive file routines
      * -----------------------------------
@@ -131,6 +156,8 @@ public:
     */
     virtual Files::IStreamPtr getFile(const FileStruct* file);
 
+    virtual void addFile(const std::string& filename, std::istream& file);
+
     /// Get a list of all files
     /// @note Thread safe.
     const FileList &getList() const

components/bsa/compressedbsafile.cpp

@@ -226,7 +226,6 @@ void CompressedBSAFile::readHeader()
             FileStruct fileStruct{};
             fileStruct.fileSize = file.getSizeWithoutCompressionFlag();
             fileStruct.offset = file.offset;
-            fileStruct.name = nullptr;
             mFiles.push_back(fileStruct);
 
             fullPaths.push_back(folder);
@@ -249,7 +248,7 @@ void CompressedBSAFile::readHeader()
         }
 
         //The vector guarantees that its elements occupy contiguous memory
-        mFiles[fileIndex].name = reinterpret_cast<char*>(mStringBuf.data() + mStringBuffOffset);
+        mFiles[fileIndex].setNameInfos(mStringBuffOffset, &mStringBuf);
 
         fullPaths.at(fileIndex) += "\\" + std::string(mStringBuf.data() + mStringBuffOffset);
 
@@ -276,7 +275,7 @@ void CompressedBSAFile::readHeader()
             fullPaths.at(fileIndex).c_str() + stringLength + 1u,
             mStringBuf.data() + mStringBuffOffset);
 
-        mFiles[fileIndex].name = reinterpret_cast<char*>(mStringBuf.data() + mStringBuffOffset);
+        mFiles[fileIndex].setNameInfos(mStringBuffOffset, &mStringBuf);
 
         mLookup[reinterpret_cast<char*>(mStringBuf.data() + mStringBuffOffset)] = fileIndex;
         mStringBuffOffset += stringLength + 1u;
@@ -320,13 +319,19 @@ CompressedBSAFile::FileRecord CompressedBSAFile::getFileRecord(const std::string
 
 Files::IStreamPtr CompressedBSAFile::getFile(const FileStruct* file) 
 {
-    FileRecord fileRec = getFileRecord(file->name);
+    FileRecord fileRec = getFileRecord(file->name());
     if (!fileRec.isValid()) {
-        fail("File not found: " + std::string(file->name));
+        fail("File not found: " + std::string(file->name()));
     }
     return getFile(fileRec);
 }
 
+void CompressedBSAFile::addFile(const std::string& filename, std::istream& file)
+{
+    assert(false); //not implemented yet
+    fail("Add file is not implemented for compressed BSA: " + filename);
+}
+
 Files::IStreamPtr CompressedBSAFile::getFile(const char* file)
 {
     FileRecord fileRec = getFileRecord(file);
@@ -430,10 +435,10 @@ void CompressedBSAFile::convertCompressedSizesToUncompressed()
 {
     for (auto & mFile : mFiles)
     {
-        const FileRecord& fileRecord = getFileRecord(mFile.name);
+        const FileRecord& fileRecord = getFileRecord(mFile.name());
         if (!fileRecord.isValid())
         {
-            fail("Could not find file " + std::string(mFile.name) + " in BSA");
+            fail("Could not find file " + std::string(mFile.name()) + " in BSA");
         }
 
         if (!fileRecord.isCompressed(mCompressedByDefault))

components/bsa/compressedbsafile.hpp

@@ -94,7 +94,7 @@ namespace Bsa
        
         Files::IStreamPtr getFile(const char* filePath) override;
         Files::IStreamPtr getFile(const FileStruct* fileStruct) override;
-        
+        void addFile(const std::string& filename, std::istream& file) override;
     };
 }
 

components/vfs/bsaarchive.cpp

@@ -32,7 +32,7 @@ void BsaArchive::listResources(std::map<std::string, File *> &out, char (*normal
 {
     for (std::vector<BsaArchiveFile>::iterator it = mResources.begin(); it != mResources.end(); ++it)
     {
-        std::string ent = it->mInfo->name;
+        std::string ent = it->mInfo->name();
         std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function);
 
         out[ent] = &*it;
@@ -43,7 +43,7 @@ bool BsaArchive::contains(const std::string& file, char (*normalize_function)(ch
 {
     for (const auto& it : mResources)
     {
-        std::string ent = it.mInfo->name;
+        std::string ent = it.mInfo->name();
         std::transform(ent.begin(), ent.end(), ent.begin(), normalize_function);
         if(file == ent)
             return true;