diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index a289f6647c..76641899d7 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -33,7 +33,7 @@ add_component_dir (settings ) add_component_dir (bsa - bsa_file tes4bsa_file memorystream + bsa_file compressedbsafile memorystream ) add_component_dir (vfs diff --git a/components/bsa/compressedbsafile.cpp b/components/bsa/compressedbsafile.cpp new file mode 100644 index 0000000000..598f65250f --- /dev/null +++ b/components/bsa/compressedbsafile.cpp @@ -0,0 +1,425 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (compressedbsafile.cpp) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + Compressed BSA stuff added by cc9cii 2018 + + */ +#include "compressedbsafile.hpp" + +#include +#include + +#include +#include +#include +#include + +#include // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation + +#include +#include +#include +#include +#include +#include + +namespace Bsa +{ +//special marker for invalid records, +//equal to max uint32_t value +const uint32_t CompressedBSAFile::sInvalidOffset = std::numeric_limits::max(); + +//bit marking compression on file size +const uint32_t CompressedBSAFile::sCompressedFlag = 1u << 30u; + + +CompressedBSAFile::FileRecord::FileRecord() : size(0), offset(sInvalidOffset) +{ } + +bool CompressedBSAFile::FileRecord::isValid() const +{ + return offset != sInvalidOffset; +} + +bool CompressedBSAFile::FileRecord::isCompressed(bool bsaCompressedByDefault) const +{ + bool compressionFlagEnabled = ((size & sCompressedFlag) == sCompressedFlag); + + if (bsaCompressedByDefault) { + return !compressionFlagEnabled; + } + return compressionFlagEnabled; +} + +std::uint32_t CompressedBSAFile::FileRecord::getSizeWithoutCompressionFlag() const { + return size & (~sCompressedFlag); +} + +void CompressedBSAFile::getBZString(std::string& str, std::istream& filestream) +{ + char size = 0; + filestream.read(&size, 1); + + boost::scoped_array buf(new char[size]); + filestream.read(buf.get(), size); + + if (buf[size - 1] != 0) + { + str.assign(buf.get(), size); + if (str.size() != ((size_t)size)) { + fail("getBZString string size mismatch"); + } + } + else + { + str.assign(buf.get(), size - 1); // don't copy null terminator + if (str.size() != ((size_t)size - 1)) { + fail("getBZString string size mismatch (null terminator)"); + } + } +} + +CompressedBSAFile::CompressedBSAFile() + : mCompressedByDefault(false), mEmbeddedFileNames(false) +{ } + +CompressedBSAFile::~CompressedBSAFile() +{ } + +/// Read header information from the input source +void CompressedBSAFile::readHeader() +{ + assert(!mIsLoaded); + + namespace bfs = boost::filesystem; + bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if(input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if(fsize < 36) // header is 36 bytes + fail("File too small to be a valid BSA archive"); + + // Get essential header numbers + //size_t dirsize, filenum; + std::uint32_t archiveFlags, folderCount, totalFileNameLength; + { + // First 36 bytes + std::uint32_t header[9]; + + input.read(reinterpret_cast(header), 36); + + if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/)) + fail("Unrecognized TES4 BSA header"); + + // header[2] is offset, should be 36 = 0x24 which is the size of the header + + // Oblivion - Meshes.bsa + // + // 0111 1000 0111 = 0x0787 + // ^^^ ^ ^^^ + // ||| | ||+-- has names for dirs (mandatory?) + // ||| | |+--- has names for files (mandatory?) + // ||| | +---- files are compressed by default + // ||| | + // ||| +---------- unknown (TES5: retain strings during startup) + // ||+------------ unknown (TES5: embedded file names) + // |+------------- unknown + // +-------------- unknown + // + archiveFlags = header[3]; + folderCount = header[4]; + // header[5] - fileCount + // totalFolderNameLength = header[6]; + totalFileNameLength = header[7]; + // header[8]; // fileFlags : an opportunity to optimize here + + mCompressedByDefault = (archiveFlags & 0x4) != 0; + mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0; + } + + // folder records + std::uint64_t hash; + FolderRecord fr; + for (std::uint32_t i = 0; i < folderCount; ++i) + { + input.read(reinterpret_cast(&hash), 8); + input.read(reinterpret_cast(&fr.count), 4); // not sure purpose of count + input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset + + std::map::const_iterator lb = mFolders.lower_bound(hash); + if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first))) + fail("Archive found duplicate folder name hash"); + else + mFolders.insert(lb, std::pair(hash, fr)); + } + + // file record blocks + std::uint64_t fileHash; + FileRecord file; + + std::string folder(""); + std::uint64_t folderHash; + if ((archiveFlags & 0x1) == 0) + folderCount = 1; // TODO: not tested - unit test necessary + + mFiles.clear(); + std::vector fullPaths; + + for (std::uint32_t i = 0; i < folderCount; ++i) + { + if ((archiveFlags & 0x1) != 0) + getBZString(folder, input); + + std::string emptyString; + folderHash = GenOBHash(folder, emptyString); + + std::map::iterator iter = mFolders.find(folderHash); + if (iter == mFolders.end()) + fail("Archive folder name hash not found"); + + for (std::uint32_t j = 0; j < iter->second.count; ++j) + { + input.read(reinterpret_cast(&fileHash), 8); + input.read(reinterpret_cast(&file.size), 4); + input.read(reinterpret_cast(&file.offset), 4); + + std::map::const_iterator lb = iter->second.files.lower_bound(fileHash); + if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first))) + fail("Archive found duplicate file name hash"); + + iter->second.files.insert(lb, std::pair(fileHash, file)); + + FileStruct fileStruct; + fileStruct.fileSize = file.getSizeWithoutCompressionFlag(); + fileStruct.offset = file.offset; + fileStruct.name = nullptr; + mFiles.push_back(fileStruct); + + fullPaths.push_back(folder); + } + } + + // file record blocks + if ((archiveFlags & 0x2) != 0) + { + mStringBuf.resize(totalFileNameLength); + input.read(&mStringBuf[0], mStringBuf.size()); // TODO: maybe useful in building a lookup map? + } + + size_t mStringBuffOffset = 0; + size_t totalStringsSize = 0; + for (std::uint32_t fileIndex = 0; fileIndex < mFiles.size(); ++fileIndex) { + + if (mStringBuffOffset >= totalFileNameLength) { + fail("Corrupted names record in BSA file"); + } + + //The vector guarantees that its elements occupy contiguous memory + mFiles[fileIndex].name = reinterpret_cast(mStringBuf.data() + mStringBuffOffset); + + fullPaths.at(fileIndex) += "\\" + std::string(mStringBuf.data() + mStringBuffOffset); + + while (mStringBuffOffset < totalFileNameLength) { + if (mStringBuf[mStringBuffOffset] != '\0') { + mStringBuffOffset++; + } + else { + mStringBuffOffset++; + break; + } + } + //we want to keep one more 0 character at the end of each string + totalStringsSize += fullPaths.at(fileIndex).length() + 1u; + } + mStringBuf.resize(totalStringsSize); + + mStringBuffOffset = 0; + for (std::uint32_t fileIndex = 0u; fileIndex < mFiles.size(); fileIndex++) { + size_t stringLength = fullPaths.at(fileIndex).length(); + + std::copy(fullPaths.at(fileIndex).c_str(), + //plus 1 because we also want to copy 0 at the end of the string + fullPaths.at(fileIndex).c_str() + stringLength + 1u, + mStringBuf.data() + mStringBuffOffset); + + mFiles[fileIndex].name = reinterpret_cast(mStringBuf.data() + mStringBuffOffset); + + mLookup[reinterpret_cast(mStringBuf.data() + mStringBuffOffset)] = fileIndex; + mStringBuffOffset += stringLength + 1u; + } + + if (mStringBuffOffset != mStringBuf.size()) { + fail("Could not resolve names of files in BSA file"); + } + + convertCompressedSizesToUncompressed(); + mIsLoaded = true; +} + +CompressedBSAFile::FileRecord CompressedBSAFile::getFileRecord(const std::string& str) const +{ + boost::filesystem::path p(str); + std::string stem = p.stem().string(); + std::string ext = p.extension().string(); + std::string filename = p.filename().string(); + p.remove_filename(); + + std::string folder = p.string(); + // GenOBHash already converts to lowercase and replaces file separators but not for path + boost::algorithm::to_lower(folder); + std::replace(folder.begin(), folder.end(), '/', '\\'); + + std::string emptyString; + std::uint64_t folderHash = GenOBHash(folder, emptyString); + + std::map::const_iterator it = mFolders.find(folderHash); + if (it == mFolders.end()) + return FileRecord(); // folder not found, return default which has offset of sInvalidOffset + + boost::algorithm::to_lower(stem); + boost::algorithm::to_lower(ext); + std::uint64_t fileHash = GenOBHashPair(stem, ext); + std::map::const_iterator iter = it->second.files.find(fileHash); + if (iter == it->second.files.end()) + return FileRecord(); // file not found, return default which has offset of sInvalidOffset + + return iter->second; +} + +Files::IStreamPtr CompressedBSAFile::getFile(const FileStruct* file) +{ + FileRecord fileRec = getFileRecord(file->name); + if (!fileRec.isValid()) { + fail("File not found: " + std::string(file->name)); + } + return getFile(fileRec); +} + +Files::IStreamPtr CompressedBSAFile::getFile(const char* file) +{ + FileRecord fileRec = getFileRecord(file); + if (!fileRec.isValid()) { + fail("File not found: " + std::string(file)); + } + return getFile(fileRec); +} + +Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord) +{ + if (fileRecord.isCompressed(mCompressedByDefault)) { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag()); + + std::istream* fileStream = streamPtr.get(); + + if (mEmbeddedFileNames) { + std::string embeddedFileName; + getBZString(embeddedFileName, *fileStream); + } + + uint32_t uncompressedSize = 0u; + fileStream->read(reinterpret_cast(&uncompressedSize), sizeof(uncompressedSize)); + + boost::iostreams::filtering_streambuf inputStreamBuf; + inputStreamBuf.push(boost::iostreams::zlib_decompressor()); + inputStreamBuf.push(*fileStream); + + std::shared_ptr memoryStreamPtr = std::make_shared(uncompressedSize); + + boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); + boost::iostreams::copy(inputStreamBuf, sr); + + return std::shared_ptr(memoryStreamPtr, (std::istream*)memoryStreamPtr.get()); + } + + return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size); +} + +BsaVersion CompressedBSAFile::detectVersion(std::string filePath) +{ + namespace bfs = boost::filesystem; + bfs::ifstream input(bfs::path(filePath), std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if (input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if (fsize < 12) { + return BSAVER_UNKNOWN; + } + + // Get essential header numbers + + // First 12 bytes + uint32_t head[3]; + + input.read(reinterpret_cast(head), 12); + + if (head[0] == static_cast(BSAVER_UNCOMPRESSED)) { + return BSAVER_UNCOMPRESSED; + } + + if (head[0] == static_cast(BSAVER_COMPRESSED)) { + return BSAVER_COMPRESSED; + } + + return BSAVER_UNKNOWN; +} + +//mFiles used by OpenMW expects uncompressed sizes +void CompressedBSAFile::convertCompressedSizesToUncompressed() +{ + for (auto iter = mFiles.begin(); iter != mFiles.end(); ++iter) + { + const FileRecord& fileRecord = getFileRecord(iter->name); + if (!fileRecord.isValid()) + { + fail("Could not find file " + std::string(iter->name) + " in BSA"); + } + + if (!fileRecord.isCompressed(mCompressedByDefault)) + { + //no need to fix fileSize in mFiles - uncompressed size already set + continue; + } + + Files::IStreamPtr dataBegin = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag()); + + if (mEmbeddedFileNames) + { + std::string embeddedFileName; + getBZString(embeddedFileName, *(dataBegin.get())); + } + + dataBegin->read(reinterpret_cast(&(iter->fileSize)), sizeof(iter->fileSize)); + } +} + +} //namespace Bsa diff --git a/components/bsa/compressedbsafile.hpp b/components/bsa/compressedbsafile.hpp new file mode 100644 index 0000000000..b607eb3eff --- /dev/null +++ b/components/bsa/compressedbsafile.hpp @@ -0,0 +1,101 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (compressedbsafile.hpp) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + Compressed BSA stuff added by cc9cii 2018 + + */ + +#ifndef BSA_COMPRESSED_BSA_FILE_H +#define BSA_COMPRESSED_BSA_FILE_H + +#include +#include +#include +#include +#include + +namespace Bsa +{ + enum BsaVersion + { + BSAVER_UNKNOWN = 0x0, + BSAVER_UNCOMPRESSED = 0x100, + BSAVER_COMPRESSED = 0x415342 //B, S, A + }; + + class CompressedBSAFile : public BSAFile + { + private: + //special marker for invalid records, + //equal to max uint32_t value + static const uint32_t sInvalidOffset; + + //bit marking compression on file size + static const uint32_t sCompressedFlag; + + struct FileRecord + { + std::uint32_t size; + std::uint32_t offset; + + FileRecord(); + bool isCompressed(bool bsaCompressedByDefault) const; + bool isValid() const; + std::uint32_t getSizeWithoutCompressionFlag() const; + }; + + //if files in BSA without 30th bit enabled are compressed + bool mCompressedByDefault; + + //if each file record begins with BZ string with file name + bool mEmbeddedFileNames; + + struct FolderRecord + { + std::uint32_t count; + std::uint32_t offset; + std::map files; + }; + std::map mFolders; + + FileRecord getFileRecord(const std::string& str) const; + + void getBZString(std::string& str, std::istream& filestream); + //mFiles used by OpenMW will contain uncompressed file sizes + void convertCompressedSizesToUncompressed(); + Files::IStreamPtr getFile(const FileRecord& fileRecord); + public: + CompressedBSAFile(); + virtual ~CompressedBSAFile(); + + //checks version of BSA from file header + static BsaVersion detectVersion(std::string filePath); + + /// Read header information from the input source + virtual void readHeader(); + + Files::IStreamPtr getFile(const char* filePath); + Files::IStreamPtr getFile(const FileStruct* fileStruct); + + }; +} + +#endif diff --git a/components/bsa/memorystream.cpp b/components/bsa/memorystream.cpp index 3cacad60f7..34e98e6b68 100644 --- a/components/bsa/memorystream.cpp +++ b/components/bsa/memorystream.cpp @@ -19,7 +19,7 @@ version 3 along with this program. If not, see http://www.gnu.org/licenses/ . - TES4 stuff upgrade added by Azdul 2019 + Compressed BSA upgrade added by Azdul 2019 */ #include "memorystream.hpp" diff --git a/components/bsa/memorystream.hpp b/components/bsa/memorystream.hpp index 0182e901ae..5dbe16ebe3 100644 --- a/components/bsa/memorystream.hpp +++ b/components/bsa/memorystream.hpp @@ -19,7 +19,7 @@ version 3 along with this program. If not, see http://www.gnu.org/licenses/ . - TES4 stuff upgrade added by Azdul 2019 + Compressed BSA upgrade added by Azdul 2019 */ diff --git a/components/bsa/tes4bsa_file.cpp b/components/bsa/tes4bsa_file.cpp deleted file mode 100644 index 33caefd0b3..0000000000 --- a/components/bsa/tes4bsa_file.cpp +++ /dev/null @@ -1,339 +0,0 @@ -/* - OpenMW - The completely unofficial reimplementation of Morrowind - Copyright (C) 2008-2010 Nicolay Korslund - Email: < korslund@gmail.com > - WWW: http://openmw.sourceforge.net/ - - This file (bsa_file.cpp) is part of the OpenMW package. - - OpenMW is distributed as free software: you can redistribute it - and/or modify it under the terms of the GNU General Public License - version 3, as published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - version 3 along with this program. If not, see - http://www.gnu.org/licenses/ . - - TES4 stuff added by cc9cii 2018 - - */ -#include "tes4bsa_file.hpp" - -#include -#include - -#include -#include -#include -#include - -#include - -#include // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation - -#undef TEST_UNIQUE_HASH - -namespace -{ - void getBZString(std::string& str, boost::filesystem::ifstream& filestream) - { - char size = 0; - filestream.read(&size, 1); - - boost::scoped_array buf(new char[size]); - filestream.read(buf.get(), size); - - if (buf[size - 1] != 0) - str.assign(buf.get(), size); - else - str.assign(buf.get(), size - 1); // don't copy null terminator - assert((size_t)size-1 == str.size() && "getBZString string size mismatch"); - return; - } -} - -using namespace Bsa; - - -/// Error handling -void TES4BSAFile::fail(const std::string& msg) -{ - throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename); -} - -/// Read header information from the input source -void TES4BSAFile::readHeader() -{ - assert(!isLoaded); - - namespace bfs = boost::filesystem; - bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary); - - // Total archive size - std::streamoff fsize = 0; - if(input.seekg(0, std::ios_base::end)) - { - fsize = input.tellg(); - input.seekg(0); - } - - if(fsize < 36) // header is 36 bytes - fail("File too small to be a valid BSA archive"); - - // Get essential header numbers - //size_t dirsize, filenum; - std::uint32_t archiveFlags, folderCount, fileCount, totalFileNameLength; - { - // First 36 bytes - std::uint32_t header[9]; - - input.read(reinterpret_cast(header), 36); - - if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/)) - fail("Unrecognized TES4 BSA header"); - - // header[2] is offset, should be 36 = 0x24 which is the size of the header - - // Oblivion - Meshes.bsa - // - // 0111 1000 0111 = 0x0787 - // ^^^ ^ ^^^ - // ||| | ||+-- has names for dirs (mandatory?) - // ||| | |+--- has names for files (mandatory?) - // ||| | +---- files are compressed by default - // ||| | - // ||| +---------- unknown (TES5: retain strings during startup) - // ||+------------ unknown (TES5: embedded file names) - // |+------------- unknown - // +-------------- unknown - // - archiveFlags = header[3]; - folderCount = header[4]; - fileCount = header[5]; - //totalFolderNameLength = header[6]; - totalFileNameLength = header[7]; - //fileFlags = header[8]; // FIXME: an opportunity to optimize here - - mCompressedByDefault = (archiveFlags & 0x4) != 0; - mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0; - } - - // TODO: more checks for BSA file corruption - - // folder records - std::uint64_t hash; - FolderRecord fr; - for (std::uint32_t i = 0; i < folderCount; ++i) - { - input.read(reinterpret_cast(&hash), 8); - input.read(reinterpret_cast(&fr.count), 4); // not sure purpose of count - input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset - - std::map::const_iterator lb = mFolders.lower_bound(hash); - if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first))) - fail("Archive found duplicate folder name hash"); - else - mFolders.insert(lb, std::pair(hash, fr)); - } - - // file record blocks - std::uint64_t fileHash; - FileRecord file; - - std::string folder(""); - std::uint64_t folderHash; - if ((archiveFlags & 0x1) == 0) - folderCount = 1; // TODO: not tested - - for (std::uint32_t i = 0; i < folderCount; ++i) - { - if ((archiveFlags & 0x1) != 0) - getBZString(folder, input); - - folderHash = GenOBHash(folder, std::string("")); - - std::map::iterator iter = mFolders.find(folderHash); - if (iter == mFolders.end()) - fail("Archive folder name hash not found"); - - for (std::uint32_t j = 0; j < iter->second.count; ++j) - { - input.read(reinterpret_cast(&fileHash), 8); - input.read(reinterpret_cast(&file.size), 4); - input.read(reinterpret_cast(&file.offset), 4); - - std::map::const_iterator lb = iter->second.files.lower_bound(fileHash); - if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first))) - fail("Archive found duplicate file name hash"); - - iter->second.files.insert(lb, std::pair(fileHash, file)); - } - } - - // file record blocks - if ((archiveFlags & 0x2) != 0) - { - mStringBuf.resize(totalFileNameLength); - input.read(&mStringBuf[0], mStringBuf.size()); // TODO: maybe useful in building a lookup map? - } - - // TODO: more checks for BSA file corruption - - isLoaded = true; -} - -TES4BSAFile::FileRecord TES4BSAFile::getFileRecord(const std::string& str) const -{ - boost::filesystem::path p(str); - std::string stem = p.stem().string(); - std::string ext = p.extension().string(); - std::string filename = p.filename().string(); - p.remove_filename(); - - std::string folder = p.string(); - // GenOBHash already converts to lowercase and replaces file separators but not for path - boost::algorithm::to_lower(folder); - std::replace(folder.begin(), folder.end(), '/', '\\'); - - std::uint64_t folderHash = GenOBHash(folder, std::string("")); - - std::map::const_iterator it = mFolders.find(folderHash); - if (it == mFolders.end()) - return FileRecord(); // folder not found, return default which has offset of -1 - - boost::algorithm::to_lower(stem); - boost::algorithm::to_lower(ext); - std::uint64_t fileHash = GenOBHashPair(stem, ext); - std::map::const_iterator iter = it->second.files.find(fileHash); - if (iter == it->second.files.end()) - return FileRecord(); // file not found, return default which has offset of -1 - - // cache for next time - std::uint64_t hash = GenOBHash(folder, filename); - -#if defined (TEST_UNIQUE_HASH) - FileList::const_iterator lb = mFiles.lower_bound(hash); - if (lb != mFiles.end() && !(mFiles.key_comp()(hash, lb->first))) - { - // found, check if same filename - if (lb->second.fileName == str) - return iter->second; // same name, should not have got here!! - else - { - // different filename, hash is not unique! - std::cerr << "BSA hash collision: " << str << std::hex << "0x" << hash << std::endl; - - return iter->second; // return without cashing - } - } - - // not found, cache for later - const_cast(mFiles).insert(lb, std::pair(hash, iter->second)); - const_cast(mFiles)[hash].fileName = str; -#else - const_cast(mFiles)[hash] = iter->second; // NOTE: const hack -#endif - return iter->second; -} - -bool TES4BSAFile::exists(const std::string& str) const -{ - // check cache first - boost::filesystem::path p(str); - std::string filename = p.filename().string(); - p.remove_filename(); - - std::string folder = p.string(); - // GenOBHash already converts to lowercase and replaces file separators but not for path - boost::algorithm::to_lower(folder); - std::replace(folder.begin(), folder.end(), '/', '\\'); - - std::uint64_t hash = GenOBHash(folder, filename); - - std::map::const_iterator it = mFiles.find(hash); -#if defined (TEST_UNIQUE_HASH) - if (it != mFiles.end() && it->second.fileName == str) -#else - if (it != mFiles.end()) -#endif - return true; - else - return getFileRecord(str).offset != -1; -} - -void TES4BSAFile::open(const std::string& file) -{ - mFilename = file; - readHeader(); -} - -Ogre::DataStreamPtr TES4BSAFile::getFile(const std::string& file) -{ - assert(file); - - FileRecord fileRec = getFileRecord(file); - if(fileRec.offset == -1) - fail("File not found: " + std::string(file)); - - boost::filesystem::ifstream input(boost::filesystem::path(mFilename), std::ios_base::binary); - input.seekg(fileRec.offset); - - std::string fullPath; - if (mEmbeddedFileNames) - getBZString(fullPath, input); // TODO: maybe cache the hash and/or offset of frequently used ones? - - if (( mCompressedByDefault && (fileRec.size & (1<<30)) == 0) - || - (!mCompressedByDefault && (fileRec.size & (1<<30)) != 0)) - { - std::uint32_t bufSize = 0; - boost::scoped_array inBuf; - inBuf.reset(new unsigned char[fileRec.size-4]); - input.read(reinterpret_cast(&bufSize), 4); - input.read(reinterpret_cast(inBuf.get()), fileRec.size-4); - Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(bufSize); - Ogre::SharedPtr streamPtr(outBuf); - - int ret; - z_stream strm; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = bufSize; - strm.next_in = inBuf.get(); - ret = inflateInit(&strm); - if (ret != Z_OK) - throw std::runtime_error("TES4BSAFile::getFile - inflateInit failed"); - - strm.avail_out = bufSize; - strm.next_out = outBuf->getPtr(); - ret = inflate(&strm, Z_NO_FLUSH); - assert(ret != Z_STREAM_ERROR && "TES4BSAFile::getFile - inflate - state clobbered"); - switch (ret) - { - case Z_NEED_DICT: - ret = Z_DATA_ERROR; /* and fall through */ - case Z_DATA_ERROR: - case Z_MEM_ERROR: - inflateEnd(&strm); - throw std::runtime_error("TES4BSAFile::getFile - inflate failed"); - } - assert(ret == Z_OK || ret == Z_STREAM_END); - inflateEnd(&strm); - - return streamPtr; - } - else // not compressed TODO: not tested - { - Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(fileRec.size); - Ogre::SharedPtr streamPtr(outBuf); - input.read(reinterpret_cast(outBuf->getPtr()), fileRec.size); - - return streamPtr; - } -} diff --git a/components/bsa/tes4bsa_file.hpp b/components/bsa/tes4bsa_file.hpp deleted file mode 100644 index 979a6af1a0..0000000000 --- a/components/bsa/tes4bsa_file.hpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - OpenMW - The completely unofficial reimplementation of Morrowind - Copyright (C) 2008-2010 Nicolay Korslund - Email: < korslund@gmail.com > - WWW: http://openmw.sourceforge.net/ - - This file (bsa_file.h) is part of the OpenMW package. - - OpenMW is distributed as free software: you can redistribute it - and/or modify it under the terms of the GNU General Public License - version 3, as published by the Free Software Foundation. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - version 3 along with this program. If not, see - http://www.gnu.org/licenses/ . - - TES4 stuff added by cc9cii 2018 - - */ - -#ifndef BSA_TES4BSA_FILE_H -#define BSA_TES4BSA_FILE_H - -#include -#include -#include -#include - -#include - - -namespace Bsa -{ - class TES4BSAFile - { - public: - struct FileRecord - { - std::uint32_t size; - std::uint32_t offset; - - std::string fileName; // NOTE: for testing hash collision only, see TEST_UNIQUE_HASH - - FileRecord() : size(0), offset(-1) {} - }; - - private: - /// Filenames string buffer - std::vector mStringBuf; - - /// True when an archive has been loaded - bool isLoaded; - - bool mCompressedByDefault; - bool mEmbeddedFileNames; - - std::map mFiles; - typedef std::map FileList; - - struct FolderRecord - { - std::uint32_t count; - std::uint32_t offset; - std::map files; - }; - std::map mFolders; - - FileRecord getFileRecord(const std::string& str) const; - - /// Used for error messages and getting files - std::string mFilename; - - /// Error handling - void fail(const std::string &msg); - - /// Read header information from the input source - void readHeader(); - - public: - TES4BSAFile() - : isLoaded(false), mCompressedByDefault(false), mEmbeddedFileNames(false) - { } - - /// Open an archive file. - void open(const std::string &file); - - /// Check if a file exists - bool exists(const std::string& file) const; - - Ogre::DataStreamPtr getFile(const std::string& file); - - /// Get a list of all files - const FileList &getList() const // FIXME - { return mFiles; } - }; -} - -#endif diff --git a/components/vfs/bsaarchive.cpp b/components/vfs/bsaarchive.cpp index 268a530472..925c1076e3 100644 --- a/components/vfs/bsaarchive.cpp +++ b/components/vfs/bsaarchive.cpp @@ -1,5 +1,5 @@ #include "bsaarchive.hpp" -#include +#include #include namespace VFS @@ -7,10 +7,10 @@ namespace VFS BsaArchive::BsaArchive(const std::string &filename) { - Bsa::BsaVersion bsaVersion = Bsa::TES4BSAFile::detectVersion(filename); + Bsa::BsaVersion bsaVersion = Bsa::CompressedBSAFile::detectVersion(filename); - if (bsaVersion == Bsa::BSAVER_TES4PLUS) { - mFile = std::unique_ptr(new Bsa::TES4BSAFile()); + if (bsaVersion == Bsa::BSAVER_COMPRESSED) { + mFile = std::unique_ptr(new Bsa::CompressedBSAFile()); } else { mFile = std::unique_ptr(new Bsa::BSAFile());