From 3982573035b726dd8a1ffbe261bc342f79c8aecb Mon Sep 17 00:00:00 2001 From: cc9cii Date: Sat, 6 Oct 2018 13:35:51 +1000 Subject: [PATCH] BSA enhancements. * Implement hash based lookup for TES3 BSA files. * Added TES4/TES5 BSA support. * Implemented a hack (non-portable code) in an attempt to reduce startup time under Windows because Boost::filesystem seems to take forever on GetFileAttributeW. This implementation uses FindFirstFile/FindNextFile/FindClose instead. --- CMakeLists.txt | 7 +- apps/opencs/CMakeLists.txt | 2 + apps/opencs/editor.cpp | 17 + apps/opencs/editor.hpp | 1 + apps/openmw/CMakeLists.txt | 2 + apps/openmw/engine.cpp | 6 + apps/openmw/engine.hpp | 2 + apps/openmw/main.cpp | 9 + components/CMakeLists.txt | 2 +- components/bsa/bsa_archive.cpp | 195 ++++++++++- components/bsa/bsa_archive.hpp | 1 + components/bsa/bsa_file.cpp | 69 +++- components/bsa/bsa_file.hpp | 2 + components/bsa/resources.cpp | 9 +- components/bsa/resources.hpp | 2 +- components/bsa/tes4bsa_file.cpp | 339 +++++++++++++++++++ components/bsa/tes4bsa_file.hpp | 103 ++++++ extern/BSAOpt/CMakeLists.txt | 17 + extern/BSAOpt/hash.cpp | 108 ++++++ extern/BSAOpt/hash.hpp | 42 +++ plugins/mygui_resource_plugin/CMakeLists.txt | 2 + 21 files changed, 917 insertions(+), 20 deletions(-) create mode 100644 components/bsa/tes4bsa_file.cpp create mode 100644 components/bsa/tes4bsa_file.hpp create mode 100644 extern/BSAOpt/CMakeLists.txt create mode 100644 extern/BSAOpt/hash.cpp create mode 100644 extern/BSAOpt/hash.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index ff5191c46..2ec616cbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,9 @@ if(USE_SYSTEM_TINYXML) endif() endif() +# TES4 +find_package(ZLIB REQUIRED) + # Platform specific if (WIN32) if(NOT MINGW) @@ -257,9 +260,10 @@ include_directories("." ${LIBS_DIR} ${MYGUI_PLATFORM_INCLUDE_DIRS} ${OPENAL_INCLUDE_DIR} ${BULLET_INCLUDE_DIRS} + ${ZLIB_INCLUDE_DIRS} ) -link_directories(${SDL2_LIBRARY_DIRS} ${Boost_LIBRARY_DIRS} ${OGRE_LIB_DIR} ${MYGUI_LIB_DIR}) +link_directories(${SDL2_LIBRARY_DIRS} ${Boost_LIBRARY_DIRS} ${OGRE_LIB_DIR} ${MYGUI_LIB_DIR} ${ZLIB_LIB_DIR}) if(MYGUI_STATIC) add_definitions(-DMYGUI_STATIC) @@ -577,6 +581,7 @@ add_subdirectory (extern/ogre-ffmpeg-videoplayer) add_subdirectory (extern/oics) add_subdirectory (extern/sdl4ogre) add_subdirectory (extern/murmurhash) +add_subdirectory (extern/BSAOpt) # Components add_subdirectory (components) diff --git a/apps/opencs/CMakeLists.txt b/apps/opencs/CMakeLists.txt index 7f3709b8a..ec686c4e2 100644 --- a/apps/opencs/CMakeLists.txt +++ b/apps/opencs/CMakeLists.txt @@ -211,7 +211,9 @@ target_link_libraries(openmw-cs ${OGRE_Overlay_LIBRARIES} ${OGRE_STATIC_PLUGINS} ${SHINY_LIBRARIES} + ${ZLIB_LIBRARY} ${MURMURHASH_LIBRARIES} + ${BSAOPTHASH_LIBRARIES} ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/apps/opencs/editor.cpp b/apps/opencs/editor.cpp index 79f32619e..cd5f6753d 100644 --- a/apps/opencs/editor.cpp +++ b/apps/opencs/editor.cpp @@ -30,6 +30,7 @@ CS::Editor::Editor (OgreInit::OgreInit& ogreInit) mIpcServerName ("org.openmw.OpenCS"), mServer(NULL), mClientSocket(NULL) { std::pair > config = readConfig(); + std::vector tes4config = readTES4Config(); setupDataFiles (config.first); @@ -44,6 +45,9 @@ CS::Editor::Editor (OgreInit::OgreInit& ogreInit) Bsa::registerResources (Files::Collections (config.first, !mFsStrict), config.second, true, mFsStrict); + // useLooseFiles is set false, since it is already done above + Bsa::registerResources (Files::Collections (config.first, !mFsStrict), tes4config, /*useLooseFiles*/false, + mFsStrict, /*isTes4*/true); mDocumentManager.listResources(); @@ -182,6 +186,19 @@ std::pair > CS::Editor::readConfi return std::make_pair (canonicalPaths, variables["fallback-archive"].as >()); } +std::vector CS::Editor::readTES4Config() +{ + boost::program_options::variables_map variables; + boost::program_options::options_description desc("Syntax: openmw-cs \nAllowed options"); + + desc.add_options() + ("fallback-tes4archive", boost::program_options::value >()-> + default_value(std::vector(), "fallback-tes4archive")->multitoken()); + + mCfgMgr.readConfiguration(variables, desc, /*quiet*/true); + return variables["fallback-tes4archive"].as >(); +} + void CS::Editor::createGame() { mStartup.hide(); diff --git a/apps/opencs/editor.hpp b/apps/opencs/editor.hpp index 33f5fd3b3..5eae1258a 100644 --- a/apps/opencs/editor.hpp +++ b/apps/opencs/editor.hpp @@ -73,6 +73,7 @@ namespace CS void setupDataFiles (const Files::PathContainer& dataDirs); std::pair > readConfig(bool quiet=false); + std::vector readTES4Config(); ///< \return data paths // not implemented diff --git a/apps/openmw/CMakeLists.txt b/apps/openmw/CMakeLists.txt index ffa52042c..cdb1691f0 100644 --- a/apps/openmw/CMakeLists.txt +++ b/apps/openmw/CMakeLists.txt @@ -117,6 +117,8 @@ target_link_libraries(openmw ${OGRE_LIBRARIES} ${OGRE_STATIC_PLUGINS} ${SHINY_LIBRARIES} + ${BSAOPTHASH_LIBRARIES} + ${ZLIB_LIBRARY} ${OPENAL_LIBRARY} ${SOUND_INPUT_LIBRARY} ${BULLET_LIBRARIES} diff --git a/apps/openmw/engine.cpp b/apps/openmw/engine.cpp index 539539db9..1bc3b48c5 100644 --- a/apps/openmw/engine.cpp +++ b/apps/openmw/engine.cpp @@ -252,6 +252,10 @@ void OMW::Engine::addArchive (const std::string& archive) { mArchives.push_back(archive); } +void OMW::Engine::addTES4Archive (const std::string& archive) { + mTES4Archives.push_back(archive); +} + // Set resource dir void OMW::Engine::setResourceDir (const boost::filesystem::path& parResDir) { @@ -365,6 +369,8 @@ void OMW::Engine::prepareEngine (Settings::Manager & settings) mOgre->createWindow("OpenMW", windowSettings); Bsa::registerResources (mFileCollections, mArchives, true, mFSStrict); + // useLooseFiles is set false, since it is already done above + Bsa::registerResources (mFileCollections, mTES4Archives, /*useLooseFiles*/false, mFSStrict, /*isTes4*/true); // Create input and UI first to set up a bootstrapping environment for // showing a loading screen and keeping the window responsive while doing so diff --git a/apps/openmw/engine.hpp b/apps/openmw/engine.hpp index 3b088595c..79b047cd1 100644 --- a/apps/openmw/engine.hpp +++ b/apps/openmw/engine.hpp @@ -67,6 +67,7 @@ namespace OMW ToUTF8::Utf8Encoder* mEncoder; Files::PathContainer mDataDirs; std::vector mArchives; + std::vector mTES4Archives; boost::filesystem::path mResDir; OEngine::Render::OgreRenderer *mOgre; std::string mCellName; @@ -137,6 +138,7 @@ namespace OMW /// Add BSA archive void addArchive(const std::string& archive); + void addTES4Archive(const std::string& archive); /// Set resource dir void setResourceDir(const boost::filesystem::path& parResDir); diff --git a/apps/openmw/main.cpp b/apps/openmw/main.cpp index 2d2c9af0c..c36fcc5e9 100644 --- a/apps/openmw/main.cpp +++ b/apps/openmw/main.cpp @@ -112,6 +112,9 @@ bool parseOptions (int argc, char** argv, OMW::Engine& engine, Files::Configurat ("fallback-archive", bpo::value()->default_value(StringsVector(), "fallback-archive") ->multitoken(), "set fallback BSA archives (later archives have higher priority)") + ("fallback-tes4archive", bpo::value()->default_value(StringsVector(), "fallback-tes4archive") + ->multitoken(), "set fallback TES4 BSA archives (later archives have higher priority)") + ("resources", bpo::value()->default_value("resources"), "set resources directory") @@ -240,6 +243,12 @@ bool parseOptions (int argc, char** argv, OMW::Engine& engine, Files::Configurat engine.addArchive(*it); } + StringsVector tes4archives = variables["fallback-tes4archive"].as(); + for (StringsVector::const_iterator it = tes4archives.begin(); it != tes4archives.end(); ++it) + { + engine.addTES4Archive(*it); + } + engine.setResourceDir(variables["resources"].as()); StringsVector content = variables["content"].as(); diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index da0428ec2..9b6f9fef9 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -31,7 +31,7 @@ add_component_dir (nifoverrides ) add_component_dir (bsa - bsa_archive bsa_file resources + bsa_archive bsa_file resources tes4bsa_file ) add_component_dir (nif diff --git a/components/bsa/bsa_archive.cpp b/components/bsa/bsa_archive.cpp index 12836ea70..114abee20 100644 --- a/components/bsa/bsa_archive.cpp +++ b/components/bsa/bsa_archive.cpp @@ -23,7 +23,10 @@ #include "bsa_archive.hpp" +#include + #include +#include #include #include @@ -41,10 +44,92 @@ #define OGRE_CONST #endif +#include #include "bsa_file.hpp" +#include "tes4bsa_file.hpp" #include "../files/constrainedfiledatastream.hpp" +namespace +{ +// Concepts from answer by Remy Lebeau +// https://stackoverflow.com/questions/15068475/recursive-hard-disk-search-with-findfirstfile-findnextfile-c +// +// Also see https://msdn.microsoft.com/en-us/library/aa365200%28VS.85%29.aspx +// +// From 34.5 sec down to 18.5 sec on laptop with many of the data files on an external USB drive +#if defined _WIN32 || defined _WIN64 + +#include +#include // auto_ptr + + // FIXME: not tested unicode path and filenames + DWORD indexFiles(const std::string& rootDir, const std::string& subdir, + std::map& files, std::map& index) + { + HANDLE hFind = INVALID_HANDLE_VALUE; + WIN32_FIND_DATA ffd; + std::string path = rootDir + ((subdir == "") ? "" : "\\" +subdir); + + hFind = FindFirstFile((path + "\\*").c_str(), &ffd); + if (INVALID_HANDLE_VALUE == hFind) + return ERROR_INVALID_HANDLE; + + std::auto_ptr > subDirs; + std::string filename; + + do + { + filename = std::string(ffd.cFileName); + + if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + { + if (filename != "." && filename != "..") + { + if (subDirs.get() == nullptr) + subDirs.reset(new std::vector); + + subDirs->push_back(filename); + } + } + else + { + std::uint64_t folderHash = GenOBHash(subdir, filename); + + std::map::iterator iter = files.find(folderHash); + if (iter != files.end()) + throw std::runtime_error ("duplicate hash found"); + + files[folderHash] = path + "\\" + filename; + + std::string entry = ((subdir == "") ? "" : subdir + "\\") + filename; + std::replace(entry.begin(), entry.end(), '\\', '/'); + index.insert(std::make_pair (entry, path + "/" + filename)); + } + } while (FindNextFile(hFind, &ffd) != 0); + + FindClose(hFind); + + DWORD dwError = GetLastError(); + if (dwError != ERROR_NO_MORE_FILES) + return dwError; + + if (subDirs.get() != nullptr) + { + for (size_t i = 0; i < subDirs->size(); ++i) + { + std::string dir = subDirs->at(i); + boost::algorithm::to_lower(dir); + // FIXME: ignoring errors for now + dwError = indexFiles(rootDir, ((subdir == "") ? "" : subdir + "\\") + dir, files, index); + } + } + + return 0; + } +#endif +} + using namespace Ogre; static bool fsstrict = false; @@ -78,6 +163,8 @@ class DirArchive: public Ogre::Archive index mIndex; + std::map mFiles; + index::const_iterator lookup_filename (std::string const & filename) const { std::string normalized = normalize_path (filename.begin (), filename.end ()); @@ -89,6 +176,10 @@ public: DirArchive(const String& name) : Archive(name, "Dir") { +#if defined _WIN32 || defined _WIN64 + indexFiles(name, "", mFiles, mIndex); +#else + typedef boost::filesystem::recursive_directory_iterator directory_iterator; directory_iterator end; @@ -109,6 +200,7 @@ public: mIndex.insert (std::make_pair (searchable, proper)); } +#endif } bool isCaseSensitive() const { return fsstrict; } @@ -119,6 +211,27 @@ public: virtual DataStreamPtr open(const String& filename, bool readonly = true) OGRE_CONST { +#if defined _WIN32 || defined _WIN64 + boost::filesystem::path p(filename); + std::string file = p.filename().string(); + + p.remove_filename(); + std::string dir = p.string(); + boost::algorithm::to_lower(dir); + std::replace(dir.begin(), dir.end(), '/', '\\'); + + std::uint64_t hash = GenOBHash(dir, file); + + std::map::const_iterator it = mFiles.find(hash); + if (it == mFiles.end()) + { + std::ostringstream os; + os << "The file '" << filename << "' could not be found."; + throw std::runtime_error (os.str()); + } + + return openConstrainedFileDataStream (it->second.c_str()); +#else index::const_iterator i = lookup_filename (filename); if (i == mIndex.end ()) @@ -129,6 +242,7 @@ public: } return openConstrainedFileDataStream (i->second.c_str ()); +#endif } StringVectorPtr list(bool recursive = true, bool dirs = false) @@ -157,7 +271,25 @@ public: bool exists(const String& filename) { +#if defined _WIN32 || defined _WIN64 + boost::filesystem::path p(filename); + std::string file = p.filename().string(); + + p.remove_filename(); + std::string dir = p.string(); + boost::algorithm::to_lower(dir); + std::replace(dir.begin(), dir.end(), '/', '\\'); + + std::uint64_t hash = GenOBHash(dir, file); + + std::map::const_iterator it = mFiles.find(hash); + if (it == mFiles.end()) + return false; + + return true; +#else return lookup_filename(filename) != mIndex.end (); +#endif } time_t getModifiedTime(const String&) { return 0; } @@ -223,6 +355,8 @@ public: : Archive(name, "BSA") { arc.open(name); } + BSAArchive(const String& name, const std::string& type) : Archive(name, type) {} + bool isCaseSensitive() const { return false; } // The archive is loaded in the constructor, and never unloaded. @@ -241,7 +375,7 @@ public: return narc->getFile(filename.c_str()); } - bool exists(const String& filename) { + virtual bool exists(const String& filename) { return arc.exists(filename.c_str()); } @@ -259,7 +393,7 @@ public: return findFileInfo ("*", recursive, dirs); } - StringVectorPtr find(const String& pattern, bool recursive = true, + virtual StringVectorPtr find(const String& pattern, bool recursive = true, bool dirs = false) { std::string normalizedPattern = normalize_path(pattern.begin(), pattern.end()); @@ -306,6 +440,24 @@ public: } }; +class TES4BSAArchive : public BSAArchive +{ + Bsa::TES4BSAFile arc; + +public: + TES4BSAArchive::TES4BSAArchive(const String& name) : BSAArchive(name, "TES4BSA") { arc.open(name); } + + virtual DataStreamPtr open(const String& filename, bool readonly = true) + { + return arc.getFile(filename); + } + + virtual bool exists(const String& filename) + { + return arc.exists(filename); + } +}; + // An archive factory for BSA archives class BSAArchiveFactory : public ArchiveFactory { @@ -351,9 +503,32 @@ public: void destroyInstance( Archive* arch) { delete arch; } }; +class TES4BSAArchiveFactory : public ArchiveFactory +{ +public: + const String& getType() const + { + static String name = "TES4BSA"; + return name; + } + + Archive *createInstance( const String& name ) + { + return new TES4BSAArchive(name); + } + + virtual Archive* createInstance(const String& name, bool readOnly) + { + return new TES4BSAArchive(name); + } + + void destroyInstance( Archive* arch) { delete arch; } +}; + static bool init = false; static bool init2 = false; +static bool init3 = false; static void insertBSAFactory() { @@ -364,6 +539,15 @@ static void insertBSAFactory() } } +static void insertTES4BSAFactory() +{ + if(!init3) + { + ArchiveManager::getSingleton().addArchiveFactory( new TES4BSAArchiveFactory ); + init3 = true; + } +} + static void insertDirFactory() { if(!init2) @@ -386,6 +570,13 @@ void addBSA(const std::string& name, const std::string& group) addResourceLocation(name, "BSA", group, true); } +void addTES4BSA(const std::string& name, const std::string& group) +{ + insertTES4BSAFactory(); + ResourceGroupManager::getSingleton(). + addResourceLocation(name, "TES4BSA", group, true); +} + void addDir(const std::string& name, const bool& fs, const std::string& group) { fsstrict = fs; diff --git a/components/bsa/bsa_archive.hpp b/components/bsa/bsa_archive.hpp index 7f9ebaae1..d930c2571 100644 --- a/components/bsa/bsa_archive.hpp +++ b/components/bsa/bsa_archive.hpp @@ -33,6 +33,7 @@ namespace Bsa /// Add the given BSA file as an input archive in the Ogre resource /// system. void addBSA(const std::string& file, const std::string& group="General"); +void addTES4BSA(const std::string& file, const std::string& group="General"); void addDir(const std::string& file, const bool& fs, const std::string& group="General"); } diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index 3bf73ede2..40bfce2ac 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -27,9 +27,41 @@ #include #include +#include #include "../files/constrainedfiledatastream.hpp" +namespace +{ + // see: http://en.uesp.net/wiki/Tes3Mod:BSA_File_Format + std::uint64_t getHash(const char *name) + { + unsigned int len = (unsigned int)strlen(name); + std::uint64_t hash; + + unsigned l = (len>>1); + unsigned sum, off, temp, i, n, hash1; + + for(sum = off = i = 0; i < l; i++) { + sum ^= (((unsigned)(name[i]))<<(off&0x1F)); + off += 8; + } + hash1 = sum; + + for(sum = off = 0; i < len; i++) { + temp = (((unsigned)(name[i]))<<(off&0x1F)); + sum ^= temp; + n = temp & 0x1F; + sum = (sum << (32-n)) | (sum >> n); // binary "rotate right" + off += 8; + } + hash = sum; + hash <<= 32; + hash += hash1; + return hash; + } +} + using namespace std; using namespace Bsa; @@ -143,7 +175,14 @@ void BSAFile::readHeader() fail("Archive contains offsets outside itself"); // Add the file name to the lookup - lookup[fs.name] = i; + //lookup[fs.name] = i; + } + + std::uint64_t hash; + for (size_t i = 0; i < filenum; ++i) + { + input.read(reinterpret_cast(&hash), 8); + mFiles[hash] = files[i]; } isLoaded = true; @@ -152,13 +191,14 @@ void BSAFile::readHeader() /// Get the index of a given file name, or -1 if not found int BSAFile::getIndex(const char *str) const { - Lookup::const_iterator it = lookup.find(str); - if(it == lookup.end()) + std::string name(str); + boost::algorithm::to_lower(name); + std::uint64_t hash = getHash(name.c_str()); + std::map::const_iterator iter = mFiles.find(hash); + if (iter != mFiles.end()) + return 0; // NOTE: this is a bit of a hack, exists() only checks for '-1' + else return -1; - - int res = it->second; - assert(res >= 0 && (size_t)res < files.size()); - return res; } /// Open an archive file. @@ -171,10 +211,15 @@ void BSAFile::open(const string &file) Ogre::DataStreamPtr BSAFile::getFile(const char *file) { assert(file); - int i = getIndex(file); - if(i == -1) - fail("File not found: " + string(file)); - const FileStruct &fs = files[i]; - return openConstrainedFileDataStream (filename.c_str (), fs.offset, fs.fileSize); + std::string name(file); + boost::algorithm::to_lower(name); + std::uint64_t hash = getHash(name.c_str()); + std::map::const_iterator it = mFiles.find(hash); + if (it != mFiles.end()) + { + const FileStruct &fs = it->second; + return openConstrainedFileDataStream (filename.c_str (), fs.offset, fs.fileSize); + } + fail("File not found: " + string(file)); } diff --git a/components/bsa/bsa_file.hpp b/components/bsa/bsa_file.hpp index 017adf1e3..5379d20e3 100644 --- a/components/bsa/bsa_file.hpp +++ b/components/bsa/bsa_file.hpp @@ -82,6 +82,8 @@ private: typedef std::map Lookup; Lookup lookup; + std::map mFiles; + /// Error handling void fail(const std::string &msg); diff --git a/components/bsa/resources.cpp b/components/bsa/resources.cpp index b66da1a76..c5075ec69 100644 --- a/components/bsa/resources.cpp +++ b/components/bsa/resources.cpp @@ -9,7 +9,7 @@ #include "bsa_archive.hpp" void Bsa::registerResources (const Files::Collections& collections, - const std::vector& archives, bool useLooseFiles, bool fsStrict) + const std::vector& archives, bool useLooseFiles, bool fsStrict, bool isTes4) { const Files::PathContainer& dataDirs = collections.getPaths(); @@ -34,13 +34,16 @@ void Bsa::registerResources (const Files::Collections& collections, if (collections.doesExist(*archive)) { // Last BSA has the highest priority - std::string groupName = "DataBSA" + Ogre::StringConverter::toString(archives.size()-i, 8, '0'); + std::string groupName = (isTes4 ? "TES4BSA" : "DataBSA") + Ogre::StringConverter::toString(archives.size()-i, 8, '0'); Ogre::ResourceGroupManager::getSingleton ().createResourceGroup (groupName); const std::string archivePath = collections.getPath(*archive).string(); std::cout << "Adding BSA archive " << archivePath << std::endl; - Bsa::addBSA(archivePath, groupName); + if (!isTes4) + Bsa::addBSA(archivePath, groupName); + else + Bsa::addTES4BSA(archivePath, groupName); ++i; } else diff --git a/components/bsa/resources.hpp b/components/bsa/resources.hpp index 8c3fb7bef..b0723b3eb 100644 --- a/components/bsa/resources.hpp +++ b/components/bsa/resources.hpp @@ -9,7 +9,7 @@ namespace Bsa { void registerResources (const Files::Collections& collections, - const std::vector& archives, bool useLooseFiles, bool fsStrict); + const std::vector& archives, bool useLooseFiles, bool fsStrict, bool isTes4=false); ///< Register resources directories and archives as OGRE resources groups } diff --git a/components/bsa/tes4bsa_file.cpp b/components/bsa/tes4bsa_file.cpp new file mode 100644 index 000000000..33caefd0b --- /dev/null +++ b/components/bsa/tes4bsa_file.cpp @@ -0,0 +1,339 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (bsa_file.cpp) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + TES4 stuff added by cc9cii 2018 + + */ +#include "tes4bsa_file.hpp" + +#include +#include + +#include +#include +#include +#include + +#include + +#include // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation + +#undef TEST_UNIQUE_HASH + +namespace +{ + void getBZString(std::string& str, boost::filesystem::ifstream& filestream) + { + char size = 0; + filestream.read(&size, 1); + + boost::scoped_array buf(new char[size]); + filestream.read(buf.get(), size); + + if (buf[size - 1] != 0) + str.assign(buf.get(), size); + else + str.assign(buf.get(), size - 1); // don't copy null terminator + assert((size_t)size-1 == str.size() && "getBZString string size mismatch"); + return; + } +} + +using namespace Bsa; + + +/// Error handling +void TES4BSAFile::fail(const std::string& msg) +{ + throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename); +} + +/// Read header information from the input source +void TES4BSAFile::readHeader() +{ + assert(!isLoaded); + + namespace bfs = boost::filesystem; + bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if(input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if(fsize < 36) // header is 36 bytes + fail("File too small to be a valid BSA archive"); + + // Get essential header numbers + //size_t dirsize, filenum; + std::uint32_t archiveFlags, folderCount, fileCount, totalFileNameLength; + { + // First 36 bytes + std::uint32_t header[9]; + + input.read(reinterpret_cast(header), 36); + + if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/)) + fail("Unrecognized TES4 BSA header"); + + // header[2] is offset, should be 36 = 0x24 which is the size of the header + + // Oblivion - Meshes.bsa + // + // 0111 1000 0111 = 0x0787 + // ^^^ ^ ^^^ + // ||| | ||+-- has names for dirs (mandatory?) + // ||| | |+--- has names for files (mandatory?) + // ||| | +---- files are compressed by default + // ||| | + // ||| +---------- unknown (TES5: retain strings during startup) + // ||+------------ unknown (TES5: embedded file names) + // |+------------- unknown + // +-------------- unknown + // + archiveFlags = header[3]; + folderCount = header[4]; + fileCount = header[5]; + //totalFolderNameLength = header[6]; + totalFileNameLength = header[7]; + //fileFlags = header[8]; // FIXME: an opportunity to optimize here + + mCompressedByDefault = (archiveFlags & 0x4) != 0; + mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0; + } + + // TODO: more checks for BSA file corruption + + // folder records + std::uint64_t hash; + FolderRecord fr; + for (std::uint32_t i = 0; i < folderCount; ++i) + { + input.read(reinterpret_cast(&hash), 8); + input.read(reinterpret_cast(&fr.count), 4); // not sure purpose of count + input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset + + std::map::const_iterator lb = mFolders.lower_bound(hash); + if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first))) + fail("Archive found duplicate folder name hash"); + else + mFolders.insert(lb, std::pair(hash, fr)); + } + + // file record blocks + std::uint64_t fileHash; + FileRecord file; + + std::string folder(""); + std::uint64_t folderHash; + if ((archiveFlags & 0x1) == 0) + folderCount = 1; // TODO: not tested + + for (std::uint32_t i = 0; i < folderCount; ++i) + { + if ((archiveFlags & 0x1) != 0) + getBZString(folder, input); + + folderHash = GenOBHash(folder, std::string("")); + + std::map::iterator iter = mFolders.find(folderHash); + if (iter == mFolders.end()) + fail("Archive folder name hash not found"); + + for (std::uint32_t j = 0; j < iter->second.count; ++j) + { + input.read(reinterpret_cast(&fileHash), 8); + input.read(reinterpret_cast(&file.size), 4); + input.read(reinterpret_cast(&file.offset), 4); + + std::map::const_iterator lb = iter->second.files.lower_bound(fileHash); + if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first))) + fail("Archive found duplicate file name hash"); + + iter->second.files.insert(lb, std::pair(fileHash, file)); + } + } + + // file record blocks + if ((archiveFlags & 0x2) != 0) + { + mStringBuf.resize(totalFileNameLength); + input.read(&mStringBuf[0], mStringBuf.size()); // TODO: maybe useful in building a lookup map? + } + + // TODO: more checks for BSA file corruption + + isLoaded = true; +} + +TES4BSAFile::FileRecord TES4BSAFile::getFileRecord(const std::string& str) const +{ + boost::filesystem::path p(str); + std::string stem = p.stem().string(); + std::string ext = p.extension().string(); + std::string filename = p.filename().string(); + p.remove_filename(); + + std::string folder = p.string(); + // GenOBHash already converts to lowercase and replaces file separators but not for path + boost::algorithm::to_lower(folder); + std::replace(folder.begin(), folder.end(), '/', '\\'); + + std::uint64_t folderHash = GenOBHash(folder, std::string("")); + + std::map::const_iterator it = mFolders.find(folderHash); + if (it == mFolders.end()) + return FileRecord(); // folder not found, return default which has offset of -1 + + boost::algorithm::to_lower(stem); + boost::algorithm::to_lower(ext); + std::uint64_t fileHash = GenOBHashPair(stem, ext); + std::map::const_iterator iter = it->second.files.find(fileHash); + if (iter == it->second.files.end()) + return FileRecord(); // file not found, return default which has offset of -1 + + // cache for next time + std::uint64_t hash = GenOBHash(folder, filename); + +#if defined (TEST_UNIQUE_HASH) + FileList::const_iterator lb = mFiles.lower_bound(hash); + if (lb != mFiles.end() && !(mFiles.key_comp()(hash, lb->first))) + { + // found, check if same filename + if (lb->second.fileName == str) + return iter->second; // same name, should not have got here!! + else + { + // different filename, hash is not unique! + std::cerr << "BSA hash collision: " << str << std::hex << "0x" << hash << std::endl; + + return iter->second; // return without cashing + } + } + + // not found, cache for later + const_cast(mFiles).insert(lb, std::pair(hash, iter->second)); + const_cast(mFiles)[hash].fileName = str; +#else + const_cast(mFiles)[hash] = iter->second; // NOTE: const hack +#endif + return iter->second; +} + +bool TES4BSAFile::exists(const std::string& str) const +{ + // check cache first + boost::filesystem::path p(str); + std::string filename = p.filename().string(); + p.remove_filename(); + + std::string folder = p.string(); + // GenOBHash already converts to lowercase and replaces file separators but not for path + boost::algorithm::to_lower(folder); + std::replace(folder.begin(), folder.end(), '/', '\\'); + + std::uint64_t hash = GenOBHash(folder, filename); + + std::map::const_iterator it = mFiles.find(hash); +#if defined (TEST_UNIQUE_HASH) + if (it != mFiles.end() && it->second.fileName == str) +#else + if (it != mFiles.end()) +#endif + return true; + else + return getFileRecord(str).offset != -1; +} + +void TES4BSAFile::open(const std::string& file) +{ + mFilename = file; + readHeader(); +} + +Ogre::DataStreamPtr TES4BSAFile::getFile(const std::string& file) +{ + assert(file); + + FileRecord fileRec = getFileRecord(file); + if(fileRec.offset == -1) + fail("File not found: " + std::string(file)); + + boost::filesystem::ifstream input(boost::filesystem::path(mFilename), std::ios_base::binary); + input.seekg(fileRec.offset); + + std::string fullPath; + if (mEmbeddedFileNames) + getBZString(fullPath, input); // TODO: maybe cache the hash and/or offset of frequently used ones? + + if (( mCompressedByDefault && (fileRec.size & (1<<30)) == 0) + || + (!mCompressedByDefault && (fileRec.size & (1<<30)) != 0)) + { + std::uint32_t bufSize = 0; + boost::scoped_array inBuf; + inBuf.reset(new unsigned char[fileRec.size-4]); + input.read(reinterpret_cast(&bufSize), 4); + input.read(reinterpret_cast(inBuf.get()), fileRec.size-4); + Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(bufSize); + Ogre::SharedPtr streamPtr(outBuf); + + int ret; + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = bufSize; + strm.next_in = inBuf.get(); + ret = inflateInit(&strm); + if (ret != Z_OK) + throw std::runtime_error("TES4BSAFile::getFile - inflateInit failed"); + + strm.avail_out = bufSize; + strm.next_out = outBuf->getPtr(); + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR && "TES4BSAFile::getFile - inflate - state clobbered"); + switch (ret) + { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + inflateEnd(&strm); + throw std::runtime_error("TES4BSAFile::getFile - inflate failed"); + } + assert(ret == Z_OK || ret == Z_STREAM_END); + inflateEnd(&strm); + + return streamPtr; + } + else // not compressed TODO: not tested + { + Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(fileRec.size); + Ogre::SharedPtr streamPtr(outBuf); + input.read(reinterpret_cast(outBuf->getPtr()), fileRec.size); + + return streamPtr; + } +} diff --git a/components/bsa/tes4bsa_file.hpp b/components/bsa/tes4bsa_file.hpp new file mode 100644 index 000000000..979a6af1a --- /dev/null +++ b/components/bsa/tes4bsa_file.hpp @@ -0,0 +1,103 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (bsa_file.h) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + TES4 stuff added by cc9cii 2018 + + */ + +#ifndef BSA_TES4BSA_FILE_H +#define BSA_TES4BSA_FILE_H + +#include +#include +#include +#include + +#include + + +namespace Bsa +{ + class TES4BSAFile + { + public: + struct FileRecord + { + std::uint32_t size; + std::uint32_t offset; + + std::string fileName; // NOTE: for testing hash collision only, see TEST_UNIQUE_HASH + + FileRecord() : size(0), offset(-1) {} + }; + + private: + /// Filenames string buffer + std::vector mStringBuf; + + /// True when an archive has been loaded + bool isLoaded; + + bool mCompressedByDefault; + bool mEmbeddedFileNames; + + std::map mFiles; + typedef std::map FileList; + + struct FolderRecord + { + std::uint32_t count; + std::uint32_t offset; + std::map files; + }; + std::map mFolders; + + FileRecord getFileRecord(const std::string& str) const; + + /// Used for error messages and getting files + std::string mFilename; + + /// Error handling + void fail(const std::string &msg); + + /// Read header information from the input source + void readHeader(); + + public: + TES4BSAFile() + : isLoaded(false), mCompressedByDefault(false), mEmbeddedFileNames(false) + { } + + /// Open an archive file. + void open(const std::string &file); + + /// Check if a file exists + bool exists(const std::string& file) const; + + Ogre::DataStreamPtr getFile(const std::string& file); + + /// Get a list of all files + const FileList &getList() const // FIXME + { return mFiles; } + }; +} + +#endif diff --git a/extern/BSAOpt/CMakeLists.txt b/extern/BSAOpt/CMakeLists.txt new file mode 100644 index 000000000..a7f1dcf74 --- /dev/null +++ b/extern/BSAOpt/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 2.8) + +# This is NOT intended as a stand-alone build system! Instead, you should include this from the main CMakeLists of your project. + +set(BSAOPTHASH_LIBRARY "bsaopthash") + +# Sources +set(SOURCE_FILES + hash.cpp +) + +add_library(${BSAOPTHASH_LIBRARY} STATIC ${SOURCE_FILES}) + +set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARY}) + +link_directories(${CMAKE_CURRENT_BINARY_DIR}) +set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARIES} PARENT_SCOPE) diff --git a/extern/BSAOpt/hash.cpp b/extern/BSAOpt/hash.cpp new file mode 100644 index 000000000..029c54279 --- /dev/null +++ b/extern/BSAOpt/hash.cpp @@ -0,0 +1,108 @@ +/* Version: MPL 1.1/LGPL 3.0 + * + * "The contents of this file are subject to the Mozilla Public License + * Version 1.1 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + * License for the specific language governing rights and limitations + * under the License. + * + * The Original Code is BSAopt. + * + * The Initial Developer of the Original Code is + * Ethatron . Portions created by The Initial + * Developer are Copyright (C) 2011 The Initial Developer. + * All Rights Reserved. + * + * Alternatively, the contents of this file may be used under the terms + * of the GNU Library General Public License Version 3 license (the + * "LGPL License"), in which case the provisions of LGPL License are + * applicable instead of those above. If you wish to allow use of your + * version of this file only under the terms of the LGPL License and not + * to allow others to use your version of this file under the MPL, + * indicate your decision by deleting the provisions above and replace + * them with the notice and other provisions required by the LGPL License. + * If you do not delete the provisions above, a recipient may use your + * version of this file under either the MPL or the LGPL License." + */ + +#include +#include + +#include "hash.hpp" + +std::uint32_t GenOBHashStr(const std::string& s) { + std::uint32_t hash = 0; + + for (std::size_t i = 0; i < s.length(); i++) { + hash *= 0x1003F; + hash += (unsigned char)s[i]; + } + + return hash; +} + +std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext) { + std::uint64_t hash = 0; + + if (fle.length() > 0) { + hash = (std::uint64_t)( + (((unsigned char)fle[fle.length() - 1]) * 0x1) + + ((fle.length() > 2 ? (unsigned char)fle[fle.length() - 2] : (unsigned char)0) * 0x100) + + (fle.length() * 0x10000) + + (((unsigned char)fle[0]) * 0x1000000) + ); + + if (fle.length() > 3) { + hash += (std::uint64_t)(GenOBHashStr(fle.substr(1, fle.length() - 3)) * 0x100000000); + } + } + + if (ext.length() > 0) { + hash += (std::uint64_t)(GenOBHashStr(ext) * 0x100000000LL); + + unsigned char i = 0; + if (ext == ".nif") i = 1; + if (ext == ".kf" ) i = 2; + if (ext == ".dds") i = 3; + if (ext == ".wav") i = 4; + + if (i != 0) { + unsigned char a = (unsigned char)(((i & 0xfc ) << 5) + (unsigned char)((hash & 0xff000000) >> 24)); + unsigned char b = (unsigned char)(((i & 0xfe ) << 6) + (unsigned char)( hash & 0x000000ff) ); + unsigned char c = (unsigned char)(( i << 7) + (unsigned char)((hash & 0x0000ff00) >> 8)); + + hash -= hash & 0xFF00FFFF; + hash += (std::uint32_t)((a << 24) + b + (c << 8)); + } + } + + return hash; +} + +std::uint64_t GenOBHash(const std::string& path, std::string& file) { + std::transform(file.begin(), file.end(), file.begin(), ::tolower); + std::replace(file.begin(), file.end(), '/', '\\'); + + std::string fle; + std::string ext; + + const char *_fle = file.data(); + const char *_ext = strrchr(_fle, '.'); + if (_ext) { + ext = file.substr((0 + _ext) - _fle); + fle = file.substr(0, ( _ext) - _fle); + } + else { + ext = ""; + fle = file; + } + + if (path.length() && fle.length()) + return GenOBHashPair(path + "\\" + fle, ext); + else + return GenOBHashPair(path + fle, ext); +} diff --git a/extern/BSAOpt/hash.hpp b/extern/BSAOpt/hash.hpp new file mode 100644 index 000000000..cd936530a --- /dev/null +++ b/extern/BSAOpt/hash.hpp @@ -0,0 +1,42 @@ +/* Version: MPL 1.1/LGPL 3.0 + * + * "The contents of this file are subject to the Mozilla Public License + * Version 1.1 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + * License for the specific language governing rights and limitations + * under the License. + * + * The Original Code is BSAopt. + * + * The Initial Developer of the Original Code is + * Ethatron . Portions created by The Initial + * Developer are Copyright (C) 2011 The Initial Developer. + * All Rights Reserved. + * + * Alternatively, the contents of this file may be used under the terms + * of the GNU Library General Public License Version 3 license (the + * "LGPL License"), in which case the provisions of LGPL License are + * applicable instead of those above. If you wish to allow use of your + * version of this file only under the terms of the LGPL License and not + * to allow others to use your version of this file under the MPL, + * indicate your decision by deleting the provisions above and replace + * them with the notice and other provisions required by the LGPL License. + * If you do not delete the provisions above, a recipient may use your + * version of this file under either the MPL or the LGPL License." + */ +#ifndef BSAOPT_HASH_H +#define BSAOPT_HASH_H + +#include + +std::uint32_t GenOBHashStr(const std::string& s); + +std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext); + +std::uint64_t GenOBHash(const std::string& path, std::string& file); + +#endif // BSAOPT_HASH_H diff --git a/plugins/mygui_resource_plugin/CMakeLists.txt b/plugins/mygui_resource_plugin/CMakeLists.txt index 72965c917..6ffa5bc90 100644 --- a/plugins/mygui_resource_plugin/CMakeLists.txt +++ b/plugins/mygui_resource_plugin/CMakeLists.txt @@ -28,5 +28,7 @@ set_target_properties(${MYGUI_RESOURCE_PLUGIN_LIBRARY} PROPERTIES PREFIX "") target_link_libraries(${MYGUI_RESOURCE_PLUGIN_LIBRARY} ${OGRE_LIBRARIES} ${MYGUI_LIBRARIES} + ${BSAOPTHASH_LIBRARIES} # components/bsa + ${ZLIB_LIBRARIES} # components/bsa components )