mirror/openmw-tes3mp
1
0
Fork 1
mirror of https://github.com/TES3MP/openmw-tes3mp.git synced 2025-01-16 19:19:56 +00:00
Code Issues Releases Wiki Activity

BSA enhancements.

Browse source 
* Implement hash based lookup for TES3 BSA files.
* Added TES4/TES5 BSA support.
* Implemented a hack (non-portable code) in an attempt to reduce startup time under Windows because  Boost::filesystem seems to take forever on GetFileAttributeW.  This implementation uses FindFirstFile/FindNextFile/FindClose instead.
This commit is contained in:
cc9cii 2018-10-06 13:35:51 +10:00
parent 4cd4cf8479
commit 3982573035
21 changed files with 917 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
CMakeLists.txt
View file

@ -134,6 +134,9 @@ if(USE_SYSTEM_TINYXML)
endif()
endif()
# TES4
find_package(ZLIB REQUIRED)
# Platform specific
if (WIN32)
if(NOT MINGW)
@ -257,9 +260,10 @@ include_directories("." ${LIBS_DIR}
${MYGUI_PLATFORM_INCLUDE_DIRS}
${OPENAL_INCLUDE_DIR}
${BULLET_INCLUDE_DIRS}
${ZLIB_INCLUDE_DIRS}
)
link_directories(${SDL2_LIBRARY_DIRS} ${Boost_LIBRARY_DIRS} ${OGRE_LIB_DIR} ${MYGUI_LIB_DIR})
link_directories(${SDL2_LIBRARY_DIRS} ${Boost_LIBRARY_DIRS} ${OGRE_LIB_DIR} ${MYGUI_LIB_DIR} ${ZLIB_LIB_DIR})
if(MYGUI_STATIC)
add_definitions(-DMYGUI_STATIC)
@ -577,6 +581,7 @@ add_subdirectory (extern/ogre-ffmpeg-videoplayer)
add_subdirectory (extern/oics)
add_subdirectory (extern/sdl4ogre)
add_subdirectory (extern/murmurhash)
add_subdirectory (extern/BSAOpt)
# Components
add_subdirectory (components)

2
apps/opencs/CMakeLists.txt
View file

@ -211,7 +211,9 @@ target_link_libraries(openmw-cs
${OGRE_Overlay_LIBRARIES}
${OGRE_STATIC_PLUGINS}
${SHINY_LIBRARIES}
${ZLIB_LIBRARY}
${MURMURHASH_LIBRARIES}
${BSAOPTHASH_LIBRARIES}
${Boost_SYSTEM_LIBRARY}
${Boost_FILESYSTEM_LIBRARY}
${Boost_PROGRAM_OPTIONS_LIBRARY}

17
apps/opencs/editor.cpp
View file

@ -30,6 +30,7 @@ CS::Editor::Editor (OgreInit::OgreInit& ogreInit)
mIpcServerName ("org.openmw.OpenCS"), mServer(NULL), mClientSocket(NULL)
{
std::pair<Files::PathContainer, std::vector<std::string> > config = readConfig();
std::vector<std::string> tes4config = readTES4Config();
setupDataFiles (config.first);
@ -44,6 +45,9 @@ CS::Editor::Editor (OgreInit::OgreInit& ogreInit)
Bsa::registerResources (Files::Collections (config.first, !mFsStrict), config.second, true,
mFsStrict);
// useLooseFiles is set false, since it is already done above
Bsa::registerResources (Files::Collections (config.first, !mFsStrict), tes4config, /*useLooseFiles*/false,
mFsStrict, /*isTes4*/true);
mDocumentManager.listResources();
@ -182,6 +186,19 @@ std::pair<Files::PathContainer, std::vector<std::string> > CS::Editor::readConfi
return std::make_pair (canonicalPaths, variables["fallback-archive"].as<std::vector<std::string> >());
}
std::vector<std::string> CS::Editor::readTES4Config()
{
boost::program_options::variables_map variables;
boost::program_options::options_description desc("Syntax: openmw-cs <options>\nAllowed options");
desc.add_options()
("fallback-tes4archive", boost::program_options::value<std::vector<std::string> >()->
default_value(std::vector<std::string>(), "fallback-tes4archive")->multitoken());
mCfgMgr.readConfiguration(variables, desc, /*quiet*/true);
return variables["fallback-tes4archive"].as<std::vector<std::string> >();
}
void CS::Editor::createGame()
{
mStartup.hide();

1
apps/opencs/editor.hpp
View file

@ -73,6 +73,7 @@ namespace CS
void setupDataFiles (const Files::PathContainer& dataDirs);
std::pair<Files::PathContainer, std::vector<std::string> > readConfig(bool quiet=false);
std::vector<std::string> readTES4Config();
///< \return data paths
// not implemented

2
apps/openmw/CMakeLists.txt
View file

@ -117,6 +117,8 @@ target_link_libraries(openmw
${OGRE_LIBRARIES}
${OGRE_STATIC_PLUGINS}
${SHINY_LIBRARIES}
${BSAOPTHASH_LIBRARIES}
${ZLIB_LIBRARY}
${OPENAL_LIBRARY}
${SOUND_INPUT_LIBRARY}
${BULLET_LIBRARIES}

6
apps/openmw/engine.cpp
View file

@ -252,6 +252,10 @@ void OMW::Engine::addArchive (const std::string& archive) {
mArchives.push_back(archive);
}
void OMW::Engine::addTES4Archive (const std::string& archive) {
mTES4Archives.push_back(archive);
}
// Set resource dir
void OMW::Engine::setResourceDir (const boost::filesystem::path& parResDir)
{
@ -365,6 +369,8 @@ void OMW::Engine::prepareEngine (Settings::Manager & settings)
mOgre->createWindow("OpenMW", windowSettings);
Bsa::registerResources (mFileCollections, mArchives, true, mFSStrict);
// useLooseFiles is set false, since it is already done above
Bsa::registerResources (mFileCollections, mTES4Archives, /*useLooseFiles*/false, mFSStrict, /*isTes4*/true);
// Create input and UI first to set up a bootstrapping environment for
// showing a loading screen and keeping the window responsive while doing so

2
apps/openmw/engine.hpp
View file

@ -67,6 +67,7 @@ namespace OMW
ToUTF8::Utf8Encoder* mEncoder;
Files::PathContainer mDataDirs;
std::vector<std::string> mArchives;
std::vector<std::string> mTES4Archives;
boost::filesystem::path mResDir;
OEngine::Render::OgreRenderer *mOgre;
std::string mCellName;
@ -137,6 +138,7 @@ namespace OMW
/// Add BSA archive
void addArchive(const std::string& archive);
void addTES4Archive(const std::string& archive);
/// Set resource dir
void setResourceDir(const boost::filesystem::path& parResDir);

9
apps/openmw/main.cpp
View file

@ -112,6 +112,9 @@ bool parseOptions (int argc, char** argv, OMW::Engine& engine, Files::Configurat
("fallback-archive", bpo::value<StringsVector>()->default_value(StringsVector(), "fallback-archive")
->multitoken(), "set fallback BSA archives (later archives have higher priority)")
("fallback-tes4archive", bpo::value<StringsVector>()->default_value(StringsVector(), "fallback-tes4archive")
->multitoken(), "set fallback TES4 BSA archives (later archives have higher priority)")
("resources", bpo::value<std::string>()->default_value("resources"),
"set resources directory")
@ -240,6 +243,12 @@ bool parseOptions (int argc, char** argv, OMW::Engine& engine, Files::Configurat
engine.addArchive(*it);
}
StringsVector tes4archives = variables["fallback-tes4archive"].as<StringsVector>();
for (StringsVector::const_iterator it = tes4archives.begin(); it != tes4archives.end(); ++it)
{
engine.addTES4Archive(*it);
}
engine.setResourceDir(variables["resources"].as<std::string>());
StringsVector content = variables["content"].as<StringsVector>();

2
components/CMakeLists.txt
View file

@ -31,7 +31,7 @@ add_component_dir (nifoverrides
)
add_component_dir (bsa
bsa_archive bsa_file resources
bsa_archive bsa_file resources tes4bsa_file
)
add_component_dir (nif

195
components/bsa/bsa_archive.cpp
View file

@ -23,7 +23,10 @@
#include "bsa_archive.hpp"
#include <map>
#include <boost/filesystem.hpp>
#include <boost/algorithm/string.hpp>
#include <OgreFileSystem.h>
#include <OgreArchive.h>
@ -41,10 +44,92 @@
#define OGRE_CONST
#endif
#include <extern/BSAOpt/hash.hpp>
#include "bsa_file.hpp"
#include "tes4bsa_file.hpp"
#include "../files/constrainedfiledatastream.hpp"
namespace
{
// Concepts from answer by Remy Lebeau
// https://stackoverflow.com/questions/15068475/recursive-hard-disk-search-with-findfirstfile-findnextfile-c
//
// Also see https://msdn.microsoft.com/en-us/library/aa365200%28VS.85%29.aspx
//
// From 34.5 sec down to 18.5 sec on laptop with many of the data files on an external USB drive
#if defined _WIN32 || defined _WIN64
#include <windows.h>
#include <memory> // auto_ptr
// FIXME: not tested unicode path and filenames
DWORD indexFiles(const std::string& rootDir, const std::string& subdir,
std::map<std::uint64_t, std::string>& files, std::map<std::string, std::string>& index)
{
HANDLE hFind = INVALID_HANDLE_VALUE;
WIN32_FIND_DATA ffd;
std::string path = rootDir + ((subdir == "") ? "" : "\\" +subdir);
hFind = FindFirstFile((path + "\\*").c_str(), &ffd);
if (INVALID_HANDLE_VALUE == hFind)
return ERROR_INVALID_HANDLE;
std::auto_ptr<std::vector<std::string> > subDirs;
std::string filename;
do
{
filename = std::string(ffd.cFileName);
if (ffd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
{
if (filename != "." && filename != "..")
{
if (subDirs.get() == nullptr)
subDirs.reset(new std::vector<std::string>);
subDirs->push_back(filename);
}
}
else
{
std::uint64_t folderHash = GenOBHash(subdir, filename);
std::map<std::uint64_t, std::string>::iterator iter = files.find(folderHash);
if (iter != files.end())
throw std::runtime_error ("duplicate hash found");
files[folderHash] = path + "\\" + filename;
std::string entry = ((subdir == "") ? "" : subdir + "\\") + filename;
std::replace(entry.begin(), entry.end(), '\\', '/');
index.insert(std::make_pair (entry, path + "/" + filename));
}
} while (FindNextFile(hFind, &ffd) != 0);
FindClose(hFind);
DWORD dwError = GetLastError();
if (dwError != ERROR_NO_MORE_FILES)
return dwError;
if (subDirs.get() != nullptr)
{
for (size_t i = 0; i < subDirs->size(); ++i)
{
std::string dir = subDirs->at(i);
boost::algorithm::to_lower(dir);
// FIXME: ignoring errors for now
dwError = indexFiles(rootDir, ((subdir == "") ? "" : subdir + "\\") + dir, files, index);
}
}
return 0;
}
#endif
}
using namespace Ogre;
static bool fsstrict = false;
@ -78,6 +163,8 @@ class DirArchive: public Ogre::Archive
index mIndex;
std::map <std::uint64_t, std::string> mFiles;
index::const_iterator lookup_filename (std::string const & filename) const
{
std::string normalized = normalize_path (filename.begin (), filename.end ());
@ -89,6 +176,10 @@ public:
DirArchive(const String& name)
: Archive(name, "Dir")
{
#if defined _WIN32 || defined _WIN64
indexFiles(name, "", mFiles, mIndex);
#else
typedef boost::filesystem::recursive_directory_iterator directory_iterator;
directory_iterator end;
@ -109,6 +200,7 @@ public:
mIndex.insert (std::make_pair (searchable, proper));
}
#endif
}
bool isCaseSensitive() const { return fsstrict; }
@ -119,6 +211,27 @@ public:
virtual DataStreamPtr open(const String& filename, bool readonly = true) OGRE_CONST
{
#if defined _WIN32 || defined _WIN64
boost::filesystem::path p(filename);
std::string file = p.filename().string();
p.remove_filename();
std::string dir = p.string();
boost::algorithm::to_lower(dir);
std::replace(dir.begin(), dir.end(), '/', '\\');
std::uint64_t hash = GenOBHash(dir, file);
std::map<std::uint64_t, std::string>::const_iterator it = mFiles.find(hash);
if (it == mFiles.end())
{
std::ostringstream os;
os << "The file '" << filename << "' could not be found.";
throw std::runtime_error (os.str());
}
return openConstrainedFileDataStream (it->second.c_str());
#else
index::const_iterator i = lookup_filename (filename);
if (i == mIndex.end ())
@ -129,6 +242,7 @@ public:
}
return openConstrainedFileDataStream (i->second.c_str ());
#endif
}
StringVectorPtr list(bool recursive = true, bool dirs = false)
@ -157,7 +271,25 @@ public:
bool exists(const String& filename)
{
#if defined _WIN32 || defined _WIN64
boost::filesystem::path p(filename);
std::string file = p.filename().string();
p.remove_filename();
std::string dir = p.string();
boost::algorithm::to_lower(dir);
std::replace(dir.begin(), dir.end(), '/', '\\');
std::uint64_t hash = GenOBHash(dir, file);
std::map<std::uint64_t, std::string>::const_iterator it = mFiles.find(hash);
if (it == mFiles.end())
return false;
return true;
#else
return lookup_filename(filename) != mIndex.end ();
#endif
}
time_t getModifiedTime(const String&) { return 0; }
@ -223,6 +355,8 @@ public:
: Archive(name, "BSA")
{ arc.open(name); }
BSAArchive(const String& name, const std::string& type) : Archive(name, type) {}
bool isCaseSensitive() const { return false; }
// The archive is loaded in the constructor, and never unloaded.
@ -241,7 +375,7 @@ public:
return narc->getFile(filename.c_str());
}
bool exists(const String& filename) {
virtual bool exists(const String& filename) {
return arc.exists(filename.c_str());
}
@ -259,7 +393,7 @@ public:
return findFileInfo ("*", recursive, dirs);
}
StringVectorPtr find(const String& pattern, bool recursive = true,
virtual StringVectorPtr find(const String& pattern, bool recursive = true,
bool dirs = false)
{
std::string normalizedPattern = normalize_path(pattern.begin(), pattern.end());
@ -306,6 +440,24 @@ public:
}
};
class TES4BSAArchive : public BSAArchive
{
Bsa::TES4BSAFile arc;
public:
TES4BSAArchive::TES4BSAArchive(const String& name) : BSAArchive(name, "TES4BSA") { arc.open(name); }
virtual DataStreamPtr open(const String& filename, bool readonly = true)
{
return arc.getFile(filename);
}
virtual bool exists(const String& filename)
{
return arc.exists(filename);
}
};
// An archive factory for BSA archives
class BSAArchiveFactory : public ArchiveFactory
{
@ -351,9 +503,32 @@ public:
void destroyInstance( Archive* arch) { delete arch; }
};
class TES4BSAArchiveFactory : public ArchiveFactory
{
public:
const String& getType() const
{
static String name = "TES4BSA";
return name;
}
Archive *createInstance( const String& name )
{
return new TES4BSAArchive(name);
}
virtual Archive* createInstance(const String& name, bool readOnly)
{
return new TES4BSAArchive(name);
}
void destroyInstance( Archive* arch) { delete arch; }
};
static bool init = false;
static bool init2 = false;
static bool init3 = false;
static void insertBSAFactory()
{
@ -364,6 +539,15 @@ static void insertBSAFactory()
}
}
static void insertTES4BSAFactory()
{
if(!init3)
{
ArchiveManager::getSingleton().addArchiveFactory( new TES4BSAArchiveFactory );
init3 = true;
}
}
static void insertDirFactory()
{
if(!init2)
@ -386,6 +570,13 @@ void addBSA(const std::string& name, const std::string& group)
addResourceLocation(name, "BSA", group, true);
}
void addTES4BSA(const std::string& name, const std::string& group)
{
insertTES4BSAFactory();
ResourceGroupManager::getSingleton().
addResourceLocation(name, "TES4BSA", group, true);
}
void addDir(const std::string& name, const bool& fs, const std::string& group)
{
fsstrict = fs;

1
components/bsa/bsa_archive.hpp
View file

@ -33,6 +33,7 @@ namespace Bsa
/// Add the given BSA file as an input archive in the Ogre resource
/// system.
void addBSA(const std::string& file, const std::string& group="General");
void addTES4BSA(const std::string& file, const std::string& group="General");
void addDir(const std::string& file, const bool& fs, const std::string& group="General");
}

69
components/bsa/bsa_file.cpp
View file

@ -27,9 +27,41 @@
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/algorithm/string.hpp>
#include "../files/constrainedfiledatastream.hpp"
namespace
{
// see: http://en.uesp.net/wiki/Tes3Mod:BSA_File_Format
std::uint64_t getHash(const char *name)
{
unsigned int len = (unsigned int)strlen(name);
std::uint64_t hash;
unsigned l = (len>>1);
unsigned sum, off, temp, i, n, hash1;
for(sum = off = i = 0; i < l; i++) {
sum ^= (((unsigned)(name[i]))<<(off&0x1F));
off += 8;
}
hash1 = sum;
for(sum = off = 0; i < len; i++) {
temp = (((unsigned)(name[i]))<<(off&0x1F));
sum ^= temp;
n = temp & 0x1F;
sum = (sum << (32-n)) | (sum >> n); // binary "rotate right"
off += 8;
}
hash = sum;
hash <<= 32;
hash += hash1;
return hash;
}
}
using namespace std;
using namespace Bsa;
@ -143,7 +175,14 @@ void BSAFile::readHeader()
fail("Archive contains offsets outside itself");
// Add the file name to the lookup
lookup[fs.name] = i;
//lookup[fs.name] = i;
}
std::uint64_t hash;
for (size_t i = 0; i < filenum; ++i)
{
input.read(reinterpret_cast<char*>(&hash), 8);
mFiles[hash] = files[i];
}
isLoaded = true;
@ -152,13 +191,14 @@ void BSAFile::readHeader()
/// Get the index of a given file name, or -1 if not found
int BSAFile::getIndex(const char *str) const
{
Lookup::const_iterator it = lookup.find(str);
if(it == lookup.end())
std::string name(str);
boost::algorithm::to_lower(name);
std::uint64_t hash = getHash(name.c_str());
std::map<std::uint64_t, FileStruct>::const_iterator iter = mFiles.find(hash);
if (iter != mFiles.end())
return 0; // NOTE: this is a bit of a hack, exists() only checks for '-1'
else
return -1;
int res = it->second;
assert(res >= 0 && (size_t)res < files.size());
return res;
}
/// Open an archive file.
@ -171,10 +211,15 @@ void BSAFile::open(const string &file)
Ogre::DataStreamPtr BSAFile::getFile(const char *file)
{
assert(file);
int i = getIndex(file);
if(i == -1)
fail("File not found: " + string(file));
const FileStruct &fs = files[i];
return openConstrainedFileDataStream (filename.c_str (), fs.offset, fs.fileSize);
std::string name(file);
boost::algorithm::to_lower(name);
std::uint64_t hash = getHash(name.c_str());
std::map<std::uint64_t, FileStruct>::const_iterator it = mFiles.find(hash);
if (it != mFiles.end())
{
const FileStruct &fs = it->second;
return openConstrainedFileDataStream (filename.c_str (), fs.offset, fs.fileSize);
}
fail("File not found: " + string(file));
}

2
components/bsa/bsa_file.hpp
View file

@ -82,6 +82,8 @@ private:
typedef std::map<const char*, int, iltstr> Lookup;
Lookup lookup;
std::map<std::uint64_t, FileStruct> mFiles;
/// Error handling
void fail(const std::string &msg);

9
components/bsa/resources.cpp
View file

@ -9,7 +9,7 @@
#include "bsa_archive.hpp"
void Bsa::registerResources (const Files::Collections& collections,
const std::vector<std::string>& archives, bool useLooseFiles, bool fsStrict)
const std::vector<std::string>& archives, bool useLooseFiles, bool fsStrict, bool isTes4)
{
const Files::PathContainer& dataDirs = collections.getPaths();
@ -34,13 +34,16 @@ void Bsa::registerResources (const Files::Collections& collections,
if (collections.doesExist(*archive))
{
// Last BSA has the highest priority
std::string groupName = "DataBSA" + Ogre::StringConverter::toString(archives.size()-i, 8, '0');
std::string groupName = (isTes4 ? "TES4BSA" : "DataBSA") + Ogre::StringConverter::toString(archives.size()-i, 8, '0');
Ogre::ResourceGroupManager::getSingleton ().createResourceGroup (groupName);
const std::string archivePath = collections.getPath(*archive).string();
std::cout << "Adding BSA archive " << archivePath << std::endl;
Bsa::addBSA(archivePath, groupName);
if (!isTes4)
Bsa::addBSA(archivePath, groupName);
else
Bsa::addTES4BSA(archivePath, groupName);
++i;
}
else

2
components/bsa/resources.hpp
View file

@ -9,7 +9,7 @@
namespace Bsa
{
void registerResources (const Files::Collections& collections,
const std::vector<std::string>& archives, bool useLooseFiles, bool fsStrict);
const std::vector<std::string>& archives, bool useLooseFiles, bool fsStrict, bool isTes4=false);
///< Register resources directories and archives as OGRE resources groups
}

339
components/bsa/tes4bsa_file.cpp Normal file
View file

@ -0,0 +1,339 @@
/*
OpenMW - The completely unofficial reimplementation of Morrowind
Copyright (C) 2008-2010 Nicolay Korslund
Email: < korslund@gmail.com >
WWW: http://openmw.sourceforge.net/
This file (bsa_file.cpp) is part of the OpenMW package.
OpenMW is distributed as free software: you can redistribute it
and/or modify it under the terms of the GNU General Public License
version 3, as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
version 3 along with this program. If not, see
http://www.gnu.org/licenses/ .
TES4 stuff added by cc9cii 2018
*/
#include "tes4bsa_file.hpp"
#include <stdexcept>
#include <cassert>
#include <boost/scoped_array.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/fstream.hpp>
#include <zlib.h>
#include <extern/BSAOpt/hash.hpp> // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation
#undef TEST_UNIQUE_HASH
namespace
{
void getBZString(std::string& str, boost::filesystem::ifstream& filestream)
{
char size = 0;
filestream.read(&size, 1);
boost::scoped_array<char> buf(new char[size]);
filestream.read(buf.get(), size);
if (buf[size - 1] != 0)
str.assign(buf.get(), size);
else
str.assign(buf.get(), size - 1); // don't copy null terminator
assert((size_t)size-1 == str.size() && "getBZString string size mismatch");
return;
}
}
using namespace Bsa;
/// Error handling
void TES4BSAFile::fail(const std::string& msg)
{
throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename);
}
/// Read header information from the input source
void TES4BSAFile::readHeader()
{
assert(!isLoaded);
namespace bfs = boost::filesystem;
bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary);
// Total archive size
std::streamoff fsize = 0;
if(input.seekg(0, std::ios_base::end))
{
fsize = input.tellg();
input.seekg(0);
}
if(fsize < 36) // header is 36 bytes
fail("File too small to be a valid BSA archive");
// Get essential header numbers
//size_t dirsize, filenum;
std::uint32_t archiveFlags, folderCount, fileCount, totalFileNameLength;
{
// First 36 bytes
std::uint32_t header[9];
input.read(reinterpret_cast<char*>(header), 36);
if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/))
fail("Unrecognized TES4 BSA header");
// header[2] is offset, should be 36 = 0x24 which is the size of the header
// Oblivion - Meshes.bsa
//
// 0111 1000 0111 = 0x0787
// ^^^ ^ ^^^
// ||| | ||+-- has names for dirs (mandatory?)
// ||| | |+--- has names for files (mandatory?)
// ||| | +---- files are compressed by default
// ||| |
// ||| +---------- unknown (TES5: retain strings during startup)
// ||+------------ unknown (TES5: embedded file names)
// |+------------- unknown
// +-------------- unknown
//
archiveFlags = header[3];
folderCount = header[4];
fileCount = header[5];
//totalFolderNameLength = header[6];
totalFileNameLength = header[7];
//fileFlags = header[8]; // FIXME: an opportunity to optimize here
mCompressedByDefault = (archiveFlags & 0x4) != 0;
mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0;
}
// TODO: more checks for BSA file corruption
// folder records
std::uint64_t hash;
FolderRecord fr;
for (std::uint32_t i = 0; i < folderCount; ++i)
{
input.read(reinterpret_cast<char*>(&hash), 8);
input.read(reinterpret_cast<char*>(&fr.count), 4); // not sure purpose of count
input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
std::map<std::uint64_t, FolderRecord>::const_iterator lb = mFolders.lower_bound(hash);
if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first)))
fail("Archive found duplicate folder name hash");
else
mFolders.insert(lb, std::pair<std::uint64_t, FolderRecord>(hash, fr));
}
// file record blocks
std::uint64_t fileHash;
FileRecord file;
std::string folder("");
std::uint64_t folderHash;
if ((archiveFlags & 0x1) == 0)
folderCount = 1; // TODO: not tested
for (std::uint32_t i = 0; i < folderCount; ++i)
{
if ((archiveFlags & 0x1) != 0)
getBZString(folder, input);
folderHash = GenOBHash(folder, std::string(""));
std::map<std::uint64_t, FolderRecord>::iterator iter = mFolders.find(folderHash);
if (iter == mFolders.end())
fail("Archive folder name hash not found");
for (std::uint32_t j = 0; j < iter->second.count; ++j)
{
input.read(reinterpret_cast<char*>(&fileHash), 8);
input.read(reinterpret_cast<char*>(&file.size), 4);
input.read(reinterpret_cast<char*>(&file.offset), 4);
std::map<std::uint64_t, FileRecord>::const_iterator lb = iter->second.files.lower_bound(fileHash);
if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first)))
fail("Archive found duplicate file name hash");
iter->second.files.insert(lb, std::pair<std::uint64_t, FileRecord>(fileHash, file));
}
}
// file record blocks
if ((archiveFlags & 0x2) != 0)
{
mStringBuf.resize(totalFileNameLength);
input.read(&mStringBuf[0], mStringBuf.size()); // TODO: maybe useful in building a lookup map?
}
// TODO: more checks for BSA file corruption
isLoaded = true;
}
TES4BSAFile::FileRecord TES4BSAFile::getFileRecord(const std::string& str) const
{
boost::filesystem::path p(str);
std::string stem = p.stem().string();
std::string ext = p.extension().string();
std::string filename = p.filename().string();
p.remove_filename();
std::string folder = p.string();
// GenOBHash already converts to lowercase and replaces file separators but not for path
boost::algorithm::to_lower(folder);
std::replace(folder.begin(), folder.end(), '/', '\\');
std::uint64_t folderHash = GenOBHash(folder, std::string(""));
std::map<std::uint64_t, FolderRecord>::const_iterator it = mFolders.find(folderHash);
if (it == mFolders.end())
return FileRecord(); // folder not found, return default which has offset of -1
boost::algorithm::to_lower(stem);
boost::algorithm::to_lower(ext);
std::uint64_t fileHash = GenOBHashPair(stem, ext);
std::map<std::uint64_t, FileRecord>::const_iterator iter = it->second.files.find(fileHash);
if (iter == it->second.files.end())
return FileRecord(); // file not found, return default which has offset of -1
// cache for next time
std::uint64_t hash = GenOBHash(folder, filename);
#if defined (TEST_UNIQUE_HASH)
FileList::const_iterator lb = mFiles.lower_bound(hash);
if (lb != mFiles.end() && !(mFiles.key_comp()(hash, lb->first)))
{
// found, check if same filename
if (lb->second.fileName == str)
return iter->second; // same name, should not have got here!!
else
{
// different filename, hash is not unique!
std::cerr << "BSA hash collision: " << str << std::hex << "0x" << hash << std::endl;
return iter->second; // return without cashing
}
}
// not found, cache for later
const_cast<FileList&>(mFiles).insert(lb, std::pair<std::uint64_t, FileRecord>(hash, iter->second));
const_cast<FileList&>(mFiles)[hash].fileName = str;
#else
const_cast<FileList&>(mFiles)[hash] = iter->second; // NOTE: const hack
#endif
return iter->second;
}
bool TES4BSAFile::exists(const std::string& str) const
{
// check cache first
boost::filesystem::path p(str);
std::string filename = p.filename().string();
p.remove_filename();
std::string folder = p.string();
// GenOBHash already converts to lowercase and replaces file separators but not for path
boost::algorithm::to_lower(folder);
std::replace(folder.begin(), folder.end(), '/', '\\');
std::uint64_t hash = GenOBHash(folder, filename);
std::map<std::uint64_t, FileRecord>::const_iterator it = mFiles.find(hash);
#if defined (TEST_UNIQUE_HASH)
if (it != mFiles.end() && it->second.fileName == str)
#else
if (it != mFiles.end())
#endif
return true;
else
return getFileRecord(str).offset != -1;
}
void TES4BSAFile::open(const std::string& file)
{
mFilename = file;
readHeader();
}
Ogre::DataStreamPtr TES4BSAFile::getFile(const std::string& file)
{
assert(file);
FileRecord fileRec = getFileRecord(file);
if(fileRec.offset == -1)
fail("File not found: " + std::string(file));
boost::filesystem::ifstream input(boost::filesystem::path(mFilename), std::ios_base::binary);
input.seekg(fileRec.offset);
std::string fullPath;
if (mEmbeddedFileNames)
getBZString(fullPath, input); // TODO: maybe cache the hash and/or offset of frequently used ones?
if (( mCompressedByDefault && (fileRec.size & (1<<30)) == 0)
||
(!mCompressedByDefault && (fileRec.size & (1<<30)) != 0))
{
std::uint32_t bufSize = 0;
boost::scoped_array<unsigned char> inBuf;
inBuf.reset(new unsigned char[fileRec.size-4]);
input.read(reinterpret_cast<char*>(&bufSize), 4);
input.read(reinterpret_cast<char*>(inBuf.get()), fileRec.size-4);
Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(bufSize);
Ogre::SharedPtr<Ogre::DataStream> streamPtr(outBuf);
int ret;
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = bufSize;
strm.next_in = inBuf.get();
ret = inflateInit(&strm);
if (ret != Z_OK)
throw std::runtime_error("TES4BSAFile::getFile - inflateInit failed");
strm.avail_out = bufSize;
strm.next_out = outBuf->getPtr();
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR && "TES4BSAFile::getFile - inflate - state clobbered");
switch (ret)
{
case Z_NEED_DICT:
ret = Z_DATA_ERROR; /* and fall through */
case Z_DATA_ERROR:
case Z_MEM_ERROR:
inflateEnd(&strm);
throw std::runtime_error("TES4BSAFile::getFile - inflate failed");
}
assert(ret == Z_OK || ret == Z_STREAM_END);
inflateEnd(&strm);
return streamPtr;
}
else // not compressed TODO: not tested
{
Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(fileRec.size);
Ogre::SharedPtr<Ogre::DataStream> streamPtr(outBuf);
input.read(reinterpret_cast<char*>(outBuf->getPtr()), fileRec.size);
return streamPtr;
}
}

103
components/bsa/tes4bsa_file.hpp Normal file
View file

@ -0,0 +1,103 @@
/*
OpenMW - The completely unofficial reimplementation of Morrowind
Copyright (C) 2008-2010 Nicolay Korslund
Email: < korslund@gmail.com >
WWW: http://openmw.sourceforge.net/
This file (bsa_file.h) is part of the OpenMW package.
OpenMW is distributed as free software: you can redistribute it
and/or modify it under the terms of the GNU General Public License
version 3, as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
version 3 along with this program. If not, see
http://www.gnu.org/licenses/ .
TES4 stuff added by cc9cii 2018
*/
#ifndef BSA_TES4BSA_FILE_H
#define BSA_TES4BSA_FILE_H
#include <stdint.h>
#include <string>
#include <vector>
#include <map>
#include <OgreDataStream.h>
namespace Bsa
{
class TES4BSAFile
{
public:
struct FileRecord
{
std::uint32_t size;
std::uint32_t offset;
std::string fileName; // NOTE: for testing hash collision only, see TEST_UNIQUE_HASH
FileRecord() : size(0), offset(-1) {}
};
private:
/// Filenames string buffer
std::vector<char> mStringBuf;
/// True when an archive has been loaded
bool isLoaded;
bool mCompressedByDefault;
bool mEmbeddedFileNames;
std::map<std::uint64_t, FileRecord> mFiles;
typedef std::map<std::uint64_t, FileRecord> FileList;
struct FolderRecord
{
std::uint32_t count;
std::uint32_t offset;
std::map<std::uint64_t, FileRecord> files;
};
std::map<std::uint64_t, FolderRecord> mFolders;
FileRecord getFileRecord(const std::string& str) const;
/// Used for error messages and getting files
std::string mFilename;
/// Error handling
void fail(const std::string &msg);
/// Read header information from the input source
void readHeader();
public:
TES4BSAFile()
: isLoaded(false), mCompressedByDefault(false), mEmbeddedFileNames(false)
{ }
/// Open an archive file.
void open(const std::string &file);
/// Check if a file exists
bool exists(const std::string& file) const;
Ogre::DataStreamPtr getFile(const std::string& file);
/// Get a list of all files
const FileList &getList() const // FIXME
{ return mFiles; }
};
}
#endif

17
extern/BSAOpt/CMakeLists.txt vendored Normal file
View file

@ -0,0 +1,17 @@
cmake_minimum_required(VERSION 2.8)
# This is NOT intended as a stand-alone build system! Instead, you should include this from the main CMakeLists of your project.
set(BSAOPTHASH_LIBRARY "bsaopthash")
# Sources
set(SOURCE_FILES
hash.cpp
)
add_library(${BSAOPTHASH_LIBRARY} STATIC ${SOURCE_FILES})
set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARY})
link_directories(${CMAKE_CURRENT_BINARY_DIR})
set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARIES} PARENT_SCOPE)

108
extern/BSAOpt/hash.cpp vendored Normal file
View file

@ -0,0 +1,108 @@
/* Version: MPL 1.1/LGPL 3.0
*
* "The contents of this file are subject to the Mozilla Public License
* Version 1.1 (the "License"); you may not use this file except in
* compliance with the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
* License for the specific language governing rights and limitations
* under the License.
*
* The Original Code is BSAopt.
*
* The Initial Developer of the Original Code is
* Ethatron <niels@paradice-insight.us>. Portions created by The Initial
* Developer are Copyright (C) 2011 The Initial Developer.
* All Rights Reserved.
*
* Alternatively, the contents of this file may be used under the terms
* of the GNU Library General Public License Version 3 license (the
* "LGPL License"), in which case the provisions of LGPL License are
* applicable instead of those above. If you wish to allow use of your
* version of this file only under the terms of the LGPL License and not
* to allow others to use your version of this file under the MPL,
* indicate your decision by deleting the provisions above and replace
* them with the notice and other provisions required by the LGPL License.
* If you do not delete the provisions above, a recipient may use your
* version of this file under either the MPL or the LGPL License."
*/
#include <cstdint>
#include <algorithm>
#include "hash.hpp"
std::uint32_t GenOBHashStr(const std::string& s) {
std::uint32_t hash = 0;
for (std::size_t i = 0; i < s.length(); i++) {
hash *= 0x1003F;
hash += (unsigned char)s[i];
}
return hash;
}
std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext) {
std::uint64_t hash = 0;
if (fle.length() > 0) {
hash = (std::uint64_t)(
(((unsigned char)fle[fle.length() - 1]) * 0x1) +
((fle.length() > 2 ? (unsigned char)fle[fle.length() - 2] : (unsigned char)0) * 0x100) +
(fle.length() * 0x10000) +
(((unsigned char)fle[0]) * 0x1000000)
);
if (fle.length() > 3) {
hash += (std::uint64_t)(GenOBHashStr(fle.substr(1, fle.length() - 3)) * 0x100000000);
}
}
if (ext.length() > 0) {
hash += (std::uint64_t)(GenOBHashStr(ext) * 0x100000000LL);
unsigned char i = 0;
if (ext == ".nif") i = 1;
if (ext == ".kf" ) i = 2;
if (ext == ".dds") i = 3;
if (ext == ".wav") i = 4;
if (i != 0) {
unsigned char a = (unsigned char)(((i & 0xfc ) << 5) + (unsigned char)((hash & 0xff000000) >> 24));
unsigned char b = (unsigned char)(((i & 0xfe ) << 6) + (unsigned char)( hash & 0x000000ff) );
unsigned char c = (unsigned char)(( i << 7) + (unsigned char)((hash & 0x0000ff00) >> 8));
hash -= hash & 0xFF00FFFF;
hash += (std::uint32_t)((a << 24) + b + (c << 8));
}
}
return hash;
}
std::uint64_t GenOBHash(const std::string& path, std::string& file) {
std::transform(file.begin(), file.end(), file.begin(), ::tolower);
std::replace(file.begin(), file.end(), '/', '\\');
std::string fle;
std::string ext;
const char *_fle = file.data();
const char *_ext = strrchr(_fle, '.');
if (_ext) {
ext = file.substr((0 + _ext) - _fle);
fle = file.substr(0, ( _ext) - _fle);
}
else {
ext = "";
fle = file;
}
if (path.length() && fle.length())
return GenOBHashPair(path + "\\" + fle, ext);
else
return GenOBHashPair(path + fle, ext);
}

42
extern/BSAOpt/hash.hpp vendored Normal file
View file

@ -0,0 +1,42 @@
/* Version: MPL 1.1/LGPL 3.0
*
* "The contents of this file are subject to the Mozilla Public License
* Version 1.1 (the "License"); you may not use this file except in
* compliance with the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
* License for the specific language governing rights and limitations
* under the License.
*
* The Original Code is BSAopt.
*
* The Initial Developer of the Original Code is
* Ethatron <niels@paradice-insight.us>. Portions created by The Initial
* Developer are Copyright (C) 2011 The Initial Developer.
* All Rights Reserved.
*
* Alternatively, the contents of this file may be used under the terms
* of the GNU Library General Public License Version 3 license (the
* "LGPL License"), in which case the provisions of LGPL License are
* applicable instead of those above. If you wish to allow use of your
* version of this file only under the terms of the LGPL License and not
* to allow others to use your version of this file under the MPL,
* indicate your decision by deleting the provisions above and replace
* them with the notice and other provisions required by the LGPL License.
* If you do not delete the provisions above, a recipient may use your
* version of this file under either the MPL or the LGPL License."
*/
#ifndef BSAOPT_HASH_H
#define BSAOPT_HASH_H
#include <string>
std::uint32_t GenOBHashStr(const std::string& s);
std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext);
std::uint64_t GenOBHash(const std::string& path, std::string& file);
#endif // BSAOPT_HASH_H

2
plugins/mygui_resource_plugin/CMakeLists.txt
View file

@ -28,5 +28,7 @@ set_target_properties(${MYGUI_RESOURCE_PLUGIN_LIBRARY} PROPERTIES PREFIX "")
target_link_libraries(${MYGUI_RESOURCE_PLUGIN_LIBRARY}
${OGRE_LIBRARIES}
${MYGUI_LIBRARIES}
${BSAOPTHASH_LIBRARIES} # components/bsa
${ZLIB_LIBRARIES} # components/bsa
components
)