Merge branch 'cc9cii' of https://github.com/cc9cii/openmw into tes4_bsa

pull/2138/head
Azdul 6 years ago
commit a3bcd95546

@ -610,7 +610,7 @@ add_subdirectory (extern/oics)
if (BUILD_OPENCS)
add_subdirectory (extern/osgQt)
endif()
add_subdirectory (extern/bsaopthash)
add_subdirectory (extern/BSAOpt)
# Components
add_subdirectory (components)

@ -0,0 +1,339 @@
/*
OpenMW - The completely unofficial reimplementation of Morrowind
Copyright (C) 2008-2010 Nicolay Korslund
Email: < korslund@gmail.com >
WWW: http://openmw.sourceforge.net/
This file (bsa_file.cpp) is part of the OpenMW package.
OpenMW is distributed as free software: you can redistribute it
and/or modify it under the terms of the GNU General Public License
version 3, as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
version 3 along with this program. If not, see
http://www.gnu.org/licenses/ .
TES4 stuff added by cc9cii 2018
*/
#include "tes4bsa_file.hpp"
#include <stdexcept>
#include <cassert>
#include <boost/scoped_array.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/fstream.hpp>
#include <zlib.h>
#include <extern/BSAOpt/hash.hpp> // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation
#undef TEST_UNIQUE_HASH
namespace
{
void getBZString(std::string& str, boost::filesystem::ifstream& filestream)
{
char size = 0;
filestream.read(&size, 1);
boost::scoped_array<char> buf(new char[size]);
filestream.read(buf.get(), size);
if (buf[size - 1] != 0)
str.assign(buf.get(), size);
else
str.assign(buf.get(), size - 1); // don't copy null terminator
assert((size_t)size-1 == str.size() && "getBZString string size mismatch");
return;
}
}
using namespace Bsa;
/// Error handling
void TES4BSAFile::fail(const std::string& msg)
{
throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename);
}
/// Read header information from the input source
void TES4BSAFile::readHeader()
{
assert(!isLoaded);
namespace bfs = boost::filesystem;
bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary);
// Total archive size
std::streamoff fsize = 0;
if(input.seekg(0, std::ios_base::end))
{
fsize = input.tellg();
input.seekg(0);
}
if(fsize < 36) // header is 36 bytes
fail("File too small to be a valid BSA archive");
// Get essential header numbers
//size_t dirsize, filenum;
std::uint32_t archiveFlags, folderCount, fileCount, totalFileNameLength;
{
// First 36 bytes
std::uint32_t header[9];
input.read(reinterpret_cast<char*>(header), 36);
if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/))
fail("Unrecognized TES4 BSA header");
// header[2] is offset, should be 36 = 0x24 which is the size of the header
// Oblivion - Meshes.bsa
//
// 0111 1000 0111 = 0x0787
// ^^^ ^ ^^^
// ||| | ||+-- has names for dirs (mandatory?)
// ||| | |+--- has names for files (mandatory?)
// ||| | +---- files are compressed by default
// ||| |
// ||| +---------- unknown (TES5: retain strings during startup)
// ||+------------ unknown (TES5: embedded file names)
// |+------------- unknown
// +-------------- unknown
//
archiveFlags = header[3];
folderCount = header[4];
fileCount = header[5];
//totalFolderNameLength = header[6];
totalFileNameLength = header[7];
//fileFlags = header[8]; // FIXME: an opportunity to optimize here
mCompressedByDefault = (archiveFlags & 0x4) != 0;
mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0;
}
// TODO: more checks for BSA file corruption
// folder records
std::uint64_t hash;
FolderRecord fr;
for (std::uint32_t i = 0; i < folderCount; ++i)
{
input.read(reinterpret_cast<char*>(&hash), 8);
input.read(reinterpret_cast<char*>(&fr.count), 4); // not sure purpose of count
input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
std::map<std::uint64_t, FolderRecord>::const_iterator lb = mFolders.lower_bound(hash);
if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first)))
fail("Archive found duplicate folder name hash");
else
mFolders.insert(lb, std::pair<std::uint64_t, FolderRecord>(hash, fr));
}
// file record blocks
std::uint64_t fileHash;
FileRecord file;
std::string folder("");
std::uint64_t folderHash;
if ((archiveFlags & 0x1) == 0)
folderCount = 1; // TODO: not tested
for (std::uint32_t i = 0; i < folderCount; ++i)
{
if ((archiveFlags & 0x1) != 0)
getBZString(folder, input);
folderHash = GenOBHash(folder, std::string(""));
std::map<std::uint64_t, FolderRecord>::iterator iter = mFolders.find(folderHash);
if (iter == mFolders.end())
fail("Archive folder name hash not found");
for (std::uint32_t j = 0; j < iter->second.count; ++j)
{
input.read(reinterpret_cast<char*>(&fileHash), 8);
input.read(reinterpret_cast<char*>(&file.size), 4);
input.read(reinterpret_cast<char*>(&file.offset), 4);
std::map<std::uint64_t, FileRecord>::const_iterator lb = iter->second.files.lower_bound(fileHash);
if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first)))
fail("Archive found duplicate file name hash");
iter->second.files.insert(lb, std::pair<std::uint64_t, FileRecord>(fileHash, file));
}
}
// file record blocks
if ((archiveFlags & 0x2) != 0)
{
mStringBuf.resize(totalFileNameLength);
input.read(&mStringBuf[0], mStringBuf.size()); // TODO: maybe useful in building a lookup map?
}
// TODO: more checks for BSA file corruption
isLoaded = true;
}
TES4BSAFile::FileRecord TES4BSAFile::getFileRecord(const std::string& str) const
{
boost::filesystem::path p(str);
std::string stem = p.stem().string();
std::string ext = p.extension().string();
std::string filename = p.filename().string();
p.remove_filename();
std::string folder = p.string();
// GenOBHash already converts to lowercase and replaces file separators but not for path
boost::algorithm::to_lower(folder);
std::replace(folder.begin(), folder.end(), '/', '\\');
std::uint64_t folderHash = GenOBHash(folder, std::string(""));
std::map<std::uint64_t, FolderRecord>::const_iterator it = mFolders.find(folderHash);
if (it == mFolders.end())
return FileRecord(); // folder not found, return default which has offset of -1
boost::algorithm::to_lower(stem);
boost::algorithm::to_lower(ext);
std::uint64_t fileHash = GenOBHashPair(stem, ext);
std::map<std::uint64_t, FileRecord>::const_iterator iter = it->second.files.find(fileHash);
if (iter == it->second.files.end())
return FileRecord(); // file not found, return default which has offset of -1
// cache for next time
std::uint64_t hash = GenOBHash(folder, filename);
#if defined (TEST_UNIQUE_HASH)
FileList::const_iterator lb = mFiles.lower_bound(hash);
if (lb != mFiles.end() && !(mFiles.key_comp()(hash, lb->first)))
{
// found, check if same filename
if (lb->second.fileName == str)
return iter->second; // same name, should not have got here!!
else
{
// different filename, hash is not unique!
std::cerr << "BSA hash collision: " << str << std::hex << "0x" << hash << std::endl;
return iter->second; // return without cashing
}
}
// not found, cache for later
const_cast<FileList&>(mFiles).insert(lb, std::pair<std::uint64_t, FileRecord>(hash, iter->second));
const_cast<FileList&>(mFiles)[hash].fileName = str;
#else
const_cast<FileList&>(mFiles)[hash] = iter->second; // NOTE: const hack
#endif
return iter->second;
}
bool TES4BSAFile::exists(const std::string& str) const
{
// check cache first
boost::filesystem::path p(str);
std::string filename = p.filename().string();
p.remove_filename();
std::string folder = p.string();
// GenOBHash already converts to lowercase and replaces file separators but not for path
boost::algorithm::to_lower(folder);
std::replace(folder.begin(), folder.end(), '/', '\\');
std::uint64_t hash = GenOBHash(folder, filename);
std::map<std::uint64_t, FileRecord>::const_iterator it = mFiles.find(hash);
#if defined (TEST_UNIQUE_HASH)
if (it != mFiles.end() && it->second.fileName == str)
#else
if (it != mFiles.end())
#endif
return true;
else
return getFileRecord(str).offset != -1;
}
void TES4BSAFile::open(const std::string& file)
{
mFilename = file;
readHeader();
}
Ogre::DataStreamPtr TES4BSAFile::getFile(const std::string& file)
{
assert(file);
FileRecord fileRec = getFileRecord(file);
if(fileRec.offset == -1)
fail("File not found: " + std::string(file));
boost::filesystem::ifstream input(boost::filesystem::path(mFilename), std::ios_base::binary);
input.seekg(fileRec.offset);
std::string fullPath;
if (mEmbeddedFileNames)
getBZString(fullPath, input); // TODO: maybe cache the hash and/or offset of frequently used ones?
if (( mCompressedByDefault && (fileRec.size & (1<<30)) == 0)
||
(!mCompressedByDefault && (fileRec.size & (1<<30)) != 0))
{
std::uint32_t bufSize = 0;
boost::scoped_array<unsigned char> inBuf;
inBuf.reset(new unsigned char[fileRec.size-4]);
input.read(reinterpret_cast<char*>(&bufSize), 4);
input.read(reinterpret_cast<char*>(inBuf.get()), fileRec.size-4);
Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(bufSize);
Ogre::SharedPtr<Ogre::DataStream> streamPtr(outBuf);
int ret;
z_stream strm;
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = bufSize;
strm.next_in = inBuf.get();
ret = inflateInit(&strm);
if (ret != Z_OK)
throw std::runtime_error("TES4BSAFile::getFile - inflateInit failed");
strm.avail_out = bufSize;
strm.next_out = outBuf->getPtr();
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR && "TES4BSAFile::getFile - inflate - state clobbered");
switch (ret)
{
case Z_NEED_DICT:
ret = Z_DATA_ERROR; /* and fall through */
case Z_DATA_ERROR:
case Z_MEM_ERROR:
inflateEnd(&strm);
throw std::runtime_error("TES4BSAFile::getFile - inflate failed");
}
assert(ret == Z_OK || ret == Z_STREAM_END);
inflateEnd(&strm);
return streamPtr;
}
else // not compressed TODO: not tested
{
Ogre::MemoryDataStream *outBuf = new Ogre::MemoryDataStream(fileRec.size);
Ogre::SharedPtr<Ogre::DataStream> streamPtr(outBuf);
input.read(reinterpret_cast<char*>(outBuf->getPtr()), fileRec.size);
return streamPtr;
}
}

@ -0,0 +1,103 @@
/*
OpenMW - The completely unofficial reimplementation of Morrowind
Copyright (C) 2008-2010 Nicolay Korslund
Email: < korslund@gmail.com >
WWW: http://openmw.sourceforge.net/
This file (bsa_file.h) is part of the OpenMW package.
OpenMW is distributed as free software: you can redistribute it
and/or modify it under the terms of the GNU General Public License
version 3, as published by the Free Software Foundation.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public License
version 3 along with this program. If not, see
http://www.gnu.org/licenses/ .
TES4 stuff added by cc9cii 2018
*/
#ifndef BSA_TES4BSA_FILE_H
#define BSA_TES4BSA_FILE_H
#include <stdint.h>
#include <string>
#include <vector>
#include <map>
#include <OgreDataStream.h>
namespace Bsa
{
class TES4BSAFile
{
public:
struct FileRecord
{
std::uint32_t size;
std::uint32_t offset;
std::string fileName; // NOTE: for testing hash collision only, see TEST_UNIQUE_HASH
FileRecord() : size(0), offset(-1) {}
};
private:
/// Filenames string buffer
std::vector<char> mStringBuf;
/// True when an archive has been loaded
bool isLoaded;
bool mCompressedByDefault;
bool mEmbeddedFileNames;
std::map<std::uint64_t, FileRecord> mFiles;
typedef std::map<std::uint64_t, FileRecord> FileList;
struct FolderRecord
{
std::uint32_t count;
std::uint32_t offset;
std::map<std::uint64_t, FileRecord> files;
};
std::map<std::uint64_t, FolderRecord> mFolders;
FileRecord getFileRecord(const std::string& str) const;
/// Used for error messages and getting files
std::string mFilename;
/// Error handling
void fail(const std::string &msg);
/// Read header information from the input source
void readHeader();
public:
TES4BSAFile()
: isLoaded(false), mCompressedByDefault(false), mEmbeddedFileNames(false)
{ }
/// Open an archive file.
void open(const std::string &file);
/// Check if a file exists
bool exists(const std::string& file) const;
Ogre::DataStreamPtr getFile(const std::string& file);
/// Get a list of all files
const FileList &getList() const // FIXME
{ return mFiles; }
};
}
#endif

@ -0,0 +1,17 @@
cmake_minimum_required(VERSION 2.8)
# This is NOT intended as a stand-alone build system! Instead, you should include this from the main CMakeLists of your project.
set(BSAOPTHASH_LIBRARY "bsaopthash")
# Sources
set(SOURCE_FILES
hash.cpp
)
add_library(${BSAOPTHASH_LIBRARY} STATIC ${SOURCE_FILES})
set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARY})
link_directories(${CMAKE_CURRENT_BINARY_DIR})
set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARIES} PARENT_SCOPE)

@ -0,0 +1,108 @@
/* Version: MPL 1.1/LGPL 3.0
*
* "The contents of this file are subject to the Mozilla Public License
* Version 1.1 (the "License"); you may not use this file except in
* compliance with the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
* License for the specific language governing rights and limitations
* under the License.
*
* The Original Code is BSAopt.
*
* The Initial Developer of the Original Code is
* Ethatron <niels@paradice-insight.us>. Portions created by The Initial
* Developer are Copyright (C) 2011 The Initial Developer.
* All Rights Reserved.
*
* Alternatively, the contents of this file may be used under the terms
* of the GNU Library General Public License Version 3 license (the
* "LGPL License"), in which case the provisions of LGPL License are
* applicable instead of those above. If you wish to allow use of your
* version of this file only under the terms of the LGPL License and not
* to allow others to use your version of this file under the MPL,
* indicate your decision by deleting the provisions above and replace
* them with the notice and other provisions required by the LGPL License.
* If you do not delete the provisions above, a recipient may use your
* version of this file under either the MPL or the LGPL License."
*/
#include <cstdint>
#include <algorithm>
#include "hash.hpp"
std::uint32_t GenOBHashStr(const std::string& s) {
std::uint32_t hash = 0;
for (std::size_t i = 0; i < s.length(); i++) {
hash *= 0x1003F;
hash += (unsigned char)s[i];
}
return hash;
}
std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext) {
std::uint64_t hash = 0;
if (fle.length() > 0) {
hash = (std::uint64_t)(
(((unsigned char)fle[fle.length() - 1]) * 0x1) +
((fle.length() > 2 ? (unsigned char)fle[fle.length() - 2] : (unsigned char)0) * 0x100) +
(fle.length() * 0x10000) +
(((unsigned char)fle[0]) * 0x1000000)
);
if (fle.length() > 3) {
hash += (std::uint64_t)(GenOBHashStr(fle.substr(1, fle.length() - 3)) * 0x100000000);
}
}
if (ext.length() > 0) {
hash += (std::uint64_t)(GenOBHashStr(ext) * 0x100000000LL);
unsigned char i = 0;
if (ext == ".nif") i = 1;
if (ext == ".kf" ) i = 2;
if (ext == ".dds") i = 3;
if (ext == ".wav") i = 4;
if (i != 0) {
unsigned char a = (unsigned char)(((i & 0xfc ) << 5) + (unsigned char)((hash & 0xff000000) >> 24));
unsigned char b = (unsigned char)(((i & 0xfe ) << 6) + (unsigned char)( hash & 0x000000ff) );
unsigned char c = (unsigned char)(( i << 7) + (unsigned char)((hash & 0x0000ff00) >> 8));
hash -= hash & 0xFF00FFFF;
hash += (std::uint32_t)((a << 24) + b + (c << 8));
}
}
return hash;
}
std::uint64_t GenOBHash(const std::string& path, std::string& file) {
std::transform(file.begin(), file.end(), file.begin(), ::tolower);
std::replace(file.begin(), file.end(), '/', '\\');
std::string fle;
std::string ext;
const char *_fle = file.data();
const char *_ext = strrchr(_fle, '.');
if (_ext) {
ext = file.substr((0 + _ext) - _fle);
fle = file.substr(0, ( _ext) - _fle);
}
else {
ext = "";
fle = file;
}
if (path.length() && fle.length())
return GenOBHashPair(path + "\\" + fle, ext);
else
return GenOBHashPair(path + fle, ext);
}

@ -0,0 +1,42 @@
/* Version: MPL 1.1/LGPL 3.0
*
* "The contents of this file are subject to the Mozilla Public License
* Version 1.1 (the "License"); you may not use this file except in
* compliance with the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS"
* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
* License for the specific language governing rights and limitations
* under the License.
*
* The Original Code is BSAopt.
*
* The Initial Developer of the Original Code is
* Ethatron <niels@paradice-insight.us>. Portions created by The Initial
* Developer are Copyright (C) 2011 The Initial Developer.
* All Rights Reserved.
*
* Alternatively, the contents of this file may be used under the terms
* of the GNU Library General Public License Version 3 license (the
* "LGPL License"), in which case the provisions of LGPL License are
* applicable instead of those above. If you wish to allow use of your
* version of this file only under the terms of the LGPL License and not
* to allow others to use your version of this file under the MPL,
* indicate your decision by deleting the provisions above and replace
* them with the notice and other provisions required by the LGPL License.
* If you do not delete the provisions above, a recipient may use your
* version of this file under either the MPL or the LGPL License."
*/
#ifndef BSAOPT_HASH_H
#define BSAOPT_HASH_H
#include <string>
std::uint32_t GenOBHashStr(const std::string& s);
std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext);
std::uint64_t GenOBHash(const std::string& path, std::string& file);
#endif // BSAOPT_HASH_H
Loading…
Cancel
Save