From 66d2b9c1958cddb4210fe30f4fb3af3e932a6943 Mon Sep 17 00:00:00 2001 From: Alexei Dobrohotov Date: Sat, 10 Oct 2020 14:00:03 +0300 Subject: [PATCH] Add Skyrim SE BSA version support Fix embedded file name loading --- CHANGELOG.md | 1 + components/CMakeLists.txt | 2 +- components/bsa/compressedbsafile.cpp | 87 ++++++++++++++++++++-------- components/bsa/compressedbsafile.hpp | 4 +- 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad3bcfad0..7ae5f506d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ Feature #5579: MCP SetAngle enhancement Feature #5610: Actors movement should be smoother Feature #5642: Ability to attach arrows to actor skeleton instead of bow mesh + Feature #5649: Skyrim SE compressed BSA format support Task #5480: Drop Qt4 support Task #5520: Improve cell name autocompleter implementation diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 7f44cf6fb..686bd8caa 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -233,7 +233,7 @@ target_link_libraries(components ${SDL2_LIBRARIES} ${OPENGL_gl_LIBRARY} ${MyGUI_LIBRARIES} - ${BSAOPTHASH_LIBRARIES} + ${LZ4_LIBRARIES} RecastNavigation::DebugUtils RecastNavigation::Detour RecastNavigation::Recast diff --git a/components/bsa/compressedbsafile.cpp b/components/bsa/compressedbsafile.cpp index d0bebe3c1..bf805e5f7 100644 --- a/components/bsa/compressedbsafile.cpp +++ b/components/bsa/compressedbsafile.cpp @@ -27,6 +27,8 @@ #include #include +#include + #include #include #include @@ -132,8 +134,11 @@ void CompressedBSAFile::readHeader() input.read(reinterpret_cast(header), 36); - if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/)) - fail("Unrecognized TES4 BSA header"); + if (header[0] != 0x00415342) /*"BSA\x00"*/ + fail("Unrecognized compressed BSA format"); + mVersion = header[1]; + if (mVersion != 0x67 /*TES4*/ && mVersion != 0x68 /*FO3, FNV, TES5*/ && mVersion != 0x69 /*SSE*/) + fail("Unrecognized compressed BSA version"); // header[2] is offset, should be 36 = 0x24 which is the size of the header @@ -158,7 +163,8 @@ void CompressedBSAFile::readHeader() // header[8]; // fileFlags : an opportunity to optimize here mCompressedByDefault = (archiveFlags & 0x4) != 0; - mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0; + if (mVersion == 0x68 || mVersion == 0x69) /*FO3, FNV, TES5, SSE*/ + mEmbeddedFileNames = (archiveFlags & 0x100) != 0; } // folder records @@ -168,7 +174,14 @@ void CompressedBSAFile::readHeader() { input.read(reinterpret_cast(&hash), 8); input.read(reinterpret_cast(&fr.count), 4); // not sure purpose of count - input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset + if (mVersion == 0x69) // SSE + { + std::uint32_t unknown; + input.read(reinterpret_cast(&unknown), 4); + input.read(reinterpret_cast(&fr.offset), 8); + } + else + input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset std::map::const_iterator lb = mFolders.lower_bound(hash); if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first))) @@ -327,32 +340,56 @@ Files::IStreamPtr CompressedBSAFile::getFile(const char* file) Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord) { - if (fileRecord.isCompressed(mCompressedByDefault)) { - Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag()); + size_t size = fileRecord.getSizeWithoutCompressionFlag(); + size_t uncompressedSize = size; + bool compressed = fileRecord.isCompressed(mCompressedByDefault); + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, size); + std::istream* fileStream = streamPtr.get(); + if (mEmbeddedFileNames) + { + // Skip over the embedded file name + char length = 0; + fileStream->read(&length, 1); + fileStream->ignore(length); + size -= length + sizeof(char); + } + if (compressed) + { + fileStream->read(reinterpret_cast(&uncompressedSize), sizeof(uint32_t)); + size -= sizeof(uint32_t); + } + std::shared_ptr memoryStreamPtr = std::make_shared(uncompressedSize); - std::istream* fileStream = streamPtr.get(); + if (compressed) + { + if (mVersion != 0x69) // Non-SSE: zlib + { + boost::iostreams::filtering_streambuf inputStreamBuf; + inputStreamBuf.push(boost::iostreams::zlib_decompressor()); + inputStreamBuf.push(*fileStream); - if (mEmbeddedFileNames) { - std::string embeddedFileName; - getBZString(embeddedFileName, *fileStream); + boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); + boost::iostreams::copy(inputStreamBuf, sr); } - - uint32_t uncompressedSize = 0u; - fileStream->read(reinterpret_cast(&uncompressedSize), sizeof(uncompressedSize)); - - boost::iostreams::filtering_streambuf inputStreamBuf; - inputStreamBuf.push(boost::iostreams::zlib_decompressor()); - inputStreamBuf.push(*fileStream); - - std::shared_ptr memoryStreamPtr = std::make_shared(uncompressedSize); - - boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); - boost::iostreams::copy(inputStreamBuf, sr); - - return std::shared_ptr(memoryStreamPtr, (std::istream*)memoryStreamPtr.get()); + else // SSE: lz4 + { + boost::scoped_array buffer(new char[size]); + fileStream->read(buffer.get(), size); + LZ4F_dctx* context = nullptr; + LZ4F_createDecompressionContext(&context, LZ4F_VERSION); + LZ4F_decompressOptions_t options = {}; + LZ4F_decompress(context, memoryStreamPtr->getRawData(), &uncompressedSize, buffer.get(), &size, &options); + LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(context); + if (LZ4F_isError(errorCode)) + fail("LZ4 decompression error (file " + mFilename + "): " + LZ4F_getErrorName(errorCode)); + } + } + else + { + fileStream->read(memoryStreamPtr->getRawData(), size); } - return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size); + return std::shared_ptr(memoryStreamPtr, (std::istream*)memoryStreamPtr.get()); } BsaVersion CompressedBSAFile::detectVersion(std::string filePath) diff --git a/components/bsa/compressedbsafile.hpp b/components/bsa/compressedbsafile.hpp index 3c28b5a5f..f3ad584d8 100644 --- a/components/bsa/compressedbsafile.hpp +++ b/components/bsa/compressedbsafile.hpp @@ -64,10 +64,12 @@ namespace Bsa //if each file record begins with BZ string with file name bool mEmbeddedFileNames; + std::uint32_t mVersion{0u}; + struct FolderRecord { std::uint32_t count; - std::uint32_t offset; + std::uint64_t offset; std::map files; }; std::map mFolders;