diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5831a73b8e..3201dd6b7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: before_script: - export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR - apt-get update -yq - - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev ccache git clang + - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev liblz4-dev ccache git clang stage: build script: - export CCACHE_BASEDIR="`pwd`" diff --git a/.travis.yml b/.travis.yml index 8d98d0dd43..2f457798a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ addons: # FFmpeg libavcodec-dev, libavformat-dev, libavutil-dev, libswresample-dev, libswscale-dev, # Audio, Video and Misc. deps - libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev, + libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev, liblz4-dev # The other ones from OpenMW ppa libbullet-dev, libopenscenegraph-dev, libmygui-dev ] diff --git a/CHANGELOG.md b/CHANGELOG.md index ad3bcfad07..7ae5f506d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ Feature #5579: MCP SetAngle enhancement Feature #5610: Actors movement should be smoother Feature #5642: Ability to attach arrows to actor skeleton instead of bow mesh + Feature #5649: Skyrim SE compressed BSA format support Task #5480: Drop Qt4 support Task #5520: Improve cell name autocompleter implementation diff --git a/CI/before_install.osx.sh b/CI/before_install.osx.sh index c3514e25be..9edfdf0d29 100755 --- a/CI/before_install.osx.sh +++ b/CI/before_install.osx.sh @@ -5,5 +5,5 @@ command -v ccache >/dev/null 2>&1 || brew install ccache command -v cmake >/dev/null 2>&1 || brew install cmake command -v qmake >/dev/null 2>&1 || brew install qt -curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-ef2462c.zip -o ~/openmw-deps.zip +curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-20201016.zip -o ~/openmw-deps.zip unzip -o ~/openmw-deps.zip -d /private/tmp/openmw-deps > /dev/null diff --git a/CI/before_script.msvc.sh b/CI/before_script.msvc.sh index 8b3ece2bd6..2a0db9c91d 100644 --- a/CI/before_script.msvc.sh +++ b/CI/before_script.msvc.sh @@ -571,6 +571,11 @@ if [ -z $SKIP_DOWNLOAD ]; then "https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/SDL2-2.0.12.zip" \ "SDL2-2.0.12.zip" + # LZ4 + download "LZ4 1.9.2" \ + "https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/lz4_win${BITS}_v1_9_2.7z" \ + "lz4_win${BITS}_v1_9_2.7z" + # Google test and mock if [ ! -z $TEST_FRAMEWORK ]; then echo "Google test 1.10.0..." @@ -901,6 +906,31 @@ printf "SDL 2.0.12... " } cd $DEPS echo +# LZ4 +printf "LZ4 1.9.2... " +{ + if [ -d LZ4_1.9.2 ]; then + printf "Exists. " + elif [ -z $SKIP_EXTRACT ]; then + rm -rf LZ4_1.9.2 + eval 7z x -y lz4_win${BITS}_v1_9_2.7z -o./LZ4_1.9.2 $STRIP + fi + export LZ4DIR="$(real_pwd)/LZ4_1.9.2" + add_cmake_opts -DLZ4_INCLUDE_DIR="${LZ4DIR}/include" \ + -DLZ4_LIBRARY="${LZ4DIR}/lib/liblz4.lib" + for CONFIGURATION in ${CONFIGURATIONS[@]}; do + if [ $CONFIGURATION == "Debug" ]; then + LZ4_CONFIGURATION="Debug" + else + SUFFIX="" + LZ4_CONFIGURATION="Release" + fi + add_runtime_dlls $CONFIGURATION "$(pwd)/LZ4_1.9.2/bin/${LZ4_CONFIGURATION}/liblz4.dll" + done + echo Done. +} +cd $DEPS +echo # Google Test and Google Mock if [ ! -z $TEST_FRAMEWORK ]; then printf "Google test 1.10.0 ..." diff --git a/CMakeLists.txt b/CMakeLists.txt index 829927de67..c4002ea500 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,6 +152,8 @@ endif() # Dependencies find_package(OpenGL REQUIRED) +find_package(LZ4 REQUIRED) + if (USE_QT) find_package(Qt5Core 5.12 REQUIRED) find_package(Qt5Widgets REQUIRED) diff --git a/cmake/FindLZ4.cmake b/cmake/FindLZ4.cmake new file mode 100644 index 0000000000..ec854c6b18 --- /dev/null +++ b/cmake/FindLZ4.cmake @@ -0,0 +1,130 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +FindLZ4 +------- + +Find the LZ4 include directory and library. + +Use this module by invoking find_package with the form:: + +.. code-block:: cmake + + find_package(LZ4 + [version] # Minimum version e.g. 1.8.0 + [REQUIRED] # Fail with error if LZ4 is not found + ) + +Imported targets +^^^^^^^^^^^^^^^^ + +This module defines the following :prop_tgt:`IMPORTED` targets: + +.. variable:: LZ4::LZ4 + + Imported target for using the LZ4 library, if found. + +Result variables +^^^^^^^^^^^^^^^^ + +.. variable:: LZ4_FOUND + + Set to true if LZ4 library found, otherwise false or undefined. + +.. variable:: LZ4_INCLUDE_DIRS + + Paths to include directories listed in one variable for use by LZ4 client. + +.. variable:: LZ4_LIBRARIES + + Paths to libraries to linked against to use LZ4. + +.. variable:: LZ4_VERSION + + The version string of LZ4 found. + +Cache variables +^^^^^^^^^^^^^^^ + +For users who wish to edit and control the module behavior, this module +reads hints about search locations from the following variables:: + +.. variable:: LZ4_INCLUDE_DIR + + Path to LZ4 include directory with ``lz4.h`` header. + +.. variable:: LZ4_LIBRARY + + Path to LZ4 library to be linked. + +NOTE: The variables above should not usually be used in CMakeLists.txt files! + +#]=======================================================================] + +### Find library ############################################################## + +if(NOT LZ4_LIBRARY) + find_library(LZ4_LIBRARY_RELEASE NAMES lz4) + find_library(LZ4_LIBRARY_DEBUG NAMES lz4d) + + include(SelectLibraryConfigurations) + select_library_configurations(LZ4) +else() + file(TO_CMAKE_PATH "${LZ4_LIBRARY}" LZ4_LIBRARY) +endif() + +### Find include directory #################################################### +find_path(LZ4_INCLUDE_DIR NAMES lz4.h) + +if(LZ4_INCLUDE_DIR AND EXISTS "${LZ4_INCLUDE_DIR}/lz4.h") + file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_h_contents + REGEX "#define LZ4_VERSION_[A-Z]+[ ]+[0-9]+") + string(REGEX REPLACE "#define LZ4_VERSION_MAJOR[ ]+([0-9]+).+" "\\1" + LZ4_VERSION_MAJOR "${_lz4_h_contents}") + string(REGEX REPLACE ".+#define LZ4_VERSION_MINOR[ ]+([0-9]+).+" "\\1" + LZ4_VERSION_MINOR "${_lz4_h_contents}") + string(REGEX REPLACE ".+#define LZ4_VERSION_RELEASE[ ]+([0-9]+).*" "\\1" + LZ4_VERSION_RELEASE "${_lz4_h_contents}") + set(LZ4_VERSION "${LZ4_VERSION_MAJOR}.${LZ4_VERSION_MINOR}.${LZ4_VERSION_RELEASE}") + unset(_lz4_h_contents) +endif() + +### Set result variables ###################################################### +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LZ4 DEFAULT_MSG + LZ4_LIBRARY LZ4_INCLUDE_DIR LZ4_VERSION) + +mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY) + +set(LZ4_LIBRARIES ${LZ4_LIBRARY}) +set(LZ4_INCLUDE_DIRS ${LZ4_INCLUDE_DIR}) + +### Import targets ############################################################ +if(LZ4_FOUND) + if(NOT TARGET LZ4::LZ4) + add_library(LZ4::LZ4 UNKNOWN IMPORTED) + set_target_properties(LZ4::LZ4 PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}") + + if(LZ4_LIBRARY_RELEASE) + set_property(TARGET LZ4::LZ4 APPEND PROPERTY + IMPORTED_CONFIGURATIONS RELEASE) + set_target_properties(LZ4::LZ4 PROPERTIES + IMPORTED_LOCATION_RELEASE "${LZ4_LIBRARY_RELEASE}") + endif() + + if(LZ4_LIBRARY_DEBUG) + set_property(TARGET LZ4::LZ4 APPEND PROPERTY + IMPORTED_CONFIGURATIONS DEBUG) + set_target_properties(LZ4::LZ4 PROPERTIES + IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}") + endif() + + if(NOT LZ4_LIBRARY_RELEASE AND NOT LZ4_LIBRARY_DEBUG) + set_property(TARGET LZ4::LZ4 APPEND PROPERTY + IMPORTED_LOCATION "${LZ4_LIBRARY}") + endif() + endif() +endif() diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 7f44cf6fb5..c399a70103 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -233,7 +233,7 @@ target_link_libraries(components ${SDL2_LIBRARIES} ${OPENGL_gl_LIBRARY} ${MyGUI_LIBRARIES} - ${BSAOPTHASH_LIBRARIES} + LZ4::LZ4 RecastNavigation::DebugUtils RecastNavigation::Detour RecastNavigation::Recast diff --git a/components/bsa/compressedbsafile.cpp b/components/bsa/compressedbsafile.cpp index d0bebe3c1f..bf805e5f7f 100644 --- a/components/bsa/compressedbsafile.cpp +++ b/components/bsa/compressedbsafile.cpp @@ -27,6 +27,8 @@ #include #include +#include + #include #include #include @@ -132,8 +134,11 @@ void CompressedBSAFile::readHeader() input.read(reinterpret_cast(header), 36); - if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/)) - fail("Unrecognized TES4 BSA header"); + if (header[0] != 0x00415342) /*"BSA\x00"*/ + fail("Unrecognized compressed BSA format"); + mVersion = header[1]; + if (mVersion != 0x67 /*TES4*/ && mVersion != 0x68 /*FO3, FNV, TES5*/ && mVersion != 0x69 /*SSE*/) + fail("Unrecognized compressed BSA version"); // header[2] is offset, should be 36 = 0x24 which is the size of the header @@ -158,7 +163,8 @@ void CompressedBSAFile::readHeader() // header[8]; // fileFlags : an opportunity to optimize here mCompressedByDefault = (archiveFlags & 0x4) != 0; - mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0; + if (mVersion == 0x68 || mVersion == 0x69) /*FO3, FNV, TES5, SSE*/ + mEmbeddedFileNames = (archiveFlags & 0x100) != 0; } // folder records @@ -168,7 +174,14 @@ void CompressedBSAFile::readHeader() { input.read(reinterpret_cast(&hash), 8); input.read(reinterpret_cast(&fr.count), 4); // not sure purpose of count - input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset + if (mVersion == 0x69) // SSE + { + std::uint32_t unknown; + input.read(reinterpret_cast(&unknown), 4); + input.read(reinterpret_cast(&fr.offset), 8); + } + else + input.read(reinterpret_cast(&fr.offset), 4); // not sure purpose of offset std::map::const_iterator lb = mFolders.lower_bound(hash); if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first))) @@ -327,32 +340,56 @@ Files::IStreamPtr CompressedBSAFile::getFile(const char* file) Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord) { - if (fileRecord.isCompressed(mCompressedByDefault)) { - Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag()); + size_t size = fileRecord.getSizeWithoutCompressionFlag(); + size_t uncompressedSize = size; + bool compressed = fileRecord.isCompressed(mCompressedByDefault); + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, size); + std::istream* fileStream = streamPtr.get(); + if (mEmbeddedFileNames) + { + // Skip over the embedded file name + char length = 0; + fileStream->read(&length, 1); + fileStream->ignore(length); + size -= length + sizeof(char); + } + if (compressed) + { + fileStream->read(reinterpret_cast(&uncompressedSize), sizeof(uint32_t)); + size -= sizeof(uint32_t); + } + std::shared_ptr memoryStreamPtr = std::make_shared(uncompressedSize); - std::istream* fileStream = streamPtr.get(); + if (compressed) + { + if (mVersion != 0x69) // Non-SSE: zlib + { + boost::iostreams::filtering_streambuf inputStreamBuf; + inputStreamBuf.push(boost::iostreams::zlib_decompressor()); + inputStreamBuf.push(*fileStream); - if (mEmbeddedFileNames) { - std::string embeddedFileName; - getBZString(embeddedFileName, *fileStream); + boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); + boost::iostreams::copy(inputStreamBuf, sr); } - - uint32_t uncompressedSize = 0u; - fileStream->read(reinterpret_cast(&uncompressedSize), sizeof(uncompressedSize)); - - boost::iostreams::filtering_streambuf inputStreamBuf; - inputStreamBuf.push(boost::iostreams::zlib_decompressor()); - inputStreamBuf.push(*fileStream); - - std::shared_ptr memoryStreamPtr = std::make_shared(uncompressedSize); - - boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); - boost::iostreams::copy(inputStreamBuf, sr); - - return std::shared_ptr(memoryStreamPtr, (std::istream*)memoryStreamPtr.get()); + else // SSE: lz4 + { + boost::scoped_array buffer(new char[size]); + fileStream->read(buffer.get(), size); + LZ4F_dctx* context = nullptr; + LZ4F_createDecompressionContext(&context, LZ4F_VERSION); + LZ4F_decompressOptions_t options = {}; + LZ4F_decompress(context, memoryStreamPtr->getRawData(), &uncompressedSize, buffer.get(), &size, &options); + LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(context); + if (LZ4F_isError(errorCode)) + fail("LZ4 decompression error (file " + mFilename + "): " + LZ4F_getErrorName(errorCode)); + } + } + else + { + fileStream->read(memoryStreamPtr->getRawData(), size); } - return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size); + return std::shared_ptr(memoryStreamPtr, (std::istream*)memoryStreamPtr.get()); } BsaVersion CompressedBSAFile::detectVersion(std::string filePath) diff --git a/components/bsa/compressedbsafile.hpp b/components/bsa/compressedbsafile.hpp index 3c28b5a5f8..f3ad584d8f 100644 --- a/components/bsa/compressedbsafile.hpp +++ b/components/bsa/compressedbsafile.hpp @@ -64,10 +64,12 @@ namespace Bsa //if each file record begins with BZ string with file name bool mEmbeddedFileNames; + std::uint32_t mVersion{0u}; + struct FolderRecord { std::uint32_t count; - std::uint32_t offset; + std::uint64_t offset; std::map files; }; std::map mFolders;