mirror/openmw
1
0
Fork 0
mirror of https://github.com/OpenMW/openmw.git synced 2025-01-30 02:15:34 +00:00
Code Issues Releases Wiki Activity

Use zlib directly for ESM4

Browse source 
Fallback to decompression by block of fixed size when decompressing the
whole archive fails.
This commit is contained in:
elsid 2023-05-26 13:06:52 +02:00
parent f841a1377f
commit 11204d35d6
No known key found for this signature in database
GPG key ID: 4DE04C198CBA7625
4 changed files with 125 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
CI/before_script.msvc.sh
View file

@ -652,6 +652,10 @@ if [ -z $SKIP_DOWNLOAD ]; then
download "ICU ${ICU_VER/_/.}"\
"https://github.com/unicode-org/icu/releases/download/release-${ICU_VER/_/-}/icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip" \
"icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip"
download "zlib 1.2.11"\
"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/zlib-1.2.11-msvc2017-win64.7z" \
"zlib-1.2.11-msvc2017-win64.7z"
fi
cd .. #/..
@ -849,10 +853,10 @@ printf "${OSG_ARCHIVE_NAME}... "
fi
if ! [ -z $OSG_MULTIVIEW_BUILD ]; then
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,zlib,libpng16}${SUFFIX}.dll \
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,libpng16}${SUFFIX}.dll \
"$(pwd)/OSG/bin/osg162-osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll
else
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16,zlib}${SUFFIX}.dll \
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16}${SUFFIX}.dll \
"$(pwd)/OSG/bin/libxml2"${SUFFIX_UPCASE}.dll \
"$(pwd)/OSG/bin/osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/icudt58.dll"
@ -1028,6 +1032,27 @@ printf "ICU ${ICU_VER/_/.}... "
echo Done.
}
cd $DEPS
echo
printf "zlib 1.2.11... "
{
if [ -d zlib-1.2.11-msvc2017-win64 ]; then
printf "Exists. "
elif [ -z $SKIP_EXTRACT ]; then
rm -rf zlib-1.2.11-msvc2017-win64
eval 7z x -y zlib-1.2.11-msvc2017-win64.7z $STRIP
fi
add_cmake_opts -DZLIB_ROOT="$(real_pwd)/zlib-1.2.11-msvc2017-win64"
for config in ${CONFIGURATIONS[@]}; do
if [ $CONFIGURATION == "Debug" ]; then
add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlibd.dll"
else
add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlib.dll"
fi
done
echo Done.
}
echo
cd $DEPS_INSTALL/..
echo

1
CMakeLists.txt
View file

@ -479,6 +479,7 @@ if(OPENMW_USE_SYSTEM_MYGUI)
endif()
find_package(SDL2 2.0.9 REQUIRED)
find_package(OpenAL REQUIRED)
find_package(ZLIB REQUIRED)
option(USE_LUAJIT "Switch Lua/LuaJit (TRUE is highly recommended)" TRUE)
if(USE_LUAJIT)

1
components/CMakeLists.txt
View file

@ -522,6 +522,7 @@ target_link_libraries(components
smhasher
${ICU_LIBRARIES}
yaml-cpp
ZLIB::ZLIB
)
if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.77.0)

118
components/esm4/reader.cpp
View file

@ -24,21 +24,15 @@
#undef DEBUG_GROUPSTACK
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <optional>
#include <span>
#include <sstream>
#include <stdexcept>
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable : 4706)
#include <boost/iostreams/filter/zlib.hpp>
#pragma warning(pop)
#else
#include <boost/iostreams/filter/zlib.hpp>
#endif
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <zlib.h>
#include <components/bsa/memorystream.hpp>
#include <components/debug/debuglog.hpp>
@ -68,6 +62,92 @@ namespace ESM4
throw std::logic_error("Unsupported LocalizedStringType: " + std::to_string(static_cast<int>(type)));
}
struct InflateEnd
{
void operator()(z_stream* stream) const { inflateEnd(stream); }
};
std::optional<std::string> tryDecompressAll(std::span<char> compressed, std::span<char> decompressed)
{
z_stream stream{};
stream.next_in = reinterpret_cast<Bytef*>(compressed.data());
stream.next_out = reinterpret_cast<Bytef*>(decompressed.data());
stream.avail_in = compressed.size();
stream.avail_out = decompressed.size();
if (const int ec = inflateInit(&stream); ec != Z_OK)
return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg);
const std::unique_ptr<z_stream, InflateEnd> streamPtr(&stream);
if (const int ec = inflate(&stream, Z_NO_FLUSH); ec != Z_STREAM_END)
return "inflate error: " + std::to_string(ec) + " " + std::string(stream.msg);
return std::nullopt;
}
std::optional<std::string> tryDecompressByBlock(
std::span<char> compressed, std::span<char> decompressed, std::size_t blockSize)
{
z_stream stream{};
if (const int ec = inflateInit(&stream); ec != Z_OK)
return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg);
const std::unique_ptr<z_stream, InflateEnd> streamPtr(&stream);
while (!compressed.empty() && !decompressed.empty())
{
const auto prevTotalIn = stream.total_in;
const auto prevTotalOut = stream.total_out;
stream.next_in = reinterpret_cast<Bytef*>(compressed.data());
stream.avail_in = std::min(blockSize, compressed.size());
stream.next_out = reinterpret_cast<Bytef*>(decompressed.data());
stream.avail_out = std::min(blockSize, decompressed.size());
const int ec = inflate(&stream, Z_NO_FLUSH);
if (ec == Z_STREAM_END)
break;
if (ec != Z_OK)
return "inflate error after reading " + std::to_string(stream.total_in)
+ " bytes: " + std::to_string(ec) + " " + std::string(stream.msg);
compressed = compressed.subspan(stream.total_in - prevTotalIn);
decompressed = decompressed.subspan(stream.total_out - prevTotalOut);
}
return std::nullopt;
}
std::unique_ptr<Bsa::MemoryInputStream> decompress(
std::streamoff position, std::span<char> compressed, std::uint32_t uncompressedSize)
{
auto result = std::make_unique<Bsa::MemoryInputStream>(uncompressedSize);
const std::span decompressed(result->getRawData(), uncompressedSize);
const auto allError = tryDecompressAll(compressed, decompressed);
if (!allError.has_value())
return result;
Log(Debug::Warning) << "Failed to decompress record data at 0x" << std::hex << position
<< std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size()
<< " uncompressed size = " << uncompressedSize << ": " << *allError
<< ". Trying to decompress by block...";
std::memset(result->getRawData(), 0, uncompressedSize);
constexpr std::size_t blockSize = 4;
const auto blockError = tryDecompressByBlock(compressed, decompressed, blockSize);
if (!blockError.has_value())
return result;
std::ostringstream s;
s << "Failed to decompress record data by block of " << blockSize << " bytes at 0x" << std::hex << position
<< std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size()
<< " uncompressed size = " << uncompressedSize << ": " << *blockError;
throw std::runtime_error(s.str());
}
}
ReaderContext::ReaderContext()
@ -423,22 +503,16 @@ namespace ESM4
{
mStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(std::uint32_t));
std::size_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t);
Bsa::MemoryInputStream compressedRecord(recordSize);
mStream->read(compressedRecord.getRawData(), recordSize);
std::istream* fileStream = (std::istream*)&compressedRecord;
const std::streamoff position = mStream->tellg();
const std::uint32_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t);
std::vector<char> compressed(recordSize);
mStream->read(compressed.data(), recordSize);
mSavedStream = std::move(mStream);
mCtx.recordHeader.record.dataSize = uncompressedSize - sizeof(uncompressedSize);
auto memoryStreamPtr = std::make_unique<Bsa::MemoryInputStream>(uncompressedSize);
boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
inputStreamBuf.push(boost::iostreams::zlib_decompressor());
inputStreamBuf.push(*fileStream);
boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
boost::iostreams::copy(inputStreamBuf, sr);
auto memoryStreamPtr = decompress(position, compressed, uncompressedSize);
// For debugging only
// #if 0