mirror
/
openmw
mirror of https://github.com/OpenMW/openmw.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Use zlib directly for ESM4

Browse Source 
Fallback to decompression by block of fixed size when decompressing the
whole archive fails.
revert-6246b479
elsid 2 years ago
parent f841a1377f
commit 11204d35d6
No known key found for this signature in database
GPG Key ID: 4DE04C198CBA7625
4 changed files with 125 additions and 24 deletions
Show Stats Download Patch File Download Diff File

29
CI/before_script.msvc.sh
Unescape Escape View File

@ -652,6 +652,10 @@ if [ -z $SKIP_DOWNLOAD ]; then
download "ICU ${ICU_VER/_/.}"\
"https://github.com/unicode-org/icu/releases/download/release-${ICU_VER/_/-}/icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip" \
"icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip"
download "zlib 1.2.11"\
"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/zlib-1.2.11-msvc2017-win64.7z" \
"zlib-1.2.11-msvc2017-win64.7z"
fi
cd .. #/..
@ -849,10 +853,10 @@ printf "${OSG_ARCHIVE_NAME}... "
fi
if ! [ -z $OSG_MULTIVIEW_BUILD ]; then
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,zlib,libpng16}${SUFFIX}.dll \
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,libpng16}${SUFFIX}.dll \
"$(pwd)/OSG/bin/osg162-osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll
else
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16,zlib}${SUFFIX}.dll \
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16}${SUFFIX}.dll \
"$(pwd)/OSG/bin/libxml2"${SUFFIX_UPCASE}.dll \
"$(pwd)/OSG/bin/osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll
add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/icudt58.dll"
@ -1028,6 +1032,27 @@ printf "ICU ${ICU_VER/_/.}... "
echo Done.
}
cd $DEPS
echo
printf "zlib 1.2.11... "
{
if [ -d zlib-1.2.11-msvc2017-win64 ]; then
printf "Exists. "
elif [ -z $SKIP_EXTRACT ]; then
rm -rf zlib-1.2.11-msvc2017-win64
eval 7z x -y zlib-1.2.11-msvc2017-win64.7z $STRIP
fi
add_cmake_opts -DZLIB_ROOT="$(real_pwd)/zlib-1.2.11-msvc2017-win64"
for config in ${CONFIGURATIONS[@]}; do
if [ $CONFIGURATION == "Debug" ]; then
add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlibd.dll"
else
add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlib.dll"
fi
done
echo Done.
}
echo
cd $DEPS_INSTALL/..
echo

1
CMakeLists.txt
Unescape Escape View File

@ -479,6 +479,7 @@ if(OPENMW_USE_SYSTEM_MYGUI)
endif()
find_package(SDL2 2.0.9 REQUIRED)
find_package(OpenAL REQUIRED)
find_package(ZLIB REQUIRED)
option(USE_LUAJIT "Switch Lua/LuaJit (TRUE is highly recommended)" TRUE)
if(USE_LUAJIT)

1
components/CMakeLists.txt
Unescape Escape View File

@ -522,6 +522,7 @@ target_link_libraries(components
smhasher
${ICU_LIBRARIES}
yaml-cpp
ZLIB::ZLIB
)
if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.77.0)

118
components/esm4/reader.cpp
Unescape Escape View File

@ -24,21 +24,15 @@
#undef DEBUG_GROUPSTACK
#include <algorithm>
#include <iomanip>
#include <iostream>
#include <optional>
#include <span>
#include <sstream>
#include <stdexcept>
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable : 4706)
#include <boost/iostreams/filter/zlib.hpp>
#pragma warning(pop)
#else
#include <boost/iostreams/filter/zlib.hpp>
#endif
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/filtering_streambuf.hpp>
#include <zlib.h>
#include <components/bsa/memorystream.hpp>
#include <components/debug/debuglog.hpp>
@ -68,6 +62,92 @@ namespace ESM4
throw std::logic_error("Unsupported LocalizedStringType: " + std::to_string(static_cast<int>(type)));
}
struct InflateEnd
{
void operator()(z_stream* stream) const { inflateEnd(stream); }
};
std::optional<std::string> tryDecompressAll(std::span<char> compressed, std::span<char> decompressed)
{
z_stream stream{};
stream.next_in = reinterpret_cast<Bytef*>(compressed.data());
stream.next_out = reinterpret_cast<Bytef*>(decompressed.data());
stream.avail_in = compressed.size();
stream.avail_out = decompressed.size();
if (const int ec = inflateInit(&stream); ec != Z_OK)
return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg);
const std::unique_ptr<z_stream, InflateEnd> streamPtr(&stream);
if (const int ec = inflate(&stream, Z_NO_FLUSH); ec != Z_STREAM_END)
return "inflate error: " + std::to_string(ec) + " " + std::string(stream.msg);
return std::nullopt;
}
std::optional<std::string> tryDecompressByBlock(
std::span<char> compressed, std::span<char> decompressed, std::size_t blockSize)
{
z_stream stream{};
if (const int ec = inflateInit(&stream); ec != Z_OK)
return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg);
const std::unique_ptr<z_stream, InflateEnd> streamPtr(&stream);
while (!compressed.empty() && !decompressed.empty())
{
const auto prevTotalIn = stream.total_in;
const auto prevTotalOut = stream.total_out;
stream.next_in = reinterpret_cast<Bytef*>(compressed.data());
stream.avail_in = std::min(blockSize, compressed.size());
stream.next_out = reinterpret_cast<Bytef*>(decompressed.data());
stream.avail_out = std::min(blockSize, decompressed.size());
const int ec = inflate(&stream, Z_NO_FLUSH);
if (ec == Z_STREAM_END)
break;
if (ec != Z_OK)
return "inflate error after reading " + std::to_string(stream.total_in)
+ " bytes: " + std::to_string(ec) + " " + std::string(stream.msg);
compressed = compressed.subspan(stream.total_in - prevTotalIn);
decompressed = decompressed.subspan(stream.total_out - prevTotalOut);
}
return std::nullopt;
}
std::unique_ptr<Bsa::MemoryInputStream> decompress(
std::streamoff position, std::span<char> compressed, std::uint32_t uncompressedSize)
{
auto result = std::make_unique<Bsa::MemoryInputStream>(uncompressedSize);
const std::span decompressed(result->getRawData(), uncompressedSize);
const auto allError = tryDecompressAll(compressed, decompressed);
if (!allError.has_value())
return result;
Log(Debug::Warning) << "Failed to decompress record data at 0x" << std::hex << position
<< std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size()
<< " uncompressed size = " << uncompressedSize << ": " << *allError
<< ". Trying to decompress by block...";
std::memset(result->getRawData(), 0, uncompressedSize);
constexpr std::size_t blockSize = 4;
const auto blockError = tryDecompressByBlock(compressed, decompressed, blockSize);
if (!blockError.has_value())
return result;
std::ostringstream s;
s << "Failed to decompress record data by block of " << blockSize << " bytes at 0x" << std::hex << position
<< std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size()
<< " uncompressed size = " << uncompressedSize << ": " << *blockError;
throw std::runtime_error(s.str());
}
}
ReaderContext::ReaderContext()
@ -423,22 +503,16 @@ namespace ESM4
{
mStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(std::uint32_t));
std::size_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t);
Bsa::MemoryInputStream compressedRecord(recordSize);
mStream->read(compressedRecord.getRawData(), recordSize);
std::istream* fileStream = (std::istream*)&compressedRecord;
const std::streamoff position = mStream->tellg();
const std::uint32_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t);
std::vector<char> compressed(recordSize);
mStream->read(compressed.data(), recordSize);
mSavedStream = std::move(mStream);
mCtx.recordHeader.record.dataSize = uncompressedSize - sizeof(uncompressedSize);
auto memoryStreamPtr = std::make_unique<Bsa::MemoryInputStream>(uncompressedSize);
boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
inputStreamBuf.push(boost::iostreams::zlib_decompressor());
inputStreamBuf.push(*fileStream);
boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
boost::iostreams::copy(inputStreamBuf, sr);
auto memoryStreamPtr = decompress(position, compressed, uncompressedSize);
// For debugging only
// #if 0

Write Preview
Loading…
Cancel
Save