Exchange binary messages between navmeshtool and launcher

apps/launcher/datafilespage.cpp

@@ -20,11 +20,73 @@
 #include <components/config/gamesettings.hpp>
 #include <components/config/launchersettings.hpp>
 
+#include <components/navmeshtool/protocol.hpp>
+
 #include "utils/textinputdialog.hpp"
 
-
 const char *Launcher::DataFilesPage::mDefaultContentListName = "Default";
 
+namespace Launcher
+{
+    namespace
+    {
+        struct HandleNavMeshToolMessage
+        {
+            int mCellsCount;
+            int mExpectedMaxProgress;
+            int mMaxProgress;
+            int mProgress;
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::ExpectedCells&& message) const
+            {
+                return HandleNavMeshToolMessage {
+                    static_cast<int>(message.mCount),
+                    mExpectedMaxProgress,
+                    static_cast<int>(message.mCount) * 100,
+                    mProgress
+                };
+            }
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::ProcessedCells&& message) const
+            {
+                return HandleNavMeshToolMessage {
+                    mCellsCount,
+                    mExpectedMaxProgress,
+                    mMaxProgress,
+                    std::max(mProgress, static_cast<int>(message.mCount))
+                };
+            }
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::ExpectedTiles&& message) const
+            {
+                const int expectedMaxProgress = mCellsCount + static_cast<int>(message.mCount);
+                return HandleNavMeshToolMessage {
+                    mCellsCount,
+                    expectedMaxProgress,
+                    std::max(mMaxProgress, expectedMaxProgress),
+                    mProgress
+                };
+            }
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::GeneratedTiles&& message) const
+            {
+                int progress = mCellsCount + static_cast<int>(message.mCount);
+                if (mExpectedMaxProgress < mMaxProgress)
+                    progress += static_cast<int>(std::round(
+                        (mMaxProgress - mExpectedMaxProgress)
+                        * (static_cast<float>(progress) / static_cast<float>(mExpectedMaxProgress))
+                    ));
+                return HandleNavMeshToolMessage {
+                    mCellsCount,
+                    mExpectedMaxProgress,
+                    mMaxProgress,
+                    std::max(mProgress, progress)
+                };
+            }
+        };
+    }
+}
+
 Launcher::DataFilesPage::DataFilesPage(Files::ConfigurationManager &cfg, Config::GameSettings &gameSettings,
                                        Config::LauncherSettings &launcherSettings, MainDialog *parent)
     : QWidget(parent)
@@ -95,8 +157,8 @@ void Launcher::DataFilesPage::buildView()
     connect(ui.updateNavMeshButton, SIGNAL(clicked()), this, SLOT(startNavMeshTool()));
     connect(ui.cancelNavMeshButton, SIGNAL(clicked()), this, SLOT(killNavMeshTool()));
 
-    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardOutput()), this, SLOT(updateNavMeshProgress()));
+    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardOutput()), this, SLOT(readNavMeshToolStdout()));
-    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardError()), this, SLOT(updateNavMeshProgress()));
+    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardError()), this, SLOT(readNavMeshToolStderr()));
     connect(mNavMeshToolInvoker->getProcess(), SIGNAL(finished(int, QProcess::ExitStatus)), this, SLOT(navMeshToolFinished(int, QProcess::ExitStatus)));
 }
 
@@ -429,7 +491,9 @@ void Launcher::DataFilesPage::startNavMeshTool()
     ui.navMeshProgressBar->setValue(0);
     ui.navMeshProgressBar->setMaximum(1);
 
-    if (!mNavMeshToolInvoker->startProcess(QLatin1String("openmw-navmeshtool")))
+    mNavMeshToolProgress = NavMeshToolProgress {};
+
+    if (!mNavMeshToolInvoker->startProcess(QLatin1String("openmw-navmeshtool"), QStringList({"--write-binary-log"})))
         return;
 
     ui.cancelNavMeshButton->setEnabled(true);
@@ -441,39 +505,60 @@ void Launcher::DataFilesPage::killNavMeshTool()
     mNavMeshToolInvoker->killProcess();
 }
 
-void Launcher::DataFilesPage::updateNavMeshProgress()
+void Launcher::DataFilesPage::readNavMeshToolStderr()
+{
+    updateNavMeshProgress(4096);
+}
+
+void Launcher::DataFilesPage::updateNavMeshProgress(int minDataSize)
 {
     QProcess& process = *mNavMeshToolInvoker->getProcess();
-    QString text;
+    mNavMeshToolProgress.mMessagesData.append(process.readAllStandardError());
-    while (process.canReadLine())
+    if (mNavMeshToolProgress.mMessagesData.size() < minDataSize)
-    {
-        const QByteArray line = process.readLine();
-        const auto end = std::find_if(line.rbegin(), line.rend(), [] (auto v) { return v != '\n' && v != '\r'; });
-        text = QString::fromUtf8(line.mid(0, line.size() - (end - line.rbegin())));
-        ui.navMeshLogPlainTextEdit->appendPlainText(text);
-    }
-    const QRegularExpression pattern(R"([\( ](\d+)/(\d+)[\) ])");
-    QRegularExpressionMatch match = pattern.match(text);
-    if (!match.hasMatch())
         return;
-    int value = match.captured(1).toInt();
+    const std::byte* const begin = reinterpret_cast<const std::byte*>(mNavMeshToolProgress.mMessagesData.constData());
-    const int maximum = match.captured(2).toInt();
+    const std::byte* const end = begin + mNavMeshToolProgress.mMessagesData.size();
-    if (text.contains("cell"))
+    const std::byte* position = begin;
-        ui.navMeshProgressBar->setMaximum(maximum * 100);
+    HandleNavMeshToolMessage handle {
-    else if (maximum > ui.navMeshProgressBar->maximum())
+        mNavMeshToolProgress.mCellsCount,
-        ui.navMeshProgressBar->setMaximum(maximum);
+        mNavMeshToolProgress.mExpectedMaxProgress,
-    else
+        ui.navMeshProgressBar->maximum(),
-        value += static_cast<int>(std::round(
+        ui.navMeshProgressBar->value(),
-            (ui.navMeshProgressBar->maximum() - maximum)
+    };
-            * (static_cast<float>(value) / static_cast<float>(maximum))
+    while (true)
-        ));
+    {
-    ui.navMeshProgressBar->setValue(value);
+        NavMeshTool::Message message;
+        const std::byte* const nextPosition = NavMeshTool::deserialize(position, end, message);
+        if (nextPosition == position)
+            break;
+        position = nextPosition;
+        handle = std::visit(handle, NavMeshTool::decode(message));
+    }
+    if (position != begin)
+        mNavMeshToolProgress.mMessagesData = mNavMeshToolProgress.mMessagesData.mid(position - begin);
+    mNavMeshToolProgress.mCellsCount = handle.mCellsCount;
+    mNavMeshToolProgress.mExpectedMaxProgress = handle.mExpectedMaxProgress;
+    ui.navMeshProgressBar->setMaximum(handle.mMaxProgress);
+    ui.navMeshProgressBar->setValue(handle.mProgress);
+}
+
+void Launcher::DataFilesPage::readNavMeshToolStdout()
+{
+    QProcess& process = *mNavMeshToolInvoker->getProcess();
+    QByteArray& logData = mNavMeshToolProgress.mLogData;
+    logData.append(process.readAllStandardOutput());
+    const int lineEnd = logData.lastIndexOf('\n');
+    if (lineEnd == -1)
+        return;
+    const int size = logData.size() >= lineEnd && logData[lineEnd - 1] == '\r' ? lineEnd - 1 : lineEnd;
+    ui.navMeshLogPlainTextEdit->appendPlainText(QString::fromUtf8(logData.data(), size));
+    logData = logData.mid(lineEnd + 1);
 }
 
 void Launcher::DataFilesPage::navMeshToolFinished(int exitCode, QProcess::ExitStatus exitStatus)
 {
-    updateNavMeshProgress();
+    updateNavMeshProgress(0);
-    ui.navMeshLogPlainTextEdit->appendPlainText(QString::fromUtf8(mNavMeshToolInvoker->getProcess()->readAll()));
+    ui.navMeshLogPlainTextEdit->appendPlainText(QString::fromUtf8(mNavMeshToolInvoker->getProcess()->readAllStandardOutput()));
     if (exitCode == 0 && exitStatus == QProcess::ExitStatus::NormalExit)
         ui.navMeshProgressBar->setValue(ui.navMeshProgressBar->maximum());
     ui.cancelNavMeshButton->setEnabled(false);

apps/launcher/datafilespage.hpp

@@ -73,7 +73,8 @@ namespace Launcher
 
         void startNavMeshTool();
         void killNavMeshTool();
-        void updateNavMeshProgress();
+        void readNavMeshToolStdout();
+        void readNavMeshToolStderr();
         void navMeshToolFinished(int exitCode, QProcess::ExitStatus exitStatus);
 
     public:
@@ -81,6 +82,14 @@ namespace Launcher
         const static char *mDefaultContentListName;
 
     private:
+        struct NavMeshToolProgress
+        {
+            QByteArray mLogData;
+            QByteArray mMessagesData;
+            std::map<std::uint64_t, std::string> mWorldspaces;
+            int mCellsCount = 0;
+            int mExpectedMaxProgress = 0;
+        };
 
         MainDialog *mMainDialog;
         TextInputDialog *mNewProfileDialog;
@@ -96,6 +105,7 @@ namespace Launcher
         QString mDataLocal;
 
         Process::ProcessInvoker* mNavMeshToolInvoker;
+        NavMeshToolProgress mNavMeshToolProgress;
 
         void buildView();
         void setProfile (int index, bool savePrevious);
@@ -107,6 +117,7 @@ namespace Launcher
         void populateFileViews(const QString& contentModelName);
         void reloadCells(QStringList selectedFiles);
         void refreshDataFilesView ();
+        void updateNavMeshProgress(int minDataSize);
 
         class PathIterator
         {

apps/navmeshtool/main.cpp

@@ -31,6 +31,11 @@
 #include <thread>
 #include <vector>
 
+#ifdef WIN32
+#include <fcntl.h>
+#include <io.h>
+#endif
+
 namespace NavMeshTool
 {
     namespace
@@ -86,6 +91,9 @@ namespace NavMeshTool
 
                 ("remove-unused-tiles", bpo::value<bool>()->implicit_value(true)
                     ->default_value(false), "remove tiles from cache that will not be used with current content profile")
+
+                ("write-binary-log", bpo::value<bool>()->implicit_value(true)
+                    ->default_value(false), "write progress in binary messages to be consumed by the launcher")
             ;
             Files::ConfigurationManager::addCommonOptions(result);
 
@@ -145,6 +153,12 @@ namespace NavMeshTool
 
             const bool processInteriorCells = variables["process-interior-cells"].as<bool>();
             const bool removeUnusedTiles = variables["remove-unused-tiles"].as<bool>();
+            const bool writeBinaryLog = variables["write-binary-log"].as<bool>();
+
+#ifdef WIN32
+            if (writeBinaryLog)
+                _setmode(_fileno(stderr), _O_BINARY);
+#endif
 
             Fallback::Map::init(variables["fallback"].as<Fallback::FallbackMap>().mMap);
 
@@ -180,10 +194,10 @@ namespace NavMeshTool
             navigatorSettings.mRecast.mSwimHeightScale = EsmLoader::getGameSetting(esmData.mGameSettings, "fSwimHeightScale").getFloat();
 
             WorldspaceData cellsData = gatherWorldspaceData(navigatorSettings, readers, vfs, bulletShapeManager,
-                                                            esmData, processInteriorCells);
+                                                            esmData, processInteriorCells, writeBinaryLog);
 
             generateAllNavMeshTiles(agentHalfExtents, navigatorSettings, threadsNumber, removeUnusedTiles,
-                                    cellsData, std::move(db));
+                                    writeBinaryLog, cellsData, std::move(db));
 
             Log(Debug::Info) << "Done";
 

apps/navmeshtool/navmesh.cpp

@@ -15,6 +15,8 @@
 #include <components/misc/progressreporter.hpp>
 #include <components/sceneutil/workqueue.hpp>
 #include <components/sqlite3/transaction.hpp>
+#include <components/debug/debugging.hpp>
+#include <components/navmeshtool/protocol.hpp>
 
 #include <osg/Vec3f>
 
@@ -51,6 +53,18 @@ namespace NavMeshTool
                 << "%) navmesh tiles are generated";
         }
 
+        template <class T>
+        void serializeToStderr(const T& value)
+        {
+            const std::vector<std::byte> data = serialize(value);
+            getLockedRawStderr()->write(reinterpret_cast<const char*>(data.data()), static_cast<std::streamsize>(data.size()));
+        }
+
+        void logGeneratedTilesMessage(std::size_t number)
+        {
+            serializeToStderr(GeneratedTiles {static_cast<std::uint64_t>(number)});
+        }
+
         struct LogGeneratedTiles
         {
             void operator()(std::size_t provided, std::size_t expected) const
@@ -64,9 +78,10 @@ namespace NavMeshTool
         public:
             std::atomic_size_t mExpected {0};
 
-            explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles)
+            explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles, bool writeBinaryLog)
                 : mDb(std::move(db))
                 , mRemoveUnusedTiles(removeUnusedTiles)
+                , mWriteBinaryLog(writeBinaryLog)
                 , mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate))
                 , mNextTileId(mDb.getMaxTileId() + 1)
                 , mNextShapeId(mDb.getMaxShapeId() + 1)
@@ -178,6 +193,8 @@ namespace NavMeshTool
                     }
                 }
                 logGeneratedTiles(mProvided, mExpected);
+                if (mWriteBinaryLog)
+                    logGeneratedTilesMessage(mProvided);
                 return !mCancelled;
             }
 
@@ -211,6 +228,7 @@ namespace NavMeshTool
             mutable std::mutex mMutex;
             NavMeshDb mDb;
             const bool mRemoveUnusedTiles;
+            const bool mWriteBinaryLog;
             Transaction mTransaction;
             TileId mNextTileId;
             std::condition_variable mHasTile;
@@ -223,18 +241,20 @@ namespace NavMeshTool
                 const std::size_t provided = mProvided.fetch_add(1, std::memory_order_relaxed) + 1;
                 mReporter(provided, mExpected);
                 mHasTile.notify_one();
+                if (mWriteBinaryLog)
+                    logGeneratedTilesMessage(provided);
             }
         };
     }
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const Settings& settings,
-        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data,
+        std::size_t threadsNumber, bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& data,
         NavMeshDb&& db)
     {
         Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers...";
 
         SceneUtil::WorkQueue workQueue(threadsNumber);
-        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles);
+        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles, writeBinaryLog);
         std::size_t tiles = 0;
         std::mt19937_64 random;
 
@@ -256,6 +276,9 @@ namespace NavMeshTool
 
             tiles += worldspaceTiles.size();
 
+            if (writeBinaryLog)
+                serializeToStderr(ExpectedTiles {static_cast<std::uint64_t>(tiles)});
+
             navMeshTileConsumer->mExpected = tiles;
 
             std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random);

apps/navmeshtool/navmesh.hpp

@@ -16,7 +16,7 @@ namespace NavMeshTool
     struct WorldspaceData;
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const DetourNavigator::Settings& settings,
-        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& cellsData,
+        std::size_t threadsNumber, bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& cellsData,
         DetourNavigator::NavMeshDb&& db);
 }
 

apps/navmeshtool/worldspacedata.cpp

@@ -18,6 +18,8 @@
 #include <components/resource/bulletshapemanager.hpp>
 #include <components/settings/settings.hpp>
 #include <components/vfs/manager.hpp>
+#include <components/debug/debugging.hpp>
+#include <components/navmeshtool/protocol.hpp>
 
 #include <LinearMath/btVector3.h>
 
@@ -214,6 +216,13 @@ namespace NavMeshTool
             surface.mSize = static_cast<std::size_t>(ESM::Land::LAND_SIZE);
             return {surface, landData.mMinHeight, landData.mMaxHeight};
         }
+
+        template <class T>
+        void serializeToStderr(const T& value)
+        {
+            const std::vector<std::byte> data = serialize(value);
+            getRawStderr().write(reinterpret_cast<const char*>(data.data()), static_cast<std::streamsize>(data.size()));
+        }
     }
 
     WorldspaceNavMeshInput::WorldspaceNavMeshInput(std::string worldspace, const DetourNavigator::RecastSettings& settings)
@@ -226,7 +235,7 @@ namespace NavMeshTool
 
     WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
         const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
-        bool processInteriorCells)
+        bool processInteriorCells, bool writeBinaryLog)
     {
         Log(Debug::Info) << "Processing " << esmData.mCells.size() << " cells...";
 
@@ -235,6 +244,9 @@ namespace NavMeshTool
 
         std::size_t objectsCounter = 0;
 
+        if (writeBinaryLog)
+            serializeToStderr(ExpectedCells {static_cast<std::uint64_t>(esmData.mCells.size())});
+
         for (std::size_t i = 0; i < esmData.mCells.size(); ++i)
         {
             const ESM::Cell& cell = esmData.mCells[i];
@@ -242,6 +254,8 @@ namespace NavMeshTool
 
             if (!exterior && !processInteriorCells)
             {
+                if (writeBinaryLog)
+                    serializeToStderr(ProcessedCells {static_cast<std::uint64_t>(i + 1)});
                 Log(Debug::Info) << "Skipped interior"
                     << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \"" << cell.getDescription() << "\"";
                 continue;
@@ -311,8 +325,13 @@ namespace NavMeshTool
                     data.mObjects.emplace_back(std::move(object));
                 });
 
+            const auto cellDescription = cell.getDescription();
+
+            if (writeBinaryLog)
+                serializeToStderr(ProcessedCells {static_cast<std::uint64_t>(i + 1)});
+
             Log(Debug::Info) << "Processed " << (exterior ? "exterior" : "interior")
-                << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") " << cell.getDescription()
+                << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") " << cellDescription
                 << " with " << (data.mObjects.size() - cellObjectsBegin) << " objects";
         }
 

apps/navmeshtool/worldspacedata.hpp

@@ -91,7 +91,7 @@ namespace NavMeshTool
 
     WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
         const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
-        bool processInteriorCells);
+        bool processInteriorCells, bool writeBinaryLog);
 }
 
 #endif

apps/openmw_test_suite/serialization/binaryreader.cpp

@@ -64,4 +64,27 @@ namespace
         const TestFormat<Mode::Read> format;
         EXPECT_THROW(binaryReader(format, values.data(), values.size()), std::runtime_error);
     }
+
+    TEST(DetourNavigatorSerializationBinaryReaderTest, shouldSetPointerToCurrentBufferPosition)
+    {
+        std::vector<std::byte> data(8);
+        BinaryReader binaryReader(data.data(), data.data() + data.size());
+        const std::byte* ptr = nullptr;
+        const TestFormat<Mode::Read> format;
+        binaryReader(format, ptr);
+        EXPECT_EQ(ptr, data.data());
+    }
+
+    TEST(DetourNavigatorSerializationBinaryReaderTest, shouldNotAdvanceAfterPointer)
+    {
+        std::vector<std::byte> data(8);
+        BinaryReader binaryReader(data.data(), data.data() + data.size());
+        const std::byte* ptr1 = nullptr;
+        const std::byte* ptr2 = nullptr;
+        const TestFormat<Mode::Read> format;
+        binaryReader(format, ptr1);
+        binaryReader(format, ptr2);
+        EXPECT_EQ(ptr1, data.data());
+        EXPECT_EQ(ptr2, data.data());
+    }
 }

components/CMakeLists.txt

@@ -319,6 +319,10 @@ add_component_dir(esmloader
     esmdata
 )
 
+add_component_dir(navmeshtool
+    protocol
+    )
+
 set (ESM_UI ${CMAKE_SOURCE_DIR}/files/ui/contentselector.ui
     )
 

components/debug/debugging.cpp

@@ -138,6 +138,7 @@ namespace Debug
 
 static std::unique_ptr<std::ostream> rawStdout = nullptr;
 static std::unique_ptr<std::ostream> rawStderr = nullptr;
+static std::unique_ptr<std::mutex> rawStderrMutex = nullptr;
 static boost::filesystem::ofstream logfile;
 
 #if defined(_WIN32) && defined(_DEBUG)
@@ -152,6 +153,16 @@ std::ostream& getRawStdout()
     return rawStdout ? *rawStdout : std::cout;
 }
 
+std::ostream& getRawStderr()
+{
+    return rawStderr ? *rawStderr : std::cerr;
+}
+
+Misc::Locked<std::ostream&> getLockedRawStderr()
+{
+    return Misc::Locked<std::ostream&>(*rawStderrMutex, getRawStderr());
+}
+
 // Redirect cout and cerr to the log file
 void setupLogging(const std::string& logDir, const std::string& appName, std::ios_base::openmode mode)
 {
@@ -180,6 +191,7 @@ int wrapApplication(int (*innerApplication)(int argc, char *argv[]), int argc, c
 #endif
     rawStdout = std::make_unique<std::ostream>(std::cout.rdbuf());
     rawStderr = std::make_unique<std::ostream>(std::cerr.rdbuf());
+    rawStderrMutex = std::make_unique<std::mutex>();
 
     int ret = 0;
     try

components/debug/debugging.hpp

@@ -5,6 +5,7 @@
 #include <boost/iostreams/stream.hpp>
 
 #include <components/files/configurationmanager.hpp>
+#include <components/misc/guarded.hpp>
 
 #include <SDL_messagebox.h>
 
@@ -140,6 +141,10 @@ namespace Debug
 // Can be used to print messages without timestamps
 std::ostream& getRawStdout();
 
+std::ostream& getRawStderr();
+
+Misc::Locked<std::ostream&> getLockedRawStderr();
+
 void setupLogging(const std::string& logDir, const std::string& appName, std::ios_base::openmode = std::ios::out);
 
 int wrapApplication(int (*innerApplication)(int argc, char *argv[]), int argc, char *argv[],

components/misc/guarded.hpp

@@ -4,6 +4,7 @@
 #include <mutex>
 #include <memory>
 #include <condition_variable>
+#include <type_traits>
 
 namespace Misc
 {
@@ -11,28 +12,28 @@ namespace Misc
     class Locked
     {
         public:
-            Locked(std::mutex& mutex, T& value)
+            Locked(std::mutex& mutex, std::remove_reference_t<T>& value)
                 : mLock(mutex), mValue(value)
             {}
 
-            T& get() const
+            std::remove_reference_t<T>& get() const
             {
                 return mValue.get();
             }
 
-            T* operator ->() const
+            std::remove_reference_t<T>* operator ->() const
             {
-                return std::addressof(get());
+                return &get();
             }
 
-            T& operator *() const
+            std::remove_reference_t<T>& operator *() const
             {
                 return get();
             }
 
         private:
             std::unique_lock<std::mutex> mLock;
-            std::reference_wrapper<T> mValue;
+            std::reference_wrapper<std::remove_reference_t<T>> mValue;
     };
 
     template <class T>

components/navmeshtool/protocol.cpp

@@ -0,0 +1,159 @@
+#include "protocol.hpp"
+
+#include <components/serialization/format.hpp>
+#include <components/serialization/sizeaccumulator.hpp>
+#include <components/serialization/binarywriter.hpp>
+#include <components/serialization/binaryreader.hpp>
+
+#include <stdexcept>
+#include <string>
+
+namespace NavMeshTool
+{
+    namespace
+    {
+        template <Serialization::Mode mode>
+        struct Format : Serialization::Format<mode, Format<mode>>
+        {
+            using Serialization::Format<mode, Format<mode>>::operator();
+
+            template <class Visitor, class T>
+            auto operator()(Visitor&& visitor, T& value) const
+                -> std::enable_if_t<std::is_same_v<std::decay_t<T>, Message>>
+            {
+                if constexpr (mode == Serialization::Mode::Write)
+                    visitor(*this, messageMagic);
+                else
+                {
+                    static_assert(mode == Serialization::Mode::Read);
+                    char magic[std::size(messageMagic)];
+                    visitor(*this, magic);
+                    if (std::memcmp(magic, messageMagic, sizeof(magic)) != 0)
+                        throw BadMessageMagic();
+                }
+                visitor(*this, value.mType);
+                visitor(*this, value.mSize);
+                if constexpr (mode == Serialization::Mode::Write)
+                    visitor(*this, value.mData, value.mSize);
+                else
+                    visitor(*this, value.mData);
+            }
+
+            template <class Visitor, class T>
+            auto operator()(Visitor&& visitor, T& value) const
+                -> std::enable_if_t<std::is_same_v<std::decay_t<T>, ExpectedCells>>
+            {
+                visitor(*this, value.mCount);
+            }
+
+            template <class Visitor, class T>
+            auto operator()(Visitor&& visitor, T& value) const
+                -> std::enable_if_t<std::is_same_v<std::decay_t<T>, ProcessedCells>>
+            {
+                visitor(*this, value.mCount);
+            }
+
+            template <class Visitor, class T>
+            auto operator()(Visitor&& visitor, T& value) const
+                -> std::enable_if_t<std::is_same_v<std::decay_t<T>, ExpectedTiles>>
+            {
+                visitor(*this, value.mCount);
+            }
+
+            template <class Visitor, class T>
+            auto operator()(Visitor&& visitor, T& value) const
+                -> std::enable_if_t<std::is_same_v<std::decay_t<T>, GeneratedTiles>>
+            {
+                visitor(*this, value.mCount);
+            }
+        };
+
+        template <class T>
+        std::vector<std::byte> serializeToVector(const T& value)
+        {
+            constexpr Format<Serialization::Mode::Write> format;
+            Serialization::SizeAccumulator sizeAccumulator;
+            format(sizeAccumulator, value);
+            std::vector<std::byte> buffer(sizeAccumulator.value());
+            format(Serialization::BinaryWriter(buffer.data(), buffer.data() + buffer.size()), value);
+            return buffer;
+        }
+
+        template <class T>
+        std::vector<std::byte> serializeImpl(const T& value)
+        {
+            const auto data = serializeToVector(value);
+            const Message message {static_cast<std::uint64_t>(T::sMessageType), static_cast<std::uint64_t>(data.size()), data.data()};
+            return serializeToVector(message);
+        }
+    }
+
+    std::vector<std::byte> serialize(const ExpectedCells& value)
+    {
+        return serializeImpl(value);
+    }
+
+    std::vector<std::byte> serialize(const ProcessedCells& value)
+    {
+        return serializeImpl(value);
+    }
+
+    std::vector<std::byte> serialize(const ExpectedTiles& value)
+    {
+        return serializeImpl(value);
+    }
+
+    std::vector<std::byte> serialize(const GeneratedTiles& value)
+    {
+        return serializeImpl(value);
+    }
+
+    const std::byte* deserialize(const std::byte* begin, const std::byte* end, Message& message)
+    {
+        try
+        {
+            constexpr Format<Serialization::Mode::Read> format;
+            Serialization::BinaryReader reader(begin, end);
+            format(reader, message);
+            return message.mData + message.mSize;
+        }
+        catch (const Serialization::NotEnoughData&)
+        {
+            return begin;
+        }
+    }
+
+    TypedMessage decode(const Message& message)
+    {
+        constexpr Format<Serialization::Mode::Read> format;
+        Serialization::BinaryReader reader(message.mData, message.mData + message.mSize);
+        switch (static_cast<MessageType>(message.mType))
+        {
+            case MessageType::ExpectedCells:
+            {
+                ExpectedCells value;
+                format(reader, value);
+                return value;
+            }
+            case MessageType::ProcessedCells:
+            {
+                ProcessedCells value;
+                format(reader, value);
+                return value;
+            }
+            case MessageType::ExpectedTiles:
+            {
+                ExpectedTiles value;
+                format(reader, value);
+                return value;
+            }
+            case MessageType::GeneratedTiles:
+            {
+                GeneratedTiles value;
+                format(reader, value);
+                return value;
+            }
+        }
+        throw std::logic_error("Unsupported message type: " + std::to_string(message.mType));
+    }
+}

components/navmeshtool/protocol.hpp

@@ -0,0 +1,78 @@
+#ifndef OPENMW_COMPONENTS_NAVMESHTOOL_PROTOCOL_H
+#define OPENMW_COMPONENTS_NAVMESHTOOL_PROTOCOL_H
+
+#include <cstddef>
+#include <cstdint>
+#include <stdexcept>
+#include <variant>
+#include <vector>
+
+namespace NavMeshTool
+{
+    inline constexpr char messageMagic[] = {'n', 'v', 't', 'm'};
+
+    struct BadMessageMagic : std::runtime_error
+    {
+        BadMessageMagic() : std::runtime_error("Bad Message magic") {}
+    };
+
+    enum class MessageType : std::uint64_t
+    {
+        ExpectedCells = 1,
+        ProcessedCells = 2,
+        ExpectedTiles = 3,
+        GeneratedTiles = 4,
+    };
+
+    struct Message
+    {
+        std::uint64_t mType = 0;
+        std::uint64_t mSize = 0;
+        const std::byte* mData = nullptr;
+    };
+
+    struct ExpectedCells
+    {
+        static constexpr MessageType sMessageType = MessageType::ExpectedCells;
+        std::uint64_t mCount = 0;
+    };
+
+    struct ProcessedCells
+    {
+        static constexpr MessageType sMessageType = MessageType::ProcessedCells;
+        std::uint64_t mCount = 0;
+    };
+
+    struct ExpectedTiles
+    {
+        static constexpr MessageType sMessageType = MessageType::ExpectedTiles;
+        std::uint64_t mCount = 0;
+    };
+
+    struct GeneratedTiles
+    {
+        static constexpr MessageType sMessageType = MessageType::GeneratedTiles;
+        std::uint64_t mCount = 0;
+    };
+
+    using TypedMessage = std::variant<
+        ExpectedCells,
+        ProcessedCells,
+        ExpectedTiles,
+        GeneratedTiles
+    >;
+
+    std::vector<std::byte> serialize(const ExpectedCells& value);
+
+    std::vector<std::byte> serialize(const ProcessedCells& value);
+
+    std::vector<std::byte> serialize(const ExpectedTiles& value);
+
+    std::vector<std::byte> serialize(const GeneratedTiles& value);
+
+    const std::byte* deserialize(const std::byte* begin, const std::byte* end, Message& message);
+
+    TypedMessage decode(const Message& message);
+}
+
+#endif

components/serialization/binaryreader.hpp

@@ -12,6 +12,11 @@
 
 namespace Serialization
 {
+    struct NotEnoughData : std::runtime_error
+    {
+        NotEnoughData() : std::runtime_error("Not enough data") {}
+    };
+
     class BinaryReader
     {
     public:
@@ -31,11 +36,13 @@ namespace Serialization
             else if constexpr (std::is_arithmetic_v<T>)
             {
                 if (mEnd - mPos < static_cast<std::ptrdiff_t>(sizeof(T)))
-                    throw std::runtime_error("Not enough data");
+                    throw NotEnoughData();
                 std::memcpy(&value, mPos, sizeof(T));
                 mPos += sizeof(T);
                 value = Misc::toLittleEndian(value);
             }
+            else if constexpr (std::is_pointer_v<T>)
+                value = reinterpret_cast<T>(mPos);
             else
             {
                 format(*this, value);
@@ -51,7 +58,7 @@ namespace Serialization
             {
                 const std::size_t size = sizeof(T) * count;
                 if (mEnd - mPos < static_cast<std::ptrdiff_t>(size))
-                    throw std::runtime_error("Not enough data");
+                    throw NotEnoughData();
                 std::memcpy(data, mPos, size);
                 mPos += size;
                 if constexpr (!Misc::IS_LITTLE_ENDIAN)

components/serialization/binarywriter.hpp

@@ -12,6 +12,11 @@
 
 namespace Serialization
 {
+    struct NotEnoughSpace : std::runtime_error
+    {
+        NotEnoughSpace() : std::runtime_error("Not enough space") {}
+    };
+
     struct BinaryWriter
     {
     public:
@@ -31,7 +36,7 @@ namespace Serialization
             else if constexpr (std::is_arithmetic_v<T>)
             {
                 if (mEnd - mDest < static_cast<std::ptrdiff_t>(sizeof(T)))
-                    throw std::runtime_error("Not enough space");
+                    throw NotEnoughSpace();
                 writeValue(value);
             }
             else
@@ -49,7 +54,7 @@ namespace Serialization
             {
                 const std::size_t size = sizeof(T) * count;
                 if (mEnd - mDest < static_cast<std::ptrdiff_t>(size))
-                    throw std::runtime_error("Not enough space");
+                    throw NotEnoughSpace();
                 if constexpr (Misc::IS_LITTLE_ENDIAN)
                 {
                     std::memcpy(mDest, data, size);