Limit max navmeshdb file size

Use "pragma max_page_count" to define max allowed file size in combination with
"pragma page_size" based on a new setting "max navmeshdb file size".

* Stop navmeshtool on the first db error.
* Disable writes to db in the engine on first "database or disk is full"
  SQLite3 error. There is no special error code for this error.
* Change default "write to navmeshdb" to true.
* Use time intervals for transaction duration instead of number of changes.

apps/navmeshtool/main.cpp

@@ -156,8 +156,9 @@ namespace NavMeshTool
             settings.load(config);
 
             const osg::Vec3f agentHalfExtents = Settings::Manager::getVector3("default actor pathfind half extents", "Game");
+            const std::uint64_t maxDbFileSize = static_cast<std::uint64_t>(Settings::Manager::getInt64("max navmeshdb file size", "Navigator"));
 
-            DetourNavigator::NavMeshDb db((config.getUserDataPath() / "navmesh.db").string());
+            DetourNavigator::NavMeshDb db((config.getUserDataPath() / "navmesh.db").string(), maxDbFileSize);
 
             std::vector<ESM::ESMReader> readers(contentFiles.size());
             EsmLoader::Query query;

apps/navmeshtool/navmesh.cpp

@@ -67,7 +67,7 @@ namespace NavMeshTool
             explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles)
                 : mDb(std::move(db))
                 , mRemoveUnusedTiles(removeUnusedTiles)
-                , mTransaction(mDb.startTransaction())
+                , mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate))
                 , mNextTileId(mDb.getMaxTileId() + 1)
                 , mNextShapeId(mDb.getMaxShapeId() + 1)
             {}
@@ -128,14 +128,11 @@ namespace NavMeshTool
             void insert(std::string_view worldspace, const TilePosition& tilePosition,
                 std::int64_t version, const std::vector<std::byte>& input, PreparedNavMeshData& data) override
             {
-                if (mRemoveUnusedTiles)
-                {
-                    std::lock_guard lock(mMutex);
-                    mDeleted += static_cast<std::size_t>(mDb.deleteTilesAt(worldspace, tilePosition));
-                }
-                data.mUserId = static_cast<unsigned>(mNextTileId);
                 {
                     std::lock_guard lock(mMutex);
+                    if (mRemoveUnusedTiles)
+                        mDeleted += static_cast<std::size_t>(mDb.deleteTilesAt(worldspace, tilePosition));
+                    data.mUserId = static_cast<unsigned>(mNextTileId);
                     mDb.insertTile(mNextTileId, worldspace, tilePosition, TileVersion {version}, input, serialize(data));
                     ++mNextTileId;
                 }
@@ -157,25 +154,44 @@ namespace NavMeshTool
                 report();
             }
 
-            void wait()
+            void cancel() override
+            {
+                std::unique_lock lock(mMutex);
+                mCancelled = true;
+                mHasTile.notify_one();
+            }
+
+            bool wait()
             {
-                constexpr std::size_t tilesPerTransaction = 3000;
+                constexpr std::chrono::seconds transactionInterval(1);
                 std::unique_lock lock(mMutex);
-                while (mProvided < mExpected)
+                auto start = std::chrono::steady_clock::now();
+                while (mProvided < mExpected && !mCancelled)
                 {
                     mHasTile.wait(lock);
-                    if (mProvided % tilesPerTransaction == 0)
+                    const auto now = std::chrono::steady_clock::now();
+                    if (now - start > transactionInterval)
                     {
                         mTransaction.commit();
-                        mTransaction = mDb.startTransaction();
+                        mTransaction = mDb.startTransaction(Sqlite3::TransactionMode::Immediate);
+                        start = now;
                     }
                 }
                 logGeneratedTiles(mProvided, mExpected);
+                return !mCancelled;
             }
 
-            void commit() { mTransaction.commit(); }
+            void commit()
+            {
+                const std::lock_guard lock(mMutex);
+                mTransaction.commit();
+            }
 
-            void vacuum() { mDb.vacuum(); }
+            void vacuum()
+            {
+                const std::lock_guard lock(mMutex);
+                mDb.vacuum();
+            }
 
             void removeTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range)
             {
@@ -183,7 +199,7 @@ namespace NavMeshTool
                 mTransaction.commit();
                 Log(Debug::Info) << "Removing tiles outside processed range for worldspace \"" << worldspace << "\"...";
                 mDeleted += static_cast<std::size_t>(mDb.deleteTilesOutsideRange(worldspace, range));
-                mTransaction = mDb.startTransaction();
+                mTransaction = mDb.startTransaction(Sqlite3::TransactionMode::Immediate);
             }
 
         private:
@@ -191,6 +207,7 @@ namespace NavMeshTool
             std::atomic_size_t mInserted {0};
             std::atomic_size_t mUpdated {0};
             std::size_t mDeleted = 0;
+            bool mCancelled = false;
             mutable std::mutex mMutex;
             NavMeshDb mDb;
             const bool mRemoveUnusedTiles;
@@ -211,7 +228,8 @@ namespace NavMeshTool
     }
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const Settings& settings,
-        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data, NavMeshDb&& db)
+        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data,
+        NavMeshDb&& db)
     {
         Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers...";
 
@@ -253,8 +271,8 @@ namespace NavMeshTool
                 ));
         }
 
-        navMeshTileConsumer->wait();
+        if (navMeshTileConsumer->wait())
-        navMeshTileConsumer->commit();
+            navMeshTileConsumer->commit();
 
         const auto inserted = navMeshTileConsumer->getInserted();
         const auto updated = navMeshTileConsumer->getUpdated();

apps/openmw_test_suite/detournavigator/asyncnavmeshupdater.cpp

@@ -12,6 +12,7 @@
 #include <gtest/gtest.h>
 
 #include <map>
+#include <limits>
 
 namespace
 {
@@ -19,9 +20,8 @@ namespace
     using namespace DetourNavigator;
     using namespace DetourNavigator::Tests;
 
-    void addHeightFieldPlane(TileCachedRecastMeshManager& recastMeshManager)
+    void addHeightFieldPlane(TileCachedRecastMeshManager& recastMeshManager, const osg::Vec2i cellPosition = osg::Vec2i(0, 0))
     {
-        const osg::Vec2i cellPosition(0, 0);
         const int cellSize = 8192;
         recastMeshManager.addHeightfield(cellPosition, cellSize, HeightfieldPlane {0});
     }
@@ -130,7 +130,7 @@ namespace
         mRecastMeshManager.setWorldspace(mWorldspace);
         addHeightFieldPlane(mRecastMeshManager);
         addObject(mBox, mRecastMeshManager);
-        auto db = std::make_unique<NavMeshDb>(":memory:");
+        auto db = std::make_unique<NavMeshDb>(":memory:", std::numeric_limits<std::uint64_t>::max());
         NavMeshDb* const dbPtr = db.get();
         AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db));
         const auto navMeshCacheItem = std::make_shared<GuardedNavMeshCacheItem>(makeEmptyNavMesh(mSettings), 1);
@@ -138,6 +138,7 @@ namespace
         const std::map<TilePosition, ChangeType> changedTiles {{tilePosition, ChangeType::add}};
         updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles);
         updater.wait(mListener, WaitConditionType::allJobsDone);
+        updater.stop();
         const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition);
         ASSERT_NE(recastMesh, nullptr);
         ShapeId nextShapeId {1};
@@ -154,7 +155,7 @@ namespace
         mRecastMeshManager.setWorldspace(mWorldspace);
         addHeightFieldPlane(mRecastMeshManager);
         addObject(mBox, mRecastMeshManager);
-        auto db = std::make_unique<NavMeshDb>(":memory:");
+        auto db = std::make_unique<NavMeshDb>(":memory:", std::numeric_limits<std::uint64_t>::max());
         NavMeshDb* const dbPtr = db.get();
         mSettings.mWriteToNavMeshDb = false;
         AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db));
@@ -163,6 +164,7 @@ namespace
         const std::map<TilePosition, ChangeType> changedTiles {{tilePosition, ChangeType::add}};
         updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles);
         updater.wait(mListener, WaitConditionType::allJobsDone);
+        updater.stop();
         const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition);
         ASSERT_NE(recastMesh, nullptr);
         ShapeId nextShapeId {1};
@@ -177,7 +179,7 @@ namespace
         mRecastMeshManager.setWorldspace(mWorldspace);
         addHeightFieldPlane(mRecastMeshManager);
         addObject(mBox, mRecastMeshManager);
-        auto db = std::make_unique<NavMeshDb>(":memory:");
+        auto db = std::make_unique<NavMeshDb>(":memory:", std::numeric_limits<std::uint64_t>::max());
         NavMeshDb* const dbPtr = db.get();
         mSettings.mWriteToNavMeshDb = false;
         AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db));
@@ -186,6 +188,7 @@ namespace
         const std::map<TilePosition, ChangeType> changedTiles {{tilePosition, ChangeType::add}};
         updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles);
         updater.wait(mListener, WaitConditionType::allJobsDone);
+        updater.stop();
         const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition);
         ASSERT_NE(recastMesh, nullptr);
         const auto objects = makeDbRefGeometryObjects(recastMesh->getMeshSources(),
@@ -198,7 +201,8 @@ namespace
         mRecastMeshManager.setWorldspace(mWorldspace);
         addHeightFieldPlane(mRecastMeshManager);
         mSettings.mMaxNavMeshTilesCacheSize = 0;
-        AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::make_unique<NavMeshDb>(":memory:"));
+        AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager,
+                                    std::make_unique<NavMeshDb>(":memory:", std::numeric_limits<std::uint64_t>::max()));
         const auto navMeshCacheItem = std::make_shared<GuardedNavMeshCacheItem>(makeEmptyNavMesh(mSettings), 1);
         const std::map<TilePosition, ChangeType> changedTiles {{TilePosition {0, 0}, ChangeType::add}};
         updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles);
@@ -239,4 +243,37 @@ namespace
         updater.wait(mListener, WaitConditionType::allJobsDone);
         EXPECT_EQ(navMeshCacheItem->lockConst()->getImpl().getTileRefAt(0, 0, 0), 0);
     }
+
+    TEST_F(DetourNavigatorAsyncNavMeshUpdaterTest, should_stop_writing_to_db_when_size_limit_is_reached)
+    {
+        mRecastMeshManager.setWorldspace(mWorldspace);
+        for (int x = -1; x <= 1; ++x)
+            for (int y = -1; y <= 1; ++y)
+                addHeightFieldPlane(mRecastMeshManager, osg::Vec2i(x, y));
+        addObject(mBox, mRecastMeshManager);
+        auto db = std::make_unique<NavMeshDb>(":memory:", 4097);
+        NavMeshDb* const dbPtr = db.get();
+        AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db));
+        const auto navMeshCacheItem = std::make_shared<GuardedNavMeshCacheItem>(makeEmptyNavMesh(mSettings), 1);
+        std::map<TilePosition, ChangeType> changedTiles;
+        for (int x = -5; x <= 5; ++x)
+            for (int y = -5; y <= 5; ++y)
+                changedTiles.emplace(TilePosition {x, y}, ChangeType::add);
+        updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles);
+        updater.wait(mListener, WaitConditionType::allJobsDone);
+        updater.stop();
+        for (int x = -5; x <= 5; ++x)
+            for (int y = -5; y <= 5; ++y)
+            {
+                const TilePosition tilePosition(x, y);
+                const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition);
+                ASSERT_NE(recastMesh, nullptr);
+                const std::optional<std::vector<DbRefGeometryObject>> objects = makeDbRefGeometryObjects(recastMesh->getMeshSources(),
+                    [&] (const MeshSource& v) { return resolveMeshSource(*dbPtr, v); });
+                if (!objects.has_value())
+                    continue;
+                EXPECT_FALSE(dbPtr->findTile(mWorldspace, tilePosition, serialize(mSettings.mRecast, *recastMesh, *objects)).has_value())
+                    << tilePosition.x() << " " << tilePosition.y();
+            }
+    }
 }

apps/openmw_test_suite/detournavigator/navigator.cpp

@@ -23,6 +23,7 @@
 #include <array>
 #include <deque>
 #include <memory>
+#include <limits>
 
 MATCHER_P3(Vec3fEq, x, y, z, "")
 {
@@ -64,7 +65,7 @@ namespace
             , mOut(mPath)
             , mStepSize(28.333332061767578125f)
         {
-            mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique<NavMeshDb>(":memory:")));
+            mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique<NavMeshDb>(":memory:", std::numeric_limits<std::uint64_t>::max())));
         }
     };
 
@@ -831,7 +832,7 @@ namespace
     TEST_F(DetourNavigatorNavigatorTest, multiple_threads_should_lock_tiles)
     {
         mSettings.mAsyncNavMeshUpdaterThreads = 2;
-        mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique<NavMeshDb>(":memory:")));
+        mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique<NavMeshDb>(":memory:", std::numeric_limits<std::uint64_t>::max())));
 
         const std::array<float, 5 * 5> heightfieldData {{
             0,   0,    0,    0,    0,

apps/openmw_test_suite/detournavigator/navmeshdb.cpp

@@ -10,6 +10,7 @@
 
 #include <numeric>
 #include <random>
+#include <limits>
 
 namespace
 {
@@ -27,7 +28,7 @@ namespace
 
     struct DetourNavigatorNavMeshDbTest : Test
     {
-        NavMeshDb mDb {":memory:"};
+        NavMeshDb mDb {":memory:", std::numeric_limits<std::uint64_t>::max()};
         std::minstd_rand mRandom;
 
         std::vector<std::byte> generateData()
@@ -166,4 +167,15 @@ namespace
                 ASSERT_EQ(mDb.findTile(worldspace, TilePosition {x, y}, input).has_value(),
                           -1 <= x && x <= 1 && -1 <= y && y <= 1) << "x=" << x << " y=" << y;
     }
+
+    TEST_F(DetourNavigatorNavMeshDbTest, should_support_file_size_limit)
+    {
+        mDb = NavMeshDb(":memory:", 4096);
+        const auto f = [&]
+        {
+            for (std::int64_t i = 1; i <= 100; ++i)
+                insertTile(TileId {i}, TileVersion {1});
+        };
+        EXPECT_THROW(f(), std::runtime_error);
+    }
 }

components/detournavigator/asyncnavmeshupdater.cpp

@@ -140,15 +140,7 @@ namespace DetourNavigator
 
     AsyncNavMeshUpdater::~AsyncNavMeshUpdater()
     {
-        mShouldStop = true;
+        stop();
-        if (mDbWorker != nullptr)
-            mDbWorker->stop();
-        std::unique_lock<std::mutex> lock(mMutex);
-        mWaiting.clear();
-        mHasJob.notify_all();
-        lock.unlock();
-        for (auto& thread : mThreads)
-            thread.join();
     }
 
     void AsyncNavMeshUpdater::post(const osg::Vec3f& agentHalfExtents, const SharedNavMeshCacheItem& navMeshCacheItem,
@@ -235,6 +227,20 @@ namespace DetourNavigator
         }
     }
 
+    void AsyncNavMeshUpdater::stop()
+    {
+        mShouldStop = true;
+        if (mDbWorker != nullptr)
+            mDbWorker->stop();
+        std::unique_lock<std::mutex> lock(mMutex);
+        mWaiting.clear();
+        mHasJob.notify_all();
+        lock.unlock();
+        for (auto& thread : mThreads)
+            if (thread.joinable())
+                thread.join();
+    }
+
     int AsyncNavMeshUpdater::waitUntilJobsDoneForNotPresentTiles(const std::size_t initialJobsLeft, std::size_t& maxProgress, Loading::Listener& listener)
     {
         std::size_t prevJobsLeft = initialJobsLeft;
@@ -672,10 +678,10 @@ namespace DetourNavigator
         mHasJob.notify_all();
     }
 
-    std::optional<JobIt> DbJobQueue::pop()
+    std::optional<JobIt> DbJobQueue::pop(std::chrono::steady_clock::duration timeout)
     {
         std::unique_lock lock(mMutex);
-        mHasJob.wait(lock, [&] { return mShouldStop || !mJobs.empty(); });
+        mHasJob.wait_for(lock, timeout, [&] { return mShouldStop || !mJobs.empty(); });
         if (mJobs.empty())
             return std::nullopt;
         const JobIt job = mJobs.front();
@@ -720,7 +726,6 @@ namespace DetourNavigator
     DbWorker::~DbWorker()
     {
         stop();
-        mThread.join();
     }
 
     void DbWorker::enqueueJob(JobIt job)
@@ -741,23 +746,35 @@ namespace DetourNavigator
     {
         mShouldStop = true;
         mQueue.stop();
+        if (mThread.joinable())
+            mThread.join();
     }
 
     void DbWorker::run() noexcept
     {
-        constexpr std::size_t writesPerTransaction = 100;
+        constexpr std::chrono::seconds transactionInterval(1);
-        auto transaction = mDb->startTransaction();
+        auto transaction = mDb->startTransaction(Sqlite3::TransactionMode::Immediate);
+        auto start = std::chrono::steady_clock::now();
         while (!mShouldStop)
         {
             try
             {
-                if (const auto job = mQueue.pop())
+                if (const auto job = mQueue.pop(transactionInterval))
                     processJob(*job);
-                if (mWrites > writesPerTransaction)
+                const auto now = std::chrono::steady_clock::now();
+                if (mHasChanges && now - start > transactionInterval)
                 {
-                    mWrites = 0;
+                    mHasChanges = false;
-                    transaction.commit();
+                    try
-                    transaction = mDb->startTransaction();
+                    {
+                        transaction.commit();
+                    }
+                    catch (const std::exception& e)
+                    {
+                        Log(Debug::Error) << "DbWorker exception on commit: " << e.what();
+                    }
+                    transaction = mDb->startTransaction(Sqlite3::TransactionMode::Immediate);
+                    start = now;
                 }
             }
             catch (const std::exception& e)
@@ -765,7 +782,15 @@ namespace DetourNavigator
                 Log(Debug::Error) << "DbWorker exception: " << e.what();
             }
         }
-        transaction.commit();
+        if (mHasChanges)
+            try
+            {
+                transaction.commit();
+            }
+            catch (const std::exception& e)
+            {
+                Log(Debug::Error) << "DbWorker exception on final commit: " << e.what();
+            }
     }
 
     void DbWorker::processJob(JobIt job)
@@ -779,6 +804,11 @@ namespace DetourNavigator
             catch (const std::exception& e)
             {
                 Log(Debug::Error) << "DbWorker exception while processing job " << job->mId << ": " << e.what();
+                if (std::string_view(e.what()).find("database or disk is full") != std::string_view::npos)
+                {
+                    mWriteToDb = false;
+                    Log(Debug::Warning) << "Writes to navmeshdb are disabled because file size limit is reached or disk is full";
+                }
             }
         };
 
@@ -807,7 +837,7 @@ namespace DetourNavigator
                 const auto objects = makeDbRefGeometryObjects(job->mRecastMesh->getMeshSources(),
                     [&] (const MeshSource& v) { return resolveMeshSource(*mDb, v, mNextShapeId); });
                 if (shapeId != mNextShapeId)
-                    ++mWrites;
+                    mHasChanges = true;
                 job->mInput = serialize(mRecastSettings, *job->mRecastMesh, objects);
             }
             else
@@ -826,7 +856,11 @@ namespace DetourNavigator
 
     void DbWorker::processWritingJob(JobIt job)
     {
-        ++mWrites;
+        if (!mWriteToDb)
+        {
+            Log(Debug::Debug) << "Ignored db write job " << job->mId;
+            return;
+        }
 
         Log(Debug::Debug) << "Processing db write job " << job->mId;
 
@@ -843,6 +877,7 @@ namespace DetourNavigator
             Log(Debug::Debug) << "Update db tile by job " << job->mId;
             job->mGeneratedNavMeshData->mUserId = cachedTileData->mTileId;
             mDb->updateTile(cachedTileData->mTileId, mVersion, serialize(*job->mGeneratedNavMeshData));
+            mHasChanges = true;
             return;
         }
 
@@ -858,5 +893,6 @@ namespace DetourNavigator
         mDb->insertTile(mNextTileId, job->mWorldspace, job->mChangedTile,
                         mVersion, job->mInput, serialize(*job->mGeneratedNavMeshData));
         ++mNextTileId;
+        mHasChanges = true;
     }
 }

components/detournavigator/asyncnavmeshupdater.hpp

@@ -87,7 +87,7 @@ namespace DetourNavigator
     public:
         void push(JobIt job);
 
-        std::optional<JobIt> pop();
+        std::optional<JobIt> pop(std::chrono::steady_clock::duration timeout);
 
         void update(TilePosition playerTile, int maxTiles);
 
@@ -131,13 +131,13 @@ namespace DetourNavigator
         const RecastSettings& mRecastSettings;
         const std::unique_ptr<NavMeshDb> mDb;
         const TileVersion mVersion;
-        const bool mWriteToDb;
+        bool mWriteToDb;
         TileId mNextTileId;
         ShapeId mNextShapeId;
         DbJobQueue mQueue;
         std::atomic_bool mShouldStop {false};
         std::atomic_size_t mGetTileCount {0};
-        std::size_t mWrites = 0;
+        bool mHasChanges = false;
         std::thread mThread;
 
         inline void run() noexcept;
@@ -173,6 +173,8 @@ namespace DetourNavigator
 
         void wait(Loading::Listener& listener, WaitConditionType waitConditionType);
 
+        void stop();
+
         Stats getStats() const;
 
         void enqueueJob(JobIt job);

components/detournavigator/generatenavmeshtile.cpp

@@ -96,6 +96,7 @@ namespace DetourNavigator
         {
             Log(Debug::Warning) << "Failed to generate navmesh for worldspace \"" << mWorldspace
                                 << "\" tile " << mTilePosition << ": " << e.what();
+            consumer->cancel();
         }
     }
 }

components/detournavigator/generatenavmeshtile.hpp

@@ -49,6 +49,8 @@ namespace DetourNavigator
 
         virtual void update(std::string_view worldspace, const TilePosition& tilePosition,
                             std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0;
+
+        virtual void cancel() = 0;
     };
 
     class GenerateNavMeshTile final : public SceneUtil::WorkItem

components/detournavigator/navigator.cpp

@@ -16,7 +16,7 @@ namespace DetourNavigator
         {
             try
             {
-                db = std::make_unique<NavMeshDb>(userDataPath + "/navmesh.db");
+                db = std::make_unique<NavMeshDb>(userDataPath + "/navmesh.db", settings.mMaxDbFileSize);
             }
             catch (const std::exception& e)
             {

components/detournavigator/navmeshdb.cpp

@@ -4,6 +4,7 @@
 #include <components/misc/compression.hpp>
 #include <components/sqlite3/db.hpp>
 #include <components/sqlite3/request.hpp>
+#include <components/misc/stringops.hpp>
 
 #include <DetourAlloc.h>
 
@@ -130,6 +131,27 @@ namespace DetourNavigator
         constexpr std::string_view vacuumQuery = R"(
             VACUUM;
         )";
+
+        struct GetPageSize
+        {
+            static std::string_view text() noexcept { return "pragma page_size;"; }
+            static void bind(sqlite3&, sqlite3_stmt&) {}
+        };
+
+        std::uint64_t getPageSize(sqlite3& db)
+        {
+            Sqlite3::Statement<GetPageSize> statement(db);
+            std::uint64_t value = 0;
+            request(db, statement, &value, 1);
+            return value;
+        }
+
+        void setMaxPageCount(sqlite3& db, std::uint64_t value)
+        {
+            const auto query = Misc::StringUtils::format("pragma max_page_count = %lu;", value);
+            if (const int ec = sqlite3_exec(&db, query.c_str(), nullptr, nullptr, nullptr); ec != SQLITE_OK)
+                throw std::runtime_error("Failed set max page count: " + std::string(sqlite3_errmsg(&db)));
+        }
     }
 
     std::ostream& operator<<(std::ostream& stream, ShapeType value)
@@ -142,7 +164,7 @@ namespace DetourNavigator
         return stream << "unknown shape type (" << static_cast<std::underlying_type_t<ShapeType>>(value) << ")";
     }
 
-    NavMeshDb::NavMeshDb(std::string_view path)
+    NavMeshDb::NavMeshDb(std::string_view path, std::uint64_t maxFileSize)
         : mDb(Sqlite3::makeDb(path, schema))
         , mGetMaxTileId(*mDb, DbQueries::GetMaxTileId {})
         , mFindTile(*mDb, DbQueries::FindTile {})
@@ -157,11 +179,13 @@ namespace DetourNavigator
         , mInsertShape(*mDb, DbQueries::InsertShape {})
         , mVacuum(*mDb, DbQueries::Vacuum {})
     {
+        const auto dbPageSize = getPageSize(*mDb);
+        setMaxPageCount(*mDb, maxFileSize / dbPageSize + static_cast<std::uint64_t>((maxFileSize % dbPageSize) != 0));
     }
 
-    Sqlite3::Transaction NavMeshDb::startTransaction()
+    Sqlite3::Transaction NavMeshDb::startTransaction(Sqlite3::TransactionMode mode)
     {
-        return Sqlite3::Transaction(*mDb);
+        return Sqlite3::Transaction(*mDb, mode);
     }
 
     TileId NavMeshDb::getMaxTileId()

components/detournavigator/navmeshdb.hpp

@@ -141,9 +141,9 @@ namespace DetourNavigator
     class NavMeshDb
     {
     public:
-        explicit NavMeshDb(std::string_view path);
+        explicit NavMeshDb(std::string_view path, std::uint64_t maxFileSize);
 
-        Sqlite3::Transaction startTransaction();
+        Sqlite3::Transaction startTransaction(Sqlite3::TransactionMode mode = Sqlite3::TransactionMode::Default);
 
         TileId getMaxTileId();
 

components/detournavigator/settings.cpp

@@ -64,6 +64,7 @@ namespace DetourNavigator
         result.mNavMeshVersion = ::Settings::Manager::getInt("nav mesh version", "Navigator");
         result.mEnableNavMeshDiskCache = ::Settings::Manager::getBool("enable nav mesh disk cache", "Navigator");
         result.mWriteToNavMeshDb = ::Settings::Manager::getBool("write to navmeshdb", "Navigator");
+        result.mMaxDbFileSize = static_cast<std::uint64_t>(::Settings::Manager::getInt64("max navmeshdb file size", "Navigator"));
 
         return result;
     }

components/detournavigator/settings.hpp

@@ -51,6 +51,7 @@ namespace DetourNavigator
         std::string mNavMeshPathPrefix;
         std::chrono::milliseconds mMinUpdateInterval;
         std::int64_t mNavMeshVersion = 0;
+        std::uint64_t mMaxDbFileSize = 0;
     };
 
     RecastSettings makeRecastSettingsFromSettingsManager();

components/sqlite3/db.cpp

@@ -10,7 +10,7 @@ namespace Sqlite3
 {
     void CloseSqlite3::operator()(sqlite3* handle) const noexcept
     {
-        sqlite3_close(handle);
+        sqlite3_close_v2(handle);
     }
 
     Db makeDb(std::string_view path, const char* schema)

components/sqlite3/transaction.cpp

@@ -9,25 +9,43 @@
 
 namespace Sqlite3
 {
+    namespace
+    {
+        const char* getBeginStatement(TransactionMode mode)
+        {
+            switch (mode)
+            {
+                case TransactionMode::Default: return "BEGIN";
+                case TransactionMode::Deferred: return "BEGIN DEFERRED";
+                case TransactionMode::Immediate: return "BEGIN IMMEDIATE";
+                case TransactionMode::Exclusive: return "BEGIN EXCLUSIVE";
+            }
+            throw std::logic_error("Invalid transaction mode: " + std::to_string(static_cast<std::underlying_type_t<TransactionMode>>(mode)));
+        }
+    }
+
     void Rollback::operator()(sqlite3* db) const
     {
         if (db == nullptr)
             return;
         if (const int ec = sqlite3_exec(db, "ROLLBACK", nullptr, nullptr, nullptr); ec != SQLITE_OK)
-            Log(Debug::Warning) << "Failed to rollback SQLite3 transaction: " << std::string(sqlite3_errmsg(db));
+            Log(Debug::Debug) << "Failed to rollback SQLite3 transaction: " << sqlite3_errmsg(db) << " (" << ec << ")";
     }
 
-    Transaction::Transaction(sqlite3& db)
+    Transaction::Transaction(sqlite3& db, TransactionMode mode)
         : mDb(&db)
     {
-        if (const int ec = sqlite3_exec(mDb.get(), "BEGIN", nullptr, nullptr, nullptr); ec != SQLITE_OK)
+        if (const int ec = sqlite3_exec(&db, getBeginStatement(mode), nullptr, nullptr, nullptr); ec != SQLITE_OK)
-            throw std::runtime_error("Failed to start transaction: " + std::string(sqlite3_errmsg(mDb.get())));
+        {
+            (void) mDb.release();
+            throw std::runtime_error("Failed to start transaction: " + std::string(sqlite3_errmsg(&db)) + " (" + std::to_string(ec) + ")");
+        }
     }
 
     void Transaction::commit()
     {
         if (const int ec = sqlite3_exec(mDb.get(), "COMMIT", nullptr, nullptr, nullptr); ec != SQLITE_OK)
-            throw std::runtime_error("Failed to commit transaction: " + std::string(sqlite3_errmsg(mDb.get())));
+            throw std::runtime_error("Failed to commit transaction: " + std::string(sqlite3_errmsg(mDb.get())) + " (" + std::to_string(ec) + ")");
         (void) mDb.release();
     }
 }

components/sqlite3/transaction.hpp

@@ -12,10 +12,18 @@ namespace Sqlite3
         void operator()(sqlite3* handle) const;
     };
 
+    enum class TransactionMode
+    {
+        Default,
+        Deferred,
+        Immediate,
+        Exclusive,
+    };
+
     class Transaction
     {
     public:
-        Transaction(sqlite3& db);
+        explicit Transaction(sqlite3& db, TransactionMode mode = TransactionMode::Default);
 
         void commit();
 

docs/source/reference/modding/settings/navigator.rst

@@ -70,10 +70,19 @@ write to navmeshdb
 
 :Type:		boolean
 :Range:		True/False
-:Default:	False
+:Default:	True
 
 If true generated navmesh tiles will be stored into disk cache while game is running.
 
+max navmeshdb file size
+-----------------------
+
+:Type:		integer
+:Range:		> 0
+:Default:	2147483648
+
+Approximate maximum file size of navigation mesh cache stored on disk in bytes (value > 0).
+
 Advanced settings
 *****************
 

files/settings-default.cfg

@@ -938,7 +938,10 @@ nav mesh version = 1
 enable nav mesh disk cache = true
 
 # Cache navigation mesh tiles to disk (true, false)
-write to navmeshdb = false
+write to navmeshdb = true
+
+# Approximate maximum file size of navigation mesh cache stored on disk in bytes (value > 0)
+max navmeshdb file size = 2147483648
 
 [Shadows]