diff --git a/apps/navmeshtool/main.cpp b/apps/navmeshtool/main.cpp index 894ec6b3b1..af411f03dd 100644 --- a/apps/navmeshtool/main.cpp +++ b/apps/navmeshtool/main.cpp @@ -156,8 +156,9 @@ namespace NavMeshTool settings.load(config); const osg::Vec3f agentHalfExtents = Settings::Manager::getVector3("default actor pathfind half extents", "Game"); + const std::uint64_t maxDbFileSize = static_cast(Settings::Manager::getInt64("max navmeshdb file size", "Navigator")); - DetourNavigator::NavMeshDb db((config.getUserDataPath() / "navmesh.db").string()); + DetourNavigator::NavMeshDb db((config.getUserDataPath() / "navmesh.db").string(), maxDbFileSize); std::vector readers(contentFiles.size()); EsmLoader::Query query; diff --git a/apps/navmeshtool/navmesh.cpp b/apps/navmeshtool/navmesh.cpp index 8b85029949..3acddb821d 100644 --- a/apps/navmeshtool/navmesh.cpp +++ b/apps/navmeshtool/navmesh.cpp @@ -67,7 +67,7 @@ namespace NavMeshTool explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles) : mDb(std::move(db)) , mRemoveUnusedTiles(removeUnusedTiles) - , mTransaction(mDb.startTransaction()) + , mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate)) , mNextTileId(mDb.getMaxTileId() + 1) , mNextShapeId(mDb.getMaxShapeId() + 1) {} @@ -128,14 +128,11 @@ namespace NavMeshTool void insert(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t version, const std::vector& input, PreparedNavMeshData& data) override { - if (mRemoveUnusedTiles) - { - std::lock_guard lock(mMutex); - mDeleted += static_cast(mDb.deleteTilesAt(worldspace, tilePosition)); - } - data.mUserId = static_cast(mNextTileId); { std::lock_guard lock(mMutex); + if (mRemoveUnusedTiles) + mDeleted += static_cast(mDb.deleteTilesAt(worldspace, tilePosition)); + data.mUserId = static_cast(mNextTileId); mDb.insertTile(mNextTileId, worldspace, tilePosition, TileVersion {version}, input, serialize(data)); ++mNextTileId; } @@ -157,25 +154,44 @@ namespace NavMeshTool report(); } - void wait() + void cancel() override + { + std::unique_lock lock(mMutex); + mCancelled = true; + mHasTile.notify_one(); + } + + bool wait() { - constexpr std::size_t tilesPerTransaction = 3000; + constexpr std::chrono::seconds transactionInterval(1); std::unique_lock lock(mMutex); - while (mProvided < mExpected) + auto start = std::chrono::steady_clock::now(); + while (mProvided < mExpected && !mCancelled) { mHasTile.wait(lock); - if (mProvided % tilesPerTransaction == 0) + const auto now = std::chrono::steady_clock::now(); + if (now - start > transactionInterval) { mTransaction.commit(); - mTransaction = mDb.startTransaction(); + mTransaction = mDb.startTransaction(Sqlite3::TransactionMode::Immediate); + start = now; } } logGeneratedTiles(mProvided, mExpected); + return !mCancelled; } - void commit() { mTransaction.commit(); } + void commit() + { + const std::lock_guard lock(mMutex); + mTransaction.commit(); + } - void vacuum() { mDb.vacuum(); } + void vacuum() + { + const std::lock_guard lock(mMutex); + mDb.vacuum(); + } void removeTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range) { @@ -183,7 +199,7 @@ namespace NavMeshTool mTransaction.commit(); Log(Debug::Info) << "Removing tiles outside processed range for worldspace \"" << worldspace << "\"..."; mDeleted += static_cast(mDb.deleteTilesOutsideRange(worldspace, range)); - mTransaction = mDb.startTransaction(); + mTransaction = mDb.startTransaction(Sqlite3::TransactionMode::Immediate); } private: @@ -191,6 +207,7 @@ namespace NavMeshTool std::atomic_size_t mInserted {0}; std::atomic_size_t mUpdated {0}; std::size_t mDeleted = 0; + bool mCancelled = false; mutable std::mutex mMutex; NavMeshDb mDb; const bool mRemoveUnusedTiles; @@ -211,7 +228,8 @@ namespace NavMeshTool } void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const Settings& settings, - std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data, NavMeshDb&& db) + std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data, + NavMeshDb&& db) { Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers..."; @@ -253,8 +271,8 @@ namespace NavMeshTool )); } - navMeshTileConsumer->wait(); - navMeshTileConsumer->commit(); + if (navMeshTileConsumer->wait()) + navMeshTileConsumer->commit(); const auto inserted = navMeshTileConsumer->getInserted(); const auto updated = navMeshTileConsumer->getUpdated(); diff --git a/apps/openmw_test_suite/detournavigator/asyncnavmeshupdater.cpp b/apps/openmw_test_suite/detournavigator/asyncnavmeshupdater.cpp index 71c013d9f4..1f7125d1f2 100644 --- a/apps/openmw_test_suite/detournavigator/asyncnavmeshupdater.cpp +++ b/apps/openmw_test_suite/detournavigator/asyncnavmeshupdater.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace { @@ -19,9 +20,8 @@ namespace using namespace DetourNavigator; using namespace DetourNavigator::Tests; - void addHeightFieldPlane(TileCachedRecastMeshManager& recastMeshManager) + void addHeightFieldPlane(TileCachedRecastMeshManager& recastMeshManager, const osg::Vec2i cellPosition = osg::Vec2i(0, 0)) { - const osg::Vec2i cellPosition(0, 0); const int cellSize = 8192; recastMeshManager.addHeightfield(cellPosition, cellSize, HeightfieldPlane {0}); } @@ -130,7 +130,7 @@ namespace mRecastMeshManager.setWorldspace(mWorldspace); addHeightFieldPlane(mRecastMeshManager); addObject(mBox, mRecastMeshManager); - auto db = std::make_unique(":memory:"); + auto db = std::make_unique(":memory:", std::numeric_limits::max()); NavMeshDb* const dbPtr = db.get(); AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db)); const auto navMeshCacheItem = std::make_shared(makeEmptyNavMesh(mSettings), 1); @@ -138,6 +138,7 @@ namespace const std::map changedTiles {{tilePosition, ChangeType::add}}; updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles); updater.wait(mListener, WaitConditionType::allJobsDone); + updater.stop(); const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition); ASSERT_NE(recastMesh, nullptr); ShapeId nextShapeId {1}; @@ -154,7 +155,7 @@ namespace mRecastMeshManager.setWorldspace(mWorldspace); addHeightFieldPlane(mRecastMeshManager); addObject(mBox, mRecastMeshManager); - auto db = std::make_unique(":memory:"); + auto db = std::make_unique(":memory:", std::numeric_limits::max()); NavMeshDb* const dbPtr = db.get(); mSettings.mWriteToNavMeshDb = false; AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db)); @@ -163,6 +164,7 @@ namespace const std::map changedTiles {{tilePosition, ChangeType::add}}; updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles); updater.wait(mListener, WaitConditionType::allJobsDone); + updater.stop(); const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition); ASSERT_NE(recastMesh, nullptr); ShapeId nextShapeId {1}; @@ -177,7 +179,7 @@ namespace mRecastMeshManager.setWorldspace(mWorldspace); addHeightFieldPlane(mRecastMeshManager); addObject(mBox, mRecastMeshManager); - auto db = std::make_unique(":memory:"); + auto db = std::make_unique(":memory:", std::numeric_limits::max()); NavMeshDb* const dbPtr = db.get(); mSettings.mWriteToNavMeshDb = false; AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db)); @@ -186,6 +188,7 @@ namespace const std::map changedTiles {{tilePosition, ChangeType::add}}; updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles); updater.wait(mListener, WaitConditionType::allJobsDone); + updater.stop(); const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition); ASSERT_NE(recastMesh, nullptr); const auto objects = makeDbRefGeometryObjects(recastMesh->getMeshSources(), @@ -198,7 +201,8 @@ namespace mRecastMeshManager.setWorldspace(mWorldspace); addHeightFieldPlane(mRecastMeshManager); mSettings.mMaxNavMeshTilesCacheSize = 0; - AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::make_unique(":memory:")); + AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, + std::make_unique(":memory:", std::numeric_limits::max())); const auto navMeshCacheItem = std::make_shared(makeEmptyNavMesh(mSettings), 1); const std::map changedTiles {{TilePosition {0, 0}, ChangeType::add}}; updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles); @@ -239,4 +243,37 @@ namespace updater.wait(mListener, WaitConditionType::allJobsDone); EXPECT_EQ(navMeshCacheItem->lockConst()->getImpl().getTileRefAt(0, 0, 0), 0); } + + TEST_F(DetourNavigatorAsyncNavMeshUpdaterTest, should_stop_writing_to_db_when_size_limit_is_reached) + { + mRecastMeshManager.setWorldspace(mWorldspace); + for (int x = -1; x <= 1; ++x) + for (int y = -1; y <= 1; ++y) + addHeightFieldPlane(mRecastMeshManager, osg::Vec2i(x, y)); + addObject(mBox, mRecastMeshManager); + auto db = std::make_unique(":memory:", 4097); + NavMeshDb* const dbPtr = db.get(); + AsyncNavMeshUpdater updater(mSettings, mRecastMeshManager, mOffMeshConnectionsManager, std::move(db)); + const auto navMeshCacheItem = std::make_shared(makeEmptyNavMesh(mSettings), 1); + std::map changedTiles; + for (int x = -5; x <= 5; ++x) + for (int y = -5; y <= 5; ++y) + changedTiles.emplace(TilePosition {x, y}, ChangeType::add); + updater.post(mAgentHalfExtents, navMeshCacheItem, mPlayerTile, mWorldspace, changedTiles); + updater.wait(mListener, WaitConditionType::allJobsDone); + updater.stop(); + for (int x = -5; x <= 5; ++x) + for (int y = -5; y <= 5; ++y) + { + const TilePosition tilePosition(x, y); + const auto recastMesh = mRecastMeshManager.getMesh(mWorldspace, tilePosition); + ASSERT_NE(recastMesh, nullptr); + const std::optional> objects = makeDbRefGeometryObjects(recastMesh->getMeshSources(), + [&] (const MeshSource& v) { return resolveMeshSource(*dbPtr, v); }); + if (!objects.has_value()) + continue; + EXPECT_FALSE(dbPtr->findTile(mWorldspace, tilePosition, serialize(mSettings.mRecast, *recastMesh, *objects)).has_value()) + << tilePosition.x() << " " << tilePosition.y(); + } + } } diff --git a/apps/openmw_test_suite/detournavigator/navigator.cpp b/apps/openmw_test_suite/detournavigator/navigator.cpp index 50bf1fe735..ec4b669692 100644 --- a/apps/openmw_test_suite/detournavigator/navigator.cpp +++ b/apps/openmw_test_suite/detournavigator/navigator.cpp @@ -23,6 +23,7 @@ #include #include #include +#include MATCHER_P3(Vec3fEq, x, y, z, "") { @@ -64,7 +65,7 @@ namespace , mOut(mPath) , mStepSize(28.333332061767578125f) { - mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique(":memory:"))); + mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique(":memory:", std::numeric_limits::max()))); } }; @@ -831,7 +832,7 @@ namespace TEST_F(DetourNavigatorNavigatorTest, multiple_threads_should_lock_tiles) { mSettings.mAsyncNavMeshUpdaterThreads = 2; - mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique(":memory:"))); + mNavigator.reset(new NavigatorImpl(mSettings, std::make_unique(":memory:", std::numeric_limits::max()))); const std::array heightfieldData {{ 0, 0, 0, 0, 0, diff --git a/apps/openmw_test_suite/detournavigator/navmeshdb.cpp b/apps/openmw_test_suite/detournavigator/navmeshdb.cpp index f7d9327e65..55d26e3b78 100644 --- a/apps/openmw_test_suite/detournavigator/navmeshdb.cpp +++ b/apps/openmw_test_suite/detournavigator/navmeshdb.cpp @@ -10,6 +10,7 @@ #include #include +#include namespace { @@ -27,7 +28,7 @@ namespace struct DetourNavigatorNavMeshDbTest : Test { - NavMeshDb mDb {":memory:"}; + NavMeshDb mDb {":memory:", std::numeric_limits::max()}; std::minstd_rand mRandom; std::vector generateData() @@ -166,4 +167,15 @@ namespace ASSERT_EQ(mDb.findTile(worldspace, TilePosition {x, y}, input).has_value(), -1 <= x && x <= 1 && -1 <= y && y <= 1) << "x=" << x << " y=" << y; } + + TEST_F(DetourNavigatorNavMeshDbTest, should_support_file_size_limit) + { + mDb = NavMeshDb(":memory:", 4096); + const auto f = [&] + { + for (std::int64_t i = 1; i <= 100; ++i) + insertTile(TileId {i}, TileVersion {1}); + }; + EXPECT_THROW(f(), std::runtime_error); + } } diff --git a/components/detournavigator/asyncnavmeshupdater.cpp b/components/detournavigator/asyncnavmeshupdater.cpp index 4af229da8a..18bffae131 100644 --- a/components/detournavigator/asyncnavmeshupdater.cpp +++ b/components/detournavigator/asyncnavmeshupdater.cpp @@ -140,15 +140,7 @@ namespace DetourNavigator AsyncNavMeshUpdater::~AsyncNavMeshUpdater() { - mShouldStop = true; - if (mDbWorker != nullptr) - mDbWorker->stop(); - std::unique_lock lock(mMutex); - mWaiting.clear(); - mHasJob.notify_all(); - lock.unlock(); - for (auto& thread : mThreads) - thread.join(); + stop(); } void AsyncNavMeshUpdater::post(const osg::Vec3f& agentHalfExtents, const SharedNavMeshCacheItem& navMeshCacheItem, @@ -235,6 +227,20 @@ namespace DetourNavigator } } + void AsyncNavMeshUpdater::stop() + { + mShouldStop = true; + if (mDbWorker != nullptr) + mDbWorker->stop(); + std::unique_lock lock(mMutex); + mWaiting.clear(); + mHasJob.notify_all(); + lock.unlock(); + for (auto& thread : mThreads) + if (thread.joinable()) + thread.join(); + } + int AsyncNavMeshUpdater::waitUntilJobsDoneForNotPresentTiles(const std::size_t initialJobsLeft, std::size_t& maxProgress, Loading::Listener& listener) { std::size_t prevJobsLeft = initialJobsLeft; @@ -672,10 +678,10 @@ namespace DetourNavigator mHasJob.notify_all(); } - std::optional DbJobQueue::pop() + std::optional DbJobQueue::pop(std::chrono::steady_clock::duration timeout) { std::unique_lock lock(mMutex); - mHasJob.wait(lock, [&] { return mShouldStop || !mJobs.empty(); }); + mHasJob.wait_for(lock, timeout, [&] { return mShouldStop || !mJobs.empty(); }); if (mJobs.empty()) return std::nullopt; const JobIt job = mJobs.front(); @@ -720,7 +726,6 @@ namespace DetourNavigator DbWorker::~DbWorker() { stop(); - mThread.join(); } void DbWorker::enqueueJob(JobIt job) @@ -741,23 +746,35 @@ namespace DetourNavigator { mShouldStop = true; mQueue.stop(); + if (mThread.joinable()) + mThread.join(); } void DbWorker::run() noexcept { - constexpr std::size_t writesPerTransaction = 100; - auto transaction = mDb->startTransaction(); + constexpr std::chrono::seconds transactionInterval(1); + auto transaction = mDb->startTransaction(Sqlite3::TransactionMode::Immediate); + auto start = std::chrono::steady_clock::now(); while (!mShouldStop) { try { - if (const auto job = mQueue.pop()) + if (const auto job = mQueue.pop(transactionInterval)) processJob(*job); - if (mWrites > writesPerTransaction) + const auto now = std::chrono::steady_clock::now(); + if (mHasChanges && now - start > transactionInterval) { - mWrites = 0; - transaction.commit(); - transaction = mDb->startTransaction(); + mHasChanges = false; + try + { + transaction.commit(); + } + catch (const std::exception& e) + { + Log(Debug::Error) << "DbWorker exception on commit: " << e.what(); + } + transaction = mDb->startTransaction(Sqlite3::TransactionMode::Immediate); + start = now; } } catch (const std::exception& e) @@ -765,7 +782,15 @@ namespace DetourNavigator Log(Debug::Error) << "DbWorker exception: " << e.what(); } } - transaction.commit(); + if (mHasChanges) + try + { + transaction.commit(); + } + catch (const std::exception& e) + { + Log(Debug::Error) << "DbWorker exception on final commit: " << e.what(); + } } void DbWorker::processJob(JobIt job) @@ -779,6 +804,11 @@ namespace DetourNavigator catch (const std::exception& e) { Log(Debug::Error) << "DbWorker exception while processing job " << job->mId << ": " << e.what(); + if (std::string_view(e.what()).find("database or disk is full") != std::string_view::npos) + { + mWriteToDb = false; + Log(Debug::Warning) << "Writes to navmeshdb are disabled because file size limit is reached or disk is full"; + } } }; @@ -807,7 +837,7 @@ namespace DetourNavigator const auto objects = makeDbRefGeometryObjects(job->mRecastMesh->getMeshSources(), [&] (const MeshSource& v) { return resolveMeshSource(*mDb, v, mNextShapeId); }); if (shapeId != mNextShapeId) - ++mWrites; + mHasChanges = true; job->mInput = serialize(mRecastSettings, *job->mRecastMesh, objects); } else @@ -826,7 +856,11 @@ namespace DetourNavigator void DbWorker::processWritingJob(JobIt job) { - ++mWrites; + if (!mWriteToDb) + { + Log(Debug::Debug) << "Ignored db write job " << job->mId; + return; + } Log(Debug::Debug) << "Processing db write job " << job->mId; @@ -843,6 +877,7 @@ namespace DetourNavigator Log(Debug::Debug) << "Update db tile by job " << job->mId; job->mGeneratedNavMeshData->mUserId = cachedTileData->mTileId; mDb->updateTile(cachedTileData->mTileId, mVersion, serialize(*job->mGeneratedNavMeshData)); + mHasChanges = true; return; } @@ -858,5 +893,6 @@ namespace DetourNavigator mDb->insertTile(mNextTileId, job->mWorldspace, job->mChangedTile, mVersion, job->mInput, serialize(*job->mGeneratedNavMeshData)); ++mNextTileId; + mHasChanges = true; } } diff --git a/components/detournavigator/asyncnavmeshupdater.hpp b/components/detournavigator/asyncnavmeshupdater.hpp index fa8cba03af..dc6e9b5a81 100644 --- a/components/detournavigator/asyncnavmeshupdater.hpp +++ b/components/detournavigator/asyncnavmeshupdater.hpp @@ -87,7 +87,7 @@ namespace DetourNavigator public: void push(JobIt job); - std::optional pop(); + std::optional pop(std::chrono::steady_clock::duration timeout); void update(TilePosition playerTile, int maxTiles); @@ -131,13 +131,13 @@ namespace DetourNavigator const RecastSettings& mRecastSettings; const std::unique_ptr mDb; const TileVersion mVersion; - const bool mWriteToDb; + bool mWriteToDb; TileId mNextTileId; ShapeId mNextShapeId; DbJobQueue mQueue; std::atomic_bool mShouldStop {false}; std::atomic_size_t mGetTileCount {0}; - std::size_t mWrites = 0; + bool mHasChanges = false; std::thread mThread; inline void run() noexcept; @@ -173,6 +173,8 @@ namespace DetourNavigator void wait(Loading::Listener& listener, WaitConditionType waitConditionType); + void stop(); + Stats getStats() const; void enqueueJob(JobIt job); diff --git a/components/detournavigator/generatenavmeshtile.cpp b/components/detournavigator/generatenavmeshtile.cpp index 360c05931f..a725c5bc23 100644 --- a/components/detournavigator/generatenavmeshtile.cpp +++ b/components/detournavigator/generatenavmeshtile.cpp @@ -96,6 +96,7 @@ namespace DetourNavigator { Log(Debug::Warning) << "Failed to generate navmesh for worldspace \"" << mWorldspace << "\" tile " << mTilePosition << ": " << e.what(); + consumer->cancel(); } } } diff --git a/components/detournavigator/generatenavmeshtile.hpp b/components/detournavigator/generatenavmeshtile.hpp index e6d9e26c1d..2cd669c8ec 100644 --- a/components/detournavigator/generatenavmeshtile.hpp +++ b/components/detournavigator/generatenavmeshtile.hpp @@ -49,6 +49,8 @@ namespace DetourNavigator virtual void update(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0; + + virtual void cancel() = 0; }; class GenerateNavMeshTile final : public SceneUtil::WorkItem diff --git a/components/detournavigator/navigator.cpp b/components/detournavigator/navigator.cpp index 5b9341523c..d40f330771 100644 --- a/components/detournavigator/navigator.cpp +++ b/components/detournavigator/navigator.cpp @@ -16,7 +16,7 @@ namespace DetourNavigator { try { - db = std::make_unique(userDataPath + "/navmesh.db"); + db = std::make_unique(userDataPath + "/navmesh.db", settings.mMaxDbFileSize); } catch (const std::exception& e) { diff --git a/components/detournavigator/navmeshdb.cpp b/components/detournavigator/navmeshdb.cpp index 621c97f390..4d9ab3dafa 100644 --- a/components/detournavigator/navmeshdb.cpp +++ b/components/detournavigator/navmeshdb.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -130,6 +131,27 @@ namespace DetourNavigator constexpr std::string_view vacuumQuery = R"( VACUUM; )"; + + struct GetPageSize + { + static std::string_view text() noexcept { return "pragma page_size;"; } + static void bind(sqlite3&, sqlite3_stmt&) {} + }; + + std::uint64_t getPageSize(sqlite3& db) + { + Sqlite3::Statement statement(db); + std::uint64_t value = 0; + request(db, statement, &value, 1); + return value; + } + + void setMaxPageCount(sqlite3& db, std::uint64_t value) + { + const auto query = Misc::StringUtils::format("pragma max_page_count = %lu;", value); + if (const int ec = sqlite3_exec(&db, query.c_str(), nullptr, nullptr, nullptr); ec != SQLITE_OK) + throw std::runtime_error("Failed set max page count: " + std::string(sqlite3_errmsg(&db))); + } } std::ostream& operator<<(std::ostream& stream, ShapeType value) @@ -142,7 +164,7 @@ namespace DetourNavigator return stream << "unknown shape type (" << static_cast>(value) << ")"; } - NavMeshDb::NavMeshDb(std::string_view path) + NavMeshDb::NavMeshDb(std::string_view path, std::uint64_t maxFileSize) : mDb(Sqlite3::makeDb(path, schema)) , mGetMaxTileId(*mDb, DbQueries::GetMaxTileId {}) , mFindTile(*mDb, DbQueries::FindTile {}) @@ -157,11 +179,13 @@ namespace DetourNavigator , mInsertShape(*mDb, DbQueries::InsertShape {}) , mVacuum(*mDb, DbQueries::Vacuum {}) { + const auto dbPageSize = getPageSize(*mDb); + setMaxPageCount(*mDb, maxFileSize / dbPageSize + static_cast((maxFileSize % dbPageSize) != 0)); } - Sqlite3::Transaction NavMeshDb::startTransaction() + Sqlite3::Transaction NavMeshDb::startTransaction(Sqlite3::TransactionMode mode) { - return Sqlite3::Transaction(*mDb); + return Sqlite3::Transaction(*mDb, mode); } TileId NavMeshDb::getMaxTileId() diff --git a/components/detournavigator/navmeshdb.hpp b/components/detournavigator/navmeshdb.hpp index 35aaeaf677..812452206e 100644 --- a/components/detournavigator/navmeshdb.hpp +++ b/components/detournavigator/navmeshdb.hpp @@ -141,9 +141,9 @@ namespace DetourNavigator class NavMeshDb { public: - explicit NavMeshDb(std::string_view path); + explicit NavMeshDb(std::string_view path, std::uint64_t maxFileSize); - Sqlite3::Transaction startTransaction(); + Sqlite3::Transaction startTransaction(Sqlite3::TransactionMode mode = Sqlite3::TransactionMode::Default); TileId getMaxTileId(); diff --git a/components/detournavigator/settings.cpp b/components/detournavigator/settings.cpp index cc2c685992..040a0a7b29 100644 --- a/components/detournavigator/settings.cpp +++ b/components/detournavigator/settings.cpp @@ -64,6 +64,7 @@ namespace DetourNavigator result.mNavMeshVersion = ::Settings::Manager::getInt("nav mesh version", "Navigator"); result.mEnableNavMeshDiskCache = ::Settings::Manager::getBool("enable nav mesh disk cache", "Navigator"); result.mWriteToNavMeshDb = ::Settings::Manager::getBool("write to navmeshdb", "Navigator"); + result.mMaxDbFileSize = static_cast(::Settings::Manager::getInt64("max navmeshdb file size", "Navigator")); return result; } diff --git a/components/detournavigator/settings.hpp b/components/detournavigator/settings.hpp index e6be8017d5..b107a6eb85 100644 --- a/components/detournavigator/settings.hpp +++ b/components/detournavigator/settings.hpp @@ -51,6 +51,7 @@ namespace DetourNavigator std::string mNavMeshPathPrefix; std::chrono::milliseconds mMinUpdateInterval; std::int64_t mNavMeshVersion = 0; + std::uint64_t mMaxDbFileSize = 0; }; RecastSettings makeRecastSettingsFromSettingsManager(); diff --git a/components/sqlite3/db.cpp b/components/sqlite3/db.cpp index b1e3afb1ae..f341fef064 100644 --- a/components/sqlite3/db.cpp +++ b/components/sqlite3/db.cpp @@ -10,7 +10,7 @@ namespace Sqlite3 { void CloseSqlite3::operator()(sqlite3* handle) const noexcept { - sqlite3_close(handle); + sqlite3_close_v2(handle); } Db makeDb(std::string_view path, const char* schema) diff --git a/components/sqlite3/transaction.cpp b/components/sqlite3/transaction.cpp index 3012538652..bafd6e8d32 100644 --- a/components/sqlite3/transaction.cpp +++ b/components/sqlite3/transaction.cpp @@ -9,25 +9,43 @@ namespace Sqlite3 { + namespace + { + const char* getBeginStatement(TransactionMode mode) + { + switch (mode) + { + case TransactionMode::Default: return "BEGIN"; + case TransactionMode::Deferred: return "BEGIN DEFERRED"; + case TransactionMode::Immediate: return "BEGIN IMMEDIATE"; + case TransactionMode::Exclusive: return "BEGIN EXCLUSIVE"; + } + throw std::logic_error("Invalid transaction mode: " + std::to_string(static_cast>(mode))); + } + } + void Rollback::operator()(sqlite3* db) const { if (db == nullptr) return; if (const int ec = sqlite3_exec(db, "ROLLBACK", nullptr, nullptr, nullptr); ec != SQLITE_OK) - Log(Debug::Warning) << "Failed to rollback SQLite3 transaction: " << std::string(sqlite3_errmsg(db)); + Log(Debug::Debug) << "Failed to rollback SQLite3 transaction: " << sqlite3_errmsg(db) << " (" << ec << ")"; } - Transaction::Transaction(sqlite3& db) + Transaction::Transaction(sqlite3& db, TransactionMode mode) : mDb(&db) { - if (const int ec = sqlite3_exec(mDb.get(), "BEGIN", nullptr, nullptr, nullptr); ec != SQLITE_OK) - throw std::runtime_error("Failed to start transaction: " + std::string(sqlite3_errmsg(mDb.get()))); + if (const int ec = sqlite3_exec(&db, getBeginStatement(mode), nullptr, nullptr, nullptr); ec != SQLITE_OK) + { + (void) mDb.release(); + throw std::runtime_error("Failed to start transaction: " + std::string(sqlite3_errmsg(&db)) + " (" + std::to_string(ec) + ")"); + } } void Transaction::commit() { if (const int ec = sqlite3_exec(mDb.get(), "COMMIT", nullptr, nullptr, nullptr); ec != SQLITE_OK) - throw std::runtime_error("Failed to commit transaction: " + std::string(sqlite3_errmsg(mDb.get()))); + throw std::runtime_error("Failed to commit transaction: " + std::string(sqlite3_errmsg(mDb.get())) + " (" + std::to_string(ec) + ")"); (void) mDb.release(); } } diff --git a/components/sqlite3/transaction.hpp b/components/sqlite3/transaction.hpp index 88b780a0a5..ff23bc03b5 100644 --- a/components/sqlite3/transaction.hpp +++ b/components/sqlite3/transaction.hpp @@ -12,10 +12,18 @@ namespace Sqlite3 void operator()(sqlite3* handle) const; }; + enum class TransactionMode + { + Default, + Deferred, + Immediate, + Exclusive, + }; + class Transaction { public: - Transaction(sqlite3& db); + explicit Transaction(sqlite3& db, TransactionMode mode = TransactionMode::Default); void commit(); diff --git a/docs/source/reference/modding/settings/navigator.rst b/docs/source/reference/modding/settings/navigator.rst index 6d00c770bc..85924ca8cc 100644 --- a/docs/source/reference/modding/settings/navigator.rst +++ b/docs/source/reference/modding/settings/navigator.rst @@ -70,10 +70,19 @@ write to navmeshdb :Type: boolean :Range: True/False -:Default: False +:Default: True If true generated navmesh tiles will be stored into disk cache while game is running. +max navmeshdb file size +----------------------- + +:Type: integer +:Range: > 0 +:Default: 2147483648 + +Approximate maximum file size of navigation mesh cache stored on disk in bytes (value > 0). + Advanced settings ***************** diff --git a/files/settings-default.cfg b/files/settings-default.cfg index 3d461d2dce..f76b67a7e5 100644 --- a/files/settings-default.cfg +++ b/files/settings-default.cfg @@ -938,7 +938,10 @@ nav mesh version = 1 enable nav mesh disk cache = true # Cache navigation mesh tiles to disk (true, false) -write to navmeshdb = false +write to navmeshdb = true + +# Approximate maximum file size of navigation mesh cache stored on disk in bytes (value > 0) +max navmeshdb file size = 2147483648 [Shadows]