diff --git a/apps/navmeshtool/main.cpp b/apps/navmeshtool/main.cpp index 63effd41a2..ec40ada33c 100644 --- a/apps/navmeshtool/main.cpp +++ b/apps/navmeshtool/main.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ #include #include #include -#include #include #include @@ -236,11 +236,42 @@ namespace NavMeshTool navigatorSettings.mRecast.mSwimHeightScale = EsmLoader::getGameSetting(esmData.mGameSettings, "fSwimHeightScale").getFloat(); - WorldspaceData cellsData = gatherWorldspaceData(navigatorSettings, readers, vfs, bulletShapeManager, - esmData, processInteriorCells, writeBinaryLog, worldspaceFilter); + const std::unordered_map> worldspaceCells + = collectWorldspaceCells(esmData, processInteriorCells, worldspaceFilter); - const Status status = generateAllNavMeshTiles(agentBounds, navigatorSettings, threadsNumber, - removeUnusedTiles, writeBinaryLog, cellsData, std::move(db)); + Status status = Status::Ok; + bool needVacuum = false; + std::size_t count = 0; + + SceneUtil::WorkQueue workQueue(threadsNumber); + + Log(Debug::Info) << "Using " << threadsNumber << " parallel workers..."; + + for (const auto& [worldspace, cells] : worldspaceCells) + { + const WorldspaceData worldspaceData = gatherWorldspaceData( + navigatorSettings, readers, vfs, bulletShapeManager, esmData, writeBinaryLog, worldspace, cells); + + const Result result = generateAllNavMeshTiles( + agentBounds, navigatorSettings, removeUnusedTiles, writeBinaryLog, worldspaceData, db, workQueue); + + ++count; + + Log(Debug::Info) << "Processed worldspace (" << count << "/" << worldspaceCells.size() << ") " + << worldspace; + + status = result.mStatus; + needVacuum = needVacuum || result.mNeedVacuum; + + if (status != Status::Ok) + break; + } + + if (status == Status::Ok && needVacuum) + { + Log(Debug::Info) << "Vacuuming the database..."; + db.vacuum(); + } switch (status) { diff --git a/apps/navmeshtool/navmesh.cpp b/apps/navmeshtool/navmesh.cpp index 6a4a708ef9..9b80e894b2 100644 --- a/apps/navmeshtool/navmesh.cpp +++ b/apps/navmeshtool/navmesh.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include namespace NavMeshTool @@ -78,8 +77,8 @@ namespace NavMeshTool public: std::atomic_size_t mExpected{ 0 }; - explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles, bool writeBinaryLog) - : mDb(std::move(db)) + explicit NavMeshTileConsumer(NavMeshDb& db, bool removeUnusedTiles, bool writeBinaryLog) + : mDb(db) , mRemoveUnusedTiles(removeUnusedTiles) , mWriteBinaryLog(writeBinaryLog) , mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate)) @@ -211,12 +210,6 @@ namespace NavMeshTool mTransaction.commit(); } - void vacuum() - { - const std::lock_guard lock(mMutex); - mDb.vacuum(); - } - void removeTilesOutsideRange(ESM::RefId worldspace, const TilesPositionsRange& range) { const std::lock_guard lock(mMutex); @@ -233,7 +226,7 @@ namespace NavMeshTool std::size_t mDeleted = 0; Status mStatus = Status::Ok; mutable std::mutex mMutex; - NavMeshDb mDb; + NavMeshDb& mDb; const bool mRemoveUnusedTiles; const bool mWriteBinaryLog; Transaction mTransaction; @@ -254,45 +247,42 @@ namespace NavMeshTool }; } - Status generateAllNavMeshTiles(const AgentBounds& agentBounds, const Settings& settings, std::size_t threadsNumber, - bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& data, NavMeshDb&& db) + Result generateAllNavMeshTiles(const AgentBounds& agentBounds, const Settings& settings, bool removeUnusedTiles, + bool writeBinaryLog, const WorldspaceData& data, NavMeshDb& db, SceneUtil::WorkQueue& workQueue) { - Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers..."; + Log(Debug::Info) << "Generating navmesh tiles for " << data.mWorldspace << " worldspace..."; - SceneUtil::WorkQueue workQueue(threadsNumber); - auto navMeshTileConsumer - = std::make_shared(std::move(db), removeUnusedTiles, writeBinaryLog); - std::size_t tiles = 0; - std::mt19937_64 random; + auto navMeshTileConsumer = std::make_shared(db, removeUnusedTiles, writeBinaryLog); + + const auto range = DetourNavigator::makeTilesPositionsRange( + Misc::Convert::toOsgXY(data.mAabb.m_min), Misc::Convert::toOsgXY(data.mAabb.m_max), settings.mRecast); + + if (removeUnusedTiles) + navMeshTileConsumer->removeTilesOutsideRange(data.mWorldspace, range); + + std::vector worldspaceTiles; + + DetourNavigator::getTilesPositions( + range, [&](const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); }); - for (const std::unique_ptr& input : data.mNavMeshInputs) { - const auto range = DetourNavigator::makeTilesPositionsRange(Misc::Convert::toOsgXY(input->mAabb.m_min), - Misc::Convert::toOsgXY(input->mAabb.m_max), settings.mRecast); - - if (removeUnusedTiles) - navMeshTileConsumer->removeTilesOutsideRange(input->mWorldspace, range); - - std::vector worldspaceTiles; - - DetourNavigator::getTilesPositions( - range, [&](const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); }); - - tiles += worldspaceTiles.size(); + const std::size_t tiles = worldspaceTiles.size(); if (writeBinaryLog) serializeToStderr(ExpectedTiles{ static_cast(tiles) }); navMeshTileConsumer->mExpected = tiles; - - std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random); - - for (const TilePosition& tilePosition : worldspaceTiles) - workQueue.addWorkItem(new GenerateNavMeshTile(input->mWorldspace, tilePosition, - RecastMeshProvider(input->mTileCachedRecastMeshManager), agentBounds, settings, - navMeshTileConsumer)); } + { + std::mt19937_64 random; + std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random); + } + + for (const TilePosition& tilePosition : worldspaceTiles) + workQueue.addWorkItem(new GenerateNavMeshTile(data.mWorldspace, tilePosition, + RecastMeshProvider(*data.mTileCachedRecastMeshManager), agentBounds, settings, navMeshTileConsumer)); + const Status status = navMeshTileConsumer->wait(); if (status == Status::Ok) navMeshTileConsumer->commit(); @@ -304,12 +294,9 @@ namespace NavMeshTool Log(Debug::Info) << "Generated navmesh for " << navMeshTileConsumer->getProvided() << " tiles, " << inserted << " are inserted, " << updated << " updated and " << deleted << " deleted"; - if (inserted + updated + deleted > 0) - { - Log(Debug::Info) << "Vacuuming the database..."; - navMeshTileConsumer->vacuum(); - } - - return status; + return Result{ + .mStatus = status, + .mNeedVacuum = inserted + updated + deleted > 0, + }; } } diff --git a/apps/navmeshtool/navmesh.hpp b/apps/navmeshtool/navmesh.hpp index 4e607f4f2b..2a0eed2fb0 100644 --- a/apps/navmeshtool/navmesh.hpp +++ b/apps/navmeshtool/navmesh.hpp @@ -1,8 +1,6 @@ #ifndef OPENMW_NAVMESHTOOL_NAVMESH_H #define OPENMW_NAVMESHTOOL_NAVMESH_H -#include - namespace DetourNavigator { class NavMeshDb; @@ -10,6 +8,11 @@ namespace DetourNavigator struct AgentBounds; } +namespace SceneUtil +{ + class WorkQueue; +} + namespace NavMeshTool { struct WorldspaceData; @@ -21,9 +24,15 @@ namespace NavMeshTool NotEnoughSpace, }; - Status generateAllNavMeshTiles(const DetourNavigator::AgentBounds& agentBounds, - const DetourNavigator::Settings& settings, std::size_t threadsNumber, bool removeUnusedTiles, - bool writeBinaryLog, WorldspaceData& cellsData, DetourNavigator::NavMeshDb&& db); + struct Result + { + Status mStatus; + bool mNeedVacuum; + }; + + Result generateAllNavMeshTiles(const DetourNavigator::AgentBounds& agentBounds, + const DetourNavigator::Settings& settings, bool removeUnusedTiles, bool writeBinaryLog, + const WorldspaceData& data, DetourNavigator::NavMeshDb& db, SceneUtil::WorkQueue& workQueue); } #endif diff --git a/apps/navmeshtool/worldspacedata.cpp b/apps/navmeshtool/worldspacedata.cpp index 43b3ae5da3..12ed47a333 100644 --- a/apps/navmeshtool/worldspacedata.cpp +++ b/apps/navmeshtool/worldspacedata.cpp @@ -234,28 +234,20 @@ namespace NavMeshTool } } - WorldspaceNavMeshInput::WorldspaceNavMeshInput( - ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings) + WorldspaceData::WorldspaceData(ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings) : mWorldspace(worldspace) - , mTileCachedRecastMeshManager(settings) + , mTileCachedRecastMeshManager(std::make_unique(settings)) { mAabb.m_min = btVector3(0, 0, 0); mAabb.m_max = btVector3(0, 0, 0); } - WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers, - const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData, - bool processInteriorCells, bool writeBinaryLog, const std::regex& worldspaceFilter) + std::unordered_map> collectWorldspaceCells( + const EsmLoader::EsmData& esmData, bool processInteriorCells, const std::regex& worldspaceFilter) { - Log(Debug::Info) << "Processing " << esmData.mCells.size() << " cells..."; + Log(Debug::Info) << "Collecting worldspaces from " << esmData.mCells.size() << " cells..."; - std::unordered_map> navMeshInputs; - WorldspaceData data; - - std::size_t objectsCounter = 0; - - if (writeBinaryLog) - serializeToStderr(ExpectedCells{ static_cast(esmData.mCells.size()) }); + std::unordered_map> result; for (std::size_t i = 0; i < esmData.mCells.size(); ++i) { @@ -264,11 +256,9 @@ namespace NavMeshTool if (!exterior && !processInteriorCells) { - if (writeBinaryLog) - serializeToStderr(ProcessedCells{ static_cast(i + 1) }); - Log(Debug::Info) << "Skipped interior" - << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \"" - << cell.getDescription() << "\""; + Log(Debug::Verbose) << "Skipped interior" + << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \"" + << cell.getDescription() << "\""; continue; } @@ -276,34 +266,52 @@ namespace NavMeshTool if (!std::regex_match(cellWorldspace.toString(), worldspaceFilter)) { - Log(Debug::Info) << "Skipped filtered out" - << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \"" - << cell.getDescription() << "\" from " << cellWorldspace << " worldspace"; + Log(Debug::Verbose) << "Skipped filtered out" + << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \"" + << cell.getDescription() << "\" from " << cellWorldspace << " worldspace"; continue; } + result[cellWorldspace].push_back(i); + + Log(Debug::Info) << "Collected " << (exterior ? "exterior" : "interior") << " cell (" << (i + 1) << "/" + << esmData.mCells.size() << ") " << cell.getDescription(); + } + + Log(Debug::Info) << "Collected " << result.size() << " worldspaces"; + + return result; + } + + WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers, + const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData, + bool writeBinaryLog, ESM::RefId worldspace, std::span cells) + { + Log(Debug::Info) << "Processing " << cells.size() << " cells from worldspace " << worldspace << "..."; + + if (writeBinaryLog) + serializeToStderr(ExpectedCells{ static_cast(cells.size()) }); + + WorldspaceData data(worldspace, settings.mRecast); + TileCachedRecastMeshManager& manager = *data.mTileCachedRecastMeshManager; + + const auto guard = manager.makeUpdateGuard(); + + manager.setWorldspace(worldspace, guard.get()); + + std::size_t objectsCounter = 0; + + for (std::size_t i = 0; i < cells.size(); ++i) + { + const ESM::Cell& cell = esmData.mCells[cells[i]]; + const bool exterior = cell.isExterior(); + Log(Debug::Debug) << "Processing " << (exterior ? "exterior" : "interior") << " cell (" << (i + 1) << "/" - << esmData.mCells.size() << ") \"" << cell.getDescription() << "\" from " - << cellWorldspace << " worldspace"; + << cells.size() << ") \"" << cell.getDescription() << "\""; const osg::Vec2i cellPosition(cell.mData.mX, cell.mData.mY); const std::size_t cellObjectsBegin = data.mObjects.size(); - WorldspaceNavMeshInput& navMeshInput = [&]() -> WorldspaceNavMeshInput& { - auto it = navMeshInputs.find(cellWorldspace); - if (it == navMeshInputs.end()) - { - it = navMeshInputs - .emplace(cellWorldspace, - std::make_unique(cellWorldspace, settings.mRecast)) - .first; - it->second->mTileCachedRecastMeshManager.setWorldspace(cellWorldspace, nullptr); - } - return *it->second; - }(); - - const auto guard = navMeshInput.mTileCachedRecastMeshManager.makeUpdateGuard(); - if (exterior) { const auto it @@ -312,19 +320,16 @@ namespace NavMeshTool = makeHeightfieldShape(it == esmData.mLands.end() ? std::optional() : *it, cellPosition, data.mHeightfields, data.mLandData); - mergeOrAssign( - getAabb(cellPosition, minHeight, maxHeight), navMeshInput.mAabb, navMeshInput.mAabbInitialized); + mergeOrAssign(getAabb(cellPosition, minHeight, maxHeight), data.mAabb, data.mAabbInitialized); - navMeshInput.mTileCachedRecastMeshManager.addHeightfield( - cellPosition, ESM::Land::REAL_SIZE, heightfieldShape, guard.get()); + manager.addHeightfield(cellPosition, ESM::Land::REAL_SIZE, heightfieldShape, guard.get()); - navMeshInput.mTileCachedRecastMeshManager.addWater(cellPosition, ESM::Land::REAL_SIZE, -1, guard.get()); + manager.addWater(cellPosition, ESM::Land::REAL_SIZE, -1, guard.get()); } else { if ((cell.mData.mFlags & ESM::Cell::HasWater) != 0) - navMeshInput.mTileCachedRecastMeshManager.addWater( - cellPosition, std::numeric_limits::max(), cell.mWater, guard.get()); + manager.addWater(cellPosition, std::numeric_limits::max(), cell.mWater, guard.get()); } forEachObject(cell, esmData, vfs, bulletShapeManager, readers, [&](BulletObject object) { @@ -333,16 +338,15 @@ namespace NavMeshTool const btTransform& transform = object.getCollisionObject().getWorldTransform(); const btAABB aabb = BulletHelpers::getAabb(*object.getCollisionObject().getCollisionShape(), transform); - mergeOrAssign(aabb, navMeshInput.mAabb, navMeshInput.mAabbInitialized); + mergeOrAssign(aabb, data.mAabb, data.mAabbInitialized); if (const btCollisionShape* avoid = object.getShapeInstance()->mAvoidCollisionShape.get()) - navMeshInput.mAabb.merge(BulletHelpers::getAabb(*avoid, transform)); + data.mAabb.merge(BulletHelpers::getAabb(*avoid, transform)); const ObjectId objectId(++objectsCounter); const CollisionShape shape(object.getShapeInstance(), *object.getCollisionObject().getCollisionShape(), object.getObjectTransform()); - if (!navMeshInput.mTileCachedRecastMeshManager.addObject( - objectId, shape, transform, DetourNavigator::AreaType_ground, guard.get())) + if (!manager.addObject(objectId, shape, transform, DetourNavigator::AreaType_ground, guard.get())) throw std::logic_error( makeAddObjectErrorMessage(objectId, DetourNavigator::AreaType_ground, shape)); @@ -350,7 +354,7 @@ namespace NavMeshTool { const ObjectId avoidObjectId(++objectsCounter); const CollisionShape avoidShape(object.getShapeInstance(), *avoid, object.getObjectTransform()); - if (!navMeshInput.mTileCachedRecastMeshManager.addObject( + if (!manager.addObject( avoidObjectId, avoidShape, transform, DetourNavigator::AreaType_null, guard.get())) throw std::logic_error( makeAddObjectErrorMessage(avoidObjectId, DetourNavigator::AreaType_null, avoidShape)); @@ -359,22 +363,16 @@ namespace NavMeshTool data.mObjects.emplace_back(std::move(object)); }); - const auto cellDescription = cell.getDescription(); - if (writeBinaryLog) serializeToStderr(ProcessedCells{ static_cast(i + 1) }); Log(Debug::Info) << "Processed " << (exterior ? "exterior" : "interior") << " cell (" << (i + 1) << "/" - << esmData.mCells.size() << ") " << cellDescription << " from " << cellWorldspace - << " worldspace with " << (data.mObjects.size() - cellObjectsBegin) << " objects"; + << cells.size() << ") " << cell.getDescription() << " with " + << (data.mObjects.size() - cellObjectsBegin) << " objects"; } - data.mNavMeshInputs.reserve(navMeshInputs.size()); - std::transform(navMeshInputs.begin(), navMeshInputs.end(), std::back_inserter(data.mNavMeshInputs), - [](auto& v) { return std::move(v.second); }); - - Log(Debug::Info) << "Processed " << esmData.mCells.size() << " cells, added " << data.mObjects.size() - << " objects and " << data.mHeightfields.size() << " height fields"; + Log(Debug::Info) << "Processed " << cells.size() << " cells, added " << data.mObjects.size() << " objects and " + << data.mHeightfields.size() << " height fields"; return data; } diff --git a/apps/navmeshtool/worldspacedata.hpp b/apps/navmeshtool/worldspacedata.hpp index 5a74a57342..858cb42e7a 100644 --- a/apps/navmeshtool/worldspacedata.hpp +++ b/apps/navmeshtool/worldspacedata.hpp @@ -13,6 +13,7 @@ #include #include +#include #include namespace ESM @@ -46,16 +47,6 @@ namespace NavMeshTool using DetourNavigator::ObjectTransform; using DetourNavigator::TileCachedRecastMeshManager; - struct WorldspaceNavMeshInput - { - ESM::RefId mWorldspace; - TileCachedRecastMeshManager mTileCachedRecastMeshManager; - btAABB mAabb; - bool mAabbInitialized = false; - - explicit WorldspaceNavMeshInput(ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings); - }; - class BulletObject { public: @@ -82,15 +73,23 @@ namespace NavMeshTool struct WorldspaceData { - std::vector> mNavMeshInputs; + ESM::RefId mWorldspace; + std::unique_ptr mTileCachedRecastMeshManager; + btAABB mAabb; + bool mAabbInitialized = false; std::vector mObjects; std::vector> mLandData; std::vector> mHeightfields; + + WorldspaceData(ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings); }; + std::unordered_map> collectWorldspaceCells( + const EsmLoader::EsmData& esmData, bool processInteriorCells, const std::regex& worldspaceFilter); + WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers, const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData, - bool processInteriorCells, bool writeBinaryLog, const std::regex& worldspaceFilter); + bool writeBinaryLog, ESM::RefId worldspace, std::span cells); } #endif diff --git a/components/navmeshtool/protocol.hpp b/components/navmeshtool/protocol.hpp index 3ee8baa949..47cfc4d77c 100644 --- a/components/navmeshtool/protocol.hpp +++ b/components/navmeshtool/protocol.hpp @@ -3,7 +3,6 @@ #include #include -#include #include #include