mirror/openmw
1
0
Fork 0
mirror of https://github.com/OpenMW/openmw.git synced 2026-01-04 01:43:07 +00:00
Code Issues Releases Wiki Activity

Process worldspaces sequentially by navmeshtool

Browse source 
Gathering all cells data for all worldspaces may consume a lot memory if
interior cells processing is enabled.
This commit is contained in:
elsid 2025-12-27 15:18:37 +01:00
parent adcec8fded
commit e9468267fc
No known key found for this signature in database
GPG key ID: B845CB9FEE18AB40
6 changed files with 152 additions and 129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
apps/navmeshtool/main.cpp
View file

@ -24,6 +24,7 @@
#include <components/resource/imagemanager.hpp>
#include <components/resource/niffilemanager.hpp>
#include <components/resource/scenemanager.hpp>
#include <components/sceneutil/workqueue.hpp>
#include <components/settings/values.hpp>
#include <components/toutf8/toutf8.hpp>
#include <components/version/version.hpp>
@ -42,7 +43,6 @@
#include <regex>
#include <string>
#include <thread>
#include <type_traits>
#include <utility>
#include <vector>
@ -236,11 +236,42 @@ namespace NavMeshTool
navigatorSettings.mRecast.mSwimHeightScale
= EsmLoader::getGameSetting(esmData.mGameSettings, "fSwimHeightScale").getFloat();
WorldspaceData cellsData = gatherWorldspaceData(navigatorSettings, readers, vfs, bulletShapeManager,
esmData, processInteriorCells, writeBinaryLog, worldspaceFilter);
const std::unordered_map<ESM::RefId, std::vector<std::size_t>> worldspaceCells
= collectWorldspaceCells(esmData, processInteriorCells, worldspaceFilter);
const Status status = generateAllNavMeshTiles(agentBounds, navigatorSettings, threadsNumber,
removeUnusedTiles, writeBinaryLog, cellsData, std::move(db));
Status status = Status::Ok;
bool needVacuum = false;
std::size_t count = 0;
SceneUtil::WorkQueue workQueue(threadsNumber);
Log(Debug::Info) << "Using " << threadsNumber << " parallel workers...";
for (const auto& [worldspace, cells] : worldspaceCells)
{
const WorldspaceData worldspaceData = gatherWorldspaceData(
navigatorSettings, readers, vfs, bulletShapeManager, esmData, writeBinaryLog, worldspace, cells);
const Result result = generateAllNavMeshTiles(
agentBounds, navigatorSettings, removeUnusedTiles, writeBinaryLog, worldspaceData, db, workQueue);
++count;
Log(Debug::Info) << "Processed worldspace (" << count << "/" << worldspaceCells.size() << ") "
<< worldspace;
status = result.mStatus;
needVacuum = needVacuum || result.mNeedVacuum;
if (status != Status::Ok)
break;
}
if (status == Status::Ok && needVacuum)
{
Log(Debug::Info) << "Vacuuming the database...";
db.vacuum();
}
switch (status)
{

77
apps/navmeshtool/navmesh.cpp
View file

@ -26,7 +26,6 @@
#include <cstddef>
#include <random>
#include <string_view>
#include <utility>
#include <vector>
namespace NavMeshTool
@ -78,8 +77,8 @@ namespace NavMeshTool
public:
std::atomic_size_t mExpected{ 0 };
explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles, bool writeBinaryLog)
: mDb(std::move(db))
explicit NavMeshTileConsumer(NavMeshDb& db, bool removeUnusedTiles, bool writeBinaryLog)
: mDb(db)
, mRemoveUnusedTiles(removeUnusedTiles)
, mWriteBinaryLog(writeBinaryLog)
, mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate))
@ -211,12 +210,6 @@ namespace NavMeshTool
mTransaction.commit();
}
void vacuum()
{
const std::lock_guard lock(mMutex);
mDb.vacuum();
}
void removeTilesOutsideRange(ESM::RefId worldspace, const TilesPositionsRange& range)
{
const std::lock_guard lock(mMutex);
@ -233,7 +226,7 @@ namespace NavMeshTool
std::size_t mDeleted = 0;
Status mStatus = Status::Ok;
mutable std::mutex mMutex;
NavMeshDb mDb;
NavMeshDb& mDb;
const bool mRemoveUnusedTiles;
const bool mWriteBinaryLog;
Transaction mTransaction;
@ -254,45 +247,42 @@ namespace NavMeshTool
};
}
Status generateAllNavMeshTiles(const AgentBounds& agentBounds, const Settings& settings, std::size_t threadsNumber,
bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& data, NavMeshDb&& db)
Result generateAllNavMeshTiles(const AgentBounds& agentBounds, const Settings& settings, bool removeUnusedTiles,
bool writeBinaryLog, const WorldspaceData& data, NavMeshDb& db, SceneUtil::WorkQueue& workQueue)
{
Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers...";
Log(Debug::Info) << "Generating navmesh tiles for " << data.mWorldspace << " worldspace...";
SceneUtil::WorkQueue workQueue(threadsNumber);
auto navMeshTileConsumer
= std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles, writeBinaryLog);
std::size_t tiles = 0;
std::mt19937_64 random;
auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(db, removeUnusedTiles, writeBinaryLog);
const auto range = DetourNavigator::makeTilesPositionsRange(
Misc::Convert::toOsgXY(data.mAabb.m_min), Misc::Convert::toOsgXY(data.mAabb.m_max), settings.mRecast);
if (removeUnusedTiles)
navMeshTileConsumer->removeTilesOutsideRange(data.mWorldspace, range);
std::vector<TilePosition> worldspaceTiles;
DetourNavigator::getTilesPositions(
range, [&](const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); });
for (const std::unique_ptr<WorldspaceNavMeshInput>& input : data.mNavMeshInputs)
{
const auto range = DetourNavigator::makeTilesPositionsRange(Misc::Convert::toOsgXY(input->mAabb.m_min),
Misc::Convert::toOsgXY(input->mAabb.m_max), settings.mRecast);
if (removeUnusedTiles)
navMeshTileConsumer->removeTilesOutsideRange(input->mWorldspace, range);
std::vector<TilePosition> worldspaceTiles;
DetourNavigator::getTilesPositions(
range, [&](const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); });
tiles += worldspaceTiles.size();
const std::size_t tiles = worldspaceTiles.size();
if (writeBinaryLog)
serializeToStderr(ExpectedTiles{ static_cast<std::uint64_t>(tiles) });
navMeshTileConsumer->mExpected = tiles;
std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random);
for (const TilePosition& tilePosition : worldspaceTiles)
workQueue.addWorkItem(new GenerateNavMeshTile(input->mWorldspace, tilePosition,
RecastMeshProvider(input->mTileCachedRecastMeshManager), agentBounds, settings,
navMeshTileConsumer));
}
{
std::mt19937_64 random;
std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random);
}
for (const TilePosition& tilePosition : worldspaceTiles)
workQueue.addWorkItem(new GenerateNavMeshTile(data.mWorldspace, tilePosition,
RecastMeshProvider(*data.mTileCachedRecastMeshManager), agentBounds, settings, navMeshTileConsumer));
const Status status = navMeshTileConsumer->wait();
if (status == Status::Ok)
navMeshTileConsumer->commit();
@ -304,12 +294,9 @@ namespace NavMeshTool
Log(Debug::Info) << "Generated navmesh for " << navMeshTileConsumer->getProvided() << " tiles, " << inserted
<< " are inserted, " << updated << " updated and " << deleted << " deleted";
if (inserted + updated + deleted > 0)
{
Log(Debug::Info) << "Vacuuming the database...";
navMeshTileConsumer->vacuum();
}
return status;
return Result{
.mStatus = status,
.mNeedVacuum = inserted + updated + deleted > 0,
};
}
}

19
apps/navmeshtool/navmesh.hpp
View file

@ -1,8 +1,6 @@
#ifndef OPENMW_NAVMESHTOOL_NAVMESH_H
#define OPENMW_NAVMESHTOOL_NAVMESH_H
#include <cstddef>
namespace DetourNavigator
{
class NavMeshDb;
@ -10,6 +8,11 @@ namespace DetourNavigator
struct AgentBounds;
}
namespace SceneUtil
{
class WorkQueue;
}
namespace NavMeshTool
{
struct WorldspaceData;
@ -21,9 +24,15 @@ namespace NavMeshTool
NotEnoughSpace,
};
Status generateAllNavMeshTiles(const DetourNavigator::AgentBounds& agentBounds,
const DetourNavigator::Settings& settings, std::size_t threadsNumber, bool removeUnusedTiles,
bool writeBinaryLog, WorldspaceData& cellsData, DetourNavigator::NavMeshDb&& db);
struct Result
{
Status mStatus;
bool mNeedVacuum;
};
Result generateAllNavMeshTiles(const DetourNavigator::AgentBounds& agentBounds,
const DetourNavigator::Settings& settings, bool removeUnusedTiles, bool writeBinaryLog,
const WorldspaceData& data, DetourNavigator::NavMeshDb& db, SceneUtil::WorkQueue& workQueue);
}
#endif

120
apps/navmeshtool/worldspacedata.cpp
View file

@ -234,28 +234,20 @@ namespace NavMeshTool
}
}
WorldspaceNavMeshInput::WorldspaceNavMeshInput(
ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings)
WorldspaceData::WorldspaceData(ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings)
: mWorldspace(worldspace)
, mTileCachedRecastMeshManager(settings)
, mTileCachedRecastMeshManager(std::make_unique<TileCachedRecastMeshManager>(settings))
{
mAabb.m_min = btVector3(0, 0, 0);
mAabb.m_max = btVector3(0, 0, 0);
}
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
bool processInteriorCells, bool writeBinaryLog, const std::regex& worldspaceFilter)
std::unordered_map<ESM::RefId, std::vector<std::size_t>> collectWorldspaceCells(
const EsmLoader::EsmData& esmData, bool processInteriorCells, const std::regex& worldspaceFilter)
{
Log(Debug::Info) << "Processing " << esmData.mCells.size() << " cells...";
Log(Debug::Info) << "Collecting worldspaces from " << esmData.mCells.size() << " cells...";
std::unordered_map<ESM::RefId, std::unique_ptr<WorldspaceNavMeshInput>> navMeshInputs;
WorldspaceData data;
std::size_t objectsCounter = 0;
if (writeBinaryLog)
serializeToStderr(ExpectedCells{ static_cast<std::uint64_t>(esmData.mCells.size()) });
std::unordered_map<ESM::RefId, std::vector<std::size_t>> result;
for (std::size_t i = 0; i < esmData.mCells.size(); ++i)
{
@ -264,11 +256,9 @@ namespace NavMeshTool
if (!exterior && !processInteriorCells)
{
if (writeBinaryLog)
serializeToStderr(ProcessedCells{ static_cast<std::uint64_t>(i + 1) });
Log(Debug::Info) << "Skipped interior"
<< " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \""
<< cell.getDescription() << "\"";
Log(Debug::Verbose) << "Skipped interior"
<< " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \""
<< cell.getDescription() << "\"";
continue;
}
@ -276,34 +266,52 @@ namespace NavMeshTool
if (!std::regex_match(cellWorldspace.toString(), worldspaceFilter))
{
Log(Debug::Info) << "Skipped filtered out"
<< " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \""
<< cell.getDescription() << "\" from " << cellWorldspace << " worldspace";
Log(Debug::Verbose) << "Skipped filtered out"
<< " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \""
<< cell.getDescription() << "\" from " << cellWorldspace << " worldspace";
continue;
}
result[cellWorldspace].push_back(i);
Log(Debug::Info) << "Collected " << (exterior ? "exterior" : "interior") << " cell (" << (i + 1) << "/"
<< esmData.mCells.size() << ") " << cell.getDescription();
}
Log(Debug::Info) << "Collected " << result.size() << " worldspaces";
return result;
}
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
bool writeBinaryLog, ESM::RefId worldspace, std::span<const std::size_t> cells)
{
Log(Debug::Info) << "Processing " << cells.size() << " cells from worldspace " << worldspace << "...";
if (writeBinaryLog)
serializeToStderr(ExpectedCells{ static_cast<std::uint64_t>(cells.size()) });
WorldspaceData data(worldspace, settings.mRecast);
TileCachedRecastMeshManager& manager = *data.mTileCachedRecastMeshManager;
const auto guard = manager.makeUpdateGuard();
manager.setWorldspace(worldspace, guard.get());
std::size_t objectsCounter = 0;
for (std::size_t i = 0; i < cells.size(); ++i)
{
const ESM::Cell& cell = esmData.mCells[cells[i]];
const bool exterior = cell.isExterior();
Log(Debug::Debug) << "Processing " << (exterior ? "exterior" : "interior") << " cell (" << (i + 1) << "/"
<< esmData.mCells.size() << ") \"" << cell.getDescription() << "\" from "
<< cellWorldspace << " worldspace";
<< cells.size() << ") \"" << cell.getDescription() << "\"";
const osg::Vec2i cellPosition(cell.mData.mX, cell.mData.mY);
const std::size_t cellObjectsBegin = data.mObjects.size();
WorldspaceNavMeshInput& navMeshInput = [&]() -> WorldspaceNavMeshInput& {
auto it = navMeshInputs.find(cellWorldspace);
if (it == navMeshInputs.end())
{
it = navMeshInputs
.emplace(cellWorldspace,
std::make_unique<WorldspaceNavMeshInput>(cellWorldspace, settings.mRecast))
.first;
it->second->mTileCachedRecastMeshManager.setWorldspace(cellWorldspace, nullptr);
}
return *it->second;
}();
const auto guard = navMeshInput.mTileCachedRecastMeshManager.makeUpdateGuard();
if (exterior)
{
const auto it
@ -312,19 +320,16 @@ namespace NavMeshTool
= makeHeightfieldShape(it == esmData.mLands.end() ? std::optional<ESM::Land>() : *it, cellPosition,
data.mHeightfields, data.mLandData);
mergeOrAssign(
getAabb(cellPosition, minHeight, maxHeight), navMeshInput.mAabb, navMeshInput.mAabbInitialized);
mergeOrAssign(getAabb(cellPosition, minHeight, maxHeight), data.mAabb, data.mAabbInitialized);
navMeshInput.mTileCachedRecastMeshManager.addHeightfield(
cellPosition, ESM::Land::REAL_SIZE, heightfieldShape, guard.get());
manager.addHeightfield(cellPosition, ESM::Land::REAL_SIZE, heightfieldShape, guard.get());
navMeshInput.mTileCachedRecastMeshManager.addWater(cellPosition, ESM::Land::REAL_SIZE, -1, guard.get());
manager.addWater(cellPosition, ESM::Land::REAL_SIZE, -1, guard.get());
}
else
{
if ((cell.mData.mFlags & ESM::Cell::HasWater) != 0)
navMeshInput.mTileCachedRecastMeshManager.addWater(
cellPosition, std::numeric_limits<int>::max(), cell.mWater, guard.get());
manager.addWater(cellPosition, std::numeric_limits<int>::max(), cell.mWater, guard.get());
}
forEachObject(cell, esmData, vfs, bulletShapeManager, readers, [&](BulletObject object) {
@ -333,16 +338,15 @@ namespace NavMeshTool
const btTransform& transform = object.getCollisionObject().getWorldTransform();
const btAABB aabb = BulletHelpers::getAabb(*object.getCollisionObject().getCollisionShape(), transform);
mergeOrAssign(aabb, navMeshInput.mAabb, navMeshInput.mAabbInitialized);
mergeOrAssign(aabb, data.mAabb, data.mAabbInitialized);
if (const btCollisionShape* avoid = object.getShapeInstance()->mAvoidCollisionShape.get())
navMeshInput.mAabb.merge(BulletHelpers::getAabb(*avoid, transform));
data.mAabb.merge(BulletHelpers::getAabb(*avoid, transform));
const ObjectId objectId(++objectsCounter);
const CollisionShape shape(object.getShapeInstance(), *object.getCollisionObject().getCollisionShape(),
object.getObjectTransform());
if (!navMeshInput.mTileCachedRecastMeshManager.addObject(
objectId, shape, transform, DetourNavigator::AreaType_ground, guard.get()))
if (!manager.addObject(objectId, shape, transform, DetourNavigator::AreaType_ground, guard.get()))
throw std::logic_error(
makeAddObjectErrorMessage(objectId, DetourNavigator::AreaType_ground, shape));
@ -350,7 +354,7 @@ namespace NavMeshTool
{
const ObjectId avoidObjectId(++objectsCounter);
const CollisionShape avoidShape(object.getShapeInstance(), *avoid, object.getObjectTransform());
if (!navMeshInput.mTileCachedRecastMeshManager.addObject(
if (!manager.addObject(
avoidObjectId, avoidShape, transform, DetourNavigator::AreaType_null, guard.get()))
throw std::logic_error(
makeAddObjectErrorMessage(avoidObjectId, DetourNavigator::AreaType_null, avoidShape));
@ -359,22 +363,16 @@ namespace NavMeshTool
data.mObjects.emplace_back(std::move(object));
});
const auto cellDescription = cell.getDescription();
if (writeBinaryLog)
serializeToStderr(ProcessedCells{ static_cast<std::uint64_t>(i + 1) });
Log(Debug::Info) << "Processed " << (exterior ? "exterior" : "interior") << " cell (" << (i + 1) << "/"
<< esmData.mCells.size() << ") " << cellDescription << " from " << cellWorldspace
<< " worldspace with " << (data.mObjects.size() - cellObjectsBegin) << " objects";
<< cells.size() << ") " << cell.getDescription() << " with "
<< (data.mObjects.size() - cellObjectsBegin) << " objects";
}
data.mNavMeshInputs.reserve(navMeshInputs.size());
std::transform(navMeshInputs.begin(), navMeshInputs.end(), std::back_inserter(data.mNavMeshInputs),
[](auto& v) { return std::move(v.second); });
Log(Debug::Info) << "Processed " << esmData.mCells.size() << " cells, added " << data.mObjects.size()
<< " objects and " << data.mHeightfields.size() << " height fields";
Log(Debug::Info) << "Processed " << cells.size() << " cells, added " << data.mObjects.size() << " objects and "
<< data.mHeightfields.size() << " height fields";
return data;
}

23
apps/navmeshtool/worldspacedata.hpp
View file

@ -13,6 +13,7 @@
#include <memory>
#include <regex>
#include <span>
#include <vector>
namespace ESM
@ -46,16 +47,6 @@ namespace NavMeshTool
using DetourNavigator::ObjectTransform;
using DetourNavigator::TileCachedRecastMeshManager;
struct WorldspaceNavMeshInput
{
ESM::RefId mWorldspace;
TileCachedRecastMeshManager mTileCachedRecastMeshManager;
btAABB mAabb;
bool mAabbInitialized = false;
explicit WorldspaceNavMeshInput(ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings);
};
class BulletObject
{
public:
@ -82,15 +73,23 @@ namespace NavMeshTool
struct WorldspaceData
{
std::vector<std::unique_ptr<WorldspaceNavMeshInput>> mNavMeshInputs;
ESM::RefId mWorldspace;
std::unique_ptr<TileCachedRecastMeshManager> mTileCachedRecastMeshManager;
btAABB mAabb;
bool mAabbInitialized = false;
std::vector<BulletObject> mObjects;
std::vector<std::unique_ptr<ESM::Land::LandData>> mLandData;
std::vector<std::vector<float>> mHeightfields;
WorldspaceData(ESM::RefId worldspace, const DetourNavigator::RecastSettings& settings);
};
std::unordered_map<ESM::RefId, std::vector<std::size_t>> collectWorldspaceCells(
const EsmLoader::EsmData& esmData, bool processInteriorCells, const std::regex& worldspaceFilter);
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
bool processInteriorCells, bool writeBinaryLog, const std::regex& worldspaceFilter);
bool writeBinaryLog, ESM::RefId worldspace, std::span<const std::size_t> cells);
}
#endif

1
components/navmeshtool/protocol.hpp
View file

@ -3,7 +3,6 @@
#include <cstddef>
#include <cstdint>
#include <stdexcept>
#include <variant>
#include <vector>