Prepare navmesh scene asynchronously

It is expensive operation to generate new osg::Group for updated navmesh tile
which noticeably slows down main thread primarily because of
SceneManager::recreateShaders call. Move it to the preload work queue that is
used by RenderingManager. Leave to main thread only manipulations on the root
node.

Also move deallocation of no more needed data to the work queue. It's also
quite expensive operation because SceneManager::recreateShaders allocates a
new state set for each osg::Geometry. Deallocating them takes time.

Avoid creating another work item if there is existing one that is not started
yet.

Make sure results are accepted in the proper serialized order by selecting
completed work item with maximum {id, version}.

apps/openmw/mwrender/navmesh.cpp

@@ -6,6 +6,8 @@
 #include <components/resource/scenemanager.hpp>
 #include <components/detournavigator/navmeshcacheitem.hpp>
 #include <components/sceneutil/detourdebugdraw.hpp>
+#include <components/sceneutil/workqueue.hpp>
+#include <components/detournavigator/settings.hpp>
 
 #include <osg/PositionAttitudeTransform>
 #include <osg/StateSet>
@@ -17,8 +19,137 @@
 
 namespace MWRender
 {
-    NavMesh::NavMesh(const osg::ref_ptr<osg::Group>& root, bool enabled)
+    struct NavMesh::LessByTilePosition
+    {
+        bool operator()(const DetourNavigator::TilePosition& lhs,
+                        const std::pair<DetourNavigator::TilePosition, DetourNavigator::Version>& rhs) const
+        {
+            return lhs < rhs.first;
+        }
+
+        bool operator()(const std::pair<DetourNavigator::TilePosition, DetourNavigator::Version>& lhs,
+                        const DetourNavigator::TilePosition& rhs) const
+        {
+            return lhs.first < rhs;
+        }
+    };
+
+    struct NavMesh::CreateNavMeshTileGroups final : SceneUtil::WorkItem
+    {
+        std::size_t mId;
+        DetourNavigator::Version mVersion;
+        const std::weak_ptr<DetourNavigator::GuardedNavMeshCacheItem> mNavMesh;
+        const osg::ref_ptr<osg::StateSet> mGroupStateSet;
+        const osg::ref_ptr<osg::StateSet> mDebugDrawStateSet;
+        const DetourNavigator::Settings mSettings;
+        std::map<DetourNavigator::TilePosition, Tile> mTiles;
+        std::atomic_bool mAborted {false};
+        std::mutex mMutex;
+        bool mStarted = false;
+        std::vector<std::pair<DetourNavigator::TilePosition, Tile>> mUpdatedTiles;
+        std::vector<DetourNavigator::TilePosition> mRemovedTiles;
+
+        explicit CreateNavMeshTileGroups(std::size_t id, DetourNavigator::Version version,
+            std::weak_ptr<DetourNavigator::GuardedNavMeshCacheItem> navMesh,
+            const osg::ref_ptr<osg::StateSet>& groupStateSet, const osg::ref_ptr<osg::StateSet>& debugDrawStateSet,
+            const DetourNavigator::Settings& settings, const std::map<DetourNavigator::TilePosition, Tile>& tiles)
+            : mId(id)
+            , mVersion(version)
+            , mNavMesh(navMesh)
+            , mGroupStateSet(groupStateSet)
+            , mDebugDrawStateSet(debugDrawStateSet)
+            , mSettings(settings)
+            , mTiles(tiles)
+        {
+        }
+
+        void doWork() final
+        {
+            using DetourNavigator::TilePosition;
+            using DetourNavigator::Version;
+
+            const std::lock_guard lock(mMutex);
+            mStarted = true;
+
+            if (mAborted.load(std::memory_order_acquire))
+                return;
+
+            const auto navMeshPtr = mNavMesh.lock();
+            if (navMeshPtr == nullptr)
+                return;
+
+            std::vector<std::pair<DetourNavigator::TilePosition, Version>> existingTiles;
+
+            navMeshPtr->lockConst()->forEachUsedTile([&] (const TilePosition& position, const Version& version, const dtMeshTile& /*meshTile*/)
+            {
+                existingTiles.emplace_back(position, version);
+            });
+
+            if (mAborted.load(std::memory_order_acquire))
+                return;
+
+            std::sort(existingTiles.begin(), existingTiles.end());
+
+            std::vector<DetourNavigator::TilePosition> removedTiles;
+
+            for (const auto& [position, tile] : mTiles)
+                if (!std::binary_search(existingTiles.begin(), existingTiles.end(), position, LessByTilePosition {}))
+                    removedTiles.push_back(position);
+
+            std::vector<std::pair<TilePosition, Tile>> updatedTiles;
+
+            for (const auto& [position, version] : existingTiles)
+            {
+                const auto it = mTiles.find(position);
+                if (it != mTiles.end() && it->second.mGroup != nullptr && it->second.mVersion == version)
+                    continue;
+
+                osg::ref_ptr<osg::Group> group;
+                {
+                    const auto navMesh = navMeshPtr->lockConst();
+                    const dtMeshTile* meshTile = DetourNavigator::getTile(navMesh->getImpl(), position);
+                    if (meshTile == nullptr)
+                        continue;
+
+                    if (mAborted.load(std::memory_order_acquire))
+                        return;
+
+                    group = SceneUtil::createNavMeshTileGroup(navMesh->getImpl(), *meshTile, mSettings, mGroupStateSet, mDebugDrawStateSet);
+                }
+                if (group == nullptr)
+                {
+                    removedTiles.push_back(position);
+                    continue;
+                }
+                MWBase::Environment::get().getResourceSystem()->getSceneManager()->recreateShaders(group, "debug");
+                group->setNodeMask(Mask_Debug);
+                updatedTiles.emplace_back(position, Tile {version, std::move(group)});
+            }
+
+            if (mAborted.load(std::memory_order_acquire))
+                return;
+
+            mUpdatedTiles = std::move(updatedTiles);
+            mRemovedTiles = std::move(removedTiles);
+        }
+
+        void abort() final
+        {
+            mAborted.store(true, std::memory_order_release);
+        }
+    };
+
+    struct NavMesh::DeallocateCreateNavMeshTileGroups final : SceneUtil::WorkItem
+    {
+        osg::ref_ptr<NavMesh::CreateNavMeshTileGroups> mWorkItem;
+
+        explicit DeallocateCreateNavMeshTileGroups(osg::ref_ptr<NavMesh::CreateNavMeshTileGroups>&& workItem)
+            : mWorkItem(std::move(workItem)) {}
+    };
+
+    NavMesh::NavMesh(const osg::ref_ptr<osg::Group>& root, const osg::ref_ptr<SceneUtil::WorkQueue>& workQueue, bool enabled)
         : mRootNode(root)
+        , mWorkQueue(workQueue)
         , mGroupStateSet(SceneUtil::makeNavMeshTileStateSet())
         , mDebugDrawStateSet(SceneUtil::DebugDraw::makeStateSet())
         , mEnabled(enabled)
@@ -30,6 +161,8 @@ namespace MWRender
     {
         if (mEnabled)
             disable();
+        for (const auto& workItem : mWorkItems)
+            workItem->abort();
     }
 
     bool NavMesh::toggle()
@@ -42,13 +175,69 @@ namespace MWRender
         return mEnabled;
     }
 
-    void NavMesh::update(const DetourNavigator::NavMeshCacheItem& navMesh, std::size_t id,
+    void NavMesh::update(const std::shared_ptr<DetourNavigator::GuardedNavMeshCacheItem>& navMesh, std::size_t id,
         const DetourNavigator::Settings& settings)
     {
         using DetourNavigator::TilePosition;
         using DetourNavigator::Version;
 
-        if (!mEnabled || (!mTiles.empty() && mId == id && mVersion == navMesh.getVersion()))
+        if (!mEnabled)
+            return;
+
+        {
+            std::pair<std::size_t, Version> lastest {0, Version {}};
+            osg::ref_ptr<CreateNavMeshTileGroups> latestCandidate;
+            for (auto it = mWorkItems.begin(); it != mWorkItems.end();)
+            {
+                if (!(*it)->isDone())
+                {
+                    ++it;
+                    continue;
+                }
+                const std::pair<std::size_t, Version> order {(*it)->mId, (*it)->mVersion};
+                if (lastest < order)
+                {
+                    lastest = order;
+                    std::swap(latestCandidate, *it);
+                }
+                if (*it != nullptr)
+                    mWorkQueue->addWorkItem(new DeallocateCreateNavMeshTileGroups(std::move(*it)));
+                it = mWorkItems.erase(it);
+            }
+
+            if (latestCandidate != nullptr)
+            {
+                for (const TilePosition& position : latestCandidate->mRemovedTiles)
+                {
+                    const auto it = mTiles.find(position);
+                    if (it == mTiles.end())
+                        continue;
+                    mRootNode->removeChild(it->second.mGroup);
+                    mTiles.erase(it);
+                }
+
+                for (auto& [position, tile] : latestCandidate->mUpdatedTiles)
+                {
+                    const auto it = mTiles.find(position);
+                    if (it == mTiles.end())
+                    {
+                        mRootNode->addChild(tile.mGroup);
+                        mTiles.emplace_hint(it, position, std::move(tile));
+                    }
+                    else
+                    {
+                        mRootNode->replaceChild(it->second.mGroup, tile.mGroup);
+                        std::swap(it->second, tile);
+                    }
+                }
+
+                mWorkQueue->addWorkItem(new DeallocateCreateNavMeshTileGroups(std::move(latestCandidate)));
+            }
+        }
+
+        const auto version = navMesh->lock()->getVersion();
+
+        if (!mTiles.empty() && mId == id && mVersion == version)
             return;
 
         if (mId != id)
@@ -57,40 +246,35 @@ namespace MWRender
             mId = id;
         }
 
-        mVersion = navMesh.getVersion();
+        mVersion = version;
 
-        std::vector<TilePosition> updated;
-        navMesh.forEachUsedTile([&] (const TilePosition& position, const Version& version, const dtMeshTile& meshTile)
+        for (auto& workItem : mWorkItems)
         {
-            updated.push_back(position);
-            Tile& tile = mTiles[position];
-            if (tile.mGroup != nullptr && tile.mVersion == version)
-                return;
-            if (tile.mGroup != nullptr)
-                mRootNode->removeChild(tile.mGroup);
-            tile.mGroup = SceneUtil::createNavMeshTileGroup(navMesh.getImpl(), meshTile, settings,
-                                                            mGroupStateSet, mDebugDrawStateSet);
-            if (tile.mGroup == nullptr)
-                return;
-            MWBase::Environment::get().getResourceSystem()->getSceneManager()->recreateShaders(tile.mGroup, "debug");
-            tile.mGroup->setNodeMask(Mask_Debug);
-            mRootNode->addChild(tile.mGroup);
-        });
-        std::sort(updated.begin(), updated.end());
-        for (auto it = mTiles.begin(); it != mTiles.end();)
-        {
-            if (!std::binary_search(updated.begin(), updated.end(), it->first))
-            {
-                mRootNode->removeChild(it->second.mGroup);
-                it = mTiles.erase(it);
-            }
-            else
-                ++it;
+            const std::unique_lock lock(workItem->mMutex, std::try_to_lock);
+
+            if (!lock.owns_lock())
+                continue;
+
+            if (workItem->mStarted)
+                continue;
+
+            workItem->mId = id;
+            workItem->mVersion = version;
+            workItem->mTiles = mTiles;
+
+            return;
         }
+
+        osg::ref_ptr<CreateNavMeshTileGroups> workItem = new CreateNavMeshTileGroups(id, version, navMesh, mGroupStateSet, mDebugDrawStateSet, settings, mTiles);
+        mWorkQueue->addWorkItem(workItem);
+        mWorkItems.push_back(std::move(workItem));
     }
 
     void NavMesh::reset()
     {
+        for (auto& workItem : mWorkItems)
+            workItem->abort();
+        mWorkItems.clear();
         for (auto& [position, tile] : mTiles)
             mRootNode->removeChild(tile.mGroup);
         mTiles.clear();
@@ -98,15 +282,12 @@ namespace MWRender
 
     void NavMesh::enable()
     {
-        for (const auto& [position, tile] : mTiles)
-            mRootNode->addChild(tile.mGroup);
         mEnabled = true;
     }
 
     void NavMesh::disable()
     {
-        for (const auto& [position, tile] : mTiles)
-            mRootNode->removeChild(tile.mGroup);
+        reset();
         mEnabled = false;
     }
 }

apps/openmw/mwrender/navmesh.hpp

@@ -3,11 +3,14 @@
 
 #include <components/detournavigator/version.hpp>
 #include <components/detournavigator/tileposition.hpp>
+#include <components/misc/guarded.hpp>
 
 #include <osg/ref_ptr>
 
 #include <cstddef>
 #include <map>
+#include <memory>
+#include <vector>
 
 class dtNavMesh;
 
@@ -24,18 +27,24 @@ namespace DetourNavigator
     struct Settings;
 }
 
+namespace SceneUtil
+{
+    class WorkQueue;
+}
+
 namespace MWRender
 {
     class NavMesh
     {
     public:
-        NavMesh(const osg::ref_ptr<osg::Group>& root, bool enabled);
+        explicit NavMesh(const osg::ref_ptr<osg::Group>& root, const osg::ref_ptr<SceneUtil::WorkQueue>& workQueue,
+                         bool enabled);
         ~NavMesh();
 
         bool toggle();
 
-        void update(const DetourNavigator::NavMeshCacheItem& navMesh, std::size_t id,
-            const DetourNavigator::Settings& settings);
+        void update(const std::shared_ptr<Misc::ScopeGuarded<DetourNavigator::NavMeshCacheItem>>& navMesh,
+            std::size_t id, const DetourNavigator::Settings& settings);
 
         void reset();
 
@@ -55,13 +64,19 @@ namespace MWRender
             osg::ref_ptr<osg::Group> mGroup;
         };
 
+        struct LessByTilePosition;
+        struct CreateNavMeshTileGroups;
+        struct DeallocateCreateNavMeshTileGroups;
+
         osg::ref_ptr<osg::Group> mRootNode;
+        osg::ref_ptr<SceneUtil::WorkQueue> mWorkQueue;
         osg::ref_ptr<osg::StateSet> mGroupStateSet;
         osg::ref_ptr<osg::StateSet> mDebugDrawStateSet;
         bool mEnabled;
         std::size_t mId;
         DetourNavigator::Version mVersion;
         std::map<DetourNavigator::TilePosition, Tile> mTiles;
+        std::vector<osg::ref_ptr<CreateNavMeshTileGroups>> mWorkItems;
     };
 }
 

apps/openmw/mwrender/renderingmanager.cpp

@@ -384,7 +384,7 @@ namespace MWRender
         // It is unnecessary to stop/start the viewer as no frames are being rendered yet.
         mResourceSystem->getSceneManager()->getShaderManager().setGlobalDefines(globalDefines);
 
-        mNavMesh.reset(new NavMesh(mRootNode, Settings::Manager::getBool("enable nav mesh render", "Navigator")));
+        mNavMesh.reset(new NavMesh(mRootNode, mWorkQueue, Settings::Manager::getBool("enable nav mesh render", "Navigator")));
         mActorsPaths.reset(new ActorsPaths(mRootNode, Settings::Manager::getBool("enable agents paths render", "Navigator")));
         mRecastMesh.reset(new RecastMesh(mRootNode, Settings::Manager::getBool("enable recast mesh render", "Navigator")));
         mPathgrid.reset(new Pathgrid(mRootNode));
@@ -1395,7 +1395,7 @@ namespace MWRender
         {
             try
             {
-                mNavMesh->update(*it->second->lockConst(), mNavMeshNumber, mNavigator.getSettings());
+                mNavMesh->update(it->second, mNavMeshNumber, mNavigator.getSettings());
             }
             catch (const std::exception& e)
             {