Merge branch 'fix_new_game_guard_3' into 'master'

Fix new game guard 3 (#6126)

Closes #6126

See merge request OpenMW/openmw!976

(cherry picked from commit 89ca56632c6bc37f07e430e7f5cd23d855610b19)

a3942a1e Remove redundant check for y coordinate in inRange function
cc08a45c Move include where it is needed
7e1630a7 Remove redundant getPolyHeight wrapper
793c30ab Check dtNavMeshQuery::getPolyHeight status
a54c4bc2 Check dtNavMeshQuery::findStraightPath status
94e460ba Use proper check for distance
5624fe19 Consider path not found when there is navmesh query error

apps/openmw/mwmechanics/pathfinding.cpp

@@ -369,7 +369,13 @@ namespace MWMechanics
         mPath.clear();
 
         // If it's not possible to build path over navmesh due to disabled navmesh generation fallback to straight path
-        if (!buildPathByNavigatorImpl(actor, startPoint, endPoint, halfExtents, flags, areaCosts, std::back_inserter(mPath)))
+        DetourNavigator::Status status = buildPathByNavigatorImpl(actor, startPoint, endPoint, halfExtents, flags,
+                                                                  areaCosts, std::back_inserter(mPath));
+
+        if (status != DetourNavigator::Status::Success)
+            mPath.clear();
+
+        if (status == DetourNavigator::Status::NavMeshNotFound)
             mPath.push_back(endPoint);
 
         mConstructed = !mPath.empty();
@@ -382,25 +388,33 @@ namespace MWMechanics
         mPath.clear();
         mCell = cell;
 
-        bool hasNavMesh = false;
+        DetourNavigator::Status status = DetourNavigator::Status::NavMeshNotFound;
 
         if (!actor.getClass().isPureWaterCreature(actor) && !actor.getClass().isPureFlyingCreature(actor))
-            hasNavMesh = buildPathByNavigatorImpl(actor, startPoint, endPoint, halfExtents, flags, areaCosts, std::back_inserter(mPath));
+        {
+            status = buildPathByNavigatorImpl(actor, startPoint, endPoint, halfExtents, flags, areaCosts, std::back_inserter(mPath));
+            if (status != DetourNavigator::Status::Success)
+                mPath.clear();
+        }
 
-        if (hasNavMesh && mPath.empty())
-            buildPathByNavigatorImpl(actor, startPoint, endPoint, halfExtents,
+        if (status != DetourNavigator::Status::NavMeshNotFound && mPath.empty())
+        {
+            status = buildPathByNavigatorImpl(actor, startPoint, endPoint, halfExtents,
                                      flags | DetourNavigator::Flag_usePathgrid, areaCosts, std::back_inserter(mPath));
+            if (status != DetourNavigator::Status::Success)
+                mPath.clear();
+        }
 
         if (mPath.empty())
             buildPathByPathgridImpl(startPoint, endPoint, pathgridGraph, std::back_inserter(mPath));
 
-        if (!hasNavMesh && mPath.empty())
+        if (status == DetourNavigator::Status::NavMeshNotFound && mPath.empty())
             mPath.push_back(endPoint);
 
         mConstructed = !mPath.empty();
     }
 
-    bool PathFinder::buildPathByNavigatorImpl(const MWWorld::ConstPtr& actor, const osg::Vec3f& startPoint,
+    DetourNavigator::Status PathFinder::buildPathByNavigatorImpl(const MWWorld::ConstPtr& actor, const osg::Vec3f& startPoint,
         const osg::Vec3f& endPoint, const osg::Vec3f& halfExtents, const DetourNavigator::Flags flags,
         const DetourNavigator::AreaCosts& areaCosts, std::back_insert_iterator<std::deque<osg::Vec3f>> out)
     {
@@ -409,9 +423,6 @@ namespace MWMechanics
         const auto navigator = world->getNavigator();
         const auto status = navigator->findPath(halfExtents, stepSize, startPoint, endPoint, flags, areaCosts, out);
 
-        if (status == DetourNavigator::Status::NavMeshNotFound)
-            return false;
-
         if (status != DetourNavigator::Status::Success)
         {
             Log(Debug::Debug) << "Build path by navigator error: \"" << DetourNavigator::getMessage(status)
@@ -420,7 +431,7 @@ namespace MWMechanics
                 << DetourNavigator::WriteFlags {flags} << ")";
         }
 
-        return true;
+        return status;
     }
 
     void PathFinder::buildPathByNavMeshToNextPoint(const MWWorld::ConstPtr& actor, const osg::Vec3f& halfExtents,

apps/openmw/mwmechanics/pathfinding.hpp

@@ -7,6 +7,7 @@
 
 #include <components/detournavigator/flags.hpp>
 #include <components/detournavigator/areatype.hpp>
+#include <components/detournavigator/status.hpp>
 #include <components/esm/defs.hpp>
 #include <components/esm/loadpgrd.hpp>
 
@@ -209,9 +210,10 @@ namespace MWMechanics
             void buildPathByPathgridImpl(const osg::Vec3f& startPoint, const osg::Vec3f& endPoint,
                 const PathgridGraph& pathgridGraph, std::back_insert_iterator<std::deque<osg::Vec3f>> out);
 
-            bool buildPathByNavigatorImpl(const MWWorld::ConstPtr& actor, const osg::Vec3f& startPoint,
-                const osg::Vec3f& endPoint, const osg::Vec3f& halfExtents, const DetourNavigator::Flags flags,
-                const DetourNavigator::AreaCosts& areaCosts, std::back_insert_iterator<std::deque<osg::Vec3f>> out);
+            [[nodiscard]] DetourNavigator::Status buildPathByNavigatorImpl(const MWWorld::ConstPtr& actor,
+                const osg::Vec3f& startPoint, const osg::Vec3f& endPoint, const osg::Vec3f& halfExtents,
+                const DetourNavigator::Flags flags, const DetourNavigator::AreaCosts& areaCosts,
+                std::back_insert_iterator<std::deque<osg::Vec3f>> out);
     };
 }
 

components/detournavigator/findsmoothpath.cpp

@@ -1,5 +1,7 @@
 #include "findsmoothpath.hpp"
 
+#include <components/misc/convert.hpp>
+
 #include <algorithm>
 #include <array>
 
@@ -103,7 +105,7 @@ namespace DetourNavigator
         return result;
     }
 
-    std::optional<SteerTarget> getSteerTarget(const dtNavMeshQuery& navQuery, const osg::Vec3f& startPos,
+    std::optional<SteerTarget> getSteerTarget(const dtNavMeshQuery& navMeshQuery, const osg::Vec3f& startPos,
             const osg::Vec3f& endPos, const float minTargetDist, const std::vector<dtPolyRef>& path)
     {
         // Find steer target.
@@ -113,8 +115,11 @@ namespace DetourNavigator
         std::array<unsigned char, maxSteerPoints> steerPathFlags;
         std::array<dtPolyRef, maxSteerPoints> steerPathPolys;
         int nsteerPath = 0;
-        navQuery.findStraightPath(startPos.ptr(), endPos.ptr(), path.data(), int(path.size()), steerPath.data(),
-            steerPathFlags.data(), steerPathPolys.data(), &nsteerPath, maxSteerPoints);
+        const dtStatus status = navMeshQuery.findStraightPath(startPos.ptr(), endPos.ptr(), path.data(),
+            static_cast<int>(path.size()), steerPath.data(), steerPathFlags.data(), steerPathPolys.data(),
+            &nsteerPath, maxSteerPoints);
+        if (dtStatusFailed(status))
+            return std::nullopt;
         assert(nsteerPath >= 0);
         if (!nsteerPath)
             return std::nullopt;
@@ -125,7 +130,7 @@ namespace DetourNavigator
         {
             // Stop at Off-Mesh link or when point is further than slop away.
             if ((steerPathFlags[ns] & DT_STRAIGHTPATH_OFFMESH_CONNECTION) ||
-                    !inRange(Misc::Convert::makeOsgVec3f(&steerPath[ns * 3]), startPos, minTargetDist, 1000.0f))
+                    !inRange(Misc::Convert::makeOsgVec3f(&steerPath[ns * 3]), startPos, minTargetDist))
                 break;
             ns++;
         }

components/detournavigator/findsmoothpath.hpp

@@ -14,9 +14,8 @@
 #include <DetourNavMesh.h>
 #include <DetourNavMeshQuery.h>
 
-#include <components/misc/convert.hpp>
-
 #include <osg/Vec3f>
+
 #include <cassert>
 #include <vector>
 
@@ -26,10 +25,9 @@ namespace DetourNavigator
 {
     struct Settings;
 
-    inline bool inRange(const osg::Vec3f& v1, const osg::Vec3f& v2, const float r, const float h)
+    inline bool inRange(const osg::Vec3f& v1, const osg::Vec3f& v2, const float r)
     {
-        const auto d = v2 - v1;
-        return (d.x() * d.x() + d.z() * d.z()) < r * r && std::abs(d.y()) < h;
+        return (osg::Vec2f(v1.x(), v1.z()) - osg::Vec2f(v2.x(), v2.z())).length() < r;
     }
 
     std::vector<dtPolyRef> fixupCorridor(const std::vector<dtPolyRef>& path, const std::vector<dtPolyRef>& visited);
@@ -143,15 +141,6 @@ namespace DetourNavigator
         return {std::move(result)};
     }
 
-    inline std::optional<float> getPolyHeight(const dtNavMeshQuery& navMeshQuery, const dtPolyRef ref, const osg::Vec3f& pos)
-    {
-        float result = 0.0f;
-        const auto status = navMeshQuery.getPolyHeight(ref, pos.ptr(), &result);
-        if (!dtStatusSucceed(status))
-            return {};
-        return result;
-    }
-
     template <class OutputIterator>
     Status makeSmoothPath(const dtNavMesh& navMesh, const dtNavMeshQuery& navMeshQuery,
             const dtQueryFilter& filter, const osg::Vec3f& start, const osg::Vec3f& end, const float stepSize,
@@ -201,13 +190,8 @@ namespace DetourNavigator
             polygonPath = fixupCorridor(polygonPath, result->mVisited);
             polygonPath = fixupShortcuts(polygonPath, navMeshQuery);
 
-            float h = 0;
-            navMeshQuery.getPolyHeight(polygonPath.front(), result->mResultPos.ptr(), &h);
-            iterPos = result->mResultPos;
-            iterPos.y() = h;
-
             // Handle end of path and off-mesh links when close enough.
-            if (endOfPath && inRange(iterPos, steerTarget->steerPos, slop, 1.0f))
+            if (endOfPath && inRange(result->mResultPos, steerTarget->steerPos, slop))
             {
                 // Reached end of path.
                 iterPos = targetPos;
@@ -215,7 +199,7 @@ namespace DetourNavigator
                 ++smoothPathSize;
                 break;
             }
-            else if (offMeshConnection && inRange(iterPos, steerTarget->steerPos, slop, 1.0f))
+            else if (offMeshConnection && inRange(result->mResultPos, steerTarget->steerPos, slop))
             {
                 // Advance the path up to and over the off-mesh connection.
                 dtPolyRef prevRef = 0;
@@ -249,16 +233,18 @@ namespace DetourNavigator
                     }
 
                     // Move position at the other side of the off-mesh link.
-                    iterPos = endPos;
-                    const auto height = getPolyHeight(navMeshQuery, polygonPath.front(), iterPos);
-
-                    if (!height)
+                    if (dtStatusFailed(navMeshQuery.getPolyHeight(polygonPath.front(), endPos.ptr(), &iterPos.y())))
                         return Status::GetPolyHeightFailed;
-
-                    iterPos.y() = *height;
+                    iterPos.x() = endPos.x();
+                    iterPos.z() = endPos.z();
                 }
             }
 
+            if (dtStatusFailed(navMeshQuery.getPolyHeight(polygonPath.front(), result->mResultPos.ptr(), &iterPos.y())))
+                return Status::GetPolyHeightFailed;
+            iterPos.x() = result->mResultPos.x();
+            iterPos.z() = result->mResultPos.z();
+
             // Store results.
             *out++ = iterPos;
             ++smoothPathSize;