Merge branch 'fix_far_away_navmesh_crash' into 'master'

Limit navmesh vertices coordinates values (#6574)

Closes #6574

See merge request OpenMW/openmw!3825

CHANGELOG.md

@@ -38,6 +38,7 @@
     Bug #6402: The sound of a thunderstorm does not stop playing after entering the premises
     Bug #6427: Enemy health bar disappears before damaging effect ends
     Bug #6550: Cloned body parts don't inherit texture effects
+    Bug #6574: Crash at far away from world origin coordinates
     Bug #6645: Enemy block sounds align with animation instead of blocked hits
     Bug #6657: Distant terrain tiles become black when using FWIW mod
     Bug #6661: Saved games that have no preview screenshot cause issues or crashes

apps/openmw_test_suite/detournavigator/navigator.cpp

@@ -871,7 +871,7 @@ namespace
 
     TEST_F(DetourNavigatorNavigatorTest, update_for_very_big_object_should_be_limited)
     {
-        const float size = static_cast<float>(2 * static_cast<std::int64_t>(std::numeric_limits<int>::max()) - 1);
+        const float size = static_cast<float>((1 << 22) - 1);
         CollisionShapeInstance bigBox(std::make_unique<btBoxShape>(btVector3(size, size, 1)));
         const ObjectTransform objectTransform{
             .mPosition = ESM::Position{ .pos = { 0, 0, 0 }, .rot{ 0, 0, 0 } },

components/detournavigator/makenavmesh.cpp

@@ -186,16 +186,35 @@ namespace DetourNavigator
                 &context, solid, width, height, bmin.ptr(), bmax.ptr(), settings.mCellSize, settings.mCellHeight);
         }
 
+        bool isSupportedCoordinate(float value)
+        {
+            constexpr float maxVertexCoordinate = static_cast<float>(1 << 22);
+            return -maxVertexCoordinate < value && value < maxVertexCoordinate;
+        }
+
+        template <class Iterator>
+        bool isSupportedCoordinates(Iterator begin, Iterator end)
+        {
+            return std::all_of(begin, end, isSupportedCoordinate);
+        }
+
         [[nodiscard]] bool rasterizeTriangles(RecastContext& context, const Mesh& mesh, const RecastSettings& settings,
             const RecastParams& params, rcHeightfield& solid)
         {
             std::vector<unsigned char> areas(mesh.getAreaTypes().begin(), mesh.getAreaTypes().end());
             std::vector<float> vertices = mesh.getVertices();
 
-            for (std::size_t i = 0; i < vertices.size(); i += 3)
+            constexpr std::size_t verticesPerTriangle = 3;
+
+            for (std::size_t i = 0; i < vertices.size(); i += verticesPerTriangle)
             {
-                for (std::size_t j = 0; j < 3; ++j)
-                    vertices[i + j] = toNavMeshCoordinates(settings, vertices[i + j]);
+                for (std::size_t j = 0; j < verticesPerTriangle; ++j)
+                {
+                    const float coordinate = toNavMeshCoordinates(settings, vertices[i + j]);
+                    if (!isSupportedCoordinate(coordinate))
+                        return false;
+                    vertices[i + j] = coordinate;
+                }
                 std::swap(vertices[i + 1], vertices[i + 2]);
             }
 
@@ -217,6 +236,9 @@ namespace DetourNavigator
                 rectangle.mBounds.mMax.x(), rectangle.mHeight, rectangle.mBounds.mMin.y(), // vertex 3
             };
 
+            if (!isSupportedCoordinates(vertices.begin(), vertices.end()))
+                return false;
+
             const std::array indices{
                 0, 1, 2, // triangle 0
                 0, 2, 3, // triangle 1