Limit navmesh vertices coordinates values

Float values with more than 22 significant fraction bits may cause out of bounds
access in recastnavigation on triangles rasterization. Prevent passing such
values there.

CHANGELOG.md

@@ -37,6 +37,7 @@
     Bug #6402: The sound of a thunderstorm does not stop playing after entering the premises
     Bug #6427: Enemy health bar disappears before damaging effect ends
     Bug #6550: Cloned body parts don't inherit texture effects
+    Bug #6574: Crash at far away from world origin coordinates
     Bug #6645: Enemy block sounds align with animation instead of blocked hits
     Bug #6657: Distant terrain tiles become black when using FWIW mod
     Bug #6661: Saved games that have no preview screenshot cause issues or crashes

apps/openmw_test_suite/detournavigator/navigator.cpp

@@ -871,7 +871,7 @@ namespace
 
     TEST_F(DetourNavigatorNavigatorTest, update_for_very_big_object_should_be_limited)
     {
-        const float size = static_cast<float>(2 * static_cast<std::int64_t>(std::numeric_limits<int>::max()) - 1);
+        const float size = static_cast<float>((1 << 22) - 1);
         CollisionShapeInstance bigBox(std::make_unique<btBoxShape>(btVector3(size, size, 1)));
         const ObjectTransform objectTransform{
             .mPosition = ESM::Position{ .pos = { 0, 0, 0 }, .rot{ 0, 0, 0 } },

components/detournavigator/makenavmesh.cpp

@@ -186,16 +186,35 @@ namespace DetourNavigator
                 &context, solid, width, height, bmin.ptr(), bmax.ptr(), settings.mCellSize, settings.mCellHeight);
         }
 
+        bool isSupportedCoordinate(float value)
+        {
+            constexpr float maxVertexCoordinate = static_cast<float>(1 << 22);
+            return -maxVertexCoordinate < value && value < maxVertexCoordinate;
+        }
+
+        template <class Iterator>
+        bool isSupportedCoordinates(Iterator begin, Iterator end)
+        {
+            return std::all_of(begin, end, isSupportedCoordinate);
+        }
+
         [[nodiscard]] bool rasterizeTriangles(RecastContext& context, const Mesh& mesh, const RecastSettings& settings,
             const RecastParams& params, rcHeightfield& solid)
         {
             std::vector<unsigned char> areas(mesh.getAreaTypes().begin(), mesh.getAreaTypes().end());
             std::vector<float> vertices = mesh.getVertices();
 
-            for (std::size_t i = 0; i < vertices.size(); i += 3)
+            constexpr std::size_t verticesPerTriangle = 3;
+
+            for (std::size_t i = 0; i < vertices.size(); i += verticesPerTriangle)
             {
-                for (std::size_t j = 0; j < 3; ++j)
+                for (std::size_t j = 0; j < verticesPerTriangle; ++j)
-                    vertices[i + j] = toNavMeshCoordinates(settings, vertices[i + j]);
+                {
+                    const float coordinate = toNavMeshCoordinates(settings, vertices[i + j]);
+                    if (!isSupportedCoordinate(coordinate))
+                        return false;
+                    vertices[i + j] = coordinate;
+                }
                 std::swap(vertices[i + 1], vertices[i + 2]);
             }
 
@@ -217,6 +236,9 @@ namespace DetourNavigator
                 rectangle.mBounds.mMax.x(), rectangle.mHeight, rectangle.mBounds.mMin.y(), // vertex 3
             };
 
+            if (!isSupportedCoordinates(vertices.begin(), vertices.end()))
+                return false;
+
             const std::array indices{
                 0, 1, 2, // triangle 0
                 0, 2, 3, // triangle 1