Filter recast mesh triangles by global bounding box

Except heightfields to reduce slow down.

apps/openmw_test_suite/detournavigator/recastmeshbuilder.cpp

@@ -361,9 +361,9 @@ namespace
         );
         const auto recastMesh = builder.create();
         EXPECT_EQ(recastMesh->getVertices(), std::vector<float>({
-            0.707107067108154296875, 0, -3.535533905029296875,
+            1.41421353816986083984375, 0, 1.1920928955078125e-07,
-            -0.70710659027099609375, 0, -3.535533905029296875,
+            -1.41421353816986083984375, 0, -1.1920928955078125e-07,
-            2.384185791015625e-07, 0, -4.24264049530029296875,
+            1.1920928955078125e-07, 0, -1.41421353816986083984375,
         }));
         EXPECT_EQ(recastMesh->getIndices(), std::vector<int>({0, 1, 2}));
         EXPECT_EQ(recastMesh->getAreaTypes(), std::vector<AreaType>({AreaType_ground}));

components/bullethelpers/transformboundingbox.hpp

@@ -0,0 +1,38 @@
+#ifndef OPENMW_COMPONENTS_BULLETHELPERS_TRANSFORMBOUNDINGBOX_H
+#define OPENMW_COMPONENTS_BULLETHELPERS_TRANSFORMBOUNDINGBOX_H
+
+#include <LinearMath/btVector3.h>
+#include <LinearMath/btTransform.h>
+
+#include <algorithm>
+
+namespace BulletHelpers
+{
+    inline btVector3 min(const btVector3& a, const btVector3& b)
+    {
+        return btVector3(std::min(a.x(), b.x()), std::min(a.y(), b.y()), std::min(a.z(), b.z()));
+    }
+
+    inline btVector3 max(const btVector3& a, const btVector3& b)
+    {
+        return btVector3(std::max(a.x(), b.x()), std::max(a.y(), b.y()), std::max(a.z(), b.z()));
+    }
+
+    // http://dev.theomader.com/transform-bounding-boxes/
+    inline void transformBoundingBox(const btTransform& transform, btVector3& aabbMin, btVector3& aabbMax)
+    {
+        const btVector3 xa(transform.getBasis().getColumn(0) * aabbMin.x());
+        const btVector3 xb(transform.getBasis().getColumn(0) * aabbMax.x());
+
+        const btVector3 ya(transform.getBasis().getColumn(1) * aabbMin.y());
+        const btVector3 yb(transform.getBasis().getColumn(1) * aabbMax.y());
+
+        const btVector3 za(transform.getBasis().getColumn(2) * aabbMin.z());
+        const btVector3 zb(transform.getBasis().getColumn(2) * aabbMax.z());
+
+        aabbMin = min(xa, xb) + min(ya, yb) + min(za, zb) + transform.getOrigin();
+        aabbMax = max(xa, xb) + max(ya, yb) + max(za, zb) + transform.getOrigin();
+    }
+}
+
+#endif

components/detournavigator/recastmeshbuilder.cpp

@@ -5,6 +5,7 @@
 #include "settingsutils.hpp"
 #include "exceptions.hpp"
 
+#include <components/bullethelpers/transformboundingbox.hpp>
 #include <components/bullethelpers/processtrianglecallback.hpp>
 #include <components/misc/convert.hpp>
 
@@ -13,6 +14,7 @@
 #include <BulletCollision/CollisionShapes/btConcaveShape.h>
 #include <BulletCollision/CollisionShapes/btHeightfieldTerrainShape.h>
 #include <LinearMath/btTransform.h>
+#include <LinearMath/btAabbUtil2.h>
 
 #include <algorithm>
 
@@ -57,7 +59,7 @@ namespace DetourNavigator
         return addObject(shape, transform, makeProcessTriangleCallback([&] (btVector3* triangle, int, int)
         {
             for (std::size_t i = 3; i > 0; --i)
-                addTriangleVertex(transform(triangle[i - 1]));
+                addTriangleVertex(triangle[i - 1]);
             mAreaTypes.push_back(areaType);
         }));
     }
@@ -68,7 +70,7 @@ namespace DetourNavigator
         return addObject(shape, transform, makeProcessTriangleCallback([&] (btVector3* triangle, int, int)
         {
             for (std::size_t i = 0; i < 3; ++i)
-                addTriangleVertex(transform(triangle[i]));
+                addTriangleVertex(triangle[i]);
             mAreaTypes.push_back(areaType);
         }));
     }
@@ -131,8 +133,34 @@ namespace DetourNavigator
 
         shape.getAabb(btTransform::getIdentity(), aabbMin, aabbMax);
 
-        aabbMin = transform(aabbMin);
+        const btVector3 boundsMin(mBounds.mMin.x(), mBounds.mMin.y(),
-        aabbMax = transform(aabbMax);
+            -std::numeric_limits<btScalar>::max() * std::numeric_limits<btScalar>::epsilon());
+        const btVector3 boundsMax(mBounds.mMax.x(), mBounds.mMax.y(),
+            std::numeric_limits<btScalar>::max() * std::numeric_limits<btScalar>::epsilon());
+
+        auto wrapper = makeProcessTriangleCallback([&] (btVector3* triangle, int partId, int triangleIndex)
+        {
+            std::array<btVector3, 3> transformed;
+            for (std::size_t i = 0; i < 3; ++i)
+                transformed[i] = transform(triangle[i]);
+            if (TestTriangleAgainstAabb2(transformed.data(), boundsMin, boundsMax))
+                callback.processTriangle(transformed.data(), partId, triangleIndex);
+        });
+
+        shape.processAllTriangles(&wrapper, aabbMin, aabbMax);
+    }
+
+    void RecastMeshBuilder::addObject(const btHeightfieldTerrainShape& shape, const btTransform& transform,
+                                      btTriangleCallback&& callback)
+    {
+        using BulletHelpers::transformBoundingBox;
+
+        btVector3 aabbMin;
+        btVector3 aabbMax;
+
+        shape.getAabb(btTransform::getIdentity(), aabbMin, aabbMax);
+
+        transformBoundingBox(transform, aabbMin, aabbMax);
 
         aabbMin.setX(std::max(mBounds.mMin.x(), aabbMin.x()));
         aabbMin.setX(std::min(mBounds.mMax.x(), aabbMin.x()));
@@ -144,20 +172,17 @@ namespace DetourNavigator
         aabbMax.setY(std::max(mBounds.mMin.y(), aabbMax.y()));
         aabbMax.setY(std::min(mBounds.mMax.y(), aabbMax.y()));
 
-        const auto inversedTransform = transform.inverse();
+        transformBoundingBox(transform.inverse(), aabbMin, aabbMax);
 
-        aabbMin = inversedTransform(aabbMin);
+        auto wrapper = makeProcessTriangleCallback([&] (btVector3* triangle, int partId, int triangleIndex)
-        aabbMax = inversedTransform(aabbMax);
+        {
+            std::array<btVector3, 3> transformed;
+            for (std::size_t i = 0; i < 3; ++i)
+                transformed[i] = transform(triangle[i]);
+            callback.processTriangle(transformed.data(), partId, triangleIndex);
+        });
 
-        aabbMin.setX(std::min(aabbMin.x(), aabbMax.x()));
+        shape.processAllTriangles(&wrapper, aabbMin, aabbMax);
-        aabbMin.setY(std::min(aabbMin.y(), aabbMax.y()));
-        aabbMin.setZ(std::min(aabbMin.z(), aabbMax.z()));
-
-        aabbMax.setX(std::max(aabbMin.x(), aabbMax.x()));
-        aabbMax.setY(std::max(aabbMin.y(), aabbMax.y()));
-        aabbMax.setZ(std::max(aabbMin.z(), aabbMax.z()));
-
-        shape.processAllTriangles(&callback, aabbMin, aabbMax);
     }
 
     void RecastMeshBuilder::addTriangleVertex(const btVector3& worldPosition)

components/detournavigator/recastmeshbuilder.hpp

@@ -48,6 +48,8 @@ namespace DetourNavigator
 
         void addObject(const btConcaveShape& shape, const btTransform& transform, btTriangleCallback&& callback);
 
+        void addObject(const btHeightfieldTerrainShape& shape, const btTransform& transform, btTriangleCallback&& callback);
+
         void addTriangleVertex(const btVector3& worldPosition);
 
         void addVertex(const btVector3& worldPosition);