Maybe compute an accurate near and far plane using primitives for groundcover

I couldn't reproduce the original bug, so can't verify whether this
fixes it.
There's a strong chance I've boffed the matrix multiplication order.
If it's the same as GLSL, then everything *should* be fine.

apps/openmw/mwrender/groundcover.cpp

@@ -10,6 +10,7 @@
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/loadland.hpp>
 #include <components/esm3/readerscache.hpp>
+#include <components/misc/convert.hpp>
 #include <components/sceneutil/lightmanager.hpp>
 #include <components/sceneutil/nodecallback.hpp>
 #include <components/shader/shadermanager.hpp>
@@ -21,6 +22,113 @@
 
 namespace MWRender
 {
+    namespace
+    {
+        using value_type = osgUtil::CullVisitor::value_type;
+
+        // From OSG's CullVisitor.cpp
+        inline value_type distance(const osg::Vec3& coord, const osg::Matrix& matrix)
+        {
+            return -((value_type)coord[0] * (value_type)matrix(0, 2) + (value_type)coord[1] * (value_type)matrix(1, 2)
+                + (value_type)coord[2] * (value_type)matrix(2, 2) + matrix(3, 2));
+        }
+
+        inline osg::Matrix computeInstanceMatrix(const Groundcover::GroundcoverEntry& entry)
+        {
+            return osg::Matrix::translate(entry.mPos.asVec3()) * osg::Matrix(Misc::Convert::makeOsgQuat(entry.mPos))
+                * osg::Matrix::scale(entry.mScale, entry.mScale, entry.mScale);
+        }
+
+        class InstancedComputeNearFarCullCallback : public osg::DrawableCullCallback
+        {
+        public:
+            InstancedComputeNearFarCullCallback(const std::vector<Groundcover::GroundcoverEntry>& instances)
+                : mInstances(instances)
+            {
+            }
+
+            bool cull(osg::NodeVisitor* nv, osg::Drawable* drawable, osg::RenderInfo* renderInfo) const override
+            {
+                osgUtil::CullVisitor& cullVisitor = *nv->asCullVisitor();
+                osg::CullSettings::ComputeNearFarMode cnfMode = cullVisitor.getComputeNearFarMode();
+                const osg::BoundingBox& boundingBox = drawable->getBoundingBox();
+                osg::RefMatrix& matrix = *cullVisitor.getModelViewMatrix();
+
+                if (cnfMode != osg::CullSettings::COMPUTE_NEAR_FAR_USING_PRIMITIVES
+                    && cnfMode != osg::CullSettings::COMPUTE_NEAR_USING_PRIMITIVES)
+                    return true;
+
+                if (drawable->isCullingActive() && cullVisitor.isCulled(boundingBox))
+                    return false;
+
+                osg::Vec3 lookVector = cullVisitor.getLookVectorLocal();
+                unsigned int bbCornerFar
+                    = (lookVector.x() >= 0 ? 1 : 0) | (lookVector.y() >= 0 ? 2 : 0) | (lookVector.z() >= 0 ? 4 : 0);
+                unsigned int bbCornerNear = (~bbCornerFar) & 7;
+                value_type dNear = distance(boundingBox.corner(bbCornerNear), matrix);
+                value_type dFar = distance(boundingBox.corner(bbCornerFar), matrix);
+
+                if (dNear > dFar)
+                    std::swap(dNear, dFar);
+
+                if (dFar < 0)
+                    return false;
+
+                value_type computedZNear = cullVisitor.getCalculatedNearPlane();
+                value_type computedZFar = cullVisitor.getCalculatedFarPlane();
+
+                if (dNear < computedZNear || dFar > computedZFar)
+                {
+                    osg::Polytope& frustum = cullVisitor.getCurrentCullingSet().getFrustum();
+                    osg::Polytope::ClippingMask resultMask = frustum.getResultMask();
+                    if (resultMask)
+                    {
+                        osg::Polytope::PlaneList planes;
+                        osg::Polytope::ClippingMask selectorMask = 0x1;
+                        for (const auto& plane : frustum.getPlaneList())
+                        {
+                            if (resultMask & selectorMask)
+                                planes.push_back(plane);
+                            selectorMask <<= 1;
+                        }
+
+                        if (dNear < computedZNear)
+                        {
+                            dNear = std::numeric_limits<value_type>::max();
+                            for (const auto& entry : mInstances)
+                            {
+                                osg::Matrix instanceMatrix = computeInstanceMatrix(entry);
+                                value_type newNear = cullVisitor.computeNearestPointInFrustum(
+                                    matrix * instanceMatrix, planes, *drawable);
+                                dNear = std::min(dNear, newNear);
+                            }
+                            if (dNear < computedZNear)
+                                cullVisitor.setCalculatedNearPlane(dNear);
+                        }
+
+                        if (cnfMode == osg::CullSettings::COMPUTE_NEAR_FAR_USING_PRIMITIVES && dFar > computedZFar)
+                        {
+                            dFar = -std::numeric_limits<value_type>::max();
+                            for (const auto& entry : mInstances)
+                            {
+                                osg::Matrix instanceMatrix = computeInstanceMatrix(entry);
+                                value_type newFar = cullVisitor.computeFurthestPointInFrustum(
+                                    matrix * instanceMatrix, planes, *drawable);
+                                dFar = std::max(dFar, newFar);
+                            }
+                            if (dFar > computedZFar)
+                                cullVisitor.setCalculatedFarPlane(dFar);
+                        }
+                    }
+                }
+
+                return true;
+            }
+
+        private:
+            std::vector<Groundcover::GroundcoverEntry> mInstances;
+        };
+
         class InstancingVisitor : public osg::NodeVisitor
         {
         public:
@@ -67,6 +175,8 @@ namespace MWRender
 
                 geom.setVertexAttribArray(6, transforms.get(), osg::Array::BIND_PER_VERTEX);
                 geom.setVertexAttribArray(7, rotations.get(), osg::Array::BIND_PER_VERTEX);
+
+                geom.addCullCallback(new InstancedComputeNearFarCullCallback(mInstances));
             }
 
         private:
@@ -137,6 +247,7 @@ namespace MWRender
 
             return true;
         }
+    }
 
     osg::ref_ptr<osg::Node> Groundcover::getChunk(float size, const osg::Vec2f& center, unsigned char lod,
         unsigned int lodFlags, bool activeGrid, const osg::Vec3f& viewPoint, bool compile)