Merge branch 'object_paging_retry' into 'master'

Object Paging

See merge request OpenMW/openmw!209

apps/openmw/CMakeLists.txt

@@ -21,7 +21,7 @@ add_openmw_dir (mwrender
     actors objects renderingmanager animation rotatecontroller sky npcanimation vismask
     creatureanimation effectmanager util renderinginterface pathgrid rendermode weaponanimation
     bulletdebugdraw globalmap characterpreview camera localmap water terrainstorage ripplesimulation
-    renderbin actoranimation landmanager navmesh actorspaths recastmesh fogmanager
+    renderbin actoranimation landmanager navmesh actorspaths recastmesh fogmanager objectpaging
     )
 
 add_openmw_dir (mwinput

apps/openmw/mwgui/loadingscreen.cpp

@@ -16,6 +16,7 @@
 #include <components/myguiplatform/myguitexture.hpp>
 #include <components/settings/settings.hpp>
 #include <components/vfs/manager.hpp>
+#include <components/resource/resourcesystem.hpp>
 
 #include "../mwbase/environment.hpp"
 #include "../mwbase/statemanager.hpp"
@@ -29,9 +30,9 @@
 namespace MWGui
 {
 
-    LoadingScreen::LoadingScreen(const VFS::Manager* vfs, osgViewer::Viewer* viewer)
+    LoadingScreen::LoadingScreen(Resource::ResourceSystem* resourceSystem, osgViewer::Viewer* viewer)
         : WindowBase("openmw_loading_screen.layout")
-        , mVFS(vfs)
+        , mResourceSystem(resourceSystem)
         , mViewer(viewer)
         , mTargetFrameRate(120.0)
         , mLastWallpaperChangeTime(0.0)
@@ -64,9 +65,9 @@ namespace MWGui
 
     void LoadingScreen::findSplashScreens()
     {
-        const std::map<std::string, VFS::File*>& index = mVFS->getIndex();
+        const std::map<std::string, VFS::File*>& index = mResourceSystem->getVFS()->getIndex();
         std::string pattern = "Splash/";
-        mVFS->normalizeFilename(pattern);
+        mResourceSystem->getVFS()->normalizeFilename(pattern);
 
         /* priority given to the left */
         const std::array<std::string, 7> supported_extensions {{".tga", ".dds", ".ktx", ".png", ".bmp", ".jpeg", ".jpg"}};
@@ -171,6 +172,11 @@ namespace MWGui
         // We are already using node masks to avoid the scene from being updated/rendered, but node masks don't work for computeBound()
         mViewer->getSceneData()->setComputeBoundingSphereCallback(new DontComputeBoundCallback);
 
+        if (const osgUtil::IncrementalCompileOperation* ico = mViewer->getIncrementalCompileOperation()) {
+            mOldIcoMin = ico->getMinimumTimeAvailableForGLCompileAndDeletePerFrame();
+            mOldIcoMax = ico->getMaximumNumOfObjectsToCompilePerFrame();
+        }
+
         mVisible = visible;
         mLoadingBox->setVisible(mVisible);
         setVisible(true);
@@ -215,6 +221,12 @@ namespace MWGui
         //std::cout << "loading took " << mTimer.time_m() - mLoadingOnTime << std::endl;
         setVisible(false);
 
+        if (osgUtil::IncrementalCompileOperation* ico = mViewer->getIncrementalCompileOperation())
+        {
+            ico->setMinimumTimeAvailableForGLCompileAndDeletePerFrame(mOldIcoMin);
+            ico->setMaximumNumOfObjectsToCompilePerFrame(mOldIcoMax);
+        }
+
         MWBase::Environment::get().getWindowManager()->removeGuiMode(GM_Loading);
         MWBase::Environment::get().getWindowManager()->removeGuiMode(GM_LoadingWallpaper);
     }
@@ -336,7 +348,13 @@ namespace MWGui
 
         MWBase::Environment::get().getInputManager()->update(0, true, true);
 
-        //osg::Timer timer;
+        mResourceSystem->reportStats(mViewer->getFrameStamp()->getFrameNumber(), mViewer->getViewerStats());
+        if (osgUtil::IncrementalCompileOperation* ico = mViewer->getIncrementalCompileOperation())
+        {
+            ico->setMinimumTimeAvailableForGLCompileAndDeletePerFrame(1.f/getTargetFrameRate());
+            ico->setMaximumNumOfObjectsToCompilePerFrame(1000);
+        }
+
         // at the time this function is called we are in the middle of a frame,
         // so out of order calls are necessary to get a correct frameNumber for the next frame.
         // refer to the advance() and frame() order in Engine::go()
@@ -344,10 +362,6 @@ namespace MWGui
         mViewer->updateTraversal();
         mViewer->renderingTraversals();
         mViewer->advance(mViewer->getFrameStamp()->getSimulationTime());
-        //std::cout << "frame took " << timer.time_m() << std::endl;
-
-        //if (mViewer->getIncrementalCompileOperation())
-            //std::cout << "num to compile " << mViewer->getIncrementalCompileOperation()->getToCompile().size() << std::endl;
 
         mLastRenderTime = mTimer.time_m();
     }

apps/openmw/mwgui/loadingscreen.hpp

@@ -20,9 +20,9 @@ namespace osg
     class Texture2D;
 }
 
-namespace VFS
+namespace Resource
 {
-    class Manager;
+    class ResourceSystem;
 }
 
 namespace MWGui
@@ -32,7 +32,7 @@ namespace MWGui
     class LoadingScreen : public WindowBase, public Loading::Listener
     {
     public:
-        LoadingScreen(const VFS::Manager* vfs, osgViewer::Viewer* viewer);
+        LoadingScreen(Resource::ResourceSystem* resourceSystem, osgViewer::Viewer* viewer);
         virtual ~LoadingScreen();
 
         /// Overridden from Loading::Listener, see the Loading::Listener documentation for usage details
@@ -53,7 +53,7 @@ namespace MWGui
 
         void setupCopyFramebufferToTextureCallback();
 
-        const VFS::Manager* mVFS;
+        Resource::ResourceSystem* mResourceSystem;
         osg::ref_ptr<osgViewer::Viewer> mViewer;
 
         double mTargetFrameRate;
@@ -70,6 +70,8 @@ namespace MWGui
         size_t mProgress;
 
         bool mShowWallpaper;
+        float mOldIcoMin = 0.f;
+        unsigned int mOldIcoMax = 0;
 
         MyGUI::Widget* mLoadingBox;
 

apps/openmw/mwgui/windowmanagerimp.cpp

@@ -230,7 +230,7 @@ namespace MWGui
         mKeyboardNavigation->setEnabled(keyboardNav);
         Gui::ImageButton::setDefaultNeedKeyFocus(keyboardNav);
 
-        mLoadingScreen = new LoadingScreen(mResourceSystem->getVFS(), mViewer);
+        mLoadingScreen = new LoadingScreen(mResourceSystem, mViewer);
         mWindows.push_back(mLoadingScreen);
 
         //set up the hardware cursor manager

apps/openmw/mwmechanics/mechanicsmanagerimp.cpp

@@ -7,6 +7,8 @@
 #include <components/esm/esmwriter.hpp>
 #include <components/esm/stolenitems.hpp>
 
+#include <components/detournavigator/navigator.hpp>
+
 #include <components/sceneutil/positionattitudetransform.hpp>
 
 #include "../mwworld/esmstore.hpp"
@@ -900,6 +902,12 @@ namespace MWMechanics
     bool MechanicsManager::toggleAI()
     {
         mAI = !mAI;
+
+        MWBase::World* world = MWBase::Environment::get().getWorld();
+        world->getNavigator()->setUpdatesEnabled(mAI);
+        if (mAI)
+           world->getNavigator()->update(world->getPlayerPtr().getRefData().getPosition().asVec3());
+
         return mAI;
     }
 

apps/openmw/mwphysics/physicssystem.cpp

@@ -397,6 +397,15 @@ namespace MWPhysics
             return osg::Vec3f();
     }
 
+    osg::BoundingBox PhysicsSystem::getBoundingBox(const MWWorld::ConstPtr &object) const
+    {
+        const Object * physobject = getObject(object);
+        if (!physobject) return osg::BoundingBox();
+        btVector3 min, max;
+        physobject->getCollisionObject()->getCollisionShape()->getAabb(physobject->getCollisionObject()->getWorldTransform(), min, max);
+        return osg::BoundingBox(Misc::Convert::toOsg(min), Misc::Convert::toOsg(max));
+    }
+
     osg::Vec3f PhysicsSystem::getCollisionObjectPosition(const MWWorld::ConstPtr &actor) const
     {
         const Actor* physactor = getActor(actor);

apps/openmw/mwphysics/physicssystem.hpp

@@ -7,6 +7,7 @@
 #include <algorithm>
 
 #include <osg/Quat>
+#include <osg/BoundingBox>
 #include <osg/ref_ptr>
 
 #include "../mwworld/ptr.hpp"
@@ -144,6 +145,9 @@ namespace MWPhysics
             /// @note The collision shape's origin is in its center, so the position returned can be described as center of the actor collision box in world space.
             osg::Vec3f getCollisionObjectPosition(const MWWorld::ConstPtr& actor) const;
 
+            /// Get bounding box in world space of the given object.
+            osg::BoundingBox getBoundingBox(const MWWorld::ConstPtr &object) const;
+
             /// Queues velocity movement for a Ptr. If a Ptr is already queued, its velocity will
             /// be overwritten. Valid until the next call to applyQueuedMovement.
             void queueObjectMovement(const MWWorld::Ptr &ptr, const osg::Vec3f &velocity);

apps/openmw/mwrender/animation.cpp

@@ -513,6 +513,9 @@ namespace MWRender
             if (mShadowUniform)
                 stateset->addUniform(mShadowUniform);
 
+            stateset->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
+            stateset->setRenderBinMode(osg::StateSet::OVERRIDE_RENDERBIN_DETAILS);
+
             // FIXME: overriding diffuse/ambient/emissive colors
             osg::Material* material = new osg::Material;
             material->setColorMode(osg::Material::OFF);
@@ -1369,7 +1372,7 @@ namespace MWRender
             osg::Group* sheathParent = findVisitor.mFoundNode;
             if (sheathParent)
             {
-                osg::Node* copy = osg::clone(nodePair.first, osg::CopyOp::DEEP_COPY_NODES);
+                osg::Node* copy = static_cast<osg::Node*>(nodePair.first->clone(osg::CopyOp::DEEP_COPY_NODES));
                 sheathParent->addChild(copy);
             }
         }
@@ -1741,31 +1744,16 @@ namespace MWRender
             if (mTransparencyUpdater == nullptr)
             {
                 mTransparencyUpdater = new TransparencyUpdater(alpha, mResourceSystem->getSceneManager()->getShaderManager().getShadowMapAlphaTestEnableUniform());
-                mObjectRoot->addUpdateCallback(mTransparencyUpdater);
+                mObjectRoot->addCullCallback(mTransparencyUpdater);
             }
             else
                 mTransparencyUpdater->setAlpha(alpha);
         }
         else
         {
-            mObjectRoot->removeUpdateCallback(mTransparencyUpdater);
+            mObjectRoot->removeCullCallback(mTransparencyUpdater);
             mTransparencyUpdater = nullptr;
-            mObjectRoot->setStateSet(nullptr);
         }
-
-        setRenderBin();
-    }
-
-    void Animation::setRenderBin()
-    {
-        if (mAlpha != 1.f)
-        {
-            osg::StateSet* stateset = mObjectRoot->getOrCreateStateSet();
-            stateset->setRenderingHint(osg::StateSet::TRANSPARENT_BIN);
-            stateset->setRenderBinMode(osg::StateSet::OVERRIDE_RENDERBIN_DETAILS);
-        }
-        else if (osg::StateSet* stateset = mObjectRoot->getStateSet())
-            stateset->setRenderBinToInherit();
     }
 
     void Animation::setLightEffect(float effect)

apps/openmw/mwrender/animation.hpp

@@ -336,9 +336,6 @@ protected:
      */
     virtual void addControllers();
 
-    /// Set the render bin for this animation's object root. May be customized by subclasses.
-    virtual void setRenderBin();
-
 public:
 
     Animation(const MWWorld::Ptr &ptr, osg::ref_ptr<osg::Group> parentNode, Resource::ResourceSystem* resourceSystem);

apps/openmw/mwrender/localmap.cpp

@@ -168,11 +168,10 @@ void LocalMap::saveFogOfWar(MWWorld::CellStore* cell)
 osg::ref_ptr<osg::Camera> LocalMap::createOrthographicCamera(float x, float y, float width, float height, const osg::Vec3d& upVector, float zmin, float zmax)
 {
     osg::ref_ptr<osg::Camera> camera (new osg::Camera);
-
     camera->setProjectionMatrixAsOrtho(-width/2, width/2, -height/2, height/2, 5, (zmax-zmin) + 10);
     camera->setComputeNearFarMode(osg::Camera::DO_NOT_COMPUTE_NEAR_FAR);
     camera->setViewMatrixAsLookAt(osg::Vec3d(x, y, zmax + 5), osg::Vec3d(x, y, zmin), upVector);
-    camera->setReferenceFrame(osg::Camera::ABSOLUTE_RF);
+    camera->setReferenceFrame(osg::Camera::ABSOLUTE_RF_INHERIT_VIEWPOINT);
     camera->setRenderTargetImplementation(osg::Camera::FRAME_BUFFER_OBJECT, osg::Camera::PIXEL_BUFFER_RTT);
     camera->setClearColor(osg::Vec4(0.f, 0.f, 0.f, 1.f));
     camera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
@@ -360,11 +359,6 @@ void LocalMap::requestExteriorMap(const MWWorld::CellStore* cell)
 
     osg::ref_ptr<osg::Camera> camera = createOrthographicCamera(x*mMapWorldSize + mMapWorldSize/2.f, y*mMapWorldSize + mMapWorldSize/2.f, mMapWorldSize, mMapWorldSize,
                                                                 osg::Vec3d(0,1,0), zmin, zmax);
-    camera->getOrCreateUserDataContainer()->addDescription("NoTerrainLod");
-    std::ostringstream stream;
-    stream << x << " " << y;
-    camera->getOrCreateUserDataContainer()->addDescription(stream.str());
-
     setupRenderToTexture(camera, cell->getCell()->getGridX(), cell->getCell()->getGridY());
 
     MapSegment& segment = mSegments[std::make_pair(cell->getCell()->getGridX(), cell->getCell()->getGridY())];

apps/openmw/mwrender/npcanimation.cpp

@@ -435,12 +435,10 @@ void NpcAnimation::setRenderBin()
             osgUtil::RenderBin::addRenderBinPrototype("DepthClear", depthClearBin);
             prototypeAdded = true;
         }
-
-        osg::StateSet* stateset = mObjectRoot->getOrCreateStateSet();
-        stateset->setRenderBinDetails(RenderBin_FirstPerson, "DepthClear", osg::StateSet::OVERRIDE_RENDERBIN_DETAILS);
+        mObjectRoot->getOrCreateStateSet()->setRenderBinDetails(RenderBin_FirstPerson, "DepthClear", osg::StateSet::OVERRIDE_RENDERBIN_DETAILS);
     }
-    else
-        Animation::setRenderBin();
+    else if (osg::StateSet* stateset = mObjectRoot->getStateSet())
+        stateset->setRenderBinToInherit();
 }
 
 void NpcAnimation::rebuild()

apps/openmw/mwrender/npcanimation.hpp

@@ -88,7 +88,7 @@ private:
     void addPartGroup(int group, int priority, const std::vector<ESM::PartReference> &parts,
                                     bool enchantedGlow=false, osg::Vec4f* glowColor=nullptr);
 
-    virtual void setRenderBin();
+    void setRenderBin();
 
     osg::ref_ptr<NeckController> mFirstPersonNeckController;
 

apps/openmw/mwrender/objectpaging.cpp

@@ -0,0 +1,781 @@
+#include "objectpaging.hpp"
+
+#include <unordered_map>
+
+#include <osg/Version>
+#include <osg/LOD>
+#include <osg/Switch>
+#include <osg/MatrixTransform>
+#include <osg/Material>
+#include <osgUtil/IncrementalCompileOperation>
+
+#include <components/esm/esmreader.hpp>
+#include <components/misc/resourcehelpers.hpp>
+#include <components/resource/scenemanager.hpp>
+#include <components/sceneutil/optimizer.hpp>
+#include <components/sceneutil/clone.hpp>
+#include <components/sceneutil/util.hpp>
+#include <components/vfs/manager.hpp>
+
+#include <osgParticle/ParticleProcessor>
+#include <osgParticle/ParticleSystemUpdater>
+
+#include <components/sceneutil/lightmanager.hpp>
+#include <components/sceneutil/morphgeometry.hpp>
+#include <components/sceneutil/riggeometry.hpp>
+#include <components/settings/settings.hpp>
+#include <components/misc/rng.hpp>
+
+#include "apps/openmw/mwworld/esmstore.hpp"
+#include "apps/openmw/mwbase/environment.hpp"
+#include "apps/openmw/mwbase/world.hpp"
+
+#include "vismask.hpp"
+
+namespace MWRender
+{
+
+    bool typeFilter(int type, bool far)
+    {
+        switch (type)
+        {
+          case ESM::REC_STAT:
+          case ESM::REC_ACTI:
+          case ESM::REC_DOOR:
+            return true;
+          case ESM::REC_CONT:
+            return !far;
+
+        default:
+            return false;
+        }
+    }
+
+    std::string getModel(int type, const std::string& id, const MWWorld::ESMStore& store)
+    {
+        switch (type)
+        {
+          case ESM::REC_STAT:
+            return store.get<ESM::Static>().searchStatic(id)->mModel;
+          case ESM::REC_ACTI:
+            return store.get<ESM::Activator>().searchStatic(id)->mModel;
+          case ESM::REC_DOOR:
+            return store.get<ESM::Door>().searchStatic(id)->mModel;
+          case ESM::REC_CONT:
+            return store.get<ESM::Container>().searchStatic(id)->mModel;
+          default:
+            return std::string();
+        }
+    }
+
+    osg::ref_ptr<osg::Node> ObjectPaging::getChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool activeGrid, const osg::Vec3f& viewPoint, bool compile)
+    {
+        if (activeGrid && !mActiveGrid)
+            return nullptr;
+
+        ChunkId id = std::make_tuple(center, size, activeGrid);
+
+        osg::ref_ptr<osg::Object> obj = mCache->getRefFromObjectCache(id);
+        if (obj)
+            return obj->asNode();
+        else
+        {
+            osg::ref_ptr<osg::Node> node = createChunk(size, center, activeGrid, viewPoint, compile);
+            mCache->addEntryToObjectCache(id, node.get());
+            return node;
+        }
+    }
+
+    class CanOptimizeCallback : public SceneUtil::Optimizer::IsOperationPermissibleForObjectCallback
+    {
+    public:
+        virtual bool isOperationPermissibleForObjectImplementation(const SceneUtil::Optimizer* optimizer, const osg::Drawable* node,unsigned int option) const
+        {
+            return true;
+        }
+        virtual bool isOperationPermissibleForObjectImplementation(const SceneUtil::Optimizer* optimizer, const osg::Node* node,unsigned int option) const
+        {
+            return (node->getDataVariance() != osg::Object::DYNAMIC);
+        }
+    };
+
+    class CopyOp : public osg::CopyOp
+    {
+    public:
+        bool mOptimizeBillboards = true;
+        float mSqrDistance = 0.f;
+        osg::Vec3f mViewVector;
+        mutable std::vector<const osg::Node*> mNodePath;
+
+        void copy(const osg::Node* toCopy, osg::Group* attachTo)
+        {
+            const osg::Group* groupToCopy = toCopy->asGroup();
+            if (toCopy->getStateSet() || toCopy->asTransform() || !groupToCopy)
+                attachTo->addChild(operator()(toCopy));
+            else
+            {
+                for (unsigned int i=0; i<groupToCopy->getNumChildren(); ++i)
+                    attachTo->addChild(operator()(groupToCopy->getChild(i)));
+            }
+        }
+
+        virtual osg::Node* operator() (const osg::Node* node) const
+        {
+            if (const osg::Drawable* d = node->asDrawable())
+                return operator()(d);
+
+            if (dynamic_cast<const osgParticle::ParticleProcessor*>(node))
+                return nullptr;
+            if (dynamic_cast<const osgParticle::ParticleSystemUpdater*>(node))
+                return nullptr;
+
+            if (const osg::Switch* sw = node->asSwitch())
+            {
+                osg::Group* n = new osg::Group;
+                for (unsigned int i=0; i<sw->getNumChildren(); ++i)
+                    if (sw->getValue(i))
+                        n->addChild(operator()(sw->getChild(i)));
+                n->setDataVariance(osg::Object::STATIC);
+                return n;
+            }
+            if (const osg::LOD* lod = dynamic_cast<const osg::LOD*>(node))
+            {
+                osg::Group* n = new osg::Group;
+                for (unsigned int i=0; i<lod->getNumChildren(); ++i)
+                    if (lod->getMinRange(i) * lod->getMinRange(i) <= mSqrDistance && mSqrDistance < lod->getMaxRange(i) * lod->getMaxRange(i))
+                        n->addChild(operator()(lod->getChild(i)));
+                n->setDataVariance(osg::Object::STATIC);
+                return n;
+            }
+
+            mNodePath.push_back(node);
+
+            osg::Node* cloned = static_cast<osg::Node*>(node->clone(*this));
+            cloned->setDataVariance(osg::Object::STATIC);
+            cloned->setUserDataContainer(nullptr);
+            cloned->setName("");
+
+            mNodePath.pop_back();
+
+            handleCallbacks(node, cloned);
+
+            return cloned;
+        }
+        void handleCallbacks(const osg::Node* node, osg::Node *cloned) const
+        {
+            for (const osg::Callback* callback = node->getCullCallback(); callback != nullptr; callback = callback->getNestedCallback())
+            {
+                if (callback->className() == std::string("BillboardCallback"))
+                {
+                    if (mOptimizeBillboards)
+                    {
+                        handleBillboard(cloned);
+                        continue;
+                    }
+                    else
+                        cloned->setDataVariance(osg::Object::DYNAMIC);
+                }
+
+                if (node->getCullCallback()->getNestedCallback())
+                {
+                    osg::Callback *clonedCallback = osg::clone(callback, osg::CopyOp::SHALLOW_COPY);
+                    clonedCallback->setNestedCallback(nullptr);
+                    cloned->addCullCallback(clonedCallback);
+                }
+                else
+                    cloned->addCullCallback(const_cast<osg::Callback*>(callback));
+            }
+        }
+        void handleBillboard(osg::Node* node) const
+        {
+            osg::Transform* transform = node->asTransform();
+            if (!transform) return;
+            osg::MatrixTransform* matrixTransform = transform->asMatrixTransform();
+            if (!matrixTransform) return;
+
+            osg::Matrix worldToLocal = osg::Matrix::identity();
+            for (auto node : mNodePath)
+                if (const osg::Transform* t = node->asTransform())
+                    t->computeWorldToLocalMatrix(worldToLocal, nullptr);
+            worldToLocal = osg::Matrix::orthoNormal(worldToLocal);
+
+            osg::Matrix billboardMatrix;
+            osg::Vec3f viewVector = -(mViewVector + worldToLocal.getTrans());
+            viewVector.normalize();
+            osg::Vec3f right = viewVector ^ osg::Vec3f(0,0,1);
+            right.normalize();
+            osg::Vec3f up = right ^ viewVector;
+            up.normalize();
+            billboardMatrix.makeLookAt(osg::Vec3f(0,0,0), viewVector, up);
+            billboardMatrix.invert(billboardMatrix);
+
+            const osg::Matrix& oldMatrix = matrixTransform->getMatrix();
+            float mag[3]; // attempt to preserve scale
+            for (int i=0;i<3;++i)
+                mag[i] = std::sqrt(oldMatrix(0,i) * oldMatrix(0,i) + oldMatrix(1,i) * oldMatrix(1,i) + oldMatrix(2,i) * oldMatrix(2,i));
+            osg::Matrix newMatrix;
+            worldToLocal.setTrans(0,0,0);
+            newMatrix *= worldToLocal;
+            newMatrix.preMult(billboardMatrix);
+            newMatrix.preMultScale(osg::Vec3f(mag[0], mag[1], mag[2]));
+            newMatrix.setTrans(oldMatrix.getTrans());
+
+            matrixTransform->setMatrix(newMatrix);
+        }
+        virtual osg::Drawable* operator() (const osg::Drawable* drawable) const
+        {
+            if (dynamic_cast<const osgParticle::ParticleSystem*>(drawable))
+                return nullptr;
+
+            if (const SceneUtil::RigGeometry* rig = dynamic_cast<const SceneUtil::RigGeometry*>(drawable))
+                return operator()(rig->getSourceGeometry());
+            if (const SceneUtil::MorphGeometry* morph = dynamic_cast<const SceneUtil::MorphGeometry*>(drawable))
+                return operator()(morph->getSourceGeometry());
+
+            if (getCopyFlags() & DEEP_COPY_DRAWABLES)
+            {
+                osg::Drawable* d = static_cast<osg::Drawable*>(drawable->clone(*this));
+                d->setDataVariance(osg::Object::STATIC);
+                d->setUserDataContainer(nullptr);
+                d->setName("");
+                return d;
+            }
+            else
+                return const_cast<osg::Drawable*>(drawable);
+        }
+        virtual osg::Callback* operator() (const osg::Callback* callback) const
+        {
+            return nullptr;
+        }
+    };
+
+    class TemplateRef : public osg::Object
+    {
+    public:
+        TemplateRef() {}
+        TemplateRef(const TemplateRef& copy, const osg::CopyOp&) : mObjects(copy.mObjects) {}
+        META_Object(MWRender, TemplateRef)
+        std::vector<osg::ref_ptr<const Object>> mObjects;
+    };
+
+    class RefnumSet : public osg::Object
+    {
+    public:
+        RefnumSet(){}
+        RefnumSet(const RefnumSet& copy, const osg::CopyOp&) : mRefnums(copy.mRefnums) {}
+        META_Object(MWRender, RefnumSet)
+        std::set<ESM::RefNum> mRefnums;
+    };
+
+    class AnalyzeVisitor : public osg::NodeVisitor
+    {
+    public:
+        AnalyzeVisitor()
+         : osg::NodeVisitor(TRAVERSE_ALL_CHILDREN)
+         , mCurrentStateSet(nullptr) {}
+
+        typedef std::unordered_map<osg::StateSet*, unsigned int> StateSetCounter;
+        struct Result
+        {
+            StateSetCounter mStateSetCounter;
+            unsigned int mNumVerts = 0;
+        };
+
+        virtual void apply(osg::Node& node)
+        {
+            if (node.getStateSet())
+                mCurrentStateSet = node.getStateSet();
+            traverse(node);
+        }
+        virtual void apply(osg::Geometry& geom)
+        {
+            mResult.mNumVerts += geom.getVertexArray()->getNumElements();
+            ++mResult.mStateSetCounter[mCurrentStateSet];
+            ++mGlobalStateSetCounter[mCurrentStateSet];
+        }
+        Result retrieveResult()
+        {
+            Result result = mResult;
+            mResult = Result();
+            mCurrentStateSet = nullptr;
+            return result;
+        }
+        void addInstance(const Result& result)
+        {
+            for (auto pair : result.mStateSetCounter)
+                mGlobalStateSetCounter[pair.first] += pair.second;
+        }
+        float getMergeBenefit(const Result& result)
+        {
+            if (result.mStateSetCounter.empty()) return 1;
+            float mergeBenefit = 0;
+            for (auto pair : result.mStateSetCounter)
+            {
+                mergeBenefit += mGlobalStateSetCounter[pair.first];
+            }
+            mergeBenefit /= result.mStateSetCounter.size();
+            return mergeBenefit;
+        }
+
+        Result mResult;
+        osg::StateSet* mCurrentStateSet;
+        StateSetCounter mGlobalStateSetCounter;
+    };
+
+    class DebugVisitor : public osg::NodeVisitor
+    {
+    public:
+        DebugVisitor() : osg::NodeVisitor(TRAVERSE_ALL_CHILDREN) {}
+        virtual void apply(osg::Drawable& node)
+        {
+            osg::ref_ptr<osg::Material> m (new osg::Material);
+            osg::Vec4f color(Misc::Rng::rollProbability(), Misc::Rng::rollProbability(), Misc::Rng::rollProbability(), 0.f);
+            color.normalize();
+            m->setDiffuse(osg::Material::FRONT_AND_BACK, osg::Vec4f(0.1f,0.1f,0.1f,1.f));
+            m->setAmbient(osg::Material::FRONT_AND_BACK, osg::Vec4f(0.1f,0.1f,0.1f,1.f));
+            m->setColorMode(osg::Material::OFF);
+            m->setEmission(osg::Material::FRONT_AND_BACK, osg::Vec4f(color));
+            osg::ref_ptr<osg::StateSet> stateset = node.getStateSet() ? osg::clone(node.getStateSet(), osg::CopyOp::SHALLOW_COPY) : new osg::StateSet;
+            stateset->setAttribute(m);
+            stateset->addUniform(new osg::Uniform("colorMode", 0));
+            node.setStateSet(stateset);
+        }
+    };
+
+    class AddRefnumMarkerVisitor : public osg::NodeVisitor
+    {
+    public:
+        AddRefnumMarkerVisitor(const ESM::RefNum &refnum) : osg::NodeVisitor(TRAVERSE_ALL_CHILDREN), mRefnum(refnum) {}
+        ESM::RefNum mRefnum;
+        virtual void apply(osg::Geometry &node)
+        {
+            osg::ref_ptr<RefnumMarker> marker (new RefnumMarker);
+            marker->mRefnum = mRefnum;
+            if (osg::Array* array = node.getVertexArray())
+                marker->mNumVertices = array->getNumElements();
+            node.getOrCreateUserDataContainer()->addUserObject(marker);
+        }
+    };
+
+    ObjectPaging::ObjectPaging(Resource::SceneManager* sceneManager)
+            : GenericResourceManager<ChunkId>(nullptr)
+         , mSceneManager(sceneManager)
+         , mRefTrackerLocked(false)
+    {
+        mActiveGrid = Settings::Manager::getBool("object paging active grid", "Terrain");
+        mDebugBatches = Settings::Manager::getBool("object paging debug batches", "Terrain");
+        mMergeFactor = Settings::Manager::getFloat("object paging merge factor", "Terrain");
+        mMinSize = Settings::Manager::getFloat("object paging min size", "Terrain");
+        mMinSizeMergeFactor = Settings::Manager::getFloat("object paging min size merge factor", "Terrain");
+        mMinSizeCostMultiplier = Settings::Manager::getFloat("object paging min size cost multiplier", "Terrain");
+    }
+
+    osg::ref_ptr<osg::Node> ObjectPaging::createChunk(float size, const osg::Vec2f& center, bool activeGrid, const osg::Vec3f& viewPoint, bool compile)
+    {
+        osg::Vec2i startCell = osg::Vec2i(std::floor(center.x() - size/2.f), std::floor(center.y() - size/2.f));
+
+        osg::Vec3f worldCenter = osg::Vec3f(center.x(), center.y(), 0)*ESM::Land::REAL_SIZE;
+        osg::Vec3f relativeViewPoint = viewPoint - worldCenter;
+
+        std::map<ESM::RefNum, ESM::CellRef> refs;
+        std::vector<ESM::ESMReader> esm;
+        const MWWorld::ESMStore& store = MWBase::Environment::get().getWorld()->getStore();
+
+        for (int cellX = startCell.x(); cellX < startCell.x() + size; ++cellX)
+        {
+            for (int cellY = startCell.y(); cellY < startCell.y() + size; ++cellY)
+            {
+                const ESM::Cell* cell = store.get<ESM::Cell>().searchStatic(cellX, cellY);
+                if (!cell) continue;
+                for (size_t i=0; i<cell->mContextList.size(); ++i)
+                {
+                    try
+                    {
+                        unsigned int index = cell->mContextList.at(i).index;
+                        if (esm.size()<=index)
+                            esm.resize(index+1);
+                        cell->restore(esm[index], i);
+                        ESM::CellRef ref;
+                        ref.mRefNum.mContentFile = ESM::RefNum::RefNum_NoContentFile;
+                        bool deleted = false;
+                        while(cell->getNextRef(esm[index], ref, deleted))
+                        {
+                            Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
+                            if (std::find(cell->mMovedRefs.begin(), cell->mMovedRefs.end(), ref.mRefNum) != cell->mMovedRefs.end()) continue;
+                            int type = store.findStatic(ref.mRefID);
+                            if (!typeFilter(type,size>=2)) continue;
+                            if (deleted) { refs.erase(ref.mRefNum); continue; }
+                            refs[ref.mRefNum] = ref;
+                        }
+                    }
+                    catch (std::exception& e)
+                    {
+                        continue;
+                    }
+                }
+                for (ESM::CellRefTracker::const_iterator it = cell->mLeasedRefs.begin(); it != cell->mLeasedRefs.end(); ++it)
+                {
+                    ESM::CellRef ref = it->first;
+                    Misc::StringUtils::lowerCaseInPlace(ref.mRefID);
+                    bool deleted = it->second;
+                    if (deleted) { refs.erase(ref.mRefNum); continue; }
+                    int type = store.findStatic(ref.mRefID);
+                    if (!typeFilter(type,size>=2)) continue;
+                    refs[ref.mRefNum] = ref;
+                }
+            }
+        }
+
+        if (activeGrid)
+        {
+            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mRefTrackerMutex);
+            for (auto ref : getRefTracker().mBlacklist)
+                refs.erase(ref);
+        }
+
+        osg::Vec2f minBound = (center - osg::Vec2f(size/2.f, size/2.f));
+        osg::Vec2f maxBound = (center + osg::Vec2f(size/2.f, size/2.f));
+        struct InstanceList
+        {
+            std::vector<const ESM::CellRef*> mInstances;
+            AnalyzeVisitor::Result mAnalyzeResult;
+            bool mNeedCompile = false;
+        };
+        typedef std::map<osg::ref_ptr<const osg::Node>, InstanceList> NodeMap;
+        NodeMap nodes;
+        osg::ref_ptr<RefnumSet> refnumSet = activeGrid ? new RefnumSet : nullptr;
+        AnalyzeVisitor analyzeVisitor;
+        float minSize = mMinSize;
+        if (mMinSizeMergeFactor)
+            minSize *= mMinSizeMergeFactor;
+        for (const auto& pair : refs)
+        {
+            const ESM::CellRef& ref = pair.second;
+
+            osg::Vec3f pos = ref.mPos.asVec3();
+            if (size < 1.f)
+            {
+                osg::Vec3f cellPos = pos / ESM::Land::REAL_SIZE;
+                if ((minBound.x() > std::floor(minBound.x()) && cellPos.x() < minBound.x()) || (minBound.y() > std::floor(minBound.y()) && cellPos.y() < minBound.y())
+                 || (maxBound.x() < std::ceil(maxBound.x()) && cellPos.x() >= maxBound.x()) || (minBound.y() < std::ceil(maxBound.y()) && cellPos.y() >= maxBound.y()))
+                    continue;
+            }
+
+            float dSqr = (viewPoint - pos).length2();
+            if (!activeGrid)
+            {
+                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mSizeCacheMutex);
+                SizeCache::iterator found = mSizeCache.find(pair.first);
+                if (found != mSizeCache.end() && found->second < dSqr*minSize*minSize)
+                    continue;
+            }
+
+            if (ref.mRefID == "prisonmarker" || ref.mRefID == "divinemarker" || ref.mRefID == "templemarker" || ref.mRefID == "northmarker")
+                continue; // marker objects that have a hardcoded function in the game logic, should be hidden from the player
+
+            int type = store.findStatic(ref.mRefID);
+            std::string model = getModel(type, ref.mRefID, store);
+            if (model.empty()) continue;
+            model = "meshes/" + model;
+
+            if (activeGrid && type != ESM::REC_STAT)
+            {
+                model = Misc::ResourceHelpers::correctActorModelPath(model, mSceneManager->getVFS());
+                std::string kfname = Misc::StringUtils::lowerCase(model);
+                if(kfname.size() > 4 && kfname.compare(kfname.size()-4, 4, ".nif") == 0)
+                {
+                    kfname.replace(kfname.size()-4, 4, ".kf");
+                    if (mSceneManager->getVFS()->exists(kfname))
+                        continue;
+                }
+            }
+
+            osg::ref_ptr<const osg::Node> cnode = mSceneManager->getTemplate(model, false);
+
+            if (activeGrid)
+            {
+                if (cnode->getNumChildrenRequiringUpdateTraversal() > 0 || SceneUtil::hasUserDescription(cnode, Constants::NightDayLabel) || SceneUtil::hasUserDescription(cnode, Constants::HerbalismLabel))
+                    continue;
+                else
+                    refnumSet->mRefnums.insert(pair.first);
+            }
+
+            {
+                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mRefTrackerMutex);
+                if (getRefTracker().mDisabled.count(pair.first))
+                    continue;
+            }
+
+            float radius2 = cnode->getBound().radius2() * ref.mScale*ref.mScale;
+            if (radius2 < dSqr*minSize*minSize && !activeGrid)
+            {
+                OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mSizeCacheMutex);
+                mSizeCache[pair.first] = radius2;
+                continue;
+            }
+
+            auto emplaced = nodes.emplace(cnode, InstanceList());
+            if (emplaced.second)
+            {
+                const_cast<osg::Node*>(cnode.get())->accept(analyzeVisitor); // const-trickery required because there is no const version of NodeVisitor
+                emplaced.first->second.mAnalyzeResult = analyzeVisitor.retrieveResult();
+                emplaced.first->second.mNeedCompile = compile && cnode->referenceCount() <= 3;
+            }
+            else
+                analyzeVisitor.addInstance(emplaced.first->second.mAnalyzeResult);
+            emplaced.first->second.mInstances.push_back(&ref);
+        }
+
+        osg::ref_ptr<osg::Group> group = new osg::Group;
+        osg::ref_ptr<osg::Group> mergeGroup = new osg::Group;
+        osg::ref_ptr<TemplateRef> templateRefs = new TemplateRef;
+        osgUtil::StateToCompile stateToCompile(0, nullptr);
+        CopyOp copyop;
+        for (const auto& pair : nodes)
+        {
+            const osg::Node* cnode = pair.first;
+
+            const AnalyzeVisitor::Result& analyzeResult = pair.second.mAnalyzeResult;
+
+            float mergeCost = analyzeResult.mNumVerts * size;
+            float mergeBenefit = analyzeVisitor.getMergeBenefit(analyzeResult) * mMergeFactor;
+            bool merge = mergeBenefit > mergeCost;
+
+            float minSizeMerged = mMinSize;
+            float factor2 = mergeBenefit > 0 ? std::min(1.f, mergeCost * mMinSizeCostMultiplier / mergeBenefit) : 1;
+            float minSizeMergeFactor2 = (1-factor2) * mMinSizeMergeFactor + factor2;
+            if (minSizeMergeFactor2 > 0)
+                minSizeMerged *= minSizeMergeFactor2;
+
+            unsigned int numinstances = 0;
+            for (auto cref : pair.second.mInstances)
+            {
+                const ESM::CellRef& ref = *cref;
+                osg::Vec3f pos = ref.mPos.asVec3();
+
+                if (!activeGrid && minSizeMerged != minSize && cnode->getBound().radius2() * cref->mScale*cref->mScale < (viewPoint-pos).length2()*minSizeMerged*minSizeMerged)
+                    continue;
+
+                osg::Matrixf matrix;
+                matrix.preMultTranslate(pos - worldCenter);
+                matrix.preMultRotate( osg::Quat(ref.mPos.rot[2], osg::Vec3f(0,0,-1)) *
+                                        osg::Quat(ref.mPos.rot[1], osg::Vec3f(0,-1,0)) *
+                                        osg::Quat(ref.mPos.rot[0], osg::Vec3f(-1,0,0)) );
+                matrix.preMultScale(osg::Vec3f(ref.mScale, ref.mScale, ref.mScale));
+                osg::ref_ptr<osg::MatrixTransform> trans = new osg::MatrixTransform(matrix);
+                trans->setDataVariance(osg::Object::STATIC);
+
+                copyop.setCopyFlags(merge ? osg::CopyOp::DEEP_COPY_NODES|osg::CopyOp::DEEP_COPY_DRAWABLES : osg::CopyOp::DEEP_COPY_NODES);
+                copyop.mOptimizeBillboards = (size > 1/4.f);
+                copyop.mNodePath.push_back(trans);
+                copyop.mSqrDistance = (viewPoint - pos).length2();
+                copyop.mViewVector = (viewPoint - worldCenter);
+                copyop.copy(cnode, trans);
+
+                if (activeGrid)
+                {
+                    if (merge)
+                    {
+                        AddRefnumMarkerVisitor visitor(ref.mRefNum);
+                        trans->accept(visitor);
+                    }
+                    else
+                    {
+                        osg::ref_ptr<RefnumMarker> marker = new RefnumMarker; marker->mRefnum = ref.mRefNum;
+                        trans->getOrCreateUserDataContainer()->addUserObject(marker);
+                    }
+                }
+
+                osg::Group* attachTo = merge ? mergeGroup : group;
+                attachTo->addChild(trans);
+                ++numinstances;
+            }
+            if (numinstances > 0)
+            {
+                // add a ref to the original template, to hint to the cache that it's still being used and should be kept in cache
+                templateRefs->mObjects.push_back(cnode);
+
+                if (pair.second.mNeedCompile)
+                {
+                    int mode = osgUtil::GLObjectsVisitor::COMPILE_STATE_ATTRIBUTES;
+                    if (!merge)
+                        mode |= osgUtil::GLObjectsVisitor::COMPILE_DISPLAY_LISTS;
+                    stateToCompile._mode = mode;
+                    const_cast<osg::Node*>(cnode)->accept(stateToCompile);
+                }
+            }
+        }
+
+        if (mergeGroup->getNumChildren())
+        {
+            SceneUtil::Optimizer optimizer;
+            if (size > 1/8.f)
+            {
+                optimizer.setViewPoint(relativeViewPoint);
+                optimizer.setMergeAlphaBlending(true);
+            }
+            optimizer.setIsOperationPermissibleForObjectCallback(new CanOptimizeCallback);
+            unsigned int options = SceneUtil::Optimizer::FLATTEN_STATIC_TRANSFORMS|SceneUtil::Optimizer::REMOVE_REDUNDANT_NODES|SceneUtil::Optimizer::MERGE_GEOMETRY;
+            optimizer.optimize(mergeGroup, options);
+
+            group->addChild(mergeGroup);
+
+            if (mDebugBatches)
+            {
+                DebugVisitor dv;
+                mergeGroup->accept(dv);
+            }
+            if (compile)
+            {
+                stateToCompile._mode = osgUtil::GLObjectsVisitor::COMPILE_DISPLAY_LISTS;
+                mergeGroup->accept(stateToCompile);
+            }
+        }
+
+        auto ico = mSceneManager->getIncrementalCompileOperation();
+        if (!stateToCompile.empty() && ico)
+        {
+            auto compileSet = new osgUtil::IncrementalCompileOperation::CompileSet(group);
+            compileSet->buildCompileMap(ico->getContextSet(), stateToCompile);
+            ico->add(compileSet, false);
+        }
+
+        group->getBound();
+        group->setNodeMask(Mask_Static);
+        osg::UserDataContainer* udc = group->getOrCreateUserDataContainer();
+        if (activeGrid)
+        {
+            udc->addUserObject(refnumSet);
+            group->addCullCallback(new SceneUtil::LightListCallback);
+        }
+        udc->addUserObject(templateRefs);
+
+        return group;
+    }
+
+    unsigned int ObjectPaging::getNodeMask()
+    {
+        return Mask_Static;
+    }
+
+    struct ClearCacheFunctor
+    {
+        void operator()(MWRender::ChunkId id, osg::Object* obj)
+        {
+            if (intersects(id, mPosition))
+                mToClear.insert(id);
+        }
+        bool intersects(ChunkId id, osg::Vec3f pos)
+        {
+            if (mActiveGridOnly && !std::get<2>(id)) return false;
+            pos /= ESM::Land::REAL_SIZE;
+            osg::Vec2f center = std::get<0>(id);
+            float halfSize = std::get<1>(id)/2;
+            return pos.x() >= center.x()-halfSize && pos.y() >= center.y()-halfSize && pos.x() <= center.x()+halfSize && pos.y() <= center.y()+halfSize;
+        }
+        osg::Vec3f mPosition;
+        std::set<MWRender::ChunkId> mToClear;
+        bool mActiveGridOnly = false;
+    };
+
+    bool ObjectPaging::enableObject(int type, const ESM::RefNum & refnum, const osg::Vec3f& pos, bool enabled)
+    {
+        if (!typeFilter(type, false))
+            return false;
+
+        {
+            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mRefTrackerMutex);
+            if (enabled && !getWritableRefTracker().mDisabled.erase(refnum)) return false;
+            if (!enabled && !getWritableRefTracker().mDisabled.insert(refnum).second) return false;
+            if (mRefTrackerLocked) return false;
+        }
+
+        ClearCacheFunctor ccf;
+        ccf.mPosition = pos;
+        mCache->call(ccf);
+        if (ccf.mToClear.empty()) return false;
+        for (auto chunk : ccf.mToClear)
+            mCache->removeFromObjectCache(chunk);
+        return true;
+    }
+
+    bool ObjectPaging::blacklistObject(int type, const ESM::RefNum & refnum, const osg::Vec3f& pos)
+    {
+        if (!typeFilter(type, false))
+            return false;
+
+        {
+            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mRefTrackerMutex);
+            if (!getWritableRefTracker().mBlacklist.insert(refnum).second) return false;
+            if (mRefTrackerLocked) return false;
+        }
+
+        ClearCacheFunctor ccf;
+        ccf.mPosition = pos;
+        ccf.mActiveGridOnly = true;
+        mCache->call(ccf);
+        if (ccf.mToClear.empty()) return false;
+        for (auto chunk : ccf.mToClear)
+            mCache->removeFromObjectCache(chunk);
+        return true;
+    }
+
+
+    void ObjectPaging::clear()
+    {
+        OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mRefTrackerMutex);
+        mRefTrackerNew.mDisabled.clear();
+        mRefTrackerNew.mBlacklist.clear();
+        mRefTrackerLocked = true;
+    }
+
+    bool ObjectPaging::unlockCache()
+    {
+        if (!mRefTrackerLocked) return false;
+        {
+            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(mRefTrackerMutex);
+            mRefTrackerLocked = false;
+            if (mRefTracker == mRefTrackerNew)
+                return false;
+            else
+                mRefTracker = mRefTrackerNew;
+        }
+        mCache->clear();
+        return true;
+    }
+
+    struct GetRefnumsFunctor
+    {
+        GetRefnumsFunctor(std::set<ESM::RefNum>& output) : mOutput(output) {}
+        void operator()(MWRender::ChunkId chunkId, osg::Object* obj)
+        {
+            if (!std::get<2>(chunkId)) return;
+            const osg::Vec2f& center = std::get<0>(chunkId);
+            bool activeGrid = (center.x() > mActiveGrid.x() || center.y() > mActiveGrid.y() || center.x() < mActiveGrid.z() || center.y() < mActiveGrid.w());
+            if (!activeGrid) return;
+
+            osg::UserDataContainer* udc = obj->getUserDataContainer();
+            if (udc && udc->getNumUserObjects())
+            {
+                RefnumSet* refnums = dynamic_cast<RefnumSet*>(udc->getUserObject(0));
+                if (!refnums) return;
+                mOutput.insert(refnums->mRefnums.begin(), refnums->mRefnums.end());
+            }
+        }
+        osg::Vec4i mActiveGrid;
+        std::set<ESM::RefNum>& mOutput;
+    };
+
+    void ObjectPaging::getPagedRefnums(const osg::Vec4i &activeGrid, std::set<ESM::RefNum> &out)
+    {
+        GetRefnumsFunctor grf(out);
+        grf.mActiveGrid = activeGrid;
+        mCache->call(grf);
+    }
+
+    void ObjectPaging::reportStats(unsigned int frameNumber, osg::Stats *stats) const
+    {
+        stats->setAttribute(frameNumber, "Object Chunk", mCache->getCacheSize());
+    }
+
+}

apps/openmw/mwrender/objectpaging.hpp

@@ -0,0 +1,92 @@
+#ifndef OPENMW_COMPONENTS_ESMPAGING_CHUNKMANAGER_H
+#define OPENMW_COMPONENTS_ESMPAGING_CHUNKMANAGER_H
+
+#include <components/terrain/quadtreeworld.hpp>
+#include <components/resource/resourcemanager.hpp>
+#include <components/esm/loadcell.hpp>
+
+#include <OpenThreads/Mutex>
+
+namespace Resource
+{
+    class SceneManager;
+}
+namespace MWWorld
+{
+    class ESMStore;
+}
+
+namespace MWRender
+{
+
+    typedef std::tuple<osg::Vec2f, float, bool> ChunkId; // Center, Size, ActiveGrid
+
+    class ObjectPaging : public Resource::GenericResourceManager<ChunkId>, public Terrain::QuadTreeWorld::ChunkManager
+    {
+    public:
+        ObjectPaging(Resource::SceneManager* sceneManager);
+        ~ObjectPaging() = default;
+
+        osg::ref_ptr<osg::Node> getChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool activeGrid, const osg::Vec3f& viewPoint, bool compile) override;
+
+        osg::ref_ptr<osg::Node> createChunk(float size, const osg::Vec2f& center, bool activeGrid, const osg::Vec3f& viewPoint, bool compile);
+
+        virtual unsigned int getNodeMask() override;
+
+        /// @return true if view needs rebuild
+        bool enableObject(int type, const ESM::RefNum & refnum, const osg::Vec3f& pos, bool enabled);
+
+        /// @return true if view needs rebuild
+        bool blacklistObject(int type, const ESM::RefNum & refnum, const osg::Vec3f& pos);
+
+        void clear();
+
+        /// Must be called after clear() before rendering starts.
+        /// @return true if view needs rebuild
+        bool unlockCache();
+
+        void reportStats(unsigned int frameNumber, osg::Stats* stats) const override;
+
+        void getPagedRefnums(const osg::Vec4i &activeGrid, std::set<ESM::RefNum> &out);
+
+    private:
+        Resource::SceneManager* mSceneManager;
+        bool mActiveGrid;
+        bool mDebugBatches;
+        float mMergeFactor;
+        float mMinSize;
+        float mMinSizeMergeFactor;
+        float mMinSizeCostMultiplier;
+
+        OpenThreads::Mutex mRefTrackerMutex;
+        struct RefTracker
+        {
+            std::set<ESM::RefNum> mDisabled;
+            std::set<ESM::RefNum> mBlacklist;
+            bool operator==(const RefTracker&other) { return mDisabled == other.mDisabled && mBlacklist == other.mBlacklist; }
+        };
+        RefTracker mRefTracker;
+        RefTracker mRefTrackerNew;
+        bool mRefTrackerLocked;
+
+        const RefTracker& getRefTracker() const { return mRefTracker; }
+        RefTracker& getWritableRefTracker() { return mRefTrackerLocked ? mRefTrackerNew : mRefTracker; }
+
+        OpenThreads::Mutex mSizeCacheMutex;
+        typedef std::map<ESM::RefNum, float> SizeCache;
+        SizeCache mSizeCache;
+    };
+
+    class RefnumMarker : public osg::Object
+    {
+    public:
+        RefnumMarker() : mNumVertices(0) {}
+        RefnumMarker(const RefnumMarker &copy, osg::CopyOp co) : mRefnum(copy.mRefnum), mNumVertices(copy.mNumVertices) {}
+        META_Object(MWRender, RefnumMarker)
+
+        ESM::RefNum mRefnum;
+        unsigned int mNumVertices;
+    };
+}
+
+#endif

apps/openmw/mwrender/renderingmanager.cpp

@@ -70,6 +70,8 @@
 #include "actorspaths.hpp"
 #include "recastmesh.hpp"
 #include "fogmanager.hpp"
+#include "objectpaging.hpp"
+
 
 namespace MWRender
 {
@@ -258,7 +260,6 @@ namespace MWRender
         {
             mViewer->setIncrementalCompileOperation(new osgUtil::IncrementalCompileOperation);
             mViewer->getIncrementalCompileOperation()->setTargetFrameRate(Settings::Manager::getFloat("target framerate", "Cells"));
-            mViewer->getIncrementalCompileOperation()->setMaximumNumOfObjectsToCompilePerFrame(100);
         }
 
         mResourceSystem->getSceneManager()->setIncrementalCompileOperation(mViewer->getIncrementalCompileOperation());
@@ -286,6 +287,12 @@ namespace MWRender
             mTerrain.reset(new Terrain::QuadTreeWorld(
                 sceneRoot, mRootNode, mResourceSystem, mTerrainStorage, Mask_Terrain, Mask_PreCompile, Mask_Debug,
                 compMapResolution, compMapLevel, lodFactor, vertexLodMod, maxCompGeometrySize));
+            if (Settings::Manager::getBool("object paging", "Terrain"))
+            {
+                mObjectPaging.reset(new ObjectPaging(mResourceSystem->getSceneManager()));
+                static_cast<Terrain::QuadTreeWorld*>(mTerrain.get())->addChunkManager(mObjectPaging.get());
+                mResourceSystem->addResourceManager(mObjectPaging.get());
+            }
         }
         else
             mTerrain.reset(new Terrain::TerrainGrid(sceneRoot, mRootNode, mResourceSystem, mTerrainStorage, Mask_Terrain, Mask_PreCompile, Mask_Debug));
@@ -970,20 +977,14 @@ namespace MWRender
         renderCameraToImage(rttCamera.get(),image,w,h);
     }
 
-    osg::Vec4f RenderingManager::getScreenBounds(const MWWorld::Ptr& ptr)
+    osg::Vec4f RenderingManager::getScreenBounds(const osg::BoundingBox &worldbb)
     {
-        if (!ptr.getRefData().getBaseNode())
-            return osg::Vec4f();
-
-        osg::ComputeBoundsVisitor computeBoundsVisitor;
-        computeBoundsVisitor.setTraversalMask(~(Mask_ParticleSystem|Mask_Effect));
-        ptr.getRefData().getBaseNode()->accept(computeBoundsVisitor);
-
+        if (!worldbb.valid()) return osg::Vec4f();
         osg::Matrix viewProj = mViewer->getCamera()->getViewMatrix() * mViewer->getCamera()->getProjectionMatrix();
         float min_x = 1.0f, max_x = 0.0f, min_y = 1.0f, max_y = 0.0f;
         for (int i=0; i<8; ++i)
         {
-            osg::Vec3f corner = computeBoundsVisitor.getBoundingBox().corner(i);
+            osg::Vec3f corner = worldbb.corner(i);
             corner = corner * viewProj;
 
             float x = (corner.x() + 1.f) * 0.5f;
@@ -1009,6 +1010,7 @@ namespace MWRender
     {
         RenderingManager::RayResult result;
         result.mHit = false;
+        result.mHitRefnum.mContentFile = -1;
         result.mRatio = 0;
         if (intersector->containsIntersections())
         {
@@ -1020,6 +1022,7 @@ namespace MWRender
             result.mRatio = intersection.ratio;
 
             PtrHolder* ptrHolder = nullptr;
+            std::vector<RefnumMarker*> refnumMarkers;
             for (osg::NodePath::const_iterator it = intersection.nodePath.begin(); it != intersection.nodePath.end(); ++it)
             {
                 osg::UserDataContainer* userDataContainer = (*it)->getUserDataContainer();
@@ -1029,11 +1032,25 @@ namespace MWRender
                 {
                     if (PtrHolder* p = dynamic_cast<PtrHolder*>(userDataContainer->getUserObject(i)))
                         ptrHolder = p;
+                    if (RefnumMarker* r = dynamic_cast<RefnumMarker*>(userDataContainer->getUserObject(i)))
+                        refnumMarkers.push_back(r);
                 }
             }
 
             if (ptrHolder)
                 result.mHitObject = ptrHolder->mPtr;
+
+            unsigned int vertexCounter = 0;
+            for (unsigned int i=0; i<refnumMarkers.size(); ++i)
+            {
+                unsigned int intersectionIndex = intersection.indexList.empty() ? 0 : intersection.indexList[0];
+                if (!refnumMarkers[i]->mNumVertices || (intersectionIndex >= vertexCounter && intersectionIndex < vertexCounter + refnumMarkers[i]->mNumVertices))
+                {
+                    result.mHitRefnum = refnumMarkers[i]->mRefnum;
+                    break;
+                }
+                vertexCounter += refnumMarkers[i]->mNumVertices;
+            }
         }
 
         return result;
@@ -1046,6 +1063,7 @@ namespace MWRender
             mIntersectionVisitor = new osgUtil::IntersectionVisitor;
 
         mIntersectionVisitor->setTraversalNumber(mViewer->getFrameStamp()->getFrameNumber());
+        mIntersectionVisitor->setFrameStamp(mViewer->getFrameStamp());
         mIntersectionVisitor->setIntersector(intersector);
 
         int mask = ~0;
@@ -1111,6 +1129,8 @@ namespace MWRender
         mSky->setMoonColour(false);
 
         notifyWorldSpaceChanged();
+        if (mObjectPaging)
+            mObjectPaging->clear();
     }
 
     MWRender::Animation* RenderingManager::getAnimation(const MWWorld::Ptr &ptr)
@@ -1467,4 +1487,43 @@ namespace MWRender
 
         mRecastMesh->update(mNavigator.getRecastMeshTiles(), mNavigator.getSettings());
     }
+
+    void RenderingManager::setActiveGrid(const osg::Vec4i &grid)
+    {
+        mTerrain->setActiveGrid(grid);
+    }
+    bool RenderingManager::pagingEnableObject(int type, const MWWorld::ConstPtr& ptr, bool enabled)
+    {
+        if (!ptr.isInCell() || !ptr.getCell()->isExterior() || !mObjectPaging)
+            return false;
+        if (mObjectPaging->enableObject(type, ptr.getCellRef().getRefNum(), ptr.getCellRef().getPosition().asVec3(), enabled))
+        {
+            mTerrain->rebuildViews();
+            return true;
+        }
+        return false;
+    }
+    void RenderingManager::pagingBlacklistObject(int type, const MWWorld::ConstPtr &ptr)
+    {
+        if (!ptr.isInCell() || !ptr.getCell()->isExterior() || !mObjectPaging)
+            return;
+        const ESM::RefNum & refnum = ptr.getCellRef().getRefNum();
+        if (!refnum.hasContentFile()) return;
+        if (mObjectPaging->blacklistObject(type, refnum, ptr.getCellRef().getPosition().asVec3()))
+            mTerrain->rebuildViews();
+    }
+    bool RenderingManager::pagingUnlockCache()
+    {
+        if (mObjectPaging && mObjectPaging->unlockCache())
+        {
+            mTerrain->rebuildViews();
+            return true;
+        }
+        return false;
+    }
+    void RenderingManager::getPagedRefnums(const osg::Vec4i &activeGrid, std::set<ESM::RefNum> &out)
+    {
+        if (mObjectPaging)
+            mObjectPaging->getPagedRefnums(activeGrid, out);
+    }
 }

apps/openmw/mwrender/renderingmanager.hpp

@@ -42,6 +42,7 @@ namespace osgViewer
 namespace ESM
 {
     struct Cell;
+    struct RefNum;
 }
 
 namespace Terrain
@@ -84,6 +85,7 @@ namespace MWRender
     class NavMesh;
     class ActorsPaths;
     class RecastMesh;
+    class ObjectPaging;
 
     class RenderingManager : public MWRender::RenderingInterface
     {
@@ -155,6 +157,7 @@ namespace MWRender
             osg::Vec3f mHitNormalWorld;
             osg::Vec3f mHitPointWorld;
             MWWorld::Ptr mHitObject;
+            ESM::RefNum mHitRefnum;
             float mRatio;
         };
 
@@ -165,7 +168,7 @@ namespace MWRender
         RayResult castCameraToViewportRay(const float nX, const float nY, float maxDistance, bool ignorePlayer, bool ignoreActors=false);
 
         /// Get the bounding box of the given object in screen coordinates as (minX, minY, maxX, maxY), with (0,0) being the top left corner.
-        osg::Vec4f getScreenBounds(const MWWorld::Ptr& ptr);
+        osg::Vec4f getScreenBounds(const osg::BoundingBox &worldbb);
 
         void setSkyEnabled(bool enabled);
 
@@ -237,6 +240,13 @@ namespace MWRender
 
         void setNavMeshNumber(const std::size_t value);
 
+        void setActiveGrid(const osg::Vec4i &grid);
+
+        bool pagingEnableObject(int type, const MWWorld::ConstPtr& ptr, bool enabled);
+        void pagingBlacklistObject(int type, const MWWorld::ConstPtr &ptr);
+        bool pagingUnlockCache();
+        void getPagedRefnums(const osg::Vec4i &activeGrid, std::set<ESM::RefNum> &out);
+
     private:
         void updateProjectionMatrix();
         void updateTextureFiltering();
@@ -275,6 +285,7 @@ namespace MWRender
         std::unique_ptr<Water> mWater;
         std::unique_ptr<Terrain::World> mTerrain;
         TerrainStorage* mTerrainStorage;
+        std::unique_ptr<ObjectPaging> mObjectPaging;
         std::unique_ptr<SkyManager> mSky;
         std::unique_ptr<FogManager> mFog;
         std::unique_ptr<EffectManager> mEffectManager;

apps/openmw/mwworld/cellpreloader.cpp

@@ -8,6 +8,7 @@
 #include <components/resource/resourcesystem.hpp>
 #include <components/resource/bulletshapemanager.hpp>
 #include <components/resource/keyframemanager.hpp>
+#include <components/vfs/manager.hpp>
 #include <components/misc/resourcehelpers.hpp>
 #include <components/misc/stringops.hpp>
 #include <components/terrain/world.hpp>
@@ -65,24 +66,8 @@ namespace MWWorld
             mTerrainView = mTerrain->createView();
 
             ListModelsVisitor visitor (mMeshes);
-            if (cell->getState() == MWWorld::CellStore::State_Loaded)
-            {
             cell->forEach(visitor);
         }
-            else
-            {
-                const std::vector<std::string>& objectIds = cell->getPreloadedIds();
-
-                // could possibly build the model list in the worker thread if we manage to make the Store thread safe
-                for (const std::string& id : objectIds)
-                {
-                    MWWorld::ManualRef ref(MWBase::Environment::get().getWorld()->getStore(), id);
-                    std::string model = ref.getPtr().getClass().getModel(ref.getPtr());
-                    if (!model.empty())
-                        mMeshes.push_back(model);
-                }
-            }
-        }
 
         virtual void abort()
         {
@@ -97,7 +82,7 @@ namespace MWWorld
                 try
                 {
                     mTerrain->cacheCell(mTerrainView.get(), mX, mY);
-                    mPreloadedObjects.push_back(mLandManager->getLand(mX, mY));
+                    mPreloadedObjects.insert(mLandManager->getLand(mX, mY));
                 }
                 catch(std::exception& e)
                 {
@@ -113,17 +98,7 @@ namespace MWWorld
                 {
                     mesh = Misc::ResourceHelpers::correctActorModelPath(mesh, mSceneManager->getVFS());
 
-                    if (mPreloadInstances)
-                    {
-                        mPreloadedObjects.push_back(mSceneManager->cacheInstance(mesh));
-                        mPreloadedObjects.push_back(mBulletShapeManager->cacheInstance(mesh));
-                    }
-                    else
-                    {
-                        mPreloadedObjects.push_back(mSceneManager->getTemplate(mesh));
-                        mPreloadedObjects.push_back(mBulletShapeManager->getShape(mesh));
-                    }
-
+                    bool animated = false;
                     size_t slashpos = mesh.find_last_of("/\\");
                     if (slashpos != std::string::npos && slashpos != mesh.size()-1)
                     {
@@ -134,12 +109,24 @@ namespace MWWorld
                             if(kfname.size() > 4 && kfname.compare(kfname.size()-4, 4, ".nif") == 0)
                             {
                                 kfname.replace(kfname.size()-4, 4, ".kf");
-                                mPreloadedObjects.push_back(mKeyframeManager->get(kfname));
+                                if (mSceneManager->getVFS()->exists(kfname))
+                                {
+                                    mPreloadedObjects.insert(mKeyframeManager->get(kfname));
+                                    animated = true;
                                 }
+                            }
+                        }
+                    }
+                    if (mPreloadInstances && animated)
+                        mPreloadedObjects.insert(mSceneManager->cacheInstance(mesh));
+                    else
+                        mPreloadedObjects.insert(mSceneManager->getTemplate(mesh));
+                    if (mPreloadInstances)
+                        mPreloadedObjects.insert(mBulletShapeManager->cacheInstance(mesh));
+                    else
+                        mPreloadedObjects.insert(mBulletShapeManager->getShape(mesh));
 
                 }
-                    }
-                }
                 catch (std::exception& e)
                 {
                     // ignore error for now, would spam the log too much
@@ -166,24 +153,28 @@ namespace MWWorld
         osg::ref_ptr<Terrain::View> mTerrainView;
 
         // keep a ref to the loaded objects to make sure it stays loaded as long as this cell is in the preloaded state
-        std::vector<osg::ref_ptr<const osg::Object> > mPreloadedObjects;
+        std::set<osg::ref_ptr<const osg::Object> > mPreloadedObjects;
     };
 
     class TerrainPreloadItem : public SceneUtil::WorkItem
     {
     public:
-        TerrainPreloadItem(const std::vector<osg::ref_ptr<Terrain::View> >& views, Terrain::World* world, const std::vector<osg::Vec3f>& preloadPositions)
+        TerrainPreloadItem(const std::vector<osg::ref_ptr<Terrain::View> >& views, Terrain::World* world, const std::vector<CellPreloader::PositionCellGrid>& preloadPositions)
             : mAbort(false)
+            , mProgress(views.size())
+            , mProgressRange(0)
             , mTerrainViews(views)
             , mWorld(world)
             , mPreloadPositions(preloadPositions)
         {
         }
 
-        void storeViews(double referenceTime)
+        bool storeViews(double referenceTime)
         {
             for (unsigned int i=0; i<mTerrainViews.size() && i<mPreloadPositions.size(); ++i)
-                mWorld->storeView(mTerrainViews[i], referenceTime);
+                if (!mWorld->storeView(mTerrainViews[i], referenceTime))
+                    return false;
+            return true;
         }
 
         virtual void doWork()
@@ -191,7 +182,7 @@ namespace MWWorld
             for (unsigned int i=0; i<mTerrainViews.size() && i<mPreloadPositions.size() && !mAbort; ++i)
             {
                 mTerrainViews[i]->reset();
-                mWorld->preload(mTerrainViews[i], mPreloadPositions[i], mAbort);
+                mWorld->preload(mTerrainViews[i], mPreloadPositions[i].first, mPreloadPositions[i].second, mAbort, mProgress[i], mProgressRange);
             }
         }
 
@@ -200,11 +191,16 @@ namespace MWWorld
             mAbort = true;
         }
 
+        int getProgress() const { return !mProgress.empty() ? mProgress[0].load() : 0; }
+        int getProgressRange() const { return !mProgress.empty() && mProgress[0].load() ? mProgressRange : 0; }
+
     private:
         std::atomic<bool> mAbort;
+        std::vector<std::atomic<int>> mProgress;
+        int mProgressRange;
         std::vector<osg::ref_ptr<Terrain::View> > mTerrainViews;
         Terrain::World* mWorld;
-        std::vector<osg::Vec3f> mPreloadPositions;
+        std::vector<CellPreloader::PositionCellGrid> mPreloadPositions;
     };
 
     /// Worker thread item: update the resource system's cache, effectively deleting unused entries.
@@ -237,6 +233,7 @@ namespace MWWorld
         , mMaxCacheSize(0)
         , mPreloadInstances(true)
         , mLastResourceCacheUpdate(0.0)
+        , mStoreViewsFailCount(0)
     {
     }
 
@@ -328,9 +325,6 @@ namespace MWWorld
             }
 
             mPreloadCells.erase(found);
-
-            if (cell->isExterior() && mTerrainPreloadItem && mTerrainPreloadItem->isDone())
-                mTerrainPreloadItem->storeViews(0.0);
         }
     }
 
@@ -375,7 +369,17 @@ namespace MWWorld
 
         if (mTerrainPreloadItem && mTerrainPreloadItem->isDone())
         {
-            mTerrainPreloadItem->storeViews(timestamp);
+            if (!mTerrainPreloadItem->storeViews(timestamp))
+            {
+                if (++mStoreViewsFailCount > 100)
+                {
+                    OSG_ALWAYS << "paging views are rebuilt every frame, please check for faulty enable/disable scripts." << std::endl;
+                    mStoreViewsFailCount = 0;
+                }
+                setTerrainPreloadPositions(std::vector<PositionCellGrid>());
+            }
+            else
+                mStoreViewsFailCount = 0;
             mTerrainPreloadItem = nullptr;
         }
     }
@@ -415,11 +419,71 @@ namespace MWWorld
         mUnrefQueue = unrefQueue;
     }
 
-    void CellPreloader::setTerrainPreloadPositions(const std::vector<osg::Vec3f> &positions)
+    bool CellPreloader::syncTerrainLoad(const std::vector<CellPreloader::PositionCellGrid> &positions, int& progress, int& progressRange, double timestamp)
     {
-        if (mTerrainPreloadItem && !mTerrainPreloadItem->isDone())
+        if (!mTerrainPreloadItem)
+            return true;
+        else if (mTerrainPreloadItem->isDone())
+        {
+            if (mTerrainPreloadItem->storeViews(timestamp))
+            {
+                mTerrainPreloadItem = nullptr;
+                return true;
+            }
+            else
+            {
+                setTerrainPreloadPositions(std::vector<CellPreloader::PositionCellGrid>());
+                setTerrainPreloadPositions(positions);
+                return false;
+            }
+        }
+        else
+        {
+            progress = mTerrainPreloadItem->getProgress();
+            progressRange = mTerrainPreloadItem->getProgressRange();
+            return false;
+        }
+    }
+
+    void CellPreloader::abortTerrainPreloadExcept(const CellPreloader::PositionCellGrid *exceptPos)
+    {
+        const float resetThreshold = ESM::Land::REAL_SIZE;
+        for (auto pos : mTerrainPreloadPositions)
+            if (exceptPos && (pos.first-exceptPos->first).length2() < resetThreshold*resetThreshold && pos.second == exceptPos->second)
                 return;
-        else if (positions == mTerrainPreloadPositions)
+        if (mTerrainPreloadItem && !mTerrainPreloadItem->isDone())
+        {
+            mTerrainPreloadItem->abort();
+            mTerrainPreloadItem->waitTillDone();
+        }
+        setTerrainPreloadPositions(std::vector<CellPreloader::PositionCellGrid>());
+    }
+
+    bool contains(const std::vector<CellPreloader::PositionCellGrid>& container, const std::vector<CellPreloader::PositionCellGrid>& contained)
+    {
+        for (auto pos : contained)
+        {
+            bool found = false;
+            for (auto pos2 : container)
+            {
+                if ((pos.first-pos2.first).length2() < 1 && pos.second == pos2.second)
+                {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) return false;
+        }
+        return true;
+    }
+
+    void CellPreloader::setTerrainPreloadPositions(const std::vector<CellPreloader::PositionCellGrid> &positions)
+    {
+        if (positions.empty())
+            mTerrainPreloadPositions.clear();
+        else if (contains(mTerrainPreloadPositions, positions))
+            return;
+        if (mTerrainPreloadItem && !mTerrainPreloadItem->isDone())
             return;
         else
         {
@@ -436,9 +500,12 @@ namespace MWWorld
             }
 
             mTerrainPreloadPositions = positions;
+            if (!positions.empty())
+            {
                 mTerrainPreloadItem = new TerrainPreloadItem(mTerrainViews, mTerrain, positions);
                 mWorkQueue->addWorkItem(mTerrainPreloadItem);
             }
         }
+    }
 
 }

apps/openmw/mwworld/cellpreloader.hpp

@@ -4,6 +4,7 @@
 #include <map>
 #include <osg/ref_ptr>
 #include <osg/Vec3f>
+#include <osg/Vec4i>
 #include <components/sceneutil/workqueue.hpp>
 
 namespace Resource
@@ -68,7 +69,11 @@ namespace MWWorld
 
         void setUnrefQueue(SceneUtil::UnrefQueue* unrefQueue);
 
-        void setTerrainPreloadPositions(const std::vector<osg::Vec3f>& positions);
+        typedef std::pair<osg::Vec3f, osg::Vec4i> PositionCellGrid;
+        void setTerrainPreloadPositions(const std::vector<PositionCellGrid>& positions);
+
+        bool syncTerrainLoad(const std::vector<CellPreloader::PositionCellGrid> &positions, int& progress, int& progressRange, double timestamp);
+        void abortTerrainPreloadExcept(const PositionCellGrid *exceptPos);
 
     private:
         Resource::ResourceSystem* mResourceSystem;
@@ -83,6 +88,7 @@ namespace MWWorld
         bool mPreloadInstances;
 
         double mLastResourceCacheUpdate;
+        int mStoreViewsFailCount;
 
         struct PreloadEntry
         {
@@ -105,7 +111,7 @@ namespace MWWorld
         PreloadMap mPreloadCells;
 
         std::vector<osg::ref_ptr<Terrain::View> > mTerrainViews;
-        std::vector<osg::Vec3f> mTerrainPreloadPositions;
+        std::vector<PositionCellGrid> mTerrainPreloadPositions;
         osg::ref_ptr<TerrainPreloadItem> mTerrainPreloadItem;
         osg::ref_ptr<SceneUtil::WorkItem> mUpdateCacheItem;
     };

apps/openmw/mwworld/cellstore.cpp

@@ -106,7 +106,7 @@ namespace
 
     template<typename RecordType, typename T>
     void readReferenceCollection (ESM::ESMReader& reader,
-        MWWorld::CellRefList<T>& collection, const ESM::CellRef& cref, const std::map<int, int>& contentFileMap)
+        MWWorld::CellRefList<T>& collection, const ESM::CellRef& cref, const std::map<int, int>& contentFileMap, MWWorld::CellStore* cellstore)
     {
         const MWWorld::ESMStore& esmStore = MWBase::Environment::get().getWorld()->getStore();
 
@@ -141,7 +141,18 @@ namespace
                 if (iter->mRef.getRefNum()==state.mRef.mRefNum && *iter->mRef.getRefIdPtr() == state.mRef.mRefID)
                 {
                     // overwrite existing reference
+                    float oldscale = iter->mRef.getScale();
                     iter->load (state);
+                    const ESM::Position & oldpos = iter->mRef.getPosition();
+                    const ESM::Position & newpos = iter->mData.getPosition();
+                    const MWWorld::Ptr ptr(&*iter, cellstore);
+                    if ((oldscale != iter->mRef.getScale() || oldpos.asVec3() != newpos.asVec3() || oldpos.rot[0] != newpos.rot[0] || oldpos.rot[1] != newpos.rot[1] || oldpos.rot[2] != newpos.rot[2]) && !ptr.getClass().isActor())
+                        MWBase::Environment::get().getWorld()->moveObject(ptr, newpos.pos[0], newpos.pos[1], newpos.pos[2]);
+                    if (!iter->mData.isEnabled())
+                    {
+                        iter->mData.enable();
+                        MWBase::Environment::get().getWorld()->disable(MWWorld::Ptr(&*iter, cellstore));
+                    }
                     return;
                 }
 
@@ -154,28 +165,6 @@ namespace
         ref.load (state);
         collection.mList.push_back (ref);
     }
-
-    struct SearchByRefNumVisitor
-    {
-        MWWorld::LiveCellRefBase* mFound;
-        ESM::RefNum mRefNumToFind;
-
-        SearchByRefNumVisitor(const ESM::RefNum& toFind)
-            : mFound(nullptr)
-            , mRefNumToFind(toFind)
-        {
-        }
-
-        bool operator()(const MWWorld::Ptr& ptr)
-        {
-            if (ptr.getCellRef().getRefNum() == mRefNumToFind)
-            {
-                mFound = ptr.getBase();
-                return false;
-            }
-            return true;
-        }
-    };
 }
 
 namespace MWWorld
@@ -252,9 +241,7 @@ namespace MWWorld
             throw std::runtime_error("moveTo: can't move object from a non-loaded cell (how did you get this object anyway?)");
 
         // Ensure that the object actually exists in the cell
-        SearchByRefNumVisitor searchVisitor(object.getCellRef().getRefNum());
-        forEach(searchVisitor);
-        if (!searchVisitor.mFound)
+        if (searchViaRefNum(object.getCellRef().getRefNum()).isEmpty())
             throw std::runtime_error("moveTo: object is not in this cell");
 
 
@@ -809,107 +796,107 @@ namespace MWWorld
             {
                 case ESM::REC_ACTI:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mActivators, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mActivators, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_ALCH:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mPotions, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mPotions, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_APPA:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mAppas, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mAppas, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_ARMO:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mArmors, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mArmors, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_BOOK:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mBooks, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mBooks, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_CLOT:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mClothes, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mClothes, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_CONT:
 
-                    readReferenceCollection<ESM::ContainerState> (reader, mContainers, cref, contentFileMap);
+                    readReferenceCollection<ESM::ContainerState> (reader, mContainers, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_CREA:
 
-                    readReferenceCollection<ESM::CreatureState> (reader, mCreatures, cref, contentFileMap);
+                    readReferenceCollection<ESM::CreatureState> (reader, mCreatures, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_DOOR:
 
-                    readReferenceCollection<ESM::DoorState> (reader, mDoors, cref, contentFileMap);
+                    readReferenceCollection<ESM::DoorState> (reader, mDoors, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_INGR:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mIngreds, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mIngreds, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_LEVC:
 
-                    readReferenceCollection<ESM::CreatureLevListState> (reader, mCreatureLists, cref, contentFileMap);
+                    readReferenceCollection<ESM::CreatureLevListState> (reader, mCreatureLists, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_LEVI:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mItemLists, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mItemLists, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_LIGH:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mLights, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mLights, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_LOCK:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mLockpicks, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mLockpicks, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_MISC:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mMiscItems, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mMiscItems, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_NPC_:
 
-                    readReferenceCollection<ESM::NpcState> (reader, mNpcs, cref, contentFileMap);
+                    readReferenceCollection<ESM::NpcState> (reader, mNpcs, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_PROB:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mProbes, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mProbes, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_REPA:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mRepairs, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mRepairs, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_STAT:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mStatics, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mStatics, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_WEAP:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mWeapons, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mWeapons, cref, contentFileMap, this);
                     break;
 
                 case ESM::REC_BODY:
 
-                    readReferenceCollection<ESM::ObjectState> (reader, mBodyParts, cref, contentFileMap);
+                    readReferenceCollection<ESM::ObjectState> (reader, mBodyParts, cref, contentFileMap, this);
                     break;
 
                 default:
@@ -931,26 +918,22 @@ namespace MWWorld
             movedTo.load(reader);
 
             // Search for the reference. It might no longer exist if its content file was removed.
-            SearchByRefNumVisitor visitor(refnum);
-            forEachInternal(visitor);
-
-            if (!visitor.mFound)
+            Ptr movedRef = searchViaRefNum(refnum);
+            if (movedRef.isEmpty())
             {
                 Log(Debug::Warning) << "Warning: Dropping moved ref tag for " << refnum.mIndex << " (moved object no longer exists)";
                 continue;
             }
 
-            MWWorld::LiveCellRefBase* movedRef = visitor.mFound;
-
             CellStore* otherCell = callback->getCellStore(movedTo);
 
             if (otherCell == nullptr)
             {
-                Log(Debug::Warning) << "Warning: Dropping moved ref tag for " << movedRef->mRef.getRefId()
+                Log(Debug::Warning) << "Warning: Dropping moved ref tag for " << movedRef.getCellRef().getRefId()
                                     << " (target cell " << movedTo.mWorldspace << " no longer exists). Reference moved back to its original location.";
                 // Note by dropping tag the object will automatically re-appear in its original cell, though potentially at inapproriate coordinates.
                 // Restore original coordinates:
-                movedRef->mData.setPosition(movedRef->mRef.getPosition());
+                movedRef.getRefData().setPosition(movedRef.getCellRef().getPosition());
                 continue;
             }
 
@@ -961,7 +944,7 @@ namespace MWWorld
                 continue;
             }
 
-            moveTo(MWWorld::Ptr(movedRef, this), otherCell);
+            moveTo(movedRef, otherCell);
         }
     }
 

apps/openmw/mwworld/esmstore.cpp

@@ -136,6 +136,10 @@ void ESMStore::setUp(bool validateRecords)
                 mIds[*record] = storeIt->first;
         }
     }
+
+    if (mStaticIds.empty())
+        mStaticIds = mIds;
+
     mSkills.setUp();
     mMagicEffects.setUp();
     mAttributes.setUp();

apps/openmw/mwworld/esmstore.hpp

@@ -68,6 +68,8 @@ namespace MWWorld
         // Lookup of all IDs. Makes looking up references faster. Just
         // maps the id name to the record type.
         std::map<std::string, int> mIds;
+        std::map<std::string, int> mStaticIds;
+
         std::map<int, StoreBase *> mStores;
 
         ESM::NPC mPlayerTemplate;
@@ -99,6 +101,14 @@ namespace MWWorld
             }
             return it->second;
         }
+        int findStatic(const std::string &id) const
+        {
+            std::map<std::string, int>::const_iterator it = mStaticIds.find(id);
+            if (it == mStaticIds.end()) {
+                return 0;
+            }
+            return it->second;
+        }
 
         ESMStore()
           : mDynamicCount(0)

apps/openmw/mwworld/scene.cpp

@@ -13,6 +13,7 @@
 #include <components/resource/scenemanager.hpp>
 #include <components/resource/bulletshape.hpp>
 #include <components/sceneutil/unrefqueue.hpp>
+#include <components/sceneutil/positionattitudetransform.hpp>
 #include <components/detournavigator/navigator.hpp>
 #include <components/detournavigator/debug.hpp>
 #include <components/misc/convert.hpp>
@@ -86,8 +87,21 @@ namespace
         );
     }
 
+    std::string getModel(const MWWorld::Ptr &ptr, const VFS::Manager *vfs)
+    {
+        bool useAnim = ptr.getClass().useAnim();
+        std::string model = ptr.getClass().getModel(ptr);
+        if (useAnim)
+            model = Misc::ResourceHelpers::correctActorModelPath(model, vfs);
+
+        const std::string &id = ptr.getCellRef().getRefId();
+        if (id == "prisonmarker" || id == "divinemarker" || id == "templemarker" || id == "northmarker")
+            model = ""; // marker objects that have a hardcoded function in the game logic, should be hidden from the player
+        return model;
+    }
+
     void addObject(const MWWorld::Ptr& ptr, MWPhysics::PhysicsSystem& physics,
-                   MWRender::RenderingManager& rendering)
+                   MWRender::RenderingManager& rendering, std::set<ESM::RefNum>& pagedRefs)
     {
         if (ptr.getRefData().getBaseNode() || physics.getActor(ptr))
         {
@@ -96,15 +110,13 @@ namespace
         }
 
         bool useAnim = ptr.getClass().useAnim();
-        std::string model = ptr.getClass().getModel(ptr);
-        if (useAnim)
-            model = Misc::ResourceHelpers::correctActorModelPath(model, rendering.getResourceSystem()->getVFS());
-
-        std::string id = ptr.getCellRef().getRefId();
-        if (id == "prisonmarker" || id == "divinemarker" || id == "templemarker" || id == "northmarker")
-            model = ""; // marker objects that have a hardcoded function in the game logic, should be hidden from the player
+        std::string model = getModel(ptr, rendering.getResourceSystem()->getVFS());
 
+        const ESM::RefNum& refnum = ptr.getCellRef().getRefNum();
+        if (!refnum.hasContentFile() || pagedRefs.find(refnum) == pagedRefs.end())
             ptr.getClass().insertObjectRendering(ptr, model, rendering);
+        else
+            ptr.getRefData().setBaseNode(new SceneUtil::PositionAttitudeTransform); // FIXME remove this when physics code is fixed not to depend on basenode
         setNodeRotation(ptr, rendering, RotationOrder::direct);
 
         ptr.getClass().insertObject (ptr, model, physics);
@@ -183,27 +195,6 @@ namespace
         }
     }
 
-    void updateObjectRotation (const MWWorld::Ptr& ptr, MWPhysics::PhysicsSystem& physics,
-                                    MWRender::RenderingManager& rendering, RotationOrder order)
-    {
-        setNodeRotation(ptr, rendering, order);
-        physics.updateRotation(ptr);
-    }
-
-    void updateObjectScale(const MWWorld::Ptr& ptr, MWPhysics::PhysicsSystem& physics,
-                            MWRender::RenderingManager& rendering)
-    {
-        if (ptr.getRefData().getBaseNode() != nullptr)
-        {
-            float scale = ptr.getCellRef().getScale();
-            osg::Vec3f scaleVec (scale, scale, scale);
-            ptr.getClass().adjustScale(ptr, scaleVec, true);
-            rendering.scaleObject(ptr, scaleVec);
-
-            physics.updateScale(ptr);
-        }
-    }
-
     struct InsertVisitor
     {
         MWWorld::CellStore& mCell;
@@ -276,50 +267,48 @@ namespace
 namespace MWWorld
 {
 
+    void Scene::removeFromPagedRefs(const Ptr &ptr)
+    {
+        const ESM::RefNum& refnum = ptr.getCellRef().getRefNum();
+        if (refnum.hasContentFile() && mPagedRefs.erase(refnum))
+        {
+            if (!ptr.getRefData().getBaseNode()) return;
+            ptr.getClass().insertObjectRendering(ptr, getModel(ptr, mRendering.getResourceSystem()->getVFS()), mRendering);
+            setNodeRotation(ptr, mRendering, RotationOrder::direct);
+            reloadTerrain();
+        }
+    }
+
+    void Scene::updateObjectPosition(const Ptr &ptr, const osg::Vec3f &pos, bool movePhysics)
+    {
+        mRendering.moveObject(ptr, pos);
+        if (movePhysics)
+        {
+            mPhysics->updatePosition(ptr);
+        }
+    }
+
     void Scene::updateObjectRotation(const Ptr &ptr, RotationOrder order)
     {
-        ::updateObjectRotation(ptr, *mPhysics, mRendering, order);
+        setNodeRotation(ptr, mRendering, order);
+        mPhysics->updateRotation(ptr);
     }
 
     void Scene::updateObjectScale(const Ptr &ptr)
     {
-        ::updateObjectScale(ptr, *mPhysics, mRendering);
-    }
-
-    void Scene::getGridCenter(int &cellX, int &cellY)
-    {
-        int maxX = std::numeric_limits<int>::min();
-        int maxY = std::numeric_limits<int>::min();
-        int minX = std::numeric_limits<int>::max();
-        int minY = std::numeric_limits<int>::max();
-        CellStoreCollection::iterator iter = mActiveCells.begin();
-        while (iter!=mActiveCells.end())
-        {
-            assert ((*iter)->getCell()->isExterior());
-            int x = (*iter)->getCell()->getGridX();
-            int y = (*iter)->getCell()->getGridY();
-            maxX = std::max(x, maxX);
-            maxY = std::max(y, maxY);
-            minX = std::min(x, minX);
-            minY = std::min(y, minY);
-            ++iter;
-        }
-        cellX = (minX + maxX) / 2;
-        cellY = (minY + maxY) / 2;
+        float scale = ptr.getCellRef().getScale();
+        osg::Vec3f scaleVec (scale, scale, scale);
+        ptr.getClass().adjustScale(ptr, scaleVec, true);
+        mRendering.scaleObject(ptr, scaleVec);
+        mPhysics->updateScale(ptr);
     }
 
     void Scene::update (float duration, bool paused)
     {
-        mPreloadTimer += duration;
-        if (mPreloadTimer > 0.1f)
-        {
-            preloadCells(0.1f);
-            mPreloadTimer = 0.f;
-        }
+        mPreloader->updateCache(mRendering.getReferenceTime());
+        preloadCells(duration);
 
         mRendering.update (duration, paused);
-
-        mPreloader->updateCache(mRendering.getReferenceTime());
     }
 
     void Scene::unloadCell (CellStoreCollection::iterator iter, bool test)
@@ -488,6 +477,27 @@ namespace MWWorld
         mPreloader->clear();
     }
 
+    osg::Vec4i Scene::gridCenterToBounds(const osg::Vec2i& centerCell) const
+    {
+        return osg::Vec4i(centerCell.x()-mHalfGridSize,centerCell.y()-mHalfGridSize,centerCell.x()+mHalfGridSize+1,centerCell.y()+mHalfGridSize+1);
+    }
+
+    osg::Vec2i Scene::getNewGridCenter(const osg::Vec3f &pos, const osg::Vec2i* currentGridCenter) const
+    {
+        if (currentGridCenter)
+        {
+            float centerX, centerY;
+            MWBase::Environment::get().getWorld()->indexToPosition(currentGridCenter->x(), currentGridCenter->y(), centerX, centerY, true);
+            float distance = std::max(std::abs(centerX-pos.x()), std::abs(centerY-pos.y()));
+            const float maxDistance = Constants::CellSizeInUnits / 2 + mCellLoadingThreshold; // 1/2 cell size + threshold
+            if (distance <= maxDistance)
+                return *currentGridCenter;
+        }
+        osg::Vec2i newCenter;
+        MWBase::Environment::get().getWorld()->positionToIndex(pos.x(), pos.y(), newCenter.x(), newCenter.y());
+        return newCenter;
+    }
+
     void Scene::playerMoved(const osg::Vec3f &pos)
     {
         const auto navigator = MWBase::Environment::get().getWorld()->getNavigator();
@@ -497,30 +507,13 @@ namespace MWWorld
         if (!mCurrentCell || !mCurrentCell->isExterior())
             return;
 
-        // figure out the center of the current cell grid (*not* necessarily mCurrentCell, which is the cell the player is in)
-        int cellX, cellY;
-        getGridCenter(cellX, cellY);
-        float centerX, centerY;
-        MWBase::Environment::get().getWorld()->indexToPosition(cellX, cellY, centerX, centerY, true);
-        const float maxDistance = Constants::CellSizeInUnits / 2 + mCellLoadingThreshold; // 1/2 cell size + threshold
-        float distance = std::max(std::abs(centerX-pos.x()), std::abs(centerY-pos.y()));
-        if (distance > maxDistance)
-        {
-            int newX, newY;
-            MWBase::Environment::get().getWorld()->positionToIndex(pos.x(), pos.y(), newX, newY);
-            changeCellGrid(newX, newY);
-        }
+        osg::Vec2i newCell = getNewGridCenter(pos, &mCurrentGridCenter);
+        if (newCell != mCurrentGridCenter)
+            changeCellGrid(pos, newCell.x(), newCell.y());
     }
 
-    void Scene::changeCellGrid (int playerCellX, int playerCellY, bool changeEvent)
+    void Scene::changeCellGrid (const osg::Vec3f &pos, int playerCellX, int playerCellY, bool changeEvent)
     {
-        Loading::Listener* loadingListener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
-        Loading::ScopedLoad load(loadingListener);
-
-        int messagesCount = MWBase::Environment::get().getWindowManager()->getMessagesCount();
-        std::string loadingExteriorText = "#{sLoadingMessage3}";
-        loadingListener->setLabel(loadingExteriorText, false, messagesCount > 0);
-
         CellStoreCollection::iterator active = mActiveCells.begin();
         while (active!=mActiveCells.end())
         {
@@ -537,6 +530,14 @@ namespace MWWorld
             unloadCell (active++);
         }
 
+        mCurrentGridCenter = osg::Vec2i(playerCellX, playerCellY);
+        osg::Vec4i newGrid = gridCenterToBounds(mCurrentGridCenter);
+        mRendering.setActiveGrid(newGrid);
+
+        preloadTerrain(pos, true);
+        mPagedRefs.clear();
+        mRendering.getPagedRefnums(newGrid, mPagedRefs);
+
         std::size_t refsToLoad = 0;
         std::vector<std::pair<int, int>> cellsPositionsToLoad;
         // get the number of refs to load
@@ -565,6 +566,11 @@ namespace MWWorld
             }
         }
 
+        Loading::Listener* loadingListener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
+        Loading::ScopedLoad load(loadingListener);
+        int messagesCount = MWBase::Environment::get().getWindowManager()->getMessagesCount();
+        std::string loadingExteriorText = "#{sLoadingMessage3}";
+        loadingListener->setLabel(loadingExteriorText, false, messagesCount > 0);
         loadingListener->setProgressRange(refsToLoad);
 
         const auto getDistanceToPlayerCell = [&] (const std::pair<int, int>& cellPosition)
@@ -745,13 +751,12 @@ namespace MWWorld
 
         MWBase::Environment::get().getWorld()->adjustSky();
 
-        mLastPlayerPos = pos.asVec3();
+        mLastPlayerPos = player.getRefData().getPosition().asVec3();
     }
 
     Scene::Scene (MWRender::RenderingManager& rendering, MWPhysics::PhysicsSystem *physics,
                   DetourNavigator::Navigator& navigator)
     : mCurrentCell (0), mCellChanged (false), mPhysics(physics), mRendering(rendering), mNavigator(navigator)
-    , mPreloadTimer(0.f)
     , mHalfGridSize(Settings::Manager::getInt("exterior cell load distance", "Cells"))
     , mCellLoadingThreshold(1024.f)
     , mPreloadDistance(Settings::Manager::getInt("preload distance", "Cells"))
@@ -828,6 +833,7 @@ namespace MWWorld
         loadingListener->setProgressRange(cell->count());
 
         // Load cell.
+        mPagedRefs.clear();
         loadCell (cell, loadingListener, changeEvent);
 
         changePlayerCell(cell, position, adjustPlayerPos);
@@ -857,7 +863,7 @@ namespace MWWorld
         if (changeEvent)
             MWBase::Environment::get().getWindowManager()->fadeScreenOut(0.5);
 
-        changeCellGrid(x, y, changeEvent);
+        changeCellGrid(position.asVec3(), x, y, changeEvent);
 
         CellStore* current = MWBase::Environment::get().getWorld()->getExterior(x, y);
         changePlayerCell(current, position, adjustPlayerPos);
@@ -880,7 +886,7 @@ namespace MWWorld
     {
         InsertVisitor insertVisitor (cell, *loadingListener, test);
         cell.forEach (insertVisitor);
-        insertVisitor.insert([&] (const MWWorld::Ptr& ptr) { addObject(ptr, *mPhysics, mRendering); });
+        insertVisitor.insert([&] (const MWWorld::Ptr& ptr) { addObject(ptr, *mPhysics, mRendering, mPagedRefs); });
         insertVisitor.insert([&] (const MWWorld::Ptr& ptr) { addObject(ptr, *mPhysics, mNavigator); });
 
         // do adjustPosition (snapping actors to ground) after objects are loaded, so we don't depend on the loading order
@@ -892,7 +898,7 @@ namespace MWWorld
     {
         try
         {
-            addObject(ptr, *mPhysics, mRendering);
+            addObject(ptr, *mPhysics, mRendering, mPagedRefs);
             addObject(ptr, *mPhysics, mNavigator);
             MWBase::Environment::get().getWorld()->scaleObject(ptr, ptr.getCellRef().getScale());
             const auto navigator = MWBase::Environment::get().getWorld()->getNavigator();
@@ -924,6 +930,7 @@ namespace MWWorld
         mRendering.removeObject (ptr);
         if (ptr.getClass().isActor())
             mRendering.removeWaterRippleEmitter(ptr);
+        ptr.getRefData().setBaseNode(nullptr);
     }
 
     bool Scene::isCellActive(const CellStore &cell)
@@ -983,7 +990,8 @@ namespace MWWorld
 
     void Scene::preloadCells(float dt)
     {
-        std::vector<osg::Vec3f> exteriorPositions;
+        if (dt<=1e-06) return;
+        std::vector<PositionCellGrid> exteriorPositions;
 
         const MWWorld::ConstPtr player = MWBase::Environment::get().getWorld()->getPlayerPtr();
         osg::Vec3f playerPos = player.getRefData().getPosition().asVec3();
@@ -991,7 +999,7 @@ namespace MWWorld
         osg::Vec3f predictedPos = playerPos + moved / dt * mPredictionTime;
 
         if (mCurrentCell->isExterior())
-            exteriorPositions.push_back(predictedPos);
+            exteriorPositions.emplace_back(predictedPos, gridCenterToBounds(getNewGridCenter(predictedPos, &mCurrentGridCenter)));
 
         mLastPlayerPos = playerPos;
 
@@ -1008,7 +1016,7 @@ namespace MWWorld
         mPreloader->setTerrainPreloadPositions(exteriorPositions);
     }
 
-    void Scene::preloadTeleportDoorDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos, std::vector<osg::Vec3f>& exteriorPositions)
+    void Scene::preloadTeleportDoorDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos, std::vector<PositionCellGrid>& exteriorPositions)
     {
         std::vector<MWWorld::ConstPtr> teleportDoors;
         for (const MWWorld::CellStore* cellStore : mActiveCells)
@@ -1042,7 +1050,7 @@ namespace MWWorld
                         int x,y;
                         MWBase::Environment::get().getWorld()->positionToIndex (pos.x(), pos.y(), x, y);
                         preloadCell(MWBase::Environment::get().getWorld()->getExterior(x,y), true);
-                        exteriorPositions.push_back(pos);
+                        exteriorPositions.emplace_back(pos, gridCenterToBounds(getNewGridCenter(pos)));
                     }
                 }
                 catch (std::exception& e)
@@ -1062,7 +1070,7 @@ namespace MWWorld
 
 
         int cellX,cellY;
-        getGridCenter(cellX,cellY);
+        cellX = mCurrentGridCenter.x(); cellY = mCurrentGridCenter.y();
 
         float centerX, centerY;
         MWBase::Environment::get().getWorld()->indexToPosition(cellX, cellY, centerX, centerY, true);
@@ -1108,11 +1116,41 @@ namespace MWWorld
             mPreloader->preload(cell, mRendering.getReferenceTime());
     }
 
-    void Scene::preloadTerrain(const osg::Vec3f &pos)
+    void Scene::preloadTerrain(const osg::Vec3f &pos, bool sync)
     {
-        std::vector<osg::Vec3f> vec;
-        vec.push_back(pos);
+        std::vector<PositionCellGrid> vec;
+        vec.emplace_back(pos, gridCenterToBounds(getNewGridCenter(pos)));
+        if (sync && mRendering.pagingUnlockCache())
+            mPreloader->abortTerrainPreloadExcept(nullptr);
+        else
+            mPreloader->abortTerrainPreloadExcept(&vec[0]);
         mPreloader->setTerrainPreloadPositions(vec);
+        if (!sync) return;
+
+        Loading::Listener* loadingListener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
+        Loading::ScopedLoad load(loadingListener);
+        int progress = 0, initialProgress = -1, progressRange = 0;
+        while (!mPreloader->syncTerrainLoad(vec, progress, progressRange, mRendering.getReferenceTime()))
+        {
+            if (initialProgress == -1)
+            {
+                loadingListener->setLabel("#{sLoadingMessage4}");
+                initialProgress = progress;
+            }
+            if (progress)
+            {
+                loadingListener->setProgressRange(std::max(0, progressRange-initialProgress));
+                loadingListener->setProgress(progress-initialProgress);
+            }
+            else
+                loadingListener->setProgress(0);
+            OpenThreads::Thread::microSleep(5000);
+        }
+    }
+
+    void Scene::reloadTerrain()
+    {
+        mPreloader->setTerrainPreloadPositions(std::vector<PositionCellGrid>());
     }
 
     struct ListFastTravelDestinationsVisitor
@@ -1145,7 +1183,7 @@ namespace MWWorld
         std::vector<ESM::Transport::Dest> mList;
     };
 
-    void Scene::preloadFastTravelDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& /*predictedPos*/, std::vector<osg::Vec3f>& exteriorPositions) // ignore predictedPos here since opening dialogue with travel service takes extra time
+    void Scene::preloadFastTravelDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& /*predictedPos*/, std::vector<PositionCellGrid>& exteriorPositions) // ignore predictedPos here since opening dialogue with travel service takes extra time
     {
         const MWWorld::ConstPtr player = MWBase::Environment::get().getWorld()->getPlayerPtr();
         ListFastTravelDestinationsVisitor listVisitor(mPreloadDistance, player.getRefData().getPosition().asVec3());
@@ -1166,7 +1204,7 @@ namespace MWWorld
                 int x,y;
                 MWBase::Environment::get().getWorld()->positionToIndex( pos.x(), pos.y(), x, y);
                 preloadCell(MWBase::Environment::get().getWorld()->getExterior(x,y), true);
-                exteriorPositions.push_back(pos);
+                exteriorPositions.emplace_back(pos, gridCenterToBounds(getNewGridCenter(pos)));
             }
         }
     }

apps/openmw/mwworld/scene.hpp

@@ -1,6 +1,9 @@
 #ifndef GAME_MWWORLD_SCENE_H
 #define GAME_MWWORLD_SCENE_H
 
+#include <osg/Vec4i>
+#include <osg/Vec2i>
+
 #include "ptr.hpp"
 #include "globals.hpp"
 
@@ -72,7 +75,6 @@ namespace MWWorld
             MWRender::RenderingManager& mRendering;
             DetourNavigator::Navigator& mNavigator;
             std::unique_ptr<CellPreloader> mPreloader;
-            float mPreloadTimer;
             int mHalfGridSize;
             float mCellLoadingThreshold;
             float mPreloadDistance;
@@ -85,17 +87,23 @@ namespace MWWorld
 
             osg::Vec3f mLastPlayerPos;
 
+            std::set<ESM::RefNum> mPagedRefs;
+
             void insertCell (CellStore &cell, Loading::Listener* loadingListener, bool test = false);
+            osg::Vec2i mCurrentGridCenter;
 
             // Load and unload cells as necessary to create a cell grid with "X" and "Y" in the center
-            void changeCellGrid (int playerCellX, int playerCellY, bool changeEvent = true);
+            void changeCellGrid (const osg::Vec3f &pos, int playerCellX, int playerCellY, bool changeEvent = true);
 
-            void getGridCenter(int& cellX, int& cellY);
+            typedef std::pair<osg::Vec3f, osg::Vec4i> PositionCellGrid;
 
             void preloadCells(float dt);
-            void preloadTeleportDoorDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos, std::vector<osg::Vec3f>& exteriorPositions);
+            void preloadTeleportDoorDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos, std::vector<PositionCellGrid>& exteriorPositions);
             void preloadExteriorGrid(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos);
-            void preloadFastTravelDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos, std::vector<osg::Vec3f>& exteriorPositions);
+            void preloadFastTravelDestinations(const osg::Vec3f& playerPos, const osg::Vec3f& predictedPos, std::vector<PositionCellGrid>& exteriorPositions);
+
+            osg::Vec4i gridCenterToBounds(const osg::Vec2i &centerCell) const;
+            osg::Vec2i getNewGridCenter(const osg::Vec3f &pos, const osg::Vec2i *currentGridCenter = nullptr) const;
 
         public:
 
@@ -105,7 +113,8 @@ namespace MWWorld
             ~Scene();
 
             void preloadCell(MWWorld::CellStore* cell, bool preloadSurrounding=false);
-            void preloadTerrain(const osg::Vec3f& pos);
+            void preloadTerrain(const osg::Vec3f& pos, bool sync=false);
+            void reloadTerrain();
 
             void unloadCell (CellStoreCollection::iterator iter, bool test = false);
 
@@ -143,8 +152,11 @@ namespace MWWorld
             void removeObjectFromScene (const Ptr& ptr);
             ///< Remove an object from the scene, but not from the world model.
 
+            void removeFromPagedRefs(const Ptr &ptr);
+
             void updateObjectRotation(const Ptr& ptr, RotationOrder order);
             void updateObjectScale(const Ptr& ptr);
+            void updateObjectPosition(const Ptr &ptr, const osg::Vec3f &pos, bool movePhysics);
 
             bool isCellActive(const CellStore &cell);
 

apps/openmw/mwworld/store.cpp

@@ -139,18 +139,26 @@ namespace MWWorld
         std::string idLower = Misc::StringUtils::lowerCase(id);
 
         typename Dynamic::const_iterator dit = mDynamic.find(idLower);
-        if (dit != mDynamic.end()) {
+        if (dit != mDynamic.end())
             return &dit->second;
-        }
 
         typename std::map<std::string, T>::const_iterator it = mStatic.find(idLower);
-
-        if (it != mStatic.end() && Misc::StringUtils::ciEqual(it->second.mId, id)) {
+        if (it != mStatic.end())
             return &(it->second);
-        }
 
         return 0;
     }
+    template<typename T>
+    const T *Store<T>::searchStatic(const std::string &id) const
+    {
+        std::string idLower = Misc::StringUtils::lowerCase(id);
+        typename std::map<std::string, T>::const_iterator it = mStatic.find(idLower);
+        if (it != mStatic.end())
+            return &(it->second);
+
+        return 0;
+    }
+
     template<typename T>
     bool Store<T>::isDynamic(const std::string &id) const
     {
@@ -276,7 +284,7 @@ namespace MWWorld
 
         typename std::map<std::string, T>::iterator it = mStatic.find(idLower);
 
-        if (it != mStatic.end() && Misc::StringUtils::ciEqual(it->second.mId, id)) {
+        if (it != mStatic.end()) {
             // delete from the static part of mShared
             typename std::vector<T *>::iterator sharedIter = mShared.begin();
             typename std::vector<T *>::iterator end = sharedIter + mStatic.size();
@@ -553,7 +561,7 @@ namespace MWWorld
 
         std::map<std::string, ESM::Cell>::const_iterator it = mInt.find(cell.mName);
 
-        if (it != mInt.end() && Misc::StringUtils::ciEqual(it->second.mName, id)) {
+        if (it != mInt.end()) {
             return &(it->second);
         }
 
@@ -582,6 +590,18 @@ namespace MWWorld
 
         return 0;
     }
+    const ESM::Cell *Store<ESM::Cell>::searchStatic(int x, int y) const
+    {
+        ESM::Cell cell;
+        cell.mData.mX = x, cell.mData.mY = y;
+
+        std::pair<int, int> key(x, y);
+        DynamicExt::const_iterator it = mExt.find(key);
+        if (it != mExt.end()) {
+            return &(it->second);
+        }
+        return 0;
+    }
     const ESM::Cell *Store<ESM::Cell>::searchOrCreate(int x, int y)
     {
         std::pair<int, int> key(x, y);
@@ -1104,9 +1124,8 @@ namespace MWWorld
     {
         auto it = mStatic.find(Misc::StringUtils::lowerCase(id));
 
-        if (it != mStatic.end() && Misc::StringUtils::ciEqual(it->second.mId, id)) {
+        if (it != mStatic.end())
             mStatic.erase(it);
-        }
 
         return true;
     }

apps/openmw/mwworld/store.hpp

@@ -167,6 +167,7 @@ namespace MWWorld
         void setUp();
 
         const T *search(const std::string &id) const;
+        const T *searchStatic(const std::string &id) const;
 
         /**
          * Does the record with this ID come from the dynamic store?
@@ -297,6 +298,7 @@ namespace MWWorld
 
         const ESM::Cell *search(const std::string &id) const;
         const ESM::Cell *search(int x, int y) const;
+        const ESM::Cell *searchStatic(int x, int y) const;
         const ESM::Cell *searchOrCreate(int x, int y);
 
         const ESM::Cell *find(const std::string &id) const;

apps/openmw/mwworld/worldimp.cpp

@@ -243,7 +243,6 @@ namespace MWWorld
         if (bypass && !mStartCell.empty())
         {
             ESM::Position pos;
-
             if (findExteriorPosition (mStartCell, pos))
             {
                 changeToExteriorCell (pos, true);
@@ -384,9 +383,9 @@ namespace MWWorld
                 mPlayer->readRecord(reader, type);
                 if (getPlayerPtr().isInCell())
                 {
-                    mWorldScene->preloadCell(getPlayerPtr().getCell(), true);
                     if (getPlayerPtr().getCell()->isExterior())
                         mWorldScene->preloadTerrain(getPlayerPtr().getRefData().getPosition().asVec3());
+                    mWorldScene->preloadCell(getPlayerPtr().getCell(), true);
                 }
                 break;
             default:
@@ -814,6 +813,13 @@ namespace MWWorld
 
             if(mWorldScene->getActiveCells().find (reference.getCell()) != mWorldScene->getActiveCells().end() && reference.getRefData().getCount())
                 mWorldScene->addObjectToScene (reference);
+
+            if (reference.getCellRef().getRefNum().hasContentFile())
+            {
+                int type = mStore.find(Misc::StringUtils::lowerCase(reference.getCellRef().getRefId()));
+                if (mRendering->pagingEnableObject(type, reference, true))
+                    mWorldScene->reloadTerrain();
+            }
         }
     }
 
@@ -838,21 +844,28 @@ namespace MWWorld
 
     void World::disable (const Ptr& reference)
     {
+        if (!reference.getRefData().isEnabled())
+            return;
+
         // disable is a no-op for items in containers
         if (!reference.isInCell())
             return;
 
-        if (reference.getRefData().isEnabled())
-        {
         if (reference == getPlayerPtr())
             throw std::runtime_error("can not disable player object");
 
         reference.getRefData().disable();
 
+        if (reference.getCellRef().getRefNum().hasContentFile())
+        {
+            int type = mStore.find(Misc::StringUtils::lowerCase(reference.getCellRef().getRefId()));
+            if (mRendering->pagingEnableObject(type, reference, false))
+                mWorldScene->reloadTerrain();
+        }
+
         if(mWorldScene->getActiveCells().find (reference.getCell())!=mWorldScene->getActiveCells().end() && reference.getRefData().getCount())
             mWorldScene->removeObjectFromScene (reference);
     }
-    }
 
     void World::advanceTime (double hours, bool incremental)
     {
@@ -1191,20 +1204,22 @@ namespace MWWorld
         }
         if (haveToMove && newPtr.getRefData().getBaseNode())
         {
-            mRendering->moveObject(newPtr, vec);
+            mWorldScene->updateObjectPosition(newPtr, vec, movePhysics);
             if (movePhysics)
             {
-                mPhysics->updatePosition(newPtr);
-                mPhysics->updatePtr(ptr, newPtr);
-
-                if (const auto object = mPhysics->getObject(newPtr))
+                if (const auto object = mPhysics->getObject(ptr))
                     updateNavigatorObject(object);
             }
         }
+
         if (isPlayer)
-        {
             mWorldScene->playerMoved(vec);
+        else
+        {
+            mRendering->pagingBlacklistObject(mStore.find(ptr.getCellRef().getRefId()), ptr);
+            mWorldScene->removeFromPagedRefs(newPtr);
         }
+
         return newPtr;
     }
 
@@ -1233,8 +1248,14 @@ namespace MWWorld
         if (mPhysics->getActor(ptr))
             mNavigator->removeAgent(getPathfindingHalfExtents(ptr));
 
+        if (scale != ptr.getCellRef().getScale())
+        {
             ptr.getCellRef().setScale(scale);
+            mRendering->pagingBlacklistObject(mStore.find(ptr.getCellRef().getRefId()), ptr);
+            mWorldScene->removeFromPagedRefs(ptr);
+        }
 
+        if(ptr.getRefData().getBaseNode() != 0)
             mWorldScene->updateObjectScale(ptr);
 
         if (mPhysics->getActor(ptr))
@@ -1279,6 +1300,9 @@ namespace MWWorld
 
         ptr.getRefData().setPosition(pos);
 
+        mRendering->pagingBlacklistObject(mStore.find(ptr.getCellRef().getRefId()), ptr);
+        mWorldScene->removeFromPagedRefs(ptr);
+
         if(ptr.getRefData().getBaseNode() != 0)
         {
             const auto order = flags & MWBase::RotationFlag_inverseOrder
@@ -1910,8 +1934,15 @@ namespace MWWorld
             // retrieve object dimensions so we know where to place the floating label
             if (!object.isEmpty ())
             {
-                osg::Vec4f screenBounds = mRendering->getScreenBounds(object);
-
+                osg::BoundingBox bb = mPhysics->getBoundingBox(object);
+                if (!bb.valid() && object.getRefData().getBaseNode())
+                {
+                    osg::ComputeBoundsVisitor computeBoundsVisitor;
+                    computeBoundsVisitor.setTraversalMask(~(MWRender::Mask_ParticleSystem|MWRender::Mask_Effect));
+                    object.getRefData().getBaseNode()->accept(computeBoundsVisitor);
+                    bb = computeBoundsVisitor.getBoundingBox();
+                }
+                osg::Vec4f screenBounds = mRendering->getScreenBounds(bb);
                 MWBase::Environment::get().getWindowManager()->setFocusObjectScreenCoords(
                     screenBounds.x(), screenBounds.y(), screenBounds.z(), screenBounds.w());
             }
@@ -1941,6 +1972,14 @@ namespace MWWorld
             rayToObject = mRendering->castCameraToViewportRay(0.5f, 0.5f, maxDistance, ignorePlayer);
 
         facedObject = rayToObject.mHitObject;
+        if (facedObject.isEmpty() && rayToObject.mHitRefnum.hasContentFile())
+        {
+            for (CellStore* cellstore : mWorldScene->getActiveCells())
+            {
+                facedObject = cellstore->searchViaRefNum(rayToObject.mHitRefnum);
+                if (!facedObject.isEmpty()) break;
+            }
+        }
         if (rayToObject.mHit)
             mDistanceToFacedObject = (rayToObject.mRatio * maxDistance) - camDist;
         else
@@ -3552,6 +3591,8 @@ namespace MWWorld
     std::string World::exportSceneGraph(const Ptr &ptr)
     {
         std::string file = mUserDataPath + "/openmw.osgt";
+        mRendering->pagingBlacklistObject(mStore.find(ptr.getCellRef().getRefId()), ptr);
+        mWorldScene->removeFromPagedRefs(ptr);
         mRendering->exportSceneGraph(ptr, file, "Ascii");
         return file;
     }

components/detournavigator/navigator.hpp

@@ -155,6 +155,11 @@ namespace DetourNavigator
          */
         virtual void update(const osg::Vec3f& playerPosition) = 0;
 
+        /**
+         * @brief disable navigator updates
+         */
+        virtual void setUpdatesEnabled(bool enabled) = 0;
+
         /**
          * @brief wait locks thread until all tiles are updated from last update call.
          */

components/detournavigator/navigatorimpl.cpp

@@ -12,6 +12,7 @@ namespace DetourNavigator
     NavigatorImpl::NavigatorImpl(const Settings& settings)
         : mSettings(settings)
         , mNavMeshManager(mSettings)
+        , mUpdatesEnabled(true)
     {
     }
 
@@ -138,11 +139,18 @@ namespace DetourNavigator
 
     void NavigatorImpl::update(const osg::Vec3f& playerPosition)
     {
+        if (!mUpdatesEnabled)
+            return;
         removeUnusedNavMeshes();
         for (const auto& v : mAgents)
             mNavMeshManager.update(playerPosition, v.first);
     }
 
+    void NavigatorImpl::setUpdatesEnabled(bool enabled)
+    {
+        mUpdatesEnabled = enabled;
+    }
+
     void NavigatorImpl::wait()
     {
         mNavMeshManager.wait();

components/detournavigator/navigatorimpl.hpp

@@ -46,6 +46,8 @@ namespace DetourNavigator
 
         void update(const osg::Vec3f& playerPosition) override;
 
+        void setUpdatesEnabled(bool enabled) override;
+
         void wait() override;
 
         SharedNavMeshCacheItem getNavMesh(const osg::Vec3f& agentHalfExtents) const override;
@@ -61,6 +63,7 @@ namespace DetourNavigator
     private:
         Settings mSettings;
         NavMeshManager mNavMeshManager;
+        bool mUpdatesEnabled;
         std::map<osg::Vec3f, std::size_t> mAgents;
         std::unordered_map<ObjectId, ObjectId> mAvoidIds;
         std::unordered_map<ObjectId, ObjectId> mWaterIds;

components/detournavigator/navigatorstub.hpp

@@ -66,6 +66,8 @@ namespace DetourNavigator
 
         void update(const osg::Vec3f& /*playerPosition*/) override {}
 
+        void setUpdatesEnabled(bool enabled) override {}
+
         void wait() override {}
 
         SharedNavMeshCacheItem getNavMesh(const osg::Vec3f& /*agentHalfExtents*/) const override

components/esm/cellref.cpp

@@ -232,19 +232,3 @@ void ESM::CellRef::blank()
         mPos.rot[i] = 0;
     }
 }
-
-bool ESM::operator== (const RefNum& left, const RefNum& right)
-{
-    return left.mIndex==right.mIndex && left.mContentFile==right.mContentFile;
-}
-
-bool ESM::operator< (const RefNum& left, const RefNum& right)
-{
-    if (left.mIndex<right.mIndex)
-        return true;
-
-    if (left.mIndex>right.mIndex)
-        return false;
-
-    return left.mContentFile<right.mContentFile;
-}

components/esm/cellref.hpp

@@ -114,8 +114,20 @@ namespace ESM
             void blank();
     };
 
-    bool operator== (const RefNum& left, const RefNum& right);
-    bool operator< (const RefNum& left, const RefNum& right);
+    inline bool operator== (const RefNum& left, const RefNum& right)
+    {
+        return left.mIndex==right.mIndex && left.mContentFile==right.mContentFile;
+    }
+
+    inline bool operator< (const RefNum& left, const RefNum& right)
+    {
+        if (left.mIndex<right.mIndex)
+            return true;
+        if (left.mIndex>right.mIndex)
+            return false;
+        return left.mContentFile<right.mContentFile;
+    }
+
 }
 
 #endif

components/nifosg/controller.cpp

@@ -352,8 +352,9 @@ void RollController::operator() (osg::Node* node, osg::NodeVisitor* nv)
     }
 }
 
-AlphaController::AlphaController(const Nif::NiFloatData *data)
+AlphaController::AlphaController(const Nif::NiFloatData *data, const osg::Material* baseMaterial)
     : mData(data->mKeyList, 1.f)
+    , mBaseMaterial(baseMaterial)
 {
 
 }
@@ -365,14 +366,13 @@ AlphaController::AlphaController()
 AlphaController::AlphaController(const AlphaController &copy, const osg::CopyOp &copyop)
     : StateSetUpdater(copy, copyop), Controller(copy)
     , mData(copy.mData)
+    , mBaseMaterial(copy.mBaseMaterial)
 {
 }
 
 void AlphaController::setDefaults(osg::StateSet *stateset)
 {
-    // need to create a deep copy of StateAttributes we will modify
-    osg::Material* mat = static_cast<osg::Material*>(stateset->getAttribute(osg::StateAttribute::MATERIAL));
-    stateset->setAttribute(osg::clone(mat, osg::CopyOp::DEEP_COPY_ALL), osg::StateAttribute::ON);
+    stateset->setAttribute(static_cast<osg::Material*>(mBaseMaterial->clone(osg::CopyOp::DEEP_COPY_ALL)), osg::StateAttribute::ON);
 }
 
 void AlphaController::apply(osg::StateSet *stateset, osg::NodeVisitor *nv)
@@ -387,9 +387,10 @@ void AlphaController::apply(osg::StateSet *stateset, osg::NodeVisitor *nv)
     }
 }
 
-MaterialColorController::MaterialColorController(const Nif::NiPosData *data, TargetColor color)
+MaterialColorController::MaterialColorController(const Nif::NiPosData *data, TargetColor color, const osg::Material* baseMaterial)
     : mData(data->mKeyList, osg::Vec3f(1,1,1))
     , mTargetColor(color)
+    , mBaseMaterial(baseMaterial)
 {
 }
 
@@ -401,14 +402,13 @@ MaterialColorController::MaterialColorController(const MaterialColorController &
     : StateSetUpdater(copy, copyop), Controller(copy)
     , mData(copy.mData)
     , mTargetColor(copy.mTargetColor)
+    , mBaseMaterial(copy.mBaseMaterial)
 {
 }
 
 void MaterialColorController::setDefaults(osg::StateSet *stateset)
 {
-    // need to create a deep copy of StateAttributes we will modify
-    osg::Material* mat = static_cast<osg::Material*>(stateset->getAttribute(osg::StateAttribute::MATERIAL));
-    stateset->setAttribute(osg::clone(mat, osg::CopyOp::DEEP_COPY_ALL), osg::StateAttribute::ON);
+    stateset->setAttribute(static_cast<osg::Material*>(mBaseMaterial->clone(osg::CopyOp::DEEP_COPY_ALL)), osg::StateAttribute::ON);
 }
 
 void MaterialColorController::apply(osg::StateSet *stateset, osg::NodeVisitor *nv)

components/nifosg/controller.hpp

@@ -24,6 +24,7 @@ namespace osg
 {
     class Node;
     class StateSet;
+    class Material;
 }
 
 namespace osgParticle
@@ -268,9 +269,9 @@ namespace NifOsg
     {
     private:
         FloatInterpolator mData;
-
+        osg::ref_ptr<const osg::Material> mBaseMaterial;
     public:
-        AlphaController(const Nif::NiFloatData *data);
+        AlphaController(const Nif::NiFloatData *data, const osg::Material* baseMaterial);
         AlphaController();
         AlphaController(const AlphaController& copy, const osg::CopyOp& copyop);
 
@@ -291,7 +292,7 @@ namespace NifOsg
             Specular = 2,
             Emissive = 3
         };
-        MaterialColorController(const Nif::NiPosData *data, TargetColor color);
+        MaterialColorController(const Nif::NiPosData *data, TargetColor color, const osg::Material* baseMaterial);
         MaterialColorController();
         MaterialColorController(const MaterialColorController& copy, const osg::CopyOp& copyop);
 
@@ -304,6 +305,7 @@ namespace NifOsg
     private:
         Vec3Interpolator mData;
         TargetColor mTargetColor = Ambient;
+        osg::ref_ptr<const osg::Material> mBaseMaterial;
     };
 
     class FlipController : public SceneUtil::StateSetUpdater, public SceneUtil::Controller

components/nifosg/nifloader.cpp

@@ -639,7 +639,15 @@ namespace NifOsg
                 handleParticleSystem(nifNode, node, composite, animflags, rootNode);
 
             if (composite->getNumControllers() > 0)
-                node->addUpdateCallback(composite);
+            {
+                osg::Callback *cb = composite;
+                if (composite->getNumControllers() == 1)
+                    cb = composite->getController(0);
+                if (animflags & Nif::NiNode::AnimFlag_AutoPlay)
+                    node->addCullCallback(cb);
+                else
+                    node->addUpdateCallback(cb); // have to remain as UpdateCallback so AssignControllerSourcesVisitor can find it.
+            }
 
             bool isAnimated = false;
             handleNodeControllers(nifNode, node, animflags, isAnimated);
@@ -778,7 +786,7 @@ namespace NifOsg
             }
         }
 
-        void handleMaterialControllers(const Nif::Property *materialProperty, SceneUtil::CompositeStateSetUpdater* composite, int animflags)
+        void handleMaterialControllers(const Nif::Property *materialProperty, SceneUtil::CompositeStateSetUpdater* composite, int animflags, const osg::Material* baseMaterial)
         {
             for (Nif::ControllerPtr ctrl = materialProperty->controller; !ctrl.empty(); ctrl = ctrl->next)
             {
@@ -789,7 +797,7 @@ namespace NifOsg
                     const Nif::NiAlphaController* alphactrl = static_cast<const Nif::NiAlphaController*>(ctrl.getPtr());
                     if (alphactrl->data.empty())
                         continue;
-                    osg::ref_ptr<AlphaController> osgctrl(new AlphaController(alphactrl->data.getPtr()));
+                    osg::ref_ptr<AlphaController> osgctrl(new AlphaController(alphactrl->data.getPtr(), baseMaterial));
                     setupController(alphactrl, osgctrl, animflags);
                     composite->addController(osgctrl);
                 }
@@ -799,7 +807,7 @@ namespace NifOsg
                     if (matctrl->data.empty())
                         continue;
                     auto targetColor = static_cast<MaterialColorController::TargetColor>(matctrl->targetColor);
-                    osg::ref_ptr<MaterialColorController> osgctrl(new MaterialColorController(matctrl->data.getPtr(), targetColor));
+                    osg::ref_ptr<MaterialColorController> osgctrl(new MaterialColorController(matctrl->data.getPtr(), targetColor, baseMaterial));
                     setupController(matctrl, osgctrl, animflags);
                     composite->addController(osgctrl);
                 }
@@ -1767,7 +1775,7 @@ namespace NifOsg
                     if (!matprop->controller.empty())
                     {
                         hasMatCtrl = true;
-                        handleMaterialControllers(matprop, composite, animflags);
+                        handleMaterialControllers(matprop, composite, animflags, mat);
                     }
 
                     break;

components/nifosg/particle.cpp

@@ -277,7 +277,7 @@ Emitter::Emitter(const Emitter &copy, const osg::CopyOp &copyop)
     , mPlacer(copy.mPlacer)
     , mShooter(copy.mShooter)
     // need a deep copy because the remainder is stored in the object
-    , mCounter(osg::clone(copy.mCounter.get(), osg::CopyOp::DEEP_COPY_ALL))
+    , mCounter(static_cast<osgParticle::Counter*>(copy.mCounter->clone(osg::CopyOp::DEEP_COPY_ALL)))
 {
 }
 

components/resource/objectcache.hpp

@@ -161,13 +161,13 @@ class GenericObjectCache : public osg::Referenced
             }
         }
 
-        /** call operator()(osg::Object*) for each object in the cache. */
+        /** call operator()(KeyType, osg::Object*) for each object in the cache. */
         template <class Functor>
         void call(Functor& f)
         {
             OpenThreads::ScopedLock<OpenThreads::Mutex> lock(_objectCacheMutex);
             for (typename ObjectCacheMap::iterator it = _objectCache.begin(); it != _objectCache.end(); ++it)
-                f(it->second.first.get());
+                f(it->first, it->second.first.get());
         }
 
         /** Get the number of objects in the cache. */

components/resource/scenemanager.cpp

@@ -466,7 +466,7 @@ namespace Resource
         return options;
     }
 
-    osg::ref_ptr<const osg::Node> SceneManager::getTemplate(const std::string &name)
+    osg::ref_ptr<const osg::Node> SceneManager::getTemplate(const std::string &name, bool compile)
     {
         std::string normalized = name;
         mVFS->normalizeFilename(normalized);
@@ -529,7 +529,7 @@ namespace Resource
                 optimizer.optimize(loaded, options);
             }
 
-            if (mIncrementalCompileOperation)
+            if (compile && mIncrementalCompileOperation)
                 mIncrementalCompileOperation->add(loaded);
             else
                 loaded->getBound();
@@ -577,7 +577,7 @@ namespace Resource
 
     osg::ref_ptr<osg::Node> SceneManager::createInstance(const osg::Node *base)
     {
-        osg::ref_ptr<osg::Node> cloned = osg::clone(base, SceneUtil::CopyOp());
+        osg::ref_ptr<osg::Node> cloned = static_cast<osg::Node*>(base->clone(SceneUtil::CopyOp()));
 
         // add a ref to the original template, to hint to the cache that it's still being used and should be kept in cache
         cloned->getOrCreateUserDataContainer()->addUserObject(new TemplateRef(base));
@@ -713,6 +713,24 @@ namespace Resource
         mSharedStateMutex.lock();
         mSharedStateManager->prune();
         mSharedStateMutex.unlock();
+
+        if (mIncrementalCompileOperation)
+        {
+            OpenThreads::ScopedLock<OpenThreads::Mutex> lock(*mIncrementalCompileOperation->getToCompiledMutex());
+            osgUtil::IncrementalCompileOperation::CompileSets& sets = mIncrementalCompileOperation->getToCompile();
+            for(osgUtil::IncrementalCompileOperation::CompileSets::iterator it = sets.begin(); it != sets.end();)
+            {
+                int refcount = (*it)->_subgraphToCompile->referenceCount();
+                if ((*it)->_subgraphToCompile->asDrawable()) refcount -= 1; // ref by CompileList.
+                if (refcount <= 2) // ref by ObjectCache + ref by _subgraphToCompile.
+                {
+                    // no other ref = not needed anymore.
+                    it = sets.erase(it);
+                }
+                else
+                    ++it;
+            }
+        }
     }
 
     void SceneManager::clearCache()

components/resource/scenemanager.hpp

@@ -81,7 +81,7 @@ namespace Resource
         /// @note If the given filename does not exist or fails to load, an error marker mesh will be used instead.
         ///  If even the error marker mesh can not be found, an exception is thrown.
         /// @note Thread safe.
-        osg::ref_ptr<const osg::Node> getTemplate(const std::string& name);
+        osg::ref_ptr<const osg::Node> getTemplate(const std::string& name, bool compile=true);
 
         /// Create an instance of the given scene template and cache it for later use, so that future calls to getInstance() can simply
         /// return this cached object instead of creating a new one.

components/resource/stats.cpp

@@ -292,6 +292,7 @@ void StatsHandler::setUpScene(osgViewer::ViewerBase *viewer)
             "Nif",
             "Keyframe",
             "",
+            "Object Chunk",
             "Terrain Chunk",
             "Terrain Texture",
             "Land",

components/sceneutil/clone.cpp

@@ -22,20 +22,11 @@ namespace SceneUtil
                      | osg::CopyOp::DEEP_COPY_USERDATA);
     }
 
-    osg::StateSet* CopyOp::operator ()(const osg::StateSet* stateset) const
-    {
-        if (!stateset)
-            return nullptr;
-        if (stateset->getDataVariance() == osg::StateSet::DYNAMIC)
-            return osg::clone(stateset, *this);
-        return const_cast<osg::StateSet*>(stateset);
-    }
-
     osg::Object* CopyOp::operator ()(const osg::Object* node) const
     {
         // We should copy node transformations when we copy node
-        if (const NifOsg::NodeUserData* data = dynamic_cast<const NifOsg::NodeUserData*>(node))
-            return osg::clone(data, *this);
+        if (dynamic_cast<const NifOsg::NodeUserData*>(node))
+            return static_cast<NifOsg::NodeUserData*>(node->clone(*this));
 
         return osg::CopyOp::operator()(node);
     }
@@ -60,7 +51,7 @@ namespace SceneUtil
 
         if (dynamic_cast<const SceneUtil::RigGeometry*>(drawable) || dynamic_cast<const SceneUtil::MorphGeometry*>(drawable))
         {
-            return osg::clone(drawable, *this);
+            return static_cast<osg::Drawable*>(drawable->clone(*this));
         }
 
         return osg::CopyOp::operator()(drawable);
@@ -68,7 +59,7 @@ namespace SceneUtil
 
     osgParticle::ParticleProcessor* CopyOp::operator() (const osgParticle::ParticleProcessor* processor) const
     {
-        osgParticle::ParticleProcessor* cloned = osg::clone(processor, osg::CopyOp::DEEP_COPY_CALLBACKS);
+        osgParticle::ParticleProcessor* cloned = static_cast<osgParticle::ParticleProcessor*>(processor->clone(osg::CopyOp::DEEP_COPY_CALLBACKS));
         for (const auto& oldPsNewPsPair : mOldPsToNewPs)
         {
             if (processor->getParticleSystem() == oldPsNewPsPair.first)
@@ -84,7 +75,7 @@ namespace SceneUtil
 
     osgParticle::ParticleSystem* CopyOp::operator ()(const osgParticle::ParticleSystem* partsys) const
     {
-        osgParticle::ParticleSystem* cloned = osg::clone(partsys, *this);
+        osgParticle::ParticleSystem* cloned = static_cast<osgParticle::ParticleSystem*>(partsys->clone(*this));
 
         for (const auto& processorPsPair : mProcessorToOldPs)
         {

components/sceneutil/clone.hpp

@@ -17,7 +17,6 @@ namespace SceneUtil
 
     /// @par Defines the cloning behaviour we need:
     /// * Assigns updated ParticleSystem pointers on cloned emitters and programs.
-    /// * Creates deep copy of StateSets if they have a DYNAMIC data variance.
     /// * Deep copies RigGeometry and MorphGeometry so they can animate without affecting clones.
     /// @warning Do not use an object of this class for more than one copy operation.
     class CopyOp : public osg::CopyOp
@@ -31,7 +30,6 @@ namespace SceneUtil
         virtual osg::Node* operator() (const osg::Node* node) const;
         virtual osg::Drawable* operator() (const osg::Drawable* drawable) const;
 
-        virtual osg::StateSet* operator() (const osg::StateSet* stateset) const;
         virtual osg::Object* operator ()(const osg::Object* node) const;
 
     private:

components/sceneutil/morphgeometry.cpp

@@ -44,7 +44,7 @@ void MorphGeometry::setSourceGeometry(osg::ref_ptr<osg::Geometry> sourceGeom)
         osg::ref_ptr<osg::VertexBufferObject> vbo (new osg::VertexBufferObject);
         vbo->setUsage(GL_DYNAMIC_DRAW_ARB);
 
-        osg::ref_ptr<osg::Array> vertexArray = osg::clone(from.getVertexArray(), osg::CopyOp::DEEP_COPY_ALL);
+        osg::ref_ptr<osg::Array> vertexArray = static_cast<osg::Array*>(from.getVertexArray()->clone(osg::CopyOp::DEEP_COPY_ALL));
         if (vertexArray)
         {
             vertexArray->setVertexBufferObject(vbo);

components/sceneutil/mwshadowtechnique.cpp

@@ -22,6 +22,7 @@
 #include <osg/CullFace>
 #include <osg/Geometry>
 #include <osg/io_utils>
+#include <osg/Depth>
 
 #include <sstream>
 
@@ -1580,7 +1581,9 @@ void MWShadowTechnique::createShaders()
     _shadowCastingStateSet->setTextureAttributeAndModes(0, _fallbackBaseTexture.get(), osg::StateAttribute::ON);
     _shadowCastingStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
     _shadowCastingStateSet->addUniform(_shadowMapAlphaTestDisableUniform);
-
+    osg::ref_ptr<osg::Depth> depth = new osg::Depth;
+    depth->setWriteMask(true);
+    _shadowCastingStateSet->setAttribute(depth, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
     _shadowCastingStateSet->setMode(GL_DEPTH_CLAMP, osg::StateAttribute::ON);
 
     _shadowCastingStateSet->setRenderBinDetails(osg::StateSet::OPAQUE_BIN, "RenderBin", osg::StateSet::OVERRIDE_PROTECTED_RENDERBIN_DETAILS);

components/sceneutil/optimizer.cpp

@@ -18,6 +18,7 @@
 
 #include "optimizer.hpp"
 
+#include <osg/Version>
 #include <osg/Transform>
 #include <osg/MatrixTransform>
 #include <osg/PositionAttitudeTransform>
@@ -27,6 +28,7 @@
 #include <osg/Notify>
 #include <osg/Timer>
 #include <osg/io_utils>
+#include <osg/Depth>
 
 #include <osgUtil/TransformAttributeFunctor>
 #include <osgUtil/Statistics>
@@ -103,7 +105,9 @@ void Optimizer::optimize(osg::Node* node, unsigned int options)
         osg::Timer_t startTick = osg::Timer::instance()->tick();
 
         MergeGeometryVisitor mgv(this);
-        mgv.setTargetMaximumNumberOfVertices(10000);
+        mgv.setTargetMaximumNumberOfVertices(1000000);
+        mgv.setMergeAlphaBlending(_mergeAlphaBlending);
+        mgv.setViewPoint(_viewPoint);
         node->accept(mgv);
 
         osg::Timer_t endTick = osg::Timer::instance()->tick();
@@ -585,17 +589,36 @@ void Optimizer::FlattenStaticTransformsVisitor::apply(osg::Node& node)
     traverse(node);
 }
 
+bool needvbo(const osg::Geometry* geom)
+{
+#if OSG_MIN_VERSION_REQUIRED(3,5,6)
+    return true;
+#else
+    return geom->getUseVertexBufferObjects();
+#endif
+}
+
+osg::Array* cloneArray(osg::Array* array, osg::VertexBufferObject*& vbo, const osg::Geometry* geom)
+{
+    array = static_cast<osg::Array*>(array->clone(osg::CopyOp::DEEP_COPY_ALL));
+    if (!vbo && needvbo(geom))
+        vbo = new osg::VertexBufferObject;
+    if (vbo)
+        array->setVertexBufferObject(vbo);
+    return array;
+}
 
 void Optimizer::FlattenStaticTransformsVisitor::apply(osg::Drawable& drawable)
 {
     osg::Geometry *geometry = drawable.asGeometry();
     if((geometry) && (isOperationPermissibleForObject(&drawable)))
     {
+        osg::VertexBufferObject* vbo = nullptr;
         if(geometry->getVertexArray() && geometry->getVertexArray()->referenceCount() > 1) {
-            geometry->setVertexArray(dynamic_cast<osg::Array*>(geometry->getVertexArray()->clone(osg::CopyOp::DEEP_COPY_ALL)));
+            geometry->setVertexArray(cloneArray(geometry->getVertexArray(), vbo, geometry));
         }
         if(geometry->getNormalArray() && geometry->getNormalArray()->referenceCount() > 1) {
-            geometry->setNormalArray(dynamic_cast<osg::Array*>(geometry->getNormalArray()->clone(osg::CopyOp::DEEP_COPY_ALL)));
+            geometry->setNormalArray(cloneArray(geometry->getNormalArray(), vbo, geometry));
         }
     }
     _drawableSet.insert(&drawable);
@@ -988,6 +1011,17 @@ struct LessGeometry
     }
 };
 
+struct LessGeometryViewPoint
+{
+    osg::Vec3f _viewPoint;
+    bool operator() (const osg::ref_ptr<osg::Geometry>& lhs,const osg::ref_ptr<osg::Geometry>& rhs) const
+    {
+        float len1 = (lhs->getBoundingBox().center() - _viewPoint).length2();
+        float len2 = (rhs->getBoundingBox().center() - _viewPoint).length2();
+        return len2 < len1;
+    }
+};
+
 struct LessGeometryPrimitiveType
 {
     bool operator() (const osg::ref_ptr<osg::Geometry>& lhs,const osg::ref_ptr<osg::Geometry>& rhs) const
@@ -1055,16 +1089,16 @@ bool isAbleToMerge(const osg::Geometry& g1, const osg::Geometry& g2)
 void Optimizer::MergeGeometryVisitor::pushStateSet(osg::StateSet *stateSet)
 {
     _stateSetStack.push_back(stateSet);
-    checkAllowedToMerge();
+    checkAlphaBlendingActive();
 }
 
 void Optimizer::MergeGeometryVisitor::popStateSet()
 {
     _stateSetStack.pop_back();
-    checkAllowedToMerge();
+    checkAlphaBlendingActive();
 }
 
-void Optimizer::MergeGeometryVisitor::checkAllowedToMerge()
+void Optimizer::MergeGeometryVisitor::checkAlphaBlendingActive()
 {
     int renderingHint = 0;
     bool override = false;
@@ -1080,7 +1114,7 @@ void Optimizer::MergeGeometryVisitor::checkAllowedToMerge()
             override = true;
     }
     // Can't merge Geometry that are using a transparent sorting bin as that would cause the sorting to break.
-    _allowedToMerge = renderingHint != osg::StateSet::TRANSPARENT_BIN;
+    _alphaBlendingActive = renderingHint == osg::StateSet::TRANSPARENT_BIN;
 }
 
 void Optimizer::MergeGeometryVisitor::apply(osg::Group &group)
@@ -1088,7 +1122,7 @@ void Optimizer::MergeGeometryVisitor::apply(osg::Group &group)
     if (group.getStateSet())
         pushStateSet(group.getStateSet());
 
-    if (_allowedToMerge)
+    if (!_alphaBlendingActive || _mergeAlphaBlending)
         mergeGroup(group);
 
     traverse(group);
@@ -1097,6 +1131,30 @@ void Optimizer::MergeGeometryVisitor::apply(osg::Group &group)
         popStateSet();
 }
 
+osg::PrimitiveSet* clonePrimitive(osg::PrimitiveSet* ps, osg::ElementBufferObject*& ebo, const osg::Geometry* geom)
+{
+    if (ps->referenceCount() <= 1)
+        return ps;
+    ps = static_cast<osg::PrimitiveSet*>(ps->clone(osg::CopyOp::DEEP_COPY_ALL));
+
+    osg::DrawElements* drawElements = ps->getDrawElements();
+    if (!drawElements) return ps;
+
+    if (!ebo && needvbo(geom))
+        ebo = new osg::ElementBufferObject;
+    if (ebo)
+        drawElements->setElementBufferObject(ebo);
+
+    return ps;
+}
+
+bool containsSharedPrimitives(const osg::Geometry* geom)
+{
+    for (unsigned int i=0; i<geom->getNumPrimitiveSets(); ++i)
+        if (geom->getPrimitiveSet(i)->referenceCount() > 1) return true;
+    return false;
+}
+
 bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
 {
     if (!isOperationPermissibleForObject(&group)) return false;
@@ -1120,7 +1178,7 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
             osg::Geometry* geom = child->asGeometry();
             if (geom)
             {
-                if (!geometryContainsSharedArrays(*geom) &&
+                if (
                     geom->getDataVariance()!=osg::Object::DYNAMIC &&
                     isOperationPermissibleForObject(geom))
                 {
@@ -1254,6 +1312,12 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
                 DuplicateList& duplicateList = *mitr;
                 if (!duplicateList.empty())
                 {
+                    if (_alphaBlendingActive)
+                    {
+                        LessGeometryViewPoint lgvp;
+                        lgvp._viewPoint = _viewPoint;
+                        std::sort(duplicateList.begin(), duplicateList.end(), lgvp);
+                    }
                     DuplicateList::iterator ditr = duplicateList.begin();
                     osg::ref_ptr<osg::Geometry> lhs = *ditr++;
                     group.addChild(lhs.get());
@@ -1278,6 +1342,7 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
         if (!drawable)
             continue;
         osg::Geometry* geom = drawable->asGeometry();
+        osg::ElementBufferObject* ebo = nullptr;
         if (geom)
         {
             osg::Geometry::PrimitiveSetList& primitives = geom->getPrimitiveSetList();
@@ -1290,10 +1355,12 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
                 {
                     if (prim->getNumIndices()==3)
                     {
+                        prim = clonePrimitive(prim, ebo, geom); (*itr) = prim;
                         prim->setMode(osg::PrimitiveSet::TRIANGLES);
                     }
                     else if (prim->getNumIndices()==4)
                     {
+                        prim = clonePrimitive(prim, ebo, geom); (*itr) = prim;
                         prim->setMode(osg::PrimitiveSet::QUADS);
                     }
                 }
@@ -1308,6 +1375,7 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
         if (!drawable)
             continue;
         osg::Geometry* geom = drawable->asGeometry();
+        osg::ElementBufferObject* ebo = nullptr;
         if (geom)
         {
             if (geom->getNumPrimitiveSets()>0 &&
@@ -1320,6 +1388,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
 #if 1
                 bool doneCombine = false;
 
+                std::set<osg::PrimitiveSet*> toremove;
+
                 osg::Geometry::PrimitiveSetList& primitives = geom->getPrimitiveSetList();
                 unsigned int lhsNo=0;
                 unsigned int rhsNo=1;
@@ -1348,6 +1418,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
 
                     if (combine)
                     {
+                        lhs = clonePrimitive(lhs, ebo, geom);
+                        primitives[lhsNo] = lhs;
 
                         switch(lhs->getType())
                         {
@@ -1375,7 +1447,7 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
                     if (combine)
                     {
                         // make this primitive set as invalid and needing cleaning up.
-                        rhs->setMode(0xffffff);
+                        toremove.insert(rhs);
                         doneCombine = true;
                         ++rhsNo;
                     }
@@ -1390,7 +1462,6 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
                 if (doneCombine)
                 {
                     // now need to clean up primitiveset so it no longer contains the rhs combined primitives.
-
                     // first swap with a empty primitiveSet to empty it completely.
                     osg::Geometry::PrimitiveSetList oldPrimitives;
                     primitives.swap(oldPrimitives);
@@ -1400,7 +1471,7 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
                         pitr != oldPrimitives.end();
                         ++pitr)
                     {
-                        if ((*pitr)->getMode()!=0xffffff) primitives.push_back(*pitr);
+                        if (!toremove.count(*pitr)) primitives.push_back(*pitr);
                     }
                 }
     #endif
@@ -1467,6 +1538,18 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
                     }
                 }
 #endif
+                if (doneCombine && !geom->containsSharedArrays() && !containsSharedPrimitives(geom))
+                {
+                    // prefer to use vbo for merged geometries as vbo uses less memory than display lists.
+                    geom->setUseVertexBufferObjects(true);
+                    geom->setUseDisplayList(false);
+                }
+                if (_alphaBlendingActive && _mergeAlphaBlending && !geom->getStateSet())
+                {
+                    osg::Depth* d = new osg::Depth;
+                    d->setWriteMask(0);
+                    geom->getOrCreateStateSet()->setAttribute(d);
+                }
             }
         }
 
@@ -1479,34 +1562,6 @@ bool Optimizer::MergeGeometryVisitor::mergeGroup(osg::Group& group)
     return false;
 }
 
-bool Optimizer::MergeGeometryVisitor::geometryContainsSharedArrays(osg::Geometry& geom)
-{
-    if (geom.getVertexArray() && geom.getVertexArray()->referenceCount()>1) return true;
-    if (geom.getNormalArray() && geom.getNormalArray()->referenceCount()>1) return true;
-    if (geom.getColorArray() && geom.getColorArray()->referenceCount()>1) return true;
-    if (geom.getSecondaryColorArray() && geom.getSecondaryColorArray()->referenceCount()>1) return true;
-    if (geom.getFogCoordArray() && geom.getFogCoordArray()->referenceCount()>1) return true;
-
-
-    for(unsigned int unit=0;unit<geom.getNumTexCoordArrays();++unit)
-    {
-        osg::Array* tex = geom.getTexCoordArray(unit);
-        if (tex && tex->referenceCount()>1) return true;
-    }
-
-    // shift the indices of the incoming primitives to account for the pre existing geometry.
-    for(osg::Geometry::PrimitiveSetList::iterator primItr=geom.getPrimitiveSetList().begin();
-        primItr!=geom.getPrimitiveSetList().end();
-        ++primItr)
-    {
-        if ((*primItr)->referenceCount()>1) return true;
-    }
-
-
-    return false;
-}
-
-
 class MergeArrayVisitor : public osg::ArrayVisitor
 {
     protected:
@@ -1574,14 +1629,14 @@ class MergeArrayVisitor : public osg::ArrayVisitor
 
 bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geometry& rhs)
 {
-
     MergeArrayVisitor merger;
-
+    osg::VertexBufferObject* vbo = nullptr;
     unsigned int base = 0;
     if (lhs.getVertexArray() && rhs.getVertexArray())
     {
-
         base = lhs.getVertexArray()->getNumElements();
+        if (lhs.getVertexArray()->referenceCount() > 1)
+            lhs.setVertexArray(cloneArray(lhs.getVertexArray(), vbo, &lhs));
         if (!merger.merge(lhs.getVertexArray(),rhs.getVertexArray()))
         {
             OSG_DEBUG << "MergeGeometry: vertex array not merged. Some data may be lost." <<std::endl;
@@ -1596,6 +1651,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
 
     if (lhs.getNormalArray() && rhs.getNormalArray() && lhs.getNormalArray()->getBinding()!=osg::Array::BIND_OVERALL)
     {
+        if (lhs.getNormalArray()->referenceCount() > 1)
+            lhs.setNormalArray(cloneArray(lhs.getNormalArray(), vbo, &lhs));
         if (!merger.merge(lhs.getNormalArray(),rhs.getNormalArray()))
         {
             OSG_DEBUG << "MergeGeometry: normal array not merged. Some data may be lost." <<std::endl;
@@ -1609,6 +1666,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
 
     if (lhs.getColorArray() && rhs.getColorArray() && lhs.getColorArray()->getBinding()!=osg::Array::BIND_OVERALL)
     {
+        if (lhs.getColorArray()->referenceCount() > 1)
+            lhs.setColorArray(cloneArray(lhs.getColorArray(), vbo, &lhs));
         if (!merger.merge(lhs.getColorArray(),rhs.getColorArray()))
         {
             OSG_DEBUG << "MergeGeometry: color array not merged. Some data may be lost." <<std::endl;
@@ -1621,6 +1680,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
 
     if (lhs.getSecondaryColorArray() && rhs.getSecondaryColorArray() && lhs.getSecondaryColorArray()->getBinding()!=osg::Array::BIND_OVERALL)
     {
+        if (lhs.getSecondaryColorArray()->referenceCount() > 1)
+            lhs.setSecondaryColorArray(cloneArray(lhs.getSecondaryColorArray(), vbo, &lhs));
         if (!merger.merge(lhs.getSecondaryColorArray(),rhs.getSecondaryColorArray()))
         {
             OSG_DEBUG << "MergeGeometry: secondary color array not merged. Some data may be lost." <<std::endl;
@@ -1633,6 +1694,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
 
     if (lhs.getFogCoordArray() && rhs.getFogCoordArray() && lhs.getFogCoordArray()->getBinding()!=osg::Array::BIND_OVERALL)
     {
+        if (lhs.getFogCoordArray()->referenceCount() > 1)
+            lhs.setFogCoordArray(cloneArray(lhs.getFogCoordArray(), vbo, &lhs));
         if (!merger.merge(lhs.getFogCoordArray(),rhs.getFogCoordArray()))
         {
             OSG_DEBUG << "MergeGeometry: fog coord array not merged. Some data may be lost." <<std::endl;
@@ -1647,6 +1710,9 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
     unsigned int unit;
     for(unit=0;unit<lhs.getNumTexCoordArrays();++unit)
     {
+        if (!lhs.getTexCoordArray(unit)) continue;
+        if (lhs.getTexCoordArray(unit)->referenceCount() > 1)
+            lhs.setTexCoordArray(unit, cloneArray(lhs.getTexCoordArray(unit), vbo, &lhs));
         if (!merger.merge(lhs.getTexCoordArray(unit),rhs.getTexCoordArray(unit)))
         {
             OSG_DEBUG << "MergeGeometry: tex coord array not merged. Some data may be lost." <<std::endl;
@@ -1655,14 +1721,17 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
 
     for(unit=0;unit<lhs.getNumVertexAttribArrays();++unit)
     {
+        if (!lhs.getVertexAttribArray(unit)) continue;
+        if (lhs.getVertexAttribArray(unit)->referenceCount() > 1)
+            lhs.setVertexAttribArray(unit, cloneArray(lhs.getVertexAttribArray(unit), vbo, &lhs));
         if (!merger.merge(lhs.getVertexAttribArray(unit),rhs.getVertexAttribArray(unit)))
         {
             OSG_DEBUG << "MergeGeometry: vertex attrib array not merged. Some data may be lost." <<std::endl;
         }
     }
 
-
     // shift the indices of the incoming primitives to account for the pre existing geometry.
+    osg::ElementBufferObject* ebo = nullptr;
     osg::Geometry::PrimitiveSetList::iterator primItr;
     for(primItr=rhs.getPrimitiveSetList().begin(); primItr!=rhs.getPrimitiveSetList().end(); ++primItr)
     {
@@ -1684,6 +1753,11 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
                 {
                     // must promote to a DrawElementsUInt
                     osg::DrawElementsUInt* new_primitive = new osg::DrawElementsUInt(primitive->getMode());
+                    if (needvbo(&lhs))
+                    {
+                        if (!ebo) ebo = new osg::ElementBufferObject;
+                         new_primitive->setElementBufferObject(ebo);
+                    }
                     std::copy(primitiveUByte->begin(),primitiveUByte->end(),std::back_inserter(*new_primitive));
                     new_primitive->offsetIndices(base);
                     (*primItr) = new_primitive;
@@ -1691,13 +1765,19 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
                 {
                     // must promote to a DrawElementsUShort
                     osg::DrawElementsUShort* new_primitive = new osg::DrawElementsUShort(primitive->getMode());
+                    if (needvbo(&lhs))
+                    {
+                        if (!ebo) ebo = new osg::ElementBufferObject;
+                         new_primitive->setElementBufferObject(ebo);
+                    }
                     std::copy(primitiveUByte->begin(),primitiveUByte->end(),std::back_inserter(*new_primitive));
                     new_primitive->offsetIndices(base);
                     (*primItr) = new_primitive;
                 }
                 else
                 {
-                    primitive->offsetIndices(base);
+                    (*primItr) = clonePrimitive(primitive, ebo, &lhs);
+                    (*primItr)->offsetIndices(base);
                 }
             }
             break;
@@ -1716,13 +1796,19 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
                 {
                     // must promote to a DrawElementsUInt
                     osg::DrawElementsUInt* new_primitive = new osg::DrawElementsUInt(primitive->getMode());
+                    if (needvbo(&lhs))
+                    {
+                        if (!ebo) ebo = new osg::ElementBufferObject;
+                         new_primitive->setElementBufferObject(ebo);
+                    }
                     std::copy(primitiveUShort->begin(),primitiveUShort->end(),std::back_inserter(*new_primitive));
                     new_primitive->offsetIndices(base);
                     (*primItr) = new_primitive;
                 }
                 else
                 {
-                    primitive->offsetIndices(base);
+                    (*primItr) = clonePrimitive(primitive, ebo, &lhs);
+                    (*primItr)->offsetIndices(base);
                 }
             }
             break;
@@ -1731,7 +1817,8 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
         case(osg::PrimitiveSet::DrawArrayLengthsPrimitiveType):
         case(osg::PrimitiveSet::DrawElementsUIntPrimitiveType):
         default:
-            primitive->offsetIndices(base);
+            (*primItr) = clonePrimitive(primitive, ebo, &lhs);
+            (*primItr)->offsetIndices(base);
             break;
         }
     }
@@ -1744,6 +1831,10 @@ bool Optimizer::MergeGeometryVisitor::mergeGeometry(osg::Geometry& lhs,osg::Geom
     lhs.dirtyBound();
     lhs.dirtyDisplayList();
 
+    if (osg::UserDataContainer* rhsUserData = rhs.getUserDataContainer())
+        for (unsigned int i=0; i<rhsUserData->getNumUserObjects(); ++i)
+            lhs.getOrCreateUserDataContainer()->addUserObject(rhsUserData->getUserObject(i));
+
     return true;
 }
 

components/sceneutil/optimizer.hpp

@@ -65,7 +65,7 @@ class Optimizer
 
     public:
 
-        Optimizer() {}
+        Optimizer() : _mergeAlphaBlending(false) {}
         virtual ~Optimizer() {}
 
         enum OptimizationOptions
@@ -118,6 +118,9 @@ class Optimizer
                                 STATIC_OBJECT_DETECTION
         };
 
+        void setMergeAlphaBlending(bool merge) { _mergeAlphaBlending = merge; }
+        void setViewPoint(const osg::Vec3f& viewPoint) { _viewPoint = viewPoint; }
+
         /** Reset internal data to initial state - the getPermissibleOptionsMap is cleared.*/
         void reset();
 
@@ -252,6 +255,9 @@ class Optimizer
         typedef std::map<const osg::Object*,unsigned int> PermissibleOptimizationsMap;
         PermissibleOptimizationsMap _permissibleOptimizationsMap;
 
+        osg::Vec3f _viewPoint;
+        bool _mergeAlphaBlending;
+
     public:
 
         /** Flatten Static Transform nodes by applying their transform to the
@@ -371,7 +377,16 @@ class Optimizer
                 /// default to traversing all children.
                 MergeGeometryVisitor(Optimizer* optimizer=0) :
                     BaseOptimizerVisitor(optimizer, MERGE_GEOMETRY),
-                    _targetMaximumNumberOfVertices(10000), _allowedToMerge(true) {}
+                    _targetMaximumNumberOfVertices(10000), _alphaBlendingActive(false), _mergeAlphaBlending(false) {}
+
+                void setMergeAlphaBlending(bool merge)
+                {
+                    _mergeAlphaBlending = merge;
+                }
+                void setViewPoint(const osg::Vec3f& viewPoint)
+                {
+                    _viewPoint = viewPoint;
+                }
 
                 void setTargetMaximumNumberOfVertices(unsigned int num)
                 {
@@ -385,15 +400,13 @@ class Optimizer
 
                 void pushStateSet(osg::StateSet* stateSet);
                 void popStateSet();
-                void checkAllowedToMerge();
+                void checkAlphaBlendingActive();
 
                 virtual void apply(osg::Group& group);
                 virtual void apply(osg::Billboard&) { /* don't do anything*/ }
 
                 bool mergeGroup(osg::Group& group);
 
-                static bool geometryContainsSharedArrays(osg::Geometry& geom);
-
                 static bool mergeGeometry(osg::Geometry& lhs,osg::Geometry& rhs);
 
                 static bool mergePrimitive(osg::DrawArrays& lhs,osg::DrawArrays& rhs);
@@ -406,7 +419,9 @@ class Optimizer
 
                 unsigned int _targetMaximumNumberOfVertices;
                 std::vector<osg::StateSet*> _stateSetStack;
-                bool _allowedToMerge;
+                bool _alphaBlendingActive;
+                bool _mergeAlphaBlending;
+                osg::Vec3f _viewPoint;
         };
 
 };

components/sceneutil/riggeometry.cpp

@@ -77,7 +77,7 @@ void RigGeometry::setSourceGeometry(osg::ref_ptr<osg::Geometry> sourceGeometry)
         osg::ref_ptr<osg::VertexBufferObject> vbo (new osg::VertexBufferObject);
         vbo->setUsage(GL_DYNAMIC_DRAW_ARB);
 
-        osg::ref_ptr<osg::Array> vertexArray = osg::clone(from.getVertexArray(), osg::CopyOp::DEEP_COPY_ALL);
+        osg::ref_ptr<osg::Array> vertexArray = static_cast<osg::Array*>(from.getVertexArray()->clone(osg::CopyOp::DEEP_COPY_ALL));
         if (vertexArray)
         {
             vertexArray->setVertexBufferObject(vbo);
@@ -86,7 +86,7 @@ void RigGeometry::setSourceGeometry(osg::ref_ptr<osg::Geometry> sourceGeometry)
 
         if (const osg::Array* normals = from.getNormalArray())
         {
-            osg::ref_ptr<osg::Array> normalArray = osg::clone(normals, osg::CopyOp::DEEP_COPY_ALL);
+            osg::ref_ptr<osg::Array> normalArray = static_cast<osg::Array*>(normals->clone(osg::CopyOp::DEEP_COPY_ALL));
             if (normalArray)
             {
                 normalArray->setVertexBufferObject(vbo);
@@ -97,7 +97,7 @@ void RigGeometry::setSourceGeometry(osg::ref_ptr<osg::Geometry> sourceGeometry)
         if (const osg::Vec4Array* tangents = dynamic_cast<const osg::Vec4Array*>(from.getTexCoordArray(7)))
         {
             mSourceTangents = tangents;
-            osg::ref_ptr<osg::Array> tangentArray = osg::clone(tangents, osg::CopyOp::DEEP_COPY_ALL);
+            osg::ref_ptr<osg::Array> tangentArray = static_cast<osg::Array*>(tangents->clone(osg::CopyOp::DEEP_COPY_ALL));
             tangentArray->setVertexBufferObject(vbo);
             to.setTexCoordArray(7, tangentArray, osg::Array::BIND_PER_VERTEX);
         }
@@ -106,7 +106,7 @@ void RigGeometry::setSourceGeometry(osg::ref_ptr<osg::Geometry> sourceGeometry)
     }
 }
 
-osg::ref_ptr<osg::Geometry> RigGeometry::getSourceGeometry()
+osg::ref_ptr<osg::Geometry> RigGeometry::getSourceGeometry() const
 {
     return mSourceGeometry;
 }

components/sceneutil/riggeometry.hpp

@@ -44,7 +44,7 @@ namespace SceneUtil
         /// @note The source geometry will not be modified.
         void setSourceGeometry(osg::ref_ptr<osg::Geometry> sourceGeom);
 
-        osg::ref_ptr<osg::Geometry> getSourceGeometry();
+        osg::ref_ptr<osg::Geometry> getSourceGeometry() const;
 
         virtual void accept(osg::NodeVisitor &nv);
         virtual bool supports(const osg::PrimitiveFunctor&) const { return true; }

components/sceneutil/statesetupdater.cpp

@@ -2,30 +2,38 @@
 
 #include <osg/Node>
 #include <osg/NodeVisitor>
+#include <osgUtil/CullVisitor>
 
 namespace SceneUtil
 {
 
     void StateSetUpdater::operator()(osg::Node* node, osg::NodeVisitor* nv)
     {
+        bool isCullVisitor = nv->getVisitorType() == osg::NodeVisitor::CULL_VISITOR;
         if (!mStateSets[0])
         {
-            // first time setup
-            osg::StateSet* src = node->getOrCreateStateSet();
-            for (int i=0; i<2; ++i) // Using SHALLOW_COPY for StateAttributes, if users want to modify it is their responsibility to set a non-shared one first
-                                    // This can be done conveniently in user implementations of the setDefaults() method
+            for (int i=0; i<2; ++i)
             {
-                mStateSets[i] = new osg::StateSet(*src, osg::CopyOp::SHALLOW_COPY);
+                if (!isCullVisitor)
+                    mStateSets[i] = new osg::StateSet(*node->getOrCreateStateSet(), osg::CopyOp::SHALLOW_COPY); // Using SHALLOW_COPY for StateAttributes, if users want to modify it is their responsibility to set a non-shared one first in setDefaults
+                else
+                    mStateSets[i] = new osg::StateSet;
                 setDefaults(mStateSets[i]);
             }
         }
 
         osg::StateSet* stateset = mStateSets[nv->getTraversalNumber()%2];
-        node->setStateSet(stateset);
-
         apply(stateset, nv);
 
+        if (!isCullVisitor)
+            node->setStateSet(stateset);
+        else
+            static_cast<osgUtil::CullVisitor*>(nv)->pushStateSet(stateset);
+
         traverse(node, nv);
+
+        if (isCullVisitor)
+            static_cast<osgUtil::CullVisitor*>(nv)->popStateSet();
     }
 
     void StateSetUpdater::reset()

components/sceneutil/statesetupdater.hpp

@@ -13,7 +13,7 @@ namespace SceneUtil
     ///     traversals run in parallel can yield up to 200% framerates.
     /// @par Race conditions are prevented using a "double buffering" scheme - we have two StateSets that take turns,
     ///     one StateSet we can write to, the second one is currently in use by the draw traversal of the last frame.
-    /// @par Must be set as UpdateCallback on a Node.
+    /// @par Must be set as UpdateCallback or CullCallback on a Node. If set as a CullCallback, the StateSetUpdater operates on an empty StateSet, otherwise it operates on a clone of the node's existing StateSet.
     /// @note Do not add the same StateSetUpdater to multiple nodes.
     /// @note Do not add multiple StateSetControllers on the same Node as they will conflict - instead use the CompositeStateSetUpdater.
     class StateSetUpdater : public osg::NodeCallback

components/terrain/chunkmanager.cpp

@@ -4,13 +4,13 @@
 
 #include <osg/Texture2D>
 #include <osg/ClusterCullingCallback>
+#include <osg/Material>
 
 #include <osgUtil/IncrementalCompileOperation>
 
 #include <components/resource/objectcache.hpp>
 #include <components/resource/scenemanager.hpp>
 
-#include <components/sceneutil/positionattitudetransform.hpp>
 #include <components/sceneutil/lightmanager.hpp>
 
 #include "terraindrawable.hpp"
@@ -32,10 +32,14 @@ ChunkManager::ChunkManager(Storage *storage, Resource::SceneManager *sceneMgr, T
     , mCompositeMapLevel(1.f)
     , mMaxCompGeometrySize(1.f)
 {
-
+    mMultiPassRoot = new osg::StateSet;
+    mMultiPassRoot->setRenderingHint(osg::StateSet::OPAQUE_BIN);
+    osg::ref_ptr<osg::Material> material (new osg::Material);
+    material->setColorMode(osg::Material::AMBIENT_AND_DIFFUSE);
+    mMultiPassRoot->setAttributeAndModes(material, osg::StateAttribute::ON);
 }
 
-osg::ref_ptr<osg::Node> ChunkManager::getChunk(float size, const osg::Vec2f &center, unsigned char lod, unsigned int lodFlags)
+osg::ref_ptr<osg::Node> ChunkManager::getChunk(float size, const osg::Vec2f &center, unsigned char lod, unsigned int lodFlags, bool far, const osg::Vec3f& viewPoint, bool compile)
 {
     ChunkId id = std::make_tuple(center, lod, lodFlags);
     osg::ref_ptr<osg::Object> obj = mCache->getRefFromObjectCache(id);
@@ -43,7 +47,7 @@ osg::ref_ptr<osg::Node> ChunkManager::getChunk(float size, const osg::Vec2f &cen
         return obj->asNode();
     else
     {
-        osg::ref_ptr<osg::Node> node = createChunk(size, center, lod, lodFlags);
+        osg::ref_ptr<osg::Node> node = createChunk(size, center, lod, lodFlags, compile);
         mCache->addEntryToObjectCache(id, node.get());
         return node;
     }
@@ -161,12 +165,8 @@ std::vector<osg::ref_ptr<osg::StateSet> > ChunkManager::createPasses(float chunk
     return ::Terrain::createPasses(useShaders, &mSceneManager->getShaderManager(), layers, blendmapTextures, blendmapScale, blendmapScale);
 }
 
-osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Vec2f &chunkCenter, unsigned char lod, unsigned int lodFlags)
+osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Vec2f &chunkCenter, unsigned char lod, unsigned int lodFlags, bool compile)
 {
-    osg::Vec2f worldCenter = chunkCenter*mStorage->getCellWorldSize();
-    osg::ref_ptr<SceneUtil::PositionAttitudeTransform> transform (new SceneUtil::PositionAttitudeTransform);
-    transform->setPosition(osg::Vec3f(worldCenter.x(), worldCenter.y(), 0.f));
-
     osg::ref_ptr<osg::Vec3Array> positions (new osg::Vec3Array);
     osg::ref_ptr<osg::Vec3Array> normals (new osg::Vec3Array);
     osg::ref_ptr<osg::Vec4ubArray> colors (new osg::Vec4ubArray);
@@ -201,6 +201,8 @@ osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Ve
 
     geometry->createClusterCullingCallback();
 
+    geometry->setStateSet(mMultiPassRoot);
+
     if (useCompositeMap)
     {
         osg::ref_ptr<CompositeMap> compositeMap = new CompositeMap;
@@ -224,16 +226,15 @@ osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Ve
         geometry->setPasses(createPasses(chunkSize, chunkCenter, false));
     }
 
-    transform->addChild(geometry);
-    transform->getBound();
-
     geometry->setupWaterBoundingBox(-1, chunkSize * mStorage->getCellWorldSize() / numVerts);
 
-    if (mSceneManager->getIncrementalCompileOperation())
+    if (compile && mSceneManager->getIncrementalCompileOperation())
     {
         mSceneManager->getIncrementalCompileOperation()->add(geometry);
     }
-    return transform;
+    geometry->setNodeMask(mNodeMask);
+
+    return geometry;
 }
 
 }

components/terrain/chunkmanager.hpp

@@ -6,6 +6,7 @@
 #include <components/resource/resourcemanager.hpp>
 
 #include "buffercache.hpp"
+#include "quadtreeworld.hpp"
 
 namespace osg
 {
@@ -29,17 +30,20 @@ namespace Terrain
     typedef std::tuple<osg::Vec2f, unsigned char, unsigned int> ChunkId; // Center, Lod, Lod Flags
 
     /// @brief Handles loading and caching of terrain chunks
-    class ChunkManager : public Resource::GenericResourceManager<ChunkId>
+    class ChunkManager : public Resource::GenericResourceManager<ChunkId>, public QuadTreeWorld::ChunkManager
     {
     public:
         ChunkManager(Storage* storage, Resource::SceneManager* sceneMgr, TextureManager* textureManager, CompositeMapRenderer* renderer);
 
-        osg::ref_ptr<osg::Node> getChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags);
+        osg::ref_ptr<osg::Node> getChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool far, const osg::Vec3f& viewPoint, bool compile);
 
         void setCompositeMapSize(unsigned int size) { mCompositeMapSize = size; }
         void setCompositeMapLevel(float level) { mCompositeMapLevel = level; }
         void setMaxCompositeGeometrySize(float maxCompGeometrySize) { mMaxCompGeometrySize = maxCompGeometrySize; }
 
+        void setNodeMask(unsigned int mask) { mNodeMask = mask; }
+        virtual unsigned int getNodeMask() override { return mNodeMask; }
+
         void reportStats(unsigned int frameNumber, osg::Stats* stats) const override;
 
         void clearCache() override;
@@ -47,7 +51,7 @@ namespace Terrain
         void releaseGLObjects(osg::State* state) override;
 
     private:
-        osg::ref_ptr<osg::Node> createChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags);
+        osg::ref_ptr<osg::Node> createChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool compile);
 
         osg::ref_ptr<osg::Texture2D> createCompositeMapRTT();
 
@@ -61,6 +65,10 @@ namespace Terrain
         CompositeMapRenderer* mCompositeMapRenderer;
         BufferCache mBufferCache;
 
+        osg::ref_ptr<osg::StateSet> mMultiPassRoot;
+
+        unsigned int mNodeMask;
+
         unsigned int mCompositeMapSize;
         float mCompositeMapLevel;
         float mMaxCompGeometrySize;

components/terrain/material.cpp

@@ -183,8 +183,10 @@ namespace Terrain
 
             osg::ref_ptr<osg::StateSet> stateset (new osg::StateSet);
 
+            if (!blendmaps.empty())
+            {
                 stateset->setMode(GL_BLEND, osg::StateAttribute::ON);
-
+                stateset->setRenderBinDetails(passIndex++, "RenderBin");
                 if (!firstLayer)
                 {
                     stateset->setAttributeAndModes(BlendFunc::value(), osg::StateAttribute::ON);
@@ -195,6 +197,7 @@ namespace Terrain
                     stateset->setAttributeAndModes(BlendFuncFirst::value(), osg::StateAttribute::ON);
                     stateset->setAttributeAndModes(LequalDepth::value(), osg::StateAttribute::ON);
                 }
+            }
 
             int texunit = 0;
 
@@ -268,8 +271,6 @@ namespace Terrain
 
             }
 
-            stateset->setRenderBinDetails(passIndex++, "RenderBin");
-
             passes.push_back(stateset);
         }
         return passes;

components/terrain/quadtreenode.cpp

@@ -108,71 +108,26 @@ void QuadTreeNode::initNeighbours()
         getChild(i)->initNeighbours();
 }
 
-void QuadTreeNode::traverseNodes(ViewData* vd, const osg::Vec3f& viewPoint, LodCallback* lodCallback, float maxDist)
+void QuadTreeNode::traverseNodes(ViewData* vd, const osg::Vec3f& viewPoint, LodCallback* lodCallback)
 {
     if (!hasValidBounds())
         return;
-
-    float dist = distance(viewPoint);
-    if (dist > maxDist)
+    LodCallback::ReturnValue lodResult = lodCallback->isSufficientDetail(this, distance(viewPoint));
+    if (lodResult == LodCallback::StopTraversal)
         return;
-
-    bool stopTraversal = (lodCallback->isSufficientDetail(this, dist)) || !getNumChildren();
-
-    if (stopTraversal)
-        vd->add(this);
-    else
+    else if (lodResult == LodCallback::Deeper && getNumChildren())
     {
         for (unsigned int i=0; i<getNumChildren(); ++i)
-            getChild(i)->traverseNodes(vd, viewPoint, lodCallback, maxDist);
+            getChild(i)->traverseNodes(vd, viewPoint, lodCallback);
     }
-}
-
-void QuadTreeNode::traverseTo(ViewData* vd, float size, const osg::Vec2f& center)
-{
-    if (!hasValidBounds())
-        return;
-
-    if (getCenter().x() + getSize()/2.f <= center.x() - size/2.f
-            || getCenter().x() - getSize()/2.f >= center.x() + size/2.f
-            || getCenter().y() + getSize()/2.f <= center.y() - size/2.f
-            || getCenter().y() - getSize()/2.f >= center.y() + size/2.f)
-        return;
-
-    bool stopTraversal = (getSize() == size);
-
-    if (stopTraversal)
-        vd->add(this);
     else
-    {
-        for (unsigned int i=0; i<getNumChildren(); ++i)
-            getChild(i)->traverseTo(vd, size, center);
-    }
-}
-
-void QuadTreeNode::intersect(ViewData* vd, TerrainLineIntersector& intersector)
-{
-    if (!hasValidBounds())
-        return;
-
-    if (!intersector.intersectAndClip(getBoundingBox()))
-        return;
-
-    if (getNumChildren() == 0)
         vd->add(this);
-    else
-    {
-        for (unsigned int i=0; i<getNumChildren(); ++i)
-            getChild(i)->intersect(vd, intersector);
-    }
 }
 
 void QuadTreeNode::setBoundingBox(const osg::BoundingBox &boundingBox)
 {
     mBoundingBox = boundingBox;
     mValidBounds = boundingBox.valid();
-    dirtyBound();
-    getBound();
 }
 
 const osg::BoundingBox &QuadTreeNode::getBoundingBox() const
@@ -180,11 +135,6 @@ const osg::BoundingBox &QuadTreeNode::getBoundingBox() const
     return mBoundingBox;
 }
 
-osg::BoundingSphere QuadTreeNode::computeBound() const
-{
-    return osg::BoundingSphere(mBoundingBox);
-}
-
 float QuadTreeNode::getSize() const
 {
     return mSize;

components/terrain/quadtreenode.hpp

@@ -2,39 +2,12 @@
 #define OPENMW_COMPONENTS_TERRAIN_QUADTREENODE_H
 
 #include <osg/Group>
-#include <osgUtil/LineSegmentIntersector>
 
 #include "defs.hpp"
 
 namespace Terrain
 {
 
-    class TerrainLineIntersector : public osgUtil::LineSegmentIntersector
-    {
-    public:
-        TerrainLineIntersector(osgUtil::LineSegmentIntersector* intersector, osg::Matrix& matrix) :
-            osgUtil::LineSegmentIntersector(intersector->getStart() * matrix, intersector->getEnd() * matrix)
-        {
-            setPrecisionHint(intersector->getPrecisionHint());
-            _intersectionLimit = intersector->getIntersectionLimit();
-            _parent = intersector;
-        }
-
-        TerrainLineIntersector(osgUtil::LineSegmentIntersector* intersector) :
-            osgUtil::LineSegmentIntersector(intersector->getStart(), intersector->getEnd())
-        {
-            setPrecisionHint(intersector->getPrecisionHint());
-            _intersectionLimit = intersector->getIntersectionLimit();
-            _parent = intersector;
-        }
-
-        bool intersectAndClip(const osg::BoundingBox& bbInput)
-        {
-            osg::Vec3d s(_start), e(_end);
-            return osgUtil::LineSegmentIntersector::intersectAndClip(s, e, bbInput);
-        }
-    };
-
     enum ChildDirection
     {
         NW = 0,
@@ -50,10 +23,15 @@ namespace Terrain
     public:
         virtual ~LodCallback() {}
 
-        virtual bool isSufficientDetail(QuadTreeNode *node, float dist) = 0;
+        enum ReturnValue
+        {
+            Deeper,
+            StopTraversal,
+            StopTraversalAndUse
+        };
+        virtual ReturnValue isSufficientDetail(QuadTreeNode *node, float dist) = 0;
     };
 
-    class ViewDataMap;
     class ViewData;
 
     class QuadTreeNode : public osg::Group
@@ -91,8 +69,6 @@ namespace Terrain
         const osg::BoundingBox& getBoundingBox() const;
         bool hasValidBounds() const { return mValidBounds; }
 
-        virtual osg::BoundingSphere computeBound() const;
-
         /// size in cell coordinates
         float getSize() const;
 
@@ -100,13 +76,7 @@ namespace Terrain
         const osg::Vec2f& getCenter() const;
 
         /// Traverse nodes according to LOD selection.
-        void traverseNodes(ViewData* vd, const osg::Vec3f& viewPoint, LodCallback* lodCallback, float maxDist);
-
-        /// Traverse to a specific node and add only that node.
-        void traverseTo(ViewData* vd, float size, const osg::Vec2f& center);
-
-        /// Adds all leaf nodes which intersect the line from start to end
-        void intersect(ViewData* vd, TerrainLineIntersector& intersector);
+        void traverseNodes(ViewData* vd, const osg::Vec3f& viewPoint, LodCallback* lodCallback);
 
     private:
         QuadTreeNode* mParent;

components/terrain/quadtreeworld.cpp

@@ -9,6 +9,7 @@
 
 #include <components/misc/constants.hpp>
 #include <components/sceneutil/mwshadowtechnique.hpp>
+#include <components/sceneutil/positionattitudetransform.hpp>
 
 #include "quadtreenode.hpp"
 #include "storage.hpp"
@@ -52,25 +53,45 @@ namespace Terrain
 class DefaultLodCallback : public LodCallback
 {
 public:
-    DefaultLodCallback(float factor, float minSize)
+    DefaultLodCallback(float factor, float minSize, float viewDistance, const osg::Vec4i& grid)
         : mFactor(factor)
         , mMinSize(minSize)
+        , mViewDistance(viewDistance)
+        , mActiveGrid(grid)
     {
     }
 
-    virtual bool isSufficientDetail(QuadTreeNode* node, float dist)
+    virtual ReturnValue isSufficientDetail(QuadTreeNode* node, float dist)
     {
+        const osg::Vec2f& center = node->getCenter();
+        bool activeGrid = (center.x() > mActiveGrid.x() && center.y() > mActiveGrid.y() && center.x() < mActiveGrid.z() && center.y() < mActiveGrid.w());
+        if (dist > mViewDistance && !activeGrid) // for Scene<->ObjectPaging sync the activegrid must remain loaded
+            return StopTraversal;
+        if (node->getSize()>1)
+        {
+            float halfSize = node->getSize()/2;
+            osg::Vec4i nodeBounds (static_cast<int>(center.x() - halfSize), static_cast<int>(center.y() - halfSize), static_cast<int>(center.x() + halfSize), static_cast<int>(center.y() + halfSize));
+            bool intersects = (std::max(nodeBounds.x(), mActiveGrid.x()) < std::min(nodeBounds.z(), mActiveGrid.z()) && std::max(nodeBounds.y(), mActiveGrid.y()) < std::min(nodeBounds.w(), mActiveGrid.w()));
+            // to prevent making chunks who will cross the activegrid border
+            if (intersects)
+                return Deeper;
+        }
+
         int nativeLodLevel = Log2(static_cast<unsigned int>(node->getSize()/mMinSize));
         int lodLevel = Log2(static_cast<unsigned int>(dist/(Constants::CellSizeInUnits*mMinSize*mFactor)));
 
-        return nativeLodLevel <= lodLevel;
+        return nativeLodLevel <= lodLevel ? StopTraversalAndUse : Deeper;
     }
 
 private:
     float mFactor;
     float mMinSize;
+    float mViewDistance;
+    osg::Vec4i mActiveGrid;
 };
 
+const float MIN_SIZE = 1/8.f;
+
 class RootNode : public QuadTreeNode
 {
 public:
@@ -125,6 +146,8 @@ public:
         addChildren(mRootNode);
 
         mRootNode->initNeighbours();
+        float cellWorldSize = mStorage->getCellWorldSize();
+        mRootNode->setInitialBound(osg::BoundingSphere(osg::BoundingBox(osg::Vec3(mMinX*cellWorldSize, mMinY*cellWorldSize, 0), osg::Vec3(mMaxX*cellWorldSize, mMaxY*cellWorldSize, 0))));
     }
 
     void addChildren(QuadTreeNode* parent)
@@ -231,11 +254,11 @@ QuadTreeWorld::QuadTreeWorld(osg::Group *parent, osg::Group *compileRoot, Resour
     mChunkManager->setCompositeMapSize(compMapResolution);
     mChunkManager->setCompositeMapLevel(compMapLevel);
     mChunkManager->setMaxCompositeGeometrySize(maxCompGeometrySize);
+    mChunkManagers.push_back(mChunkManager.get());
 }
 
 QuadTreeWorld::~QuadTreeWorld()
 {
-    mViewDataMap->clear();
 }
 
 /// get the level of vertex detail to render this node at, expressed relative to the native resolution of the data set.
@@ -261,7 +284,7 @@ unsigned int getVertexLod(QuadTreeNode* node, int vertexLodMod)
 }
 
 /// get the flags to use for stitching in the index buffer so that chunks of different LOD connect seamlessly
-unsigned int getLodFlags(QuadTreeNode* node, int ourLod, int vertexLodMod, ViewData* vd)
+unsigned int getLodFlags(QuadTreeNode* node, int ourLod, int vertexLodMod, const ViewData* vd)
 {
     unsigned int lodFlags = 0;
     for (unsigned int i=0; i<4; ++i)
@@ -289,7 +312,7 @@ unsigned int getLodFlags(QuadTreeNode* node, int ourLod, int vertexLodMod, ViewD
     return lodFlags;
 }
 
-void loadRenderingNode(ViewData::Entry& entry, ViewData* vd, int vertexLodMod, ChunkManager* chunkManager)
+void loadRenderingNode(ViewData::Entry& entry, ViewData* vd, int vertexLodMod, float cellWorldSize, const osg::Vec4i &gridbounds, const std::vector<QuadTreeWorld::ChunkManager*>& chunkManagers, bool compile)
 {
     if (!vd->hasChanged() && entry.mRenderingNode)
         return;
@@ -308,7 +331,20 @@ void loadRenderingNode(ViewData::Entry& entry, ViewData* vd, int vertexLodMod, C
     }
 
     if (!entry.mRenderingNode)
-        entry.mRenderingNode = chunkManager->getChunk(entry.mNode->getSize(), entry.mNode->getCenter(), ourLod, entry.mLodFlags);
+    {
+        osg::ref_ptr<SceneUtil::PositionAttitudeTransform> pat = new SceneUtil::PositionAttitudeTransform;
+        pat->setPosition(osg::Vec3f(entry.mNode->getCenter().x()*cellWorldSize, entry.mNode->getCenter().y()*cellWorldSize, 0.f));
+
+        const osg::Vec2f& center = entry.mNode->getCenter();
+        bool activeGrid = (center.x() > gridbounds.x() && center.y() > gridbounds.y() && center.x() < gridbounds.z() && center.y() < gridbounds.w());
+
+        for (QuadTreeWorld::ChunkManager* m : chunkManagers)
+        {
+            osg::ref_ptr<osg::Node> n = m->getChunk(entry.mNode->getSize(), entry.mNode->getCenter(), ourLod, entry.mLodFlags, activeGrid, vd->getViewPoint(), compile);
+            if (n) pat->addChild(n);
+        }
+        entry.mRenderingNode = pat;
+    }
 }
 
 void updateWaterCullingView(HeightCullCallback* callback, ViewData* vd, osgUtil::CullVisitor* cv, float cellworldsize, bool outofworld)
@@ -382,71 +418,29 @@ void QuadTreeWorld::accept(osg::NodeVisitor &nv)
         return;
     }
 
+    osg::Object * viewer = isCullVisitor ? static_cast<osgUtil::CullVisitor*>(&nv)->getCurrentCamera() : nullptr;
     bool needsUpdate = true;
-    ViewData* vd = nullptr;
-    if (isCullVisitor)
-        vd = mViewDataMap->getViewData(static_cast<osgUtil::CullVisitor*>(&nv)->getCurrentCamera(), nv.getViewPoint(), needsUpdate);
-    else
-    {
-        static ViewData sIntersectionViewData;
-        vd = &sIntersectionViewData;
-    }
+    ViewData *vd = mViewDataMap->getViewData(viewer, nv.getViewPoint(), mActiveGrid, needsUpdate);
 
     if (needsUpdate)
     {
         vd->reset();
-        if (isCullVisitor)
-        {
-            osgUtil::CullVisitor* cv = static_cast<osgUtil::CullVisitor*>(&nv);
+        DefaultLodCallback lodCallback(mLodFactor, MIN_SIZE, mViewDistance, mActiveGrid);
+        mRootNode->traverseNodes(vd, nv.getViewPoint(), &lodCallback);
+    }
 
-            osg::UserDataContainer* udc = cv->getCurrentCamera()->getUserDataContainer();
-            if (udc && udc->getNumDescriptions() >= 2 && udc->getDescriptions()[0] == "NoTerrainLod")
-            {
-                std::istringstream stream(udc->getDescriptions()[1]);
-                int x,y;
-                stream >> x;
-                stream >> y;
-                mRootNode->traverseTo(vd, 1, osg::Vec2f(x+0.5,y+0.5));
-            }
-            else
-                mRootNode->traverseNodes(vd, cv->getViewPoint(), mLodCallback, mViewDistance);
-        }
-        else
-        {
-            osgUtil::IntersectionVisitor* iv = static_cast<osgUtil::IntersectionVisitor*>(&nv);
-            osgUtil::LineSegmentIntersector* lineIntersector = dynamic_cast<osgUtil::LineSegmentIntersector*>(iv->getIntersector());
-            if (!lineIntersector)
-                throw std::runtime_error("Cannot update QuadTreeWorld: node visitor is not LineSegmentIntersector");
-
-            if (lineIntersector->getCoordinateFrame() == osgUtil::Intersector::CoordinateFrame::MODEL && iv->getModelMatrix() == 0)
-            {
-                TerrainLineIntersector terrainIntersector(lineIntersector);
-                mRootNode->intersect(vd, terrainIntersector);
-            }
-            else
-            {
-                osg::Matrix matrix(lineIntersector->getTransformation(*iv, lineIntersector->getCoordinateFrame()));
-                TerrainLineIntersector terrainIntersector(lineIntersector, matrix);
-                mRootNode->intersect(vd, terrainIntersector);
-            }
-        }
-    }
+    const float cellWorldSize = mStorage->getCellWorldSize();
 
     for (unsigned int i=0; i<vd->getNumEntries(); ++i)
     {
         ViewData::Entry& entry = vd->getEntry(i);
-
-        loadRenderingNode(entry, vd, mVertexLodMod, mChunkManager.get());
-
+        loadRenderingNode(entry, vd, mVertexLodMod, cellWorldSize, mActiveGrid, mChunkManagers, false);
         entry.mRenderingNode->accept(nv);
     }
 
     if (isCullVisitor)
         updateWaterCullingView(mHeightCullCallback, vd, static_cast<osgUtil::CullVisitor*>(&nv), mStorage->getCellWorldSize(), !isGridEmpty());
 
-    if (!isCullVisitor)
-        vd->clear(); // we can't reuse intersection views in the next frame because they only contain what is touched by the intersection ray.
-
     vd->markUnchanged();
 
     double referenceTime = nv.getFrameStamp() ? nv.getFrameStamp()->getReferenceTime() : 0.0;
@@ -463,9 +457,7 @@ void QuadTreeWorld::ensureQuadTreeBuilt()
     if (mQuadTreeBuilt)
         return;
 
-    const float minSize = 1/8.f;
-    mLodCallback = new DefaultLodCallback(mLodFactor, minSize);
-    QuadTreeBuilder builder(mStorage, minSize);
+    QuadTreeBuilder builder(mStorage, MIN_SIZE);
     builder.build();
 
     mRootNode = builder.getRootNode();
@@ -487,48 +479,38 @@ void QuadTreeWorld::enable(bool enabled)
         mRootNode->setNodeMask(enabled ? ~0 : 0);
 }
 
-void QuadTreeWorld::cacheCell(View *view, int x, int y)
-{
-    ensureQuadTreeBuilt();
-    ViewData* vd = static_cast<ViewData*>(view);
-    mRootNode->traverseTo(vd, 1, osg::Vec2f(x+0.5f,y+0.5f));
-
-    for (unsigned int i=0; i<vd->getNumEntries(); ++i)
-    {
-        ViewData::Entry& entry = vd->getEntry(i);
-        loadRenderingNode(entry, vd, mVertexLodMod, mChunkManager.get());
-    }
-}
-
 View* QuadTreeWorld::createView()
 {
-    return new ViewData;
+    return mViewDataMap->createIndependentView();
 }
 
-void QuadTreeWorld::preload(View *view, const osg::Vec3f &viewPoint, std::atomic<bool> &abort)
+void QuadTreeWorld::preload(View *view, const osg::Vec3f &viewPoint, const osg::Vec4i &grid, std::atomic<bool> &abort, std::atomic<int> &progress, int& progressTotal)
 {
     ensureQuadTreeBuilt();
 
     ViewData* vd = static_cast<ViewData*>(view);
     vd->setViewPoint(viewPoint);
-    mRootNode->traverseNodes(vd, viewPoint, mLodCallback, mViewDistance);
+    vd->setActiveGrid(grid);
+    DefaultLodCallback lodCallback(mLodFactor, MIN_SIZE, mViewDistance, grid);
+    mRootNode->traverseNodes(vd, viewPoint, &lodCallback);
 
+    if (!progressTotal)
+        for (unsigned int i=0; i<vd->getNumEntries(); ++i)
+            progressTotal += vd->getEntry(i).mNode->getSize();
+
+    const float cellWorldSize = mStorage->getCellWorldSize();
     for (unsigned int i=0; i<vd->getNumEntries() && !abort; ++i)
     {
         ViewData::Entry& entry = vd->getEntry(i);
-        loadRenderingNode(entry, vd, mVertexLodMod, mChunkManager.get());
+        loadRenderingNode(entry, vd, mVertexLodMod, cellWorldSize, grid, mChunkManagers, true);
+        progress += entry.mNode->getSize();
     }
     vd->markUnchanged();
 }
 
-void QuadTreeWorld::storeView(const View* view, double referenceTime)
+bool QuadTreeWorld::storeView(const View* view, double referenceTime)
 {
-    osg::ref_ptr<osg::Object> dummy = new osg::DummyObject;
-    const ViewData* vd = static_cast<const ViewData*>(view);
-    bool needsUpdate = false;
-    ViewData* stored = mViewDataMap->getViewData(dummy, vd->getViewPoint(), needsUpdate);
-    stored->copyFrom(*vd);
-    stored->setLastUsageTimeStamp(referenceTime);
+    return mViewDataMap->storeView(static_cast<const ViewData*>(view), referenceTime);
 }
 
 void QuadTreeWorld::reportStats(unsigned int frameNumber, osg::Stats *stats)
@@ -556,5 +538,15 @@ void QuadTreeWorld::unloadCell(int x, int y)
         World::unloadCell(x,y);
 }
 
+void QuadTreeWorld::addChunkManager(QuadTreeWorld::ChunkManager* m)
+{
+    mChunkManagers.push_back(m);
+    mTerrainRoot->setNodeMask(mTerrainRoot->getNodeMask()|m->getNodeMask());
+}
+
+void QuadTreeWorld::rebuildViews()
+{
+    mViewDataMap->rebuildViews();
+}
 
 }

components/terrain/quadtreeworld.hpp

@@ -15,7 +15,6 @@ namespace Terrain
 {
     class RootNode;
     class ViewDataMap;
-    class LodCallback;
 
     /// @brief Terrain implementation that loads cells into a Quad Tree, with geometry LOD and texture LOD.
     class QuadTreeWorld : public TerrainGrid // note: derived from TerrainGrid is only to render default cells (see loadCell)
@@ -31,25 +30,36 @@ namespace Terrain
 
         virtual void setViewDistance(float distance) { mViewDistance = distance; }
 
-        void cacheCell(View *view, int x, int y);
+        void cacheCell(View *view, int x, int y) {}
         /// @note Not thread safe.
         virtual void loadCell(int x, int y);
         /// @note Not thread safe.
         virtual void unloadCell(int x, int y);
 
         View* createView();
-        void preload(View* view, const osg::Vec3f& eyePoint, std::atomic<bool>& abort);
-        void storeView(const View* view, double referenceTime);
+        void preload(View* view, const osg::Vec3f& eyePoint, const osg::Vec4i &cellgrid, std::atomic<bool>& abort, std::atomic<int>& progress, int& progressRange);
+        bool storeView(const View* view, double referenceTime);
+        void rebuildViews() override;
 
         void reportStats(unsigned int frameNumber, osg::Stats* stats);
 
+        class ChunkManager
+        {
+        public:
+            virtual ~ChunkManager(){}
+            virtual osg::ref_ptr<osg::Node> getChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool far, const osg::Vec3f& viewPoint, bool compile) = 0;
+            virtual unsigned int getNodeMask() { return 0; }
+        };
+        void addChunkManager(ChunkManager*);
+
     private:
         void ensureQuadTreeBuilt();
 
         osg::ref_ptr<RootNode> mRootNode;
 
         osg::ref_ptr<ViewDataMap> mViewDataMap;
-        osg::ref_ptr<LodCallback> mLodCallback;
+
+        std::vector<ChunkManager*> mChunkManagers;
 
         OpenThreads::Mutex mQuadTreeMutex;
         bool mQuadTreeBuilt;

components/terrain/terraindrawable.cpp

@@ -102,6 +102,10 @@ void TerrainDrawable::cull(osgUtil::CullVisitor *cv)
 
     bool pushedLight = mLightListCallback && mLightListCallback->pushLightState(this, cv);
 
+    osg::StateSet* stateset = getStateSet();
+    if (stateset)
+        cv->pushStateSet(stateset);
+
     for (PassVector::const_iterator it = mPasses.begin(); it != mPasses.end(); ++it)
     {
         cv->pushStateSet(*it);
@@ -109,6 +113,8 @@ void TerrainDrawable::cull(osgUtil::CullVisitor *cv)
         cv->popStateSet();
     }
 
+    if (stateset)
+        cv->popStateSet();
     if (pushedLight)
         cv->popStateSet();
 }

components/terrain/terraingrid.cpp

@@ -5,9 +5,10 @@
 #include <osg/Group>
 #include <osg/ComputeBoundsVisitor>
 
+#include <components/sceneutil/positionattitudetransform.hpp>
 #include "chunkmanager.hpp"
 #include "compositemaprenderer.hpp"
-
+#include "storage.hpp"
 namespace Terrain
 {
 
@@ -57,12 +58,17 @@ osg::ref_ptr<osg::Node> TerrainGrid::buildTerrain (osg::Group* parent, float chu
     }
     else
     {
-        osg::ref_ptr<osg::Node> node = mChunkManager->getChunk(chunkSize, chunkCenter, 0, 0);
+        osg::ref_ptr<osg::Node> node = mChunkManager->getChunk(chunkSize, chunkCenter, 0, 0, false, osg::Vec3f(), true);
         if (!node)
             return nullptr;
+
+        const float cellWorldSize = mStorage->getCellWorldSize();
+        osg::ref_ptr<SceneUtil::PositionAttitudeTransform> pat = new SceneUtil::PositionAttitudeTransform;
+        pat->setPosition(osg::Vec3f(chunkCenter.x()*cellWorldSize, chunkCenter.y()*cellWorldSize, 0.f));
+        pat->addChild(node);
         if (parent)
-            parent->addChild(node);
-        return node;
+            parent->addChild(pat);
+        return pat;
     }
 }
 

components/terrain/texturemanager.cpp

@@ -25,7 +25,7 @@ struct UpdateTextureFilteringFunctor
     }
     Resource::SceneManager* mSceneManager;
 
-    void operator()(osg::Object* obj)
+    void operator()(std::string, osg::Object* obj)
     {
         mSceneManager->applyFilterSettings(static_cast<osg::Texture2D*>(obj));
     }

components/terrain/viewdata.cpp

@@ -1,5 +1,7 @@
 #include "viewdata.hpp"
 
+#include "quadtreenode.hpp"
+
 namespace Terrain
 {
 
@@ -8,6 +10,7 @@ ViewData::ViewData()
     , mLastUsageTimeStamp(0.0)
     , mChanged(false)
     , mHasViewPoint(false)
+    , mWorldUpdateRevision(0)
 {
 
 }
@@ -24,6 +27,8 @@ void ViewData::copyFrom(const ViewData& other)
     mChanged = other.mChanged;
     mHasViewPoint = other.mHasViewPoint;
     mViewPoint = other.mViewPoint;
+    mActiveGrid = other.mActiveGrid;
+    mWorldUpdateRevision = other.mWorldUpdateRevision;
 }
 
 void ViewData::add(QuadTreeNode *node)
@@ -90,7 +95,12 @@ void ViewData::clear()
     mHasViewPoint = false;
 }
 
-bool ViewData::contains(QuadTreeNode *node)
+bool ViewData::suitableToUse(const osg::Vec4i &activeGrid) const
+{
+    return hasViewPoint() && activeGrid == mActiveGrid && getNumEntries();
+}
+
+bool ViewData::contains(QuadTreeNode *node) const
 {
     for (unsigned int i=0; i<mNumEntries; ++i)
         if (mEntries[i].mNode == node)
@@ -118,79 +128,110 @@ bool ViewData::Entry::set(QuadTreeNode *node)
     }
 }
 
-bool suitable(ViewData* vd, const osg::Vec3f& viewPoint, float& maxDist)
+ViewData *ViewDataMap::getViewData(osg::Object *viewer, const osg::Vec3f& viewPoint, const osg::Vec4i &activeGrid, bool& needsUpdate)
 {
-    return vd->hasViewPoint() && (vd->getViewPoint() - viewPoint).length2() < maxDist*maxDist;
-}
-
-ViewData *ViewDataMap::getViewData(osg::Object *viewer, const osg::Vec3f& viewPoint, bool& needsUpdate)
-{
-    Map::const_iterator found = mViews.find(viewer);
+    ViewerMap::const_iterator found = mViewers.find(viewer);
     ViewData* vd = nullptr;
-    if (found == mViews.end())
+    if (found == mViewers.end())
     {
         vd = createOrReuseView();
-        mViews[viewer] = vd;
+        mViewers[viewer] = vd;
     }
     else
         vd = found->second;
-
-    if (!suitable(vd, viewPoint, mReuseDistance))
-    {
-        for (Map::const_iterator other = mViews.begin(); other != mViews.end(); ++other)
-        {
-            if (suitable(other->second, viewPoint, mReuseDistance) && other->second->getNumEntries())
-            {
-                vd->copyFrom(*other->second);
     needsUpdate = false;
+
+    if (!(vd->suitableToUse(activeGrid) && (vd->getViewPoint()-viewPoint).length2() < mReuseDistance*mReuseDistance && vd->getWorldUpdateRevision() >= mWorldUpdateRevision))
+    {
+        float shortestDist = std::numeric_limits<float>::max();
+        const ViewData* mostSuitableView = nullptr;
+        for (const ViewData* other : mUsedViews)
+        {
+            if (other->suitableToUse(activeGrid) && other->getWorldUpdateRevision() >= mWorldUpdateRevision)
+            {
+                float dist = (viewPoint-other->getViewPoint()).length2();
+                if (dist < shortestDist)
+                {
+                    shortestDist = dist;
+                    mostSuitableView = other;
+                }
+            }
+        }
+        if (mostSuitableView && mostSuitableView != vd)
+        {
+            vd->copyFrom(*mostSuitableView);
             return vd;
         }
     }
+    if (!vd->suitableToUse(activeGrid))
+    {
         vd->setViewPoint(viewPoint);
+        vd->setActiveGrid(activeGrid);
         needsUpdate = true;
     }
-    else
-        needsUpdate = false;
-
     return vd;
 }
 
+bool ViewDataMap::storeView(const ViewData* view, double referenceTime)
+{
+    if (view->getWorldUpdateRevision() < mWorldUpdateRevision)
+        return false;
+    ViewData* store = createOrReuseView();
+    store->copyFrom(*view);
+    store->setLastUsageTimeStamp(referenceTime);
+    return true;
+}
+
 ViewData *ViewDataMap::createOrReuseView()
 {
+    ViewData* vd = nullptr;
     if (mUnusedViews.size())
     {
-        ViewData* vd = mUnusedViews.front();
+        vd = mUnusedViews.front();
         mUnusedViews.pop_front();
-        return vd;
     }
     else
     {
         mViewVector.push_back(ViewData());
-        return &mViewVector.back();
+        vd = &mViewVector.back();
     }
+    mUsedViews.push_back(vd);
+    vd->setWorldUpdateRevision(mWorldUpdateRevision);
+    return vd;
+}
+
+ViewData *ViewDataMap::createIndependentView() const
+{
+    ViewData* vd = new ViewData;
+    vd->setWorldUpdateRevision(mWorldUpdateRevision);
+    return vd;
 }
 
 void ViewDataMap::clearUnusedViews(double referenceTime)
 {
-    for (Map::iterator it = mViews.begin(); it != mViews.end(); )
+    for (ViewerMap::iterator it = mViewers.begin(); it != mViewers.end(); )
     {
-        ViewData* vd = it->second;
-        if (vd->getLastUsageTimeStamp() + mExpiryDelay < referenceTime)
+        if (it->second->getLastUsageTimeStamp() + mExpiryDelay < referenceTime)
+            mViewers.erase(it++);
+        else
+            ++it;
+    }
+    for (std::deque<ViewData*>::iterator it = mUsedViews.begin(); it != mUsedViews.end(); )
     {
-            vd->clear();
-            mUnusedViews.push_back(vd);
-            mViews.erase(it++);
+        if ((*it)->getLastUsageTimeStamp() + mExpiryDelay < referenceTime)
+        {
+            (*it)->clear();
+            mUnusedViews.push_back(*it);
+            it = mUsedViews.erase(it);
         }
         else
             ++it;
     }
 }
 
-void ViewDataMap::clear()
+void ViewDataMap::rebuildViews()
 {
-    mViews.clear();
-    mUnusedViews.clear();
-    mViewVector.clear();
+    ++mWorldUpdateRevision;
 }
 
 }

components/terrain/viewdata.hpp

@@ -23,9 +23,11 @@ namespace Terrain
 
         void reset();
 
+        bool suitableToUse(const osg::Vec4i& activeGrid) const;
+
         void clear();
 
-        bool contains(QuadTreeNode* node);
+        bool contains(QuadTreeNode* node) const;
 
         void copyFrom(const ViewData& other);
 
@@ -57,6 +59,12 @@ namespace Terrain
         void setViewPoint(const osg::Vec3f& viewPoint);
         const osg::Vec3f& getViewPoint() const;
 
+        void setActiveGrid(const osg::Vec4i &grid) { if (grid != mActiveGrid) {mActiveGrid = grid;mEntries.clear();mNumEntries=0;} }
+        const osg::Vec4i &getActiveGrid() const { return mActiveGrid;}
+
+        unsigned int getWorldUpdateRevision() const { return mWorldUpdateRevision; }
+        void setWorldUpdateRevision(int updateRevision) { mWorldUpdateRevision = updateRevision; }
+
     private:
         std::vector<Entry> mEntries;
         unsigned int mNumEntries;
@@ -64,34 +72,41 @@ namespace Terrain
         bool mChanged;
         osg::Vec3f mViewPoint;
         bool mHasViewPoint;
+        osg::Vec4i mActiveGrid;
+        unsigned int mWorldUpdateRevision;
     };
 
     class ViewDataMap : public osg::Referenced
     {
     public:
         ViewDataMap()
-            : mReuseDistance(300) // large value should be safe because the visibility of each node is still updated individually for each camera even if the base view was reused.
+            : mReuseDistance(150) // large value should be safe because the visibility of each node is still updated individually for each camera even if the base view was reused.
                                   // this value also serves as a threshold for when a newly loaded LOD gets unloaded again so that if you hover around an LOD transition point the LODs won't keep loading and unloading all the time.
             , mExpiryDelay(1.f)
+            , mWorldUpdateRevision(0)
         {}
 
-        ViewData* getViewData(osg::Object* viewer, const osg::Vec3f& viewPoint, bool& needsUpdate);
+        ViewData* getViewData(osg::Object* viewer, const osg::Vec3f& viewPoint, const osg::Vec4i &activeGrid, bool& needsUpdate);
 
         ViewData* createOrReuseView();
+        ViewData* createIndependentView() const;
 
         void clearUnusedViews(double referenceTime);
-
-        void clear();
+        void rebuildViews();
+        bool storeView(const ViewData* view, double referenceTime);
 
     private:
         std::list<ViewData> mViewVector;
 
-        typedef std::map<osg::ref_ptr<osg::Object>, ViewData*> Map;
-        Map mViews;
+        typedef std::map<osg::ref_ptr<osg::Object>, ViewData*> ViewerMap;
+        ViewerMap mViewers;
 
         float mReuseDistance;
         float mExpiryDelay; // time in seconds for unused view to be removed
 
+        unsigned int mWorldUpdateRevision;
+
+        std::deque<ViewData*> mUsedViews;
         std::deque<ViewData*> mUnusedViews;
     };
 

components/terrain/world.cpp

@@ -1,7 +1,6 @@
 #include "world.hpp"
 
 #include <osg/Group>
-#include <osg/Material>
 #include <osg/Camera>
 
 #include <components/resource/resourcesystem.hpp>
@@ -23,11 +22,6 @@ World::World(osg::Group* parent, osg::Group* compileRoot, Resource::ResourceSyst
 {
     mTerrainRoot = new osg::Group;
     mTerrainRoot->setNodeMask(nodeMask);
-    mTerrainRoot->getOrCreateStateSet()->setRenderingHint(osg::StateSet::OPAQUE_BIN);
-    osg::ref_ptr<osg::Material> material (new osg::Material);
-    material->setColorMode(osg::Material::AMBIENT_AND_DIFFUSE);
-    mTerrainRoot->getOrCreateStateSet()->setAttributeAndModes(material, osg::StateAttribute::ON);
-
     mTerrainRoot->setName("Terrain Root");
 
     osg::ref_ptr<osg::Camera> compositeCam = new osg::Camera;
@@ -48,6 +42,7 @@ World::World(osg::Group* parent, osg::Group* compileRoot, Resource::ResourceSyst
 
     mTextureManager.reset(new TextureManager(mResourceSystem->getSceneManager()));
     mChunkManager.reset(new ChunkManager(mStorage, mResourceSystem->getSceneManager(), mTextureManager.get(), mCompositeMapRenderer));
+    mChunkManager->setNodeMask(nodeMask);
     mCellBorder.reset(new CellBorder(this,mTerrainRoot.get(),borderMask));
 
     mResourceSystem->addResourceManager(mChunkManager.get());

components/terrain/world.hpp

@@ -147,11 +147,13 @@ namespace Terrain
 
         /// @note Thread safe, as long as you do not attempt to load into the same view from multiple threads.
 
-        virtual void preload(View* view, const osg::Vec3f& viewPoint, std::atomic<bool>& abort) {}
+        virtual void preload(View* view, const osg::Vec3f& viewPoint, const osg::Vec4i &cellgrid, std::atomic<bool>& abort, std::atomic<int>& progress, int& progressRange) {}
 
         /// Store a preloaded view into the cache with the intent that the next rendering traversal can use it.
         /// @note Not thread safe.
-        virtual void storeView(const View* view, double referenceTime) {}
+        virtual bool storeView(const View* view, double referenceTime) {return true;}
+
+        virtual void rebuildViews() {}
 
         virtual void reportStats(unsigned int frameNumber, osg::Stats* stats) {}
 
@@ -161,6 +163,8 @@ namespace Terrain
 
         osg::Callback* getHeightCullCallback(float highz, unsigned int mask);
 
+        void setActiveGrid(const osg::Vec4i &grid) { mActiveGrid = grid; }
+
     protected:
         Storage* mStorage;
 
@@ -181,6 +185,8 @@ namespace Terrain
 
         std::set<std::pair<int,int>> mLoadedCells;
         osg::ref_ptr<HeightCullCallback> mHeightCullCallback;
+
+        osg::Vec4i mActiveGrid;
     };
 }
 

files/settings-default.cfg

@@ -106,6 +106,27 @@ composite map resolution = 512
 # Controls the maximum size of composite geometry, should be >= 1.0. With low values there will be many small chunks, with high values - lesser count of bigger chunks.
 max composite geometry size = 4.0
 
+# Use object paging for non active cells
+object paging = true
+
+# Use object paging for active cells grid
+object paging active grid = true
+
+# Affects the likelyhood of objects being merged. A higher value means merging is more likely and may improve FPS at the cost of memory.
+object paging merge factor = 250
+
+# Cull objects smaller than this size divided by distance
+object paging min size = 0.01
+
+# Adjusts 'min size' based on merging decision. Allows inexpensive objects to be rendered from a greater distance.
+object paging min size merge factor = 0.3
+
+# Controls how inexpensive an object needs to be to utilize 'min size merge factor'.
+object paging min size cost multiplier = 25
+
+# Assign a random color to merged batches.
+object paging debug batches = false
+
 [Fog]
 
 # If true, use extended fog parameters for distant terrain not controlled by
@@ -808,8 +829,8 @@ enable debug hud = false
 # Enable the debug overlay to see where each shadow map affects.
 enable debug overlay = false
 
-# Attempt to better use the shadow map by making them cover a smaller area. May have a minor to major performance impact.
-compute tight scene bounds = true
+# Attempt to better use the shadow map by making them cover a smaller area. May have a major performance impact.
+compute tight scene bounds = false
 
 # How large to make the shadow map(s). Higher values increase GPU load, but can produce better-looking results. Power-of-two values may turn out to be faster on some GPU/driver combinations.
 shadow map resolution = 1024