improves paging preloader (#3126)

* Return check for distance when we try to reuse data

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* cellpreloader.cpp

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* [ci skip]

* quadtreeworld.cpp

* chunkmanager.cpp

* chunkmanager.cpp

* cellpreloader.cpp

* jvoisin

* whitespace

* whitespace

apps/openmw/mwworld/cellpreloader.cpp

@@ -25,7 +25,7 @@ namespace
 {
     template <class Contained>
     bool contains(const std::vector<MWWorld::CellPreloader::PositionCellGrid>& container,
-           const Contained& contained, float tolerance=1.f)
+           const Contained& contained, float tolerance)
     {
         for (const auto& pos : contained)
         {
@@ -180,14 +180,6 @@ namespace MWWorld
         {
         }
 
-        bool storeViews(double referenceTime)
-        {
-            for (unsigned int i=0; i<mTerrainViews.size() && i<mPreloadPositions.size(); ++i)
-                if (!mWorld->storeView(mTerrainViews[i], referenceTime))
-                    return false;
-            return true;
-        }
-
         void doWork() override
         {
             for (unsigned int i=0; i<mTerrainViews.size() && i<mPreloadPositions.size() && !mAbort; ++i)
@@ -246,7 +238,6 @@ namespace MWWorld
         , mMaxCacheSize(0)
         , mPreloadInstances(true)
         , mLastResourceCacheUpdate(0.0)
-        , mStoreViewsFailCount(0)
         , mLoadedTerrainTimestamp(0.0)
     {
     }
@@ -383,22 +374,8 @@ namespace MWWorld
 
         if (mTerrainPreloadItem && mTerrainPreloadItem->isDone())
         {
-            if (!mTerrainPreloadItem->storeViews(timestamp))
+            mLoadedTerrainPositions = mTerrainPreloadPositions;
-            {
+            mLoadedTerrainTimestamp = timestamp;
-                if (++mStoreViewsFailCount > 100)
-                {
-                    OSG_ALWAYS << "paging views are rebuilt every frame, please check for faulty enable/disable scripts." << std::endl;
-                    mStoreViewsFailCount = 0;
-                }
-                setTerrainPreloadPositions(std::vector<PositionCellGrid>());
-            }
-            else
-            {
-                mStoreViewsFailCount = 0;
-                mLoadedTerrainPositions = mTerrainPreloadPositions;
-                mLoadedTerrainTimestamp = timestamp;
-            }
-            mTerrainPreloadItem = nullptr;
         }
     }
 
@@ -443,17 +420,7 @@ namespace MWWorld
             return true;
         else if (mTerrainPreloadItem->isDone())
         {
-            if (mTerrainPreloadItem->storeViews(timestamp))
+            return true;
-            {
-                mTerrainPreloadItem = nullptr;
-                return true;
-            }
-            else
-            {
-                setTerrainPreloadPositions(std::vector<CellPreloader::PositionCellGrid>());
-                setTerrainPreloadPositions(positions);
-                return false;
-            }
         }
         else
         {
@@ -481,7 +448,7 @@ namespace MWWorld
             mTerrainPreloadPositions.clear();
             mLoadedTerrainPositions.clear();
         }
-        else if (contains(mTerrainPreloadPositions, positions))
+        else if (contains(mTerrainPreloadPositions, positions, 128.f))
             return;
         if (mTerrainPreloadItem && !mTerrainPreloadItem->isDone())
             return;

components/terrain/chunkmanager.cpp

@@ -39,6 +39,17 @@ ChunkManager::ChunkManager(Storage *storage, Resource::SceneManager *sceneMgr, T
     mMultiPassRoot->setAttributeAndModes(material, osg::StateAttribute::ON);
 }
 
+struct FindChunkTemplate
+{
+    void operator() (ChunkId id, osg::Object* obj)
+    {
+        if (std::get<0>(id) == std::get<0>(mId) && std::get<1>(id) == std::get<1>(mId))
+            mFoundTemplate = obj;
+    }
+    ChunkId mId;
+    osg::ref_ptr<osg::Object> mFoundTemplate;
+};
+
 osg::ref_ptr<osg::Node> ChunkManager::getChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool activeGrid, const osg::Vec3f& viewPoint, bool compile)
 {
     ChunkId id = std::make_tuple(center, lod, lodFlags);
@@ -47,7 +58,11 @@ osg::ref_ptr<osg::Node> ChunkManager::getChunk(float size, const osg::Vec2f& cen
         return static_cast<osg::Node*>(obj.get());
     else
     {
-        osg::ref_ptr<osg::Node> node = createChunk(size, center, lod, lodFlags, compile);
+        FindChunkTemplate find;
+        find.mId = id;
+        mCache->call(find);
+        TerrainDrawable* templateGeometry = (find.mFoundTemplate && !mDebugChunks) ? static_cast<TerrainDrawable*>(find.mFoundTemplate.get()) : nullptr;
+        osg::ref_ptr<osg::Node> node = createChunk(size, center, lod, lodFlags, compile, templateGeometry);
         mCache->addEntryToObjectCache(id, node.get());
         return node;
     }
@@ -165,24 +180,45 @@ std::vector<osg::ref_ptr<osg::StateSet> > ChunkManager::createPasses(float chunk
     return ::Terrain::createPasses(useShaders, &mSceneManager->getShaderManager(), layers, blendmapTextures, blendmapScale, blendmapScale);
 }
 
-osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Vec2f &chunkCenter, unsigned char lod, unsigned int lodFlags, bool compile)
+osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Vec2f &chunkCenter, unsigned char lod, unsigned int lodFlags, bool compile, TerrainDrawable* templateGeometry)
 {
-    osg::ref_ptr<osg::Vec3Array> positions (new osg::Vec3Array);
-    osg::ref_ptr<osg::Vec3Array> normals (new osg::Vec3Array);
-    osg::ref_ptr<osg::Vec4ubArray> colors (new osg::Vec4ubArray);
-    colors->setNormalize(true);
-
-    osg::ref_ptr<osg::VertexBufferObject> vbo (new osg::VertexBufferObject);
-    positions->setVertexBufferObject(vbo);
-    normals->setVertexBufferObject(vbo);
-    colors->setVertexBufferObject(vbo);
-
-    mStorage->fillVertexBuffers(lod, chunkSize, chunkCenter, positions, normals, colors);
-
     osg::ref_ptr<TerrainDrawable> geometry (new TerrainDrawable);
-    geometry->setVertexArray(positions);
+
-    geometry->setNormalArray(normals, osg::Array::BIND_PER_VERTEX);
+    if (!templateGeometry)
-    geometry->setColorArray(colors, osg::Array::BIND_PER_VERTEX);
+    {
+        osg::ref_ptr<osg::Vec3Array> positions (new osg::Vec3Array);
+        osg::ref_ptr<osg::Vec3Array> normals (new osg::Vec3Array);
+        osg::ref_ptr<osg::Vec4ubArray> colors (new osg::Vec4ubArray);
+        colors->setNormalize(true);
+
+        mStorage->fillVertexBuffers(lod, chunkSize, chunkCenter, positions, normals, colors);
+
+        osg::ref_ptr<osg::VertexBufferObject> vbo (new osg::VertexBufferObject);
+        positions->setVertexBufferObject(vbo);
+        normals->setVertexBufferObject(vbo);
+        colors->setVertexBufferObject(vbo);
+    
+        geometry->setVertexArray(positions);
+        geometry->setNormalArray(normals, osg::Array::BIND_PER_VERTEX);
+        geometry->setColorArray(colors, osg::Array::BIND_PER_VERTEX);
+    }
+    else
+    {
+        // Unfortunately we need to copy vertex data because of poor coupling with VertexBufferObject.
+        osg::ref_ptr<osg::Array> positions = static_cast<osg::Array*>(templateGeometry->getVertexArray()->clone(osg::CopyOp::DEEP_COPY_ALL));
+        osg::ref_ptr<osg::Array> normals = static_cast<osg::Array*>(templateGeometry->getNormalArray()->clone(osg::CopyOp::DEEP_COPY_ALL));
+        osg::ref_ptr<osg::Array> colors = static_cast<osg::Array*>(templateGeometry->getColorArray()->clone(osg::CopyOp::DEEP_COPY_ALL));
+
+        osg::ref_ptr<osg::VertexBufferObject> vbo (new osg::VertexBufferObject);
+        positions->setVertexBufferObject(vbo);
+        normals->setVertexBufferObject(vbo);
+        colors->setVertexBufferObject(vbo);
+    
+        geometry->setVertexArray(positions);
+        geometry->setNormalArray(normals, osg::Array::BIND_PER_VERTEX);
+        geometry->setColorArray(colors, osg::Array::BIND_PER_VERTEX);
+    }
+    
     geometry->setUseDisplayList(false);
     geometry->setUseVertexBufferObjects(true);
 
@@ -202,32 +238,44 @@ osg::ref_ptr<osg::Node> ChunkManager::createChunk(float chunkSize, const osg::Ve
 
     geometry->setStateSet(mMultiPassRoot);
 
-    if (useCompositeMap)
+    if (templateGeometry)
     {
-        osg::ref_ptr<CompositeMap> compositeMap = new CompositeMap;
+        if (templateGeometry->getCompositeMap())
-        compositeMap->mTexture = createCompositeMapRTT();
+        {
-
+            geometry->setCompositeMap(templateGeometry->getCompositeMap());
-        createCompositeMapGeometry(chunkSize, chunkCenter, osg::Vec4f(0,0,1,1), *compositeMap);
+            geometry->setCompositeMapRenderer(mCompositeMapRenderer);
-
+        }
-        mCompositeMapRenderer->addCompositeMap(compositeMap.get(), false);
+        geometry->setPasses(templateGeometry->getPasses());
-
-        geometry->setCompositeMap(compositeMap);
-        geometry->setCompositeMapRenderer(mCompositeMapRenderer);
-
-        TextureLayer layer;
-        layer.mDiffuseMap = compositeMap->mTexture;
-        layer.mParallax = false;
-        layer.mSpecular = false;
-        geometry->setPasses(::Terrain::createPasses(mSceneManager->getForceShaders() || !mSceneManager->getClampLighting(), &mSceneManager->getShaderManager(), std::vector<TextureLayer>(1, layer), std::vector<osg::ref_ptr<osg::Texture2D> >(), 1.f, 1.f));
     }
     else
     {
-        geometry->setPasses(createPasses(chunkSize, chunkCenter, false));
+        if (useCompositeMap)
+        {
+            osg::ref_ptr<CompositeMap> compositeMap = new CompositeMap;
+            compositeMap->mTexture = createCompositeMapRTT();
+
+            createCompositeMapGeometry(chunkSize, chunkCenter, osg::Vec4f(0,0,1,1), *compositeMap);
+
+            mCompositeMapRenderer->addCompositeMap(compositeMap.get(), false);
+
+            geometry->setCompositeMap(compositeMap);
+            geometry->setCompositeMapRenderer(mCompositeMapRenderer);
+
+            TextureLayer layer;
+            layer.mDiffuseMap = compositeMap->mTexture;
+            layer.mParallax = false;
+            layer.mSpecular = false;
+            geometry->setPasses(::Terrain::createPasses(mSceneManager->getForceShaders() || !mSceneManager->getClampLighting(), &mSceneManager->getShaderManager(), std::vector<TextureLayer>(1, layer), std::vector<osg::ref_ptr<osg::Texture2D> >(), 1.f, 1.f));
+        }
+        else
+        {
+            geometry->setPasses(createPasses(chunkSize, chunkCenter, false));
+        }
     }
 
     geometry->setupWaterBoundingBox(-1, chunkSize * mStorage->getCellWorldSize() / numVerts);
 
-    if (compile && mSceneManager->getIncrementalCompileOperation())
+    if (!templateGeometry && compile && mSceneManager->getIncrementalCompileOperation())
     {
         mSceneManager->getIncrementalCompileOperation()->add(geometry);
     }

components/terrain/chunkmanager.hpp

@@ -26,6 +26,7 @@ namespace Terrain
     class CompositeMapRenderer;
     class Storage;
     class CompositeMap;
+    class TerrainDrawable;
 
     typedef std::tuple<osg::Vec2f, unsigned char, unsigned int> ChunkId; // Center, Lod, Lod Flags
 
@@ -51,7 +52,7 @@ namespace Terrain
         void releaseGLObjects(osg::State* state) override;
 
     private:
-        osg::ref_ptr<osg::Node> createChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool compile);
+        osg::ref_ptr<osg::Node> createChunk(float size, const osg::Vec2f& center, unsigned char lod, unsigned int lodFlags, bool compile, TerrainDrawable* templateGeometry);
 
         osg::ref_ptr<osg::Texture2D> createCompositeMapRTT();
 

components/terrain/quadtreeworld.cpp

@@ -55,11 +55,12 @@ namespace Terrain
 class DefaultLodCallback : public LodCallback
 {
 public:
-    DefaultLodCallback(float factor, float minSize, float viewDistance, const osg::Vec4i& grid)
+    DefaultLodCallback(float factor, float minSize, float viewDistance, const osg::Vec4i& grid, float distanceModifier=0.f)
         : mFactor(factor)
         , mMinSize(minSize)
         , mViewDistance(viewDistance)
         , mActiveGrid(grid)
+        , mDistanceModifier(distanceModifier)
     {
     }
 
@@ -78,6 +79,8 @@ public:
                 return Deeper;
         }
 
+        dist = std::max(0.f, dist + mDistanceModifier);
+
         if (dist > mViewDistance && !activeGrid) // for Scene<->ObjectPaging sync the activegrid must remain loaded
             return StopTraversal;
 
@@ -92,6 +95,7 @@ private:
     float mMinSize;
     float mViewDistance;
     osg::Vec4i mActiveGrid;
+    float mDistanceModifier;
 };
 
 class RootNode : public QuadTreeNode
@@ -370,7 +374,7 @@ void loadRenderingNode(ViewData::Entry& entry, ViewData* vd, int vertexLodMod, f
 
         for (QuadTreeWorld::ChunkManager* m : chunkManagers)
         {
-            if (m->getViewDistance() && entry.mNode->distance(vd->getViewPoint()) > m->getViewDistance() + reuseDistance*10)
+            if (m->getViewDistance() && entry.mNode->distance(vd->getViewPoint()) > m->getViewDistance() + reuseDistance)
                 continue;
             osg::ref_ptr<osg::Node> n = m->getChunk(entry.mNode->getSize(), entry.mNode->getCenter(), ourLod, entry.mLodFlags, activeGrid, vd->getViewPoint(), compile);
             if (n) pat->addChild(n);
@@ -519,39 +523,43 @@ View* QuadTreeWorld::createView()
 void QuadTreeWorld::preload(View *view, const osg::Vec3f &viewPoint, const osg::Vec4i &grid, std::atomic<bool> &abort, Loading::Reporter& reporter)
 {
     ensureQuadTreeBuilt();
+    const float cellWorldSize = mStorage->getCellWorldSize();
 
     ViewData* vd = static_cast<ViewData*>(view);
     vd->setViewPoint(viewPoint);
     vd->setActiveGrid(grid);
-    DefaultLodCallback lodCallback(mLodFactor, mMinSize, mViewDistance, grid);
-    mRootNode->traverseNodes(vd, viewPoint, &lodCallback);
 
-    std::size_t progressTotal = 0;
+    for (unsigned int pass=0; pass<3; ++pass)
-    for (unsigned int i = 0, n = vd->getNumEntries(); i < n; ++i)
-        progressTotal += vd->getEntry(i).mNode->getSize();
-
-    reporter.addTotal(progressTotal);
-
-    const float cellWorldSize = mStorage->getCellWorldSize();
-    for (unsigned int i=0; i<vd->getNumEntries() && !abort; ++i)
     {
-        ViewData::Entry& entry = vd->getEntry(i);
+        unsigned int startEntry = vd->getNumEntries();
 
+        float distanceModifier=0.f;
+        if (pass == 1)
+            distanceModifier = 1024;
+        else if (pass == 2)
+            distanceModifier = -1024;
+        DefaultLodCallback lodCallback(mLodFactor, mMinSize, mViewDistance, grid, distanceModifier);
+        mRootNode->traverseNodes(vd, viewPoint, &lodCallback);
 
+        if (pass==0)
+        {
+            std::size_t progressTotal = 0;
+            for (unsigned int i = 0, n = vd->getNumEntries(); i < n; ++i)
+                 progressTotal += vd->getEntry(i).mNode->getSize();
 
-        loadRenderingNode(entry, vd, mVertexLodMod, cellWorldSize, grid, mChunkManagers, true, mViewDataMap->getReuseDistance());
+            reporter.addTotal(progressTotal);
-        reporter.addProgress(entry.mNode->getSize());
+        }
-
-
 
+        const float reuseDistance = std::max(mViewDataMap->getReuseDistance(), std::abs(distanceModifier));
+        for (unsigned int i=startEntry; i<vd->getNumEntries() && !abort; ++i)
+        {
+            ViewData::Entry& entry = vd->getEntry(i);
 
+            loadRenderingNode(entry, vd, mVertexLodMod, cellWorldSize, grid, mChunkManagers, true, reuseDistance);
+            if (pass==0) reporter.addProgress(entry.mNode->getSize());
+            entry.mNode = nullptr; // Clear node lest we break the neighbours search for the next pass
+        }
     }
-    vd->markUnchanged();
-}
-
-bool QuadTreeWorld::storeView(const View* view, double referenceTime)
-{
-    return mViewDataMap->storeView(static_cast<const ViewData*>(view), referenceTime);
 }
 
 void QuadTreeWorld::reportStats(unsigned int frameNumber, osg::Stats *stats)

components/terrain/quadtreeworld.hpp

@@ -40,7 +40,6 @@ namespace Terrain
 
         View* createView() override;
         void preload(View* view, const osg::Vec3f& eyePoint, const osg::Vec4i &cellgrid, std::atomic<bool>& abort, Loading::Reporter& reporter) override;
-        bool storeView(const View* view, double referenceTime) override;
         void rebuildViews() override;
 
         void reportStats(unsigned int frameNumber, osg::Stats* stats) override;

components/terrain/terraindrawable.cpp

@@ -94,10 +94,10 @@ void TerrainDrawable::cull(osgUtil::CullVisitor *cv)
         return;
     }
 
-    if (mCompositeMap)
+    if (mCompositeMap && mCompositeMapRenderer)
     {
         mCompositeMapRenderer->setImmediate(mCompositeMap);
-        mCompositeMap = nullptr;
+        mCompositeMapRenderer = nullptr;
     }
 
     bool pushedLight = mLightListCallback && mLightListCallback->pushLightState(this, cv);

components/terrain/terraindrawable.hpp

@@ -45,6 +45,7 @@ namespace Terrain
 
         typedef std::vector<osg::ref_ptr<osg::StateSet> > PassVector;
         void setPasses (const PassVector& passes);
+        const PassVector& getPasses() const {  return mPasses; }
 
         void setLightListCallback(SceneUtil::LightListCallback* lightListCallback);
 
@@ -56,6 +57,7 @@ namespace Terrain
         const osg::BoundingBox& getWaterBoundingBox() const { return mWaterBoundingBox; }
 
         void setCompositeMap(CompositeMap* map) { mCompositeMap = map; }
+        CompositeMap* getCompositeMap() { return mCompositeMap; }
         void setCompositeMapRenderer(CompositeMapRenderer* renderer) { mCompositeMapRenderer = renderer; }
 
     private:

components/terrain/viewdata.cpp

@@ -143,7 +143,7 @@ ViewData *ViewDataMap::getViewData(osg::Object *viewer, const osg::Vec3f& viewPo
 
     if (!(vd->suitableToUse(activeGrid) && (vd->getViewPoint()-viewPoint).length2() < mReuseDistance*mReuseDistance && vd->getWorldUpdateRevision() >= mWorldUpdateRevision))
     {
-        float shortestDist = std::numeric_limits<float>::max();
+        float shortestDist = mReuseDistance*mReuseDistance;
         const ViewData* mostSuitableView = nullptr;
         for (const ViewData* other : mUsedViews)
         {
@@ -157,31 +157,22 @@ ViewData *ViewDataMap::getViewData(osg::Object *viewer, const osg::Vec3f& viewPo
                 }
             }
         }
-        if (mostSuitableView && mostSuitableView != vd)
+        if (mostSuitableView)
         {
             vd->copyFrom(*mostSuitableView);
             return vd;
         }
-    }
+        else
-    if (!vd->suitableToUse(activeGrid))
+        {
-    {
+            vd->setViewPoint(viewPoint);
-        vd->setViewPoint(viewPoint);
+            vd->setActiveGrid(activeGrid);
-        vd->setActiveGrid(activeGrid);
+            vd->setWorldUpdateRevision(mWorldUpdateRevision);
-        needsUpdate = true;
+            needsUpdate = true;
+        }
     }
     return vd;
 }
 
-bool ViewDataMap::storeView(const ViewData* view, double referenceTime)
-{
-    if (view->getWorldUpdateRevision() < mWorldUpdateRevision)
-        return false;
-    ViewData* store = createOrReuseView();
-    store->copyFrom(*view);
-    store->setLastUsageTimeStamp(referenceTime);
-    return true;
-}
-
 ViewData *ViewDataMap::createOrReuseView()
 {
     ViewData* vd = nullptr;

components/terrain/viewdata.hpp

@@ -93,7 +93,6 @@ namespace Terrain
 
         void clearUnusedViews(double referenceTime);
         void rebuildViews();
-        bool storeView(const ViewData* view, double referenceTime);
 
         float getReuseDistance() const { return mReuseDistance; }
 

components/terrain/world.hpp

@@ -155,10 +155,6 @@ namespace Terrain
 
         virtual void preload(View* view, const osg::Vec3f& viewPoint, const osg::Vec4i &cellgrid, std::atomic<bool>& abort, Loading::Reporter& reporter) {}
 
-        /// Store a preloaded view into the cache with the intent that the next rendering traversal can use it.
-        /// @note Not thread safe.
-        virtual bool storeView(const View* view, double referenceTime) {return true;}
-
         virtual void rebuildViews() {}
 
         virtual void reportStats(unsigned int frameNumber, osg::Stats* stats) {}