Terrain refactoring, reduce game startup time and memory usage

apps/esmtool/record.cpp

@@ -814,7 +814,6 @@ void Record<ESM::Land>::print()
 {
     std::cout << "  Coordinates: (" << mData.mX << "," << mData.mY << ")" << std::endl;
     std::cout << "  Flags: " << landFlags(mData.mFlags) << std::endl;
-    std::cout << "  HasData: " << mData.mHasData << std::endl;
     std::cout << "  DataTypes: " << mData.mDataTypes << std::endl;
 
     // Seems like this should done with reference counting in the

apps/openmw/mwrender/globalmap.cpp

@@ -76,7 +76,7 @@ namespace MWRender
 
                 if (land)
                 {
-                    int mask = ESM::Land::DATA_VHGT | ESM::Land::DATA_VNML | ESM::Land::DATA_VCLR | ESM::Land::DATA_VTEX | ESM::Land::DATA_WNAM;
+                    int mask = ESM::Land::DATA_WNAM;
                     if (!land->isDataLoaded(mask))
                         land->loadData(mask);
                 }
@@ -133,6 +133,8 @@ namespace MWRender
                     }
                 }
                 loadingListener->increaseProgress();
+                if (land)
+                    land->unloadData();
             }
         }
 

apps/openmw/mwrender/renderingmanager.cpp

@@ -1036,10 +1036,10 @@ void RenderingManager::enableTerrain(bool enable)
         if (!mTerrain)
         {
             if (Settings::Manager::getBool("distant land", "Terrain"))
-                mTerrain = new Terrain::DefaultWorld(mRendering.getScene(), new MWRender::TerrainStorage(), RV_Terrain,
+                mTerrain = new Terrain::DefaultWorld(mRendering.getScene(), new MWRender::TerrainStorage(true), RV_Terrain,
                                                 Settings::Manager::getBool("shader", "Terrain"), Terrain::Align_XY, 1, 64);
             else
-                mTerrain = new Terrain::TerrainGrid(mRendering.getScene(), new MWRender::TerrainStorage(), RV_Terrain,
+                mTerrain = new Terrain::TerrainGrid(mRendering.getScene(), new MWRender::TerrainStorage(false), RV_Terrain,
                                                 Settings::Manager::getBool("shader", "Terrain"), Terrain::Align_XY);
             mTerrain->applyMaterials(Settings::Manager::getBool("enabled", "Shadows"),
                                      Settings::Manager::getBool("split", "Shadows"));

apps/openmw/mwrender/terrainstorage.cpp

@@ -9,6 +9,22 @@
 namespace MWRender
 {
 
+    TerrainStorage::TerrainStorage(bool preload)
+    {
+        if (preload)
+        {
+            const MWWorld::ESMStore &esmStore =
+                MWBase::Environment::get().getWorld()->getStore();
+
+            MWWorld::Store<ESM::Land>::iterator it = esmStore.get<ESM::Land>().begin();
+            for (; it != esmStore.get<ESM::Land>().end(); ++it)
+            {
+                ESM::Land* land = const_cast<ESM::Land*>(&*it); // TODO: fix store interface
+                land->loadData(ESM::Land::DATA_VCLR|ESM::Land::DATA_VHGT|ESM::Land::DATA_VNML|ESM::Land::DATA_VTEX);
+            }
+        }
+    }
+
     void TerrainStorage::getBounds(float& minX, float& maxX, float& minY, float& maxY)
     {
         minX = 0, minY = 0, maxX = 0, maxY = 0;
@@ -39,6 +55,12 @@ namespace MWRender
         const MWWorld::ESMStore &esmStore =
             MWBase::Environment::get().getWorld()->getStore();
         ESM::Land* land = esmStore.get<ESM::Land>().search(cellX, cellY);
+        if (!land)
+            return NULL;
+
+        const int flags = ESM::Land::DATA_VCLR|ESM::Land::DATA_VHGT|ESM::Land::DATA_VNML|ESM::Land::DATA_VTEX;
+        if (!land->isDataLoaded(flags))
+            land->loadData(flags);
         return land;
     }
 

apps/openmw/mwrender/terrainstorage.hpp

@@ -14,6 +14,10 @@ namespace MWRender
         virtual const ESM::LandTexture* getLandTexture(int index, short plugin);
     public:
 
+        ///@param preload Preload all Land records at startup? If using the multithreaded terrain component, this
+        /// should be set to "true" in order to avoid race conditions.
+        TerrainStorage(bool preload);
+
         /// Get bounds of the whole terrain in cell units
         virtual void getBounds(float& minX, float& maxX, float& minY, float& maxY);
     };

apps/openmw/mwworld/scene.cpp

@@ -231,6 +231,11 @@ namespace MWWorld
                         cell->getCell()->getGridY()
                     );
                 if (land) {
+                    // Actually only VHGT is needed here, but we'll need the rest for rendering anyway.
+                    // Load everything now to reduce IO overhead.
+                    const int flags = ESM::Land::DATA_VCLR|ESM::Land::DATA_VHGT|ESM::Land::DATA_VNML|ESM::Land::DATA_VTEX;
+                    if (!land->isDataLoaded(flags))
+                        land->loadData(flags);
                     mPhysics->addHeightField (
                         land->mLandData->mHeights,
                         cell->getCell()->getGridX(),

components/esm/loadland.cpp

@@ -72,7 +72,6 @@ Land::Land()
     , mDataLoaded(false)
     , mLandData(NULL)
     , mPlugin(0)
-    , mHasData(false)
 {
 }
 
@@ -97,8 +96,6 @@ void Land::load(ESMReader &esm)
     // Store the file position
     mContext = esm.getContext();
 
-    mHasData = false;
-
     // Skip these here. Load the actual data when the cell is loaded.
     if (esm.isNextSub("VNML"))
     {
@@ -126,10 +123,6 @@ void Land::load(ESMReader &esm)
         mDataTypes |= DATA_VTEX;
     }
 
-    // We need all three of VNML, VHGT and VTEX in order to use the
-    // landscape. (Though Morrowind seems to accept terrain without VTEX/VCLR entries)
-    mHasData = mDataTypes & (DATA_VNML|DATA_VHGT|DATA_WNAM);
-
     mDataLoaded = 0;
     mLandData = NULL;
 }
@@ -144,13 +137,12 @@ void Land::save(ESMWriter &esm) const
     esm.writeHNT("DATA", mFlags);
 }
 
-/// \todo remove memory allocation when only defaults needed
 void Land::loadData(int flags)
 {
     // Try to load only available data
-    int actual = flags & mDataTypes;
+    flags = flags & mDataTypes;
     // Return if all required data is loaded
-    if (flags == 0 || (actual != 0 && (mDataLoaded & actual) == actual)) {
+    if ((mDataLoaded & flags) == flags) {
         return;
     }
     // Create storage if nothing is loaded
@@ -160,15 +152,13 @@ void Land::loadData(int flags)
     }
     mEsm->restoreContext(mContext);
 
-    memset(mLandData->mNormals, 0, sizeof(mLandData->mNormals));
-
     if (mEsm->isNextSub("VNML")) {
-        condLoad(actual, DATA_VNML, mLandData->mNormals, sizeof(mLandData->mNormals));
+        condLoad(flags, DATA_VNML, mLandData->mNormals, sizeof(mLandData->mNormals));
     }
 
     if (mEsm->isNextSub("VHGT")) {
         static VHGT vhgt;
-        if (condLoad(actual, DATA_VHGT, &vhgt, sizeof(vhgt))) {
+        if (condLoad(flags, DATA_VHGT, &vhgt, sizeof(vhgt))) {
             float rowOffset = vhgt.mHeightOffset;
             for (int y = 0; y < LAND_SIZE; y++) {
                 rowOffset += vhgt.mHeightData[y * LAND_SIZE];
@@ -184,30 +174,18 @@ void Land::loadData(int flags)
             mLandData->mUnk1 = vhgt.mUnk1;
             mLandData->mUnk2 = vhgt.mUnk2;
         }
-    } else if ((flags & DATA_VHGT) && (mDataLoaded & DATA_VHGT) == 0) {
-        for (int i = 0; i < LAND_NUM_VERTS; ++i) {
-            mLandData->mHeights[i] = -256.0f * HEIGHT_SCALE;
-        }
-        mDataLoaded |= DATA_VHGT;
     }
 
     if (mEsm->isNextSub("WNAM")) {
-        condLoad(actual, DATA_WNAM, mLandData->mWnam, 81);
+        condLoad(flags, DATA_WNAM, mLandData->mWnam, 81);
-    }
-    if (mEsm->isNextSub("VCLR")) {
-        mLandData->mUsingColours = true;
-        condLoad(actual, DATA_VCLR, mLandData->mColours, 3 * LAND_NUM_VERTS);
-    } else {
-        mLandData->mUsingColours = false;
     }
+    if (mEsm->isNextSub("VCLR"))
+        condLoad(flags, DATA_VCLR, mLandData->mColours, 3 * LAND_NUM_VERTS);
     if (mEsm->isNextSub("VTEX")) {
         static uint16_t vtex[LAND_NUM_TEXTURES];
-        if (condLoad(actual, DATA_VTEX, vtex, sizeof(vtex))) {
+        if (condLoad(flags, DATA_VTEX, vtex, sizeof(vtex))) {
             LandData::transposeTextureData(vtex, mLandData->mTextures);
         }
-    } else if ((flags & DATA_VTEX) && (mDataLoaded & DATA_VTEX) == 0) {
-        memset(mLandData->mTextures, 0, sizeof(mLandData->mTextures));
-        mDataLoaded |= DATA_VTEX;
     }
 }
 
@@ -232,4 +210,9 @@ bool Land::condLoad(int flags, int dataFlag, void *ptr, unsigned int size)
     return false;
 }
 
+bool Land::isDataLoaded(int flags) const
+{
+    return (mDataLoaded & flags) == (flags & mDataTypes);
+}
+
 }

components/esm/loadland.hpp

@@ -32,7 +32,6 @@ struct Land
     ESMReader* mEsm;
     ESM_Context mContext;
 
-    bool mHasData;
     int mDataTypes;
     int mDataLoaded;
 
@@ -81,7 +80,6 @@ struct Land
         VNML mNormals[LAND_NUM_VERTS * 3];
         uint16_t mTextures[LAND_NUM_TEXTURES];
 
-        bool mUsingColours;
         char mColours[3 * LAND_NUM_VERTS];
         int mDataTypes;
 
@@ -113,10 +111,8 @@ struct Land
     void unloadData();
 
     /// Check if given data type is loaded
-    /// \todo reimplement this
+    /// @note We only check data types that *can* be loaded (present in mDataTypes)
-    bool isDataLoaded(int flags) {
+    bool isDataLoaded(int flags) const;
-        return (mDataLoaded & flags) == flags;
-    }
 
     private:
         Land(const Land& land);

components/esmterrain/storage.cpp

@@ -31,7 +31,7 @@ namespace ESMTerrain
         int cellY = origin.y;
 
         const ESM::Land* land = getLand(cellX, cellY);
-        if (!land)
+        if (!land || !(land->mDataTypes&ESM::Land::DATA_VHGT))
             return false;
 
         min = std::numeric_limits<float>::max();
@@ -73,7 +73,7 @@ namespace ESMTerrain
             row += ESM::Land::LAND_SIZE-1;
         }
         ESM::Land* land = getLand(cellX, cellY);
-        if (land && land->mHasData)
+        if (land && land->mDataTypes&ESM::Land::DATA_VNML)
         {
             normal.x = land->mLandData->mNormals[col*ESM::Land::LAND_SIZE*3+row*3];
             normal.y = land->mLandData->mNormals[col*ESM::Land::LAND_SIZE*3+row*3+1];
@@ -108,7 +108,7 @@ namespace ESMTerrain
             row = 0;
         }
         ESM::Land* land = getLand(cellX, cellY);
-        if (land && land->mLandData->mUsingColours)
+        if (land && land->mDataTypes&ESM::Land::DATA_VCLR)
         {
             color.r = land->mLandData->mColours[col*ESM::Land::LAND_SIZE*3+row*3] / 255.f;
             color.g = land->mLandData->mColours[col*ESM::Land::LAND_SIZE*3+row*3+1] / 255.f;
@@ -157,9 +157,8 @@ namespace ESMTerrain
             for (int cellX = startX; cellX < startX + std::ceil(size); ++cellX)
             {
                 ESM::Land* land = getLand(cellX, cellY);
-                if (land && !land->mHasData)
+                if (land && !(land->mDataTypes&ESM::Land::DATA_VHGT))
                     land = NULL;
-                bool hasColors = land && land->mLandData->mUsingColours;
 
                 int rowStart = 0;
                 int colStart = 0;
@@ -183,7 +182,7 @@ namespace ESMTerrain
                         else
                             positions[vertX*numVerts*3 + vertY*3 + 2] = -2048;
 
-                        if (land)
+                        if (land && land->mDataTypes&ESM::Land::DATA_VNML)
                         {
                             normal.x = land->mLandData->mNormals[col*ESM::Land::LAND_SIZE*3+row*3];
                             normal.y = land->mLandData->mNormals[col*ESM::Land::LAND_SIZE*3+row*3+1];
@@ -207,7 +206,7 @@ namespace ESMTerrain
                         normals[vertX*numVerts*3 + vertY*3 + 1] = normal.y;
                         normals[vertX*numVerts*3 + vertY*3 + 2] = normal.z;
 
-                        if (hasColors)
+                        if (land && land->mDataTypes&ESM::Land::DATA_VCLR)
                         {
                             color.r = land->mLandData->mColours[col*ESM::Land::LAND_SIZE*3+row*3] / 255.f;
                             color.g = land->mLandData->mColours[col*ESM::Land::LAND_SIZE*3+row*3+1] / 255.f;
@@ -263,7 +262,7 @@ namespace ESMTerrain
         assert(y<ESM::Land::LAND_TEXTURE_SIZE);
 
         ESM::Land* land = getLand(cellX, cellY);
-        if (land)
+        if (land && (land->mDataTypes&ESM::Land::DATA_VTEX))
         {
             int tex = land->mLandData->mTextures[y * ESM::Land::LAND_TEXTURE_SIZE + x];
             if (tex == 0)