- Add support for loading references from multiple esm/esp files. Full reference ID mangling coming soon (currently, moved references are simply cloned).

- Reference loader now (partially) supports MVRF tag.

apps/esmtool/esmtool.cpp

@@ -220,7 +220,10 @@ void loadCell(ESM::Cell &cell, ESM::ESMReader &esm, Arguments& info)
     bool save = (info.mode == "clone");
 
     // Skip back to the beginning of the reference list
-    cell.restore(esm);
+    // FIXME: Changes to the references backend required to support multiple plugins have
+    //  almost certainly broken this following line. I'll leave it as is for now, so that
+    //  the compiler does not complain.
+    cell.restore(esm, 0);
 
     // Loop through all the references
     ESM::CellRef ref;

apps/openmw/mwbase/world.hpp

@@ -106,7 +106,7 @@ namespace MWBase
 
             virtual const ESMS::ESMStore& getStore() const = 0;
 
-            virtual ESM::ESMReader& getEsmReader() = 0;
+            virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;
 
             virtual MWWorld::LocalScripts& getLocalScripts() = 0;
 

apps/openmw/mwworld/cells.cpp

@@ -85,7 +85,7 @@ MWWorld::Ptr MWWorld::Cells::getPtrAndCache (const std::string& name, Ptr::CellS
     return ptr;
 }
 
-MWWorld::Cells::Cells (const ESMS::ESMStore& store, ESM::ESMReader& reader)
+MWWorld::Cells::Cells (const ESMS::ESMStore& store, std::vector<ESM::ESMReader>& reader)
 : mStore (store), mReader (reader),
   mIdCache (20, std::pair<std::string, Ptr::CellStore *> ("", (Ptr::CellStore*)0)), /// \todo make cache size configurable
   mIdCacheIndex (0)
@@ -120,6 +120,7 @@ MWWorld::Ptr::CellStore *MWWorld::Cells::getExterior (int x, int y)
 
     if (result->second.mState!=Ptr::CellStore::State_Loaded)
     {
+        // Multiple plugin support for landscape data is much easier than for references. The last plugin wins.
         result->second.load (mStore, mReader);
         fillContainers (result->second);
     }

apps/openmw/mwworld/cells.hpp

@@ -2,6 +2,7 @@
 #define GAME_MWWORLD_CELLS_H
 
 #include <map>
+#include <list>
 #include <string>
 
 #include "ptr.hpp"
@@ -22,7 +23,7 @@ namespace MWWorld
     class Cells
     {
             const ESMS::ESMStore& mStore;
-            ESM::ESMReader& mReader;
+            std::vector<ESM::ESMReader>& mReader;
             std::map<std::string, CellStore> mInteriors;
             std::map<std::pair<int, int>, CellStore> mExteriors;
             std::vector<std::pair<std::string, CellStore *> > mIdCache;
@@ -39,7 +40,7 @@ namespace MWWorld
 
         public:
 
-            Cells (const ESMS::ESMStore& store, ESM::ESMReader& reader);
+            Cells (const ESMS::ESMStore& store, std::vector<ESM::ESMReader>& reader);
             ///< \todo pass the dynamic part of the ESMStore isntead (once it is written) of the whole
             /// world
 

apps/openmw/mwworld/cellstore.cpp

@@ -16,7 +16,7 @@ namespace MWWorld
         mWaterLevel = cell->mWater;
     }
 
-    void CellStore::load (const ESMS::ESMStore &store, ESM::ESMReader &esm)
+    void CellStore::load (const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm)
     {
         if (mState!=State_Loaded)
         {
@@ -31,7 +31,7 @@ namespace MWWorld
         }
     }
 
-    void CellStore::preload (const ESMS::ESMStore &store, ESM::ESMReader &esm)
+    void CellStore::preload (const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm)
     {
         if (mState==State_Unloaded)
         {
@@ -41,7 +41,7 @@ namespace MWWorld
         }
     }
 
-    void CellStore::listRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm)
+    void CellStore::listRefs(const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm)
     {
         assert (cell);
 
@@ -49,26 +49,24 @@ namespace MWWorld
             return; // this is a dynamically generated cell -> skipping.
 
         // Load references from all plugins that do something with this cell.
-        // HACK: only use first entry for now, full support requires some more work
+        for (size_t i = 0; i < cell->mContextList.size(); i++)
-        //for (int i = 0; i < cell->mContextList.size(); i++)
-        for (int i = 0; i < 1; i++)
         {
             // Reopen the ESM reader and seek to the right position.
-            // TODO: we will need to intoduce separate "esm"s, one per plugin!
+            int index = cell->mContextList.at(i).index;
-            cell->restore (esm);
+            cell->restore (esm[index], i);
 
             ESM::CellRef ref;
 
             // Get each reference in turn
-            while (cell->getNextRef (esm, ref))
+            while (cell->getNextRef (esm[index], ref))
             {
                 std::string lowerCase;
 
                 std::transform (ref.mRefID.begin(), ref.mRefID.end(), std::back_inserter (lowerCase),
                     (int(*)(int)) std::tolower);
 
-                // TODO: support deletion / moving references out of the cell. no simple "push_back",
+                // TODO: Fully support deletion / moving references out of the cell. no simple "push_back",
-                //  but see what the plugin wants to do.
+                //  but make sure that the reference exists only once.
                 mIds.push_back (lowerCase);
             }
         }
@@ -76,7 +74,7 @@ namespace MWWorld
         std::sort (mIds.begin(), mIds.end());
     }
 
-    void CellStore::loadRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm)
+    void CellStore::loadRefs(const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm)
     {
       assert (cell);
 
@@ -84,24 +82,24 @@ namespace MWWorld
             return; // this is a dynamically generated cell -> skipping.
 
         // Load references from all plugins that do something with this cell.
-        // HACK: only use first entry for now, full support requires some more work
+        for (size_t i = 0; i < cell->mContextList.size(); i++)
-        //for (int i = 0; i < cell->mContextList.size(); i++)
-        for (int i = 0; i < 1; i++)
         {
             // Reopen the ESM reader and seek to the right position.
-            // TODO: we will need to intoduce separate "esm"s, one per plugin!
+            int index = cell->mContextList.at(i).index;
-            cell->restore(esm);
+            cell->restore (esm[index], i);
 
             ESM::CellRef ref;
 
             // Get each reference in turn
-            while(cell->getNextRef(esm, ref))
+            while(cell->getNextRef(esm[index], ref))
             {
                 std::string lowerCase;
 
                 std::transform (ref.mRefID.begin(), ref.mRefID.end(), std::back_inserter (lowerCase),
                     (int(*)(int)) std::tolower);
 
+                // TODO: Fully support deletion / moving references out of the cell. No simple loading,
+                //  but make sure that the reference exists only once. Current code clones references.
                 int rec = store.find(ref.mRefID);
 
                 ref.mRefID = lowerCase;

apps/openmw/mwworld/cellstore.hpp

@@ -3,7 +3,7 @@
 
 #include <components/esm/records.hpp>
 
-#include <list>
+#include <deque>
 #include <algorithm>
 
 #include "refdata.hpp"
@@ -126,9 +126,9 @@ namespace MWWorld
     CellRefList<ESM::Static>            statics;
     CellRefList<ESM::Weapon>            weapons;
 
-    void load (const ESMS::ESMStore &store, ESM::ESMReader &esm);
+    void load (const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm);
 
-    void preload (const ESMS::ESMStore &store, ESM::ESMReader &esm);
+    void preload (const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm);
 
     /// Call functor (ref) for each reference. functor must return a bool. Returning
     /// false will abort the iteration.
@@ -187,9 +187,9 @@ namespace MWWorld
     }
 
     /// Run through references and store IDs
-    void listRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm);
+    void listRefs(const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm);
 
-    void loadRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm);
+    void loadRefs(const ESMS::ESMStore &store, std::vector<ESM::ESMReader> &esm);
   };
 }
 

apps/openmw/mwworld/worldimp.cpp

@@ -179,6 +179,8 @@ namespace MWWorld
         mWeatherManager = new MWWorld::WeatherManager(mRendering);
 
         int idx = 0;
+        // NOTE: We might need to reserve one more for the running game / save.
+        mEsm.resize(master.size() + plugins.size());
         for (std::vector<std::string>::size_type i = 0; i < master.size(); i++, idx++)
         {
             boost::filesystem::path masterPath (fileCollections.getCollection (".esm").getPath (master[i]));
@@ -186,10 +188,12 @@ namespace MWWorld
             std::cout << "Loading ESM " << masterPath.string() << "\n";
 
             // This parses the ESM file
-            mEsm.setEncoding(encoding);
+            ESM::ESMReader lEsm;
-            mEsm.open (masterPath.string());
+            lEsm.setEncoding(encoding);
-            mEsm.setIndex(idx);
+            lEsm.open (masterPath.string());
-            mStore.load (mEsm);
+            lEsm.setIndex(idx);
+            mEsm[idx] = lEsm;
+            mStore.load (mEsm[idx]);
         }
  
         for (std::vector<std::string>::size_type i = 0; i < plugins.size(); i++, idx++)
@@ -199,10 +203,12 @@ namespace MWWorld
             std::cout << "Loading ESP " << pluginPath.string() << "\n";
 
             // This parses the ESP file
-            mEsm.setEncoding(encoding);
+            ESM::ESMReader lEsm;
-            mEsm.open (pluginPath.string());
+            lEsm.setEncoding(encoding);
-            mEsm.setIndex(idx);
+            lEsm.open (pluginPath.string());
-            mStore.load (mEsm);
+            lEsm.setIndex(idx);
+            mEsm[idx] = lEsm;
+            mStore.load (mEsm[idx]);
         }
 
         mPlayer = new MWWorld::Player (mStore.npcs.find ("player"), *this);
@@ -282,7 +288,7 @@ namespace MWWorld
         return mStore;
     }
 
-    ESM::ESMReader& World::getEsmReader()
+    std::vector<ESM::ESMReader>& World::getEsmReader()
     {
         return mEsm;
     }

apps/openmw/mwworld/worldimp.hpp

@@ -55,7 +55,7 @@ namespace MWWorld
 
             MWWorld::Scene *mWorldScene;
             MWWorld::Player *mPlayer;
-            ESM::ESMReader mEsm;
+            std::vector<ESM::ESMReader> mEsm;
             ESMS::ESMStore mStore;
             LocalScripts mLocalScripts;
             MWWorld::Globals *mGlobalVariables;
@@ -127,7 +127,7 @@ namespace MWWorld
 
             virtual const ESMS::ESMStore& getStore() const;
 
-            virtual ESM::ESMReader& getEsmReader();
+            virtual std::vector<ESM::ESMReader>& getEsmReader();
 
             virtual LocalScripts& getLocalScripts();
 

components/esm/esmcommon.hpp

@@ -113,6 +113,10 @@ struct ESM_Context
   size_t leftFile;
   NAME recName, subName;
   HEDRstruct header;
+  // When working with multiple esX files, we will generate lists of all files that
+  //  actually contribute to a specific cell. Therefore, we need to store the index
+  //  of the file belonging to this contest. See CellStore::(list/load)refs for details.
+  int index;
 
   // True if subName has been read but not used.
   bool subCached;

components/esm/esmreader.hpp

@@ -81,7 +81,7 @@ public:
   //  to the individual load() methods. This hack allows to pass this reference
   //  indirectly to the load() method.
   int idx;
-  void setIndex(const int index) {idx = index;}
+  void setIndex(const int index) {idx = index; mCtx.index = index;}
   const int getIndex() {return idx;}
 
   /*************************************************************************

components/esm/loadcell.cpp

@@ -113,6 +113,8 @@ void Cell::load(ESMReader &esm)
 
     // Save position of the cell references and move on
     mContextList.push_back(esm.getContext());
+    if (mContextList.size() > 1)
+        std::cout << "found two plugins" << std::endl;
     esm.skipRecord();
 }
 
@@ -147,10 +149,9 @@ void Cell::save(ESMWriter &esm)
         esm.writeHNT("NAM0", mNAM0);
 }
 
-void Cell::restore(ESMReader &esm) const
+void Cell::restore(ESMReader &esm, int iCtx) const
 {
-    // TODO: support all contexts in the list!
+    esm.restoreContext(mContextList[iCtx]);
-    esm.restoreContext(mContextList[0]);
 }
 
 std::string Cell::getDescription() const
@@ -169,15 +170,28 @@ std::string Cell::getDescription() const
 
 bool Cell::getNextRef(ESMReader &esm, CellRef &ref)
 {
-    // TODO: Add support for moved references. References moved without crossing a cell boundary simply
-    //  overwrite old data. References moved across cell boundaries are using a different set of keywords,
-    //  and I'll have to think more about how this can be done.
     // TODO: Add support for multiple plugins. This requires a tricky renaming scheme for "ref.mRefnum".
     //  I'll probably add something to "ESMReader", we will need one per plugin anyway.
     // TODO: Try and document reference numbering, I don't think this has been done anywhere else.
     if (!esm.hasMoreSubs())
         return false;
     
+    if (esm.isNextSub("MVRF")) {
+        // Moved existing reference across cell boundaries, so interpret the blocks correctly.
+        // FIXME: Right now, we don't do anything with this data. This might result in weird behaviour,
+        //  where a moved reference does not appear because the owning cell (i.e. this cell) is not
+        //  loaded in memory.
+        int movedRefnum = 0;
+        int destCell[2];
+        esm.getHT(movedRefnum);
+        esm.getHNT(destCell, "CNDT");
+        // TODO: Figure out what happens when a reference has moved into an interior cell. This might
+        //  be required for NPCs following the player.
+    }
+    // If we have just parsed a MVRF entry, there should be a regular FRMR entry following right there.
+    // With the exception that this bock technically belongs to a different cell than this one.
+    // TODO: Figure out a way to handle these weird references that do not belong to this cell.
+    //  This may require some not-so-small behing-the-scenes updates.
     esm.getHNT(ref.mRefnum, "FRMR");
     ref.mRefID = esm.getHNString("NAME");
 
@@ -228,6 +242,9 @@ bool Cell::getNextRef(ESMReader &esm, CellRef &ref)
     // Number of references in the cell? Maximum once in each cell,
     // but not always at the beginning, and not always right. In other
     // words, completely useless.
+    // Update: Well, maybe not completely useless. This might actually be
+    //  number_of_references + number_of_references_moved_here_Across_boundaries,
+    //  and could be helpful for collecting these weird moved references.
     ref.mNam0 = 0;
     if (esm.isNextSub("NAM0"))
     {

components/esm/loadcell.hpp

@@ -152,7 +152,7 @@ struct Cell
   // somewhere other than the file system, you need to pre-open the
   // ESMReader, and the filename must match the stored filename
   // exactly.
-  void restore(ESMReader &esm) const;
+  void restore(ESMReader &esm, int iCtx) const;
 
   std::string getDescription() const;
   ///< Return a short string describing the cell (mostly used for debugging/logging purpose)

components/esm_store/reclists.hpp

@@ -496,6 +496,11 @@ namespace ESMS
     {
       count++;
 
+      // Don't automatically assume that a new cell must be spawned. Multiple plugins write to the same cell,
+      //  and we merge all this data into one Cell object. However, we can't simply search for the cell id,
+      //  as many exterior cells do not have a name. Instead, we need to search by (x,y) coordinates - and they
+      //  are not available until both cells have been loaded! So first, proceed as usual.
+      
       // All cells have a name record, even nameless exterior cells.
       ESM::Cell *cell = new ESM::Cell;
       cell->mName = id;
@@ -505,12 +510,22 @@ namespace ESMS
 
       if(cell->mData.mFlags & ESM::Cell::Interior)
         {
-          // Store interior cell by name
+          // Store interior cell by name, try to merge with existing parent data.
+          ESM::Cell *oldcell = const_cast<ESM::Cell*>(searchInt(id));
+          if (oldcell) {
+              cell->mContextList.push_back(oldcell->mContextList.at(0));
+              delete oldcell;
+          }
           intCells[id] = cell;
         }
       else
         {
-          // Store exterior cells by grid position
+          // Store exterior cells by grid position, try to merge with existing parent data.
+          ESM::Cell *oldcell = const_cast<ESM::Cell*>(searchExt(cell->getGridX(), cell->getGridY()));
+          if (oldcell) {
+              cell->mContextList.push_back(oldcell->mContextList.at(0));
+              delete oldcell;
+          }
           extCells[std::make_pair (cell->mData.mX, cell->mData.mY)] = cell;
         }
     }