Limit the number of simultaneously open not actively used content files

Use LRU cache for ESMReaders. When cache capacity is reached close least
recently used ESMReader. Remember the file name if a reader was open. Once the
reader requested again open the file if there is stored name for it. Put
released ESMReader to the back of the free items list. Close ESMReader's from
the front of the free items list.

Cached item can be used only by one client at the same time. If the same item is
requested twice exception is thrown. This should never happen in practice. If
this happens need to fix the client logic.

It's allowed to go over the capacity limit when requesting different readers.
Ideally this should never happen but there will be system error anyway
signalizing about too many open files. Need to fix client logic in this case.

All places that were using a vector of ESMReaders now using the cache. Cache is
local for each use case and there is no need for a thread safety.

apps/bulletobjecttool/main.cpp

@@ -15,6 +15,7 @@
 #include <components/version/version.hpp>
 #include <components/vfs/manager.hpp>
 #include <components/vfs/registerarchives.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <boost/program_options.hpp>
 
@@ -156,7 +157,7 @@ namespace
         Settings::Manager settings;
         settings.load(config);
 
-        std::vector<ESM::ESMReader> readers(contentFiles.size());
+        ESM::ReadersCache readers;
         EsmLoader::Query query;
         query.mLoadActivators = true;
         query.mLoadCells = true;

apps/navmeshtool/main.cpp

@@ -20,6 +20,7 @@
 #include <components/version/version.hpp>
 #include <components/vfs/manager.hpp>
 #include <components/vfs/registerarchives.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <osg/Vec3f>
 
@@ -174,7 +175,7 @@ namespace NavMeshTool
 
             DetourNavigator::NavMeshDb db(dbPath, maxDbFileSize);
 
-            std::vector<ESM::ESMReader> readers(contentFiles.size());
+            ESM::ReadersCache readers;
             EsmLoader::Query query;
             query.mLoadActivators = true;
             query.mLoadCells = true;

apps/navmeshtool/worldspacedata.cpp

@@ -20,6 +20,7 @@
 #include <components/vfs/manager.hpp>
 #include <components/debug/debugging.hpp>
 #include <components/navmeshtool/protocol.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <LinearMath/btVector3.h>
 
@@ -68,17 +69,17 @@ namespace NavMeshTool
         }
 
         std::vector<CellRef> loadCellRefs(const ESM::Cell& cell, const EsmLoader::EsmData& esmData,
-            std::vector<ESM::ESMReader>& readers)
+            ESM::ReadersCache& readers)
         {
             std::vector<EsmLoader::Record<CellRef>> cellRefs;
 
             for (std::size_t i = 0; i < cell.mContextList.size(); i++)
             {
-                ESM::ESMReader& reader = readers[static_cast<std::size_t>(cell.mContextList[i].index)];
+                ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(cell.mContextList[i].index));
-                cell.restore(reader, static_cast<int>(i));
+                cell.restore(*reader, static_cast<int>(i));
                 ESM::CellRef cellRef;
                 bool deleted = false;
-                while (ESM::Cell::getNextRef(reader, cellRef, deleted))
+                while (ESM::Cell::getNextRef(*reader, cellRef, deleted))
                 {
                     Misc::StringUtils::lowerCaseInPlace(cellRef.mRefID);
                     const ESM::RecNameInts type = getType(esmData, cellRef.mRefID);
@@ -101,7 +102,7 @@ namespace NavMeshTool
 
         template <class F>
         void forEachObject(const ESM::Cell& cell, const EsmLoader::EsmData& esmData, const VFS::Manager& vfs,
-            Resource::BulletShapeManager& bulletShapeManager, std::vector<ESM::ESMReader>& readers,
+            Resource::BulletShapeManager& bulletShapeManager, ESM::ReadersCache& readers,
             F&& f)
         {
             std::vector<CellRef> cellRefs = loadCellRefs(cell, esmData, readers);
@@ -233,7 +234,7 @@ namespace NavMeshTool
         mAabb.m_max = btVector3(0, 0, 0);
     }
 
-    WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
+    WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
         const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
         bool processInteriorCells, bool writeBinaryLog)
     {

apps/navmeshtool/worldspacedata.hpp

@@ -18,6 +18,7 @@
 namespace ESM
 {
     class ESMReader;
+    class ReadersCache;
 }
 
 namespace VFS
@@ -89,7 +90,7 @@ namespace NavMeshTool
         std::vector<std::vector<float>> mHeightfields;
     };
 
-    WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
+    WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
         const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
         bool processInteriorCells, bool writeBinaryLog);
 }

apps/openmw/mwbase/world.hpp

@@ -158,8 +158,6 @@ namespace MWBase
 
             virtual const MWWorld::ESMStore& getStore() const = 0;
 
-            virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;
-
             virtual MWWorld::LocalScripts& getLocalScripts() = 0;
 
             virtual bool hasCellChanged() const = 0;

apps/openmw/mwrender/groundcover.cpp

@@ -12,6 +12,7 @@
 #include <components/sceneutil/nodecallback.hpp>
 #include <components/terrain/quadtreenode.hpp>
 #include <components/shader/shadermanager.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include "../mwworld/groundcoverstore.hpp"
 
@@ -176,7 +177,7 @@ namespace MWRender
         osg::Vec2f minBound = (center - osg::Vec2f(size/2.f, size/2.f));
         osg::Vec2f maxBound = (center + osg::Vec2f(size/2.f, size/2.f));
         DensityCalculator calculator(mDensity);
-        std::vector<ESM::ESMReader> esm;
+        ESM::ReadersCache readers;
         osg::Vec2i startCell = osg::Vec2i(std::floor(center.x() - size/2.f), std::floor(center.y() - size/2.f));
         for (int cellX = startCell.x(); cellX < startCell.x() + size; ++cellX)
         {
@@ -190,14 +191,13 @@ namespace MWRender
                 std::map<ESM::RefNum, ESM::CellRef> refs;
                 for (size_t i=0; i<cell.mContextList.size(); ++i)
                 {
-                    unsigned int index = cell.mContextList[i].index;
+                    const std::size_t index = static_cast<std::size_t>(cell.mContextList[i].index);
-                    if (esm.size() <= index)
+                    const ESM::ReadersCache::BusyItem reader = readers.get(index);
-                        esm.resize(index+1);
+                    cell.restore(*reader, i);
-                    cell.restore(esm[index], i);
                     ESM::CellRef ref;
                     ref.mRefNum.unset();
                     bool deleted = false;
-                    while(cell.getNextRef(esm[index], ref, deleted))
+                    while (cell.getNextRef(*reader, ref, deleted))
                     {
                         if (!deleted && refs.find(ref.mRefNum) == refs.end() && !calculator.isInstanceEnabled()) deleted = true;
                         if (!deleted && !isInChunkBorders(ref, minBound, maxBound)) deleted = true;

apps/openmw/mwrender/objectpaging.cpp

@@ -17,6 +17,7 @@
 #include <components/sceneutil/clone.hpp>
 #include <components/sceneutil/util.hpp>
 #include <components/vfs/manager.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <osgParticle/ParticleProcessor>
 #include <osgParticle/ParticleSystemUpdater>
@@ -412,7 +413,7 @@ namespace MWRender
         osg::Vec3f relativeViewPoint = viewPoint - worldCenter;
 
         std::map<ESM::RefNum, ESM::CellRef> refs;
-        std::vector<ESM::ESMReader> esm;
+        ESM::ReadersCache readers;
         const MWWorld::ESMStore& store = MWBase::Environment::get().getWorld()->getStore();
 
         for (int cellX = startCell.x(); cellX < startCell.x() + size; ++cellX)
@@ -425,17 +426,16 @@ namespace MWRender
                 {
                     try
                     {
-                        unsigned int index = cell->mContextList[i].index;
+                        const std::size_t index = static_cast<std::size_t>(cell->mContextList[i].index);
-                        if (esm.size()<=index)
+                        const ESM::ReadersCache::BusyItem reader = readers.get(index);
-                            esm.resize(index+1);
+                        cell->restore(*reader, i);
-                        cell->restore(esm[index], i);
                         ESM::CellRef ref;
                         ref.mRefNum.unset();
                         ESM::MovedCellRef cMRef;
                         cMRef.mRefNum.mIndex = 0;
                         bool deleted = false;
                         bool moved = false;
-                        while (ESM::Cell::getNextRef(esm[index], ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
+                        while (ESM::Cell::getNextRef(*reader, ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
                         {
                             if (moved)
                                 continue;

apps/openmw/mwworld/cells.cpp

@@ -68,9 +68,7 @@ MWWorld::CellStore *MWWorld::Cells::getCellStore (const ESM::Cell *cell)
         std::map<std::string, CellStore>::iterator result = mInteriors.find (lowerName);
 
         if (result==mInteriors.end())
-        {
+            result = mInteriors.emplace(std::move(lowerName), CellStore(cell, mStore, mReaders)).first;
-            result = mInteriors.insert (std::make_pair (lowerName, CellStore (cell, mStore, mReader))).first;
-        }
 
         return &result->second;
     }
@@ -80,11 +78,8 @@ MWWorld::CellStore *MWWorld::Cells::getCellStore (const ESM::Cell *cell)
             mExteriors.find (std::make_pair (cell->getGridX(), cell->getGridY()));
 
         if (result==mExteriors.end())
-        {
+            result = mExteriors.emplace(std::make_pair(cell->getGridX(), cell->getGridY()),
-            result = mExteriors.insert (std::make_pair (
+                                        CellStore(cell, mStore, mReaders)).first;
-                std::make_pair (cell->getGridX(), cell->getGridY()), CellStore (cell, mStore, mReader))).first;
-
-        }
 
         return &result->second;
     }
@@ -130,9 +125,10 @@ void MWWorld::Cells::writeCell (ESM::ESMWriter& writer, CellStore& cell) const
     writer.endRecord (ESM::REC_CSTA);
 }
 
-MWWorld::Cells::Cells (const MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& reader)
+MWWorld::Cells::Cells (const MWWorld::ESMStore& store, ESM::ReadersCache& readers)
-: mStore (store), mReader (reader),
+    : mStore(store)
-  mIdCacheIndex (0)
+    , mReaders(readers)
+    , mIdCacheIndex(0)
 {
     int cacheSize = std::clamp(Settings::Manager::getInt("pointers cache size", "Cells"), 40, 1000);
     mIdCache = IdCache(cacheSize, std::pair<std::string, CellStore *> ("", (CellStore*)nullptr));
@@ -165,8 +161,7 @@ MWWorld::CellStore *MWWorld::Cells::getExterior (int x, int y)
             cell = MWBase::Environment::get().getWorld()->createRecord (record);
         }
 
-        result = mExteriors.insert (std::make_pair (
+        result = mExteriors.emplace(std::make_pair(x, y), CellStore(cell, mStore, mReaders)).first;
-            std::make_pair (x, y), CellStore (cell, mStore, mReader))).first;
     }
 
     if (result->second.getState()!=CellStore::State_Loaded)
@@ -186,7 +181,7 @@ MWWorld::CellStore *MWWorld::Cells::getInterior (const std::string& name)
     {
         const ESM::Cell *cell = mStore.get<ESM::Cell>().find(lowerName);
 
-        result = mInteriors.insert (std::make_pair (lowerName, CellStore (cell, mStore, mReader))).first;
+        result = mInteriors.emplace(std::move(lowerName), CellStore(cell, mStore, mReaders)).first;
     }
 
     if (result->second.getState()!=CellStore::State_Loaded)

apps/openmw/mwworld/cells.hpp

@@ -11,6 +11,7 @@ namespace ESM
 {
     class ESMReader;
     class ESMWriter;
+    class ReadersCache;
     struct CellId;
     struct Cell;
     struct RefNum;
@@ -30,7 +31,7 @@ namespace MWWorld
     {
             typedef std::vector<std::pair<std::string, CellStore *> > IdCache;
             const MWWorld::ESMStore& mStore;
-            std::vector<ESM::ESMReader>& mReader;
+            ESM::ReadersCache& mReaders;
             mutable std::map<std::string, CellStore> mInteriors;
             mutable std::map<std::pair<int, int>, CellStore> mExteriors;
             IdCache mIdCache;
@@ -51,7 +52,7 @@ namespace MWWorld
 
             void clear();
 
-            Cells (const MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& reader);
+            explicit Cells(const MWWorld::ESMStore& store, ESM::ReadersCache& reader);
 
             CellStore *getExterior (int x, int y);
 

apps/openmw/mwworld/cellstore.cpp

@@ -17,6 +17,7 @@
 #include <components/esm3/fogstate.hpp>
 #include <components/esm3/creaturelevliststate.hpp>
 #include <components/esm3/doorstate.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include "../mwbase/environment.hpp"
 #include "../mwbase/luamanager.hpp"
@@ -387,8 +388,14 @@ namespace MWWorld
         return false;
     }
 
-    CellStore::CellStore (const ESM::Cell *cell, const MWWorld::ESMStore& esmStore, std::vector<ESM::ESMReader>& readerList)
+    CellStore::CellStore(const ESM::Cell* cell, const MWWorld::ESMStore& esmStore, ESM::ReadersCache& readers)
-        : mStore(esmStore), mReader(readerList), mCell (cell), mState (State_Unloaded), mHasState (false), mLastRespawn(0,0), mRechargingItemsUpToDate(false)
+        : mStore(esmStore)
+        , mReaders(readers)
+        , mCell(cell)
+        , mState(State_Unloaded)
+        , mHasState(false)
+        , mLastRespawn(0, 0)
+        , mRechargingItemsUpToDate(false)
     {
         mWaterLevel = cell->mWater;
     }
@@ -545,8 +552,6 @@ namespace MWWorld
 
     void CellStore::listRefs()
     {
-        std::vector<ESM::ESMReader>& esm = mReader;
-
         assert (mCell);
 
         if (mCell->mContextList.empty())
@@ -558,8 +563,9 @@ namespace MWWorld
             try
             {
                 // Reopen the ESM reader and seek to the right position.
-                int index = mCell->mContextList[i].index;
+                const std::size_t index = static_cast<std::size_t>(mCell->mContextList[i].index);
-                mCell->restore (esm[index], i);
+                const ESM::ReadersCache::BusyItem reader = mReaders.get(index);
+                mCell->restore(*reader, i);
 
                 ESM::CellRef ref;
 
@@ -568,7 +574,7 @@ namespace MWWorld
                 cMRef.mRefNum.mIndex = 0;
                 bool deleted = false;
                 bool moved = false;
-                while (ESM::Cell::getNextRef(esm[index], ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
+                while (ESM::Cell::getNextRef(*reader, ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
                 {
                     if (deleted || moved)
                         continue;
@@ -602,8 +608,6 @@ namespace MWWorld
 
     void CellStore::loadRefs()
     {
-        std::vector<ESM::ESMReader>& esm = mReader;
-
         assert (mCell);
 
         if (mCell->mContextList.empty())
@@ -617,8 +621,9 @@ namespace MWWorld
             try
             {
                 // Reopen the ESM reader and seek to the right position.
-                int index = mCell->mContextList[i].index;
+                const std::size_t index = static_cast<std::size_t>(mCell->mContextList[i].index);
-                mCell->restore (esm[index], i);
+                const ESM::ReadersCache::BusyItem reader = mReaders.get(index);
+                mCell->restore(*reader, i);
 
                 ESM::CellRef ref;
                 ref.mRefNum.unset();
@@ -628,7 +633,7 @@ namespace MWWorld
                 cMRef.mRefNum.mIndex = 0;
                 bool deleted = false;
                 bool moved = false;
-                while (ESM::Cell::getNextRef(esm[index], ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
+                while (ESM::Cell::getNextRef(*reader, ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
                 {
                     if (moved)
                         continue;

apps/openmw/mwworld/cellstore.hpp

@@ -38,6 +38,7 @@
 
 namespace ESM
 {
+    class ReadersCache;
     struct Cell;
     struct CellState;
     struct CellId;
@@ -61,7 +62,7 @@ namespace MWWorld
         private:
 
             const MWWorld::ESMStore& mStore;
-            std::vector<ESM::ESMReader>& mReader;
+            ESM::ReadersCache& mReaders;
 
             // Even though fog actually belongs to the player and not cells,
             // it makes sense to store it here since we need it once for each cell.
@@ -211,9 +212,7 @@ namespace MWWorld
             }
 
             /// @param readerList The readers to use for loading of the cell on-demand.
-            CellStore (const ESM::Cell *cell_,
+            CellStore(const ESM::Cell* cell, const MWWorld::ESMStore& store, ESM::ReadersCache& readers);
-                       const MWWorld::ESMStore& store,
-                       std::vector<ESM::ESMReader>& readerList);
 
             const ESM::Cell *getCell() const;
 

apps/openmw/mwworld/esmloader.cpp

@@ -2,13 +2,13 @@
 #include "esmstore.hpp"
 
 #include <components/esm3/esmreader.hpp>
+#include <components/esm3/readerscache.hpp>
 
 namespace MWWorld
 {
 
-EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& readers,
+EsmLoader::EsmLoader(MWWorld::ESMStore& store, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder)
-    ToUTF8::Utf8Encoder* encoder)
+    : mReaders(readers)
-    : mEsm(readers)
     , mStore(store)
     , mEncoder(encoder)
     , mDialogue(nullptr) // A content file containing INFO records without a DIAL record appends them to the previous file's dialogue
@@ -17,13 +17,25 @@ EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& read
 
 void EsmLoader::load(const boost::filesystem::path& filepath, int& index, Loading::Listener* listener)
 {
-    ESM::ESMReader lEsm;
+    const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
-    lEsm.setEncoder(mEncoder);
+
-    lEsm.setIndex(index);
+    reader->setEncoder(mEncoder);
-    lEsm.open(filepath.string());
+    reader->setIndex(index);
-    lEsm.resolveParentFileIndices(mEsm);
+    reader->open(filepath.string());
-    mEsm[index] = std::move(lEsm);
+    reader->resolveParentFileIndices(mReaders);
-    mStore.load(mEsm[index], listener, mDialogue);
+
+    assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
+    for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
+        if (i == static_cast<std::size_t>(reader->getIndex()))
+            throw std::runtime_error("File " + reader->getName() + " asks for parent file "
+                + reader->getGameFiles()[i].name
+                + ", but it is not available or has been loaded in the wrong order. "
+                  "Please run the launcher to fix this issue.");
+
+    mStore.load(*reader, listener, mDialogue);
+
+    if (!mMasterFileFormat.has_value() && Misc::StringUtils::ciEndsWith(reader->getName(), ".esm") && !Misc::StringUtils::ciEndsWith(reader->getName(), ".omwgame"))
+        mMasterFileFormat = reader->getFormat();
 }
 
 } /* namespace MWWorld */

apps/openmw/mwworld/esmloader.hpp

@@ -1,7 +1,7 @@
 #ifndef ESMLOADER_HPP
 #define ESMLOADER_HPP
 
-#include <vector>
+#include <optional>
 
 #include "contentloader.hpp"
 
@@ -12,7 +12,7 @@ namespace ToUTF8
 
 namespace ESM
 {
-    class ESMReader;
+    class ReadersCache;
     struct Dialogue;
 }
 
@@ -23,16 +23,18 @@ class ESMStore;
 
 struct EsmLoader : public ContentLoader
 {
-    EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& readers,
+    explicit EsmLoader(MWWorld::ESMStore& store, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder);
-        ToUTF8::Utf8Encoder* encoder);
+
+    std::optional<int> getMasterFileFormat() const { return mMasterFileFormat; }
 
     void load(const boost::filesystem::path& filepath, int& index, Loading::Listener* listener) override;
 
     private:
-        std::vector<ESM::ESMReader>& mEsm;
+        ESM::ReadersCache& mReaders;
         MWWorld::ESMStore& mStore;
         ToUTF8::Utf8Encoder* mEncoder;
         ESM::Dialogue* mDialogue;
+        std::optional<int> mMasterFileFormat;
 };
 
 } /* namespace MWWorld */

apps/openmw/mwworld/esmstore.cpp

@@ -9,6 +9,7 @@
 #include <components/loadinglistener/loadinglistener.hpp>
 #include <components/lua/configuration.hpp>
 #include <components/misc/algorithm.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include "../mwmechanics/spelllist.hpp"
 
@@ -24,19 +25,18 @@ namespace
 
     constexpr std::size_t deletedRefID = std::numeric_limits<std::size_t>::max();
 
-    void readRefs(const ESM::Cell& cell, std::vector<Ref>& refs, std::vector<std::string>& refIDs, std::vector<ESM::ESMReader>& readers)
+    void readRefs(const ESM::Cell& cell, std::vector<Ref>& refs, std::vector<std::string>& refIDs, ESM::ReadersCache& readers)
     {
         // TODO: we have many similar copies of this code.
         for (size_t i = 0; i < cell.mContextList.size(); i++)
         {
-            size_t index = cell.mContextList[i].index;
+            const std::size_t index = static_cast<std::size_t>(cell.mContextList[i].index);
-            if (readers.size() <= index)
+            const ESM::ReadersCache::BusyItem reader = readers.get(index);
-                readers.resize(index + 1);
+            cell.restore(*reader, i);
-            cell.restore(readers[index], i);
             ESM::CellRef ref;
             ref.mRefNum.unset();
             bool deleted = false;
-            while(cell.getNextRef(readers[index], ref, deleted))
+            while (cell.getNextRef(*reader, ref, deleted))
             {
                 if(deleted)
                     refs.emplace_back(ref.mRefNum, deletedRefID);
@@ -241,7 +241,7 @@ ESM::LuaScriptsCfg ESMStore::getLuaScriptsCfg() const
     return cfg;
 }
 
-void ESMStore::setUp(bool validateRecords)
+void ESMStore::setUp()
 {
     mIds.clear();
 
@@ -267,15 +267,15 @@ void ESMStore::setUp(bool validateRecords)
     mMagicEffects.setUp();
     mAttributes.setUp();
     mDialogs.setUp();
-
-    if (validateRecords)
-    {
-        validate();
-        countAllCellRefs();
-    }
 }
 
-void ESMStore::countAllCellRefs()
+void ESMStore::validateRecords(ESM::ReadersCache& readers)
+{
+    validate();
+    countAllCellRefs(readers);
+}
+
+void ESMStore::countAllCellRefs(ESM::ReadersCache& readers)
 {
     // TODO: We currently need to read entire files here again.
     // We should consider consolidating or deferring this reading.
@@ -283,7 +283,6 @@ void ESMStore::countAllCellRefs()
         return;
     std::vector<Ref> refs;
     std::vector<std::string> refIDs;
-    std::vector<ESM::ESMReader> readers;
     for(auto it = mCells.intBegin(); it != mCells.intEnd(); ++it)
         readRefs(*it, refs, refIDs, readers);
     for(auto it = mCells.extBegin(); it != mCells.extEnd(); ++it)

apps/openmw/mwworld/esmstore.hpp

@@ -20,6 +20,11 @@ namespace MWMechanics
     class SpellList;
 }
 
+namespace ESM
+{
+    class ReadersCache;
+}
+
 namespace MWWorld
 {
     class ESMStore
@@ -90,7 +95,7 @@ namespace MWWorld
         /// Validate entries in store after setup
         void validate();
 
-        void countAllCellRefs();
+        void countAllCellRefs(ESM::ReadersCache& readers);
 
         template<class T>
         void removeMissingObjects(Store<T>& store);
@@ -266,7 +271,8 @@ namespace MWWorld
 
         // This method must be called once, after loading all master/plugin files. This can only be done
         //  from the outside, so it must be public.
-        void setUp(bool validateRecords = false);
+        void setUp();
+        void validateRecords(ESM::ReadersCache& readers);
 
         int countSavedGameRecords() const;
 

apps/openmw/mwworld/groundcoverstore.cpp

@@ -2,6 +2,7 @@
 
 #include <components/esmloader/load.hpp>
 #include <components/misc/stringops.hpp>
+#include <components/esm3/readerscache.hpp>
 
 namespace MWWorld
 {
@@ -11,7 +12,7 @@ namespace MWWorld
         query.mLoadStatics = true;
         query.mLoadCells = true;
 
-        std::vector<ESM::ESMReader> readers(groundcoverFiles.size());
+        ESM::ReadersCache readers;
         const ::EsmLoader::EsmData content = ::EsmLoader::loadEsmData(query, groundcoverFiles, fileCollections, readers, encoder);
 
         for (const ESM::Static& stat : statics)

apps/openmw/mwworld/worldimp.cpp

@@ -145,8 +145,8 @@ namespace MWWorld
         ToUTF8::Utf8Encoder* encoder, int activationDistanceOverride,
         const std::string& startCell, const std::string& startupScript,
         const std::string& resourcePath, const std::string& userDataPath)
-    : mResourceSystem(resourceSystem), mLocalScripts (mStore),
+    : mResourceSystem(resourceSystem), mLocalScripts(mStore),
-      mCells (mStore, mEsm), mSky (true),
+      mCells(mStore, mReaders), mSky(true),
       mGodMode(false), mScriptsEnabled(true), mDiscardMovements(true), mContentFiles (contentFiles),
       mUserDataPath(userDataPath),
       mDefaultHalfExtents(Settings::Manager::getVector3("default actor pathfind half extents", "Game")),
@@ -156,30 +156,20 @@ namespace MWWorld
       mLevitationEnabled(true), mGoToJail(false), mDaysInPrison(0),
       mPlayerTraveling(false), mPlayerInJail(false), mSpellPreloadTimer(0.f)
     {
-        mEsm.resize(contentFiles.size());
         Loading::Listener* listener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
         listener->loadingOn();
 
-        loadContentFiles(fileCollections, contentFiles, mStore, mEsm, encoder, listener);
+        loadContentFiles(fileCollections, contentFiles, encoder, listener);
         loadGroundcoverFiles(fileCollections, groundcoverFiles, encoder);
 
         listener->loadingOff();
 
-        // Find main game file
-        for (const ESM::ESMReader& reader : mEsm)
-        {
-            if (!Misc::StringUtils::ciEndsWith(reader.getName(), ".esm") && !Misc::StringUtils::ciEndsWith(reader.getName(), ".omwgame"))
-                continue;
-            if (reader.getFormat() == 0)
-                ensureNeededRecords();  // and insert records that may not be present in all versions of MW.
-            break;
-        }
-
         mCurrentDate = std::make_unique<DateTimeManager>();
 
         fillGlobalVariables();
 
-        mStore.setUp(true);
+        mStore.setUp();
+        mStore.validateRecords(mReaders);
         mStore.movePlayerRecord();
 
         mSwimHeightScale = mStore.get<ESM::GameSetting>().find("fSwimHeightScale")->mValue.getFloat();
@@ -417,23 +407,6 @@ namespace MWWorld
         }
     }
 
-    void World::validateMasterFiles(const std::vector<ESM::ESMReader>& readers)
-    {
-        for (const auto& esm : readers)
-        {
-            assert(esm.getGameFiles().size() == esm.getParentFileIndices().size());
-            for (unsigned int i=0; i<esm.getParentFileIndices().size(); ++i)
-            {
-                if (!esm.isValidParentFileIndex(i))
-                {
-                    std::string fstring = "File " + esm.getName() + " asks for parent file " + esm.getGameFiles()[i].name
-                        + ", but it is not available or has been loaded in the wrong order. Please run the launcher to fix this issue.";
-                    throw std::runtime_error(fstring);
-                }
-            }
-        }
-    }
-
     void World::ensureNeededRecords()
     {
         std::map<std::string, ESM::Variant> gmst;
@@ -641,11 +614,6 @@ namespace MWWorld
         return mStore;
     }
 
-    std::vector<ESM::ESMReader>& World::getEsmReader()
-    {
-        return mEsm;
-    }
-
     LocalScripts& World::getLocalScripts()
     {
         return mLocalScripts;
@@ -2939,11 +2907,11 @@ namespace MWWorld
         return mScriptsEnabled;
     }
 
-    void World::loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content, ESMStore& store, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener)
+    void World::loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content,
+        ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener)
     {
         GameContentLoader gameContentLoader;
-        EsmLoader esmLoader(store, readers, encoder);
+        EsmLoader esmLoader(mStore, mReaders, encoder);
-        validateMasterFiles(readers);
 
         gameContentLoader.addLoader(".esm", esmLoader);
         gameContentLoader.addLoader(".esp", esmLoader);
@@ -2951,7 +2919,7 @@ namespace MWWorld
         gameContentLoader.addLoader(".omwaddon", esmLoader);
         gameContentLoader.addLoader(".project", esmLoader);
 
-        OMWScriptsLoader omwScriptsLoader(store);
+        OMWScriptsLoader omwScriptsLoader(mStore);
         gameContentLoader.addLoader(".omwscripts", omwScriptsLoader);
 
         int idx = 0;
@@ -2970,6 +2938,9 @@ namespace MWWorld
             }
             idx++;
         }
+
+        if (const auto v = esmLoader.getMasterFileFormat(); v.has_value() && *v == 0)
+            ensureNeededRecords(); // Insert records that may not be present in all versions of master files.
     }
 
     void World::loadGroundcoverFiles(const Files::Collections& fileCollections, const std::vector<std::string>& groundcoverFiles, ToUTF8::Utf8Encoder* encoder)

apps/openmw/mwworld/worldimp.hpp

@@ -5,6 +5,7 @@
 
 #include <components/settings/settings.hpp>
 #include <components/misc/rng.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include "../mwbase/world.hpp"
 
@@ -81,7 +82,7 @@ namespace MWWorld
         private:
             Resource::ResourceSystem* mResourceSystem;
 
-            std::vector<ESM::ESMReader> mEsm;
+            ESM::ReadersCache mReaders;
             MWWorld::ESMStore mStore;
             GroundcoverStore mGroundcoverStore;
             LocalScripts mLocalScripts;
@@ -162,13 +163,13 @@ namespace MWWorld
             void updateNavigatorObject(const MWPhysics::Object& object);
 
             void ensureNeededRecords();
-            void validateMasterFiles(const std::vector<ESM::ESMReader>& readers);
 
             void fillGlobalVariables();
 
             void updateSkyDate();
 
-            void loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content, ESMStore& store, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener);
+            void loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content,
+                ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener);
 
             void loadGroundcoverFiles(const Files::Collections& fileCollections, const std::vector<std::string>& groundcoverFiles, ToUTF8::Utf8Encoder* encoder);
 
@@ -241,8 +242,6 @@ namespace MWWorld
 
             const MWWorld::ESMStore& getStore() const override;
 
-            std::vector<ESM::ESMReader>& getEsmReader() override;
-
             LocalScripts& getLocalScripts() override;
 
             bool hasCellChanged() const override;

apps/openmw_test_suite/CMakeLists.txt

@@ -81,6 +81,8 @@ if (GTEST_FOUND AND GMOCK_FOUND)
 
         fx/lexer.cpp
         fx/technique.cpp
+
+        esm3/readerscache.cpp
     )
 
     source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES})

apps/openmw_test_suite/esm3/readerscache.cpp

@@ -0,0 +1,91 @@
+#include <components/esm3/readerscache.hpp>
+#include <components/files/collections.hpp>
+#include <components/files/multidircollection.hpp>
+
+#include <gtest/gtest.h>
+
+#ifndef OPENMW_DATA_DIR
+#error "OPENMW_DATA_DIR is not defined"
+#endif
+
+namespace
+{
+    using namespace testing;
+    using namespace ESM;
+
+    TEST(ESM3ReadersCache, onAttemptToRequestTheSameReaderTwiceShouldThrowException)
+    {
+        ReadersCache readers(1);
+        const ReadersCache::BusyItem reader = readers.get(0);
+        EXPECT_THROW(readers.get(0), std::logic_error);
+    }
+
+    TEST(ESM3ReadersCache, shouldAllowToHaveBusyItemsMoreThanCapacity)
+    {
+        ReadersCache readers(1);
+        const ReadersCache::BusyItem reader0 = readers.get(0);
+        const ReadersCache::BusyItem reader1 = readers.get(1);
+    }
+
+    TEST(ESM3ReadersCache, shouldKeepClosedReleasedClosedItem)
+    {
+        ReadersCache readers(1);
+        readers.get(0);
+        const ReadersCache::BusyItem reader = readers.get(0);
+        EXPECT_FALSE(reader->isOpen());
+    }
+
+    struct ESM3ReadersCacheWithContentFile : Test
+    {
+        static constexpr std::size_t sInitialOffset = 324;
+        static constexpr std::size_t sSkip = 100;
+        const Files::PathContainer mDataDirs {{std::string(OPENMW_DATA_DIR)}};
+        const Files::Collections mFileCollections {mDataDirs, true};
+        const std::string mContentFile = "template.omwgame";
+        const std::string mContentFilePath = mFileCollections.getCollection(".omwgame").getPath(mContentFile).string();
+    };
+
+    TEST_F(ESM3ReadersCacheWithContentFile, shouldKeepOpenReleasedOpenReader)
+    {
+        ReadersCache readers(1);
+        {
+            const ReadersCache::BusyItem reader = readers.get(0);
+            reader->open(mContentFilePath);
+            ASSERT_TRUE(reader->isOpen());
+            ASSERT_EQ(reader->getFileOffset(), sInitialOffset);
+            ASSERT_GT(reader->getFileSize(), sInitialOffset + sSkip);
+            reader->skip(sSkip);
+            ASSERT_EQ(reader->getFileOffset(), sInitialOffset + sSkip);
+        }
+        {
+            const ReadersCache::BusyItem reader = readers.get(0);
+            EXPECT_TRUE(reader->isOpen());
+            EXPECT_EQ(reader->getName(), mContentFilePath);
+            EXPECT_EQ(reader->getFileOffset(), sInitialOffset + sSkip);
+        }
+    }
+
+    TEST_F(ESM3ReadersCacheWithContentFile, shouldCloseFreeReaderWhenReachingCapacityLimit)
+    {
+        ReadersCache readers(1);
+        {
+            const ReadersCache::BusyItem reader = readers.get(0);
+            reader->open(mContentFilePath);
+            ASSERT_TRUE(reader->isOpen());
+            ASSERT_EQ(reader->getFileOffset(), sInitialOffset);
+            ASSERT_GT(reader->getFileSize(), sInitialOffset + sSkip);
+            reader->skip(sSkip);
+            ASSERT_EQ(reader->getFileOffset(), sInitialOffset + sSkip);
+        }
+        {
+            const ReadersCache::BusyItem reader = readers.get(1);
+            reader->open(mContentFilePath);
+            ASSERT_TRUE(reader->isOpen());
+        }
+        {
+            const ReadersCache::BusyItem reader = readers.get(0);
+            EXPECT_TRUE(reader->isOpen());
+            EXPECT_EQ(reader->getFileOffset(), sInitialOffset);
+        }
+    }
+}

apps/openmw_test_suite/esmloader/load.cpp

@@ -10,6 +10,7 @@
 #include <components/files/collections.hpp>
 #include <components/files/multidircollection.hpp>
 #include <components/to_utf8/to_utf8.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <gtest/gtest.h>
 
@@ -39,7 +40,7 @@ namespace
         query.mLoadGameSettings = true;
         query.mLoadLands = true;
         query.mLoadStatics = true;
-        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ESM::ReadersCache readers;
         ToUTF8::Utf8Encoder* const encoder = nullptr;
         const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
         EXPECT_EQ(esmData.mActivators.size(), 0);
@@ -61,7 +62,7 @@ namespace
         query.mLoadGameSettings = true;
         query.mLoadLands = true;
         query.mLoadStatics = true;
-        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ESM::ReadersCache readers;
         ToUTF8::Utf8Encoder* const encoder = nullptr;
         const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
         EXPECT_EQ(esmData.mActivators.size(), 0);
@@ -83,7 +84,7 @@ namespace
         query.mLoadGameSettings = false;
         query.mLoadLands = true;
         query.mLoadStatics = true;
-        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ESM::ReadersCache readers;
         ToUTF8::Utf8Encoder* const encoder = nullptr;
         const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
         EXPECT_EQ(esmData.mActivators.size(), 0);
@@ -98,7 +99,7 @@ namespace
     TEST_F(EsmLoaderTest, shouldIgnoreAllWithDefaultQuery)
     {
         const Query query;
-        std::vector<ESM::ESMReader> readers(mContentFiles.size());
+        ESM::ReadersCache readers;
         ToUTF8::Utf8Encoder* const encoder = nullptr;
         const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
         EXPECT_EQ(esmData.mActivators.size(), 0);
@@ -121,7 +122,7 @@ namespace
         query.mLoadLands = true;
         query.mLoadStatics = true;
         const std::vector<std::string> contentFiles {{"script.omwscripts"}};
-        std::vector<ESM::ESMReader> readers(contentFiles.size());
+        ESM::ReadersCache readers;
         ToUTF8::Utf8Encoder* const encoder = nullptr;
         const EsmData esmData = loadEsmData(query, contentFiles, mFileCollections, readers, encoder);
         EXPECT_EQ(esmData.mActivators.size(), 0);

components/CMakeLists.txt

@@ -95,7 +95,7 @@ add_component_dir (esm3
     savedgame journalentry queststate locals globalscript player objectstate cellid cellstate globalmap
     inventorystate containerstate npcstate creaturestate dialoguestate statstate npcstats creaturestats
     weatherstate quickkeys fogstate spellstate activespells creaturelevliststate doorstate projectilestate debugprofile
-    aisequence magiceffects custommarkerstate stolenitems transport animationstate controlsstate mappings
+    aisequence magiceffects custommarkerstate stolenitems transport animationstate controlsstate mappings readerscache
     )
 
 add_component_dir (esm3terrain

components/esm3/esmreader.cpp

@@ -1,5 +1,7 @@
 #include "esmreader.hpp"
 
+#include "readerscache.hpp"
+
 #include <components/misc/stringops.hpp>
 #include <components/files/openfile.hpp>
 
@@ -61,21 +63,22 @@ void ESMReader::clearCtx()
    mCtx.subName.clear();
 }
 
-void ESMReader::resolveParentFileIndices(const std::vector<ESMReader>& allPlugins)
+void ESMReader::resolveParentFileIndices(ReadersCache& readers)
 {
     mCtx.parentFileIndices.clear();
-    const std::vector<Header::MasterData> &masters = getGameFiles();
+    for (const Header::MasterData &mast : getGameFiles())
-    for (size_t j = 0; j < masters.size(); j++) {
+    {
-        const Header::MasterData &mast = masters[j];
+        const std::string& fname = mast.name;
-        std::string fname = mast.name;
         int index = getIndex(); 
-        for (int i = 0; i < getIndex(); i++) {
+        for (int i = 0; i < getIndex(); i++)
-            const ESMReader& reader = allPlugins.at(i);
+        {
-            if (reader.getFileSize() == 0)
+            const ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(i));
+            if (reader->getFileSize() == 0)
                 continue;  // Content file in non-ESM format
-            const std::string& candidate = reader.getName();
+            const std::string& candidate = reader->getName();
             std::string fnamecandidate = std::filesystem::path(candidate).filename().string();
-            if (Misc::StringUtils::ciEqual(fname, fnamecandidate)) {
+            if (Misc::StringUtils::ciEqual(fname, fnamecandidate))
+            {
                 index = i;
                 break;
             }

components/esm3/esmreader.hpp

@@ -16,6 +16,8 @@
 namespace ESM
 {
 
+class ReadersCache;
+
 class ESMReader
 {
 public:
@@ -39,6 +41,7 @@ public:
   const NAME &retSubName() const { return mCtx.subName; }
   uint32_t getSubSize() const { return mCtx.leftSub; }
   const std::string& getName() const { return mCtx.filename; };
+  bool isOpen() const { return mEsm != nullptr; }
 
   /*************************************************************************
    *
@@ -85,9 +88,8 @@ public:
   // all files/readers used by the engine. This is required for correct adjustRefNum() results
   // as required for handling moved, deleted and edited CellRefs.
   /// @note Does not validate.
-  void resolveParentFileIndices(const std::vector<ESMReader>& files);
+  void resolveParentFileIndices(ReadersCache& readers);
   const std::vector<int>& getParentFileIndices() const { return mCtx.parentFileIndices; }
-  bool isValidParentFileIndex(int i) const { return i != getIndex(); }
 
   /*************************************************************************
    *

components/esm3/readerscache.cpp

@@ -0,0 +1,87 @@
+#include "readerscache.hpp"
+
+#include <stdexcept>
+
+namespace ESM
+{
+    ReadersCache::BusyItem::BusyItem(ReadersCache& owner, std::list<Item>::iterator item) noexcept
+        : mOwner(owner)
+        , mItem(item)
+    {}
+
+    ReadersCache::BusyItem::~BusyItem() noexcept
+    {
+        mOwner.releaseItem(mItem);
+    }
+
+    ReadersCache::ReadersCache(std::size_t capacity)
+        : mCapacity(capacity)
+    {}
+
+    ReadersCache::BusyItem ReadersCache::get(std::size_t index)
+    {
+        const auto indexIt = mIndex.find(index);
+        std::list<Item>::iterator it;
+        if (indexIt == mIndex.end())
+        {
+            closeExtraReaders();
+            it = mBusyItems.emplace(mBusyItems.end());
+            mIndex.emplace(index, it);
+        }
+        else
+        {
+            switch (indexIt->second->mState)
+            {
+                case State::Busy:
+                    throw std::logic_error("ESMReader at index " + std::to_string(index) + " is busy");
+                case State::Free:
+                    it = indexIt->second;
+                    mBusyItems.splice(mBusyItems.end(), mFreeItems, it);
+                    break;
+                case State::Closed:
+                    closeExtraReaders();
+                    it = indexIt->second;
+                    if (it->mName.has_value())
+                    {
+                        it->mReader.open(*it->mName);
+                        it->mName.reset();
+                    }
+                    mBusyItems.splice(mBusyItems.end(), mClosedItems, it);
+                    break;
+            }
+            it->mState = State::Busy;
+        }
+
+        return BusyItem(*this, it);
+    }
+
+    void ReadersCache::closeExtraReaders()
+    {
+        while (!mFreeItems.empty() && mBusyItems.size() + mFreeItems.size() + 1 > mCapacity)
+        {
+            const auto it = mFreeItems.begin();
+            if (it->mReader.isOpen())
+            {
+                it->mName = it->mReader.getName();
+                it->mReader.close();
+            }
+            mClosedItems.splice(mClosedItems.end(), mFreeItems, it);
+            it->mState = State::Closed;
+        }
+    }
+
+    void ReadersCache::releaseItem(std::list<Item>::iterator it) noexcept
+    {
+        assert(it->mState == State::Busy);
+        if (it->mReader.isOpen())
+        {
+            mFreeItems.splice(mFreeItems.end(), mBusyItems, it);
+            it->mState = State::Free;
+        }
+        else
+        {
+            mClosedItems.splice(mClosedItems.end(), mBusyItems, it);
+            it->mState = State::Closed;
+        }
+    }
+}

components/esm3/readerscache.hpp

@@ -0,0 +1,71 @@
+#ifndef OPENMW_COMPONENTS_ESM3_READERSCACHE_H
+#define OPENMW_COMPONENTS_ESM3_READERSCACHE_H
+
+#include "esmreader.hpp"
+
+#include <cstddef>
+#include <list>
+#include <map>
+#include <optional>
+#include <string>
+
+namespace ESM
+{
+    class ReadersCache
+    {
+        private:
+            enum class State
+            {
+                Busy,
+                Free,
+                Closed,
+            };
+
+            struct Item
+            {
+                State mState = State::Busy;
+                ESMReader mReader;
+                std::optional<std::string> mName;
+
+                Item() = default;
+            };
+
+        public:
+            class BusyItem
+            {
+                public:
+                    explicit BusyItem(ReadersCache& owner, std::list<Item>::iterator item) noexcept;
+
+                    BusyItem(const BusyItem& other) = delete;
+
+                    ~BusyItem() noexcept;
+
+                    BusyItem& operator=(const BusyItem& other) = delete;
+
+                    ESMReader& operator*() const noexcept { return mItem->mReader; }
+
+                    ESMReader* operator->() const noexcept { return &mItem->mReader; }
+
+                private:
+                    ReadersCache& mOwner;
+                    std::list<Item>::iterator mItem;
+            };
+
+            explicit ReadersCache(std::size_t capacity = 100);
+
+            BusyItem get(std::size_t index);
+
+        private:
+            const std::size_t mCapacity;
+            std::map<std::size_t, std::list<Item>::iterator> mIndex;
+            std::list<Item> mBusyItems;
+            std::list<Item> mFreeItems;
+            std::list<Item> mClosedItems;
+
+            inline void closeExtraReaders();
+
+            inline void releaseItem(std::list<Item>::iterator it) noexcept;
+    };
+}
+
+#endif

components/esmloader/load.cpp

@@ -18,6 +18,7 @@
 #include <components/files/multidircollection.hpp>
 #include <components/misc/resourcehelpers.hpp>
 #include <components/misc/stringops.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <algorithm>
 #include <filesystem>
@@ -207,7 +208,7 @@ namespace EsmLoader
         }
 
         ShallowContent shallowLoad(const Query& query, const std::vector<std::string>& contentFiles,
-            const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers,
+            const Files::Collections& fileCollections, ESM::ReadersCache& readers,
             ToUTF8::Utf8Encoder* encoder)
         {
             ShallowContent result;
@@ -233,14 +234,14 @@ namespace EsmLoader
 
                 const Files::MultiDirCollection& collection = fileCollections.getCollection(extension);
 
-                ESM::ESMReader& reader = readers[i];
+                const ESM::ReadersCache::BusyItem reader = readers.get(i);
-                reader.setEncoder(encoder);
+                reader->setEncoder(encoder);
-                reader.setIndex(static_cast<int>(i));
+                reader->setIndex(static_cast<int>(i));
-                reader.open(collection.getPath(file).string());
+                reader->open(collection.getPath(file).string());
                 if (query.mLoadCells)
-                    reader.resolveParentFileIndices(readers);
+                    reader->resolveParentFileIndices(readers);
 
-                loadEsm(query, readers[i], result);
+                loadEsm(query, *reader, result);
             }
 
             return result;
@@ -289,7 +290,7 @@ namespace EsmLoader
     }
 
     EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
-        const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder)
+        const Files::Collections& fileCollections, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder)
     {
         Log(Debug::Info) << "Loading ESM data...";
 

components/esmloader/load.hpp

@@ -32,7 +32,7 @@ namespace EsmLoader
     };
 
     EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
-        const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers,
+        const Files::Collections& fileCollections, ESM::ReadersCache& readers,
         ToUTF8::Utf8Encoder* encoder);
 }
 

components/resource/foreachbulletobject.cpp

@@ -13,6 +13,7 @@
 #include <components/resource/bulletshapemanager.hpp>
 #include <components/settings/settings.hpp>
 #include <components/vfs/manager.hpp>
+#include <components/esm3/readerscache.hpp>
 
 #include <osg/ref_ptr>
 
@@ -50,17 +51,17 @@ namespace Resource
         }
 
         std::vector<CellRef> loadCellRefs(const ESM::Cell& cell, const EsmLoader::EsmData& esmData,
-            std::vector<ESM::ESMReader>& readers)
+            ESM::ReadersCache& readers)
         {
             std::vector<EsmLoader::Record<CellRef>> cellRefs;
 
             for (std::size_t i = 0; i < cell.mContextList.size(); i++)
             {
-                ESM::ESMReader& reader = readers[static_cast<std::size_t>(cell.mContextList[i].index)];
+                const ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(cell.mContextList[i].index));
-                cell.restore(reader, static_cast<int>(i));
+                cell.restore(*reader, static_cast<int>(i));
                 ESM::CellRef cellRef;
                 bool deleted = false;
-                while (ESM::Cell::getNextRef(reader, cellRef, deleted))
+                while (ESM::Cell::getNextRef(*reader, cellRef, deleted))
                 {
                     Misc::StringUtils::lowerCaseInPlace(cellRef.mRefID);
                     const ESM::RecNameInts type = getType(esmData, cellRef.mRefID);
@@ -83,7 +84,7 @@ namespace Resource
 
         template <class F>
         void forEachObject(const ESM::Cell& cell, const EsmLoader::EsmData& esmData, const VFS::Manager& vfs,
-            Resource::BulletShapeManager& bulletShapeManager, std::vector<ESM::ESMReader>& readers,
+            Resource::BulletShapeManager& bulletShapeManager, ESM::ReadersCache& readers,
             F&& f)
         {
             std::vector<CellRef> cellRefs = loadCellRefs(cell, esmData, readers);
@@ -130,7 +131,7 @@ namespace Resource
         }
     }
 
-    void forEachBulletObject(std::vector<ESM::ESMReader>& readers, const VFS::Manager& vfs,
+    void forEachBulletObject(ESM::ReadersCache& readers, const VFS::Manager& vfs,
         Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
         std::function<void (const ESM::Cell& cell, const BulletObject& object)> callback)
     {

components/resource/foreachbulletobject.hpp

@@ -12,7 +12,7 @@
 
 namespace ESM
 {
-    class ESMReader;
+    class ReadersCache;
     struct Cell;
 }
 
@@ -40,7 +40,7 @@ namespace Resource
         float mScale;
     };
 
-    void forEachBulletObject(std::vector<ESM::ESMReader>& readers, const VFS::Manager& vfs,
+    void forEachBulletObject(ESM::ReadersCache& readers, const VFS::Manager& vfs,
         Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
         std::function<void (const ESM::Cell&, const BulletObject& object)> callback);
 }