Merge branch 'fast_cell4_load' into 'master'

Lazy loading of ESM4::reference

See merge request OpenMW/openmw!2804

apps/openmw/mwworld/cellstore.cpp

@@ -2,6 +2,7 @@
 #include "magiceffects.hpp"
 
 #include <algorithm>
+#include <fstream>
 
 #include <components/debug/debuglog.hpp>
 
@@ -42,7 +43,10 @@
 #include <components/esm4/loadligh.hpp>
 #include <components/esm4/loadrefr.hpp>
 #include <components/esm4/loadstat.hpp>
+#include <components/esm4/readerutils.hpp>
+#include <components/files/openfile.hpp>
 #include <components/misc/tuplehelpers.hpp>
+#include <components/resource/resourcesystem.hpp>
 
 #include "../mwbase/environment.hpp"
 #include "../mwbase/luamanager.hpp"
@@ -756,19 +760,47 @@ namespace MWWorld
         }
     }
 
-    void CellStore::listRefs(const ESM4::Cell& cell)
+    template <typename ReferenceInvocable>
+    static void visitCell4References(const ESM4::Cell& cell, ESM::ReadersCache& readers, ReferenceInvocable&& invocable)
     {
-        auto& refs = MWBase::Environment::get().getWorld()->getStore().get<ESM4::Reference>();
+        auto stream = Files::openBinaryInputFileStream(cell.mReaderContext.filename);
+        ESM4::Reader readerESM4(
+            std::move(stream), cell.mReaderContext.filename, MWBase::Environment::get().getResourceSystem()->getVFS());
 
-        for (const auto& ref : refs)
+        readerESM4.setEncoder(readers.getStatelessEncoder());
+        readerESM4.restoreContext(cell.mReaderContext);
+        bool continueRead = true;
+        while (ESM::RefId::formIdRefId(readerESM4.getContext().currCell) == cell.mId && readerESM4.hasMoreRecs()
+            && continueRead)
         {
-            if (ref.mParent == cell.mId)
-            {
-                mIds.push_back(ref.mBaseObj);
-            }
+            continueRead = ESM4::ReaderUtils::readItem(
+                readerESM4,
+                [&](ESM4::Reader& reader) {
+                    auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
+                    ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
+                    if (esm4RecName == ESM::RecNameInts::REC_REFR4)
+                    {
+                        ESM4::Reference ref;
+                        ref.load(reader);
+                        invocable(ref);
+                        return true;
+                    }
+                    else if (esm4RecName == ESM::RecNameInts::REC_CELL4)
+                    {
+                        ESM4::Cell cellToLoad;
+                        cellToLoad.load(reader); // This is necessary to exit
+                    }
+                    return false;
+                },
+                [&](ESM4::Reader& reader) {});
         }
     }
 
+    void CellStore::listRefs(const ESM4::Cell& cell)
+    {
+        visitCell4References(cell, mReaders, [&](ESM4::Reference& ref) { mIds.push_back(ref.mBaseObj); });
+    }
+
     void CellStore::listRefs()
     {
         ESM::visit([&](auto&& cell) { listRefs(cell); }, mCellVariant);
@@ -830,15 +862,7 @@ namespace MWWorld
 
     void CellStore::loadRefs(const ESM4::Cell& cell, std::map<ESM::RefNum, ESM::RefId>& refNumToID)
     {
-        auto& refs = MWBase::Environment::get().getWorld()->getStore().get<ESM4::Reference>();
-
-        for (const auto& ref : refs)
-        {
-            if (ref.mParent == cell.mId)
-            {
-                loadRef(ref, false);
-            }
-        }
+        visitCell4References(cell, mReaders, [&](ESM4::Reference& ref) { loadRef(ref, false); });
     }
 
     void CellStore::loadRefs()

apps/openmw/mwworld/esmloader.cpp

@@ -65,8 +65,7 @@ namespace MWWorld
             {
                 ESM4::Reader readerESM4(
                     std::move(stream), filepath, MWBase::Environment::get().getResourceSystem()->getVFS());
-                auto statelessEncoder = mEncoder->getStatelessEncoder();
-                readerESM4.setEncoder(&statelessEncoder);
+                readerESM4.setEncoder(mReaders.getStatelessEncoder());
                 mStore.loadESM4(readerESM4);
                 break;
             }

apps/openmw/mwworld/esmstore.hpp

@@ -29,7 +29,6 @@ namespace ESM4
     class Reader;
     struct Static;
     struct Cell;
-    struct Reference;
     struct Light;
 }
 
@@ -106,7 +105,7 @@ namespace MWWorld
             // Special entry which is hardcoded and not loaded from an ESM
             Store<ESM::Attribute>,
 
-            Store<ESM4::Static>, Store<ESM4::Cell>, Store<ESM4::Reference>, Store<ESM4::Light>>;
+            Store<ESM4::Static>, Store<ESM4::Cell>, Store<ESM4::Light>>;
 
     private:
         template <typename T>

apps/openmw/mwworld/worldimp.cpp

@@ -270,6 +270,8 @@ namespace MWWorld
         , mPlayerInJail(false)
         , mSpellPreloadTimer(0.f)
     {
+        if (encoder)
+            mReaders.setStatelessEncoder(encoder->getStatelessEncoder());
         mESMVersions.resize(mContentFiles.size(), -1);
         Loading::Listener* listener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
         listener->loadingOn();

components/esm3/readerscache.hpp

@@ -9,6 +9,8 @@
 #include <optional>
 #include <string>
 
+#include <components/to_utf8/to_utf8.hpp>
+
 namespace ESM
 {
     class ReadersCache
@@ -55,13 +57,23 @@ namespace ESM
 
         BusyItem get(std::size_t index);
 
+        void setStatelessEncoder(const ToUTF8::StatelessUtf8Encoder& statelessEncoderPtr)
+        {
+            mStatelessEncoder.emplace(statelessEncoderPtr);
+        }
+
+        const ToUTF8::StatelessUtf8Encoder* getStatelessEncoder()
+        {
+            return mStatelessEncoder.has_value() ? &mStatelessEncoder.value() : nullptr;
+        }
+
     private:
         const std::size_t mCapacity;
         std::map<std::size_t, std::list<Item>::iterator> mIndex;
         std::list<Item> mBusyItems;
         std::list<Item> mFreeItems;
         std::list<Item> mClosedItems;
-
+        std::optional<ToUTF8::StatelessUtf8Encoder> mStatelessEncoder;
         inline void closeExtraReaders();
 
         inline void releaseItem(std::list<Item>::iterator it) noexcept;

components/esm4/loadcell.cpp

@@ -240,6 +240,8 @@ void ESM4::Cell::load(ESM4::Reader& reader)
                 throw std::runtime_error("ESM4::CELL::load - Unknown subrecord " + ESM::printName(subHdr.typeId));
         }
     }
+
+    mReaderContext = reader.getContext();
 }
 
 // void ESM4::Cell::save(ESM4::Writer& writer) const

components/esm4/loadcell.hpp

@@ -37,6 +37,7 @@
 #include <components/esm/defs.hpp>
 #include <components/esm/refid.hpp>
 #include <components/esm3/cellid.hpp>
+#include <components/esm4/reader.hpp>
 
 namespace ESM4
 {
@@ -96,6 +97,8 @@ namespace ESM4
 
         CellGroup* mCellGroup;
 
+        ESM4::ReaderContext mReaderContext;
+
         void load(ESM4::Reader& reader);
         // void save(ESM4::Writer& writer) const;