Initial commit to load ESM4

Some data is actually loaded and store in ESM Store
Any new ESM4 will go through the same code path and be automatically sent to the right store

apps/openmw/mwworld/esmloader.cpp

@@ -1,9 +1,13 @@
 #include "esmloader.hpp"
 #include "esmstore.hpp"
 
+#include <components/esm/format.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/readerscache.hpp>
+#include <components/esm4/reader.hpp>
 #include <components/files/conversion.hpp>
+#include <components/files/openfile.hpp>
+#include <fstream>
 
 namespace MWWorld
 {
@@ -21,28 +25,50 @@ namespace MWWorld
 
     void EsmLoader::load(const std::filesystem::path& filepath, int& index, Loading::Listener* listener)
     {
-        const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
 
-        reader->setEncoder(mEncoder);
+        auto stream = Files::openBinaryInputFileStream(filepath);
-        reader->setIndex(index);
+        if (!stream->is_open())
-        reader->open(filepath);
+        {
-        reader->resolveParentFileIndices(mReaders);
+            throw std::runtime_error(std::string("File Failed to open file: ") + std::strerror(errno) + "\n");
+            return;
+        }
+        const ESM::Format format = ESM::readFormat(*stream);
+        stream->seekg(0);
 
-        assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
+        switch (format)
-        for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
+        {
-            if (i == static_cast<std::size_t>(reader->getIndex()))
+            case ESM::Format::Tes3:
-                throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file "
+            {
+                const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
+                reader->setEncoder(mEncoder);
+                reader->setIndex(index);
+                reader->open(filepath);
+                reader->resolveParentFileIndices(mReaders);
+
+                assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
+                for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
+                    if (i == static_cast<std::size_t>(reader->getIndex()))
+                        throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file "
                 + reader->getGameFiles()[i].name
                 + ", but it is not available or has been loaded in the wrong order. "
                   "Please run the launcher to fix this issue.");
 
-        mESMVersions[index] = reader->getVer();
+                mESMVersions[index] = reader->getVer();
-        mStore.load(*reader, listener, mDialogue);
+                mStore.load(*reader, listener, mDialogue);
 
-        if (!mMasterFileFormat.has_value()
+                if (!mMasterFileFormat.has_value()
-            && (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
+                    && (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
-                || Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
+                        || Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
-            mMasterFileFormat = reader->getFormat();
+                    mMasterFileFormat = reader->getFormat();
+                break;
+            }
+            case ESM::Format::Tes4:
+            {
+                ESM4::Reader readerESM4(std::move(stream), filepath);
+                readerESM4.setEncoder(mEncoder->getStatelessEncoder());
+                mStore.loadESM4(readerESM4, listener, mDialogue);
+            }
+        }
     }
 
 } /* namespace MWWorld */

apps/openmw/mwworld/esmstore.cpp

@@ -13,6 +13,11 @@
 #include <components/lua/configuration.hpp>
 #include <components/misc/algorithm.hpp>
 
+#include <components/esm4/loadcell.hpp>
+#include <components/esm4/loadrefr.hpp>
+#include <components/esm4/loadstat.hpp>
+#include <components/esm4/reader.hpp>
+
 #include <components/esm4/common.hpp>
 #include <components/esmloader/load.hpp>
 
@@ -180,6 +185,84 @@ namespace MWWorld
                 }
             }
         }
+
+        template <typename T>
+        static void typedReadRecordESM4(ESM4::Reader& reader, ESMStore& stores, Store<T>& store, int& found)
+        {
+            auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
+
+            ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
+            if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*>)
+            {
+                if constexpr (ESM::isESM4Rec(T::sRecordId))
+                {
+                    if (T::sRecordId == esm4RecName)
+                    {
+                        reader.getRecordData();
+                        T value;
+                        value.load(reader);
+                        store.insertStatic(value);
+                        found++;
+                    }
+                }
+            }
+        }
+
+        static void readRecord(ESM4::Reader& reader, ESMStore& store)
+        {
+            int found = 0;
+            std::apply([&reader, &store, &found](
+                           auto&... x) { (ESMStoreImp::typedReadRecordESM4(reader, store, x, found), ...); },
+                store.mStoreImp->mStores);
+            assert(found <= 1);
+            if (found == 0) // unhandled record
+                reader.skipRecordData();
+        }
+
+        static bool readItem(ESM4::Reader& reader, ESMStore& store)
+        {
+            if (!reader.getRecordHeader() || !reader.hasMoreRecs())
+                return false;
+
+            const ESM4::RecordHeader& header = reader.hdr();
+
+            if (header.record.typeId == ESM4::REC_GRUP)
+                return readGroup(reader, store);
+
+            readRecord(reader, store);
+            return true;
+        }
+
+        static bool readGroup(ESM4::Reader& reader, ESMStore& store)
+        {
+            const ESM4::RecordHeader& header = reader.hdr();
+
+            switch (static_cast<ESM4::GroupType>(header.group.type))
+            {
+                case ESM4::Grp_RecordType:
+                case ESM4::Grp_InteriorCell:
+                case ESM4::Grp_InteriorSubCell:
+                case ESM4::Grp_ExteriorCell:
+                case ESM4::Grp_ExteriorSubCell:
+                    reader.enterGroup();
+                    return readItem(reader, store);
+                case ESM4::Grp_WorldChild:
+                case ESM4::Grp_CellChild:
+                case ESM4::Grp_TopicChild:
+                case ESM4::Grp_CellPersistentChild:
+                case ESM4::Grp_CellTemporaryChild:
+                case ESM4::Grp_CellVisibleDistChild:
+                    reader.adjustGRUPFormId();
+                    reader.enterGroup();
+                    if (!reader.hasMoreRecs())
+                        return false;
+                    return readItem(reader, store);
+            }
+
+            reader.skipGroup();
+
+            return true;
+        }
     };
 
     int ESMStore::find(const ESM::RefId& id) const
@@ -338,6 +421,16 @@ namespace MWWorld
         }
     }
 
+    void ESMStore::loadESM4(ESM4::Reader& reader, Loading::Listener* listener, ESM::Dialogue*& dialogue)
+    {
+        while (reader.hasMoreRecs())
+        {
+            reader.exitGroupCheck();
+            if (!ESMStoreImp::readItem(reader, *this))
+                break;
+        }
+    }
+
     void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type)
     {
         mStoreImp->mIds[id] = type;

apps/openmw/mwworld/esmstore.hpp

@@ -24,6 +24,14 @@ namespace MWMechanics
     class SpellList;
 }
 
+namespace ESM4
+{
+    class Reader;
+    struct Static;
+    struct Cell;
+    struct Reference;
+}
+
 namespace ESM
 {
     class ReadersCache;
@@ -95,7 +103,9 @@ namespace MWWorld
             Store<ESM::MagicEffect>, Store<ESM::Skill>,
 
             // Special entry which is hardcoded and not loaded from an ESM
-            Store<ESM::Attribute>>;
+            Store<ESM::Attribute>,
+
+            Store<ESM4::Static>, Store<ESM4::Cell>, Store<ESM4::Reference>>;
 
         template <typename T>
         static constexpr std::size_t getTypeIndex()
@@ -162,6 +172,7 @@ namespace MWWorld
         void validateDynamic();
 
         void load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue);
+        void loadESM4(ESM4::Reader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue);
 
         template <class T>
         const Store<T>& get() const

apps/openmw/mwworld/store.cpp

@@ -13,6 +13,10 @@
 #include <sstream>
 #include <stdexcept>
 
+#include <components/esm4/loadcell.hpp>
+#include <components/esm4/loadrefr.hpp>
+#include <components/esm4/loadstat.hpp>
+
 namespace
 {
     // TODO: Switch to C++23 to get a working version of std::unordered_map::erase
@@ -161,7 +165,10 @@ namespace MWWorld
         if (ptr == nullptr)
         {
             std::stringstream msg;
-            msg << T::getRecordType() << " '" << id << "' not found";
+            if constexpr (!ESM::isESM4Rec(T::sRecordId))
+            {
+                msg << T::getRecordType() << " '" << id << "' not found";
+            }
             throw std::runtime_error(msg.str());
         }
         return ptr;
@@ -171,8 +178,10 @@ namespace MWWorld
     {
         T record;
         bool isDeleted = false;
-
+        if constexpr (!ESM::isESM4Rec(T::sRecordId))
-        record.load(esm, isDeleted);
+        {
+            record.load(esm, isDeleted);
+        }
 
         std::pair<typename Static::iterator, bool> inserted = mStatic.insert_or_assign(record.mId, record);
         if (inserted.second)
@@ -293,7 +302,11 @@ namespace MWWorld
         for (typename Dynamic::const_iterator iter(mDynamic.begin()); iter != mDynamic.end(); ++iter)
         {
             writer.startRecord(T::sRecordId);
-            iter->second.save(writer);
+            if constexpr (!ESM::isESM4Rec(T::sRecordId))
+            {
+                iter->second.save(writer);
+            }
+
             writer.endRecord(T::sRecordId);
         }
     }
@@ -302,8 +315,10 @@ namespace MWWorld
     {
         T record;
         bool isDeleted = false;
-
+        if constexpr (!ESM::isESM4Rec(T::sRecordId))
-        record.load(reader, isDeleted);
+        {
+            record.load(reader, isDeleted);
+        }
         insert(record, overrideOnly);
 
         return RecordId(record.mId, isDeleted);
@@ -1196,3 +1211,7 @@ template class MWWorld::TypedDynamicStore<ESM::Spell>;
 template class MWWorld::TypedDynamicStore<ESM::StartScript>;
 template class MWWorld::TypedDynamicStore<ESM::Static>;
 template class MWWorld::TypedDynamicStore<ESM::Weapon>;
+
+template class MWWorld::TypedDynamicStore<ESM4::Static>;
+template class MWWorld::TypedDynamicStore<ESM4::Reference>;
+template class MWWorld::TypedDynamicStore<ESM4::Cell>;

components/esm/defs.hpp

@@ -333,6 +333,11 @@ namespace ESM
         REC_MSET4 = esm4Recname(ESM4::REC_MSET) // Media Set
     };
 
+    constexpr bool isESM4Rec(RecNameInts RecName)
+    {
+        return RecName & sEsm4RecnameFlag;
+    }
+
     /// Common subrecords
     enum SubRecNameInts
     {

components/esm/refid.cpp

@@ -34,6 +34,11 @@ namespace ESM
         return newRefId;
     }
 
+    RefId RefId::formIdRefId(const ESM4::FormId id)
+    {
+        return ESM::RefId::stringRefId(ESM4::formIdToString(id));
+    }
+
     bool RefId::operator==(std::string_view rhs) const
     {
         return Misc::StringUtils::ciEqual(mId, rhs);

components/esm/refid.hpp

@@ -1,6 +1,7 @@
 #ifndef OPENMW_COMPONENTS_ESM_REFID_HPP
 #define OPENMW_COMPONENTS_ESM_REFID_HPP
 #include <compare>
+#include <components/esm4/formid.hpp>
 #include <functional>
 #include <iosfwd>
 #include <string>
@@ -26,6 +27,7 @@ namespace ESM
         // RefIds that are as string in the code. For serialization, and display. Using explicit conversions make it
         // very clear where in the code we need to convert from string to RefId and Vice versa.
         static RefId stringRefId(std::string_view id);
+        static RefId formIdRefId(const ESM4::FormId id);
         const std::string& getRefIdString() const { return mId; }
 
     private:

components/esm4/loadcell.cpp

@@ -37,7 +37,8 @@
 #include <iostream> // FIXME: debug only
 
 #include "reader.hpp"
-//#include "writer.hpp"
+#include <components/esm/refid.hpp>
+// #include "writer.hpp"
 
 // TODO: Try loading only EDID and XCLC (along with mFormId, mFlags and mParent)
 //
@@ -48,8 +49,9 @@
 // longer/shorter/same as loading the subrecords.
 void ESM4::Cell::load(ESM4::Reader& reader)
 {
-    mFormId = reader.hdr().record.id;
+    auto formId = reader.hdr().record.id;
-    reader.adjustFormId(mFormId);
+    reader.adjustFormId(formId);
+    mId = ESM::RefId::formIdRefId(formId);
     mFlags = reader.hdr().record.flags;
     mParent = reader.currWorld();
 
@@ -71,7 +73,7 @@ void ESM4::Cell::load(ESM4::Reader& reader)
     // WARN: we need to call setCurrCell (and maybe setCurrCellGrid?) again before loading
     // cell child groups if we are loading them after restoring the context
     // (may be easier to update the context before saving?)
-    reader.setCurrCell(mFormId); // save for LAND (and other children) to access later
+    reader.setCurrCell(formId); // save for LAND (and other children) to access later
     std::uint32_t esmVer = reader.esmVersion();
     bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134;
 

components/esm4/loadcell.hpp

@@ -33,6 +33,8 @@
 
 #include "formid.hpp"
 #include "lighting.hpp"
+#include <components/esm/defs.hpp>
+#include <components/esm/refid.hpp>
 
 namespace ESM4
 {
@@ -61,7 +63,7 @@ namespace ESM4
     {
         FormId mParent; // world formId (for grouping cells), from the loading sequence
 
-        FormId mFormId; // from the header
+        ESM::RefId mId; // from the header
         std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
 
         std::string mEditorId;
@@ -95,6 +97,8 @@ namespace ESM4
         // void save(ESM4::Writer& writer) const;
 
         void blank();
+
+        static constexpr ESM::RecNameInts sRecordId = ESM::REC_CELL4;
     };
 }
 

components/esm4/loadrefr.cpp

@@ -30,14 +30,15 @@
 #include <stdexcept>
 
 #include "reader.hpp"
-//#include "writer.hpp"
+// #include "writer.hpp"
 
 void ESM4::Reference::load(ESM4::Reader& reader)
 {
-    mFormId = reader.hdr().record.id;
+    auto formId = reader.hdr().record.id;
-    reader.adjustFormId(mFormId);
+    reader.adjustFormId(formId);
+    mId = ESM::RefId::formIdRefId(formId);
     mFlags = reader.hdr().record.flags;
-    mParent = reader.currCell(); // NOTE: only for persistent refs?
+    mParent = ESM::RefId::formIdRefId(reader.currCell()); // NOTE: only for persistent refs?
 
     // TODO: Let the engine apply this? Saved games?
     // mInitiallyDisabled = ((mFlags & ESM4::Rec_Disabled) != 0) ? true : false;
@@ -60,7 +61,9 @@ void ESM4::Reference::load(ESM4::Reader& reader)
                 break;
             case ESM4::SUB_NAME:
             {
-                reader.getFormId(mBaseObj);
+                FormId BaseId;
+                reader.getFormId(BaseId);
+                mBaseObj = ESM::RefId::formIdRefId(BaseId);
 #if 0
                 if (mFlags & ESM4::Rec_Disabled)
                     std::cout << "REFR disable at start " << formIdToString(mFormId) <<

components/esm4/loadrefr.hpp

@@ -30,6 +30,8 @@
 #include <cstdint>
 
 #include "reference.hpp" // FormId, Placement, EnableParent
+#include <components/esm/defs.hpp>
+#include <components/esm/refid.hpp>
 
 namespace ESM4
 {
@@ -71,15 +73,15 @@ namespace ESM4
 
     struct Reference
     {
-        FormId mParent; // cell FormId (currently persistent refs only), from the loading sequence
+        ESM::RefId mParent; // cell FormId (currently persistent refs only), from the loading sequence
-                        // NOTE: for exterior cells it will be the dummy cell FormId
+                            // NOTE: for exterior cells it will be the dummy cell FormId
 
-        FormId mFormId; // from the header
+        ESM::RefId mId; // from the header
         std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
 
         std::string mEditorId;
         std::string mFullName;
-        FormId mBaseObj;
+        ESM::RefId mBaseObj;
 
         Placement mPlacement;
         float mScale = 1.0f;
@@ -110,6 +112,8 @@ namespace ESM4
         // void save(ESM4::Writer& writer) const;
 
         void blank();
+
+        static constexpr ESM::RecNameInts sRecordId = ESM::REC_REFR4;
     };
 }
 

components/esm4/loadstat.cpp

@@ -30,12 +30,13 @@
 #include <stdexcept>
 
 #include "reader.hpp"
-//#include "writer.hpp"
+// #include "writer.hpp"
 
 void ESM4::Static::load(ESM4::Reader& reader)
 {
-    mFormId = reader.hdr().record.id;
+    FormId formId = reader.hdr().record.id;
-    reader.adjustFormId(mFormId);
+    reader.adjustFormId(formId);
+    mId = ESM::RefId::formIdRefId(formId);
     mFlags = reader.hdr().record.flags;
 
     while (reader.getSubRecordHeader())

components/esm4/loadstat.hpp

@@ -32,6 +32,8 @@
 #include <vector>
 
 #include "formid.hpp"
+#include <components/esm/defs.hpp>
+#include <components/esm/refid.hpp>
 
 namespace ESM4
 {
@@ -40,7 +42,7 @@ namespace ESM4
 
     struct Static
     {
-        FormId mFormId; // from the header
+        ESM::RefId mId; // from the header
         std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
 
         std::string mEditorId;
@@ -53,6 +55,8 @@ namespace ESM4
         // void save(ESM4::Writer& writer) const;
 
         // void blank();
+
+        static constexpr ESM::RecNameInts sRecordId = ESM::REC_STAT4;
     };
 }
 

components/to_utf8/to_utf8.hpp

@@ -68,6 +68,8 @@ namespace ToUTF8
         /// ASCII-only string. Otherwise returns a view to the input.
         std::string_view getLegacyEnc(std::string_view input);
 
+        const StatelessUtf8Encoder* getStatelessEncoder() const { return &mImpl; }
+
     private:
         std::string mBuffer;
         StatelessUtf8Encoder mImpl;