The esm4 reader logic is mutualised

to avoid copy pasting code, readerutils gives functions that take visitors as params to decide how a record must be handled

Check encoder exists, and get value of stateless encoder.

fixes code formatting conventions

Fixed output of record with RefId

also fixed readTypedRecord and readRecord to have the proper return types

Check if the type has a sRecordId

apps/esmtool/tes4.cpp

@@ -8,6 +8,7 @@
 
 #include <components/esm/esmcommon.hpp>
 #include <components/esm4/reader.hpp>
+#include <components/esm4/readerutils.hpp>
 #include <components/esm4/records.hpp>
 #include <components/to_utf8/to_utf8.hpp>
 
@@ -69,6 +70,19 @@ namespace EsmTool
         template <class T>
         constexpr bool hasFormId = HasFormId<T>::value;
 
+        template <class T, class = std::void_t<>>
+        struct HasRefId : std::false_type
+        {
+        };
+
+        template <class T>
+        struct HasRefId<T, std::void_t<decltype(T::mId)>> : std::true_type
+        {
+        };
+
+        template <class T>
+        constexpr bool hasRefId = HasRefId<T>::value;
+
         template <class T, class = std::void_t<>>
         struct HasFlags : std::false_type
         {
@@ -169,6 +183,8 @@ namespace EsmTool
             std::cout << "\n  Record: " << ESM::NAME(reader.hdr().record.typeId).toStringView();
             if constexpr (hasFormId<T>)
                 std::cout << "\n  FormId: " << value.mFormId;
+            if constexpr (hasRefId<T>)
+                std::cout << "\n  RefId: " << value.mId;
             if constexpr (hasFlags<T>)
                 std::cout << "\n  Record flags: " << recordFlags(value.mFlags);
             if constexpr (hasEditorId<T>)
@@ -180,62 +196,80 @@ namespace EsmTool
             if constexpr (hasKf<T>)
                 std::cout << "\n  Kf:" << WriteArray("\n  - ", value.mKf);
             std::cout << '\n';
+            return;
         }
 
-        void readRecord(const Params& params, ESM4::Reader& reader)
+        bool readRecord(const Params& params, ESM4::Reader& reader)
         {
             switch (static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId))
             {
                 case ESM4::REC_AACT:
                     break;
                 case ESM4::REC_ACHR:
-                    return readTypedRecord<ESM4::ActorCharacter>(params, reader);
+                    readTypedRecord<ESM4::ActorCharacter>(params, reader);
+                    return true;
                 case ESM4::REC_ACRE:
-                    return readTypedRecord<ESM4::ActorCreature>(params, reader);
+                    readTypedRecord<ESM4::ActorCreature>(params, reader);
+                    return true;
                 case ESM4::REC_ACTI:
-                    return readTypedRecord<ESM4::Activator>(params, reader);
+                    readTypedRecord<ESM4::Activator>(params, reader);
+                    return true;
                 case ESM4::REC_ADDN:
                     break;
                 case ESM4::REC_ALCH:
-                    return readTypedRecord<ESM4::Potion>(params, reader);
+                    readTypedRecord<ESM4::Potion>(params, reader);
+                    return true;
                 case ESM4::REC_ALOC:
-                    return readTypedRecord<ESM4::MediaLocationController>(params, reader);
+                    readTypedRecord<ESM4::MediaLocationController>(params, reader);
+                    return true;
                 case ESM4::REC_AMMO:
-                    return readTypedRecord<ESM4::Ammunition>(params, reader);
+                    readTypedRecord<ESM4::Ammunition>(params, reader);
+                    return true;
                 case ESM4::REC_ANIO:
-                    return readTypedRecord<ESM4::AnimObject>(params, reader);
+                    readTypedRecord<ESM4::AnimObject>(params, reader);
+                    return true;
                 case ESM4::REC_APPA:
-                    return readTypedRecord<ESM4::Apparatus>(params, reader);
+                    readTypedRecord<ESM4::Apparatus>(params, reader);
+                    return true;
                 case ESM4::REC_ARMA:
-                    return readTypedRecord<ESM4::ArmorAddon>(params, reader);
+                    readTypedRecord<ESM4::ArmorAddon>(params, reader);
+                    return true;
                 case ESM4::REC_ARMO:
-                    return readTypedRecord<ESM4::Armor>(params, reader);
+                    readTypedRecord<ESM4::Armor>(params, reader);
+                    return true;
                 case ESM4::REC_ARTO:
                     break;
                 case ESM4::REC_ASPC:
-                    return readTypedRecord<ESM4::AcousticSpace>(params, reader);
+                    readTypedRecord<ESM4::AcousticSpace>(params, reader);
+                    return true;
                 case ESM4::REC_ASTP:
                     break;
                 case ESM4::REC_AVIF:
                     break;
                 case ESM4::REC_BOOK:
-                    return readTypedRecord<ESM4::Book>(params, reader);
+                    readTypedRecord<ESM4::Book>(params, reader);
+                    return true;
                 case ESM4::REC_BPTD:
-                    return readTypedRecord<ESM4::BodyPartData>(params, reader);
+                    readTypedRecord<ESM4::BodyPartData>(params, reader);
+                    return true;
                 case ESM4::REC_CAMS:
                     break;
                 case ESM4::REC_CCRD:
                     break;
                 case ESM4::REC_CELL:
-                    return readTypedRecord<ESM4::Cell>(params, reader);
+                    readTypedRecord<ESM4::Cell>(params, reader);
+                    return true;
                 case ESM4::REC_CLAS:
-                    return readTypedRecord<ESM4::Class>(params, reader);
+                    readTypedRecord<ESM4::Class>(params, reader);
+                    return true;
                 case ESM4::REC_CLFM:
-                    return readTypedRecord<ESM4::Colour>(params, reader);
+                    readTypedRecord<ESM4::Colour>(params, reader);
+                    return true;
                 case ESM4::REC_CLMT:
                     break;
                 case ESM4::REC_CLOT:
-                    return readTypedRecord<ESM4::Clothing>(params, reader);
+                    readTypedRecord<ESM4::Clothing>(params, reader);
+                    return true;
                 case ESM4::REC_CMNY:
                     break;
                 case ESM4::REC_COBJ:
@@ -243,25 +277,30 @@ namespace EsmTool
                 case ESM4::REC_COLL:
                     break;
                 case ESM4::REC_CONT:
-                    return readTypedRecord<ESM4::Container>(params, reader);
+                    readTypedRecord<ESM4::Container>(params, reader);
+                    return true;
                 case ESM4::REC_CPTH:
                     break;
                 case ESM4::REC_CREA:
-                    return readTypedRecord<ESM4::Creature>(params, reader);
+                    readTypedRecord<ESM4::Creature>(params, reader);
+                    return true;
                 case ESM4::REC_CSTY:
                     break;
                 case ESM4::REC_DEBR:
                     break;
                 case ESM4::REC_DIAL:
-                    return readTypedRecord<ESM4::Dialogue>(params, reader);
+                    readTypedRecord<ESM4::Dialogue>(params, reader);
+                    return true;
                 case ESM4::REC_DLBR:
                     break;
                 case ESM4::REC_DLVW:
                     break;
                 case ESM4::REC_DOBJ:
-                    return readTypedRecord<ESM4::DefaultObj>(params, reader);
+                    readTypedRecord<ESM4::DefaultObj>(params, reader);
+                    return true;
                 case ESM4::REC_DOOR:
-                    return readTypedRecord<ESM4::Door>(params, reader);
+                    readTypedRecord<ESM4::Door>(params, reader);
+                    return true;
                 case ESM4::REC_DUAL:
                     break;
                 case ESM4::REC_ECZN:
@@ -275,81 +314,103 @@ namespace EsmTool
                 case ESM4::REC_EXPL:
                     break;
                 case ESM4::REC_EYES:
-                    return readTypedRecord<ESM4::Eyes>(params, reader);
+                    readTypedRecord<ESM4::Eyes>(params, reader);
+                    return true;
                 case ESM4::REC_FACT:
                     break;
                 case ESM4::REC_FLOR:
-                    return readTypedRecord<ESM4::Flora>(params, reader);
+                    readTypedRecord<ESM4::Flora>(params, reader);
+                    return true;
                 case ESM4::REC_FLST:
-                    return readTypedRecord<ESM4::FormIdList>(params, reader);
+                    readTypedRecord<ESM4::FormIdList>(params, reader);
+                    return true;
                 case ESM4::REC_FSTP:
                     break;
                 case ESM4::REC_FSTS:
                     break;
                 case ESM4::REC_FURN:
-                    return readTypedRecord<ESM4::Furniture>(params, reader);
+                    readTypedRecord<ESM4::Furniture>(params, reader);
+                    return true;
                 case ESM4::REC_GLOB:
-                    return readTypedRecord<ESM4::GlobalVariable>(params, reader);
+                    readTypedRecord<ESM4::GlobalVariable>(params, reader);
+                    return true;
                 case ESM4::REC_GMST:
                     break;
                 case ESM4::REC_GRAS:
-                    return readTypedRecord<ESM4::Grass>(params, reader);
+                    readTypedRecord<ESM4::Grass>(params, reader);
+                    return true;
                 case ESM4::REC_GRUP:
                     break;
                 case ESM4::REC_HAIR:
-                    return readTypedRecord<ESM4::Hair>(params, reader);
+                    readTypedRecord<ESM4::Hair>(params, reader);
+                    return true;
                 case ESM4::REC_HAZD:
                     break;
                 case ESM4::REC_HDPT:
-                    return readTypedRecord<ESM4::HeadPart>(params, reader);
+                    readTypedRecord<ESM4::HeadPart>(params, reader);
+                    return true;
                 case ESM4::REC_IDLE:
                     // FIXME: ESM4::IdleAnimation::load does not work with Oblivion.esm
-                    // return readTypedRecord<ESM4::IdleAnimation>(params, reader);
+                    // readTypedRecord<ESM4::IdleAnimation>(params, reader);
+                    return true;
                     break;
                 case ESM4::REC_IDLM:
-                    return readTypedRecord<ESM4::IdleMarker>(params, reader);
+                    readTypedRecord<ESM4::IdleMarker>(params, reader);
+                    return true;
                 case ESM4::REC_IMAD:
                     break;
                 case ESM4::REC_IMGS:
                     break;
                 case ESM4::REC_IMOD:
-                    return readTypedRecord<ESM4::ItemMod>(params, reader);
+                    readTypedRecord<ESM4::ItemMod>(params, reader);
+                    return true;
                 case ESM4::REC_INFO:
-                    return readTypedRecord<ESM4::DialogInfo>(params, reader);
+                    readTypedRecord<ESM4::DialogInfo>(params, reader);
+                    return true;
                 case ESM4::REC_INGR:
-                    return readTypedRecord<ESM4::Ingredient>(params, reader);
+                    readTypedRecord<ESM4::Ingredient>(params, reader);
+                    return true;
                 case ESM4::REC_IPCT:
                     break;
                 case ESM4::REC_IPDS:
                     break;
                 case ESM4::REC_KEYM:
-                    return readTypedRecord<ESM4::Key>(params, reader);
+                    readTypedRecord<ESM4::Key>(params, reader);
+                    return true;
                 case ESM4::REC_KYWD:
                     break;
                 case ESM4::REC_LAND:
-                    return readTypedRecord<ESM4::Land>(params, reader);
+                    readTypedRecord<ESM4::Land>(params, reader);
+                    return true;
                 case ESM4::REC_LCRT:
                     break;
                 case ESM4::REC_LCTN:
                     break;
                 case ESM4::REC_LGTM:
-                    return readTypedRecord<ESM4::LightingTemplate>(params, reader);
+                    readTypedRecord<ESM4::LightingTemplate>(params, reader);
+                    return true;
                 case ESM4::REC_LIGH:
-                    return readTypedRecord<ESM4::Light>(params, reader);
+                    readTypedRecord<ESM4::Light>(params, reader);
+                    return true;
                 case ESM4::REC_LSCR:
                     break;
                 case ESM4::REC_LTEX:
-                    return readTypedRecord<ESM4::LandTexture>(params, reader);
+                    readTypedRecord<ESM4::LandTexture>(params, reader);
+                    return true;
                 case ESM4::REC_LVLC:
-                    return readTypedRecord<ESM4::LevelledCreature>(params, reader);
+                    readTypedRecord<ESM4::LevelledCreature>(params, reader);
+                    return true;
                 case ESM4::REC_LVLI:
-                    return readTypedRecord<ESM4::LevelledItem>(params, reader);
+                    readTypedRecord<ESM4::LevelledItem>(params, reader);
+                    return true;
                 case ESM4::REC_LVLN:
-                    return readTypedRecord<ESM4::LevelledNpc>(params, reader);
+                    readTypedRecord<ESM4::LevelledNpc>(params, reader);
+                    return true;
                 case ESM4::REC_LVSP:
                     break;
                 case ESM4::REC_MATO:
-                    return readTypedRecord<ESM4::Material>(params, reader);
+                    readTypedRecord<ESM4::Material>(params, reader);
+                    return true;
                 case ESM4::REC_MATT:
                     break;
                 case ESM4::REC_MESG:
@@ -357,49 +418,66 @@ namespace EsmTool
                 case ESM4::REC_MGEF:
                     break;
                 case ESM4::REC_MISC:
-                    return readTypedRecord<ESM4::MiscItem>(params, reader);
+                    readTypedRecord<ESM4::MiscItem>(params, reader);
+                    return true;
                 case ESM4::REC_MOVT:
                     break;
                 case ESM4::REC_MSET:
-                    return readTypedRecord<ESM4::MediaSet>(params, reader);
+                    readTypedRecord<ESM4::MediaSet>(params, reader);
+                    return true;
                 case ESM4::REC_MSTT:
-                    return readTypedRecord<ESM4::MovableStatic>(params, reader);
+                    readTypedRecord<ESM4::MovableStatic>(params, reader);
+                    return true;
                 case ESM4::REC_MUSC:
-                    return readTypedRecord<ESM4::Music>(params, reader);
+                    readTypedRecord<ESM4::Music>(params, reader);
+                    return true;
                 case ESM4::REC_MUST:
                     break;
                 case ESM4::REC_NAVI:
-                    return readTypedRecord<ESM4::Navigation>(params, reader);
+                    readTypedRecord<ESM4::Navigation>(params, reader);
+                    return true;
                 case ESM4::REC_NAVM:
-                    return readTypedRecord<ESM4::NavMesh>(params, reader);
+                    readTypedRecord<ESM4::NavMesh>(params, reader);
+                    return true;
                 case ESM4::REC_NOTE:
-                    return readTypedRecord<ESM4::Note>(params, reader);
+                    readTypedRecord<ESM4::Note>(params, reader);
+                    return true;
                 case ESM4::REC_NPC_:
-                    return readTypedRecord<ESM4::Npc>(params, reader);
+                    readTypedRecord<ESM4::Npc>(params, reader);
+                    return true;
                 case ESM4::REC_OTFT:
-                    return readTypedRecord<ESM4::Outfit>(params, reader);
+                    readTypedRecord<ESM4::Outfit>(params, reader);
+                    return true;
                 case ESM4::REC_PACK:
-                    return readTypedRecord<ESM4::AIPackage>(params, reader);
+                    readTypedRecord<ESM4::AIPackage>(params, reader);
+                    return true;
                 case ESM4::REC_PERK:
                     break;
                 case ESM4::REC_PGRD:
-                    return readTypedRecord<ESM4::Pathgrid>(params, reader);
+                    readTypedRecord<ESM4::Pathgrid>(params, reader);
+                    return true;
                 case ESM4::REC_PGRE:
-                    return readTypedRecord<ESM4::PlacedGrenade>(params, reader);
+                    readTypedRecord<ESM4::PlacedGrenade>(params, reader);
+                    return true;
                 case ESM4::REC_PHZD:
                     break;
                 case ESM4::REC_PROJ:
                     break;
                 case ESM4::REC_PWAT:
-                    return readTypedRecord<ESM4::PlaceableWater>(params, reader);
+                    readTypedRecord<ESM4::PlaceableWater>(params, reader);
+                    return true;
                 case ESM4::REC_QUST:
-                    return readTypedRecord<ESM4::Quest>(params, reader);
+                    readTypedRecord<ESM4::Quest>(params, reader);
+                    return true;
                 case ESM4::REC_RACE:
-                    return readTypedRecord<ESM4::Race>(params, reader);
+                    readTypedRecord<ESM4::Race>(params, reader);
+                    return true;
                 case ESM4::REC_REFR:
-                    return readTypedRecord<ESM4::Reference>(params, reader);
+                    readTypedRecord<ESM4::Reference>(params, reader);
+                    return true;
                 case ESM4::REC_REGN:
-                    return readTypedRecord<ESM4::Region>(params, reader);
+                    readTypedRecord<ESM4::Region>(params, reader);
+                    return true;
                 case ESM4::REC_RELA:
                     break;
                 case ESM4::REC_REVB:
@@ -407,23 +485,30 @@ namespace EsmTool
                 case ESM4::REC_RFCT:
                     break;
                 case ESM4::REC_ROAD:
-                    return readTypedRecord<ESM4::Road>(params, reader);
+                    readTypedRecord<ESM4::Road>(params, reader);
+                    return true;
                 case ESM4::REC_SBSP:
-                    return readTypedRecord<ESM4::SubSpace>(params, reader);
+                    readTypedRecord<ESM4::SubSpace>(params, reader);
+                    return true;
                 case ESM4::REC_SCEN:
                     break;
                 case ESM4::REC_SCOL:
-                    return readTypedRecord<ESM4::StaticCollection>(params, reader);
+                    readTypedRecord<ESM4::StaticCollection>(params, reader);
+                    return true;
                 case ESM4::REC_SCPT:
-                    return readTypedRecord<ESM4::Script>(params, reader);
+                    readTypedRecord<ESM4::Script>(params, reader);
+                    return true;
                 case ESM4::REC_SCRL:
-                    return readTypedRecord<ESM4::Scroll>(params, reader);
+                    readTypedRecord<ESM4::Scroll>(params, reader);
+                    return true;
                 case ESM4::REC_SGST:
-                    return readTypedRecord<ESM4::SigilStone>(params, reader);
+                    readTypedRecord<ESM4::SigilStone>(params, reader);
+                    return true;
                 case ESM4::REC_SHOU:
                     break;
                 case ESM4::REC_SLGM:
-                    return readTypedRecord<ESM4::SoulGem>(params, reader);
+                    readTypedRecord<ESM4::SoulGem>(params, reader);
+                    return true;
                 case ESM4::REC_SMBN:
                     break;
                 case ESM4::REC_SMEN:
@@ -433,97 +518,56 @@ namespace EsmTool
                 case ESM4::REC_SNCT:
                     break;
                 case ESM4::REC_SNDR:
-                    return readTypedRecord<ESM4::SoundReference>(params, reader);
+                    readTypedRecord<ESM4::SoundReference>(params, reader);
+                    return true;
                 case ESM4::REC_SOPM:
                     break;
                 case ESM4::REC_SOUN:
-                    return readTypedRecord<ESM4::Sound>(params, reader);
+                    readTypedRecord<ESM4::Sound>(params, reader);
+                    return true;
                 case ESM4::REC_SPEL:
                     break;
                 case ESM4::REC_SPGD:
                     break;
                 case ESM4::REC_STAT:
-                    return readTypedRecord<ESM4::Static>(params, reader);
+                    readTypedRecord<ESM4::Static>(params, reader);
+                    return true;
                 case ESM4::REC_TACT:
-                    return readTypedRecord<ESM4::TalkingActivator>(params, reader);
+                    readTypedRecord<ESM4::TalkingActivator>(params, reader);
+                    return true;
                 case ESM4::REC_TERM:
-                    return readTypedRecord<ESM4::Terminal>(params, reader);
+                    readTypedRecord<ESM4::Terminal>(params, reader);
+                    return true;
                 case ESM4::REC_TES4:
-                    return readTypedRecord<ESM4::Header>(params, reader);
+                    readTypedRecord<ESM4::Header>(params, reader);
+                    return true;
                 case ESM4::REC_TREE:
-                    return readTypedRecord<ESM4::Tree>(params, reader);
+                    readTypedRecord<ESM4::Tree>(params, reader);
+                    return true;
                 case ESM4::REC_TXST:
-                    return readTypedRecord<ESM4::TextureSet>(params, reader);
+                    readTypedRecord<ESM4::TextureSet>(params, reader);
+                    return true;
                 case ESM4::REC_VTYP:
                     break;
                 case ESM4::REC_WATR:
                     break;
                 case ESM4::REC_WEAP:
-                    return readTypedRecord<ESM4::Weapon>(params, reader);
+                    readTypedRecord<ESM4::Weapon>(params, reader);
+                    return true;
                 case ESM4::REC_WOOP:
                     break;
                 case ESM4::REC_WRLD:
-                    return readTypedRecord<ESM4::World>(params, reader);
+                    readTypedRecord<ESM4::World>(params, reader);
+                    return true;
                 case ESM4::REC_WTHR:
                     break;
             }
 
             if (!params.mQuite)
                 std::cout << "\n  Unsupported record: " << ESM::NAME(reader.hdr().record.typeId).toStringView() << '\n';
-
-            reader.skipRecordData();
-        }
-
-        bool readItem(const Params& params, ESM4::Reader& reader);
-
-        bool readGroup(const Params& params, ESM4::Reader& reader)
-        {
-            const ESM4::RecordHeader& header = reader.hdr();
-
-            if (!params.mQuite)
-                std::cout << "\nGroup: " << toString(static_cast<ESM4::GroupType>(header.group.type)) << " "
-                          << ESM::NAME(header.group.typeId).toStringView() << '\n';
-
-            switch (static_cast<ESM4::GroupType>(header.group.type))
-            {
-                case ESM4::Grp_RecordType:
-                case ESM4::Grp_InteriorCell:
-                case ESM4::Grp_InteriorSubCell:
-                case ESM4::Grp_ExteriorCell:
-                case ESM4::Grp_ExteriorSubCell:
-                    reader.enterGroup();
-                    return readItem(params, reader);
-                case ESM4::Grp_WorldChild:
-                case ESM4::Grp_CellChild:
-                case ESM4::Grp_TopicChild:
-                case ESM4::Grp_CellPersistentChild:
-                case ESM4::Grp_CellTemporaryChild:
-                case ESM4::Grp_CellVisibleDistChild:
-                    reader.adjustGRUPFormId();
-                    reader.enterGroup();
-                    if (!reader.hasMoreRecs())
             return false;
-                    return readItem(params, reader);
         }
 
-            reader.skipGroup();
-
-            return true;
-        }
-
-        bool readItem(const Params& params, ESM4::Reader& reader)
-        {
-            if (!reader.getRecordHeader() || !reader.hasMoreRecs())
-                return false;
-
-            const ESM4::RecordHeader& header = reader.hdr();
-
-            if (header.record.typeId == ESM4::REC_GRUP)
-                return readGroup(params, reader);
-
-            readRecord(params, reader);
-            return true;
-        }
     }
 
     int loadTes4(const Arguments& info, std::unique_ptr<std::ifstream>&& stream)
@@ -551,12 +595,15 @@ namespace EsmTool
                 }
             }
 
-            while (reader.hasMoreRecs())
+            auto visitorRec = [&params](ESM4::Reader& reader) { return readRecord(params, reader); };
-            {
+            auto visistorGroup = [&params](ESM4::Reader& reader) {
-                reader.exitGroupCheck();
+                if (params.mQuite)
-                if (!readItem(params, reader))
+                    return;
-                    break;
+                auto groupType = static_cast<ESM4::GroupType>(reader.hdr().group.type);
-            }
+                std::cout << "\nGroup: " << toString(groupType) << " "
+                          << ESM::NAME(reader.hdr().group.typeId).toStringView() << '\n';
+            };
+            ESM4::ReaderUtils::readAll(reader, visitorRec, visistorGroup);
         }
         catch (const std::exception& e)
         {

apps/openmw/mwworld/esmloader.cpp

@@ -1,13 +1,14 @@
 #include "esmloader.hpp"
 #include "esmstore.hpp"
 
+#include <fstream>
+
 #include <components/esm/format.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/readerscache.hpp>
 #include <components/esm4/reader.hpp>
 #include <components/files/conversion.hpp>
 #include <components/files/openfile.hpp>
-#include <fstream>
 
 namespace MWWorld
 {
@@ -30,7 +31,6 @@ namespace MWWorld
         if (!stream->is_open())
         {
             throw std::runtime_error(std::string("File Failed to open file: ") + std::strerror(errno) + "\n");
-            return;
         }
         const ESM::Format format = ESM::readFormat(*stream);
         stream->seekg(0);
@@ -63,12 +63,16 @@ namespace MWWorld
                 break;
             }
             case ESM::Format::Tes4:
+            {
+                if (mEncoder)
                 {
                     ESM4::Reader readerESM4(std::move(stream), filepath);
-                readerESM4.setEncoder(mEncoder->getStatelessEncoder());
+                    auto statelessEncoder = mEncoder->getStatelessEncoder();
+                    readerESM4.setEncoder(&statelessEncoder);
                     mStore.loadESM4(readerESM4, listener, mDialogue);
                 }
             }
         }
+    }
 
 } /* namespace MWWorld */

apps/openmw/mwworld/esmstore.cpp

@@ -13,12 +13,12 @@
 #include <components/lua/configuration.hpp>
 #include <components/misc/algorithm.hpp>
 
+#include <components/esm4/common.hpp>
 #include <components/esm4/loadcell.hpp>
 #include <components/esm4/loadrefr.hpp>
 #include <components/esm4/loadstat.hpp>
 #include <components/esm4/reader.hpp>
-
+#include <components/esm4/readerutils.hpp>
-#include <components/esm4/common.hpp>
 #include <components/esmloader/load.hpp>
 
 #include "../mwmechanics/spelllist.hpp"
@@ -186,13 +186,23 @@ namespace MWWorld
             }
         }
 
+        template <class T, class = std::void_t<>>
+        struct HasRecordId : std::false_type
+        {
+        };
+
+        template <class T>
+        struct HasRecordId<T, std::void_t<decltype(T::sRecordId)>> : std::true_type
+        {
+        };
+
         template <typename T>
         static void typedReadRecordESM4(ESM4::Reader& reader, ESMStore& stores, Store<T>& store, int& found)
         {
             auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
 
             ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
-            if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*>)
+            if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*> && HasRecordId<T>::value)
             {
                 if constexpr (ESM::isESM4Rec(T::sRecordId))
                 {
@@ -208,60 +218,14 @@ namespace MWWorld
             }
         }
 
-        static void readRecord(ESM4::Reader& reader, ESMStore& store)
+        static bool readRecord(ESM4::Reader& reader, ESMStore& store)
         {
             int found = 0;
             std::apply([&reader, &store, &found](
                            auto&... x) { (ESMStoreImp::typedReadRecordESM4(reader, store, x, found), ...); },
                 store.mStoreImp->mStores);
             assert(found <= 1);
-            if (found == 0) // unhandled record
+            return found;
-                reader.skipRecordData();
-        }
-
-        static bool readItem(ESM4::Reader& reader, ESMStore& store)
-        {
-            if (!reader.getRecordHeader() || !reader.hasMoreRecs())
-                return false;
-
-            const ESM4::RecordHeader& header = reader.hdr();
-
-            if (header.record.typeId == ESM4::REC_GRUP)
-                return readGroup(reader, store);
-
-            readRecord(reader, store);
-            return true;
-        }
-
-        static bool readGroup(ESM4::Reader& reader, ESMStore& store)
-        {
-            const ESM4::RecordHeader& header = reader.hdr();
-
-            switch (static_cast<ESM4::GroupType>(header.group.type))
-            {
-                case ESM4::Grp_RecordType:
-                case ESM4::Grp_InteriorCell:
-                case ESM4::Grp_InteriorSubCell:
-                case ESM4::Grp_ExteriorCell:
-                case ESM4::Grp_ExteriorSubCell:
-                    reader.enterGroup();
-                    return readItem(reader, store);
-                case ESM4::Grp_WorldChild:
-                case ESM4::Grp_CellChild:
-                case ESM4::Grp_TopicChild:
-                case ESM4::Grp_CellPersistentChild:
-                case ESM4::Grp_CellTemporaryChild:
-                case ESM4::Grp_CellVisibleDistChild:
-                    reader.adjustGRUPFormId();
-                    reader.enterGroup();
-                    if (!reader.hasMoreRecs())
-                        return false;
-                    return readItem(reader, store);
-            }
-
-            reader.skipGroup();
-
-            return true;
         }
     };
 
@@ -423,12 +387,8 @@ namespace MWWorld
 
     void ESMStore::loadESM4(ESM4::Reader& reader, Loading::Listener* listener, ESM::Dialogue*& dialogue)
     {
-        while (reader.hasMoreRecs())
+        auto visitorRec = [this](ESM4::Reader& reader) { return ESMStoreImp::readRecord(reader, *this); };
-        {
+        ESM4::ReaderUtils::readAll(reader, visitorRec, [](ESM4::Reader&) {});
-            reader.exitGroupCheck();
-            if (!ESMStoreImp::readItem(reader, *this))
-                break;
-        }
     }
 
     void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type)

apps/openmw/mwworld/store.cpp

@@ -1,21 +1,18 @@
 #include "store.hpp"
 
-#include <components/debug/debuglog.hpp>
-
-#include <components/esm/records.hpp>
-#include <components/esm3/esmreader.hpp>
-#include <components/esm3/esmwriter.hpp>
-
-#include <components/loadinglistener/loadinglistener.hpp>
-#include <components/misc/rng.hpp>
-
 #include <iterator>
 #include <sstream>
 #include <stdexcept>
 
+#include <components/debug/debuglog.hpp>
+#include <components/esm/records.hpp>
+#include <components/esm3/esmreader.hpp>
+#include <components/esm3/esmwriter.hpp>
 #include <components/esm4/loadcell.hpp>
 #include <components/esm4/loadrefr.hpp>
 #include <components/esm4/loadstat.hpp>
+#include <components/loadinglistener/loadinglistener.hpp>
+#include <components/misc/rng.hpp>
 
 namespace
 {

components/CMakeLists.txt

@@ -4,6 +4,7 @@ project (Components)
 set(VERSION_IN_FILE "${OpenMW_SOURCE_DIR}/files/version.in")
 set(VERSION_FILE_PATH_BASE "${OpenMW_BINARY_DIR}")
 set(VERSION_FILE_PATH_RELATIVE resources/version)
+
 if(GIT_CHECKOUT)
     get_generator_is_multi_config(multi_config)
     add_custom_target(git-version
@@ -27,7 +28,6 @@ else (GIT_CHECKOUT)
 endif(GIT_CHECKOUT)
 
 # source files
-
 add_component_dir(lua
     luastate scriptscontainer utilpackage serialization configuration l10n storage
 )
@@ -191,6 +191,7 @@ add_component_dir (esm4
     loadweap
     loadwrld
     reader
+    readerutils
     reference
     script
 )
@@ -212,6 +213,7 @@ IF(NOT WIN32 AND NOT APPLE)
     add_definitions(-DGLOBAL_DATA_PATH="${GLOBAL_DATA_PATH}")
     add_definitions(-DGLOBAL_CONFIG_PATH="${GLOBAL_CONFIG_PATH}")
 ENDIF()
+
 add_component_dir(files
     linuxpath androidpath windowspath macospath fixedpath multidircollection collections configurationmanager
     constrainedfilestream memorystream hash configfileparser openfile constrainedfilestreambuf conversion
@@ -272,7 +274,6 @@ add_component_dir (lua_ui
     adapter text textedit window image container flex
 )
 
-
 if(WIN32)
     add_component_dir(crashcatcher
         windows_crashcatcher
@@ -464,6 +465,7 @@ if (OSG_STATIC AND CMAKE_SYSTEM_NAME MATCHES "Linux")
     find_package(X11 REQUIRED COMPONENTS Xinerama Xrandr)
     target_link_libraries(components ${CMAKE_DL_LIBS} X11::X11 X11::Xinerama X11::Xrandr)
     find_package(Fontconfig MODULE)
+
     if(Fontconfig_FOUND)
         target_link_libraries(components Fontconfig::Fontconfig)
     endif()
@@ -483,7 +485,6 @@ if (BUILD_WITH_CODE_COVERAGE)
     target_link_libraries(components gcov)
 endif()
 
-
 # Make the variable accessible for other subdirectories
 set(COMPONENT_FILES ${COMPONENT_FILES} PARENT_SCOPE)
 
@@ -492,8 +493,10 @@ target_compile_definitions(components PUBLIC BT_USE_DOUBLE_PRECISION)
 if(OSG_STATIC)
     unset(_osg_plugins_static_files)
     add_library(components_osg_plugins INTERFACE)
+
     foreach(_plugin ${USED_OSG_PLUGINS})
         string(TOUPPER ${_plugin} _plugin_uc)
+
         if(OPENMW_USE_SYSTEM_OSG)
             list(APPEND _osg_plugins_static_files ${${_plugin_uc}_LIBRARY})
         else()
@@ -502,6 +505,7 @@ if(OSG_STATIC)
             add_dependencies(components_osg_plugins ${${_plugin_uc}_LIBRARY})
         endif()
     endforeach()
+
     # We use --whole-archive because OSG plugins use registration.
     get_whole_archive_options(_opts ${_osg_plugins_static_files})
     target_link_options(components_osg_plugins INTERFACE ${_opts})

components/esm/refid.hpp

@@ -1,12 +1,13 @@
 #ifndef OPENMW_COMPONENTS_ESM_REFID_HPP
 #define OPENMW_COMPONENTS_ESM_REFID_HPP
 #include <compare>
-#include <components/esm4/formid.hpp>
 #include <functional>
 #include <iosfwd>
 #include <string>
 #include <string_view>
 
+#include <components/esm4/formid.hpp>
+
 namespace ESM
 {
     struct RefId

components/esm4/loadcell.cpp

@@ -32,11 +32,11 @@
 
 #include <cassert>
 #include <cfloat> // FLT_MAX for gcc
+#include <iostream> // FIXME: debug only
 #include <stdexcept>
 
-#include <iostream> // FIXME: debug only
-
 #include "reader.hpp"
+
 #include <components/esm/refid.hpp>
 // #include "writer.hpp"
 

components/esm4/loadcell.hpp

@@ -33,6 +33,7 @@
 
 #include "formid.hpp"
 #include "lighting.hpp"
+
 #include <components/esm/defs.hpp>
 #include <components/esm/refid.hpp>
 

components/esm4/loadrefr.hpp

@@ -30,6 +30,7 @@
 #include <cstdint>
 
 #include "reference.hpp" // FormId, Placement, EnableParent
+
 #include <components/esm/defs.hpp>
 #include <components/esm/refid.hpp>
 

components/esm4/loadstat.hpp

@@ -32,6 +32,7 @@
 #include <vector>
 
 #include "formid.hpp"
+
 #include <components/esm/defs.hpp>
 #include <components/esm/refid.hpp>
 

components/esm4/readerutils.hpp

@@ -0,0 +1,84 @@
+#ifndef OPENMW_COMPONENTS_ESM4_READERUTILS
+#define OPENMW_COMPONENTS_ESM4_READERUTILS
+
+#include <components/esm4/reader.hpp>
+namespace ESM4
+{
+    struct ReaderUtils
+    {
+
+        /* RecordInvocable must be an invocable, takes an ESM4::Reader& as input, and outputs a boolean that indicates
+        if the record was read or ignored. Will be invoked for every record
+
+        GroupInvocable's invocable must take a ESM4::Reader& as input, doesn't need to output anything. Will be invoked
+        for every group*/
+        template <typename RecordInvocable, typename GroupInvocable>
+        static void readAll(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
+        {
+            while (reader.hasMoreRecs())
+            {
+                reader.exitGroupCheck();
+                if (!readItem(reader, recordInvocable, groupInvocable))
+                    break;
+            }
+        }
+
+        template <typename RecordInvocable>
+        static void readRecord(ESM4::Reader& reader, RecordInvocable&& recordInvocable)
+        {
+            if (!recordInvocable(reader))
+                reader.skipRecordData();
+        }
+
+        template <typename RecordInvocable, typename GroupInvocable>
+        static bool readGroup(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
+        {
+            const ESM4::RecordHeader& header = reader.hdr();
+
+            groupInvocable(reader);
+
+            switch (static_cast<ESM4::GroupType>(header.group.type))
+            {
+                case ESM4::Grp_RecordType:
+                case ESM4::Grp_InteriorCell:
+                case ESM4::Grp_InteriorSubCell:
+                case ESM4::Grp_ExteriorCell:
+                case ESM4::Grp_ExteriorSubCell:
+                    reader.enterGroup();
+                    return readItem(reader, recordInvocable, groupInvocable);
+                case ESM4::Grp_WorldChild:
+                case ESM4::Grp_CellChild:
+                case ESM4::Grp_TopicChild:
+                case ESM4::Grp_CellPersistentChild:
+                case ESM4::Grp_CellTemporaryChild:
+                case ESM4::Grp_CellVisibleDistChild:
+                    reader.adjustGRUPFormId();
+                    reader.enterGroup();
+                    if (!reader.hasMoreRecs())
+                        return false;
+                    return readItem(reader, recordInvocable, groupInvocable);
+            }
+
+            reader.skipGroup();
+
+            return true;
+        }
+
+        template <typename RecordInvocable, typename GroupInvocable>
+        static bool readItem(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
+        {
+            if (!reader.getRecordHeader() || !reader.hasMoreRecs())
+                return false;
+
+            const ESM4::RecordHeader& header = reader.hdr();
+
+            if (header.record.typeId == ESM4::REC_GRUP)
+                return readGroup(reader, recordInvocable, groupInvocable);
+
+            readRecord(reader, recordInvocable);
+            return true;
+        }
+    };
+}
+
+#endif // !OPENMW_COMPONENTS_ESM4_READERUTILS

components/to_utf8/to_utf8.hpp

@@ -68,7 +68,7 @@ namespace ToUTF8
         /// ASCII-only string. Otherwise returns a view to the input.
         std::string_view getLegacyEnc(std::string_view input);
 
-        const StatelessUtf8Encoder* getStatelessEncoder() const { return &mImpl; }
+        const StatelessUtf8Encoder getStatelessEncoder() const { return mImpl; }
 
     private:
         std::string mBuffer;