Merge branch 'Load_ESM4' into 'master'

Loading ESM4 data and storing them in the ESMStore

See merge request OpenMW/openmw!2557

apps/esmtool/tes4.cpp

@@ -8,6 +8,7 @@
 
 #include <components/esm/esmcommon.hpp>
 #include <components/esm4/reader.hpp>
+#include <components/esm4/readerutils.hpp>
 #include <components/esm4/records.hpp>
 #include <components/to_utf8/to_utf8.hpp>
 
@@ -69,6 +70,19 @@ namespace EsmTool
         template <class T>
         constexpr bool hasFormId = HasFormId<T>::value;
 
+        template <class T, class = std::void_t<>>
+        struct HasRefId : std::false_type
+        {
+        };
+
+        template <class T>
+        struct HasRefId<T, std::void_t<decltype(T::mId)>> : std::true_type
+        {
+        };
+
+        template <class T>
+        constexpr bool hasRefId = HasRefId<T>::value;
+
         template <class T, class = std::void_t<>>
         struct HasFlags : std::false_type
         {
@@ -169,6 +183,8 @@ namespace EsmTool
             std::cout << "\n  Record: " << ESM::NAME(reader.hdr().record.typeId).toStringView();
             if constexpr (hasFormId<T>)
                 std::cout << "\n  FormId: " << value.mFormId;
+            if constexpr (hasRefId<T>)
+                std::cout << "\n  FormId: " << value.mId;
             if constexpr (hasFlags<T>)
                 std::cout << "\n  Record flags: " << recordFlags(value.mFlags);
             if constexpr (hasEditorId<T>)
@@ -182,60 +198,77 @@ namespace EsmTool
             std::cout << '\n';
         }
 
-        void readRecord(const Params& params, ESM4::Reader& reader)
+        bool readRecord(const Params& params, ESM4::Reader& reader)
         {
             switch (static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId))
             {
                 case ESM4::REC_AACT:
                     break;
                 case ESM4::REC_ACHR:
-                    return readTypedRecord<ESM4::ActorCharacter>(params, reader);
+                    readTypedRecord<ESM4::ActorCharacter>(params, reader);
+                    return true;
                 case ESM4::REC_ACRE:
-                    return readTypedRecord<ESM4::ActorCreature>(params, reader);
+                    readTypedRecord<ESM4::ActorCreature>(params, reader);
+                    return true;
                 case ESM4::REC_ACTI:
-                    return readTypedRecord<ESM4::Activator>(params, reader);
+                    readTypedRecord<ESM4::Activator>(params, reader);
+                    return true;
                 case ESM4::REC_ADDN:
                     break;
                 case ESM4::REC_ALCH:
-                    return readTypedRecord<ESM4::Potion>(params, reader);
+                    readTypedRecord<ESM4::Potion>(params, reader);
+                    return true;
                 case ESM4::REC_ALOC:
-                    return readTypedRecord<ESM4::MediaLocationController>(params, reader);
+                    readTypedRecord<ESM4::MediaLocationController>(params, reader);
+                    return true;
                 case ESM4::REC_AMMO:
-                    return readTypedRecord<ESM4::Ammunition>(params, reader);
+                    readTypedRecord<ESM4::Ammunition>(params, reader);
+                    return true;
                 case ESM4::REC_ANIO:
-                    return readTypedRecord<ESM4::AnimObject>(params, reader);
+                    readTypedRecord<ESM4::AnimObject>(params, reader);
+                    return true;
                 case ESM4::REC_APPA:
-                    return readTypedRecord<ESM4::Apparatus>(params, reader);
+                    readTypedRecord<ESM4::Apparatus>(params, reader);
+                    return true;
                 case ESM4::REC_ARMA:
-                    return readTypedRecord<ESM4::ArmorAddon>(params, reader);
+                    readTypedRecord<ESM4::ArmorAddon>(params, reader);
+                    return true;
                 case ESM4::REC_ARMO:
-                    return readTypedRecord<ESM4::Armor>(params, reader);
+                    readTypedRecord<ESM4::Armor>(params, reader);
+                    return true;
                 case ESM4::REC_ARTO:
                     break;
                 case ESM4::REC_ASPC:
-                    return readTypedRecord<ESM4::AcousticSpace>(params, reader);
+                    readTypedRecord<ESM4::AcousticSpace>(params, reader);
+                    return true;
                 case ESM4::REC_ASTP:
                     break;
                 case ESM4::REC_AVIF:
                     break;
                 case ESM4::REC_BOOK:
-                    return readTypedRecord<ESM4::Book>(params, reader);
+                    readTypedRecord<ESM4::Book>(params, reader);
+                    return true;
                 case ESM4::REC_BPTD:
-                    return readTypedRecord<ESM4::BodyPartData>(params, reader);
+                    readTypedRecord<ESM4::BodyPartData>(params, reader);
+                    return true;
                 case ESM4::REC_CAMS:
                     break;
                 case ESM4::REC_CCRD:
                     break;
                 case ESM4::REC_CELL:
-                    return readTypedRecord<ESM4::Cell>(params, reader);
+                    readTypedRecord<ESM4::Cell>(params, reader);
+                    return true;
                 case ESM4::REC_CLAS:
-                    return readTypedRecord<ESM4::Class>(params, reader);
+                    readTypedRecord<ESM4::Class>(params, reader);
+                    return true;
                 case ESM4::REC_CLFM:
-                    return readTypedRecord<ESM4::Colour>(params, reader);
+                    readTypedRecord<ESM4::Colour>(params, reader);
+                    return true;
                 case ESM4::REC_CLMT:
                     break;
                 case ESM4::REC_CLOT:
-                    return readTypedRecord<ESM4::Clothing>(params, reader);
+                    readTypedRecord<ESM4::Clothing>(params, reader);
+                    return true;
                 case ESM4::REC_CMNY:
                     break;
                 case ESM4::REC_COBJ:
@@ -243,25 +276,30 @@ namespace EsmTool
                 case ESM4::REC_COLL:
                     break;
                 case ESM4::REC_CONT:
-                    return readTypedRecord<ESM4::Container>(params, reader);
+                    readTypedRecord<ESM4::Container>(params, reader);
+                    return true;
                 case ESM4::REC_CPTH:
                     break;
                 case ESM4::REC_CREA:
-                    return readTypedRecord<ESM4::Creature>(params, reader);
+                    readTypedRecord<ESM4::Creature>(params, reader);
+                    return true;
                 case ESM4::REC_CSTY:
                     break;
                 case ESM4::REC_DEBR:
                     break;
                 case ESM4::REC_DIAL:
-                    return readTypedRecord<ESM4::Dialogue>(params, reader);
+                    readTypedRecord<ESM4::Dialogue>(params, reader);
+                    return true;
                 case ESM4::REC_DLBR:
                     break;
                 case ESM4::REC_DLVW:
                     break;
                 case ESM4::REC_DOBJ:
-                    return readTypedRecord<ESM4::DefaultObj>(params, reader);
+                    readTypedRecord<ESM4::DefaultObj>(params, reader);
+                    return true;
                 case ESM4::REC_DOOR:
-                    return readTypedRecord<ESM4::Door>(params, reader);
+                    readTypedRecord<ESM4::Door>(params, reader);
+                    return true;
                 case ESM4::REC_DUAL:
                     break;
                 case ESM4::REC_ECZN:
@@ -275,81 +313,103 @@ namespace EsmTool
                 case ESM4::REC_EXPL:
                     break;
                 case ESM4::REC_EYES:
-                    return readTypedRecord<ESM4::Eyes>(params, reader);
+                    readTypedRecord<ESM4::Eyes>(params, reader);
+                    return true;
                 case ESM4::REC_FACT:
                     break;
                 case ESM4::REC_FLOR:
-                    return readTypedRecord<ESM4::Flora>(params, reader);
+                    readTypedRecord<ESM4::Flora>(params, reader);
+                    return true;
                 case ESM4::REC_FLST:
-                    return readTypedRecord<ESM4::FormIdList>(params, reader);
+                    readTypedRecord<ESM4::FormIdList>(params, reader);
+                    return true;
                 case ESM4::REC_FSTP:
                     break;
                 case ESM4::REC_FSTS:
                     break;
                 case ESM4::REC_FURN:
-                    return readTypedRecord<ESM4::Furniture>(params, reader);
+                    readTypedRecord<ESM4::Furniture>(params, reader);
+                    return true;
                 case ESM4::REC_GLOB:
-                    return readTypedRecord<ESM4::GlobalVariable>(params, reader);
+                    readTypedRecord<ESM4::GlobalVariable>(params, reader);
+                    return true;
                 case ESM4::REC_GMST:
                     break;
                 case ESM4::REC_GRAS:
-                    return readTypedRecord<ESM4::Grass>(params, reader);
+                    readTypedRecord<ESM4::Grass>(params, reader);
+                    return true;
                 case ESM4::REC_GRUP:
                     break;
                 case ESM4::REC_HAIR:
-                    return readTypedRecord<ESM4::Hair>(params, reader);
+                    readTypedRecord<ESM4::Hair>(params, reader);
+                    return true;
                 case ESM4::REC_HAZD:
                     break;
                 case ESM4::REC_HDPT:
-                    return readTypedRecord<ESM4::HeadPart>(params, reader);
+                    readTypedRecord<ESM4::HeadPart>(params, reader);
+                    return true;
                 case ESM4::REC_IDLE:
                     // FIXME: ESM4::IdleAnimation::load does not work with Oblivion.esm
-                    // return readTypedRecord<ESM4::IdleAnimation>(params, reader);
+                    // readTypedRecord<ESM4::IdleAnimation>(params, reader);
+                    return true;
                     break;
                 case ESM4::REC_IDLM:
-                    return readTypedRecord<ESM4::IdleMarker>(params, reader);
+                    readTypedRecord<ESM4::IdleMarker>(params, reader);
+                    return true;
                 case ESM4::REC_IMAD:
                     break;
                 case ESM4::REC_IMGS:
                     break;
                 case ESM4::REC_IMOD:
-                    return readTypedRecord<ESM4::ItemMod>(params, reader);
+                    readTypedRecord<ESM4::ItemMod>(params, reader);
+                    return true;
                 case ESM4::REC_INFO:
-                    return readTypedRecord<ESM4::DialogInfo>(params, reader);
+                    readTypedRecord<ESM4::DialogInfo>(params, reader);
+                    return true;
                 case ESM4::REC_INGR:
-                    return readTypedRecord<ESM4::Ingredient>(params, reader);
+                    readTypedRecord<ESM4::Ingredient>(params, reader);
+                    return true;
                 case ESM4::REC_IPCT:
                     break;
                 case ESM4::REC_IPDS:
                     break;
                 case ESM4::REC_KEYM:
-                    return readTypedRecord<ESM4::Key>(params, reader);
+                    readTypedRecord<ESM4::Key>(params, reader);
+                    return true;
                 case ESM4::REC_KYWD:
                     break;
                 case ESM4::REC_LAND:
-                    return readTypedRecord<ESM4::Land>(params, reader);
+                    readTypedRecord<ESM4::Land>(params, reader);
+                    return true;
                 case ESM4::REC_LCRT:
                     break;
                 case ESM4::REC_LCTN:
                     break;
                 case ESM4::REC_LGTM:
-                    return readTypedRecord<ESM4::LightingTemplate>(params, reader);
+                    readTypedRecord<ESM4::LightingTemplate>(params, reader);
+                    return true;
                 case ESM4::REC_LIGH:
-                    return readTypedRecord<ESM4::Light>(params, reader);
+                    readTypedRecord<ESM4::Light>(params, reader);
+                    return true;
                 case ESM4::REC_LSCR:
                     break;
                 case ESM4::REC_LTEX:
-                    return readTypedRecord<ESM4::LandTexture>(params, reader);
+                    readTypedRecord<ESM4::LandTexture>(params, reader);
+                    return true;
                 case ESM4::REC_LVLC:
-                    return readTypedRecord<ESM4::LevelledCreature>(params, reader);
+                    readTypedRecord<ESM4::LevelledCreature>(params, reader);
+                    return true;
                 case ESM4::REC_LVLI:
-                    return readTypedRecord<ESM4::LevelledItem>(params, reader);
+                    readTypedRecord<ESM4::LevelledItem>(params, reader);
+                    return true;
                 case ESM4::REC_LVLN:
-                    return readTypedRecord<ESM4::LevelledNpc>(params, reader);
+                    readTypedRecord<ESM4::LevelledNpc>(params, reader);
+                    return true;
                 case ESM4::REC_LVSP:
                     break;
                 case ESM4::REC_MATO:
-                    return readTypedRecord<ESM4::Material>(params, reader);
+                    readTypedRecord<ESM4::Material>(params, reader);
+                    return true;
                 case ESM4::REC_MATT:
                     break;
                 case ESM4::REC_MESG:
@@ -357,49 +417,66 @@ namespace EsmTool
                 case ESM4::REC_MGEF:
                     break;
                 case ESM4::REC_MISC:
-                    return readTypedRecord<ESM4::MiscItem>(params, reader);
+                    readTypedRecord<ESM4::MiscItem>(params, reader);
+                    return true;
                 case ESM4::REC_MOVT:
                     break;
                 case ESM4::REC_MSET:
-                    return readTypedRecord<ESM4::MediaSet>(params, reader);
+                    readTypedRecord<ESM4::MediaSet>(params, reader);
+                    return true;
                 case ESM4::REC_MSTT:
-                    return readTypedRecord<ESM4::MovableStatic>(params, reader);
+                    readTypedRecord<ESM4::MovableStatic>(params, reader);
+                    return true;
                 case ESM4::REC_MUSC:
-                    return readTypedRecord<ESM4::Music>(params, reader);
+                    readTypedRecord<ESM4::Music>(params, reader);
+                    return true;
                 case ESM4::REC_MUST:
                     break;
                 case ESM4::REC_NAVI:
-                    return readTypedRecord<ESM4::Navigation>(params, reader);
+                    readTypedRecord<ESM4::Navigation>(params, reader);
+                    return true;
                 case ESM4::REC_NAVM:
-                    return readTypedRecord<ESM4::NavMesh>(params, reader);
+                    readTypedRecord<ESM4::NavMesh>(params, reader);
+                    return true;
                 case ESM4::REC_NOTE:
-                    return readTypedRecord<ESM4::Note>(params, reader);
+                    readTypedRecord<ESM4::Note>(params, reader);
+                    return true;
                 case ESM4::REC_NPC_:
-                    return readTypedRecord<ESM4::Npc>(params, reader);
+                    readTypedRecord<ESM4::Npc>(params, reader);
+                    return true;
                 case ESM4::REC_OTFT:
-                    return readTypedRecord<ESM4::Outfit>(params, reader);
+                    readTypedRecord<ESM4::Outfit>(params, reader);
+                    return true;
                 case ESM4::REC_PACK:
-                    return readTypedRecord<ESM4::AIPackage>(params, reader);
+                    readTypedRecord<ESM4::AIPackage>(params, reader);
+                    return true;
                 case ESM4::REC_PERK:
                     break;
                 case ESM4::REC_PGRD:
-                    return readTypedRecord<ESM4::Pathgrid>(params, reader);
+                    readTypedRecord<ESM4::Pathgrid>(params, reader);
+                    return true;
                 case ESM4::REC_PGRE:
-                    return readTypedRecord<ESM4::PlacedGrenade>(params, reader);
+                    readTypedRecord<ESM4::PlacedGrenade>(params, reader);
+                    return true;
                 case ESM4::REC_PHZD:
                     break;
                 case ESM4::REC_PROJ:
                     break;
                 case ESM4::REC_PWAT:
-                    return readTypedRecord<ESM4::PlaceableWater>(params, reader);
+                    readTypedRecord<ESM4::PlaceableWater>(params, reader);
+                    return true;
                 case ESM4::REC_QUST:
-                    return readTypedRecord<ESM4::Quest>(params, reader);
+                    readTypedRecord<ESM4::Quest>(params, reader);
+                    return true;
                 case ESM4::REC_RACE:
-                    return readTypedRecord<ESM4::Race>(params, reader);
+                    readTypedRecord<ESM4::Race>(params, reader);
+                    return true;
                 case ESM4::REC_REFR:
-                    return readTypedRecord<ESM4::Reference>(params, reader);
+                    readTypedRecord<ESM4::Reference>(params, reader);
+                    return true;
                 case ESM4::REC_REGN:
-                    return readTypedRecord<ESM4::Region>(params, reader);
+                    readTypedRecord<ESM4::Region>(params, reader);
+                    return true;
                 case ESM4::REC_RELA:
                     break;
                 case ESM4::REC_REVB:
@@ -407,23 +484,30 @@ namespace EsmTool
                 case ESM4::REC_RFCT:
                     break;
                 case ESM4::REC_ROAD:
-                    return readTypedRecord<ESM4::Road>(params, reader);
+                    readTypedRecord<ESM4::Road>(params, reader);
+                    return true;
                 case ESM4::REC_SBSP:
-                    return readTypedRecord<ESM4::SubSpace>(params, reader);
+                    readTypedRecord<ESM4::SubSpace>(params, reader);
+                    return true;
                 case ESM4::REC_SCEN:
                     break;
                 case ESM4::REC_SCOL:
-                    return readTypedRecord<ESM4::StaticCollection>(params, reader);
+                    readTypedRecord<ESM4::StaticCollection>(params, reader);
+                    return true;
                 case ESM4::REC_SCPT:
-                    return readTypedRecord<ESM4::Script>(params, reader);
+                    readTypedRecord<ESM4::Script>(params, reader);
+                    return true;
                 case ESM4::REC_SCRL:
-                    return readTypedRecord<ESM4::Scroll>(params, reader);
+                    readTypedRecord<ESM4::Scroll>(params, reader);
+                    return true;
                 case ESM4::REC_SGST:
-                    return readTypedRecord<ESM4::SigilStone>(params, reader);
+                    readTypedRecord<ESM4::SigilStone>(params, reader);
+                    return true;
                 case ESM4::REC_SHOU:
                     break;
                 case ESM4::REC_SLGM:
-                    return readTypedRecord<ESM4::SoulGem>(params, reader);
+                    readTypedRecord<ESM4::SoulGem>(params, reader);
+                    return true;
                 case ESM4::REC_SMBN:
                     break;
                 case ESM4::REC_SMEN:
@@ -433,97 +517,56 @@ namespace EsmTool
                 case ESM4::REC_SNCT:
                     break;
                 case ESM4::REC_SNDR:
-                    return readTypedRecord<ESM4::SoundReference>(params, reader);
+                    readTypedRecord<ESM4::SoundReference>(params, reader);
+                    return true;
                 case ESM4::REC_SOPM:
                     break;
                 case ESM4::REC_SOUN:
-                    return readTypedRecord<ESM4::Sound>(params, reader);
+                    readTypedRecord<ESM4::Sound>(params, reader);
+                    return true;
                 case ESM4::REC_SPEL:
                     break;
                 case ESM4::REC_SPGD:
                     break;
                 case ESM4::REC_STAT:
-                    return readTypedRecord<ESM4::Static>(params, reader);
+                    readTypedRecord<ESM4::Static>(params, reader);
+                    return true;
                 case ESM4::REC_TACT:
-                    return readTypedRecord<ESM4::TalkingActivator>(params, reader);
+                    readTypedRecord<ESM4::TalkingActivator>(params, reader);
+                    return true;
                 case ESM4::REC_TERM:
-                    return readTypedRecord<ESM4::Terminal>(params, reader);
+                    readTypedRecord<ESM4::Terminal>(params, reader);
+                    return true;
                 case ESM4::REC_TES4:
-                    return readTypedRecord<ESM4::Header>(params, reader);
+                    readTypedRecord<ESM4::Header>(params, reader);
+                    return true;
                 case ESM4::REC_TREE:
-                    return readTypedRecord<ESM4::Tree>(params, reader);
+                    readTypedRecord<ESM4::Tree>(params, reader);
+                    return true;
                 case ESM4::REC_TXST:
-                    return readTypedRecord<ESM4::TextureSet>(params, reader);
+                    readTypedRecord<ESM4::TextureSet>(params, reader);
+                    return true;
                 case ESM4::REC_VTYP:
                     break;
                 case ESM4::REC_WATR:
                     break;
                 case ESM4::REC_WEAP:
-                    return readTypedRecord<ESM4::Weapon>(params, reader);
+                    readTypedRecord<ESM4::Weapon>(params, reader);
+                    return true;
                 case ESM4::REC_WOOP:
                     break;
                 case ESM4::REC_WRLD:
-                    return readTypedRecord<ESM4::World>(params, reader);
+                    readTypedRecord<ESM4::World>(params, reader);
+                    return true;
                 case ESM4::REC_WTHR:
                     break;
             }
 
             if (!params.mQuite)
                 std::cout << "\n  Unsupported record: " << ESM::NAME(reader.hdr().record.typeId).toStringView() << '\n';
-
-            reader.skipRecordData();
-        }
-
-        bool readItem(const Params& params, ESM4::Reader& reader);
-
-        bool readGroup(const Params& params, ESM4::Reader& reader)
-        {
-            const ESM4::RecordHeader& header = reader.hdr();
-
-            if (!params.mQuite)
-                std::cout << "\nGroup: " << toString(static_cast<ESM4::GroupType>(header.group.type)) << " "
-                          << ESM::NAME(header.group.typeId).toStringView() << '\n';
-
-            switch (static_cast<ESM4::GroupType>(header.group.type))
-            {
-                case ESM4::Grp_RecordType:
-                case ESM4::Grp_InteriorCell:
-                case ESM4::Grp_InteriorSubCell:
-                case ESM4::Grp_ExteriorCell:
-                case ESM4::Grp_ExteriorSubCell:
-                    reader.enterGroup();
-                    return readItem(params, reader);
-                case ESM4::Grp_WorldChild:
-                case ESM4::Grp_CellChild:
-                case ESM4::Grp_TopicChild:
-                case ESM4::Grp_CellPersistentChild:
-                case ESM4::Grp_CellTemporaryChild:
-                case ESM4::Grp_CellVisibleDistChild:
-                    reader.adjustGRUPFormId();
-                    reader.enterGroup();
-                    if (!reader.hasMoreRecs())
             return false;
-                    return readItem(params, reader);
         }
 
-            reader.skipGroup();
-
-            return true;
-        }
-
-        bool readItem(const Params& params, ESM4::Reader& reader)
-        {
-            if (!reader.getRecordHeader() || !reader.hasMoreRecs())
-                return false;
-
-            const ESM4::RecordHeader& header = reader.hdr();
-
-            if (header.record.typeId == ESM4::REC_GRUP)
-                return readGroup(params, reader);
-
-            readRecord(params, reader);
-            return true;
-        }
     }
 
     int loadTes4(const Arguments& info, std::unique_ptr<std::ifstream>&& stream)
@@ -551,12 +594,15 @@ namespace EsmTool
                 }
             }
 
-            while (reader.hasMoreRecs())
+            auto visitorRec = [&params](ESM4::Reader& reader) { return readRecord(params, reader); };
-            {
+            auto visitorGroup = [&params](ESM4::Reader& reader) {
-                reader.exitGroupCheck();
+                if (params.mQuite)
-                if (!readItem(params, reader))
+                    return;
-                    break;
+                auto groupType = static_cast<ESM4::GroupType>(reader.hdr().group.type);
-            }
+                std::cout << "\nGroup: " << toString(groupType) << " "
+                          << ESM::NAME(reader.hdr().group.typeId).toStringView() << '\n';
+            };
+            ESM4::ReaderUtils::readAll(reader, visitorRec, visitorGroup);
         }
         catch (const std::exception& e)
         {

apps/openmw/mwworld/esmloader.cpp

@@ -1,9 +1,14 @@
 #include "esmloader.hpp"
 #include "esmstore.hpp"
 
+#include <fstream>
+
+#include <components/esm/format.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/readerscache.hpp>
+#include <components/esm4/reader.hpp>
 #include <components/files/conversion.hpp>
+#include <components/files/openfile.hpp>
 
 namespace MWWorld
 {
@@ -21,8 +26,16 @@ namespace MWWorld
 
     void EsmLoader::load(const std::filesystem::path& filepath, int& index, Loading::Listener* listener)
     {
-        const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
 
+        auto stream = Files::openBinaryInputFileStream(filepath);
+        const ESM::Format format = ESM::readFormat(*stream);
+        stream->seekg(0);
+
+        switch (format)
+        {
+            case ESM::Format::Tes3:
+            {
+                const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
                 reader->setEncoder(mEncoder);
                 reader->setIndex(index);
                 reader->open(filepath);
@@ -43,6 +56,17 @@ namespace MWWorld
                     && (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
                         || Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
                     mMasterFileFormat = reader->getFormat();
+                break;
+            }
+            case ESM::Format::Tes4:
+            {
+                ESM4::Reader readerESM4(std::move(stream), filepath);
+                auto statelessEncoder = mEncoder->getStatelessEncoder();
+                readerESM4.setEncoder(&statelessEncoder);
+                mStore.loadESM4(readerESM4);
+                break;
+            }
+        }
     }
 
 } /* namespace MWWorld */

apps/openmw/mwworld/esmstore.cpp

@@ -14,6 +14,11 @@
 #include <components/misc/algorithm.hpp>
 
 #include <components/esm4/common.hpp>
+#include <components/esm4/loadcell.hpp>
+#include <components/esm4/loadrefr.hpp>
+#include <components/esm4/loadstat.hpp>
+#include <components/esm4/reader.hpp>
+#include <components/esm4/readerutils.hpp>
 #include <components/esmloader/load.hpp>
 
 #include "../mwmechanics/spelllist.hpp"
@@ -180,6 +185,35 @@ namespace MWWorld
                 }
             }
         }
+
+        template <typename T>
+        static bool typedReadRecordESM4(ESM4::Reader& reader, Store<T>& store)
+        {
+            auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
+
+            ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
+            if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*> && HasRecordId<T>::value)
+            {
+                if constexpr (ESM::isESM4Rec(T::sRecordId))
+                {
+                    if (T::sRecordId == esm4RecName)
+                    {
+                        reader.getRecordData();
+                        T value;
+                        value.load(reader);
+                        store.insertStatic(value);
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+
+        static bool readRecord(ESM4::Reader& reader, ESMStore& store)
+        {
+            return std::apply([&reader](auto&... x) { return (ESMStoreImp::typedReadRecordESM4(reader, x) || ...); },
+                store.mStoreImp->mStores);
+        }
     };
 
     int ESMStore::find(const ESM::RefId& id) const
@@ -338,6 +372,12 @@ namespace MWWorld
         }
     }
 
+    void ESMStore::loadESM4(ESM4::Reader& reader)
+    {
+        auto visitorRec = [this](ESM4::Reader& reader) { return ESMStoreImp::readRecord(reader, *this); };
+        ESM4::ReaderUtils::readAll(reader, visitorRec, [](ESM4::Reader&) {});
+    }
+
     void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type)
     {
         mStoreImp->mIds[id] = type;

apps/openmw/mwworld/esmstore.hpp

@@ -24,6 +24,14 @@ namespace MWMechanics
     class SpellList;
 }
 
+namespace ESM4
+{
+    class Reader;
+    struct Static;
+    struct Cell;
+    struct Reference;
+}
+
 namespace ESM
 {
     class ReadersCache;
@@ -78,7 +86,7 @@ namespace MWWorld
     class ESMStore
     {
         friend struct ESMStoreImp; // This allows StoreImp to extend esmstore without beeing included everywhere
-
+    public:
         using StoreTuple = std::tuple<Store<ESM::Activator>, Store<ESM::Potion>, Store<ESM::Apparatus>,
             Store<ESM::Armor>, Store<ESM::BodyPart>, Store<ESM::Book>, Store<ESM::BirthSign>, Store<ESM::Class>,
             Store<ESM::Clothing>, Store<ESM::Container>, Store<ESM::Creature>, Store<ESM::Dialogue>, Store<ESM::Door>,
@@ -95,8 +103,11 @@ namespace MWWorld
             Store<ESM::MagicEffect>, Store<ESM::Skill>,
 
             // Special entry which is hardcoded and not loaded from an ESM
-            Store<ESM::Attribute>>;
+            Store<ESM::Attribute>,
+
+            Store<ESM4::Static>, Store<ESM4::Cell>, Store<ESM4::Reference>>;
 
+    private:
         template <typename T>
         static constexpr std::size_t getTypeIndex()
         {
@@ -162,6 +173,7 @@ namespace MWWorld
         void validateDynamic();
 
         void load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue);
+        void loadESM4(ESM4::Reader& esm);
 
         template <class T>
         const Store<T>& get() const
@@ -252,6 +264,16 @@ namespace MWWorld
 
     template <>
     const ESM::NPC* ESMStore::insert<ESM::NPC>(const ESM::NPC& npc);
+
+    template <class T, class = std::void_t<>>
+    struct HasRecordId : std::false_type
+    {
+    };
+
+    template <class T>
+    struct HasRecordId<T, std::void_t<decltype(T::sRecordId)>> : std::true_type
+    {
+    };
 }
 
 #endif

apps/openmw/mwworld/store.cpp

@@ -1,18 +1,19 @@
 #include "store.hpp"
 
-#include <components/debug/debuglog.hpp>
+#include <iterator>
+#include <sstream>
+#include <stdexcept>
 
+#include <components/debug/debuglog.hpp>
 #include <components/esm/records.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/esmwriter.hpp>
-
+#include <components/esm4/loadcell.hpp>
+#include <components/esm4/loadrefr.hpp>
+#include <components/esm4/loadstat.hpp>
 #include <components/loadinglistener/loadinglistener.hpp>
 #include <components/misc/rng.hpp>
 
-#include <iterator>
-#include <sstream>
-#include <stdexcept>
-
 namespace
 {
     // TODO: Switch to C++23 to get a working version of std::unordered_map::erase
@@ -161,7 +162,15 @@ namespace MWWorld
         if (ptr == nullptr)
         {
             std::stringstream msg;
-            msg << T::getRecordType() << " '" << id << "' not found";
+            if constexpr (!ESM::isESM4Rec(T::sRecordId))
+            {
+                msg << T::getRecordType();
+            }
+            else
+            {
+                msg << "ESM::REC_" << getRecNameString(T::sRecordId).toStringView();
+            }
+            msg << " '" << id << "' not found";
             throw std::runtime_error(msg.str());
         }
         return ptr;
@@ -171,8 +180,10 @@ namespace MWWorld
     {
         T record;
         bool isDeleted = false;
-
+        if constexpr (!ESM::isESM4Rec(T::sRecordId))
+        {
             record.load(esm, isDeleted);
+        }
 
         std::pair<typename Static::iterator, bool> inserted = mStatic.insert_or_assign(record.mId, record);
         if (inserted.second)
@@ -291,19 +302,24 @@ namespace MWWorld
     void TypedDynamicStore<T>::write(ESM::ESMWriter& writer, Loading::Listener& progress) const
     {
         for (typename Dynamic::const_iterator iter(mDynamic.begin()); iter != mDynamic.end(); ++iter)
+        {
+            if constexpr (!ESM::isESM4Rec(T::sRecordId))
             {
                 writer.startRecord(T::sRecordId);
                 iter->second.save(writer);
                 writer.endRecord(T::sRecordId);
             }
         }
+    }
     template <typename T>
     RecordId TypedDynamicStore<T>::read(ESM::ESMReader& reader, bool overrideOnly)
     {
         T record;
         bool isDeleted = false;
-
+        if constexpr (!ESM::isESM4Rec(T::sRecordId))
+        {
             record.load(reader, isDeleted);
+        }
         insert(record, overrideOnly);
 
         return RecordId(record.mId, isDeleted);
@@ -1152,6 +1168,20 @@ namespace MWWorld
 
         return mKeywordSearch;
     }
+
+    ESM::FixedString<6> getRecNameString(ESM::RecNameInts recName)
+    {
+        ESM::FixedString<6> name;
+        name.assign("");
+        ESM::NAME fourCCName(recName & ~ESM::sEsm4RecnameFlag);
+        for (int i = 0; i < 4; i++)
+            name.mData[i] = fourCCName.mData[i];
+        if (ESM::isESM4Rec(recName))
+        {
+            name.mData[4] = '4';
+        }
+        return name;
+    }
 }
 
 template class MWWorld::TypedDynamicStore<ESM::Activator>;
@@ -1196,3 +1226,7 @@ template class MWWorld::TypedDynamicStore<ESM::Spell>;
 template class MWWorld::TypedDynamicStore<ESM::StartScript>;
 template class MWWorld::TypedDynamicStore<ESM::Static>;
 template class MWWorld::TypedDynamicStore<ESM::Weapon>;
+
+template class MWWorld::TypedDynamicStore<ESM4::Static>;
+template class MWWorld::TypedDynamicStore<ESM4::Reference>;
+template class MWWorld::TypedDynamicStore<ESM4::Cell>;

apps/openmw/mwworld/store.hpp

@@ -517,6 +517,8 @@ namespace MWWorld
         const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() const;
     };
 
+    ESM::FixedString<6> getRecNameString(ESM::RecNameInts recName);
+
 } // end namespace
 
 #endif

apps/openmw_test_suite/mwworld/test_store.cpp

@@ -8,6 +8,12 @@
 #include <components/esm/records.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/esmwriter.hpp>
+#include <components/esm4/common.hpp>
+#include <components/esm4/loadcell.hpp>
+#include <components/esm4/loadrefr.hpp>
+#include <components/esm4/loadstat.hpp>
+#include <components/esm4/reader.hpp>
+#include <components/esm4/readerutils.hpp>
 #include <components/files/configurationmanager.hpp>
 #include <components/files/conversion.hpp>
 #include <components/loadinglistener/loadinglistener.hpp>
@@ -289,6 +295,41 @@ TEST_F(StoreTest, delete_test)
     ASSERT_TRUE(mEsmStore.get<RecordType>().getSize() == 1);
 }
 
+template <typename T>
+static unsigned int hasSameRecordId(const MWWorld::Store<T>& store, ESM::RecNameInts RecName)
+{
+    if constexpr (MWWorld::HasRecordId<T>::value)
+    {
+        return T::sRecordId == RecName ? 1 : 0;
+    }
+    else
+    {
+        return 0;
+    }
+}
+
+template <typename T>
+static void testRecNameIntCount(const MWWorld::Store<T>& store, const MWWorld::ESMStore::StoreTuple& stores)
+{
+    if constexpr (MWWorld::HasRecordId<T>::value)
+    {
+        const unsigned int recordIdCount
+            = std::apply([](auto&&... x) { return (hasSameRecordId(x, T::sRecordId) + ...); }, stores);
+        ASSERT_EQ(recordIdCount, static_cast<unsigned int>(1))
+            << "The same RecNameInt is used twice ESM::REC_" << MWWorld::getRecNameString(T::sRecordId).toStringView();
+    }
+}
+
+static void testAllRecNameIntUnique(const MWWorld::ESMStore::StoreTuple& stores)
+{
+    std::apply([&stores](auto&&... x) { (testRecNameIntCount(x, stores), ...); }, stores);
+}
+
+TEST_F(StoreTest, eachRecordTypeShouldHaveUniqueRecordId)
+{
+    testAllRecNameIntUnique(MWWorld::ESMStore::StoreTuple());
+}
+
 /// Tests overwriting of records.
 TEST_F(StoreTest, overwrite_test)
 {

components/CMakeLists.txt

@@ -193,6 +193,7 @@ add_component_dir (esm4
     reader
     reference
     script
+    readerutils
 )
 
 add_component_dir (misc

components/esm/defs.hpp

@@ -333,6 +333,11 @@ namespace ESM
         REC_MSET4 = esm4Recname(ESM4::REC_MSET) // Media Set
     };
 
+    constexpr bool isESM4Rec(RecNameInts RecName)
+    {
+        return RecName & sEsm4RecnameFlag;
+    }
+
     /// Common subrecords
     enum SubRecNameInts
     {

components/esm/refid.cpp

@@ -29,6 +29,11 @@ namespace ESM
         return newRefId;
     }
 
+    RefId RefId::formIdRefId(const ESM4::FormId id)
+    {
+        return ESM::RefId::stringRefId(ESM4::formIdToString(id));
+    }
+
     bool RefId::operator==(std::string_view rhs) const
     {
         return Misc::StringUtils::ciEqual(mId, rhs);

components/esm/refid.hpp

@@ -6,6 +6,8 @@
 #include <string>
 #include <string_view>
 
+#include <components/esm4/formid.hpp>
+
 namespace ESM
 {
     // RefId is used to represent an Id that identifies an ESM record. These Ids can then be used in
@@ -27,6 +29,7 @@ namespace ESM
         // RefIds that are as string in the code. For serialization, and display. Using explicit conversions make it
         // very clear where in the code we need to convert from string to RefId and Vice versa.
         static RefId stringRefId(std::string_view id);
+        static RefId formIdRefId(const ESM4::FormId id);
         const std::string& getRefIdString() const { return mId; }
 
     private:

components/esm4/loadcell.cpp

@@ -32,13 +32,14 @@
 
 #include <cassert>
 #include <cfloat> // FLT_MAX for gcc
-#include <stdexcept>
-
 #include <iostream> // FIXME: debug only
+#include <stdexcept>
 
 #include "reader.hpp"
 // #include "writer.hpp"
 
+#include <components/esm/refid.hpp>
+
 // TODO: Try loading only EDID and XCLC (along with mFormId, mFlags and mParent)
 //
 // But, for external cells we may be scanning the whole record since we don't know if there is
@@ -48,8 +49,9 @@
 // longer/shorter/same as loading the subrecords.
 void ESM4::Cell::load(ESM4::Reader& reader)
 {
-    mFormId = reader.hdr().record.id;
+    auto formId = reader.hdr().record.id;
-    reader.adjustFormId(mFormId);
+    reader.adjustFormId(formId);
+    mId = ESM::RefId::formIdRefId(formId);
     mFlags = reader.hdr().record.flags;
     mParent = reader.currWorld();
 
@@ -71,7 +73,7 @@ void ESM4::Cell::load(ESM4::Reader& reader)
     // WARN: we need to call setCurrCell (and maybe setCurrCellGrid?) again before loading
     // cell child groups if we are loading them after restoring the context
     // (may be easier to update the context before saving?)
-    reader.setCurrCell(mFormId); // save for LAND (and other children) to access later
+    reader.setCurrCell(formId); // save for LAND (and other children) to access later
     std::uint32_t esmVer = reader.esmVersion();
     bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134;
 

components/esm4/loadcell.hpp

@@ -34,6 +34,9 @@
 #include "formid.hpp"
 #include "lighting.hpp"
 
+#include <components/esm/defs.hpp>
+#include <components/esm/refid.hpp>
+
 namespace ESM4
 {
     class Reader;
@@ -61,7 +64,7 @@ namespace ESM4
     {
         FormId mParent; // world formId (for grouping cells), from the loading sequence
 
-        FormId mFormId; // from the header
+        ESM::RefId mId; // from the header
         std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
 
         std::string mEditorId;
@@ -95,6 +98,8 @@ namespace ESM4
         // void save(ESM4::Writer& writer) const;
 
         void blank();
+
+        static constexpr ESM::RecNameInts sRecordId = ESM::REC_CELL4;
     };
 }
 

components/esm4/loadrefr.cpp

@@ -34,10 +34,11 @@
 
 void ESM4::Reference::load(ESM4::Reader& reader)
 {
-    mFormId = reader.hdr().record.id;
+    auto formId = reader.hdr().record.id;
-    reader.adjustFormId(mFormId);
+    reader.adjustFormId(formId);
+    mId = ESM::RefId::formIdRefId(formId);
     mFlags = reader.hdr().record.flags;
-    mParent = reader.currCell(); // NOTE: only for persistent refs?
+    mParent = ESM::RefId::formIdRefId(reader.currCell()); // NOTE: only for persistent refs?
 
     // TODO: Let the engine apply this? Saved games?
     // mInitiallyDisabled = ((mFlags & ESM4::Rec_Disabled) != 0) ? true : false;
@@ -60,7 +61,9 @@ void ESM4::Reference::load(ESM4::Reader& reader)
                 break;
             case ESM4::SUB_NAME:
             {
-                reader.getFormId(mBaseObj);
+                FormId BaseId;
+                reader.getFormId(BaseId);
+                mBaseObj = ESM::RefId::formIdRefId(BaseId);
 #if 0
                 if (mFlags & ESM4::Rec_Disabled)
                     std::cout << "REFR disable at start " << formIdToString(mFormId) <<

components/esm4/loadrefr.hpp

@@ -31,6 +31,9 @@
 
 #include "reference.hpp" // FormId, Placement, EnableParent
 
+#include <components/esm/defs.hpp>
+#include <components/esm/refid.hpp>
+
 namespace ESM4
 {
     class Reader;
@@ -71,15 +74,15 @@ namespace ESM4
 
     struct Reference
     {
-        FormId mParent; // cell FormId (currently persistent refs only), from the loading sequence
+        ESM::RefId mParent; // cell FormId (currently persistent refs only), from the loading sequence
                             // NOTE: for exterior cells it will be the dummy cell FormId
 
-        FormId mFormId; // from the header
+        ESM::RefId mId; // from the header
         std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
 
         std::string mEditorId;
         std::string mFullName;
-        FormId mBaseObj;
+        ESM::RefId mBaseObj;
 
         Placement mPlacement;
         float mScale = 1.0f;
@@ -110,6 +113,8 @@ namespace ESM4
         // void save(ESM4::Writer& writer) const;
 
         void blank();
+
+        static constexpr ESM::RecNameInts sRecordId = ESM::REC_REFR4;
     };
 }
 

components/esm4/loadstat.cpp

@@ -34,8 +34,9 @@
 
 void ESM4::Static::load(ESM4::Reader& reader)
 {
-    mFormId = reader.hdr().record.id;
+    FormId formId = reader.hdr().record.id;
-    reader.adjustFormId(mFormId);
+    reader.adjustFormId(formId);
+    mId = ESM::RefId::formIdRefId(formId);
     mFlags = reader.hdr().record.flags;
 
     while (reader.getSubRecordHeader())

components/esm4/loadstat.hpp

@@ -33,6 +33,9 @@
 
 #include "formid.hpp"
 
+#include <components/esm/defs.hpp>
+#include <components/esm/refid.hpp>
+
 namespace ESM4
 {
     class Reader;
@@ -40,7 +43,7 @@ namespace ESM4
 
     struct Static
     {
-        FormId mFormId; // from the header
+        ESM::RefId mId; // from the header
         std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
 
         std::string mEditorId;
@@ -53,6 +56,8 @@ namespace ESM4
         // void save(ESM4::Writer& writer) const;
 
         // void blank();
+
+        static constexpr ESM::RecNameInts sRecordId = ESM::REC_STAT4;
     };
 }
 

components/esm4/readerutils.hpp

@@ -0,0 +1,84 @@
+#ifndef OPENMW_COMPONENTS_ESM4_READERUTILS
+#define OPENMW_COMPONENTS_ESM4_READERUTILS
+
+#include <components/esm4/reader.hpp>
+namespace ESM4
+{
+    struct ReaderUtils
+    {
+
+        /* RecordInvocable must be an invocable, takes an ESM4::Reader& as input, and outputs a boolean that indicates
+        if the record was read or ignored. Will be invoked for every record
+
+        GroupInvocable's invocable must take a ESM4::Reader& as input, doesn't need to output anything. Will be invoked
+        for every group*/
+        template <typename RecordInvocable, typename GroupInvocable>
+        static void readAll(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
+        {
+            while (reader.hasMoreRecs())
+            {
+                reader.exitGroupCheck();
+                if (!readItem(reader, recordInvocable, groupInvocable))
+                    break;
+            }
+        }
+
+        template <typename RecordInvocable>
+        static void readRecord(ESM4::Reader& reader, RecordInvocable&& recordInvocable)
+        {
+            if (!recordInvocable(reader))
+                reader.skipRecordData();
+        }
+
+        template <typename RecordInvocable, typename GroupInvocable>
+        static bool readGroup(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
+        {
+            const ESM4::RecordHeader& header = reader.hdr();
+
+            groupInvocable(reader);
+
+            switch (static_cast<ESM4::GroupType>(header.group.type))
+            {
+                case ESM4::Grp_RecordType:
+                case ESM4::Grp_InteriorCell:
+                case ESM4::Grp_InteriorSubCell:
+                case ESM4::Grp_ExteriorCell:
+                case ESM4::Grp_ExteriorSubCell:
+                    reader.enterGroup();
+                    return readItem(reader, recordInvocable, groupInvocable);
+                case ESM4::Grp_WorldChild:
+                case ESM4::Grp_CellChild:
+                case ESM4::Grp_TopicChild:
+                case ESM4::Grp_CellPersistentChild:
+                case ESM4::Grp_CellTemporaryChild:
+                case ESM4::Grp_CellVisibleDistChild:
+                    reader.adjustGRUPFormId();
+                    reader.enterGroup();
+                    if (!reader.hasMoreRecs())
+                        return false;
+                    return readItem(reader, recordInvocable, groupInvocable);
+            }
+
+            reader.skipGroup();
+
+            return true;
+        }
+
+        template <typename RecordInvocable, typename GroupInvocable>
+        static bool readItem(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
+        {
+            if (!reader.getRecordHeader() || !reader.hasMoreRecs())
+                return false;
+
+            const ESM4::RecordHeader& header = reader.hdr();
+
+            if (header.record.typeId == ESM4::REC_GRUP)
+                return readGroup(reader, recordInvocable, groupInvocable);
+
+            readRecord(reader, recordInvocable);
+            return true;
+        }
+    };
+}
+
+#endif // !OPENMW_COMPONENTS_ESM4_READERUTILS

components/to_utf8/to_utf8.hpp

@@ -68,6 +68,8 @@ namespace ToUTF8
         /// ASCII-only string. Otherwise returns a view to the input.
         std::string_view getLegacyEnc(std::string_view input);
 
+        StatelessUtf8Encoder getStatelessEncoder() const { return mImpl; }
+
     private:
         std::string mBuffer;
         StatelessUtf8Encoder mImpl;