diff --git a/apps/openmw/mwworld/esmloader.cpp b/apps/openmw/mwworld/esmloader.cpp index 3c1b6acc47..3265059a13 100644 --- a/apps/openmw/mwworld/esmloader.cpp +++ b/apps/openmw/mwworld/esmloader.cpp @@ -1,9 +1,13 @@ #include "esmloader.hpp" #include "esmstore.hpp" +#include #include #include +#include #include +#include +#include namespace MWWorld { @@ -21,28 +25,50 @@ namespace MWWorld void EsmLoader::load(const std::filesystem::path& filepath, int& index, Loading::Listener* listener) { - const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast(index)); - reader->setEncoder(mEncoder); - reader->setIndex(index); - reader->open(filepath); - reader->resolveParentFileIndices(mReaders); + auto stream = Files::openBinaryInputFileStream(filepath); + if (!stream->is_open()) + { + throw std::runtime_error(std::string("File Failed to open file: ") + std::strerror(errno) + "\n"); + return; + } + const ESM::Format format = ESM::readFormat(*stream); + stream->seekg(0); - assert(reader->getGameFiles().size() == reader->getParentFileIndices().size()); - for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i) - if (i == static_cast(reader->getIndex())) - throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file " + switch (format) + { + case ESM::Format::Tes3: + { + const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast(index)); + reader->setEncoder(mEncoder); + reader->setIndex(index); + reader->open(filepath); + reader->resolveParentFileIndices(mReaders); + + assert(reader->getGameFiles().size() == reader->getParentFileIndices().size()); + for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i) + if (i == static_cast(reader->getIndex())) + throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file " + reader->getGameFiles()[i].name + ", but it is not available or has been loaded in the wrong order. " "Please run the launcher to fix this issue."); - mESMVersions[index] = reader->getVer(); - mStore.load(*reader, listener, mDialogue); + mESMVersions[index] = reader->getVer(); + mStore.load(*reader, listener, mDialogue); - if (!mMasterFileFormat.has_value() - && (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm") - || Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame"))) - mMasterFileFormat = reader->getFormat(); + if (!mMasterFileFormat.has_value() + && (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm") + || Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame"))) + mMasterFileFormat = reader->getFormat(); + break; + } + case ESM::Format::Tes4: + { + ESM4::Reader readerESM4(std::move(stream), filepath); + readerESM4.setEncoder(mEncoder->getStatelessEncoder()); + mStore.loadESM4(readerESM4, listener, mDialogue); + } + } } } /* namespace MWWorld */ diff --git a/apps/openmw/mwworld/esmstore.cpp b/apps/openmw/mwworld/esmstore.cpp index 03de9fda5d..15bf52757e 100644 --- a/apps/openmw/mwworld/esmstore.cpp +++ b/apps/openmw/mwworld/esmstore.cpp @@ -13,6 +13,11 @@ #include #include +#include +#include +#include +#include + #include #include @@ -180,6 +185,84 @@ namespace MWWorld } } } + + template + static void typedReadRecordESM4(ESM4::Reader& reader, ESMStore& stores, Store& store, int& found) + { + auto recordType = static_cast(reader.hdr().record.typeId); + + ESM::RecNameInts esm4RecName = static_cast(ESM::esm4Recname(recordType)); + if constexpr (std::is_convertible_v*, DynamicStore*>) + { + if constexpr (ESM::isESM4Rec(T::sRecordId)) + { + if (T::sRecordId == esm4RecName) + { + reader.getRecordData(); + T value; + value.load(reader); + store.insertStatic(value); + found++; + } + } + } + } + + static void readRecord(ESM4::Reader& reader, ESMStore& store) + { + int found = 0; + std::apply([&reader, &store, &found]( + auto&... x) { (ESMStoreImp::typedReadRecordESM4(reader, store, x, found), ...); }, + store.mStoreImp->mStores); + assert(found <= 1); + if (found == 0) // unhandled record + reader.skipRecordData(); + } + + static bool readItem(ESM4::Reader& reader, ESMStore& store) + { + if (!reader.getRecordHeader() || !reader.hasMoreRecs()) + return false; + + const ESM4::RecordHeader& header = reader.hdr(); + + if (header.record.typeId == ESM4::REC_GRUP) + return readGroup(reader, store); + + readRecord(reader, store); + return true; + } + + static bool readGroup(ESM4::Reader& reader, ESMStore& store) + { + const ESM4::RecordHeader& header = reader.hdr(); + + switch (static_cast(header.group.type)) + { + case ESM4::Grp_RecordType: + case ESM4::Grp_InteriorCell: + case ESM4::Grp_InteriorSubCell: + case ESM4::Grp_ExteriorCell: + case ESM4::Grp_ExteriorSubCell: + reader.enterGroup(); + return readItem(reader, store); + case ESM4::Grp_WorldChild: + case ESM4::Grp_CellChild: + case ESM4::Grp_TopicChild: + case ESM4::Grp_CellPersistentChild: + case ESM4::Grp_CellTemporaryChild: + case ESM4::Grp_CellVisibleDistChild: + reader.adjustGRUPFormId(); + reader.enterGroup(); + if (!reader.hasMoreRecs()) + return false; + return readItem(reader, store); + } + + reader.skipGroup(); + + return true; + } }; int ESMStore::find(const ESM::RefId& id) const @@ -338,6 +421,16 @@ namespace MWWorld } } + void ESMStore::loadESM4(ESM4::Reader& reader, Loading::Listener* listener, ESM::Dialogue*& dialogue) + { + while (reader.hasMoreRecs()) + { + reader.exitGroupCheck(); + if (!ESMStoreImp::readItem(reader, *this)) + break; + } + } + void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type) { mStoreImp->mIds[id] = type; diff --git a/apps/openmw/mwworld/esmstore.hpp b/apps/openmw/mwworld/esmstore.hpp index 7f5773c957..1ed66a1c71 100644 --- a/apps/openmw/mwworld/esmstore.hpp +++ b/apps/openmw/mwworld/esmstore.hpp @@ -24,6 +24,14 @@ namespace MWMechanics class SpellList; } +namespace ESM4 +{ + class Reader; + struct Static; + struct Cell; + struct Reference; +} + namespace ESM { class ReadersCache; @@ -95,7 +103,9 @@ namespace MWWorld Store, Store, // Special entry which is hardcoded and not loaded from an ESM - Store>; + Store, + + Store, Store, Store>; template static constexpr std::size_t getTypeIndex() @@ -162,6 +172,7 @@ namespace MWWorld void validateDynamic(); void load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue); + void loadESM4(ESM4::Reader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue); template const Store& get() const diff --git a/apps/openmw/mwworld/store.cpp b/apps/openmw/mwworld/store.cpp index 11e58c0edb..27d74ddf59 100644 --- a/apps/openmw/mwworld/store.cpp +++ b/apps/openmw/mwworld/store.cpp @@ -13,6 +13,10 @@ #include #include +#include +#include +#include + namespace { // TODO: Switch to C++23 to get a working version of std::unordered_map::erase @@ -161,7 +165,10 @@ namespace MWWorld if (ptr == nullptr) { std::stringstream msg; - msg << T::getRecordType() << " '" << id << "' not found"; + if constexpr (!ESM::isESM4Rec(T::sRecordId)) + { + msg << T::getRecordType() << " '" << id << "' not found"; + } throw std::runtime_error(msg.str()); } return ptr; @@ -171,8 +178,10 @@ namespace MWWorld { T record; bool isDeleted = false; - - record.load(esm, isDeleted); + if constexpr (!ESM::isESM4Rec(T::sRecordId)) + { + record.load(esm, isDeleted); + } std::pair inserted = mStatic.insert_or_assign(record.mId, record); if (inserted.second) @@ -293,7 +302,11 @@ namespace MWWorld for (typename Dynamic::const_iterator iter(mDynamic.begin()); iter != mDynamic.end(); ++iter) { writer.startRecord(T::sRecordId); - iter->second.save(writer); + if constexpr (!ESM::isESM4Rec(T::sRecordId)) + { + iter->second.save(writer); + } + writer.endRecord(T::sRecordId); } } @@ -302,8 +315,10 @@ namespace MWWorld { T record; bool isDeleted = false; - - record.load(reader, isDeleted); + if constexpr (!ESM::isESM4Rec(T::sRecordId)) + { + record.load(reader, isDeleted); + } insert(record, overrideOnly); return RecordId(record.mId, isDeleted); @@ -1196,3 +1211,7 @@ template class MWWorld::TypedDynamicStore; template class MWWorld::TypedDynamicStore; template class MWWorld::TypedDynamicStore; template class MWWorld::TypedDynamicStore; + +template class MWWorld::TypedDynamicStore; +template class MWWorld::TypedDynamicStore; +template class MWWorld::TypedDynamicStore; diff --git a/components/esm/defs.hpp b/components/esm/defs.hpp index f3e1ad640e..431b85b9ae 100644 --- a/components/esm/defs.hpp +++ b/components/esm/defs.hpp @@ -333,6 +333,11 @@ namespace ESM REC_MSET4 = esm4Recname(ESM4::REC_MSET) // Media Set }; + constexpr bool isESM4Rec(RecNameInts RecName) + { + return RecName & sEsm4RecnameFlag; + } + /// Common subrecords enum SubRecNameInts { diff --git a/components/esm/refid.cpp b/components/esm/refid.cpp index ec18c4c642..3470ebd3d3 100644 --- a/components/esm/refid.cpp +++ b/components/esm/refid.cpp @@ -34,6 +34,11 @@ namespace ESM return newRefId; } + RefId RefId::formIdRefId(const ESM4::FormId id) + { + return ESM::RefId::stringRefId(ESM4::formIdToString(id)); + } + bool RefId::operator==(std::string_view rhs) const { return Misc::StringUtils::ciEqual(mId, rhs); diff --git a/components/esm/refid.hpp b/components/esm/refid.hpp index 6bc49313d0..8900f7ce75 100644 --- a/components/esm/refid.hpp +++ b/components/esm/refid.hpp @@ -1,6 +1,7 @@ #ifndef OPENMW_COMPONENTS_ESM_REFID_HPP #define OPENMW_COMPONENTS_ESM_REFID_HPP #include +#include #include #include #include @@ -26,6 +27,7 @@ namespace ESM // RefIds that are as string in the code. For serialization, and display. Using explicit conversions make it // very clear where in the code we need to convert from string to RefId and Vice versa. static RefId stringRefId(std::string_view id); + static RefId formIdRefId(const ESM4::FormId id); const std::string& getRefIdString() const { return mId; } private: diff --git a/components/esm4/loadcell.cpp b/components/esm4/loadcell.cpp index 919c81b8e9..1321e31578 100644 --- a/components/esm4/loadcell.cpp +++ b/components/esm4/loadcell.cpp @@ -37,7 +37,8 @@ #include // FIXME: debug only #include "reader.hpp" -//#include "writer.hpp" +#include +// #include "writer.hpp" // TODO: Try loading only EDID and XCLC (along with mFormId, mFlags and mParent) // @@ -48,8 +49,9 @@ // longer/shorter/same as loading the subrecords. void ESM4::Cell::load(ESM4::Reader& reader) { - mFormId = reader.hdr().record.id; - reader.adjustFormId(mFormId); + auto formId = reader.hdr().record.id; + reader.adjustFormId(formId); + mId = ESM::RefId::formIdRefId(formId); mFlags = reader.hdr().record.flags; mParent = reader.currWorld(); @@ -71,7 +73,7 @@ void ESM4::Cell::load(ESM4::Reader& reader) // WARN: we need to call setCurrCell (and maybe setCurrCellGrid?) again before loading // cell child groups if we are loading them after restoring the context // (may be easier to update the context before saving?) - reader.setCurrCell(mFormId); // save for LAND (and other children) to access later + reader.setCurrCell(formId); // save for LAND (and other children) to access later std::uint32_t esmVer = reader.esmVersion(); bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134; diff --git a/components/esm4/loadcell.hpp b/components/esm4/loadcell.hpp index ee77bd2bfc..bcc95f652f 100644 --- a/components/esm4/loadcell.hpp +++ b/components/esm4/loadcell.hpp @@ -33,6 +33,8 @@ #include "formid.hpp" #include "lighting.hpp" +#include +#include namespace ESM4 { @@ -61,7 +63,7 @@ namespace ESM4 { FormId mParent; // world formId (for grouping cells), from the loading sequence - FormId mFormId; // from the header + ESM::RefId mId; // from the header std::uint32_t mFlags; // from the header, see enum type RecordFlag for details std::string mEditorId; @@ -95,6 +97,8 @@ namespace ESM4 // void save(ESM4::Writer& writer) const; void blank(); + + static constexpr ESM::RecNameInts sRecordId = ESM::REC_CELL4; }; } diff --git a/components/esm4/loadrefr.cpp b/components/esm4/loadrefr.cpp index d96f52a741..b95f24e73a 100644 --- a/components/esm4/loadrefr.cpp +++ b/components/esm4/loadrefr.cpp @@ -30,14 +30,15 @@ #include #include "reader.hpp" -//#include "writer.hpp" +// #include "writer.hpp" void ESM4::Reference::load(ESM4::Reader& reader) { - mFormId = reader.hdr().record.id; - reader.adjustFormId(mFormId); + auto formId = reader.hdr().record.id; + reader.adjustFormId(formId); + mId = ESM::RefId::formIdRefId(formId); mFlags = reader.hdr().record.flags; - mParent = reader.currCell(); // NOTE: only for persistent refs? + mParent = ESM::RefId::formIdRefId(reader.currCell()); // NOTE: only for persistent refs? // TODO: Let the engine apply this? Saved games? // mInitiallyDisabled = ((mFlags & ESM4::Rec_Disabled) != 0) ? true : false; @@ -60,7 +61,9 @@ void ESM4::Reference::load(ESM4::Reader& reader) break; case ESM4::SUB_NAME: { - reader.getFormId(mBaseObj); + FormId BaseId; + reader.getFormId(BaseId); + mBaseObj = ESM::RefId::formIdRefId(BaseId); #if 0 if (mFlags & ESM4::Rec_Disabled) std::cout << "REFR disable at start " << formIdToString(mFormId) << diff --git a/components/esm4/loadrefr.hpp b/components/esm4/loadrefr.hpp index 04d537932e..baf199c4c4 100644 --- a/components/esm4/loadrefr.hpp +++ b/components/esm4/loadrefr.hpp @@ -30,6 +30,8 @@ #include #include "reference.hpp" // FormId, Placement, EnableParent +#include +#include namespace ESM4 { @@ -71,15 +73,15 @@ namespace ESM4 struct Reference { - FormId mParent; // cell FormId (currently persistent refs only), from the loading sequence - // NOTE: for exterior cells it will be the dummy cell FormId + ESM::RefId mParent; // cell FormId (currently persistent refs only), from the loading sequence + // NOTE: for exterior cells it will be the dummy cell FormId - FormId mFormId; // from the header + ESM::RefId mId; // from the header std::uint32_t mFlags; // from the header, see enum type RecordFlag for details std::string mEditorId; std::string mFullName; - FormId mBaseObj; + ESM::RefId mBaseObj; Placement mPlacement; float mScale = 1.0f; @@ -110,6 +112,8 @@ namespace ESM4 // void save(ESM4::Writer& writer) const; void blank(); + + static constexpr ESM::RecNameInts sRecordId = ESM::REC_REFR4; }; } diff --git a/components/esm4/loadstat.cpp b/components/esm4/loadstat.cpp index e8e789cb1b..edfbf0e4e6 100644 --- a/components/esm4/loadstat.cpp +++ b/components/esm4/loadstat.cpp @@ -30,12 +30,13 @@ #include #include "reader.hpp" -//#include "writer.hpp" +// #include "writer.hpp" void ESM4::Static::load(ESM4::Reader& reader) { - mFormId = reader.hdr().record.id; - reader.adjustFormId(mFormId); + FormId formId = reader.hdr().record.id; + reader.adjustFormId(formId); + mId = ESM::RefId::formIdRefId(formId); mFlags = reader.hdr().record.flags; while (reader.getSubRecordHeader()) diff --git a/components/esm4/loadstat.hpp b/components/esm4/loadstat.hpp index f34ba6a1cf..5d79f84d6f 100644 --- a/components/esm4/loadstat.hpp +++ b/components/esm4/loadstat.hpp @@ -32,6 +32,8 @@ #include #include "formid.hpp" +#include +#include namespace ESM4 { @@ -40,7 +42,7 @@ namespace ESM4 struct Static { - FormId mFormId; // from the header + ESM::RefId mId; // from the header std::uint32_t mFlags; // from the header, see enum type RecordFlag for details std::string mEditorId; @@ -53,6 +55,8 @@ namespace ESM4 // void save(ESM4::Writer& writer) const; // void blank(); + + static constexpr ESM::RecNameInts sRecordId = ESM::REC_STAT4; }; } diff --git a/components/to_utf8/to_utf8.hpp b/components/to_utf8/to_utf8.hpp index df5462dbb9..e72e24a753 100644 --- a/components/to_utf8/to_utf8.hpp +++ b/components/to_utf8/to_utf8.hpp @@ -68,6 +68,8 @@ namespace ToUTF8 /// ASCII-only string. Otherwise returns a view to the input. std::string_view getLegacyEnc(std::string_view input); + const StatelessUtf8Encoder* getStatelessEncoder() const { return &mImpl; } + private: std::string mBuffer; StatelessUtf8Encoder mImpl;