From 0992624c8bc091a1ae81603ef10dfdcd2d535378 Mon Sep 17 00:00:00 2001
From: elsid <elsid.mail@gmail.com>
Date: Sun, 12 Feb 2023 16:27:23 +0100
Subject: [PATCH] Support reading and writing typed ESM::RefId to ESM

---
 apps/esmtool/record.hpp                       |   2 +-
 apps/opencs/model/doc/savingstages.hpp        |   2 +-
 apps/opencs/model/world/columnimp.cpp         |   2 +-
 apps/opencs/model/world/columnimp.hpp         |   2 +-
 apps/opencs/model/world/refidadapterimp.hpp   |   2 +-
 apps/opencs/model/world/scope.cpp             |  52 +++++--
 apps/opencs/model/world/scope.hpp             |   7 +-
 apps/openmw/mwworld/esmstore.hpp              |   5 +-
 apps/openmw_test_suite/esm3/testsaveload.cpp  | 142 +++++++++++++++++-
 apps/openmw_test_suite/mwworld/test_store.cpp |  13 +-
 components/esm/refid.hpp                      |  17 +++
 components/esm3/esmreader.cpp                 |  62 +++++++-
 components/esm3/esmreader.hpp                 |  10 +-
 components/esm3/esmwriter.cpp                 |  77 ++++++++--
 components/esm3/esmwriter.hpp                 |  20 ++-
 components/esm3/formatversion.hpp             |   3 +-
 16 files changed, 361 insertions(+), 57 deletions(-)

diff --git a/apps/esmtool/record.hpp b/apps/esmtool/record.hpp
index e7d5240c51..57f80e5571 100644
--- a/apps/esmtool/record.hpp
+++ b/apps/esmtool/record.hpp
@@ -82,7 +82,7 @@ namespace EsmTool
         {
         }
 
-        std::string getId() const override { return mData.mId.getRefIdString(); }
+        std::string getId() const override { return mData.mId.toDebugString(); }
 
         T& get() { return mData; }
 
diff --git a/apps/opencs/model/doc/savingstages.hpp b/apps/opencs/model/doc/savingstages.hpp
index dd76c5585f..5423b8f504 100644
--- a/apps/opencs/model/doc/savingstages.hpp
+++ b/apps/opencs/model/doc/savingstages.hpp
@@ -99,7 +99,7 @@ namespace CSMDoc
     template <class CollectionT>
     void WriteCollectionStage<CollectionT>::perform(int stage, Messages& messages)
     {
-        if (CSMWorld::getScopeFromId(mCollection.getRecord(stage).get().mId.getRefIdString()) != mScope)
+        if (CSMWorld::getScopeFromId(mCollection.getRecord(stage).get().mId) != mScope)
             return;
 
         ESM::ESMWriter& writer = mState.getWriter();
diff --git a/apps/opencs/model/world/columnimp.cpp b/apps/opencs/model/world/columnimp.cpp
index d9d5971aa6..a9b2c867a0 100644
--- a/apps/opencs/model/world/columnimp.cpp
+++ b/apps/opencs/model/world/columnimp.cpp
@@ -20,7 +20,7 @@ namespace CSMWorld
 
     QVariant LandTextureNicknameColumn::get(const Record<LandTexture>& record) const
     {
-        return QString::fromUtf8(record.get().mId.getRefIdString().c_str());
+        return QString::fromStdString(record.get().mId.toString());
     }
 
     void LandTextureNicknameColumn::set(Record<LandTexture>& record, const QVariant& data)
diff --git a/apps/opencs/model/world/columnimp.hpp b/apps/opencs/model/world/columnimp.hpp
index 1b8a43cfd7..3cef2fdd00 100644
--- a/apps/opencs/model/world/columnimp.hpp
+++ b/apps/opencs/model/world/columnimp.hpp
@@ -62,7 +62,7 @@ namespace CSMWorld
 
         QVariant get(const Record<ESXRecordT>& record) const override
         {
-            return QString::fromUtf8(record.get().mId.getRefIdString().c_str());
+            return QString::fromStdString(record.get().mId.toString());
         }
 
         bool isEditable() const override { return false; }
diff --git a/apps/opencs/model/world/refidadapterimp.hpp b/apps/opencs/model/world/refidadapterimp.hpp
index 6ed23670f7..9d994a307c 100644
--- a/apps/opencs/model/world/refidadapterimp.hpp
+++ b/apps/opencs/model/world/refidadapterimp.hpp
@@ -106,7 +106,7 @@ namespace CSMWorld
             = static_cast<const Record<RecordT>&>(data.getRecord(RefIdData::LocalIndex(index, mType)));
 
         if (column == mBase.mId)
-            return QString::fromUtf8(record.get().mId.getRefIdString().c_str());
+            return QString::fromStdString(record.get().mId.toString());
 
         if (column == mBase.mModified)
         {
diff --git a/apps/opencs/model/world/scope.cpp b/apps/opencs/model/world/scope.cpp
index b54b186084..175074e49a 100644
--- a/apps/opencs/model/world/scope.cpp
+++ b/apps/opencs/model/world/scope.cpp
@@ -2,23 +2,43 @@
 
 #include <string_view>
 
-#include <components/misc/strings/lower.hpp>
+#include <components/esm/refid.hpp>
+#include <components/misc/strings/algorithm.hpp>
 
-CSMWorld::Scope CSMWorld::getScopeFromId(const std::string& id)
+namespace CSMWorld
 {
-    // get root namespace
-    std::string namespace_;
-
-    std::string::size_type i = id.find("::");
-
-    if (i != std::string::npos)
-        namespace_ = Misc::StringUtils::lowerCase(std::string_view(id).substr(0, i));
-
-    if (namespace_ == "project")
-        return Scope_Project;
-
-    if (namespace_ == "session")
-        return Scope_Session;
+    namespace
+    {
+        struct GetScope
+        {
+            Scope operator()(ESM::StringRefId v) const
+            {
+                std::string_view namespace_;
+
+                const std::string::size_type i = v.getValue().find("::");
+
+                if (i != std::string::npos)
+                    namespace_ = std::string_view(v.getValue()).substr(0, i);
+
+                if (Misc::StringUtils::ciEqual(namespace_, "project"))
+                    return Scope_Project;
+
+                if (Misc::StringUtils::ciEqual(namespace_, "session"))
+                    return Scope_Session;
+
+                return Scope_Content;
+            }
+
+            template <class T>
+            Scope operator()(const T& /*v*/) const
+            {
+                return Scope_Content;
+            }
+        };
+    }
+}
 
-    return Scope_Content;
+CSMWorld::Scope CSMWorld::getScopeFromId(ESM::RefId id)
+{
+    return visit(GetScope{}, id);
 }
diff --git a/apps/opencs/model/world/scope.hpp b/apps/opencs/model/world/scope.hpp
index dc00987858..c679a78de8 100644
--- a/apps/opencs/model/world/scope.hpp
+++ b/apps/opencs/model/world/scope.hpp
@@ -1,7 +1,10 @@
 #ifndef CSM_WOLRD_SCOPE_H
 #define CSM_WOLRD_SCOPE_H
 
-#include <string>
+namespace ESM
+{
+    class RefId;
+}
 
 namespace CSMWorld
 {
@@ -17,7 +20,7 @@ namespace CSMWorld
         Scope_Session = 4
     };
 
-    Scope getScopeFromId(const std::string& id);
+    Scope getScopeFromId(ESM::RefId id);
 }
 
 #endif
diff --git a/apps/openmw/mwworld/esmstore.hpp b/apps/openmw/mwworld/esmstore.hpp
index b1d77e8309..9b4994a457 100644
--- a/apps/openmw/mwworld/esmstore.hpp
+++ b/apps/openmw/mwworld/esmstore.hpp
@@ -221,10 +221,7 @@ namespace MWWorld
         {
             Store<T>& store = getWritable<T>();
             if (store.search(x.mId) != nullptr)
-            {
-                const std::string msg = "Try to override existing record '" + x.mId.getRefIdString() + "'";
-                throw std::runtime_error(msg);
-            }
+                throw std::runtime_error("Try to override existing record " + x.mId.toDebugString());
 
             T* ptr = store.insertStatic(x);
             if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*>)
diff --git a/apps/openmw_test_suite/esm3/testsaveload.cpp b/apps/openmw_test_suite/esm3/testsaveload.cpp
index 981c3f3ee3..2204b3fcec 100644
--- a/apps/openmw_test_suite/esm3/testsaveload.cpp
+++ b/apps/openmw_test_suite/esm3/testsaveload.cpp
@@ -1,6 +1,9 @@
 #include <components/esm/fourcc.hpp>
 #include <components/esm3/esmreader.hpp>
 #include <components/esm3/esmwriter.hpp>
+#include <components/esm3/loadcont.hpp>
+#include <components/esm3/loadregn.hpp>
+#include <components/esm3/loadscpt.hpp>
 #include <components/esm3/player.hpp>
 
 #include <gmock/gmock.h>
@@ -9,15 +12,51 @@
 #include <array>
 #include <memory>
 #include <random>
+#include <type_traits>
 
 namespace ESM
 {
+    namespace
+    {
+        auto tie(const ContItem& value)
+        {
+            return std::tie(value.mCount, value.mItem);
+        }
+
+        auto tie(const ESM::Region::SoundRef& value)
+        {
+            return std::tie(value.mSound, value.mChance);
+        }
+    }
+
+    inline bool operator==(const ESM::ContItem& lhs, const ESM::ContItem& rhs)
+    {
+        return tie(lhs) == tie(rhs);
+    }
+
+    inline std::ostream& operator<<(std::ostream& stream, const ESM::ContItem& value)
+    {
+        return stream << "ESM::ContItem {.mCount = " << value.mCount << ", .mItem = '" << value.mItem << "'}";
+    }
+
+    inline bool operator==(const ESM::Region::SoundRef& lhs, const ESM::Region::SoundRef& rhs)
+    {
+        return tie(lhs) == tie(rhs);
+    }
+
+    inline std::ostream& operator<<(std::ostream& stream, const ESM::Region::SoundRef& value)
+    {
+        return stream << "ESM::Region::SoundRef {.mSound = '" << value.mSound << "', .mChance = " << value.mChance
+                      << "}";
+    }
+
     namespace
     {
         using namespace ::testing;
 
         constexpr std::array formats = {
             MaxLimitedSizeStringsFormatVersion,
+            MaxStringRefIdFormatVersion,
             CurrentSaveGameFormatVersion,
         };
 
@@ -47,12 +86,41 @@ namespace ESM
             return stream;
         }
 
+        template <class T, class = std::void_t<>>
+        struct HasLoad : std::false_type
+        {
+        };
+
         template <class T>
-        void load(ESMReader& reader, T& record)
+        struct HasLoad<T, std::void_t<decltype(std::declval<T>().load(std::declval<ESMReader&>()))>> : std::true_type
+        {
+        };
+
+        template <class T>
+        auto load(ESMReader& reader, T& record) -> std::enable_if_t<HasLoad<std::decay_t<T>>::value>
         {
             record.load(reader);
         }
 
+        template <class T, class = std::void_t<>>
+        struct HasLoadWithDelete : std::false_type
+        {
+        };
+
+        template <class T>
+        struct HasLoadWithDelete<T,
+            std::void_t<decltype(std::declval<T>().load(std::declval<ESMReader&>(), std::declval<bool&>()))>>
+            : std::true_type
+        {
+        };
+
+        template <class T>
+        auto load(ESMReader& reader, T& record) -> std::enable_if_t<HasLoadWithDelete<std::decay_t<T>>::value>
+        {
+            bool deleted = false;
+            record.load(reader, deleted);
+        }
+
         void load(ESMReader& reader, CellRef& record)
         {
             bool deleted = false;
@@ -106,6 +174,40 @@ namespace ESM
             EXPECT_EQ(reader.getDesc(), description);
         }
 
+        TEST_F(Esm3SaveLoadRecordTest, containerContItemShouldSupportRefIdLongerThan32)
+        {
+            Container record;
+            record.blank();
+            record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 42, .mItem = generateRandomRefId(33) });
+            record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 13, .mItem = generateRandomRefId(33) });
+            Container result;
+            saveAndLoadRecord(record, CurrentSaveGameFormatVersion, result);
+            EXPECT_EQ(result.mInventory.mList, record.mInventory.mList);
+        }
+
+        TEST_F(Esm3SaveLoadRecordTest, regionSoundRefShouldSupportRefIdLongerThan32)
+        {
+            Region record;
+            record.blank();
+            record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(33), .mChance = 42 });
+            record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(33), .mChance = 13 });
+            Region result;
+            saveAndLoadRecord(record, CurrentSaveGameFormatVersion, result);
+            EXPECT_EQ(result.mSoundList, record.mSoundList);
+        }
+
+        TEST_F(Esm3SaveLoadRecordTest, scriptSoundRefShouldSupportRefIdLongerThan32)
+        {
+            Script record;
+            record.blank();
+            record.mId = generateRandomRefId(33);
+            record.mData.mNumShorts = 42;
+            Script result;
+            saveAndLoadRecord(record, CurrentSaveGameFormatVersion, result);
+            EXPECT_EQ(result.mId, record.mId);
+            EXPECT_EQ(result.mData.mNumShorts, record.mData.mNumShorts);
+        }
+
         TEST_P(Esm3SaveLoadRecordTest, playerShouldNotChange)
         {
             std::minstd_rand random;
@@ -151,6 +253,44 @@ namespace ESM
             EXPECT_EQ(record.mLastHitObject, result.mLastHitObject);
         }
 
+        TEST_P(Esm3SaveLoadRecordTest, containerShouldNotChange)
+        {
+            Container record;
+            record.blank();
+            record.mId = generateRandomRefId();
+            record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 42, .mItem = generateRandomRefId(32) });
+            record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 13, .mItem = generateRandomRefId(32) });
+            Container result;
+            saveAndLoadRecord(record, GetParam(), result);
+            EXPECT_EQ(result.mId, record.mId);
+            EXPECT_EQ(result.mInventory.mList, record.mInventory.mList);
+        }
+
+        TEST_P(Esm3SaveLoadRecordTest, regionShouldNotChange)
+        {
+            Region record;
+            record.blank();
+            record.mId = generateRandomRefId();
+            record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(32), .mChance = 42 });
+            record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(32), .mChance = 13 });
+            Region result;
+            saveAndLoadRecord(record, GetParam(), result);
+            EXPECT_EQ(result.mId, record.mId);
+            EXPECT_EQ(result.mSoundList, record.mSoundList);
+        }
+
+        TEST_P(Esm3SaveLoadRecordTest, scriptShouldNotChange)
+        {
+            Script record;
+            record.blank();
+            record.mId = generateRandomRefId(32);
+            record.mData.mNumShorts = 42;
+            Script result;
+            saveAndLoadRecord(record, GetParam(), result);
+            EXPECT_EQ(result.mId, record.mId);
+            EXPECT_EQ(result.mData.mNumShorts, record.mData.mNumShorts);
+        }
+
         INSTANTIATE_TEST_SUITE_P(FormatVersions, Esm3SaveLoadRecordTest, ValuesIn(formats));
     }
 }
diff --git a/apps/openmw_test_suite/mwworld/test_store.cpp b/apps/openmw_test_suite/mwworld/test_store.cpp
index ff0365ca37..027268e103 100644
--- a/apps/openmw_test_suite/mwworld/test_store.cpp
+++ b/apps/openmw_test_suite/mwworld/test_store.cpp
@@ -281,6 +281,7 @@ namespace
         ESM::MaxOldAiPackageFormatVersion,
         ESM::MaxOldSkillsAndAttributesFormatVersion,
         ESM::MaxOldCreatureStatsFormatVersion,
+        ESM::MaxStringRefIdFormatVersion,
         ESM::CurrentSaveGameFormatVersion,
     };
 
@@ -420,8 +421,10 @@ TYPED_TEST_P(StoreTest, overwrite_test)
 
 namespace
 {
+    using namespace ::testing;
+
     template <class T>
-    struct StoreSaveLoadTest : public ::testing::Test
+    struct StoreSaveLoadTest : public Test
     {
     };
 
@@ -445,11 +448,11 @@ namespace
         using RecordType = TypeParam;
 
         const int index = 3;
-        ESM::RefId refId;
+        decltype(RecordType::mId) refId;
         if constexpr (hasIndex<RecordType> && !std::is_same_v<RecordType, ESM::LandTexture>)
             refId = ESM::RefId::stringRefId(RecordType::indexToId(index));
         else
-            refId = ESM::RefId::stringRefId("foobar");
+            refId = ESM::StringRefId("foobar");
 
         for (const ESM::FormatVersion formatVersion : formats)
         {
@@ -473,7 +476,7 @@ namespace
             MWWorld::ESMStore esmStore;
 
             reader.open(getEsmFile(record, false, formatVersion), "filename");
-            esmStore.load(reader, &dummyListener, dialogue);
+            ASSERT_NO_THROW(esmStore.load(reader, &dummyListener, dialogue));
             esmStore.setUp();
 
             const RecordType* result = nullptr;
@@ -551,7 +554,7 @@ namespace
     template <class... T>
     struct AsTestingTypes<std::tuple<T...>>
     {
-        using Type = testing::Types<T...>;
+        using Type = Types<T...>;
     };
 
     using RecordTypes = typename ToRecordTypes<MWWorld::ESMStore::StoreTuple>::Type;
diff --git a/components/esm/refid.hpp b/components/esm/refid.hpp
index 0867740705..47108da0a7 100644
--- a/components/esm/refid.hpp
+++ b/components/esm/refid.hpp
@@ -5,6 +5,7 @@
 #include <iosfwd>
 #include <string>
 #include <string_view>
+#include <type_traits>
 #include <variant>
 
 #include <components/misc/notnullptr.hpp>
@@ -27,6 +28,14 @@ namespace ESM
         friend std::ostream& operator<<(std::ostream& stream, EmptyRefId value);
     };
 
+    enum class RefIdType : std::uint8_t
+    {
+        Empty = 0,
+        SizedString = 1,
+        UnsizedString = 2,
+        FormId = 3,
+    };
+
     // RefId is used to represent an Id that identifies an ESM record. These Ids can then be used in
     // ESM::Stores to find the actual record. These Ids can be serialized/de-serialized, stored on disk and remain
     // valid. They are used by ESM files, by records to reference other ESM records.
@@ -89,6 +98,14 @@ namespace ESM
 
         friend std::ostream& operator<<(std::ostream& stream, RefId value);
 
+        template <class F, class... T>
+        friend constexpr auto visit(F&& f, T&&... v)
+            -> std::enable_if_t<(std::is_same_v<std::decay_t<T>, RefId> && ...),
+                decltype(std::visit(std::forward<F>(f), std::forward<T>(v).mValue...))>
+        {
+            return std::visit(std::forward<F>(f), std::forward<T>(v).mValue...);
+        }
+
         friend struct std::hash<ESM::RefId>;
 
     private:
diff --git a/components/esm3/esmreader.cpp b/components/esm3/esmreader.cpp
index 62f78af80f..d96901227d 100644
--- a/components/esm3/esmreader.cpp
+++ b/components/esm3/esmreader.cpp
@@ -1,6 +1,7 @@
 #include "esmreader.hpp"
 
 #include "readerscache.hpp"
+#include "savedgame.hpp"
 
 #include <components/files/conversion.hpp>
 #include <components/files/openfile.hpp>
@@ -158,6 +159,9 @@ namespace ESM
     {
         getSubHeader();
 
+        if (mHeader.mFormatVersion > MaxStringRefIdFormatVersion)
+            return getStringView(mCtx.leftSub);
+
         // Hack to make MultiMark.esp load. Zero-length strings do not
         // occur in any of the official mods, but MultiMark makes use of
         // them. For some reason, they break the rules, and contain a byte
@@ -177,7 +181,10 @@ namespace ESM
 
     RefId ESMReader::getRefId()
     {
-        return ESM::RefId::stringRefId(getHStringView());
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return ESM::RefId::stringRefId(getHStringView());
+        getSubHeader();
+        return getRefIdImpl(mCtx.leftSub);
     }
 
     void ESMReader::skipHString()
@@ -189,7 +196,8 @@ namespace ESM
         // them. For some reason, they break the rules, and contain a byte
         // (value 0) even if the header says there is no data. If
         // Morrowind accepts it, so should we.
-        if (mCtx.leftSub == 0 && hasMoreSubs() && !mEsm->peek())
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion && mCtx.leftSub == 0 && hasMoreSubs()
+            && !mEsm->peek())
         {
             // Skip the following zero byte
             mCtx.leftRec--;
@@ -378,6 +386,13 @@ namespace ESM
         return std::string(getStringView(size));
     }
 
+    RefId ESMReader::getMaybeFixedRefIdSize(std::size_t size)
+    {
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return RefId::stringRefId(getMaybeFixedStringSize(size));
+        return getRefIdImpl(mCtx.leftSub);
+    }
+
     std::string_view ESMReader::getStringView(std::size_t size)
     {
         if (mBuffer.size() <= size)
@@ -403,7 +418,48 @@ namespace ESM
 
     RefId ESMReader::getRefId(std::size_t size)
     {
-        return RefId::stringRefId(getStringView(size));
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return ESM::RefId::stringRefId(getStringView(size));
+        return getRefIdImpl(size);
+    }
+
+    RefId ESMReader::getRefIdImpl(std::size_t size)
+    {
+        RefIdType refIdType = RefIdType::Empty;
+        getT(refIdType);
+
+        switch (refIdType)
+        {
+            case RefIdType::Empty:
+                return RefId();
+            case RefIdType::SizedString:
+            {
+                const std::size_t minSize = sizeof(refIdType) + sizeof(StringSizeType);
+                if (size < minSize)
+                    fail("Requested RefId record size is too small (" + std::to_string(size) + " < "
+                        + std::to_string(minSize) + ")");
+                StringSizeType storedSize = 0;
+                getT(storedSize);
+                const std::size_t maxSize = size - minSize;
+                if (storedSize > maxSize)
+                    fail("RefId string does not fit subrecord size (" + std::to_string(storedSize) + " > "
+                        + std::to_string(maxSize) + ")");
+                return RefId::stringRefId(getStringView(storedSize));
+            }
+            case RefIdType::UnsizedString:
+                if (size < sizeof(refIdType))
+                    fail("Requested RefId record size is too small (" + std::to_string(size) + " < "
+                        + std::to_string(sizeof(refIdType)) + ")");
+                return RefId::stringRefId(getStringView(size - sizeof(refIdType)));
+            case RefIdType::FormId:
+            {
+                ESM4::FormId formId{};
+                getT(formId);
+                return RefId::formIdRefId(formId);
+            }
+        }
+
+        fail("Unsupported RefIdType: " + std::to_string(static_cast<unsigned>(refIdType)));
     }
 
     [[noreturn]] void ESMReader::fail(const std::string& msg)
diff --git a/components/esm3/esmreader.hpp b/components/esm3/esmreader.hpp
index c1dcf99297..3e667e6972 100644
--- a/components/esm3/esmreader.hpp
+++ b/components/esm3/esmreader.hpp
@@ -273,13 +273,17 @@ namespace ESM
             skip(sizeof(T));
         }
 
-        void getExact(void* x, int size) { mEsm->read((char*)x, size); }
+        void getExact(void* x, std::size_t size)
+        {
+            mEsm->read(static_cast<char*>(x), static_cast<std::streamsize>(size));
+        }
+
         void getName(NAME& name) { getT(name); }
         void getUint(uint32_t& u) { getT(u); }
 
         std::string getMaybeFixedStringSize(std::size_t size);
 
-        RefId getMaybeFixedRefIdSize(std::size_t size) { return RefId::stringRefId(getMaybeFixedStringSize(size)); }
+        RefId getMaybeFixedRefIdSize(std::size_t size);
 
         // Read the next 'size' bytes and return them as a string. Converts
         // them from native encoding to UTF8 in the process.
@@ -315,6 +319,8 @@ namespace ESM
 
         void clearCtx();
 
+        RefId getRefIdImpl(std::size_t size);
+
         std::unique_ptr<std::istream> mEsm;
 
         ESM_Context mCtx;
diff --git a/components/esm3/esmwriter.cpp b/components/esm3/esmwriter.cpp
index 3cd534e404..60f9cea17b 100644
--- a/components/esm3/esmwriter.cpp
+++ b/components/esm3/esmwriter.cpp
@@ -4,10 +4,52 @@
 #include <fstream>
 #include <stdexcept>
 
+#include <components/debug/debuglog.hpp>
+#include <components/misc/notnullptr.hpp>
 #include <components/to_utf8/to_utf8.hpp>
 
+#include "formatversion.hpp"
+
 namespace ESM
 {
+    namespace
+    {
+        template <bool sizedString>
+        struct WriteRefId
+        {
+            ESMWriter& mWriter;
+
+            explicit WriteRefId(ESMWriter& writer)
+                : mWriter(writer)
+            {
+            }
+
+            void operator()(EmptyRefId /*v*/) const { mWriter.writeT(RefIdType::Empty); }
+
+            void operator()(StringRefId v) const
+            {
+                constexpr StringSizeType maxSize = std::numeric_limits<StringSizeType>::max();
+                if (v.getValue().size() > maxSize)
+                    throw std::runtime_error("RefId string size is too long: \"" + v.getValue().substr(0, 64)
+                        + "<...>\" (" + std::to_string(v.getValue().size()) + " > " + std::to_string(maxSize) + ")");
+                if constexpr (sizedString)
+                {
+                    mWriter.writeT(RefIdType::SizedString);
+                    mWriter.writeT(static_cast<StringSizeType>(v.getValue().size()));
+                }
+                else
+                    mWriter.writeT(RefIdType::UnsizedString);
+                mWriter.write(v.getValue().data(), v.getValue().size());
+            }
+
+            void operator()(FormIdRefId v) const
+            {
+                mWriter.writeT(RefIdType::FormId);
+                mWriter.writeT(v.getValue());
+            }
+        };
+    }
+
     ESMWriter::ESMWriter()
         : mRecords()
         , mStream(nullptr)
@@ -167,14 +209,18 @@ namespace ESM
         endRecord(name);
     }
 
-    void ESMWriter::writeHNRefId(NAME name, const RefId& value)
+    void ESMWriter::writeHNRefId(NAME name, RefId value)
     {
-        writeHNString(name, value.getRefIdString());
+        startSubRecord(name);
+        writeHRefId(value);
+        endRecord(name);
     }
 
-    void ESMWriter::writeHNRefId(NAME name, const RefId& value, std::size_t size)
+    void ESMWriter::writeHNRefId(NAME name, RefId value, std::size_t size)
     {
-        writeHNString(name, value.getRefIdString(), size);
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return writeHNString(name, value.getRefIdString(), size);
+        writeHNRefId(name, value);
     }
 
     void ESMWriter::writeMaybeFixedSizeString(const std::string& data, std::size_t size)
@@ -220,19 +266,30 @@ namespace ESM
             write("\0", 1);
     }
 
-    void ESMWriter::writeMaybeFixedSizeRefId(const RefId& value, std::size_t size)
+    void ESMWriter::writeMaybeFixedSizeRefId(RefId value, std::size_t size)
+    {
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return writeMaybeFixedSizeString(value.getRefIdString(), size);
+        visit(WriteRefId<true>(*this), value);
+    }
+
+    void ESMWriter::writeHRefId(RefId value)
     {
-        writeMaybeFixedSizeString(value.getRefIdString(), size);
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return writeHString(value.getRefIdString());
+        writeRefId(value);
     }
 
-    void ESMWriter::writeHRefId(const RefId& value)
+    void ESMWriter::writeHCRefId(RefId value)
     {
-        writeHString(value.getRefIdString());
+        if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
+            return writeHCString(value.getRefIdString());
+        writeRefId(value);
     }
 
-    void ESMWriter::writeHCRefId(const RefId& value)
+    void ESMWriter::writeRefId(RefId value)
     {
-        writeHCString(value.getRefIdString());
+        visit(WriteRefId<false>(*this), value);
     }
 
     void ESMWriter::writeName(NAME name)
diff --git a/components/esm3/esmwriter.hpp b/components/esm3/esmwriter.hpp
index d10ba26241..54abca41f2 100644
--- a/components/esm3/esmwriter.hpp
+++ b/components/esm3/esmwriter.hpp
@@ -47,6 +47,8 @@ namespace ESM
         // It is a good idea to compare this with the value you wrote into the header (setRecordCount)
         // It should be the record count you set + 1 (1 additional record for the TES3 header)
         int getRecordCount() const { return mRecordCount; }
+
+        FormatVersion getFormatVersion() const { return mHeader.mFormatVersion; }
         void setFormatVersion(FormatVersion value);
 
         void clearMaster();
@@ -78,24 +80,24 @@ namespace ESM
                 writeHNCString(name, data);
         }
 
-        void writeHNRefId(NAME name, const RefId& value);
+        void writeHNRefId(NAME name, RefId value);
 
-        void writeHNRefId(NAME name, const RefId& value, std::size_t size);
+        void writeHNRefId(NAME name, RefId value, std::size_t size);
 
-        void writeHNCRefId(NAME name, const RefId& value)
+        void writeHNCRefId(NAME name, RefId value)
         {
             startSubRecord(name);
             writeHCRefId(value);
             endRecord(name);
         }
 
-        void writeHNORefId(NAME name, const RefId& value)
+        void writeHNORefId(NAME name, RefId value)
         {
             if (!value.empty())
                 writeHNRefId(name, value);
         }
 
-        void writeHNOCRefId(NAME name, const RefId& value)
+        void writeHNOCRefId(NAME name, RefId value)
         {
             if (!value.empty())
                 writeHNCRefId(name, value);
@@ -165,11 +167,11 @@ namespace ESM
         void writeHString(const std::string& data);
         void writeHCString(const std::string& data);
 
-        void writeMaybeFixedSizeRefId(const RefId& value, std::size_t size);
+        void writeMaybeFixedSizeRefId(RefId value, std::size_t size);
 
-        void writeHRefId(const RefId& value);
+        void writeHRefId(RefId refId);
 
-        void writeHCRefId(const RefId& value);
+        void writeHCRefId(RefId refId);
 
         void writeName(NAME data);
 
@@ -184,6 +186,8 @@ namespace ESM
         bool mCounting;
 
         Header mHeader;
+
+        void writeRefId(RefId value);
     };
 }
 
diff --git a/components/esm3/formatversion.hpp b/components/esm3/formatversion.hpp
index 47c08988a7..9ef945d14d 100644
--- a/components/esm3/formatversion.hpp
+++ b/components/esm3/formatversion.hpp
@@ -20,7 +20,8 @@ namespace ESM
     inline constexpr FormatVersion MaxOldSkillsAndAttributesFormatVersion = 18;
     inline constexpr FormatVersion MaxOldCreatureStatsFormatVersion = 19;
     inline constexpr FormatVersion MaxLimitedSizeStringsFormatVersion = 22;
-    inline constexpr FormatVersion CurrentSaveGameFormatVersion = 23;
+    inline constexpr FormatVersion MaxStringRefIdFormatVersion = 23;
+    inline constexpr FormatVersion CurrentSaveGameFormatVersion = 24;
 }
 
 #endif