Support reading and writing typed ESM::RefId to ESM

depth-refraction
elsid 2 years ago
parent 069d4255b9
commit 0992624c8b
No known key found for this signature in database
GPG Key ID: 4DE04C198CBA7625

@ -82,7 +82,7 @@ namespace EsmTool
{
}
std::string getId() const override { return mData.mId.getRefIdString(); }
std::string getId() const override { return mData.mId.toDebugString(); }
T& get() { return mData; }

@ -99,7 +99,7 @@ namespace CSMDoc
template <class CollectionT>
void WriteCollectionStage<CollectionT>::perform(int stage, Messages& messages)
{
if (CSMWorld::getScopeFromId(mCollection.getRecord(stage).get().mId.getRefIdString()) != mScope)
if (CSMWorld::getScopeFromId(mCollection.getRecord(stage).get().mId) != mScope)
return;
ESM::ESMWriter& writer = mState.getWriter();

@ -20,7 +20,7 @@ namespace CSMWorld
QVariant LandTextureNicknameColumn::get(const Record<LandTexture>& record) const
{
return QString::fromUtf8(record.get().mId.getRefIdString().c_str());
return QString::fromStdString(record.get().mId.toString());
}
void LandTextureNicknameColumn::set(Record<LandTexture>& record, const QVariant& data)

@ -62,7 +62,7 @@ namespace CSMWorld
QVariant get(const Record<ESXRecordT>& record) const override
{
return QString::fromUtf8(record.get().mId.getRefIdString().c_str());
return QString::fromStdString(record.get().mId.toString());
}
bool isEditable() const override { return false; }

@ -106,7 +106,7 @@ namespace CSMWorld
= static_cast<const Record<RecordT>&>(data.getRecord(RefIdData::LocalIndex(index, mType)));
if (column == mBase.mId)
return QString::fromUtf8(record.get().mId.getRefIdString().c_str());
return QString::fromStdString(record.get().mId.toString());
if (column == mBase.mModified)
{

@ -2,23 +2,43 @@
#include <string_view>
#include <components/misc/strings/lower.hpp>
#include <components/esm/refid.hpp>
#include <components/misc/strings/algorithm.hpp>
CSMWorld::Scope CSMWorld::getScopeFromId(const std::string& id)
namespace CSMWorld
{
// get root namespace
std::string namespace_;
std::string::size_type i = id.find("::");
if (i != std::string::npos)
namespace_ = Misc::StringUtils::lowerCase(std::string_view(id).substr(0, i));
if (namespace_ == "project")
return Scope_Project;
if (namespace_ == "session")
return Scope_Session;
namespace
{
struct GetScope
{
Scope operator()(ESM::StringRefId v) const
{
std::string_view namespace_;
const std::string::size_type i = v.getValue().find("::");
if (i != std::string::npos)
namespace_ = std::string_view(v.getValue()).substr(0, i);
if (Misc::StringUtils::ciEqual(namespace_, "project"))
return Scope_Project;
if (Misc::StringUtils::ciEqual(namespace_, "session"))
return Scope_Session;
return Scope_Content;
}
template <class T>
Scope operator()(const T& /*v*/) const
{
return Scope_Content;
}
};
}
}
return Scope_Content;
CSMWorld::Scope CSMWorld::getScopeFromId(ESM::RefId id)
{
return visit(GetScope{}, id);
}

@ -1,7 +1,10 @@
#ifndef CSM_WOLRD_SCOPE_H
#define CSM_WOLRD_SCOPE_H
#include <string>
namespace ESM
{
class RefId;
}
namespace CSMWorld
{
@ -17,7 +20,7 @@ namespace CSMWorld
Scope_Session = 4
};
Scope getScopeFromId(const std::string& id);
Scope getScopeFromId(ESM::RefId id);
}
#endif

@ -221,10 +221,7 @@ namespace MWWorld
{
Store<T>& store = getWritable<T>();
if (store.search(x.mId) != nullptr)
{
const std::string msg = "Try to override existing record '" + x.mId.getRefIdString() + "'";
throw std::runtime_error(msg);
}
throw std::runtime_error("Try to override existing record " + x.mId.toDebugString());
T* ptr = store.insertStatic(x);
if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*>)

@ -1,6 +1,9 @@
#include <components/esm/fourcc.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm3/loadcont.hpp>
#include <components/esm3/loadregn.hpp>
#include <components/esm3/loadscpt.hpp>
#include <components/esm3/player.hpp>
#include <gmock/gmock.h>
@ -9,15 +12,51 @@
#include <array>
#include <memory>
#include <random>
#include <type_traits>
namespace ESM
{
namespace
{
auto tie(const ContItem& value)
{
return std::tie(value.mCount, value.mItem);
}
auto tie(const ESM::Region::SoundRef& value)
{
return std::tie(value.mSound, value.mChance);
}
}
inline bool operator==(const ESM::ContItem& lhs, const ESM::ContItem& rhs)
{
return tie(lhs) == tie(rhs);
}
inline std::ostream& operator<<(std::ostream& stream, const ESM::ContItem& value)
{
return stream << "ESM::ContItem {.mCount = " << value.mCount << ", .mItem = '" << value.mItem << "'}";
}
inline bool operator==(const ESM::Region::SoundRef& lhs, const ESM::Region::SoundRef& rhs)
{
return tie(lhs) == tie(rhs);
}
inline std::ostream& operator<<(std::ostream& stream, const ESM::Region::SoundRef& value)
{
return stream << "ESM::Region::SoundRef {.mSound = '" << value.mSound << "', .mChance = " << value.mChance
<< "}";
}
namespace
{
using namespace ::testing;
constexpr std::array formats = {
MaxLimitedSizeStringsFormatVersion,
MaxStringRefIdFormatVersion,
CurrentSaveGameFormatVersion,
};
@ -47,12 +86,41 @@ namespace ESM
return stream;
}
template <class T, class = std::void_t<>>
struct HasLoad : std::false_type
{
};
template <class T>
void load(ESMReader& reader, T& record)
struct HasLoad<T, std::void_t<decltype(std::declval<T>().load(std::declval<ESMReader&>()))>> : std::true_type
{
};
template <class T>
auto load(ESMReader& reader, T& record) -> std::enable_if_t<HasLoad<std::decay_t<T>>::value>
{
record.load(reader);
}
template <class T, class = std::void_t<>>
struct HasLoadWithDelete : std::false_type
{
};
template <class T>
struct HasLoadWithDelete<T,
std::void_t<decltype(std::declval<T>().load(std::declval<ESMReader&>(), std::declval<bool&>()))>>
: std::true_type
{
};
template <class T>
auto load(ESMReader& reader, T& record) -> std::enable_if_t<HasLoadWithDelete<std::decay_t<T>>::value>
{
bool deleted = false;
record.load(reader, deleted);
}
void load(ESMReader& reader, CellRef& record)
{
bool deleted = false;
@ -106,6 +174,40 @@ namespace ESM
EXPECT_EQ(reader.getDesc(), description);
}
TEST_F(Esm3SaveLoadRecordTest, containerContItemShouldSupportRefIdLongerThan32)
{
Container record;
record.blank();
record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 42, .mItem = generateRandomRefId(33) });
record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 13, .mItem = generateRandomRefId(33) });
Container result;
saveAndLoadRecord(record, CurrentSaveGameFormatVersion, result);
EXPECT_EQ(result.mInventory.mList, record.mInventory.mList);
}
TEST_F(Esm3SaveLoadRecordTest, regionSoundRefShouldSupportRefIdLongerThan32)
{
Region record;
record.blank();
record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(33), .mChance = 42 });
record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(33), .mChance = 13 });
Region result;
saveAndLoadRecord(record, CurrentSaveGameFormatVersion, result);
EXPECT_EQ(result.mSoundList, record.mSoundList);
}
TEST_F(Esm3SaveLoadRecordTest, scriptSoundRefShouldSupportRefIdLongerThan32)
{
Script record;
record.blank();
record.mId = generateRandomRefId(33);
record.mData.mNumShorts = 42;
Script result;
saveAndLoadRecord(record, CurrentSaveGameFormatVersion, result);
EXPECT_EQ(result.mId, record.mId);
EXPECT_EQ(result.mData.mNumShorts, record.mData.mNumShorts);
}
TEST_P(Esm3SaveLoadRecordTest, playerShouldNotChange)
{
std::minstd_rand random;
@ -151,6 +253,44 @@ namespace ESM
EXPECT_EQ(record.mLastHitObject, result.mLastHitObject);
}
TEST_P(Esm3SaveLoadRecordTest, containerShouldNotChange)
{
Container record;
record.blank();
record.mId = generateRandomRefId();
record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 42, .mItem = generateRandomRefId(32) });
record.mInventory.mList.push_back(ESM::ContItem{ .mCount = 13, .mItem = generateRandomRefId(32) });
Container result;
saveAndLoadRecord(record, GetParam(), result);
EXPECT_EQ(result.mId, record.mId);
EXPECT_EQ(result.mInventory.mList, record.mInventory.mList);
}
TEST_P(Esm3SaveLoadRecordTest, regionShouldNotChange)
{
Region record;
record.blank();
record.mId = generateRandomRefId();
record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(32), .mChance = 42 });
record.mSoundList.push_back(ESM::Region::SoundRef{ .mSound = generateRandomRefId(32), .mChance = 13 });
Region result;
saveAndLoadRecord(record, GetParam(), result);
EXPECT_EQ(result.mId, record.mId);
EXPECT_EQ(result.mSoundList, record.mSoundList);
}
TEST_P(Esm3SaveLoadRecordTest, scriptShouldNotChange)
{
Script record;
record.blank();
record.mId = generateRandomRefId(32);
record.mData.mNumShorts = 42;
Script result;
saveAndLoadRecord(record, GetParam(), result);
EXPECT_EQ(result.mId, record.mId);
EXPECT_EQ(result.mData.mNumShorts, record.mData.mNumShorts);
}
INSTANTIATE_TEST_SUITE_P(FormatVersions, Esm3SaveLoadRecordTest, ValuesIn(formats));
}
}

@ -281,6 +281,7 @@ namespace
ESM::MaxOldAiPackageFormatVersion,
ESM::MaxOldSkillsAndAttributesFormatVersion,
ESM::MaxOldCreatureStatsFormatVersion,
ESM::MaxStringRefIdFormatVersion,
ESM::CurrentSaveGameFormatVersion,
};
@ -420,8 +421,10 @@ TYPED_TEST_P(StoreTest, overwrite_test)
namespace
{
using namespace ::testing;
template <class T>
struct StoreSaveLoadTest : public ::testing::Test
struct StoreSaveLoadTest : public Test
{
};
@ -445,11 +448,11 @@ namespace
using RecordType = TypeParam;
const int index = 3;
ESM::RefId refId;
decltype(RecordType::mId) refId;
if constexpr (hasIndex<RecordType> && !std::is_same_v<RecordType, ESM::LandTexture>)
refId = ESM::RefId::stringRefId(RecordType::indexToId(index));
else
refId = ESM::RefId::stringRefId("foobar");
refId = ESM::StringRefId("foobar");
for (const ESM::FormatVersion formatVersion : formats)
{
@ -473,7 +476,7 @@ namespace
MWWorld::ESMStore esmStore;
reader.open(getEsmFile(record, false, formatVersion), "filename");
esmStore.load(reader, &dummyListener, dialogue);
ASSERT_NO_THROW(esmStore.load(reader, &dummyListener, dialogue));
esmStore.setUp();
const RecordType* result = nullptr;
@ -551,7 +554,7 @@ namespace
template <class... T>
struct AsTestingTypes<std::tuple<T...>>
{
using Type = testing::Types<T...>;
using Type = Types<T...>;
};
using RecordTypes = typename ToRecordTypes<MWWorld::ESMStore::StoreTuple>::Type;

@ -5,6 +5,7 @@
#include <iosfwd>
#include <string>
#include <string_view>
#include <type_traits>
#include <variant>
#include <components/misc/notnullptr.hpp>
@ -27,6 +28,14 @@ namespace ESM
friend std::ostream& operator<<(std::ostream& stream, EmptyRefId value);
};
enum class RefIdType : std::uint8_t
{
Empty = 0,
SizedString = 1,
UnsizedString = 2,
FormId = 3,
};
// RefId is used to represent an Id that identifies an ESM record. These Ids can then be used in
// ESM::Stores to find the actual record. These Ids can be serialized/de-serialized, stored on disk and remain
// valid. They are used by ESM files, by records to reference other ESM records.
@ -89,6 +98,14 @@ namespace ESM
friend std::ostream& operator<<(std::ostream& stream, RefId value);
template <class F, class... T>
friend constexpr auto visit(F&& f, T&&... v)
-> std::enable_if_t<(std::is_same_v<std::decay_t<T>, RefId> && ...),
decltype(std::visit(std::forward<F>(f), std::forward<T>(v).mValue...))>
{
return std::visit(std::forward<F>(f), std::forward<T>(v).mValue...);
}
friend struct std::hash<ESM::RefId>;
private:

@ -1,6 +1,7 @@
#include "esmreader.hpp"
#include "readerscache.hpp"
#include "savedgame.hpp"
#include <components/files/conversion.hpp>
#include <components/files/openfile.hpp>
@ -158,6 +159,9 @@ namespace ESM
{
getSubHeader();
if (mHeader.mFormatVersion > MaxStringRefIdFormatVersion)
return getStringView(mCtx.leftSub);
// Hack to make MultiMark.esp load. Zero-length strings do not
// occur in any of the official mods, but MultiMark makes use of
// them. For some reason, they break the rules, and contain a byte
@ -177,7 +181,10 @@ namespace ESM
RefId ESMReader::getRefId()
{
return ESM::RefId::stringRefId(getHStringView());
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return ESM::RefId::stringRefId(getHStringView());
getSubHeader();
return getRefIdImpl(mCtx.leftSub);
}
void ESMReader::skipHString()
@ -189,7 +196,8 @@ namespace ESM
// them. For some reason, they break the rules, and contain a byte
// (value 0) even if the header says there is no data. If
// Morrowind accepts it, so should we.
if (mCtx.leftSub == 0 && hasMoreSubs() && !mEsm->peek())
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion && mCtx.leftSub == 0 && hasMoreSubs()
&& !mEsm->peek())
{
// Skip the following zero byte
mCtx.leftRec--;
@ -378,6 +386,13 @@ namespace ESM
return std::string(getStringView(size));
}
RefId ESMReader::getMaybeFixedRefIdSize(std::size_t size)
{
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return RefId::stringRefId(getMaybeFixedStringSize(size));
return getRefIdImpl(mCtx.leftSub);
}
std::string_view ESMReader::getStringView(std::size_t size)
{
if (mBuffer.size() <= size)
@ -403,7 +418,48 @@ namespace ESM
RefId ESMReader::getRefId(std::size_t size)
{
return RefId::stringRefId(getStringView(size));
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return ESM::RefId::stringRefId(getStringView(size));
return getRefIdImpl(size);
}
RefId ESMReader::getRefIdImpl(std::size_t size)
{
RefIdType refIdType = RefIdType::Empty;
getT(refIdType);
switch (refIdType)
{
case RefIdType::Empty:
return RefId();
case RefIdType::SizedString:
{
const std::size_t minSize = sizeof(refIdType) + sizeof(StringSizeType);
if (size < minSize)
fail("Requested RefId record size is too small (" + std::to_string(size) + " < "
+ std::to_string(minSize) + ")");
StringSizeType storedSize = 0;
getT(storedSize);
const std::size_t maxSize = size - minSize;
if (storedSize > maxSize)
fail("RefId string does not fit subrecord size (" + std::to_string(storedSize) + " > "
+ std::to_string(maxSize) + ")");
return RefId::stringRefId(getStringView(storedSize));
}
case RefIdType::UnsizedString:
if (size < sizeof(refIdType))
fail("Requested RefId record size is too small (" + std::to_string(size) + " < "
+ std::to_string(sizeof(refIdType)) + ")");
return RefId::stringRefId(getStringView(size - sizeof(refIdType)));
case RefIdType::FormId:
{
ESM4::FormId formId{};
getT(formId);
return RefId::formIdRefId(formId);
}
}
fail("Unsupported RefIdType: " + std::to_string(static_cast<unsigned>(refIdType)));
}
[[noreturn]] void ESMReader::fail(const std::string& msg)

@ -273,13 +273,17 @@ namespace ESM
skip(sizeof(T));
}
void getExact(void* x, int size) { mEsm->read((char*)x, size); }
void getExact(void* x, std::size_t size)
{
mEsm->read(static_cast<char*>(x), static_cast<std::streamsize>(size));
}
void getName(NAME& name) { getT(name); }
void getUint(uint32_t& u) { getT(u); }
std::string getMaybeFixedStringSize(std::size_t size);
RefId getMaybeFixedRefIdSize(std::size_t size) { return RefId::stringRefId(getMaybeFixedStringSize(size)); }
RefId getMaybeFixedRefIdSize(std::size_t size);
// Read the next 'size' bytes and return them as a string. Converts
// them from native encoding to UTF8 in the process.
@ -315,6 +319,8 @@ namespace ESM
void clearCtx();
RefId getRefIdImpl(std::size_t size);
std::unique_ptr<std::istream> mEsm;
ESM_Context mCtx;

@ -4,10 +4,52 @@
#include <fstream>
#include <stdexcept>
#include <components/debug/debuglog.hpp>
#include <components/misc/notnullptr.hpp>
#include <components/to_utf8/to_utf8.hpp>
#include "formatversion.hpp"
namespace ESM
{
namespace
{
template <bool sizedString>
struct WriteRefId
{
ESMWriter& mWriter;
explicit WriteRefId(ESMWriter& writer)
: mWriter(writer)
{
}
void operator()(EmptyRefId /*v*/) const { mWriter.writeT(RefIdType::Empty); }
void operator()(StringRefId v) const
{
constexpr StringSizeType maxSize = std::numeric_limits<StringSizeType>::max();
if (v.getValue().size() > maxSize)
throw std::runtime_error("RefId string size is too long: \"" + v.getValue().substr(0, 64)
+ "<...>\" (" + std::to_string(v.getValue().size()) + " > " + std::to_string(maxSize) + ")");
if constexpr (sizedString)
{
mWriter.writeT(RefIdType::SizedString);
mWriter.writeT(static_cast<StringSizeType>(v.getValue().size()));
}
else
mWriter.writeT(RefIdType::UnsizedString);
mWriter.write(v.getValue().data(), v.getValue().size());
}
void operator()(FormIdRefId v) const
{
mWriter.writeT(RefIdType::FormId);
mWriter.writeT(v.getValue());
}
};
}
ESMWriter::ESMWriter()
: mRecords()
, mStream(nullptr)
@ -167,14 +209,18 @@ namespace ESM
endRecord(name);
}
void ESMWriter::writeHNRefId(NAME name, const RefId& value)
void ESMWriter::writeHNRefId(NAME name, RefId value)
{
writeHNString(name, value.getRefIdString());
startSubRecord(name);
writeHRefId(value);
endRecord(name);
}
void ESMWriter::writeHNRefId(NAME name, const RefId& value, std::size_t size)
void ESMWriter::writeHNRefId(NAME name, RefId value, std::size_t size)
{
writeHNString(name, value.getRefIdString(), size);
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return writeHNString(name, value.getRefIdString(), size);
writeHNRefId(name, value);
}
void ESMWriter::writeMaybeFixedSizeString(const std::string& data, std::size_t size)
@ -220,19 +266,30 @@ namespace ESM
write("\0", 1);
}
void ESMWriter::writeMaybeFixedSizeRefId(const RefId& value, std::size_t size)
void ESMWriter::writeMaybeFixedSizeRefId(RefId value, std::size_t size)
{
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return writeMaybeFixedSizeString(value.getRefIdString(), size);
visit(WriteRefId<true>(*this), value);
}
void ESMWriter::writeHRefId(RefId value)
{
writeMaybeFixedSizeString(value.getRefIdString(), size);
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return writeHString(value.getRefIdString());
writeRefId(value);
}
void ESMWriter::writeHRefId(const RefId& value)
void ESMWriter::writeHCRefId(RefId value)
{
writeHString(value.getRefIdString());
if (mHeader.mFormatVersion <= MaxStringRefIdFormatVersion)
return writeHCString(value.getRefIdString());
writeRefId(value);
}
void ESMWriter::writeHCRefId(const RefId& value)
void ESMWriter::writeRefId(RefId value)
{
writeHCString(value.getRefIdString());
visit(WriteRefId<false>(*this), value);
}
void ESMWriter::writeName(NAME name)

@ -47,6 +47,8 @@ namespace ESM
// It is a good idea to compare this with the value you wrote into the header (setRecordCount)
// It should be the record count you set + 1 (1 additional record for the TES3 header)
int getRecordCount() const { return mRecordCount; }
FormatVersion getFormatVersion() const { return mHeader.mFormatVersion; }
void setFormatVersion(FormatVersion value);
void clearMaster();
@ -78,24 +80,24 @@ namespace ESM
writeHNCString(name, data);
}
void writeHNRefId(NAME name, const RefId& value);
void writeHNRefId(NAME name, RefId value);
void writeHNRefId(NAME name, const RefId& value, std::size_t size);
void writeHNRefId(NAME name, RefId value, std::size_t size);
void writeHNCRefId(NAME name, const RefId& value)
void writeHNCRefId(NAME name, RefId value)
{
startSubRecord(name);
writeHCRefId(value);
endRecord(name);
}
void writeHNORefId(NAME name, const RefId& value)
void writeHNORefId(NAME name, RefId value)
{
if (!value.empty())
writeHNRefId(name, value);
}
void writeHNOCRefId(NAME name, const RefId& value)
void writeHNOCRefId(NAME name, RefId value)
{
if (!value.empty())
writeHNCRefId(name, value);
@ -165,11 +167,11 @@ namespace ESM
void writeHString(const std::string& data);
void writeHCString(const std::string& data);
void writeMaybeFixedSizeRefId(const RefId& value, std::size_t size);
void writeMaybeFixedSizeRefId(RefId value, std::size_t size);
void writeHRefId(const RefId& value);
void writeHRefId(RefId refId);
void writeHCRefId(const RefId& value);
void writeHCRefId(RefId refId);
void writeName(NAME data);
@ -184,6 +186,8 @@ namespace ESM
bool mCounting;
Header mHeader;
void writeRefId(RefId value);
};
}

@ -20,7 +20,8 @@ namespace ESM
inline constexpr FormatVersion MaxOldSkillsAndAttributesFormatVersion = 18;
inline constexpr FormatVersion MaxOldCreatureStatsFormatVersion = 19;
inline constexpr FormatVersion MaxLimitedSizeStringsFormatVersion = 22;
inline constexpr FormatVersion CurrentSaveGameFormatVersion = 23;
inline constexpr FormatVersion MaxStringRefIdFormatVersion = 23;
inline constexpr FormatVersion CurrentSaveGameFormatVersion = 24;
}
#endif

Loading…
Cancel
Save