You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
openmw/apps/openmw/mwworld/esmstore.cpp

758 lines
27 KiB
C++

#include "esmstore.hpp"
#include <algorithm>
#include <fstream>
#include <tuple>
#include <components/debug/debuglog.hpp>
#include <components/esm/records.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm3/readerscache.hpp>
#include <components/loadinglistener/loadinglistener.hpp>
#include <components/lua/configuration.hpp>
#include <components/misc/algorithm.hpp>
#include <components/esm4/common.hpp>
#include <components/esm4/loadwrld.hpp>
#include <components/esm4/reader.hpp>
#include <components/esm4/readerutils.hpp>
#include <components/esmloader/load.hpp>
#include "../mwmechanics/spelllist.hpp"
namespace
{
struct Ref
{
ESM::RefNum mRefNum;
std::size_t mRefID;
Ref(ESM::RefNum refNum, std::size_t refID)
: mRefNum(refNum)
, mRefID(refID)
{
}
};
constexpr std::size_t deletedRefID = std::numeric_limits<std::size_t>::max();
void readRefs(const ESM::Cell& cell, std::vector<Ref>& refs, std::vector<ESM::RefId>& refIDs,
std::set<ESM::RefId>& keyIDs, ESM::ReadersCache& readers)
{
// TODO: we have many similar copies of this code.
for (size_t i = 0; i < cell.mContextList.size(); i++)
{
const std::size_t index = static_cast<std::size_t>(cell.mContextList[i].index);
const ESM::ReadersCache::BusyItem reader = readers.get(index);
cell.restore(*reader, i);
ESM::CellRef ref;
bool deleted = false;
while (cell.getNextRef(*reader, ref, deleted))
{
if (deleted)
refs.emplace_back(ref.mRefNum, deletedRefID);
else if (std::find(cell.mMovedRefs.begin(), cell.mMovedRefs.end(), ref.mRefNum)
== cell.mMovedRefs.end())
{
if (!ref.mKey.empty())
keyIDs.insert(std::move(ref.mKey));
refs.emplace_back(ref.mRefNum, refIDs.size());
refIDs.push_back(std::move(ref.mRefID));
}
}
}
for (const auto& [value, deleted] : cell.mLeasedRefs)
{
if (deleted)
refs.emplace_back(value.mRefNum, deletedRefID);
else
{
if (!value.mKey.empty())
keyIDs.insert(std::move(value.mKey));
refs.emplace_back(value.mRefNum, refIDs.size());
refIDs.push_back(value.mRefID);
}
}
}
const ESM::RefId& getDefaultClass(const MWWorld::Store<ESM::Class>& classes)
{
auto it = classes.begin();
if (it != classes.end())
return it->mId;
throw std::runtime_error("List of NPC classes is empty!");
}
std::vector<ESM::NPC> getNPCsToReplace(const MWWorld::Store<ESM::Faction>& factions,
const MWWorld::Store<ESM::Class>& classes, const std::unordered_map<ESM::RefId, ESM::NPC>& npcs)
{
// Cache first class from store - we will use it if current class is not found
const ESM::RefId& defaultCls = getDefaultClass(classes);
// Validate NPCs for non-existing class and faction.
// We will replace invalid entries by fixed ones
std::vector<ESM::NPC> npcsToReplace;
for (const auto& npcIter : npcs)
{
ESM::NPC npc = npcIter.second;
bool changed = false;
const ESM::RefId& npcFaction = npc.mFaction;
if (!npcFaction.empty())
{
const ESM::Faction* fact = factions.search(npcFaction);
if (!fact)
{
Log(Debug::Verbose) << "NPC '" << npc.mId << "' (" << npc.mName << ") has nonexistent faction '"
<< npc.mFaction << "', ignoring it.";
npc.mFaction = ESM::RefId();
npc.mNpdt.mRank = 0;
changed = true;
}
}
const ESM::RefId& npcClass = npc.mClass;
const ESM::Class* cls = classes.search(npcClass);
if (!cls)
{
Log(Debug::Verbose) << "NPC '" << npc.mId << "' (" << npc.mName << ") has nonexistent class '"
<< npc.mClass << "', using '" << defaultCls << "' class as replacement.";
npc.mClass = defaultCls;
changed = true;
}
if (changed)
npcsToReplace.push_back(npc);
}
return npcsToReplace;
}
// Custom enchanted items can reference scripts that no longer exist, this doesn't necessarily mean the base item no
// longer exists however. So instead of removing the item altogether, we're only removing the script.
template <class MapT>
void removeMissingScripts(const MWWorld::Store<ESM::Script>& scripts, MapT& items)
{
for (auto& [id, item] : items)
{
if (!item.mScript.empty() && !scripts.search(item.mScript))
{
item.mScript = ESM::RefId();
Log(Debug::Verbose) << "Item '" << id << "' (" << item.mName << ") has nonexistent script '"
<< item.mScript << "', ignoring it.";
}
}
}
}
namespace MWWorld
{
using IDMap = std::unordered_map<ESM::RefId, int>;
struct ESMStoreImp
{
ESMStore::StoreTuple mStores;
std::map<ESM::RecNameInts, DynamicStore*> mRecNameToStore;
// Lookup of all IDs. Makes looking up references faster. Just
// maps the id name to the record type.
IDMap mIds;
IDMap mStaticIds;
template <typename T>
static void assignStoreToIndex(ESMStore& stores, Store<T>& store)
{
const std::size_t storeIndex = ESMStore::getTypeIndex<T>();
if (stores.mStores.size() <= storeIndex)
stores.mStores.resize(storeIndex + 1);
assert(&store == &std::get<Store<T>>(stores.mStoreImp->mStores));
stores.mStores[storeIndex] = &store;
if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*>)
{
stores.mDynamicStores.push_back(&store);
constexpr ESM::RecNameInts recName = T::sRecordId;
if constexpr (recName != ESM::REC_INTERNAL_PLAYER)
{
stores.mStoreImp->mRecNameToStore[recName] = &store;
}
}
}
template <typename T>
static bool typedReadRecordESM4(ESM4::Reader& reader, Store<T>& store)
{
auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
if constexpr (HasRecordId<T>::value)
{
if constexpr (ESM::isESM4Rec(T::sRecordId))
{
if (T::sRecordId == esm4RecName)
{
reader.getRecordData();
T value;
value.load(reader);
store.insertStatic(value);
return true;
}
}
}
return false;
}
static bool readRecord(ESM4::Reader& reader, ESMStore& store)
{
return std::apply(
[&reader](auto&... x) { return (typedReadRecordESM4(reader, x) || ...); }, store.mStoreImp->mStores);
}
};
int ESMStore::find(const ESM::RefId& id) const
{
IDMap::const_iterator it = mStoreImp->mIds.find(id);
if (it == mStoreImp->mIds.end())
{
return 0;
}
return it->second;
}
int ESMStore::findStatic(const ESM::RefId& id) const
{
IDMap::const_iterator it = mStoreImp->mStaticIds.find(id);
if (it == mStoreImp->mStaticIds.end())
{
return 0;
}
return it->second;
}
ESMStore::ESMStore()
{
mStoreImp = std::make_unique<ESMStoreImp>();
std::apply([this](auto&... x) { (ESMStoreImp::assignStoreToIndex(*this, x), ...); }, mStoreImp->mStores);
mDynamicCount = 0;
getWritable<ESM::Pathgrid>().setCells(getWritable<ESM::Cell>());
}
ESMStore::~ESMStore() = default;
void ESMStore::clearDynamic()
{
for (const auto& store : mDynamicStores)
store->clearDynamic();
movePlayerRecord();
}
static bool isCacheableRecord(int id)
{
switch (id)
{
case ESM::REC_ACTI:
case ESM::REC_ALCH:
case ESM::REC_APPA:
case ESM::REC_ARMO:
case ESM::REC_BOOK:
case ESM::REC_CLOT:
case ESM::REC_CONT:
case ESM::REC_CREA:
case ESM::REC_DOOR:
case ESM::REC_INGR:
case ESM::REC_LEVC:
case ESM::REC_LEVI:
case ESM::REC_LIGH:
case ESM::REC_LOCK:
case ESM::REC_MISC:
case ESM::REC_NPC_:
case ESM::REC_PROB:
case ESM::REC_REPA:
case ESM::REC_STAT:
case ESM::REC_WEAP:
case ESM::REC_BODY:
case ESM::REC_STAT4:
case ESM::REC_LIGH4:
case ESM::REC_ACTI4:
case ESM::REC_ALCH4:
case ESM::REC_AMMO4:
case ESM::REC_ARMO4:
case ESM::REC_BOOK4:
case ESM::REC_CONT4:
case ESM::REC_DOOR4:
case ESM::REC_INGR4:
case ESM::REC_MISC4:
case ESM::REC_WEAP4:
return true;
break;
}
return false;
}
void ESMStore::load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue)
{
if (listener != nullptr)
listener->setProgressRange(::EsmLoader::fileProgress);
// Land texture loading needs to use a separate internal store for each plugin.
// We set the number of plugins here so we can properly verify if valid plugin
// indices are being passed to the LandTexture Store retrieval methods.
getWritable<ESM::LandTexture>().resize(esm.getIndex() + 1);
// Loop through all records
while (esm.hasMoreRecs())
{
ESM::NAME n = esm.getRecName();
esm.getRecHeader();
if (esm.getRecordFlags() & ESM::FLAG_Ignored)
{
esm.skipRecord();
continue;
}
// Look up the record type.
ESM::RecNameInts recName = static_cast<ESM::RecNameInts>(n.toInt());
const auto& it = mStoreImp->mRecNameToStore.find(recName);
if (it == mStoreImp->mRecNameToStore.end())
{
if (recName == ESM::REC_INFO)
{
if (dialogue)
{
dialogue->readInfo(esm);
}
else
{
Log(Debug::Error) << "Error: info record without dialog";
esm.skipRecord();
}
}
else if (n.toInt() == ESM::REC_MGEF)
{
getWritable<ESM::MagicEffect>().load(esm);
}
else if (n.toInt() == ESM::REC_SKIL)
{
getWritable<ESM::Skill>().load(esm);
}
else if (n.toInt() == ESM::REC_FILT || n.toInt() == ESM::REC_DBGP)
{
// ignore project file only records
esm.skipRecord();
}
else if (n.toInt() == ESM::REC_LUAL)
{
ESM::LuaScriptsCfg cfg;
cfg.load(esm);
cfg.adjustRefNums(esm);
mLuaContent.push_back(std::move(cfg));
}
else
{
throw std::runtime_error("Unknown record: " + n.toString());
}
}
else
{
RecordId id = it->second->load(esm);
if (id.mIsDeleted)
{
it->second->eraseStatic(id.mId);
continue;
}
if (n.toInt() == ESM::REC_DIAL)
{
dialogue = const_cast<ESM::Dialogue*>(getWritable<ESM::Dialogue>().find(id.mId));
}
else
{
dialogue = nullptr;
}
}
if (listener != nullptr)
listener->setProgress(::EsmLoader::fileProgress * esm.getFileOffset() / esm.getFileSize());
}
}
void ESMStore::loadESM4(ESM4::Reader& reader)
{
auto visitorRec = [this](ESM4::Reader& reader) { return ESMStoreImp::readRecord(reader, *this); };
ESM4::ReaderUtils::readAll(reader, visitorRec, [](ESM4::Reader&) {});
}
void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type)
{
mStoreImp->mIds[id] = type;
}
ESM::LuaScriptsCfg ESMStore::getLuaScriptsCfg() const
{
ESM::LuaScriptsCfg cfg;
for (const LuaContent& c : mLuaContent)
{
if (std::holds_alternative<std::filesystem::path>(c))
{
// *.omwscripts are intentionally reloaded every time when `getLuaScriptsCfg` is called.
// It is important for the `reloadlua` console command.
try
{
auto file = std::ifstream(std::get<std::filesystem::path>(c));
std::string fileContent(std::istreambuf_iterator<char>(file), {});
LuaUtil::parseOMWScripts(cfg, fileContent);
}
catch (std::exception& e)
{
Log(Debug::Error) << e.what();
}
}
else
{
const ESM::LuaScriptsCfg& addition = std::get<ESM::LuaScriptsCfg>(c);
cfg.mScripts.insert(cfg.mScripts.end(), addition.mScripts.begin(), addition.mScripts.end());
}
}
return cfg;
}
void ESMStore::setUp()
{
mStoreImp->mIds.clear();
std::map<ESM::RecNameInts, DynamicStore*>::iterator storeIt = mStoreImp->mRecNameToStore.begin();
for (; storeIt != mStoreImp->mRecNameToStore.end(); ++storeIt)
{
storeIt->second->setUp();
if (isCacheableRecord(storeIt->first))
{
std::vector<ESM::RefId> identifiers;
storeIt->second->listIdentifier(identifiers);
for (auto& record : identifiers)
mStoreImp->mIds[record] = storeIt->first;
}
}
if (mStoreImp->mStaticIds.empty())
for (const auto& [k, v] : mStoreImp->mIds)
mStoreImp->mStaticIds.emplace(k, v);
getWritable<ESM::Skill>().setUp();
getWritable<ESM::MagicEffect>().setUp();
getWritable<ESM::Attribute>().setUp();
getWritable<ESM4::Reference>().preprocessReferences(get<ESM4::Cell>());
}
void ESMStore::validateRecords(ESM::ReadersCache& readers)
{
validate();
countAllCellRefsAndMarkKeys(readers);
}
void ESMStore::countAllCellRefsAndMarkKeys(ESM::ReadersCache& readers)
{
// TODO: We currently need to read entire files here again.
// We should consider consolidating or deferring this reading.
if (!mRefCount.empty())
return;
std::vector<Ref> refs;
std::set<ESM::RefId> keyIDs;
std::vector<ESM::RefId> refIDs;
Store<ESM::Cell> Cells = get<ESM::Cell>();
for (auto it = Cells.intBegin(); it != Cells.intEnd(); ++it)
readRefs(*it, refs, refIDs, keyIDs, readers);
for (auto it = Cells.extBegin(); it != Cells.extEnd(); ++it)
readRefs(*it, refs, refIDs, keyIDs, readers);
const auto lessByRefNum = [](const Ref& l, const Ref& r) { return l.mRefNum < r.mRefNum; };
std::stable_sort(refs.begin(), refs.end(), lessByRefNum);
const auto equalByRefNum = [](const Ref& l, const Ref& r) { return l.mRefNum == r.mRefNum; };
const auto incrementRefCount = [&](const Ref& value) {
if (value.mRefID != deletedRefID)
{
ESM::RefId& refId = refIDs[value.mRefID];
++mRefCount[std::move(refId)];
}
};
Misc::forEachUnique(refs.rbegin(), refs.rend(), equalByRefNum, incrementRefCount);
auto& store = getWritable<ESM::Miscellaneous>().mStatic;
for (const auto& id : keyIDs)
{
auto it = store.find(id);
if (it != store.end())
it->second.mData.mFlags |= ESM::Miscellaneous::Key;
}
}
int ESMStore::getRefCount(const ESM::RefId& id) const
{
auto it = mRefCount.find(id);
if (it == mRefCount.end())
return 0;
return it->second;
}
void ESMStore::validate()
{
auto& npcs = getWritable<ESM::NPC>();
std::vector<ESM::NPC> npcsToReplace
= getNPCsToReplace(getWritable<ESM::Faction>(), getWritable<ESM::Class>(), npcs.mStatic);
for (const ESM::NPC& npc : npcsToReplace)
{
npcs.eraseStatic(npc.mId);
npcs.insertStatic(npc);
}
// Validate spell effects for invalid arguments
std::vector<ESM::Spell> spellsToReplace;
auto& spells = getWritable<ESM::Spell>();
for (ESM::Spell spell : spells)
{
if (spell.mEffects.mList.empty())
continue;
bool changed = false;
auto iter = spell.mEffects.mList.begin();
while (iter != spell.mEffects.mList.end())
{
const ESM::MagicEffect* mgef = getWritable<ESM::MagicEffect>().search(iter->mEffectID);
if (!mgef)
{
Log(Debug::Verbose) << "Spell '" << spell.mId << "' has an invalid effect (index "
<< iter->mEffectID << ") present. Dropping the effect.";
iter = spell.mEffects.mList.erase(iter);
changed = true;
continue;
}
if (mgef->mData.mFlags & ESM::MagicEffect::TargetSkill)
{
if (iter->mAttribute != -1)
{
iter->mAttribute = -1;
Log(Debug::Verbose)
<< ESM::MagicEffect::effectIdToString(iter->mEffectID) << " effect of spell '" << spell.mId
<< "' has an attribute argument present. Dropping the argument.";
changed = true;
}
}
else if (mgef->mData.mFlags & ESM::MagicEffect::TargetAttribute)
{
if (iter->mSkill != -1)
{
iter->mSkill = -1;
Log(Debug::Verbose)
<< ESM::MagicEffect::effectIdToString(iter->mEffectID) << " effect of spell '" << spell.mId
<< "' has a skill argument present. Dropping the argument.";
changed = true;
}
}
else if (iter->mSkill != -1 || iter->mAttribute != -1)
{
iter->mSkill = -1;
iter->mAttribute = -1;
Log(Debug::Verbose) << ESM::MagicEffect::effectIdToString(iter->mEffectID) << " effect of spell '"
<< spell.mId << "' has argument(s) present. Dropping the argument(s).";
changed = true;
}
++iter;
}
if (changed)
spellsToReplace.emplace_back(spell);
}
for (const ESM::Spell& spell : spellsToReplace)
{
spells.eraseStatic(spell.mId);
spells.insertStatic(spell);
}
}
void ESMStore::movePlayerRecord()
{
auto& npcs = getWritable<ESM::NPC>();
auto player = npcs.find(ESM::RefId::stringRefId("Player"));
npcs.insert(*player);
}
void ESMStore::validateDynamic()
{
auto& npcs = getWritable<ESM::NPC>();
auto& scripts = getWritable<ESM::Script>();
std::vector<ESM::NPC> npcsToReplace
= getNPCsToReplace(getWritable<ESM::Faction>(), getWritable<ESM::Class>(), npcs.mDynamic);
for (const ESM::NPC& npc : npcsToReplace)
npcs.insert(npc);
removeMissingScripts(scripts, getWritable<ESM::Armor>().mDynamic);
removeMissingScripts(scripts, getWritable<ESM::Book>().mDynamic);
removeMissingScripts(scripts, getWritable<ESM::Clothing>().mDynamic);
removeMissingScripts(scripts, getWritable<ESM::Weapon>().mDynamic);
removeMissingObjects(getWritable<ESM::CreatureLevList>());
removeMissingObjects(getWritable<ESM::ItemLevList>());
}
// Leveled lists can be modified by scripts. This removes items that no longer exist (presumably because the
// plugin was removed) from modified lists
template <class T>
void ESMStore::removeMissingObjects(Store<T>& store)
{
for (auto& entry : store.mDynamic)
{
auto first = std::remove_if(entry.second.mList.begin(), entry.second.mList.end(), [&](const auto& item) {
if (!find(item.mId))
{
Log(Debug::Verbose) << "Leveled list '" << entry.first << "' has nonexistent object '" << item.mId
<< "', ignoring it.";
return true;
}
return false;
});
entry.second.mList.erase(first, entry.second.mList.end());
}
}
int ESMStore::countSavedGameRecords() const
{
return 1 // DYNA (dynamic name counter)
+ get<ESM::Potion>().getDynamicSize() + get<ESM::Armor>().getDynamicSize()
+ get<ESM::Book>().getDynamicSize() + get<ESM::Class>().getDynamicSize()
+ get<ESM::Clothing>().getDynamicSize() + get<ESM::Enchantment>().getDynamicSize()
+ get<ESM::NPC>().getDynamicSize() + get<ESM::Spell>().getDynamicSize()
+ get<ESM::Activator>().getDynamicSize() + get<ESM::Miscellaneous>().getDynamicSize()
+ get<ESM::Weapon>().getDynamicSize() + get<ESM::CreatureLevList>().getDynamicSize()
+ get<ESM::ItemLevList>().getDynamicSize() + get<ESM::Creature>().getDynamicSize()
+ get<ESM::Container>().getDynamicSize();
}
void ESMStore::write(ESM::ESMWriter& writer, Loading::Listener& progress) const
{
writer.startRecord(ESM::REC_DYNA);
writer.startSubRecord("COUN");
writer.writeT(mDynamicCount);
writer.endRecord("COUN");
writer.endRecord(ESM::REC_DYNA);
get<ESM::Potion>().write(writer, progress);
get<ESM::Armor>().write(writer, progress);
get<ESM::Book>().write(writer, progress);
get<ESM::Class>().write(writer, progress);
get<ESM::Clothing>().write(writer, progress);
get<ESM::Enchantment>().write(writer, progress);
get<ESM::NPC>().write(writer, progress);
get<ESM::Miscellaneous>().write(writer, progress);
get<ESM::Activator>().write(writer, progress);
get<ESM::Spell>().write(writer, progress);
get<ESM::Weapon>().write(writer, progress);
get<ESM::CreatureLevList>().write(writer, progress);
get<ESM::ItemLevList>().write(writer, progress);
get<ESM::Creature>().write(writer, progress);
get<ESM::Container>().write(writer, progress);
}
bool ESMStore::readRecord(ESM::ESMReader& reader, uint32_t type_id)
{
ESM::RecNameInts type = (ESM::RecNameInts)type_id;
switch (type)
{
case ESM::REC_ALCH:
case ESM::REC_MISC:
case ESM::REC_ACTI:
case ESM::REC_ARMO:
case ESM::REC_BOOK:
case ESM::REC_CLAS:
case ESM::REC_CLOT:
case ESM::REC_ENCH:
case ESM::REC_SPEL:
case ESM::REC_WEAP:
case ESM::REC_LEVI:
case ESM::REC_LEVC:
mStoreImp->mRecNameToStore[type]->read(reader);
return true;
case ESM::REC_NPC_:
case ESM::REC_CREA:
case ESM::REC_CONT:
mStoreImp->mRecNameToStore[type]->read(reader, true);
return true;
case ESM::REC_DYNA:
reader.getSubNameIs("COUN");
reader.getHT(mDynamicCount);
return true;
default:
return false;
}
}
void ESMStore::checkPlayer()
{
setUp();
const ESM::NPC* player = get<ESM::NPC>().find(ESM::RefId::stringRefId("Player"));
if (!get<ESM::Race>().find(player->mRace) || !get<ESM::Class>().find(player->mClass))
throw std::runtime_error("Invalid player record (race or class unavailable");
}
std::pair<std::shared_ptr<MWMechanics::SpellList>, bool> ESMStore::getSpellList(const ESM::RefId& id) const
{
auto result = mSpellListCache.find(id);
std::shared_ptr<MWMechanics::SpellList> ptr;
if (result != mSpellListCache.end())
ptr = result->second.lock();
if (!ptr)
{
int type = find(id);
ptr = std::make_shared<MWMechanics::SpellList>(id, type);
if (result != mSpellListCache.end())
result->second = ptr;
else
mSpellListCache.insert({ id, ptr });
return { ptr, false };
}
return { ptr, true };
}
template <>
const ESM::Cell* ESMStore::insert<ESM::Cell>(const ESM::Cell& cell)
{
return getWritable<ESM::Cell>().insert(cell);
}
template <>
const ESM::NPC* ESMStore::insert<ESM::NPC>(const ESM::NPC& npc)
{
auto& npcs = getWritable<ESM::NPC>();
if (npc.mId == "Player")
{
return npcs.insert(npc);
}
const ESM::RefId id = ESM::RefId::generated(mDynamicCount++);
if (npcs.search(id) != nullptr)
throw std::runtime_error("Try to override existing record: " + id.toDebugString());
ESM::NPC record = npc;
record.mId = id;
ESM::NPC* ptr = npcs.insert(record);
mStoreImp->mIds[ptr->mId] = ESM::REC_NPC_;
return ptr;
}
} // end namespace