Allow TES4 ESM/ESP to co-exist with TES3 ESM/ESP.

This change aims to allow TES4/TE5 content to OpenMW.  i.e. a standalone TES4 would be implemented quite differently.  That said, the key changes are:

* Use pointers rather than references for ESM readers so that they can be switched to another variant on the fly.
* Content file dependencies to be checked within each group (only 3 groups for now, TES3/TES4/TES5)
pull/541/head
cc9cii 6 years ago
parent 1b56074b53
commit 15d5cdf3cf

@ -130,6 +130,23 @@ Enhancements for both OpenMW and OpenCS:
* Experimental support of loading TES4/TES5 records (coming soon).
* Experimental support of NavMesh (eventually).
openmw.cfg example
------------------
...
fallback-archive=Morrowind.bsa
fallback-archive=Tribunal.bsa
fallback-archive=Bloodmoon.bsa
fallback-archive=TR_Data.bsa
fallback-tes4archive=Oblivion - Meshes.bsa
#fallback-tes4archive=Skyrim - Textures.bsa
#fallback-tes4archive=Dragonborn.bsa
#fallback-tes4archive=Dawnguard.bsa
...
data="C:/Program Files (x86)/Bethesda Softworks/Morrowind/Data Files"
data="C:/Program Files (x86)/Bethesda Softworks/Oblivion/Data"
...
Build Dependencies
------------------

@ -134,7 +134,7 @@ namespace MWBase
virtual const MWWorld::ESMStore& getStore() const = 0;
virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;
virtual std::vector<ESM::ESMReader*>& getEsmReader() = 0;
virtual MWWorld::LocalScripts& getLocalScripts() = 0;

@ -83,7 +83,7 @@ void MWWorld::Cells::writeCell (ESM::ESMWriter& writer, CellStore& cell) const
writer.endRecord (ESM::REC_CSTA);
}
MWWorld::Cells::Cells (const MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& reader)
MWWorld::Cells::Cells (const MWWorld::ESMStore& store, std::vector<std::vector<ESM::ESMReader*> >& reader)
: mStore (store), mReader (reader),
mIdCache (40, std::pair<std::string, CellStore *> ("", (CellStore*)0)), /// \todo make cache size configurable
mIdCacheIndex (0)

@ -28,7 +28,7 @@ namespace MWWorld
class Cells
{
const MWWorld::ESMStore& mStore;
std::vector<ESM::ESMReader>& mReader;
std::vector<std::vector<ESM::ESMReader*> >& mReader;
mutable std::map<std::string, CellStore> mInteriors;
mutable std::map<std::pair<int, int>, CellStore> mExteriors;
std::vector<std::pair<std::string, CellStore *> > mIdCache;
@ -47,7 +47,7 @@ namespace MWWorld
void clear();
Cells (const MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& reader);
Cells (const MWWorld::ESMStore& store, std::vector<std::vector<ESM::ESMReader*> >& reader);
CellStore *getExterior (int x, int y);

@ -400,7 +400,7 @@ namespace MWWorld
+ mNpcs.mList.size();
}
void CellStore::load (const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm)
void CellStore::load (const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm)
{
if (mState!=State_Loaded)
{
@ -417,7 +417,7 @@ namespace MWWorld
}
}
void CellStore::preload (const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm)
void CellStore::preload (const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm)
{
if (mState==State_Unloaded)
{
@ -427,7 +427,7 @@ namespace MWWorld
}
}
void CellStore::listRefs(const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm)
void CellStore::listRefs(const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm)
{
assert (mCell);
@ -439,13 +439,13 @@ namespace MWWorld
{
// Reopen the ESM reader and seek to the right position.
int index = mCell->mContextList.at(i).index;
mCell->restore (esm[index], i);
mCell->restore (*esm[0][index], (int)i); // FIXME: hardcoded 0 means TES3
ESM::CellRef ref;
// Get each reference in turn
bool deleted = false;
while (mCell->getNextRef (esm[index], ref, deleted))
while (mCell->getNextRef (*esm[0][index], ref, deleted)) // FIXME hardcoded 0 means TES3
{
if (deleted)
continue;
@ -472,7 +472,7 @@ namespace MWWorld
std::sort (mIds.begin(), mIds.end());
}
void CellStore::loadRefs(const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm)
void CellStore::loadRefs(const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm)
{
assert (mCell);
@ -484,14 +484,14 @@ namespace MWWorld
{
// Reopen the ESM reader and seek to the right position.
int index = mCell->mContextList.at(i).index;
mCell->restore (esm[index], i);
mCell->restore (*esm[0][index], (int)i); // FIXME: hardcoded 0 means TES3
ESM::CellRef ref;
ref.mRefNum.mContentFile = ESM::RefNum::RefNum_NoContentFile;
// Get each reference in turn
bool deleted = false;
while(mCell->getNextRef(esm[index], ref, deleted))
while(mCell->getNextRef(*esm[0][index], ref, deleted)) // FIXME: 0 means TES3
{
// Don't load reference if it was moved to a different cell.
ESM::MovedCellRefTracker::const_iterator iter =

@ -112,10 +112,10 @@ namespace MWWorld
int count() const;
///< Return total number of references, including deleted ones.
void load (const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm);
void load (const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm);
///< Load references from content file.
void preload (const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm);
void preload (const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm);
///< Build ID list from content file.
/// Call functor (ref) for each reference. functor must return a bool. Returning
@ -213,9 +213,9 @@ namespace MWWorld
}
/// Run through references and store IDs
void listRefs(const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm);
void listRefs(const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm);
void loadRefs(const MWWorld::ESMStore &store, std::vector<ESM::ESMReader> &esm);
void loadRefs(const MWWorld::ESMStore &store, std::vector<std::vector<ESM::ESMReader*> > &esm);
void loadRef (ESM::CellRef& ref, bool deleted, const ESMStore& store);
///< Make case-adjustments to \a ref and insert it into the respective container.

@ -21,7 +21,7 @@ struct ContentLoader
{
}
virtual void load(const boost::filesystem::path& filepath, int& index)
virtual void load(const boost::filesystem::path& filepath, std::vector<std::vector<std::string> >& contentFiles)
{
std::cout << "Loading content file " << filepath.string() << std::endl;
mListener.setLabel(filepath.string());

@ -6,7 +6,7 @@
namespace MWWorld
{
EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& readers,
EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<std::vector<ESM::ESMReader*> >& readers,
ToUTF8::Utf8Encoder* encoder, Loading::Listener& listener)
: ContentLoader(listener)
, mEsm(readers)
@ -15,17 +15,46 @@ EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& read
{
}
void EsmLoader::load(const boost::filesystem::path& filepath, int& index)
// FIXME: tesVerIndex stuff is rather clunky, needs to be refactored
void EsmLoader::load(const boost::filesystem::path& filepath, std::vector<std::vector<std::string> >& contentFiles)
{
ContentLoader::load(filepath.filename(), index);
ESM::ESMReader lEsm;
lEsm.setEncoder(mEncoder);
lEsm.setIndex(index);
lEsm.setGlobalReaderList(&mEsm);
lEsm.open(filepath.string());
mEsm[index] = lEsm;
mStore.load(mEsm[index], &mListener);
int tesVerIndex = 0; // FIXME: hard coded, 0 = MW, 1 = TES4, 2 = TES5 (TODO: Fallout)
int index = 0;
ContentLoader::load(filepath.filename(), contentFiles); // set the label on the loading bar
ESM::ESMReader *lEsm = new ESM::ESMReader();
lEsm->setEncoder(mEncoder);
lEsm->setGlobalReaderList(&mEsm[tesVerIndex]); // global reader list is used by ESMStore::load only
lEsm->open(filepath.string());
int esmVer = lEsm->getVer();
bool isTes4 = esmVer == ESM::VER_080 || esmVer == ESM::VER_100;
bool isTes5 = esmVer == ESM::VER_094 || esmVer == ESM::VER_17;
bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134;
if (isTes4 || isTes5 || isFONV)
{
if (isTes4)
tesVerIndex = 1;
else if (isTes5)
tesVerIndex = 2;
else
tesVerIndex = 3;
// do nothing for now
return;
}
else
{
tesVerIndex = 0; // 0 = MW
index = contentFiles[tesVerIndex].size();
contentFiles[tesVerIndex].push_back(filepath.filename().string());
lEsm->setIndex(index);
mEsm[tesVerIndex].push_back(lEsm);
}
mStore.load(*mEsm[tesVerIndex][index], &mListener);
}
} /* namespace MWWorld */

@ -22,15 +22,15 @@ class ESMStore;
struct EsmLoader : public ContentLoader
{
EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& readers,
ToUTF8::Utf8Encoder* encoder, Loading::Listener& listener);
EsmLoader(MWWorld::ESMStore& store, std::vector<std::vector<ESM::ESMReader*> >& readers,
ToUTF8::Utf8Encoder* encoder, Loading::Listener& listener);
void load(const boost::filesystem::path& filepath, int& index);
void load(const boost::filesystem::path& filepath, std::vector<std::vector<std::string> >& contentFiles);
private:
std::vector<ESM::ESMReader>& mEsm;
MWWorld::ESMStore& mStore;
ToUTF8::Utf8Encoder* mEncoder;
std::vector<std::vector<ESM::ESMReader*> >& mEsm; // Note: the ownership of the readers is with the caller
MWWorld::ESMStore& mStore;
ToUTF8::Utf8Encoder* mEncoder;
};
} /* namespace MWWorld */

@ -31,38 +31,52 @@ void ESMStore::load(ESM::ESMReader &esm, Loading::Listener* listener)
ESM::Dialogue *dialogue = 0;
// Land texture loading needs to use a separate internal store for each plugin.
// We set the number of plugins here to avoid continual resizes during loading,
// and so we can properly verify if valid plugin indices are being passed to the
// LandTexture Store retrieval methods.
mLandTextures.resize(esm.getGlobalReaderList()->size());
/// \todo Move this to somewhere else. ESMReader?
// Cache parent esX files by tracking their indices in the global list of
// all files/readers used by the engine. This will greaty accelerate
// refnumber mangling, as required for handling moved references.
const std::vector<ESM::Header::MasterData> &masters = esm.getGameFiles();
std::vector<ESM::ESMReader> *allPlugins = esm.getGlobalReaderList();
for (size_t j = 0; j < masters.size(); j++) {
ESM::Header::MasterData &mast = const_cast<ESM::Header::MasterData&>(masters[j]);
std::string fname = mast.name;
int index = ~0;
for (int i = 0; i < esm.getIndex(); i++) {
const std::string &candidate = allPlugins->at(i).getContext().filename;
std::string fnamecandidate = boost::filesystem::path(candidate).filename().string();
if (Misc::StringUtils::ciEqual(fname, fnamecandidate)) {
index = i;
break;
int esmVer = esm.getVer();
bool isTes4 = esmVer == ESM::VER_080 || esmVer == ESM::VER_100;
bool isTes5 = esmVer == ESM::VER_094 || esmVer == ESM::VER_17;
bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134;
// FIXME: temporary workaround
if (!(isTes4 || isTes5 || isFONV)) // MW only
{
// Land texture loading needs to use a separate internal store for each plugin.
// We set the number of plugins here to avoid continual resizes during loading,
// and so we can properly verify if valid plugin indices are being passed to the
// LandTexture Store retrieval methods.
mLandTextures.resize(esm.getGlobalReaderList()->size()); // FIXME: size should be for MW only
}
// FIXME: for TES4/TES5 whether a dependent file is loaded is already checked in
// ESM4::Reader::updateModIndicies() which is called in EsmLoader::load() before this
if (!(isTes4 || isTes5 || isFONV)) // MW only
{
/// \todo Move this to somewhere else. ESMReader?
// Cache parent esX files by tracking their indices in the global list of
// all files/readers used by the engine. This will greaty accelerate
// refnumber mangling, as required for handling moved references.
const std::vector<ESM::Header::MasterData> &masters = esm.getGameFiles();
std::vector<ESM::ESMReader*> *allPlugins = esm.getGlobalReaderList();
for (size_t j = 0; j < masters.size(); j++) {
ESM::Header::MasterData &mast = const_cast<ESM::Header::MasterData&>(masters[j]);
std::string fname = mast.name;
int index = ~0;
for (int i = 0; i < esm.getIndex(); i++) {
const std::string &candidate = allPlugins->at(i)->getContext().filename;
std::string fnamecandidate = boost::filesystem::path(candidate).filename().string();
if (Misc::StringUtils::ciEqual(fname, fnamecandidate)) {
index = i;
break;
}
}
if (index == (int)~0) {
// Tried to load a parent file that has not been loaded yet. This is bad,
// the launcher should have taken care of this.
std::string fstring = "File " + esm.getName() + " asks for parent file " + masters[j].name
+ ", but it has not been loaded yet. Please check your load order.";
esm.fail(fstring);
}
mast.index = index;
}
if (index == (int)~0) {
// Tried to load a parent file that has not been loaded yet. This is bad,
// the launcher should have taken care of this.
std::string fstring = "File " + esm.getName() + " asks for parent file " + masters[j].name
+ ", but it has not been loaded yet. Please check your load order.";
esm.fail(fstring);
}
mast.index = index;
}
// Loop through all records

@ -90,12 +90,12 @@ namespace MWWorld
return mLoaders.insert(std::make_pair(extension, loader)).second;
}
void load(const boost::filesystem::path& filepath, int& index)
void load(const boost::filesystem::path& filepath, std::vector<std::vector<std::string> >& contentFiles)
{
LoadersContainer::iterator it(mLoaders.find(Misc::StringUtils::lowerCase(filepath.extension().string())));
if (it != mLoaders.end())
{
it->second->load(filepath, index);
it->second->load(filepath, contentFiles);
}
else
{
@ -169,7 +169,7 @@ namespace MWWorld
mWeatherManager = new MWWorld::WeatherManager(mRendering,&mFallback);
mEsm.resize(contentFiles.size());
mEsm.resize(3); // FIXME: 0 - TES3, 1 - TES4, 2 - TES5 (TODO: Fallout)
Loading::Listener* listener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
listener->loadingOn();
@ -187,7 +187,7 @@ namespace MWWorld
listener->loadingOff();
// insert records that may not be present in all versions of MW
if (mEsm[0].getFormat() == 0)
if (mEsm[0][0]->getFormat() == 0) // FIXME: first file may not be for MW
ensureNeededRecords();
mStore.setUp();
@ -474,6 +474,13 @@ namespace MWWorld
delete mPhysics;
delete mPlayer;
for (unsigned int i = 0; i < mEsm.size(); ++i)
for (unsigned int j = 0; j < mEsm[i].size(); ++j)
{
mEsm[i][j]->close();
delete mEsm[i][j];
}
}
const ESM::Cell *World::getExterior (const std::string& cellName) const
@ -542,9 +549,9 @@ namespace MWWorld
return mStore;
}
std::vector<ESM::ESMReader>& World::getEsmReader()
std::vector<ESM::ESMReader*>& World::getEsmReader()
{
return mEsm;
return mEsm[0]; // FIXME: only MW for now (but doesn't seem to be used anywhere?)
}
LocalScripts& World::getLocalScripts()
@ -2609,18 +2616,35 @@ namespace MWWorld
return mScriptsEnabled;
}
// The aim is to allow loading various types of TES files in any combination, as long as
// the dependent files are loaded first. To achieve this, separate indicies for each TES
// versions are required.
//
// The trouble is that until the file is opened by an ESM reader to check the version from
// the header we don't know which index to increment.
//
// One option is to allow the content loader to manage.
// FIXME: Appears to be loading all the files named in 'content' located in fileCollections
// based on the extension string (e.g. .esm). This probably means that the contents are in
// the correct load order.
//
// 'contentLoader' has a number of loaders that can deal with various extension types.
void World::loadContentFiles(const Files::Collections& fileCollections,
const std::vector<std::string>& content, ContentLoader& contentLoader)
{
std::vector<std::vector<std::string> > contentFiles;
contentFiles.resize(3);
std::vector<std::string>::const_iterator it(content.begin());
std::vector<std::string>::const_iterator end(content.end());
for (int idx = 0; it != end; ++it, ++idx)
for (; it != end; ++it)
{
boost::filesystem::path filename(*it);
const Files::MultiDirCollection& col = fileCollections.getCollection(filename.extension().string());
if (col.doesExist(*it))
{
contentLoader.load(col.getPath(*it), idx);
contentLoader.load(col.getPath(*it), contentFiles);
}
else
{
@ -2937,7 +2961,7 @@ namespace MWWorld
MWWorld::ActionTeleport action(cellName, closestMarker.getRefData().getPosition(), false);
action.execute(ptr);
}
void World::updateWeather(float duration, bool paused)
{
if (mPlayer->wasTeleported())
@ -2945,7 +2969,7 @@ namespace MWWorld
mPlayer->setTeleported(false);
mWeatherManager->switchToNextWeather(true);
}
mWeatherManager->update(duration, paused);
}

@ -66,7 +66,7 @@ namespace MWWorld
MWWorld::Scene *mWorldScene;
MWWorld::Player *mPlayer;
std::vector<ESM::ESMReader> mEsm;
std::vector<std::vector<ESM::ESMReader*> > mEsm;
MWWorld::ESMStore mStore;
LocalScripts mLocalScripts;
MWWorld::Globals mGlobalVariables;
@ -199,7 +199,7 @@ namespace MWWorld
virtual const MWWorld::ESMStore& getStore() const;
virtual std::vector<ESM::ESMReader>& getEsmReader();
virtual std::vector<ESM::ESMReader*>& getEsmReader();
virtual LocalScripts& getLocalScripts();

@ -11,8 +11,15 @@ namespace ESM
{
enum Version
{
VER_12 = 0x3f99999a,
VER_13 = 0x3fa66666
VER_12 = 0x3f99999a,
VER_13 = 0x3fa66666,
VER_080 = 0x3f4ccccd, // TES4
VER_100 = 0x3f800000, // TES4
VER_132 = 0x3fa8f5c3, // FONV Courier's Stash, DeadMoney
VER_133 = 0x3faa3d71, // FONV HonestHearts
VER_134 = 0x3fab851f, // FONV, GunRunnersArsenal, LonesomeRoad, OldWorldBlues
VER_094 = 0x3f70a3d7, // TES5/FO3
VER_17 = 0x3fd9999a // TES5
};
/* A structure used for holding fixed-length strings. In the case of

@ -71,12 +71,103 @@ void ESMReader::open(Ogre::DataStreamPtr _esm, const std::string &name)
{
openRaw(_esm, name);
if (getRecName() != "TES3")
fail("Not a valid Morrowind file");
getRecHeader();
NAME modVer = getRecName();
if (modVer == "TES3")
{
getRecHeader();
mHeader.load (*this);
mHeader.load (*this);
}
else if (modVer == "TES4")
{
mHeader.mData.author.assign("");
mHeader.mData.desc.assign("");
char buf[512]; // arbitrary number
unsigned short size;
skip(16); // skip the rest of the header, note it may be 4 bytes longer
NAME rec = getRecName();
if (rec != "HEDR")
rec = getRecName(); // adjust for extra 4 bytes
bool readRec = true;
while (mEsm->size() - mEsm->tell() >= 4) // Shivering Isle or Bashed Patch can end here
{
if (!readRec) // may be already read
rec = getRecName();
else
readRec = false;
switch (rec.val)
{
case 0x52444548: // HEDR
{
skip(2); // data size
getT(mHeader.mData.version);
getT(mHeader.mData.records);
skip(4); // skip next available object id
break;
}
case 0x4d414e43: // CNAM
{
getT(size);
getExact(buf, size);
std::string author;
size = std::min(size, (unsigned short)32); // clamp for TES3 format
author.assign(buf, size - 1); // don't copy null terminator
mHeader.mData.author.assign(author);
break;
}
case 0x4d414e53: // SNAM
{
getT(size);
getExact(buf, size);
std::string desc;
size = std::min(size, (unsigned short)256); // clamp for TES3 format
desc.assign(buf, size - 1); // don't copy null terminator
mHeader.mData.desc.assign(desc);
break;
}
case 0x5453414d: // MAST
{
Header::MasterData m;
getT(size);
getExact(buf, size);
m.name.assign(buf, size-1); // don't copy null terminator
rec = getRecName();
if (rec == "DATA")
{
getT(size);
getT(m.size); // 64 bits
}
else
{
// some esp's don't have DATA subrecord
m.size = 0;
readRec = true; // don't read again at the top of while loop
}
mHeader.mMaster.push_back (m);
break;
}
case 0x56544e49: // INTV
case 0x43434e49: // INCC
case 0x4d414e4f: // ONAM
{
getT(size);
skip(size);
break;
}
case 0x50555247: // GRUP
default:
return; // all done
}
}
return;
}
else
fail("Not a valid Morrowind file");
}
void ESMReader::open(const std::string &file)

@ -23,6 +23,7 @@ class ESMReader
public:
ESMReader();
virtual ~ESMReader() {}
/*************************************************************************
*
@ -74,9 +75,9 @@ public:
void openRaw(const std::string &file);
/// Get the file size. Make sure that the file has been opened!
size_t getFileSize() { return mEsm->size(); }
virtual size_t getFileSize() { return mEsm->size(); }
/// Get the current position in the file. Make sure that the file has been opened!
size_t getFileOffset() { return mEsm->tell(); }
virtual size_t getFileOffset() { return mEsm->tell(); }
// This is a quick hack for multiple esm/esp files. Each plugin introduces its own
// terrain palette, but ESMReader does not pass a reference to the correct plugin
@ -86,8 +87,8 @@ public:
void setIndex(const int index) {mIdx = index; mCtx.index = index;}
int getIndex() {return mIdx;}
void setGlobalReaderList(std::vector<ESMReader> *list) {mGlobalReaderList = list;}
std::vector<ESMReader> *getGlobalReaderList() {return mGlobalReaderList;}
void setGlobalReaderList(std::vector<ESMReader*> *list) {mGlobalReaderList = list;}
std::vector<ESMReader*> *getGlobalReaderList() {return mGlobalReaderList;}
/*************************************************************************
*
@ -292,8 +293,6 @@ public:
private:
Ogre::DataStreamPtr mEsm;
ESM_Context mCtx;
unsigned int mRecordFlags;
// Special file signifier (see SpecialFile enum above)
@ -301,10 +300,13 @@ private:
// Buffer for ESM strings
std::vector<char> mBuffer;
Header mHeader;
std::vector<ESMReader> *mGlobalReaderList;
std::vector<ESMReader*> *mGlobalReaderList;
ToUTF8::Utf8Encoder* mEncoder;
protected:
ESM_Context mCtx;
Header mHeader;
};
}
#endif

Loading…
Cancel
Save