From 86945d19128468ad987b5bf4332602d613d25d9f Mon Sep 17 00:00:00 2001 From: cc9cii Date: Fri, 11 Dec 2015 21:00:13 +1100 Subject: [PATCH] Convert the CellRef record index lookup maps to use integer keys rather than strings. - Morrowind load over 300,000 references, so even small inefficiencies add up to longer loading times. - std::map is used, but should try others, std::unordered_map or even std::vector --- apps/opencs/model/world/data.hpp | 2 +- apps/opencs/model/world/ref.cpp | 2 + apps/opencs/model/world/ref.hpp | 2 + apps/opencs/model/world/refcollection.cpp | 178 +++++++++++++++++++++- apps/opencs/model/world/refcollection.hpp | 35 ++++- 5 files changed, 211 insertions(+), 8 deletions(-) diff --git a/apps/opencs/model/world/data.hpp b/apps/opencs/model/world/data.hpp index d1c0f1f543..47046d9de3 100644 --- a/apps/opencs/model/world/data.hpp +++ b/apps/opencs/model/world/data.hpp @@ -104,7 +104,7 @@ namespace CSMWorld const ESM::Dialogue *mDialogue; // last loaded dialogue bool mBase; bool mProject; - std::map > mRefLoadCache; + std::map > mRefLoadCache; int mReaderIndex; std::vector mLoadedFiles; diff --git a/apps/opencs/model/world/ref.cpp b/apps/opencs/model/world/ref.cpp index 8c6f1d8365..dbd9ae93d4 100644 --- a/apps/opencs/model/world/ref.cpp +++ b/apps/opencs/model/world/ref.cpp @@ -3,6 +3,7 @@ #include CSMWorld::CellRef::CellRef() + : mIdNum(0) { mId.clear(); mCell.clear(); @@ -22,6 +23,7 @@ CSMWorld::CellRef& CSMWorld::CellRef::operator= (CSMWorld::CellRef&& other) if (this != &other) { ESM::CellRef::operator= (other); + mIdNum = other.mIdNum; mId = std::move(other.mId); mCell = std::move(other.mCell); mOriginalCell = std::move(other.mOriginalCell); diff --git a/apps/opencs/model/world/ref.hpp b/apps/opencs/model/world/ref.hpp index c439e7ca5d..f9af5705f2 100644 --- a/apps/opencs/model/world/ref.hpp +++ b/apps/opencs/model/world/ref.hpp @@ -10,6 +10,8 @@ namespace CSMWorld /// \brief Wrapper for CellRef sub record struct CellRef : public ESM::CellRef { + unsigned int mIdNum; + std::string mId; std::string mCell; std::string mOriginalCell; diff --git a/apps/opencs/model/world/refcollection.cpp b/apps/opencs/model/world/refcollection.cpp index 3542a30bd0..26ec507db2 100644 --- a/apps/opencs/model/world/refcollection.cpp +++ b/apps/opencs/model/world/refcollection.cpp @@ -11,8 +11,37 @@ #include "universalid.hpp" #include "record.hpp" +namespace CSMWorld +{ + template<> + void Collection >::removeRows (int index, int count) + { + mRecords.erase(mRecords.begin()+index, mRecords.begin()+index+count); + + // index map is updated in RefCollection::removeRows() + } + + template<> + void Collection >::insertRecord (std::unique_ptr record, int index, + UniversalId::Type type) + { + int size = static_cast(mRecords.size()); + if (index < 0 || index > size) + throw std::runtime_error("index out of range"); + + std::unique_ptr > record2(static_cast*>(record.release())); + + if (index == size) + mRecords.push_back(std::move(record2)); + else + mRecords.insert(mRecords.begin()+index, std::move(record2)); + + // index map is updated in RefCollection::insertRecord() + } +} + void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool base, - std::map& cache, CSMDoc::Messages& messages) + std::map& cache, CSMDoc::Messages& messages) { Record cell = mCells.getRecord (cellIndex); @@ -74,7 +103,7 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool unsigned int refNum = (ref.mRefNum.mIndex & 0x00ffffff) | (ref.mRefNum.hasContentFile() ? ref.mRefNum.mContentFile : 0xff) << 24; - std::map::iterator iter = cache.find(refNum); + std::map::iterator iter = cache.find(refNum); if (isDeleted) { @@ -83,7 +112,11 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool CSMWorld::UniversalId id (CSMWorld::UniversalId::Type_Cell, mCells.getId (cellIndex)); - messages.add (id, "Attempt to delete a non-existing reference"); + messages.add (id, "Attempt to delete a non-existing reference - RefNum index " + + std::to_string(ref.mRefNum.mIndex) + ", refID " + ref.mRefID + ", content file index " + + std::to_string(ref.mRefNum.mContentFile), + /*hint*/"", + CSMDoc::Message::Severity_Warning); continue; } @@ -107,9 +140,10 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool if (iter==cache.end()) { // new reference + ref.mIdNum = mNextId; // FIXME: fragile ref.mId = getNewId(); - cache.insert(std::make_pair(refNum, ref.mId)); + cache.insert(std::make_pair(refNum, ref.mIdNum)); std::unique_ptr > record(new Record); record->mState = base ? RecordBase::State_BaseOnly : RecordBase::State_ModifiedOnly; @@ -120,9 +154,27 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool else { // old reference -> merge - ref.mId = iter->second; + int index = getIndex(iter->second); +#if 0 + // ref.mRefNum.mIndex : the key + // iter->second : previously cached idNum for the key + // index : position of the record for that idNum + // getRecord(index).get() : record in the index position + assert(iter->second != getRecord(index).get().mIdNum); // sanity check - int index = getIndex (ref.mId); + // check if the plugin used the same RefNum index for a different record + if (ref.mRefID != getRecord(index).get().mRefID) + { + CSMWorld::UniversalId id(CSMWorld::UniversalId::Type_Cell, mCells.getId(cellIndex)); + messages.add(id, + "RefNum renamed from RefID \"" + getRecord(index).get().mRefID + "\" to \"" + + ref.mRefID + "\" (RefNum index " + std::to_string(ref.mRefNum.mIndex) + ")", + /*hint*/"", + CSMDoc::Message::Severity_Info); + } +#endif + ref.mId = getRecord(index).get().mId; + ref.mIdNum = extractIdNum(ref.mId); std::unique_ptr > record(new Record(getRecord(index))); record->mState = base ? RecordBase::State_BaseOnly : RecordBase::State_Modified; @@ -139,3 +191,117 @@ std::string CSMWorld::RefCollection::getNewId() stream << "ref#" << mNextId++; return stream.str(); } + +unsigned int CSMWorld::RefCollection::extractIdNum (const std::string& id) const +{ + std::string::size_type separator = id.find_last_of('#'); + + if (separator == std::string::npos) + throw std::runtime_error("invalid ref ID: " + id); + + return static_cast(std::stoi(id.substr(separator+1))); +} + +int CSMWorld::RefCollection::getIndex (unsigned int id) const +{ + int index = searchId(id); + + if (index == -1) + throw std::runtime_error("invalid RefNum: " + std::to_string(id)); + + return index; +} + +int CSMWorld::RefCollection::searchId (unsigned int id) const +{ + std::map::const_iterator iter = mRefIndex.find(id); + + if (iter == mRefIndex.end()) + return -1; + + return iter->second; +} + +void CSMWorld::RefCollection::removeRows (int index, int count) +{ + Collection >::removeRows(index, count); // erase records only + + std::map::iterator iter = mRefIndex.begin(); + while (iter != mRefIndex.end()) + { + if (iter->second>=index) + { + if (iter->second >= index+count) + { + iter->second -= count; + ++iter; + } + else + mRefIndex.erase(iter++); + } + else + ++iter; + } +} + +void CSMWorld::RefCollection::appendBlankRecord (const std::string& id, UniversalId::Type type) +{ + std::unique_ptr > record2(new Record); + + record2->mState = Record::State_ModifiedOnly; + record2->mModified.blank(); + + record2->get().mId = id; + record2->get().mIdNum = extractIdNum(id); + + Collection >::appendRecord(std::move(record2)); +} + +void CSMWorld::RefCollection::cloneRecord (const std::string& origin, + const std::string& destination, + const UniversalId::Type type) +{ + std::unique_ptr > copy(new Record); + + copy->mModified = getRecord(origin).get(); + copy->mState = RecordBase::State_ModifiedOnly; + + copy->get().mId = destination; + copy->get().mIdNum = extractIdNum(destination); + + insertRecord(std::move(copy), getAppendIndex(destination, type)); // call RefCollection::insertRecord() +} + +int CSMWorld::RefCollection::searchId (const std::string& id) const +{ + return searchId(extractIdNum(id)); +} + +void CSMWorld::RefCollection::appendRecord (std::unique_ptr record, UniversalId::Type type) +{ + int index = getAppendIndex(/*id*/"", type); // for CellRef records id is ignored + + mRefIndex.insert(std::make_pair(static_cast*>(record.get())->get().mIdNum, index)); + + Collection >::insertRecord(std::move(record), index, type); // add records only +} + +void CSMWorld::RefCollection::insertRecord (std::unique_ptr record, int index, + UniversalId::Type type) +{ + int size = getAppendIndex(/*id*/"", type); // for CellRef records id is ignored + unsigned int idNum = static_cast*>(record.get())->get().mIdNum; + + Collection >::insertRecord(std::move(record), index, type); // add records only + + if (index < size-1) + { + for (std::map::iterator iter(mRefIndex.begin()); iter != mRefIndex.end(); ++iter) + { + if (iter->second >= index) + ++(iter->second); + } + } + + mRefIndex.insert(std::make_pair(idNum, index)); +} diff --git a/apps/opencs/model/world/refcollection.hpp b/apps/opencs/model/world/refcollection.hpp index 42bb363570..6b4704bc9b 100644 --- a/apps/opencs/model/world/refcollection.hpp +++ b/apps/opencs/model/world/refcollection.hpp @@ -14,12 +14,27 @@ namespace CSMWorld struct Cell; class UniversalId; + template<> + void Collection >::removeRows (int index, int count); + + template<> + void Collection >::insertRecord (std::unique_ptr record, int index, + UniversalId::Type type); + /// \brief References in cells class RefCollection : public Collection { Collection& mCells; + std::map mRefIndex; + int mNextId; + unsigned int extractIdNum(const std::string& id) const; + + int getIndex (unsigned int id) const; + + int searchId (unsigned int id) const; + public: // MSVC needs the constructor for a class inheriting a template to be defined in header RefCollection (Collection& cells) @@ -27,10 +42,28 @@ namespace CSMWorld {} void load (ESM::ESMReader& reader, int cellIndex, bool base, - std::map& cache, CSMDoc::Messages& messages); + std::map& cache, CSMDoc::Messages& messages); ///< Load a sequence of references. std::string getNewId(); + + virtual void removeRows (int index, int count); + + virtual void appendBlankRecord (const std::string& id, + UniversalId::Type type = UniversalId::Type_None); + + virtual void cloneRecord (const std::string& origin, + const std::string& destination, + const UniversalId::Type type); + + virtual int searchId (const std::string& id) const; + + virtual void appendRecord (std::unique_ptr record, + UniversalId::Type type = UniversalId::Type_None); + + virtual void insertRecord (std::unique_ptr record, + int index, + UniversalId::Type type = UniversalId::Type_None); }; }