mirror/openmw
1
0
Fork 0
mirror of https://github.com/OpenMW/openmw.git synced 2025-03-31 15:06:39 +00:00
Code Issues Releases Wiki Activity

caching dialog keyword search in Store<ESM::Dialogue>

Browse source
This commit is contained in:
myrix 2021-12-24 00:54:00 +03:00
parent 3a9cfbfa53
commit 27cc7a5172
10 changed files with 177 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
apps/openmw/mwbase/world.hpp
View file

@ -87,6 +87,11 @@ namespace MWWorld
typedef std::vector<std::pair<MWWorld::Ptr,MWMechanics::Movement> > PtrMovementList;
}
namespace MWDialogue
{
template <typename string_t, typename value_t> class KeywordSearch;
}
namespace MWBase
{
/// \brief Interface for the World (implemented in MWWorld)
@ -150,6 +155,7 @@ namespace MWBase
virtual MWWorld::ConstPtr getPlayerConstPtr() const = 0;
virtual const MWWorld::ESMStore& getStore() const = 0;
virtual const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() = 0;
virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;

3
apps/openmw/mwdialogue/dialoguemanagerimp.cpp
View file

@ -43,6 +43,7 @@
#include "../mwmechanics/actorutil.hpp"
#include "filter.hpp"
#include "hypertextparser.hpp"
namespace MWDialogue
{
@ -79,7 +80,7 @@ namespace MWDialogue
{
std::vector<std::string> topicIdList;
std::vector<HyperTextParser::Token> hypertext = mHyperTextParser.parseHyperText(text);
std::vector<HyperTextParser::Token> hypertext = HyperTextParser::parseHyperText(text);
for (std::vector<HyperTextParser::Token>::iterator tok = hypertext.begin(); tok != hypertext.end(); ++tok)
{

4
apps/openmw/mwdialogue/dialoguemanagerimp.hpp
View file

@ -16,8 +16,6 @@
#include "../mwscript/compilercontext.hpp"
#include "hypertextparser.hpp"
namespace ESM
{
struct Dialogue;
@ -59,8 +57,6 @@ namespace MWDialogue
int mCurrentDisposition;
int mPermanentDispositionChange;
HyperTextParser mHyperTextParser;
std::vector<std::string> parseTopicIdsFromText (const std::string& text);
void addTopicsFromText (const std::string& text);

119
apps/openmw/mwdialogue/hypertextparser.cpp
View file

@ -6,89 +6,78 @@
#include "../mwworld/store.hpp"
#include "../mwworld/esmstore.hpp"
#include "keywordsearch.hpp"
#include "hypertextparser.hpp"
namespace MWDialogue
{
std::vector<HyperTextParser::Token> HyperTextParser::parseHyperText(const std::string & text)
namespace HyperTextParser
{
std::vector<Token> result;
size_t pos_end = std::string::npos, iteration_pos = 0;
for(;;)
std::vector<Token> parseHyperText(const std::string & text)
{
size_t pos_begin = text.find('@', iteration_pos);
if (pos_begin != std::string::npos)
pos_end = text.find('#', pos_begin);
if (pos_begin != std::string::npos && pos_end != std::string::npos)
std::vector<Token> result;
size_t pos_end = std::string::npos, iteration_pos = 0;
for(;;)
{
if (pos_begin != iteration_pos)
tokenizeKeywords(text.substr(iteration_pos, pos_begin - iteration_pos), result);
size_t pos_begin = text.find('@', iteration_pos);
if (pos_begin != std::string::npos)
pos_end = text.find('#', pos_begin);
std::string link = text.substr(pos_begin + 1, pos_end - pos_begin - 1);
result.emplace_back(link, Token::ExplicitLink);
if (pos_begin != std::string::npos && pos_end != std::string::npos)
{
if (pos_begin != iteration_pos)
tokenizeKeywords(text.substr(iteration_pos, pos_begin - iteration_pos), result);
iteration_pos = pos_end + 1;
std::string link = text.substr(pos_begin + 1, pos_end - pos_begin - 1);
result.emplace_back(link, Token::ExplicitLink);
iteration_pos = pos_end + 1;
}
else
{
if (iteration_pos != text.size())
tokenizeKeywords(text.substr(iteration_pos), result);
break;
}
}
else
return result;
}
void tokenizeKeywords(const std::string & text, std::vector<Token> & tokens)
{
const auto& keywordSearch =
MWBase::Environment::get().getWorld()->getDialogIdKeywordSearch();
std::vector<KeywordSearch<std::string, int /*unused*/>::Match> matches;
keywordSearch.highlightKeywords(text.begin(), text.end(), matches);
for (std::vector<KeywordSearch<std::string, int /*unused*/>::Match>::const_iterator it = matches.begin(); it != matches.end(); ++it)
{
if (iteration_pos != text.size())
tokenizeKeywords(text.substr(iteration_pos), result);
break;
tokens.emplace_back(std::string(it->mBeg, it->mEnd), Token::ImplicitKeyword);
}
}
return result;
}
void HyperTextParser::tokenizeKeywords(const std::string & text, std::vector<Token> & tokens)
{
const MWWorld::Store<ESM::Dialogue> & dialogs =
MWBase::Environment::get().getWorld()->getStore().get<ESM::Dialogue>();
if (dialogs.getModPoint() != mKeywordModPoint)
size_t removePseudoAsterisks(std::string & phrase)
{
mKeywordSearch.clear();
size_t pseudoAsterisksCount = 0;
std::vector<std::string> keywordList;
keywordList.reserve(dialogs.getSize());
for (const auto& it : dialogs)
keywordList.push_back(Misc::StringUtils::lowerCase(it.mId));
sort(keywordList.begin(), keywordList.end());
for (const auto& it : keywordList)
mKeywordSearch.seed(it, 0 /*unused*/);
mKeywordModPoint = dialogs.getModPoint();
}
std::vector<KeywordSearch<std::string, int /*unused*/>::Match> matches;
mKeywordSearch.highlightKeywords(text.begin(), text.end(), matches);
for (std::vector<KeywordSearch<std::string, int /*unused*/>::Match>::const_iterator it = matches.begin(); it != matches.end(); ++it)
{
tokens.emplace_back(std::string(it->mBeg, it->mEnd), Token::ImplicitKeyword);
}
}
size_t HyperTextParser::removePseudoAsterisks(std::string & phrase)
{
size_t pseudoAsterisksCount = 0;
if( !phrase.empty() )
{
std::string::reverse_iterator rit = phrase.rbegin();
const char specialPseudoAsteriskCharacter = 127;
while( rit != phrase.rend() && *rit == specialPseudoAsteriskCharacter )
if( !phrase.empty() )
{
pseudoAsterisksCount++;
++rit;
std::string::reverse_iterator rit = phrase.rbegin();
const char specialPseudoAsteriskCharacter = 127;
while( rit != phrase.rend() && *rit == specialPseudoAsteriskCharacter )
{
pseudoAsterisksCount++;
++rit;
}
}
phrase = phrase.substr(0, phrase.length() - pseudoAsterisksCount);
return pseudoAsterisksCount;
}
phrase = phrase.substr(0, phrase.length() - pseudoAsterisksCount);
return pseudoAsterisksCount;
}
}

15
apps/openmw/mwdialogue/hypertextparser.hpp
View file

@ -4,17 +4,10 @@
#include <string>
#include <vector>
#include "keywordsearch.hpp"
namespace MWDialogue
{
class HyperTextParser
namespace HyperTextParser
{
uint64_t mKeywordModPoint;
KeywordSearch<std::string, int /*unused*/> mKeywordSearch;
public:
struct Token
{
enum Type
@ -31,14 +24,12 @@ namespace MWDialogue
Type mType;
};
HyperTextParser() : mKeywordModPoint(0) {}
// In translations (at least Russian) the links are marked with @#, so
// it should be a function to parse it
std::vector<Token> parseHyperText(const std::string & text);
void tokenizeKeywords(const std::string & text, std::vector<Token> & tokens);
static size_t removePseudoAsterisks(std::string & phrase);
};
size_t removePseudoAsterisks(std::string & phrase);
}
}
#endif

10
apps/openmw/mwdialogue/keywordsearch.hpp
View file

@ -74,13 +74,13 @@ public:
return left.mBeg < right.mBeg;
}
void highlightKeywords (Point beg, Point end, std::vector<Match>& out)
void highlightKeywords (Point beg, Point end, std::vector<Match>& out) const
{
std::vector<Match> matches;
for (Point i = beg; i != end; ++i)
{
// check first character
typename Entry::childen_t::iterator candidate = mRoot.mChildren.find (Misc::StringUtils::toLower (*i));
typename Entry::childen_t::const_iterator candidate = mRoot.mChildren.find (Misc::StringUtils::toLower (*i));
// no match, on to next character
if (candidate == mRoot.mChildren.end ())
@ -91,11 +91,11 @@ public:
// some keywords might be longer variations of other keywords, so we definitely need a list of candidates
// the first element in the pair is length of the match, i.e. depth from the first character on
std::vector< typename std::pair<int, typename Entry::childen_t::iterator> > candidates;
std::vector< typename std::pair<int, typename Entry::childen_t::const_iterator> > candidates;
while ((j + 1) != end)
{
typename Entry::childen_t::iterator next = candidate->second.mChildren.find (Misc::StringUtils::toLower (*++j));
typename Entry::childen_t::const_iterator next = candidate->second.mChildren.find (Misc::StringUtils::toLower (*++j));
if (next == candidate->second.mChildren.end ())
{
@ -116,7 +116,7 @@ public:
// shorter candidates will be added to the vector first. however, we want to check against longer candidates first
std::reverse(candidates.begin(), candidates.end());
for (typename std::vector< std::pair<int, typename Entry::childen_t::iterator> >::iterator it = candidates.begin();
for (typename std::vector< std::pair<int, typename Entry::childen_t::const_iterator> >::iterator it = candidates.begin();
it != candidates.end(); ++it)
{
candidate = it->second;

3
apps/openmw/mwworld/esmstore.hpp
View file

@ -284,6 +284,9 @@ namespace MWWorld
/// Actors with the same ID share spells, abilities, etc.
/// @return The shared spell list to use for this actor and whether or not it has already been initialized.
std::pair<std::shared_ptr<MWMechanics::SpellList>, bool> getSpellList(const std::string& id) const;
const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() {
return mDialogs.getDialogIdKeywordSearch(); }
};
template <>

87
apps/openmw/mwworld/store.cpp
View file

@ -77,13 +77,12 @@ namespace MWWorld
template<typename T>
Store<T>::Store()
: mModPoint(1)
{
}
template<typename T>
Store<T>::Store(const Store<T>& orig)
: mStatic(orig.mStatic), mModPoint(orig.mModPoint + 1)
: mStatic(orig.mStatic)
{
}
@ -94,8 +93,6 @@ namespace MWWorld
assert(mShared.size() >= mStatic.size());
mShared.erase(mShared.begin() + mStatic.size(), mShared.end());
mDynamic.clear();
mModPoint++;
}
template<typename T>
@ -165,8 +162,6 @@ namespace MWWorld
if (inserted.second)
mShared.push_back(&inserted.first->second);
mModPoint++;
return RecordId(record.mId, isDeleted);
}
template<typename T>
@ -218,9 +213,6 @@ namespace MWWorld
T *ptr = &result.first->second;
if (result.second)
mShared.push_back(ptr);
mModPoint++;
return ptr;
}
template<typename T>
@ -230,9 +222,6 @@ namespace MWWorld
T *ptr = &result.first->second;
if (result.second)
mShared.push_back(ptr);
mModPoint++;
return ptr;
}
template<typename T>
@ -253,8 +242,6 @@ namespace MWWorld
++sharedIter;
}
mStatic.erase(it);
mModPoint++;
}
return true;
@ -272,9 +259,6 @@ namespace MWWorld
for (auto it = mDynamic.begin(); it != mDynamic.end(); ++it) {
mShared.push_back(&it->second);
}
mModPoint++;
return true;
}
template<typename T>
@ -997,8 +981,11 @@ namespace MWWorld
// Dialogue
//=========================================================================
Store<ESM::Dialogue>::Store()
: mKeywordSearchModFlag(true)
{
}
template<>
void Store<ESM::Dialogue>::setUp()
{
// DialInfos marked as deleted are kept during the loading phase, so that the linked list
@ -1014,10 +1001,45 @@ namespace MWWorld
// TODO: if we require this behaviour, maybe we should move it to the place that requires it
std::sort(mShared.begin(), mShared.end(), [](const ESM::Dialogue* l, const ESM::Dialogue* r) -> bool { return l->mId < r->mId; });
mModPoint++;
mKeywordSearchModFlag = true;
}
const ESM::Dialogue *Store<ESM::Dialogue>::search(const std::string &id) const
{
typename Static::const_iterator it = mStatic.find(id);
if (it != mStatic.end())
return &(it->second);
return nullptr;
}
const ESM::Dialogue *Store<ESM::Dialogue>::find(const std::string &id) const
{
const ESM::Dialogue *ptr = search(id);
if (ptr == nullptr)
{
std::stringstream msg;
msg << ESM::Dialogue::getRecordType() << " '" << id << "' not found";
throw std::runtime_error(msg.str());
}
return ptr;
}
typename Store<ESM::Dialogue>::iterator Store<ESM::Dialogue>::begin() const
{
return mShared.begin();
}
typename Store<ESM::Dialogue>::iterator Store<ESM::Dialogue>::end() const
{
return mShared.end();
}
size_t Store<ESM::Dialogue>::getSize() const
{
return mShared.size();
}
template <>
inline RecordId Store<ESM::Dialogue>::load(ESM::ESMReader &esm) {
// The original letter case of a dialogue ID is saved, because it's printed
ESM::Dialogue dialogue;
@ -1037,20 +1059,39 @@ namespace MWWorld
dialogue.mId = found->second.mId;
}
mModPoint++;
mKeywordSearchModFlag = true;
return RecordId(dialogue.mId, isDeleted);
}
template<>
bool Store<ESM::Dialogue>::eraseStatic(const std::string &id)
{
if (mStatic.erase(id))
mModPoint++;
mKeywordSearchModFlag = true;
return true;
}
const MWDialogue::KeywordSearch<std::string, int>& Store<ESM::Dialogue>::getDialogIdKeywordSearch()
{
if (mKeywordSearchModFlag)
{
mKeywordSearch.clear();
std::vector<std::string> keywordList;
keywordList.reserve(getSize());
for (const auto& it : *this)
keywordList.push_back(Misc::StringUtils::lowerCase(it.mId));
sort(keywordList.begin(), keywordList.end());
for (const auto& it : keywordList)
mKeywordSearch.seed(it, 0 /*unused*/);
mKeywordSearchModFlag = false;
}
return mKeywordSearch;
}
}
template class MWWorld::Store<ESM::Activator>;

42
apps/openmw/mwworld/store.hpp
View file

@ -11,6 +11,8 @@
#include <components/esm/records.hpp>
#include <components/misc/stringops.hpp>
#include "../mwdialogue/keywordsearch.hpp"
namespace ESM
{
struct Land;
@ -154,13 +156,10 @@ namespace MWWorld
/// @par mShared usually preserves the record order as it came from the content files (this
/// is relevant for the spell autocalc code and selection order
/// for heads/hairs in the character creation)
/// @warning ESM::Dialogue Store currently implements a sorted order for unknown reasons.
std::vector<T*> mShared;
typedef std::unordered_map<std::string, T, Misc::StringUtils::CiHash, Misc::StringUtils::CiEqual> Dynamic;
Dynamic mDynamic;
uint64_t mModPoint;
friend class ESMStore;
public:
@ -205,8 +204,6 @@ namespace MWWorld
RecordId load(ESM::ESMReader &esm) override;
void write(ESM::ESMWriter& writer, Loading::Listener& progress) const override;
RecordId read(ESM::ESMReader& reader, bool overrideOnly = false) override;
uint64_t getModPoint() const { return mModPoint; }
};
template <>
@ -444,6 +441,41 @@ namespace MWWorld
iterator end() const;
};
template <>
class Store<ESM::Dialogue> : public StoreBase
{
typedef std::unordered_map<std::string, ESM::Dialogue, Misc::StringUtils::CiHash, Misc::StringUtils::CiEqual> Static;
Static mStatic;
/// @par mShared usually preserves the record order as it came from the content files (this
/// is relevant for the spell autocalc code and selection order
/// for heads/hairs in the character creation)
/// @warning ESM::Dialogue Store currently implements a sorted order for unknown reasons.
std::vector<ESM::Dialogue*> mShared;
bool mKeywordSearchModFlag;
MWDialogue::KeywordSearch<std::string, int /*unused*/> mKeywordSearch;
public:
Store();
typedef SharedIterator<ESM::Dialogue> iterator;
void setUp() override;
const ESM::Dialogue *search(const std::string &id) const;
const ESM::Dialogue *find(const std::string &id) const;
iterator begin() const;
iterator end() const;
size_t getSize() const override;
bool eraseStatic(const std::string &id) override;
RecordId load(ESM::ESMReader &esm) override;
const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch();
};
} //end namespace

3
apps/openmw/mwworld/worldimp.hpp
View file

@ -236,6 +236,9 @@ namespace MWWorld
const MWWorld::ESMStore& getStore() const override;
const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() override {
return mStore.getDialogIdKeywordSearch(); }
std::vector<ESM::ESMReader>& getEsmReader() override;
LocalScripts& getLocalScripts() override;