Use all topics to search for keywords in dialogues (Fixes #2013)

Move KeywordSearch to MWDialogue
Move hypertext parsing functions to a new file

apps/openmw/CMakeLists.txt

@@ -38,13 +38,13 @@ add_openmw_dir (mwgui
     itemselection spellbuyingwindow loadingscreen levelupdialog waitdialog spellcreationdialog
     enchantingdialog trainingwindow travelwindow exposedwindow cursor spellicons
     merchantrepair repair soulgemdialog companionwindow bookpage journalviewmodel journalbooks
-    keywordsearch itemmodel containeritemmodel inventoryitemmodel sortfilteritemmodel itemview
+    itemmodel containeritemmodel inventoryitemmodel sortfilteritemmodel itemview
     tradeitemmodel companionitemmodel pickpocketitemmodel controllers savegamedialog
     recharge mode videowidget backgroundimage itemwidget screenfader debugwindow
     )
 
 add_openmw_dir (mwdialogue
-    dialoguemanagerimp journalimp journalentry quest topic filter selectwrapper
+    dialoguemanagerimp journalimp journalentry quest topic filter selectwrapper hypertextparser keywordsearch
     )
 
 add_openmw_dir (mwscript

apps/openmw/mwdialogue/dialoguemanagerimp.cpp

@@ -42,6 +42,7 @@
 #include "../mwmechanics/npcstats.hpp"
 
 #include "filter.hpp"
+#include "hypertextparser.hpp"
 
 namespace MWDialogue
 {
@@ -82,42 +83,27 @@ namespace MWDialogue
 
     void DialogueManager::parseText (const std::string& text)
     {
-        std::vector<HyperTextToken> hypertext = ParseHyperText(text);
+        std::vector<HyperTextParser::Token> hypertext = HyperTextParser::parseHyperText(text);
 
-        //calculation of standard form fir all hyperlinks
-        for (size_t i = 0; i < hypertext.size(); ++i)
+        for (std::vector<HyperTextParser::Token>::iterator tok = hypertext.begin(); tok != hypertext.end(); ++tok)
         {
-            if (hypertext[i].mLink)
+            Misc::StringUtils::toLower(tok->mText);
+
+            if (tok->isExplicitLink())
             {
-                size_t asterisk_count = MWDialogue::RemovePseudoAsterisks(hypertext[i].mText);
+                // calculation of standard form for all hyperlinks
+                size_t asterisk_count = HyperTextParser::removePseudoAsterisks(tok->mText);
                 for(; asterisk_count > 0; --asterisk_count)
-                    hypertext[i].mText.append("*");
+                    tok->mText.append("*");
 
-                hypertext[i].mText = mTranslationDataStorage.topicStandardForm(hypertext[i].mText);
+                tok->mText = mTranslationDataStorage.topicStandardForm(tok->mText);
             }
-        }
 
-        for (size_t i = 0; i < hypertext.size(); ++i)
-        {
-            std::list<std::string>::iterator it;
-            for(it = mActorKnownTopics.begin(); it != mActorKnownTopics.end(); ++it)
-            {
-                if (hypertext[i].mLink)
-                {
-                    if( hypertext[i].mText == *it )
-                    {
-                        mKnownTopics[hypertext[i].mText] = true;
-                    }
-                }
-                else if( !mTranslationDataStorage.hasTranslation() )
-                {
-                    size_t pos = Misc::StringUtils::lowerCase(hypertext[i].mText).find(*it, 0);
-                    if(pos !=std::string::npos)
-                    {
-                        mKnownTopics[*it] = true;
-                    }
-                }
-            }
+            if (tok->isImplicitKeyword() && mTranslationDataStorage.hasTranslation())
+                continue;
+
+            if (std::find(mActorKnownTopics.begin(), mActorKnownTopics.end(), tok->mText) != mActorKnownTopics.end())
+                mKnownTopics[tok->mText] = true;
         }
 
         updateTopics();
@@ -724,55 +710,4 @@ namespace MWDialogue
                         mLastTopic, actor.getClass().getName(actor));
         }
     }
-
-    std::vector<HyperTextToken> ParseHyperText(const std::string& text)
-    {
-        std::vector<HyperTextToken> result;
-        MyGUI::UString utext(text);
-        size_t pos_end, iteration_pos = 0;
-        for(;;)
-        {
-            size_t pos_begin = utext.find('@', iteration_pos);
-            if (pos_begin != std::string::npos)
-                pos_end = utext.find('#', pos_begin);
-
-            if (pos_begin != std::string::npos && pos_end != std::string::npos)
-            {
-                result.push_back( HyperTextToken(utext.substr(iteration_pos, pos_begin - iteration_pos), false) );
-
-                std::string link = utext.substr(pos_begin + 1, pos_end - pos_begin - 1);
-                result.push_back( HyperTextToken(link, true) );
-
-                iteration_pos = pos_end + 1;
-            }
-            else
-            {
-                result.push_back( HyperTextToken(utext.substr(iteration_pos), false) );
-                break;
-            }
-        }
-
-        return result;
-    }
-
-    size_t RemovePseudoAsterisks(std::string& phrase)
-    {
-        size_t pseudoAsterisksCount = 0;
-
-        if( !phrase.empty() )
-        {
-            std::string::reverse_iterator rit = phrase.rbegin();
-
-            const char specialPseudoAsteriskCharacter = 127;
-            while( rit != phrase.rend() && *rit == specialPseudoAsteriskCharacter )
-            {
-                pseudoAsterisksCount++;
-                ++rit;
-            }
-        }
-
-        phrase = phrase.substr(0, phrase.length() - pseudoAsterisksCount);
-
-        return pseudoAsterisksCount;
-    }
 }

apps/openmw/mwdialogue/dialoguemanagerimp.hpp

@@ -103,21 +103,6 @@ namespace MWDialogue
             /// Removes the last added topic response for the given actor from the journal
             virtual void clearInfoActor (const MWWorld::Ptr& actor) const;
     };
-
-
-    struct HyperTextToken
-    {
-        HyperTextToken(const std::string& text, bool link) : mText(text), mLink(link) {}
-
-        std::string mText;
-        bool mLink;
-    };
-
-    // In translations (at least Russian) the links are marked with @#, so
-    // it should be a function to parse it
-    std::vector<HyperTextToken> ParseHyperText(const std::string& text);
-
-    size_t RemovePseudoAsterisks(std::string& phrase);
 }
 
 #endif

apps/openmw/mwdialogue/hypertextparser.cpp

@@ -0,0 +1,89 @@
+#include <components/esm/loaddial.hpp>
+
+#include "../mwbase/environment.hpp"
+#include "../mwbase/world.hpp"
+
+#include "../mwworld/store.hpp"
+#include "../mwworld/esmstore.hpp"
+
+#include "keywordsearch.hpp"
+
+#include "hypertextparser.hpp"
+
+namespace MWDialogue
+{
+    namespace HyperTextParser
+    {
+        std::vector<Token> parseHyperText(const std::string & text)
+        {
+            std::vector<Token> result;
+            size_t pos_end, iteration_pos = 0;
+            for(;;)
+            {
+                size_t pos_begin = text.find('@', iteration_pos);
+                if (pos_begin != std::string::npos)
+                    pos_end = text.find('#', pos_begin);
+
+                if (pos_begin != std::string::npos && pos_end != std::string::npos)
+                {
+                    if (pos_begin != iteration_pos)
+                        tokenizeKeywords(text.substr(iteration_pos, pos_begin - iteration_pos), result);
+
+                    std::string link = text.substr(pos_begin + 1, pos_end - pos_begin - 1);
+                    result.push_back(Token(link, Token::ExplicitLink));
+
+                    iteration_pos = pos_end + 1;
+                }
+                else
+                {
+                    if (iteration_pos != text.size())
+                        tokenizeKeywords(text.substr(iteration_pos), result);
+                    break;
+                }
+            }
+
+            return result;
+        }
+
+        void tokenizeKeywords(const std::string & text, std::vector<Token> & tokens)
+        {
+            const MWWorld::Store<ESM::Dialogue> & dialogs =
+                MWBase::Environment::get().getWorld()->getStore().get<ESM::Dialogue>();
+
+            std::list<std::string> keywordList;
+            for (MWWorld::Store<ESM::Dialogue>::iterator it = dialogs.begin(); it != dialogs.end(); ++it)
+                keywordList.push_back(Misc::StringUtils::lowerCase(it->mId));
+            keywordList.sort(Misc::StringUtils::ciLess);
+
+            KeywordSearch<std::string, int /*unused*/> keywordSearch;
+            KeywordSearch<std::string, int /*unused*/>::Match match;
+
+            for (std::list<std::string>::const_iterator it = keywordList.begin(); it != keywordList.end(); ++it)
+                keywordSearch.seed(*it, 0 /*unused*/);
+
+            for (std::string::const_iterator it = text.begin(); it != text.end() && keywordSearch.search(it, text.end(), match, text.begin()); it = match.mEnd)
+                tokens.push_back(Token(std::string(match.mBeg, match.mEnd), Token::ImplicitKeyword));
+        }
+
+        size_t removePseudoAsterisks(std::string & phrase)
+        {
+            size_t pseudoAsterisksCount = 0;
+
+            if( !phrase.empty() )
+            {
+                std::string::reverse_iterator rit = phrase.rbegin();
+
+                const char specialPseudoAsteriskCharacter = 127;
+                while( rit != phrase.rend() && *rit == specialPseudoAsteriskCharacter )
+                {
+                    pseudoAsterisksCount++;
+                    ++rit;
+                }
+            }
+
+            phrase = phrase.substr(0, phrase.length() - pseudoAsterisksCount);
+
+            return pseudoAsterisksCount;
+        }
+    }
+}

apps/openmw/mwdialogue/hypertextparser.hpp

@@ -0,0 +1,36 @@
+#ifndef GAME_MWDIALOGUE_HYPERTEXTPARSER_H
+#define GAME_MWDIALOGUE_HYPERTEXTPARSER_H
+
+#include <string>
+#include <vector>
+
+namespace MWDialogue
+{
+    namespace HyperTextParser
+    {
+        struct Token
+        {
+            enum Type
+            {
+                ExplicitLink, // enclosed in @#
+                ImplicitKeyword
+            };
+
+            Token(const std::string & text, Type type) : mText(text), mType(type) {}
+
+            bool isExplicitLink() { return mType == ExplicitLink; }
+            bool isImplicitKeyword() { return mType == ImplicitKeyword; }
+
+            std::string mText;
+            Type mType;
+        };
+
+        // In translations (at least Russian) the links are marked with @#, so
+        // it should be a function to parse it
+        std::vector<Token> parseHyperText(const std::string & text);
+        void tokenizeKeywords(const std::string & text, std::vector<Token> & tokens);
+        size_t removePseudoAsterisks(std::string & phrase);
+    }
+}
+
+#endif

apps/openmw/mwdialogue/keywordsearch.hpp

@@ -1,5 +1,5 @@
-#ifndef MWGUI_KEYWORDSEARCH_H
-#define MWGUI_KEYWORDSEARCH_H
+#ifndef GAME_MWDIALOGUE_KEYWORDSEARCH_H
+#define GAME_MWDIALOGUE_KEYWORDSEARCH_H
 
 #include <map>
 #include <locale>
@@ -9,7 +9,7 @@
 
 #include <components/misc/stringops.hpp>
 
-namespace MWGui
+namespace MWDialogue
 {
 
 template <typename string_t, typename value_t>
@@ -141,7 +141,6 @@ public:
                 match.mValue = candidate->second.mValue;
                 match.mBeg = i;
                 match.mEnd = k;
-
                 return true;
             }
         }

apps/openmw/mwgui/dialogue.hpp

@@ -6,7 +6,7 @@
 
 #include "bookpage.hpp"
 
-#include "keywordsearch.hpp"
+#include "../mwdialogue/keywordsearch.hpp"
 
 namespace Gui
 {
@@ -76,7 +76,7 @@ namespace MWGui
         virtual void activated ();
     };
 
-    typedef KeywordSearch <std::string, intptr_t> KeywordSearchT;
+    typedef MWDialogue::KeywordSearch <std::string, intptr_t> KeywordSearchT;
 
     struct DialogueText
     {

apps/openmw/mwgui/journalviewmodel.cpp

@@ -12,9 +12,9 @@
 #include "../mwbase/journal.hpp"
 #include "../mwbase/environment.hpp"
 #include "../mwbase/windowmanager.hpp"
-#include "../mwdialogue/journalentry.hpp"
 
-#include "keywordsearch.hpp"
+#include "../mwdialogue/journalentry.hpp"
+#include "../mwdialogue/keywordsearch.hpp"
 
 namespace MWGui {
 
@@ -22,7 +22,7 @@ struct JournalViewModelImpl;
 
 struct JournalViewModelImpl : JournalViewModel
 {
-    typedef KeywordSearch <std::string, intptr_t> KeywordSearchT;
+    typedef MWDialogue::KeywordSearch <std::string, intptr_t> KeywordSearchT;
 
     mutable bool             mKeywordSearchLoaded;
     mutable KeywordSearchT mKeywordSearch;