Merge branch 'Fix_ASCII' into 'master'

Fixed an issue where keyword search expected the text to be all ASCII characters

See merge request OpenMW/openmw!1227

AUTHORS.md

@@ -80,6 +80,7 @@ Programmers
     Federico Guerra (FedeWar)
     Fil Krynicki (filkry)
     Finbar Crago (finbar-crago)
+    Florent Teppe (Tetramir)
     Florian Weber (Florianjw)
     Frédéric Chardon (fr3dz10)
     Gaëtan Dezeiraud (Brouilles)

CHANGELOG.md

@@ -37,6 +37,7 @@
     Bug #6184: Command and Calm and Demoralize and Frenzy and Rally magic effects inconsistencies with vanilla
     Bug #6197: Infinite Casting Loop
     Bug #6273: Respawning NPCs rotation is inconsistent
+    Bug #6289: Keyword search in dialogues expected the text to be all ASCII characters 
     Feature #2554: Modifying an object triggers the instances table to scroll to the corresponding record
     Feature #2780: A way to see current OpenMW version in the console
     Feature #3616: Allow Zoom levels on the World Map

apps/openmw/mwdialogue/keywordsearch.hpp

@@ -1,8 +1,9 @@
 #ifndef GAME_MWDIALOGUE_KEYWORDSEARCH_H
 #define GAME_MWDIALOGUE_KEYWORDSEARCH_H
 
-#include <map>
 #include <cctype>
+#include <map>
+#include <limits>
 #include <stdexcept>
 #include <vector>
 #include <algorithm>    // std::reverse
@@ -68,6 +69,19 @@ public:
         return false;
     }
 
+    static bool isWhitespaceUTF8(const int utf8Char)
+    {
+        if (utf8Char >= 0 && utf8Char <= static_cast<int>( std::numeric_limits<unsigned char>::max()))
+        {
+            //That function has undefined behavior if the character doesn't fit in unsigned char
+            return std::isspace(utf8Char);
+        }
+        else
+        {
+            return false;
+        }
+    }
+
     static bool sortMatches(const Match& left, const Match& right)
     {
         return left.mBeg < right.mBeg;
@@ -83,7 +97,7 @@ public:
             {
                 Point prev = i;
                 --prev;
-                if(isalpha(*prev))
+                if(!isWhitespaceUTF8(*prev))
                     continue;
             }
 

apps/openmw_test_suite/mwdialogue/test_keywordsearch.cpp

@@ -27,9 +27,9 @@ TEST_F(KeywordSearchTest, keyword_test_conflict_resolution)
     search.highlightKeywords(text.begin(), text.end(), matches);
 
     // Should contain: "foo bar", "lock switch"
-    ASSERT_TRUE (matches.size() == 2);
+    EXPECT_EQ (matches.size() , 2);
-    ASSERT_TRUE (std::string(matches.front().mBeg, matches.front().mEnd) == "foo bar");
+    EXPECT_EQ (std::string(matches.front().mBeg, matches.front().mEnd) , "foo bar");
-    ASSERT_TRUE (std::string(matches.rbegin()->mBeg, matches.rbegin()->mEnd) == "lock switch");
+    EXPECT_EQ (std::string(matches.rbegin()->mBeg, matches.rbegin()->mEnd) , "lock switch");
 }
 
 TEST_F(KeywordSearchTest, keyword_test_conflict_resolution2)
@@ -43,8 +43,8 @@ TEST_F(KeywordSearchTest, keyword_test_conflict_resolution2)
     std::vector<MWDialogue::KeywordSearch<std::string, int>::Match> matches;
     search.highlightKeywords(text.begin(), text.end(), matches);
 
-    ASSERT_TRUE (matches.size() == 1);
+    EXPECT_EQ (matches.size() , 1);
-    ASSERT_TRUE (std::string(matches.front().mBeg, matches.front().mEnd) == "dwemer language");
+    EXPECT_EQ (std::string(matches.front().mBeg, matches.front().mEnd) , "dwemer language");
 }
 
 
@@ -62,6 +62,27 @@ TEST_F(KeywordSearchTest, keyword_test_conflict_resolution3)
     std::vector<MWDialogue::KeywordSearch<std::string, int>::Match> matches;
     search.highlightKeywords(text.begin(), text.end(), matches);
 
-    ASSERT_TRUE (matches.size() == 1);
+    EXPECT_EQ (matches.size() , 1);
-    ASSERT_TRUE (std::string(matches.front().mBeg, matches.front().mEnd) == "bar lock");
+    EXPECT_EQ (std::string(matches.front().mBeg, matches.front().mEnd) , "bar lock");
+}
+
+
+TEST_F(KeywordSearchTest, keyword_test_utf8_word_begin)
+{
+    // We make sure that the search works well even if the character is not ASCII
+    MWDialogue::KeywordSearch<std::string, int> search;
+    search.seed("états", 0);
+    search.seed("ïrradiés", 0);
+    search.seed("ça nous déçois", 0);
+
+
+    std::string text = "les nations unis ont réunis le monde entier, états units inclus pour parler du problème des gens ïrradiés et ça nous déçois";
+
+    std::vector<MWDialogue::KeywordSearch<std::string, int>::Match> matches;
+    search.highlightKeywords(text.begin(), text.end(), matches);
+
+    EXPECT_EQ (matches.size() , 3);
+    EXPECT_EQ (std::string( matches[0].mBeg, matches[0].mEnd) , "états");
+    EXPECT_EQ (std::string( matches[1].mBeg, matches[1].mEnd) , "ïrradiés");
+    EXPECT_EQ (std::string( matches[2].mBeg, matches[2].mEnd) , "ça nous déçois");
 }