Merge pull request #2555 from akortunov/encoding

Encode ID's in all places to UTF-8

CHANGELOG.md

@@ -19,6 +19,7 @@
     Bug #3778: [Mod] Improved Thrown Weapon Projectiles - weapons have wrong transformation during throw animation
     Bug #3812: Wrong multiline tooltips width when word-wrapping is enabled
     Bug #3894: Hostile spell effects not detected/present on first frame of OnPCHitMe
+    Bug #3977: Non-ASCII characters in object ID's are not supported
     Bug #4077: Enchanted items are not recharged if they are not in the player's inventory
     Bug #4202: Open .omwaddon files without needing toopen openmw-cs first
     Bug #4240: Ash storm origin coordinates and hand shielding animation behavior are incorrect

apps/esmtool/record.cpp

@@ -607,7 +607,7 @@ void Record<ESM::Container>::print()
     std::cout << "  Weight: " << mData.mWeight << std::endl;
     for (const ESM::ContItem &item : mData.mInventory.mList)
         std::cout << "  Inventory: Count: " << Misc::StringUtils::format("%4d", item.mCount)
-                  << " Item: " << item.mItem.toString() << std::endl;
+                  << " Item: " << item.mItem << std::endl;
     std::cout << "  Deleted: " << mIsDeleted << std::endl;
 }
 
@@ -653,7 +653,7 @@ void Record<ESM::Creature>::print()
 
     for (const ESM::ContItem &item : mData.mInventory.mList)
         std::cout << "  Inventory: Count: " << Misc::StringUtils::format("%4d", item.mCount)
-                  << " Item: " << item.mItem.toString() << std::endl;
+                  << " Item: " << item.mItem << std::endl;
 
     for (const std::string &spell : mData.mSpells.mList)
         std::cout << "  Spell: " << spell << std::endl;
@@ -1073,7 +1073,7 @@ void Record<ESM::NPC>::print()
 
     for (const ESM::ContItem &item : mData.mInventory.mList)
         std::cout << "  Inventory: Count: " << Misc::StringUtils::format("%4d", item.mCount)
-                  << " Item: " << item.mItem.toString() << std::endl;
+                  << " Item: " << item.mItem << std::endl;
 
     for (const std::string &spell : mData.mSpells.mList)
         std::cout << "  Spell: " << spell << std::endl;
@@ -1192,7 +1192,7 @@ void Record<ESM::Region>::print()
     if (!mData.mSleepList.empty())
         std::cout << "  Sleep List: " << mData.mSleepList << std::endl;
     for (const ESM::Region::SoundRef &soundref : mData.mSoundList)
-        std::cout << "  Sound: " << (int)soundref.mChance << " = " << soundref.mSound.toString() << std::endl;
+        std::cout << "  Sound: " << (int)soundref.mChance << " = " << soundref.mSound << std::endl;
 }
 
 template<>

apps/essimporter/convertscpt.cpp

@@ -9,7 +9,7 @@ namespace ESSImport
 
     void convertSCPT(const SCPT &scpt, ESM::GlobalScript &out)
     {
-        out.mId = Misc::StringUtils::lowerCase(scpt.mSCHD.mName.toString());
+        out.mId = Misc::StringUtils::lowerCase(scpt.mSCHD.mName);
         out.mRunning = scpt.mRunning;
         convertSCRI(scpt.mSCRI, out.mLocals);
     }

apps/essimporter/importinventory.cpp

@@ -17,7 +17,7 @@ namespace ESSImport
             esm.getHT(contItem);
 
             InventoryItem item;
-            item.mId = contItem.mItem.toString();
+            item.mId = contItem.mItem;
             item.mCount = contItem.mCount;
             item.mRelativeEquipmentSlot = -1;
             item.mLockLevel = 0;

apps/opencs/model/tools/referenceablecheck.cpp

@@ -910,7 +910,7 @@ void CSMTools::ReferenceableCheckStage::inventoryListCheck(
 {
     for (size_t i = 0; i < itemList.size(); ++i)
     {
-        std::string itemName = itemList[i].mItem.toString();
+        std::string itemName = itemList[i].mItem;
         CSMWorld::RefIdData::LocalIndex localIndex = mReferencables.searchId(itemName);
 
         if (localIndex.first == -1)

apps/opencs/model/tools/regioncheck.cpp

@@ -45,7 +45,7 @@ void CSMTools::RegionCheckStage::perform (int stage, CSMDoc::Messages& messages)
     for (const ESM::Region::SoundRef& sound : region.mSoundList)
     {
         if (sound.mChance > 100)
-            messages.add(id, "Chance of '" + sound.mSound.toString() + "' sound to play is over 100 percent", "", CSMDoc::Message::Severity_Warning);
+            messages.add(id, "Chance of '" + sound.mSound + "' sound to play is over 100 percent", "", CSMDoc::Message::Severity_Warning);
     }
 
     /// \todo check data members that can't be edited in the table view

apps/opencs/model/world/actoradapter.cpp

@@ -538,7 +538,7 @@ namespace CSMWorld
         for (auto& item : npc.mInventory.mList)
         {
             if (item.mCount <= 0) continue;
-            std::string itemId = item.mItem.toString();
+            std::string itemId = item.mItem;
             addNpcItem(itemId, data);
         }
     }

apps/opencs/model/world/nestedcoladapterimp.cpp

@@ -396,7 +396,7 @@ namespace CSMWorld
         ESM::Region::SoundRef soundRef = soundList[subRowIndex];
         switch (subColIndex)
         {
-            case 0: return QString(soundRef.mSound.toString().c_str());
+            case 0: return QString(soundRef.mSound.c_str());
             case 1: return soundRef.mChance;
             default: throw std::runtime_error("Region sounds subcolumn index out of range");
         }

apps/opencs/model/world/refidadapterimp.hpp

@@ -1257,7 +1257,7 @@ namespace CSMWorld
 
             switch (subColIndex)
             {
-                case 0: return QString::fromUtf8(content.mItem.toString().c_str());
+                case 0: return QString::fromUtf8(content.mItem.c_str());
                 case 1: return content.mCount;
                 default:
                     throw std::runtime_error("Trying to access non-existing column in the nested table!");

apps/openmw/mwsound/soundmanagerimp.cpp

@@ -924,7 +924,7 @@ namespace MWSound
         {
             if(r - pos < sndref.mChance)
             {
-                playSound(sndref.mSound.toString(), 1.0f, 1.0f);
+                playSound(sndref.mSound, 1.0f, 1.0f);
                 break;
             }
             pos += sndref.mChance;

apps/openmw/mwworld/containerstore.cpp

@@ -495,7 +495,7 @@ void MWWorld::ContainerStore::fill (const ESM::InventoryList& items, const std::
     for (std::vector<ESM::ContItem>::const_iterator iter (items.mList.begin()); iter!=items.mList.end();
         ++iter)
     {
-        std::string id = Misc::StringUtils::lowerCase(iter->mItem.toString());
+        std::string id = Misc::StringUtils::lowerCase(iter->mItem);
         addInitialItem(id, owner, iter->mCount);
     }
 
@@ -626,10 +626,10 @@ void MWWorld::ContainerStore::restock (const ESM::InventoryList& items, const MW
         if (it->mCount >= 0)
             continue;
 
-        std::string itemOrList = Misc::StringUtils::lowerCase(it->mItem.toString());
+        std::string itemOrList = Misc::StringUtils::lowerCase(it->mItem);
 
         //If it's levelled list, restock if there's need to do so.
-        if (MWBase::Environment::get().getWorld()->getStore().get<ESM::ItemLevList>().search(it->mItem.toString()))
+        if (MWBase::Environment::get().getWorld()->getStore().get<ESM::ItemLevList>().search(it->mItem))
         {
             std::map<std::string, int>::iterator listInMap = allowedForReplace.find(itemOrList);
 

components/esm/loadcont.cpp

@@ -9,8 +9,10 @@ namespace ESM
 
     void InventoryList::add(ESMReader &esm)
     {
+        esm.getSubHeader();
         ContItem ci;
-        esm.getHT(ci, 36);
+        esm.getT(ci.mCount);
+        ci.mItem.assign(esm.getString(32));
         mList.push_back(ci);
     }
 
@@ -18,7 +20,10 @@ namespace ESM
     {
         for (std::vector<ContItem>::const_iterator it = mList.begin(); it != mList.end(); ++it)
         {
-            esm.writeHNT("NPCO", *it, 36);
+            esm.startSubRecord("NPCO");
+            esm.writeT(it->mCount);
+            esm.writeFixedSizeString(it->mItem, 32);
+            esm.endRecord("NPCO");
         }
     }
 

components/esm/loadcont.hpp

@@ -19,7 +19,7 @@ class ESMWriter;
 struct ContItem
 {
     int mCount;
-    NAME32 mItem;
+    std::string mItem;
 };
 
 /// InventoryList, NPCO subrecord

components/esm/loadregn.cpp

@@ -62,8 +62,10 @@ namespace ESM
                     break;
                 case ESM::FourCC<'S','N','A','M'>::value:
                 {
+                    esm.getSubHeader();
                     SoundRef sr;
-                    esm.getHT(sr, 33);
+                    sr.mSound.assign(esm.getString(32));
+                    esm.getT(sr.mChance);
                     mSoundList.push_back(sr);
                     break;
                 }
@@ -103,7 +105,10 @@ namespace ESM
         esm.writeHNT("CNAM", mMapColor);
         for (std::vector<SoundRef>::const_iterator it = mSoundList.begin(); it != mSoundList.end(); ++it)
         {
-            esm.writeHNT<SoundRef>("SNAM", *it);
+            esm.startSubRecord("SNAM");
+            esm.writeFixedSizeString(it->mSound, 32);
+            esm.writeT(it->mChance);
+            esm.endRecord("NPCO");
         }
     }
 

components/esm/loadregn.hpp

@@ -37,7 +37,7 @@ struct Region
     // Reference to a sound that is played randomly in this region
     struct SoundRef
     {
-        NAME32 mSound;
+        std::string   mSound;
         unsigned char mChance;
     }; // 33 bytes
 #pragma pack(pop)

components/esm/loadscpt.cpp

@@ -77,10 +77,10 @@ namespace ESM
             {
                 case ESM::FourCC<'S','C','H','D'>::value:
                 {
-                    SCHD data;
+                    esm.getSubHeader();
-                    esm.getHT(data, 52);
+                    mId = esm.getString(32);
-                    mData = data.mData;
+                    esm.getT(mData);
-                    mId = data.mName.toString();
+
                     hasHeader = true;
                     break;
                 }
@@ -131,13 +131,10 @@ namespace ESM
             for (std::vector<std::string>::const_iterator it = mVarNames.begin(); it != mVarNames.end(); ++it)
                 varNameString.append(*it);
 
-        SCHD data;
+        esm.startSubRecord("SCHD");
-        memset(&data, 0, sizeof(data));
+        esm.writeFixedSizeString(mId, 32);
-
+        esm.writeT(mData, 20);
-        data.mData = mData;
+        esm.endRecord("SCHD");
-        data.mName.assign(mId);
-
-        esm.writeHNT("SCHD", data, 52);
 
         if (isDeleted)
         {

components/esm/loadscpt.hpp

@@ -31,7 +31,7 @@ public:
     };
     struct SCHD
     {
-        NAME32              mName;
+        std::string         mName;
         Script::SCHDstruct  mData;
     }; // 52 bytes