Merge branch 'umlautontherocks' into 'master'

Convert strings in nif files to utf8

Closes #7780

See merge request OpenMW/openmw!3783

CHANGELOG.md

@@ -131,6 +131,7 @@
     Bug #7761: Rain and ambient loop sounds are mutually exclusive
     Bug #7765: OpenMW-CS: Touch Record option is broken
     Bug #7770: Sword of the Perithia: Script execution failure
+    Bug #7780: Non-ASCII texture paths in NIF files don't work
     Feature #2566: Handle NAM9 records for manual cell references
     Feature #3537: Shader-based water ripples
     Feature #5173: Support for NiFogProperty

apps/bulletobjecttool/main.cpp

@@ -174,7 +174,7 @@ namespace
 
         constexpr double expiryDelay = 0;
         Resource::ImageManager imageManager(&vfs, expiryDelay);
-        Resource::NifFileManager nifFileManager(&vfs);
+        Resource::NifFileManager nifFileManager(&vfs, &encoder.getStatelessEncoder());
         Resource::SceneManager sceneManager(&vfs, &imageManager, &nifFileManager, expiryDelay);
         Resource::BulletShapeManager bulletShapeManager(&vfs, &sceneManager, &nifFileManager, expiryDelay);
 

apps/navmeshtool/main.cpp

@@ -221,7 +221,7 @@ namespace NavMeshTool
             constexpr double expiryDelay = 0;
 
             Resource::ImageManager imageManager(&vfs, expiryDelay);
-            Resource::NifFileManager nifFileManager(&vfs);
+            Resource::NifFileManager nifFileManager(&vfs, &encoder.getStatelessEncoder());
             Resource::SceneManager sceneManager(&vfs, &imageManager, &nifFileManager, expiryDelay);
             Resource::BulletShapeManager bulletShapeManager(&vfs, &sceneManager, &nifFileManager, expiryDelay);
             DetourNavigator::RecastGlobalAllocator::init();

apps/niftest/niftest.cpp

@@ -84,7 +84,7 @@ void readNIF(
     try
     {
         Nif::NIFFile file(fullPath);
-        Nif::Reader reader(file);
+        Nif::Reader reader(file, nullptr);
         if (vfs != nullptr)
             reader.parse(vfs->get(pathStr));
         else

apps/opencs/model/world/data.cpp

@@ -149,7 +149,8 @@ CSMWorld::Data::Data(ToUTF8::FromType encoding, const Files::PathContainer& data
     mResourcesManager.setVFS(mVFS.get());
 
     constexpr double expiryDelay = 0;
-    mResourceSystem = std::make_unique<Resource::ResourceSystem>(mVFS.get(), expiryDelay);
+    mResourceSystem
+        = std::make_unique<Resource::ResourceSystem>(mVFS.get(), expiryDelay, &mEncoder.getStatelessEncoder());
 
     Shader::ShaderManager::DefineMap defines
         = mResourceSystem->getSceneManager()->getShaderManager().getGlobalDefines();

apps/openmw/engine.cpp

@@ -706,7 +706,8 @@ void OMW::Engine::prepareEngine()
 
     VFS::registerArchives(mVFS.get(), mFileCollections, mArchives, true);
 
-    mResourceSystem = std::make_unique<Resource::ResourceSystem>(mVFS.get(), Settings::cells().mCacheExpiryDelay);
+    mResourceSystem = std::make_unique<Resource::ResourceSystem>(
+        mVFS.get(), Settings::cells().mCacheExpiryDelay, &mEncoder.get()->getStatelessEncoder());
     mResourceSystem->getSceneManager()->getShaderManager().setMaxTextureUnits(mGlMaxTextureImageUnits);
     mResourceSystem->getSceneManager()->setUnRefImageDataAfterApply(
         false); // keep to Off for now to allow better state sharing

components/nif/niffile.cpp

@@ -24,7 +24,7 @@
 namespace Nif
 {
 
-    Reader::Reader(NIFFile& file)
+    Reader::Reader(NIFFile& file, const ToUTF8::StatelessUtf8Encoder* encoder)
         : mVersion(file.mVersion)
         , mUserVersion(file.mUserVersion)
         , mBethVersion(file.mBethVersion)
@@ -33,6 +33,7 @@ namespace Nif
         , mRecords(file.mRecords)
         , mRoots(file.mRoots)
         , mUseSkinning(file.mUseSkinning)
+        , mEncoder(encoder)
     {
     }
 
@@ -519,7 +520,7 @@ namespace Nif
         const std::array<std::uint64_t, 2> fileHash = Files::getHash(mFilename, *stream);
         mHash.append(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t));
 
-        NIFStream nif(*this, std::move(stream));
+        NIFStream nif(*this, std::move(stream), mEncoder);
 
         // Check the header string
         std::string head = nif.getVersionString();

components/nif/niffile.hpp

@@ -11,6 +11,11 @@
 
 #include "record.hpp"
 
+namespace ToUTF8
+{
+    class StatelessUtf8Encoder;
+}
+
 namespace Nif
 {
 
@@ -112,6 +117,7 @@ namespace Nif
         std::vector<std::string> mStrings;
 
         bool& mUseSkinning;
+        const ToUTF8::StatelessUtf8Encoder* mEncoder;
 
         static std::atomic_bool sLoadUnsupportedFiles;
         static std::atomic_bool sWriteNifDebugLog;
@@ -122,7 +128,7 @@ namespace Nif
 
     public:
         /// Open a NIF stream. The name is used for error messages.
-        explicit Reader(NIFFile& file);
+        explicit Reader(NIFFile& file, const ToUTF8::StatelessUtf8Encoder* encoder);
 
         /// Parse the file
         void parse(Files::IStreamPtr&& stream);

components/nif/nifstream.cpp

@@ -4,6 +4,8 @@
 
 #include "niffile.hpp"
 
+#include "../to_utf8/to_utf8.hpp"
+
 namespace
 {
 
@@ -58,6 +60,8 @@ namespace Nif
         size_t end = str.find('\0');
         if (end != std::string::npos)
             str.erase(end);
+        if (mEncoder)
+            str = mEncoder->getUtf8(str, ToUTF8::BufferAllocationPolicy::UseGrowFactor, mBuffer);
         return str;
     }
 

components/nif/nifstream.hpp

@@ -8,6 +8,7 @@
 #include <istream>
 #include <stdexcept>
 #include <stdint.h>
+#include <string>
 #include <type_traits>
 #include <vector>
 
@@ -22,6 +23,11 @@
 
 #include "niftypes.hpp"
 
+namespace ToUTF8
+{
+    class StatelessUtf8Encoder;
+}
+
 namespace Nif
 {
 
@@ -67,11 +73,15 @@ namespace Nif
     {
         const Reader& mReader;
         Files::IStreamPtr mStream;
+        const ToUTF8::StatelessUtf8Encoder* mEncoder;
+        std::string mBuffer;
 
     public:
-        explicit NIFStream(const Reader& reader, Files::IStreamPtr&& stream)
+        explicit NIFStream(
+            const Reader& reader, Files::IStreamPtr&& stream, const ToUTF8::StatelessUtf8Encoder* encoder)
             : mReader(reader)
             , mStream(std::move(stream))
+            , mEncoder(encoder)
         {
         }
 

components/resource/keyframemanager.cpp

@@ -207,9 +207,11 @@ namespace Resource
 namespace Resource
 {
 
-    KeyframeManager::KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay)
+    KeyframeManager::KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay,
+        const ToUTF8::StatelessUtf8Encoder* encoder)
         : ResourceManager(vfs, expiryDelay)
         , mSceneManager(sceneManager)
+        , mEncoder(encoder)
     {
     }
 
@@ -226,7 +228,7 @@ namespace Resource
             if (Misc::getFileExtension(normalized) == "kf")
             {
                 auto file = std::make_shared<Nif::NIFFile>(normalized);
-                Nif::Reader reader(*file);
+                Nif::Reader reader(*file, mEncoder);
                 reader.parse(mVFS->getNormalized(normalized));
                 NifOsg::Loader::loadKf(*file, *loaded.get());
             }

components/resource/keyframemanager.hpp

@@ -9,6 +9,11 @@
 
 #include "resourcemanager.hpp"
 
+namespace ToUTF8
+{
+    class StatelessUtf8Encoder;
+}
+
 namespace Resource
 {
     /// @brief extract animations from OSG formats to OpenMW's animation system
@@ -48,7 +53,8 @@ namespace Resource
     class KeyframeManager : public ResourceManager
     {
     public:
-        explicit KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay);
+        explicit KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay,
+            const ToUTF8::StatelessUtf8Encoder* encoder);
         ~KeyframeManager() = default;
 
         /// Retrieve a read-only keyframe resource by name (case-insensitive).
@@ -59,6 +65,7 @@ namespace Resource
 
     private:
         SceneManager* mSceneManager;
+        const ToUTF8::StatelessUtf8Encoder* mEncoder;
     };
 
 }

components/resource/niffilemanager.cpp

@@ -24,21 +24,22 @@ namespace Resource
         {
         }
 
-        NifFileHolder() {}
+        NifFileHolder() = default;
 
         META_Object(Resource, NifFileHolder)
 
         Nif::NIFFilePtr mNifFile;
     };
 
-    NifFileManager::NifFileManager(const VFS::Manager* vfs)
+    NifFileManager::NifFileManager(const VFS::Manager* vfs, const ToUTF8::StatelessUtf8Encoder* encoder)
         // NIF files aren't needed any more once the converted objects are cached in SceneManager / BulletShapeManager,
         // so no point in using an expiry delay.
         : ResourceManager(vfs, 0)
+        , mEncoder(encoder)
     {
     }
 
-    NifFileManager::~NifFileManager() {}
+    NifFileManager::~NifFileManager() = default;
 
     Nif::NIFFilePtr NifFileManager::get(const std::string& name)
     {
@@ -48,7 +49,7 @@ namespace Resource
         else
         {
             auto file = std::make_shared<Nif::NIFFile>(name);
-            Nif::Reader reader(*file);
+            Nif::Reader reader(*file, mEncoder);
             reader.parse(mVFS->get(name));
             obj = new NifFileHolder(file);
             mCache->addEntryToObjectCache(name, obj);

components/resource/niffilemanager.hpp

@@ -5,6 +5,11 @@
 
 #include "resourcemanager.hpp"
 
+namespace ToUTF8
+{
+    class StatelessUtf8Encoder;
+}
+
 namespace Resource
 {
 
@@ -12,8 +17,10 @@ namespace Resource
     /// @note May be used from any thread.
     class NifFileManager : public ResourceManager
     {
+        const ToUTF8::StatelessUtf8Encoder* mEncoder;
+
     public:
-        NifFileManager(const VFS::Manager* vfs);
+        NifFileManager(const VFS::Manager* vfs, const ToUTF8::StatelessUtf8Encoder* encoder);
         ~NifFileManager();
 
         /// Retrieve a NIF file from the cache, or load it from the VFS if not cached yet.

components/resource/resourcesystem.cpp

@@ -10,13 +10,14 @@
 namespace Resource
 {
 
-    ResourceSystem::ResourceSystem(const VFS::Manager* vfs, double expiryDelay)
+    ResourceSystem::ResourceSystem(
+        const VFS::Manager* vfs, double expiryDelay, const ToUTF8::StatelessUtf8Encoder* encoder)
         : mVFS(vfs)
     {
-        mNifFileManager = std::make_unique<NifFileManager>(vfs);
+        mNifFileManager = std::make_unique<NifFileManager>(vfs, encoder);
         mImageManager = std::make_unique<ImageManager>(vfs, expiryDelay);
         mSceneManager = std::make_unique<SceneManager>(vfs, mImageManager.get(), mNifFileManager.get(), expiryDelay);
-        mKeyframeManager = std::make_unique<KeyframeManager>(vfs, mSceneManager.get(), expiryDelay);
+        mKeyframeManager = std::make_unique<KeyframeManager>(vfs, mSceneManager.get(), expiryDelay, encoder);
 
         addResourceManager(mNifFileManager.get());
         addResourceManager(mKeyframeManager.get());

components/resource/resourcesystem.hpp

@@ -15,6 +15,11 @@ namespace osg
     class State;
 }
 
+namespace ToUTF8
+{
+    class StatelessUtf8Encoder;
+}
+
 namespace Resource
 {
 
@@ -30,7 +35,8 @@ namespace Resource
     class ResourceSystem
     {
     public:
-        explicit ResourceSystem(const VFS::Manager* vfs, double expiryDelay);
+        explicit ResourceSystem(
+            const VFS::Manager* vfs, double expiryDelay, const ToUTF8::StatelessUtf8Encoder* encoder);
         ~ResourceSystem();
 
         SceneManager* getSceneManager();

components/to_utf8/to_utf8.hpp

@@ -68,7 +68,7 @@ namespace ToUTF8
         /// ASCII-only string. Otherwise returns a view to the input.
         std::string_view getLegacyEnc(std::string_view input);
 
-        StatelessUtf8Encoder getStatelessEncoder() const { return mImpl; }
+        const StatelessUtf8Encoder& getStatelessEncoder() const { return mImpl; }
 
     private:
         std::string mBuffer;