From 27fa411f4f701985079da35f91c22c54a3626fe8 Mon Sep 17 00:00:00 2001 From: Evil Eye Date: Tue, 16 Jan 2024 20:56:58 +0100 Subject: [PATCH] Convert strings in nif files to utf8 --- apps/bulletobjecttool/main.cpp | 2 +- apps/navmeshtool/main.cpp | 2 +- apps/niftest/niftest.cpp | 2 +- apps/opencs/model/world/data.cpp | 2 +- apps/openmw/engine.cpp | 3 ++- components/nif/niffile.cpp | 5 +++-- components/nif/niffile.hpp | 8 +++++++- components/nif/nifstream.cpp | 4 ++++ components/nif/nifstream.hpp | 11 ++++++++++- components/resource/keyframemanager.cpp | 6 ++++-- components/resource/keyframemanager.hpp | 9 ++++++++- components/resource/niffilemanager.cpp | 9 +++++---- components/resource/niffilemanager.hpp | 9 ++++++++- components/resource/resourcesystem.cpp | 6 +++--- components/resource/resourcesystem.hpp | 7 ++++++- 15 files changed, 64 insertions(+), 21 deletions(-) diff --git a/apps/bulletobjecttool/main.cpp b/apps/bulletobjecttool/main.cpp index 7d87899f4a..2165f93804 100644 --- a/apps/bulletobjecttool/main.cpp +++ b/apps/bulletobjecttool/main.cpp @@ -174,7 +174,7 @@ namespace constexpr double expiryDelay = 0; Resource::ImageManager imageManager(&vfs, expiryDelay); - Resource::NifFileManager nifFileManager(&vfs); + Resource::NifFileManager nifFileManager(&vfs, &encoder); Resource::SceneManager sceneManager(&vfs, &imageManager, &nifFileManager, expiryDelay); Resource::BulletShapeManager bulletShapeManager(&vfs, &sceneManager, &nifFileManager, expiryDelay); diff --git a/apps/navmeshtool/main.cpp b/apps/navmeshtool/main.cpp index 8604bcdfb0..a9c30cf23e 100644 --- a/apps/navmeshtool/main.cpp +++ b/apps/navmeshtool/main.cpp @@ -221,7 +221,7 @@ namespace NavMeshTool constexpr double expiryDelay = 0; Resource::ImageManager imageManager(&vfs, expiryDelay); - Resource::NifFileManager nifFileManager(&vfs); + Resource::NifFileManager nifFileManager(&vfs, &encoder); Resource::SceneManager sceneManager(&vfs, &imageManager, &nifFileManager, expiryDelay); Resource::BulletShapeManager bulletShapeManager(&vfs, &sceneManager, &nifFileManager, expiryDelay); DetourNavigator::RecastGlobalAllocator::init(); diff --git a/apps/niftest/niftest.cpp b/apps/niftest/niftest.cpp index 29488fb677..5d0f723ee5 100644 --- a/apps/niftest/niftest.cpp +++ b/apps/niftest/niftest.cpp @@ -84,7 +84,7 @@ void readNIF( try { Nif::NIFFile file(fullPath); - Nif::Reader reader(file); + Nif::Reader reader(file, nullptr); if (vfs != nullptr) reader.parse(vfs->get(pathStr)); else diff --git a/apps/opencs/model/world/data.cpp b/apps/opencs/model/world/data.cpp index 6322a77e66..ef2e289ee2 100644 --- a/apps/opencs/model/world/data.cpp +++ b/apps/opencs/model/world/data.cpp @@ -149,7 +149,7 @@ CSMWorld::Data::Data(ToUTF8::FromType encoding, const Files::PathContainer& data mResourcesManager.setVFS(mVFS.get()); constexpr double expiryDelay = 0; - mResourceSystem = std::make_unique(mVFS.get(), expiryDelay); + mResourceSystem = std::make_unique(mVFS.get(), expiryDelay, &mEncoder); Shader::ShaderManager::DefineMap defines = mResourceSystem->getSceneManager()->getShaderManager().getGlobalDefines(); diff --git a/apps/openmw/engine.cpp b/apps/openmw/engine.cpp index 92483bd8c3..c2e57208b4 100644 --- a/apps/openmw/engine.cpp +++ b/apps/openmw/engine.cpp @@ -706,7 +706,8 @@ void OMW::Engine::prepareEngine() VFS::registerArchives(mVFS.get(), mFileCollections, mArchives, true); - mResourceSystem = std::make_unique(mVFS.get(), Settings::cells().mCacheExpiryDelay); + mResourceSystem + = std::make_unique(mVFS.get(), Settings::cells().mCacheExpiryDelay, mEncoder.get()); mResourceSystem->getSceneManager()->getShaderManager().setMaxTextureUnits(mGlMaxTextureImageUnits); mResourceSystem->getSceneManager()->setUnRefImageDataAfterApply( false); // keep to Off for now to allow better state sharing diff --git a/components/nif/niffile.cpp b/components/nif/niffile.cpp index d6d063a254..7a38b0dc1a 100644 --- a/components/nif/niffile.cpp +++ b/components/nif/niffile.cpp @@ -24,7 +24,7 @@ namespace Nif { - Reader::Reader(NIFFile& file) + Reader::Reader(NIFFile& file, const ToUTF8::Utf8Encoder* encoder) : mVersion(file.mVersion) , mUserVersion(file.mUserVersion) , mBethVersion(file.mBethVersion) @@ -33,6 +33,7 @@ namespace Nif , mRecords(file.mRecords) , mRoots(file.mRoots) , mUseSkinning(file.mUseSkinning) + , mEncoder(encoder) { } @@ -519,7 +520,7 @@ namespace Nif const std::array fileHash = Files::getHash(mFilename, *stream); mHash.append(reinterpret_cast(fileHash.data()), fileHash.size() * sizeof(std::uint64_t)); - NIFStream nif(*this, std::move(stream)); + NIFStream nif(*this, std::move(stream), mEncoder); // Check the header string std::string head = nif.getVersionString(); diff --git a/components/nif/niffile.hpp b/components/nif/niffile.hpp index 6f0030af47..38b0712373 100644 --- a/components/nif/niffile.hpp +++ b/components/nif/niffile.hpp @@ -11,6 +11,11 @@ #include "record.hpp" +namespace ToUTF8 +{ + class Utf8Encoder; +} + namespace Nif { @@ -112,6 +117,7 @@ namespace Nif std::vector mStrings; bool& mUseSkinning; + const ToUTF8::Utf8Encoder* mEncoder; static std::atomic_bool sLoadUnsupportedFiles; static std::atomic_bool sWriteNifDebugLog; @@ -122,7 +128,7 @@ namespace Nif public: /// Open a NIF stream. The name is used for error messages. - explicit Reader(NIFFile& file); + Reader(NIFFile& file, const ToUTF8::Utf8Encoder* encoder); /// Parse the file void parse(Files::IStreamPtr&& stream); diff --git a/components/nif/nifstream.cpp b/components/nif/nifstream.cpp index 2eba746ccf..93714e37f0 100644 --- a/components/nif/nifstream.cpp +++ b/components/nif/nifstream.cpp @@ -4,6 +4,8 @@ #include "niffile.hpp" +#include "../to_utf8/to_utf8.hpp" + namespace { @@ -58,6 +60,8 @@ namespace Nif size_t end = str.find('\0'); if (end != std::string::npos) str.erase(end); + if (mEncoder) + str = mEncoder->getStatelessEncoder().getUtf8(str, ToUTF8::BufferAllocationPolicy::UseGrowFactor, mBuffer); return str; } diff --git a/components/nif/nifstream.hpp b/components/nif/nifstream.hpp index 95205c4fda..958aef7254 100644 --- a/components/nif/nifstream.hpp +++ b/components/nif/nifstream.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,11 @@ #include "niftypes.hpp" +namespace ToUTF8 +{ + class Utf8Encoder; +} + namespace Nif { @@ -67,11 +73,14 @@ namespace Nif { const Reader& mReader; Files::IStreamPtr mStream; + const ToUTF8::Utf8Encoder* mEncoder; + std::string mBuffer; public: - explicit NIFStream(const Reader& reader, Files::IStreamPtr&& stream) + explicit NIFStream(const Reader& reader, Files::IStreamPtr&& stream, const ToUTF8::Utf8Encoder* encoder) : mReader(reader) , mStream(std::move(stream)) + , mEncoder(encoder) { } diff --git a/components/resource/keyframemanager.cpp b/components/resource/keyframemanager.cpp index 574d761d09..6895a0238f 100644 --- a/components/resource/keyframemanager.cpp +++ b/components/resource/keyframemanager.cpp @@ -207,9 +207,11 @@ namespace Resource namespace Resource { - KeyframeManager::KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay) + KeyframeManager::KeyframeManager( + const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay, const ToUTF8::Utf8Encoder* encoder) : ResourceManager(vfs, expiryDelay) , mSceneManager(sceneManager) + , mEncoder(encoder) { } @@ -226,7 +228,7 @@ namespace Resource if (Misc::getFileExtension(normalized) == "kf") { auto file = std::make_shared(normalized); - Nif::Reader reader(*file); + Nif::Reader reader(*file, mEncoder); reader.parse(mVFS->getNormalized(normalized)); NifOsg::Loader::loadKf(*file, *loaded.get()); } diff --git a/components/resource/keyframemanager.hpp b/components/resource/keyframemanager.hpp index 0c92553949..f684e22ee7 100644 --- a/components/resource/keyframemanager.hpp +++ b/components/resource/keyframemanager.hpp @@ -9,6 +9,11 @@ #include "resourcemanager.hpp" +namespace ToUTF8 +{ + class Utf8Encoder; +} + namespace Resource { /// @brief extract animations from OSG formats to OpenMW's animation system @@ -48,7 +53,8 @@ namespace Resource class KeyframeManager : public ResourceManager { public: - explicit KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay); + explicit KeyframeManager(const VFS::Manager* vfs, SceneManager* sceneManager, double expiryDelay, + const ToUTF8::Utf8Encoder* encoder); ~KeyframeManager() = default; /// Retrieve a read-only keyframe resource by name (case-insensitive). @@ -59,6 +65,7 @@ namespace Resource private: SceneManager* mSceneManager; + const ToUTF8::Utf8Encoder* mEncoder; }; } diff --git a/components/resource/niffilemanager.cpp b/components/resource/niffilemanager.cpp index 5e457cdfaa..fb57bc8c85 100644 --- a/components/resource/niffilemanager.cpp +++ b/components/resource/niffilemanager.cpp @@ -24,21 +24,22 @@ namespace Resource { } - NifFileHolder() {} + NifFileHolder() = default; META_Object(Resource, NifFileHolder) Nif::NIFFilePtr mNifFile; }; - NifFileManager::NifFileManager(const VFS::Manager* vfs) + NifFileManager::NifFileManager(const VFS::Manager* vfs, const ToUTF8::Utf8Encoder* encoder) // NIF files aren't needed any more once the converted objects are cached in SceneManager / BulletShapeManager, // so no point in using an expiry delay. : ResourceManager(vfs, 0) + , mEncoder(encoder) { } - NifFileManager::~NifFileManager() {} + NifFileManager::~NifFileManager() = default; Nif::NIFFilePtr NifFileManager::get(const std::string& name) { @@ -48,7 +49,7 @@ namespace Resource else { auto file = std::make_shared(name); - Nif::Reader reader(*file); + Nif::Reader reader(*file, mEncoder); reader.parse(mVFS->get(name)); obj = new NifFileHolder(file); mCache->addEntryToObjectCache(name, obj); diff --git a/components/resource/niffilemanager.hpp b/components/resource/niffilemanager.hpp index 5aef3f3016..2225a9fd4c 100644 --- a/components/resource/niffilemanager.hpp +++ b/components/resource/niffilemanager.hpp @@ -5,6 +5,11 @@ #include "resourcemanager.hpp" +namespace ToUTF8 +{ + class Utf8Encoder; +} + namespace Resource { @@ -12,8 +17,10 @@ namespace Resource /// @note May be used from any thread. class NifFileManager : public ResourceManager { + const ToUTF8::Utf8Encoder* mEncoder; + public: - NifFileManager(const VFS::Manager* vfs); + NifFileManager(const VFS::Manager* vfs, const ToUTF8::Utf8Encoder* encoder); ~NifFileManager(); /// Retrieve a NIF file from the cache, or load it from the VFS if not cached yet. diff --git a/components/resource/resourcesystem.cpp b/components/resource/resourcesystem.cpp index 0bee08e9ac..7d704e7d1e 100644 --- a/components/resource/resourcesystem.cpp +++ b/components/resource/resourcesystem.cpp @@ -10,13 +10,13 @@ namespace Resource { - ResourceSystem::ResourceSystem(const VFS::Manager* vfs, double expiryDelay) + ResourceSystem::ResourceSystem(const VFS::Manager* vfs, double expiryDelay, const ToUTF8::Utf8Encoder* encoder) : mVFS(vfs) { - mNifFileManager = std::make_unique(vfs); + mNifFileManager = std::make_unique(vfs, encoder); mImageManager = std::make_unique(vfs, expiryDelay); mSceneManager = std::make_unique(vfs, mImageManager.get(), mNifFileManager.get(), expiryDelay); - mKeyframeManager = std::make_unique(vfs, mSceneManager.get(), expiryDelay); + mKeyframeManager = std::make_unique(vfs, mSceneManager.get(), expiryDelay, encoder); addResourceManager(mNifFileManager.get()); addResourceManager(mKeyframeManager.get()); diff --git a/components/resource/resourcesystem.hpp b/components/resource/resourcesystem.hpp index d06ac79640..554083852e 100644 --- a/components/resource/resourcesystem.hpp +++ b/components/resource/resourcesystem.hpp @@ -15,6 +15,11 @@ namespace osg class State; } +namespace ToUTF8 +{ + class Utf8Encoder; +} + namespace Resource { @@ -30,7 +35,7 @@ namespace Resource class ResourceSystem { public: - explicit ResourceSystem(const VFS::Manager* vfs, double expiryDelay); + explicit ResourceSystem(const VFS::Manager* vfs, double expiryDelay, const ToUTF8::Utf8Encoder* encoder); ~ResourceSystem(); SceneManager* getSceneManager();