Merge branch 'recurse_directory_iterator' into 'master'

VFS manager refactoring

See merge request OpenMW/openmw!1218

apps/niftest/niftest.cpp

@@ -52,11 +52,8 @@ void readVFS(VFS::Archive* anArchive,std::string archivePath = "")
     myManager.addArchive(anArchive);
     myManager.buildIndex();
 
-    std::map<std::string, VFS::File*> files=myManager.getIndex();
+    for(const auto& name : myManager.getRecursiveDirectoryIterator(""))
-    for(auto it=files.begin(); it!=files.end(); ++it)
     {
-        std::string name = it->first;
-
         try{
             if(isNIF(name))
             {

apps/opencs/model/world/resources.cpp

@@ -23,10 +23,8 @@ void CSMWorld::Resources::recreate(const VFS::Manager* vfs, const char * const *
 
     size_t baseSize = mBaseDirectory.size();
 
-    const std::map<std::string, VFS::File*>& index = vfs->getIndex();
+    for (const auto& filepath : vfs->getRecursiveDirectoryIterator(""))
-    for (std::map<std::string, VFS::File*>::const_iterator it = index.begin(); it != index.end(); ++it)
     {
-        std::string filepath = it->first;
         if (filepath.size()<baseSize+1 ||
             filepath.substr (0, baseSize)!=mBaseDirectory ||
             (filepath[baseSize]!='/' && filepath[baseSize]!='\\'))

apps/openmw/mwgui/loadingscreen.cpp

@@ -13,6 +13,7 @@
 #include <MyGUI_Gui.h>
 #include <MyGUI_TextBox.h>
 
+#include <components/misc/pathhelpers.hpp>
 #include <components/misc/rng.hpp>
 #include <components/debug/debuglog.hpp>
 #include <components/myguiplatform/myguitexture.hpp>
@@ -66,35 +67,15 @@ namespace MWGui
 
     void LoadingScreen::findSplashScreens()
     {
-        const std::map<std::string, VFS::File*>& index = mResourceSystem->getVFS()->getIndex();
+        auto isSupportedExtension = [](const std::string_view& ext) {
-        std::string pattern = "Splash/";
+            static const std::array<std::string, 7> supported_extensions{ {"tga", "dds", "ktx", "png", "bmp", "jpeg", "jpg"} };
-        mResourceSystem->getVFS()->normalizeFilename(pattern);
+            return !ext.empty() && std::find(supported_extensions.begin(), supported_extensions.end(), ext) != supported_extensions.end();
+        };
 
-        /* priority given to the left */
+        for (const auto& name : mResourceSystem->getVFS()->getRecursiveDirectoryIterator("Splash/"))
-        const std::array<std::string, 7> supported_extensions {{".tga", ".dds", ".ktx", ".png", ".bmp", ".jpeg", ".jpg"}};
-
-        auto found = index.lower_bound(pattern);
-        while (found != index.end())
         {
-            const std::string& name = found->first;
+            if (isSupportedExtension(Misc::getFileExtension(name)))
-            if (name.size() >= pattern.size() && name.substr(0, pattern.size()) == pattern)
+                mSplashScreens.push_back(name);
-            {
-                size_t pos = name.find_last_of('.');
-                if (pos != std::string::npos)
-                {
-                    for(auto const& extension: supported_extensions)
-                    {
-                        if (name.compare(pos, name.size() - pos, extension) == 0)
-                        {
-                            mSplashScreens.push_back(found->first);
-                            break;  /* based on priority */
-                        }
-                    }
-                }
-            }
-            else
-                break;
-            ++found;
         }
         if (mSplashScreens.empty())
             Log(Debug::Warning) << "Warning: no splash screens found!";

apps/openmw/mwrender/animation.cpp

@@ -17,6 +17,7 @@
 #include <components/resource/keyframemanager.hpp>
 
 #include <components/misc/constants.hpp>
+#include <components/misc/pathhelpers.hpp>
 #include <components/misc/resourcehelpers.hpp>
 
 #include <components/sceneutil/keyframe.hpp>
@@ -591,8 +592,6 @@ namespace MWRender
 
     void Animation::loadAllAnimationsInFolder(const std::string &model, const std::string &baseModel)
     {
-        const std::map<std::string, VFS::File*>& index = mResourceSystem->getVFS()->getIndex();
-
         std::string animationPath = model;
         if (animationPath.find("meshes") == 0)
         {
@@ -600,22 +599,11 @@ namespace MWRender
         }
         animationPath.replace(animationPath.size()-3, 3, "/");
 
-        mResourceSystem->getVFS()->normalizeFilename(animationPath);
+        for (const auto& name : mResourceSystem->getVFS()->getRecursiveDirectoryIterator(animationPath))
-
-        std::map<std::string, VFS::File*>::const_iterator found = index.lower_bound(animationPath);
-        while (found != index.end())
         {
-            const std::string& name = found->first;
+            if (Misc::getFileExtension(name) == "kf")
-            if (name.size() >= animationPath.size() && name.substr(0, animationPath.size()) == animationPath)
-            {
-                size_t pos = name.find_last_of('.');
-                if (pos != std::string::npos && name.compare(pos, name.size()-pos, ".kf") == 0)
                 addSingleAnimSource(name, baseModel);
         }
-            else
-                break;
-            ++found;
-        }
     }
 
     void Animation::addAnimSource(const std::string &model, const std::string& baseModel)
@@ -1295,8 +1283,6 @@ namespace MWRender
         if (model.empty())
             return;
 
-        const std::map<std::string, VFS::File*>& index = resourceSystem->getVFS()->getIndex();
-
         std::string animationPath = model;
         if (animationPath.find("meshes") == 0)
         {
@@ -1304,22 +1290,11 @@ namespace MWRender
         }
         animationPath.replace(animationPath.size()-4, 4, "/");
 
-        resourceSystem->getVFS()->normalizeFilename(animationPath);
+        for (const auto& name : resourceSystem->getVFS()->getRecursiveDirectoryIterator(animationPath))
-
-        std::map<std::string, VFS::File*>::const_iterator found = index.lower_bound(animationPath);
-        while (found != index.end())
         {
-            const std::string& name = found->first;
+            if (Misc::getFileExtension(name) == "nif")
-            if (name.size() >= animationPath.size() && name.substr(0, animationPath.size()) == animationPath)
-            {
-                size_t pos = name.find_last_of('.');
-                if (pos != std::string::npos && name.compare(pos, name.size()-pos, ".nif") == 0)
                 loadBonesFromFile(node, name, resourceSystem);
         }
-            else
-                break;
-            ++found;
-        }
     }
 
     osg::ref_ptr<osg::Node> getModelInstance(Resource::ResourceSystem* resourceSystem, const std::string& model, bool baseonly, bool inject, const std::string& defaultSkeleton)

apps/openmw/mwsound/sound_buffer.cpp

@@ -131,7 +131,7 @@ namespace MWSound
         max = std::max(min, max);
 
         Sound_Buffer& sfx = mSoundBuffers.emplace_back("Sound/" + sound.mSound, volume, min, max);
-        mVfs->normalizeFilename(sfx.mResourceName);
+        sfx.mResourceName = mVfs->normalizeFilename(sfx.mResourceName);
 
         mBufferNameMap.emplace(soundId, &sfx);
         return &sfx;

apps/openmw/mwsound/soundmanagerimp.cpp

@@ -294,20 +294,9 @@ namespace MWSound
         if (mMusicFiles.find(playlist) == mMusicFiles.end())
         {
             std::vector<std::string> filelist;
-            const std::map<std::string, VFS::File*>& index = mVFS->getIndex();
 
-            std::string pattern = "Music/" + playlist;
+            for (const auto& name : mVFS->getRecursiveDirectoryIterator("Music/" + playlist))
-            mVFS->normalizeFilename(pattern);
+                filelist.push_back(name);
-
-            std::map<std::string, VFS::File*>::const_iterator found = index.lower_bound(pattern);
-            while (found != index.end())
-            {
-                if (found->first.size() >= pattern.size() && found->first.substr(0, pattern.size()) == pattern)
-                    filelist.push_back(found->first);
-                else
-                    break;
-                ++found;
-            }
 
             mMusicFiles[playlist] = filelist;
         }
@@ -327,13 +316,11 @@ namespace MWSound
         if (mMusicFiles.find("Title") == mMusicFiles.end())
         {
             std::vector<std::string> filelist;
-            const std::map<std::string, VFS::File*>& index = mVFS->getIndex();
             // Is there an ini setting for this filename or something?
             std::string filename = "music/special/morrowind title.mp3";
-            auto found = index.find(filename);
+            if (mVFS->exists(filename))
-            if (found != index.end())
             {
-                filelist.emplace_back(found->first);
+                filelist.emplace_back(filename);
                 mMusicFiles["Title"] = filelist;
             }
             else
@@ -355,10 +342,7 @@ namespace MWSound
         if(!mOutput->isInitialized())
             return;
 
-        std::string voicefile = "Sound/"+filename;
+        DecoderPtr decoder = loadVoice(mVFS->normalizeFilename("Sound/" + filename));
-
-        mVFS->normalizeFilename(voicefile);
-        DecoderPtr decoder = loadVoice(voicefile);
         if (!decoder)
             return;
 
@@ -389,10 +373,7 @@ namespace MWSound
         if(!mOutput->isInitialized())
             return;
 
-        std::string voicefile = "Sound/"+filename;
+        DecoderPtr decoder = loadVoice(mVFS->normalizeFilename("Sound/" + filename));
-
-        mVFS->normalizeFilename(voicefile);
-        DecoderPtr decoder = loadVoice(voicefile);
         if (!decoder)
             return;
 

components/fontloader/fontloader.cpp

@@ -17,6 +17,7 @@
 
 #include <components/vfs/manager.hpp>
 
+#include <components/misc/pathhelpers.hpp>
 #include <components/misc/stringops.hpp>
 
 #include <components/myguiplatform/myguitexture.hpp>
@@ -191,25 +192,11 @@ namespace Gui
 
     void FontLoader::loadBitmapFonts(bool exportToFile)
     {
-        const std::map<std::string, VFS::File*>& index = mVFS->getIndex();
+        for (const auto& name : mVFS->getRecursiveDirectoryIterator("Fonts/"))
-
-        std::string pattern = "Fonts/";
-        mVFS->normalizeFilename(pattern);
-
-        std::map<std::string, VFS::File*>::const_iterator found = index.lower_bound(pattern);
-        while (found != index.end())
         {
-            const std::string& name = found->first;
+            if (Misc::getFileExtension(name) == "fnt")
-            if (name.size() >= pattern.size() && name.substr(0, pattern.size()) == pattern)
-            {
-                size_t pos = name.find_last_of('.');
-                if (pos != std::string::npos && name.compare(pos, name.size()-pos, ".fnt") == 0)
                 loadBitmapFont(name, exportToFile);
         }
-            else
-                break;
-            ++found;
-        }
     }
 
     void FontLoader::loadTrueTypeFonts()

components/misc/pathhelpers.hpp

@@ -0,0 +1,19 @@
+#ifndef OPENMW_COMPONENTS_MISC_PATHHELPERS_H
+#define OPENMW_COMPONENTS_MISC_PATHHELPERS_H
+
+#include <string>
+
+namespace Misc
+{
+    inline std::string_view getFileExtension(std::string_view file)
+    {
+        if (auto extPos = file.find_last_of('.'); extPos != std::string::npos)
+        {
+            file.remove_prefix(extPos + 1);
+            return file;
+        }
+        return {};
+    }
+}
+
+#endif

components/resource/bulletshapemanager.cpp

@@ -8,6 +8,7 @@
 
 #include <BulletCollision/CollisionShapes/btTriangleMesh.h>
 
+#include <components/misc/pathhelpers.hpp>
 #include <components/sceneutil/visitor.hpp>
 #include <components/vfs/manager.hpp>
 
@@ -121,8 +122,7 @@ BulletShapeManager::~BulletShapeManager()
 
 osg::ref_ptr<const BulletShape> BulletShapeManager::getShape(const std::string &name)
 {
-    std::string normalized = name;
+    const std::string normalized = mVFS->normalizeFilename(name);
-    mVFS->normalizeFilename(normalized);
 
     osg::ref_ptr<BulletShape> shape;
     osg::ref_ptr<osg::Object> obj = mCache->getRefFromObjectCache(normalized);
@@ -130,12 +130,7 @@ osg::ref_ptr<const BulletShape> BulletShapeManager::getShape(const std::string &
         shape = osg::ref_ptr<BulletShape>(static_cast<BulletShape*>(obj.get()));
     else
     {
-        size_t extPos = normalized.find_last_of('.');
+        if (Misc::getFileExtension(normalized) == "nif")
-        std::string ext;
-        if (extPos != std::string::npos && extPos+1 < normalized.size())
-            ext = normalized.substr(extPos+1);
-
-        if (ext == "nif")
         {
             NifBullet::BulletNifLoader loader;
             shape = loader.load(*mNifFileManager->get(normalized));
@@ -180,8 +175,7 @@ osg::ref_ptr<const BulletShape> BulletShapeManager::getShape(const std::string &
 
 osg::ref_ptr<BulletShapeInstance> BulletShapeManager::cacheInstance(const std::string &name)
 {
-    std::string normalized = name;
+    const std::string normalized = mVFS->normalizeFilename(name);
-    mVFS->normalizeFilename(normalized);
 
     osg::ref_ptr<BulletShapeInstance> instance = createInstance(normalized);
     if (instance)
@@ -191,8 +185,7 @@ osg::ref_ptr<BulletShapeInstance> BulletShapeManager::cacheInstance(const std::s
 
 osg::ref_ptr<BulletShapeInstance> BulletShapeManager::getInstance(const std::string &name)
 {
-    std::string normalized = name;
+    const std::string normalized = mVFS->normalizeFilename(name);
-    mVFS->normalizeFilename(normalized);
 
     osg::ref_ptr<osg::Object> obj = mInstanceCache->takeFromObjectCache(normalized);
     if (obj.get())

components/resource/imagemanager.cpp

@@ -4,6 +4,7 @@
 #include <osgDB/Registry>
 
 #include <components/debug/debuglog.hpp>
+#include <components/misc/pathhelpers.hpp>
 #include <components/vfs/manager.hpp>
 
 #include "objectcache.hpp"
@@ -83,8 +84,7 @@ namespace Resource
 
     osg::ref_ptr<osg::Image> ImageManager::getImage(const std::string &filename)
     {
-        std::string normalized = filename;
+        const std::string normalized = mVFS->normalizeFilename(filename);
-        mVFS->normalizeFilename(normalized);
 
         osg::ref_ptr<osg::Object> obj = mCache->getRefFromObjectCache(normalized);
         if (obj)
@@ -103,10 +103,7 @@ namespace Resource
                 return mWarningImage;
             }
 
-            size_t extPos = normalized.find_last_of('.');
+            const std::string ext(Misc::getFileExtension(normalized));
-            std::string ext;
-            if (extPos != std::string::npos && extPos+1 < normalized.size())
-                ext = normalized.substr(extPos+1);
             osgDB::ReaderWriter* reader = osgDB::Registry::instance()->getReaderWriterForExtension(ext);
             if (!reader)
             {

components/resource/keyframemanager.cpp

@@ -9,6 +9,7 @@
 #include <components/nifosg/nifloader.hpp>
 #include <components/sceneutil/keyframe.hpp>
 #include <components/sceneutil/osgacontroller.hpp>
+#include <components/misc/pathhelpers.hpp>
 #include <components/misc/stringops.hpp>
 
 #include "animation.hpp"
@@ -133,8 +134,7 @@ namespace Resource
 
     osg::ref_ptr<const SceneUtil::KeyframeHolder> KeyframeManager::get(const std::string &name)
     {
-        std::string normalized = name;
+        const std::string normalized = mVFS->normalizeFilename(name);
-        mVFS->normalizeFilename(normalized);
 
         osg::ref_ptr<osg::Object> obj = mCache->getRefFromObjectCache(normalized);
         if (obj)
@@ -142,8 +142,7 @@ namespace Resource
         else
         {
             osg::ref_ptr<SceneUtil::KeyframeHolder> loaded (new SceneUtil::KeyframeHolder);
-            std::string ext = Resource::getFileExtension(normalized);
+            if (Misc::getFileExtension(normalized) == "kf")
-            if (ext == "kf")
             {
                 NifOsg::Loader::loadKf(Nif::NIFFilePtr(new Nif::NIFFile(mVFS->getNormalized(normalized), normalized)), *loaded.get());
             }

components/resource/scenemanager.cpp

@@ -17,6 +17,7 @@
 #include <components/nifosg/nifloader.hpp>
 #include <components/nif/niffile.hpp>
 
+#include <components/misc/pathhelpers.hpp>
 #include <components/misc/stringops.hpp>
 
 #include <components/vfs/manager.hpp>
@@ -358,10 +359,7 @@ namespace Resource
 
     bool SceneManager::checkLoaded(const std::string &name, double timeStamp)
     {
-        std::string normalized = name;
+        return mCache->checkInObjectCache(mVFS->normalizeFilename(name), timeStamp);
-        mVFS->normalizeFilename(normalized);
-
-        return mCache->checkInObjectCache(normalized, timeStamp);
     }
 
     /// @brief Callback to read image files from the VFS.
@@ -391,12 +389,12 @@ namespace Resource
 
     osg::ref_ptr<osg::Node> load (const std::string& normalizedFilename, const VFS::Manager* vfs, Resource::ImageManager* imageManager, Resource::NifFileManager* nifFileManager)
     {
-        std::string ext = Resource::getFileExtension(normalizedFilename);
+        auto ext = Misc::getFileExtension(normalizedFilename);
         if (ext == "nif")
             return NifOsg::Loader::load(nifFileManager->get(normalizedFilename), imageManager);
         else
         {
-            osgDB::ReaderWriter* reader = osgDB::Registry::instance()->getReaderWriterForExtension(ext);
+            osgDB::ReaderWriter* reader = osgDB::Registry::instance()->getReaderWriterForExtension(std::string(ext));
             if (!reader)
             {
                 std::stringstream errormsg;
@@ -533,8 +531,7 @@ namespace Resource
 
     osg::ref_ptr<const osg::Node> SceneManager::getTemplate(const std::string &name, bool compile)
     {
-        std::string normalized = name;
+        std::string normalized = mVFS->normalizeFilename(name);
-        mVFS->normalizeFilename(normalized);
 
         osg::ref_ptr<osg::Object> obj = mCache->getRefFromObjectCache(normalized);
         if (obj)
@@ -603,8 +600,7 @@ namespace Resource
 
     osg::ref_ptr<osg::Node> SceneManager::cacheInstance(const std::string &name)
     {
-        std::string normalized = name;
+        const std::string normalized = mVFS->normalizeFilename(name);
-        mVFS->normalizeFilename(normalized);
 
         osg::ref_ptr<osg::Node> node = createInstance(normalized);
 
@@ -642,8 +638,7 @@ namespace Resource
 
     osg::ref_ptr<osg::Node> SceneManager::getInstance(const std::string &name)
     {
-        std::string normalized = name;
+        const std::string normalized = mVFS->normalizeFilename(name);
-        mVFS->normalizeFilename(normalized);
 
         osg::ref_ptr<osg::Object> obj = mInstanceCache->takeFromObjectCache(normalized);
         if (obj.get())
@@ -822,12 +817,4 @@ namespace Resource
         shaderVisitor->setTranslucentFramebuffer(translucentFramebuffer);
         return shaderVisitor;
     }
-
-    std::string getFileExtension(const std::string& file)
-    {
-        size_t extPos = file.find_last_of('.');
-        if (extPos != std::string::npos && extPos+1 < file.size())
-            return file.substr(extPos+1);
-        return std::string();
-    }
 }

components/vfs/manager.cpp

@@ -86,14 +86,11 @@ namespace VFS
         return mIndex.find(normalized) != mIndex.end();
     }
 
-    const std::map<std::string, File*>& Manager::getIndex() const
+    std::string Manager::normalizeFilename(const std::string& name) const
     {
-        return mIndex;
+        std::string result = name;
-    }
+        normalize_path(result, mStrict);
-
+        return result;
-    void Manager::normalizeFilename(std::string &name) const
-    {
-        normalize_path(name, mStrict);
     }
 
     std::string Manager::getArchive(const std::string& name) const
@@ -107,4 +104,9 @@ namespace VFS
         }
         return {};
     }
+
+    RecursiveDirectoryIterator Manager::getRecursiveDirectoryIterator(const std::string& path) const
+    {
+        return RecursiveDirectoryIterator(mIndex, normalizeFilename(path));
+    }
 }

components/vfs/manager.hpp

@@ -12,6 +12,51 @@ namespace VFS
     class Archive;
     class File;
 
+    class RecursiveDirectoryIterator;
+    RecursiveDirectoryIterator end(const RecursiveDirectoryIterator& iter);
+
+    class RecursiveDirectoryIterator
+    {
+    public:
+        RecursiveDirectoryIterator(const std::map<std::string, File*>& index, const std::string& path)
+            : mPath(path)
+            , mIndex(&index)
+            , mIt(index.lower_bound(path))
+        {}
+
+        RecursiveDirectoryIterator(const RecursiveDirectoryIterator&) = default;
+
+        const std::string& operator*() const { return mIt->first; }
+        const std::string* operator->() const { return &mIt->first; }
+
+        bool operator!=(const RecursiveDirectoryIterator& other) { return mPath != other.mPath || mIt != other.mIt; }
+
+        RecursiveDirectoryIterator& operator++()
+        {
+            if (++mIt == mIndex->end() || !starts_with(mIt->first, mPath))
+                *this = end(*this);
+            return *this;
+        }
+
+        friend RecursiveDirectoryIterator end(const RecursiveDirectoryIterator& iter);
+
+    private:
+        static bool starts_with(const std::string& text, const std::string& start) { return text.rfind(start, 0) == 0; }
+
+        std::string mPath;
+        const std::map<std::string, File*>* mIndex;
+        std::map<std::string, File*>::const_iterator mIt;
+    };
+
+    inline RecursiveDirectoryIterator begin(RecursiveDirectoryIterator iter) { return iter; }
+
+    inline RecursiveDirectoryIterator end(const RecursiveDirectoryIterator& iter)
+    {
+        RecursiveDirectoryIterator result(iter);
+        result.mIt = result.mIndex->end();
+        return result;
+    }
+
     /// @brief The main class responsible for loading files from a virtual file system.
     /// @par Various archive types (e.g. directories on the filesystem, or compressed archives)
     /// can be registered, and will be merged into a single file tree. If the same filename is
@@ -40,13 +85,9 @@ namespace VFS
         /// @note May be called from any thread once the index has been built.
         bool exists(const std::string& name) const;
 
-        /// Get a complete list of files from all archives
-        /// @note May be called from any thread once the index has been built.
-        const std::map<std::string, File*>& getIndex() const;
-
         /// Normalize the given filename, making slashes/backslashes consistent, and lower-casing if mStrict is false.
         /// @note May be called from any thread once the index has been built.
-        void normalizeFilename(std::string& name) const;
+        [[nodiscard]] std::string normalizeFilename(const std::string& name) const;
 
         /// Retrieve a file by name.
         /// @note Throws an exception if the file can not be found.
@@ -59,6 +100,13 @@ namespace VFS
         Files::IStreamPtr getNormalized(const std::string& normalizedName) const;
 
         std::string getArchive(const std::string& name) const;
+
+        /// Recursivly iterate over the elements of the given path
+        /// In practice it return all files of the VFS starting with the given path
+        /// @note the path is normalized
+        /// @note May be called from any thread once the index has been built.
+        RecursiveDirectoryIterator getRecursiveDirectoryIterator(const std::string& path) const;
+
     private:
         bool mStrict;